~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/mm/shmem.c

Version: ~ [ linux-5.18 ] ~ [ linux-5.17.9 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.41 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.117 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.195 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.244 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.280 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.315 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.302 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Resizable virtual memory filesystem for Linux.
  3  *
  4  * Copyright (C) 2000 Linus Torvalds.
  5  *               2000 Transmeta Corp.
  6  *               2000-2001 Christoph Rohland
  7  *               2000-2001 SAP AG
  8  *               2002 Red Hat Inc.
  9  * Copyright (C) 2002-2011 Hugh Dickins.
 10  * Copyright (C) 2011 Google Inc.
 11  * Copyright (C) 2002-2005 VERITAS Software Corporation.
 12  * Copyright (C) 2004 Andi Kleen, SuSE Labs
 13  *
 14  * Extended attribute support for tmpfs:
 15  * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
 16  * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
 17  *
 18  * tiny-shmem:
 19  * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
 20  *
 21  * This file is released under the GPL.
 22  */
 23 
 24 #include <linux/fs.h>
 25 #include <linux/init.h>
 26 #include <linux/vfs.h>
 27 #include <linux/mount.h>
 28 #include <linux/ramfs.h>
 29 #include <linux/pagemap.h>
 30 #include <linux/file.h>
 31 #include <linux/mm.h>
 32 #include <linux/random.h>
 33 #include <linux/sched/signal.h>
 34 #include <linux/export.h>
 35 #include <linux/swap.h>
 36 #include <linux/uio.h>
 37 #include <linux/khugepaged.h>
 38 #include <linux/hugetlb.h>
 39 #include <linux/frontswap.h>
 40 #include <linux/fs_parser.h>
 41 
 42 #include <asm/tlbflush.h> /* for arch/microblaze update_mmu_cache() */
 43 
 44 static struct vfsmount *shm_mnt;
 45 
 46 #ifdef CONFIG_SHMEM
 47 /*
 48  * This virtual memory filesystem is heavily based on the ramfs. It
 49  * extends ramfs by the ability to use swap and honor resource limits
 50  * which makes it a completely usable filesystem.
 51  */
 52 
 53 #include <linux/xattr.h>
 54 #include <linux/exportfs.h>
 55 #include <linux/posix_acl.h>
 56 #include <linux/posix_acl_xattr.h>
 57 #include <linux/mman.h>
 58 #include <linux/string.h>
 59 #include <linux/slab.h>
 60 #include <linux/backing-dev.h>
 61 #include <linux/shmem_fs.h>
 62 #include <linux/writeback.h>
 63 #include <linux/blkdev.h>
 64 #include <linux/pagevec.h>
 65 #include <linux/percpu_counter.h>
 66 #include <linux/falloc.h>
 67 #include <linux/splice.h>
 68 #include <linux/security.h>
 69 #include <linux/swapops.h>
 70 #include <linux/mempolicy.h>
 71 #include <linux/namei.h>
 72 #include <linux/ctype.h>
 73 #include <linux/migrate.h>
 74 #include <linux/highmem.h>
 75 #include <linux/seq_file.h>
 76 #include <linux/magic.h>
 77 #include <linux/syscalls.h>
 78 #include <linux/fcntl.h>
 79 #include <uapi/linux/memfd.h>
 80 #include <linux/userfaultfd_k.h>
 81 #include <linux/rmap.h>
 82 #include <linux/uuid.h>
 83 
 84 #include <linux/uaccess.h>
 85 
 86 #include "internal.h"
 87 
 88 #define BLOCKS_PER_PAGE  (PAGE_SIZE/512)
 89 #define VM_ACCT(size)    (PAGE_ALIGN(size) >> PAGE_SHIFT)
 90 
 91 /* Pretend that each entry is of this size in directory's i_size */
 92 #define BOGO_DIRENT_SIZE 20
 93 
 94 /* Symlink up to this size is kmalloc'ed instead of using a swappable page */
 95 #define SHORT_SYMLINK_LEN 128
 96 
 97 /*
 98  * shmem_fallocate communicates with shmem_fault or shmem_writepage via
 99  * inode->i_private (with i_mutex making sure that it has only one user at
100  * a time): we would prefer not to enlarge the shmem inode just for that.
101  */
102 struct shmem_falloc {
103         wait_queue_head_t *waitq; /* faults into hole wait for punch to end */
104         pgoff_t start;          /* start of range currently being fallocated */
105         pgoff_t next;           /* the next page offset to be fallocated */
106         pgoff_t nr_falloced;    /* how many new pages have been fallocated */
107         pgoff_t nr_unswapped;   /* how often writepage refused to swap out */
108 };
109 
110 struct shmem_options {
111         unsigned long long blocks;
112         unsigned long long inodes;
113         struct mempolicy *mpol;
114         kuid_t uid;
115         kgid_t gid;
116         umode_t mode;
117         int huge;
118         int seen;
119 #define SHMEM_SEEN_BLOCKS 1
120 #define SHMEM_SEEN_INODES 2
121 #define SHMEM_SEEN_HUGE 4
122 };
123 
124 #ifdef CONFIG_TMPFS
125 static unsigned long shmem_default_max_blocks(void)
126 {
127         return totalram_pages() / 2;
128 }
129 
130 static unsigned long shmem_default_max_inodes(void)
131 {
132         unsigned long nr_pages = totalram_pages();
133 
134         return min(nr_pages - totalhigh_pages(), nr_pages / 2);
135 }
136 #endif
137 
138 static bool shmem_should_replace_page(struct page *page, gfp_t gfp);
139 static int shmem_replace_page(struct page **pagep, gfp_t gfp,
140                                 struct shmem_inode_info *info, pgoff_t index);
141 static int shmem_swapin_page(struct inode *inode, pgoff_t index,
142                              struct page **pagep, enum sgp_type sgp,
143                              gfp_t gfp, struct vm_area_struct *vma,
144                              vm_fault_t *fault_type);
145 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
146                 struct page **pagep, enum sgp_type sgp,
147                 gfp_t gfp, struct vm_area_struct *vma,
148                 struct vm_fault *vmf, vm_fault_t *fault_type);
149 
150 int shmem_getpage(struct inode *inode, pgoff_t index,
151                 struct page **pagep, enum sgp_type sgp)
152 {
153         return shmem_getpage_gfp(inode, index, pagep, sgp,
154                 mapping_gfp_mask(inode->i_mapping), NULL, NULL, NULL);
155 }
156 
157 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
158 {
159         return sb->s_fs_info;
160 }
161 
162 /*
163  * shmem_file_setup pre-accounts the whole fixed size of a VM object,
164  * for shared memory and for shared anonymous (/dev/zero) mappings
165  * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
166  * consistent with the pre-accounting of private mappings ...
167  */
168 static inline int shmem_acct_size(unsigned long flags, loff_t size)
169 {
170         return (flags & VM_NORESERVE) ?
171                 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size));
172 }
173 
174 static inline void shmem_unacct_size(unsigned long flags, loff_t size)
175 {
176         if (!(flags & VM_NORESERVE))
177                 vm_unacct_memory(VM_ACCT(size));
178 }
179 
180 static inline int shmem_reacct_size(unsigned long flags,
181                 loff_t oldsize, loff_t newsize)
182 {
183         if (!(flags & VM_NORESERVE)) {
184                 if (VM_ACCT(newsize) > VM_ACCT(oldsize))
185                         return security_vm_enough_memory_mm(current->mm,
186                                         VM_ACCT(newsize) - VM_ACCT(oldsize));
187                 else if (VM_ACCT(newsize) < VM_ACCT(oldsize))
188                         vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize));
189         }
190         return 0;
191 }
192 
193 /*
194  * ... whereas tmpfs objects are accounted incrementally as
195  * pages are allocated, in order to allow large sparse files.
196  * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
197  * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
198  */
199 static inline int shmem_acct_block(unsigned long flags, long pages)
200 {
201         if (!(flags & VM_NORESERVE))
202                 return 0;
203 
204         return security_vm_enough_memory_mm(current->mm,
205                         pages * VM_ACCT(PAGE_SIZE));
206 }
207 
208 static inline void shmem_unacct_blocks(unsigned long flags, long pages)
209 {
210         if (flags & VM_NORESERVE)
211                 vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE));
212 }
213 
214 static inline bool shmem_inode_acct_block(struct inode *inode, long pages)
215 {
216         struct shmem_inode_info *info = SHMEM_I(inode);
217         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
218 
219         if (shmem_acct_block(info->flags, pages))
220                 return false;
221 
222         if (sbinfo->max_blocks) {
223                 if (percpu_counter_compare(&sbinfo->used_blocks,
224                                            sbinfo->max_blocks - pages) > 0)
225                         goto unacct;
226                 percpu_counter_add(&sbinfo->used_blocks, pages);
227         }
228 
229         return true;
230 
231 unacct:
232         shmem_unacct_blocks(info->flags, pages);
233         return false;
234 }
235 
236 static inline void shmem_inode_unacct_blocks(struct inode *inode, long pages)
237 {
238         struct shmem_inode_info *info = SHMEM_I(inode);
239         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
240 
241         if (sbinfo->max_blocks)
242                 percpu_counter_sub(&sbinfo->used_blocks, pages);
243         shmem_unacct_blocks(info->flags, pages);
244 }
245 
246 static const struct super_operations shmem_ops;
247 static const struct address_space_operations shmem_aops;
248 static const struct file_operations shmem_file_operations;
249 static const struct inode_operations shmem_inode_operations;
250 static const struct inode_operations shmem_dir_inode_operations;
251 static const struct inode_operations shmem_special_inode_operations;
252 static const struct vm_operations_struct shmem_vm_ops;
253 static struct file_system_type shmem_fs_type;
254 
255 bool vma_is_shmem(struct vm_area_struct *vma)
256 {
257         return vma->vm_ops == &shmem_vm_ops;
258 }
259 
260 static LIST_HEAD(shmem_swaplist);
261 static DEFINE_MUTEX(shmem_swaplist_mutex);
262 
263 static int shmem_reserve_inode(struct super_block *sb)
264 {
265         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
266         if (sbinfo->max_inodes) {
267                 spin_lock(&sbinfo->stat_lock);
268                 if (!sbinfo->free_inodes) {
269                         spin_unlock(&sbinfo->stat_lock);
270                         return -ENOSPC;
271                 }
272                 sbinfo->free_inodes--;
273                 spin_unlock(&sbinfo->stat_lock);
274         }
275         return 0;
276 }
277 
278 static void shmem_free_inode(struct super_block *sb)
279 {
280         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
281         if (sbinfo->max_inodes) {
282                 spin_lock(&sbinfo->stat_lock);
283                 sbinfo->free_inodes++;
284                 spin_unlock(&sbinfo->stat_lock);
285         }
286 }
287 
288 /**
289  * shmem_recalc_inode - recalculate the block usage of an inode
290  * @inode: inode to recalc
291  *
292  * We have to calculate the free blocks since the mm can drop
293  * undirtied hole pages behind our back.
294  *
295  * But normally   info->alloced == inode->i_mapping->nrpages + info->swapped
296  * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
297  *
298  * It has to be called with the spinlock held.
299  */
300 static void shmem_recalc_inode(struct inode *inode)
301 {
302         struct shmem_inode_info *info = SHMEM_I(inode);
303         long freed;
304 
305         freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
306         if (freed > 0) {
307                 info->alloced -= freed;
308                 inode->i_blocks -= freed * BLOCKS_PER_PAGE;
309                 shmem_inode_unacct_blocks(inode, freed);
310         }
311 }
312 
313 bool shmem_charge(struct inode *inode, long pages)
314 {
315         struct shmem_inode_info *info = SHMEM_I(inode);
316         unsigned long flags;
317 
318         if (!shmem_inode_acct_block(inode, pages))
319                 return false;
320 
321         /* nrpages adjustment first, then shmem_recalc_inode() when balanced */
322         inode->i_mapping->nrpages += pages;
323 
324         spin_lock_irqsave(&info->lock, flags);
325         info->alloced += pages;
326         inode->i_blocks += pages * BLOCKS_PER_PAGE;
327         shmem_recalc_inode(inode);
328         spin_unlock_irqrestore(&info->lock, flags);
329 
330         return true;
331 }
332 
333 void shmem_uncharge(struct inode *inode, long pages)
334 {
335         struct shmem_inode_info *info = SHMEM_I(inode);
336         unsigned long flags;
337 
338         /* nrpages adjustment done by __delete_from_page_cache() or caller */
339 
340         spin_lock_irqsave(&info->lock, flags);
341         info->alloced -= pages;
342         inode->i_blocks -= pages * BLOCKS_PER_PAGE;
343         shmem_recalc_inode(inode);
344         spin_unlock_irqrestore(&info->lock, flags);
345 
346         shmem_inode_unacct_blocks(inode, pages);
347 }
348 
349 /*
350  * Replace item expected in xarray by a new item, while holding xa_lock.
351  */
352 static int shmem_replace_entry(struct address_space *mapping,
353                         pgoff_t index, void *expected, void *replacement)
354 {
355         XA_STATE(xas, &mapping->i_pages, index);
356         void *item;
357 
358         VM_BUG_ON(!expected);
359         VM_BUG_ON(!replacement);
360         item = xas_load(&xas);
361         if (item != expected)
362                 return -ENOENT;
363         xas_store(&xas, replacement);
364         return 0;
365 }
366 
367 /*
368  * Sometimes, before we decide whether to proceed or to fail, we must check
369  * that an entry was not already brought back from swap by a racing thread.
370  *
371  * Checking page is not enough: by the time a SwapCache page is locked, it
372  * might be reused, and again be SwapCache, using the same swap as before.
373  */
374 static bool shmem_confirm_swap(struct address_space *mapping,
375                                pgoff_t index, swp_entry_t swap)
376 {
377         return xa_load(&mapping->i_pages, index) == swp_to_radix_entry(swap);
378 }
379 
380 /*
381  * Definitions for "huge tmpfs": tmpfs mounted with the huge= option
382  *
383  * SHMEM_HUGE_NEVER:
384  *      disables huge pages for the mount;
385  * SHMEM_HUGE_ALWAYS:
386  *      enables huge pages for the mount;
387  * SHMEM_HUGE_WITHIN_SIZE:
388  *      only allocate huge pages if the page will be fully within i_size,
389  *      also respect fadvise()/madvise() hints;
390  * SHMEM_HUGE_ADVISE:
391  *      only allocate huge pages if requested with fadvise()/madvise();
392  */
393 
394 #define SHMEM_HUGE_NEVER        0
395 #define SHMEM_HUGE_ALWAYS       1
396 #define SHMEM_HUGE_WITHIN_SIZE  2
397 #define SHMEM_HUGE_ADVISE       3
398 
399 /*
400  * Special values.
401  * Only can be set via /sys/kernel/mm/transparent_hugepage/shmem_enabled:
402  *
403  * SHMEM_HUGE_DENY:
404  *      disables huge on shm_mnt and all mounts, for emergency use;
405  * SHMEM_HUGE_FORCE:
406  *      enables huge on shm_mnt and all mounts, w/o needing option, for testing;
407  *
408  */
409 #define SHMEM_HUGE_DENY         (-1)
410 #define SHMEM_HUGE_FORCE        (-2)
411 
412 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
413 /* ifdef here to avoid bloating shmem.o when not necessary */
414 
415 static int shmem_huge __read_mostly;
416 
417 #if defined(CONFIG_SYSFS)
418 static int shmem_parse_huge(const char *str)
419 {
420         if (!strcmp(str, "never"))
421                 return SHMEM_HUGE_NEVER;
422         if (!strcmp(str, "always"))
423                 return SHMEM_HUGE_ALWAYS;
424         if (!strcmp(str, "within_size"))
425                 return SHMEM_HUGE_WITHIN_SIZE;
426         if (!strcmp(str, "advise"))
427                 return SHMEM_HUGE_ADVISE;
428         if (!strcmp(str, "deny"))
429                 return SHMEM_HUGE_DENY;
430         if (!strcmp(str, "force"))
431                 return SHMEM_HUGE_FORCE;
432         return -EINVAL;
433 }
434 #endif
435 
436 #if defined(CONFIG_SYSFS) || defined(CONFIG_TMPFS)
437 static const char *shmem_format_huge(int huge)
438 {
439         switch (huge) {
440         case SHMEM_HUGE_NEVER:
441                 return "never";
442         case SHMEM_HUGE_ALWAYS:
443                 return "always";
444         case SHMEM_HUGE_WITHIN_SIZE:
445                 return "within_size";
446         case SHMEM_HUGE_ADVISE:
447                 return "advise";
448         case SHMEM_HUGE_DENY:
449                 return "deny";
450         case SHMEM_HUGE_FORCE:
451                 return "force";
452         default:
453                 VM_BUG_ON(1);
454                 return "bad_val";
455         }
456 }
457 #endif
458 
459 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
460                 struct shrink_control *sc, unsigned long nr_to_split)
461 {
462         LIST_HEAD(list), *pos, *next;
463         LIST_HEAD(to_remove);
464         struct inode *inode;
465         struct shmem_inode_info *info;
466         struct page *page;
467         unsigned long batch = sc ? sc->nr_to_scan : 128;
468         int removed = 0, split = 0;
469 
470         if (list_empty(&sbinfo->shrinklist))
471                 return SHRINK_STOP;
472 
473         spin_lock(&sbinfo->shrinklist_lock);
474         list_for_each_safe(pos, next, &sbinfo->shrinklist) {
475                 info = list_entry(pos, struct shmem_inode_info, shrinklist);
476 
477                 /* pin the inode */
478                 inode = igrab(&info->vfs_inode);
479 
480                 /* inode is about to be evicted */
481                 if (!inode) {
482                         list_del_init(&info->shrinklist);
483                         removed++;
484                         goto next;
485                 }
486 
487                 /* Check if there's anything to gain */
488                 if (round_up(inode->i_size, PAGE_SIZE) ==
489                                 round_up(inode->i_size, HPAGE_PMD_SIZE)) {
490                         list_move(&info->shrinklist, &to_remove);
491                         removed++;
492                         goto next;
493                 }
494 
495                 list_move(&info->shrinklist, &list);
496 next:
497                 if (!--batch)
498                         break;
499         }
500         spin_unlock(&sbinfo->shrinklist_lock);
501 
502         list_for_each_safe(pos, next, &to_remove) {
503                 info = list_entry(pos, struct shmem_inode_info, shrinklist);
504                 inode = &info->vfs_inode;
505                 list_del_init(&info->shrinklist);
506                 iput(inode);
507         }
508 
509         list_for_each_safe(pos, next, &list) {
510                 int ret;
511 
512                 info = list_entry(pos, struct shmem_inode_info, shrinklist);
513                 inode = &info->vfs_inode;
514 
515                 if (nr_to_split && split >= nr_to_split)
516                         goto leave;
517 
518                 page = find_get_page(inode->i_mapping,
519                                 (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT);
520                 if (!page)
521                         goto drop;
522 
523                 /* No huge page at the end of the file: nothing to split */
524                 if (!PageTransHuge(page)) {
525                         put_page(page);
526                         goto drop;
527                 }
528 
529                 /*
530                  * Leave the inode on the list if we failed to lock
531                  * the page at this time.
532                  *
533                  * Waiting for the lock may lead to deadlock in the
534                  * reclaim path.
535                  */
536                 if (!trylock_page(page)) {
537                         put_page(page);
538                         goto leave;
539                 }
540 
541                 ret = split_huge_page(page);
542                 unlock_page(page);
543                 put_page(page);
544 
545                 /* If split failed leave the inode on the list */
546                 if (ret)
547                         goto leave;
548 
549                 split++;
550 drop:
551                 list_del_init(&info->shrinklist);
552                 removed++;
553 leave:
554                 iput(inode);
555         }
556 
557         spin_lock(&sbinfo->shrinklist_lock);
558         list_splice_tail(&list, &sbinfo->shrinklist);
559         sbinfo->shrinklist_len -= removed;
560         spin_unlock(&sbinfo->shrinklist_lock);
561 
562         return split;
563 }
564 
565 static long shmem_unused_huge_scan(struct super_block *sb,
566                 struct shrink_control *sc)
567 {
568         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
569 
570         if (!READ_ONCE(sbinfo->shrinklist_len))
571                 return SHRINK_STOP;
572 
573         return shmem_unused_huge_shrink(sbinfo, sc, 0);
574 }
575 
576 static long shmem_unused_huge_count(struct super_block *sb,
577                 struct shrink_control *sc)
578 {
579         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
580         return READ_ONCE(sbinfo->shrinklist_len);
581 }
582 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */
583 
584 #define shmem_huge SHMEM_HUGE_DENY
585 
586 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
587                 struct shrink_control *sc, unsigned long nr_to_split)
588 {
589         return 0;
590 }
591 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
592 
593 static inline bool is_huge_enabled(struct shmem_sb_info *sbinfo)
594 {
595         if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
596             (shmem_huge == SHMEM_HUGE_FORCE || sbinfo->huge) &&
597             shmem_huge != SHMEM_HUGE_DENY)
598                 return true;
599         return false;
600 }
601 
602 /*
603  * Like add_to_page_cache_locked, but error if expected item has gone.
604  */
605 static int shmem_add_to_page_cache(struct page *page,
606                                    struct address_space *mapping,
607                                    pgoff_t index, void *expected, gfp_t gfp,
608                                    struct mm_struct *charge_mm)
609 {
610         XA_STATE_ORDER(xas, &mapping->i_pages, index, compound_order(page));
611         unsigned long i = 0;
612         unsigned long nr = compound_nr(page);
613         int error;
614 
615         VM_BUG_ON_PAGE(PageTail(page), page);
616         VM_BUG_ON_PAGE(index != round_down(index, nr), page);
617         VM_BUG_ON_PAGE(!PageLocked(page), page);
618         VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
619         VM_BUG_ON(expected && PageTransHuge(page));
620 
621         page_ref_add(page, nr);
622         page->mapping = mapping;
623         page->index = index;
624 
625         if (!PageSwapCache(page)) {
626                 error = mem_cgroup_charge(page, charge_mm, gfp);
627                 if (error) {
628                         if (PageTransHuge(page)) {
629                                 count_vm_event(THP_FILE_FALLBACK);
630                                 count_vm_event(THP_FILE_FALLBACK_CHARGE);
631                         }
632                         goto error;
633                 }
634         }
635         cgroup_throttle_swaprate(page, gfp);
636 
637         do {
638                 void *entry;
639                 xas_lock_irq(&xas);
640                 entry = xas_find_conflict(&xas);
641                 if (entry != expected)
642                         xas_set_err(&xas, -EEXIST);
643                 xas_create_range(&xas);
644                 if (xas_error(&xas))
645                         goto unlock;
646 next:
647                 xas_store(&xas, page);
648                 if (++i < nr) {
649                         xas_next(&xas);
650                         goto next;
651                 }
652                 if (PageTransHuge(page)) {
653                         count_vm_event(THP_FILE_ALLOC);
654                         __inc_node_page_state(page, NR_SHMEM_THPS);
655                 }
656                 mapping->nrpages += nr;
657                 __mod_lruvec_page_state(page, NR_FILE_PAGES, nr);
658                 __mod_lruvec_page_state(page, NR_SHMEM, nr);
659 unlock:
660                 xas_unlock_irq(&xas);
661         } while (xas_nomem(&xas, gfp));
662 
663         if (xas_error(&xas)) {
664                 error = xas_error(&xas);
665                 goto error;
666         }
667 
668         return 0;
669 error:
670         page->mapping = NULL;
671         page_ref_sub(page, nr);
672         return error;
673 }
674 
675 /*
676  * Like delete_from_page_cache, but substitutes swap for page.
677  */
678 static void shmem_delete_from_page_cache(struct page *page, void *radswap)
679 {
680         struct address_space *mapping = page->mapping;
681         int error;
682 
683         VM_BUG_ON_PAGE(PageCompound(page), page);
684 
685         xa_lock_irq(&mapping->i_pages);
686         error = shmem_replace_entry(mapping, page->index, page, radswap);
687         page->mapping = NULL;
688         mapping->nrpages--;
689         __dec_lruvec_page_state(page, NR_FILE_PAGES);
690         __dec_lruvec_page_state(page, NR_SHMEM);
691         xa_unlock_irq(&mapping->i_pages);
692         put_page(page);
693         BUG_ON(error);
694 }
695 
696 /*
697  * Remove swap entry from page cache, free the swap and its page cache.
698  */
699 static int shmem_free_swap(struct address_space *mapping,
700                            pgoff_t index, void *radswap)
701 {
702         void *old;
703 
704         old = xa_cmpxchg_irq(&mapping->i_pages, index, radswap, NULL, 0);
705         if (old != radswap)
706                 return -ENOENT;
707         free_swap_and_cache(radix_to_swp_entry(radswap));
708         return 0;
709 }
710 
711 /*
712  * Determine (in bytes) how many of the shmem object's pages mapped by the
713  * given offsets are swapped out.
714  *
715  * This is safe to call without i_mutex or the i_pages lock thanks to RCU,
716  * as long as the inode doesn't go away and racy results are not a problem.
717  */
718 unsigned long shmem_partial_swap_usage(struct address_space *mapping,
719                                                 pgoff_t start, pgoff_t end)
720 {
721         XA_STATE(xas, &mapping->i_pages, start);
722         struct page *page;
723         unsigned long swapped = 0;
724 
725         rcu_read_lock();
726         xas_for_each(&xas, page, end - 1) {
727                 if (xas_retry(&xas, page))
728                         continue;
729                 if (xa_is_value(page))
730                         swapped++;
731 
732                 if (need_resched()) {
733                         xas_pause(&xas);
734                         cond_resched_rcu();
735                 }
736         }
737 
738         rcu_read_unlock();
739 
740         return swapped << PAGE_SHIFT;
741 }
742 
743 /*
744  * Determine (in bytes) how many of the shmem object's pages mapped by the
745  * given vma is swapped out.
746  *
747  * This is safe to call without i_mutex or the i_pages lock thanks to RCU,
748  * as long as the inode doesn't go away and racy results are not a problem.
749  */
750 unsigned long shmem_swap_usage(struct vm_area_struct *vma)
751 {
752         struct inode *inode = file_inode(vma->vm_file);
753         struct shmem_inode_info *info = SHMEM_I(inode);
754         struct address_space *mapping = inode->i_mapping;
755         unsigned long swapped;
756 
757         /* Be careful as we don't hold info->lock */
758         swapped = READ_ONCE(info->swapped);
759 
760         /*
761          * The easier cases are when the shmem object has nothing in swap, or
762          * the vma maps it whole. Then we can simply use the stats that we
763          * already track.
764          */
765         if (!swapped)
766                 return 0;
767 
768         if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size)
769                 return swapped << PAGE_SHIFT;
770 
771         /* Here comes the more involved part */
772         return shmem_partial_swap_usage(mapping,
773                         linear_page_index(vma, vma->vm_start),
774                         linear_page_index(vma, vma->vm_end));
775 }
776 
777 /*
778  * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists.
779  */
780 void shmem_unlock_mapping(struct address_space *mapping)
781 {
782         struct pagevec pvec;
783         pgoff_t indices[PAGEVEC_SIZE];
784         pgoff_t index = 0;
785 
786         pagevec_init(&pvec);
787         /*
788          * Minor point, but we might as well stop if someone else SHM_LOCKs it.
789          */
790         while (!mapping_unevictable(mapping)) {
791                 /*
792                  * Avoid pagevec_lookup(): find_get_pages() returns 0 as if it
793                  * has finished, if it hits a row of PAGEVEC_SIZE swap entries.
794                  */
795                 pvec.nr = find_get_entries(mapping, index,
796                                            PAGEVEC_SIZE, pvec.pages, indices);
797                 if (!pvec.nr)
798                         break;
799                 index = indices[pvec.nr - 1] + 1;
800                 pagevec_remove_exceptionals(&pvec);
801                 check_move_unevictable_pages(&pvec);
802                 pagevec_release(&pvec);
803                 cond_resched();
804         }
805 }
806 
807 /*
808  * Check whether a hole-punch or truncation needs to split a huge page,
809  * returning true if no split was required, or the split has been successful.
810  *
811  * Eviction (or truncation to 0 size) should never need to split a huge page;
812  * but in rare cases might do so, if shmem_undo_range() failed to trylock on
813  * head, and then succeeded to trylock on tail.
814  *
815  * A split can only succeed when there are no additional references on the
816  * huge page: so the split below relies upon find_get_entries() having stopped
817  * when it found a subpage of the huge page, without getting further references.
818  */
819 static bool shmem_punch_compound(struct page *page, pgoff_t start, pgoff_t end)
820 {
821         if (!PageTransCompound(page))
822                 return true;
823 
824         /* Just proceed to delete a huge page wholly within the range punched */
825         if (PageHead(page) &&
826             page->index >= start && page->index + HPAGE_PMD_NR <= end)
827                 return true;
828 
829         /* Try to split huge page, so we can truly punch the hole or truncate */
830         return split_huge_page(page) >= 0;
831 }
832 
833 /*
834  * Remove range of pages and swap entries from page cache, and free them.
835  * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate.
836  */
837 static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
838                                                                  bool unfalloc)
839 {
840         struct address_space *mapping = inode->i_mapping;
841         struct shmem_inode_info *info = SHMEM_I(inode);
842         pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
843         pgoff_t end = (lend + 1) >> PAGE_SHIFT;
844         unsigned int partial_start = lstart & (PAGE_SIZE - 1);
845         unsigned int partial_end = (lend + 1) & (PAGE_SIZE - 1);
846         struct pagevec pvec;
847         pgoff_t indices[PAGEVEC_SIZE];
848         long nr_swaps_freed = 0;
849         pgoff_t index;
850         int i;
851 
852         if (lend == -1)
853                 end = -1;       /* unsigned, so actually very big */
854 
855         pagevec_init(&pvec);
856         index = start;
857         while (index < end) {
858                 pvec.nr = find_get_entries(mapping, index,
859                         min(end - index, (pgoff_t)PAGEVEC_SIZE),
860                         pvec.pages, indices);
861                 if (!pvec.nr)
862                         break;
863                 for (i = 0; i < pagevec_count(&pvec); i++) {
864                         struct page *page = pvec.pages[i];
865 
866                         index = indices[i];
867                         if (index >= end)
868                                 break;
869 
870                         if (xa_is_value(page)) {
871                                 if (unfalloc)
872                                         continue;
873                                 nr_swaps_freed += !shmem_free_swap(mapping,
874                                                                 index, page);
875                                 continue;
876                         }
877 
878                         VM_BUG_ON_PAGE(page_to_pgoff(page) != index, page);
879 
880                         if (!trylock_page(page))
881                                 continue;
882 
883                         if ((!unfalloc || !PageUptodate(page)) &&
884                             page_mapping(page) == mapping) {
885                                 VM_BUG_ON_PAGE(PageWriteback(page), page);
886                                 if (shmem_punch_compound(page, start, end))
887                                         truncate_inode_page(mapping, page);
888                         }
889                         unlock_page(page);
890                 }
891                 pagevec_remove_exceptionals(&pvec);
892                 pagevec_release(&pvec);
893                 cond_resched();
894                 index++;
895         }
896 
897         if (partial_start) {
898                 struct page *page = NULL;
899                 shmem_getpage(inode, start - 1, &page, SGP_READ);
900                 if (page) {
901                         unsigned int top = PAGE_SIZE;
902                         if (start > end) {
903                                 top = partial_end;
904                                 partial_end = 0;
905                         }
906                         zero_user_segment(page, partial_start, top);
907                         set_page_dirty(page);
908                         unlock_page(page);
909                         put_page(page);
910                 }
911         }
912         if (partial_end) {
913                 struct page *page = NULL;
914                 shmem_getpage(inode, end, &page, SGP_READ);
915                 if (page) {
916                         zero_user_segment(page, 0, partial_end);
917                         set_page_dirty(page);
918                         unlock_page(page);
919                         put_page(page);
920                 }
921         }
922         if (start >= end)
923                 return;
924 
925         index = start;
926         while (index < end) {
927                 cond_resched();
928 
929                 pvec.nr = find_get_entries(mapping, index,
930                                 min(end - index, (pgoff_t)PAGEVEC_SIZE),
931                                 pvec.pages, indices);
932                 if (!pvec.nr) {
933                         /* If all gone or hole-punch or unfalloc, we're done */
934                         if (index == start || end != -1)
935                                 break;
936                         /* But if truncating, restart to make sure all gone */
937                         index = start;
938                         continue;
939                 }
940                 for (i = 0; i < pagevec_count(&pvec); i++) {
941                         struct page *page = pvec.pages[i];
942 
943                         index = indices[i];
944                         if (index >= end)
945                                 break;
946 
947                         if (xa_is_value(page)) {
948                                 if (unfalloc)
949                                         continue;
950                                 if (shmem_free_swap(mapping, index, page)) {
951                                         /* Swap was replaced by page: retry */
952                                         index--;
953                                         break;
954                                 }
955                                 nr_swaps_freed++;
956                                 continue;
957                         }
958 
959                         lock_page(page);
960 
961                         if (!unfalloc || !PageUptodate(page)) {
962                                 if (page_mapping(page) != mapping) {
963                                         /* Page was replaced by swap: retry */
964                                         unlock_page(page);
965                                         index--;
966                                         break;
967                                 }
968                                 VM_BUG_ON_PAGE(PageWriteback(page), page);
969                                 if (shmem_punch_compound(page, start, end))
970                                         truncate_inode_page(mapping, page);
971                                 else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
972                                         /* Wipe the page and don't get stuck */
973                                         clear_highpage(page);
974                                         flush_dcache_page(page);
975                                         set_page_dirty(page);
976                                         if (index <
977                                             round_up(start, HPAGE_PMD_NR))
978                                                 start = index + 1;
979                                 }
980                         }
981                         unlock_page(page);
982                 }
983                 pagevec_remove_exceptionals(&pvec);
984                 pagevec_release(&pvec);
985                 index++;
986         }
987 
988         spin_lock_irq(&info->lock);
989         info->swapped -= nr_swaps_freed;
990         shmem_recalc_inode(inode);
991         spin_unlock_irq(&info->lock);
992 }
993 
994 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
995 {
996         shmem_undo_range(inode, lstart, lend, false);
997         inode->i_ctime = inode->i_mtime = current_time(inode);
998 }
999 EXPORT_SYMBOL_GPL(shmem_truncate_range);
1000 
1001 static int shmem_getattr(const struct path *path, struct kstat *stat,
1002                          u32 request_mask, unsigned int query_flags)
1003 {
1004         struct inode *inode = path->dentry->d_inode;
1005         struct shmem_inode_info *info = SHMEM_I(inode);
1006         struct shmem_sb_info *sb_info = SHMEM_SB(inode->i_sb);
1007 
1008         if (info->alloced - info->swapped != inode->i_mapping->nrpages) {
1009                 spin_lock_irq(&info->lock);
1010                 shmem_recalc_inode(inode);
1011                 spin_unlock_irq(&info->lock);
1012         }
1013         generic_fillattr(inode, stat);
1014 
1015         if (is_huge_enabled(sb_info))
1016                 stat->blksize = HPAGE_PMD_SIZE;
1017 
1018         return 0;
1019 }
1020 
1021 static int shmem_setattr(struct dentry *dentry, struct iattr *attr)
1022 {
1023         struct inode *inode = d_inode(dentry);
1024         struct shmem_inode_info *info = SHMEM_I(inode);
1025         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1026         int error;
1027 
1028         error = setattr_prepare(dentry, attr);
1029         if (error)
1030                 return error;
1031 
1032         if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
1033                 loff_t oldsize = inode->i_size;
1034                 loff_t newsize = attr->ia_size;
1035 
1036                 /* protected by i_mutex */
1037                 if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) ||
1038                     (newsize > oldsize && (info->seals & F_SEAL_GROW)))
1039                         return -EPERM;
1040 
1041                 if (newsize != oldsize) {
1042                         error = shmem_reacct_size(SHMEM_I(inode)->flags,
1043                                         oldsize, newsize);
1044                         if (error)
1045                                 return error;
1046                         i_size_write(inode, newsize);
1047                         inode->i_ctime = inode->i_mtime = current_time(inode);
1048                 }
1049                 if (newsize <= oldsize) {
1050                         loff_t holebegin = round_up(newsize, PAGE_SIZE);
1051                         if (oldsize > holebegin)
1052                                 unmap_mapping_range(inode->i_mapping,
1053                                                         holebegin, 0, 1);
1054                         if (info->alloced)
1055                                 shmem_truncate_range(inode,
1056                                                         newsize, (loff_t)-1);
1057                         /* unmap again to remove racily COWed private pages */
1058                         if (oldsize > holebegin)
1059                                 unmap_mapping_range(inode->i_mapping,
1060                                                         holebegin, 0, 1);
1061 
1062                         /*
1063                          * Part of the huge page can be beyond i_size: subject
1064                          * to shrink under memory pressure.
1065                          */
1066                         if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
1067                                 spin_lock(&sbinfo->shrinklist_lock);
1068                                 /*
1069                                  * _careful to defend against unlocked access to
1070                                  * ->shrink_list in shmem_unused_huge_shrink()
1071                                  */
1072                                 if (list_empty_careful(&info->shrinklist)) {
1073                                         list_add_tail(&info->shrinklist,
1074                                                         &sbinfo->shrinklist);
1075                                         sbinfo->shrinklist_len++;
1076                                 }
1077                                 spin_unlock(&sbinfo->shrinklist_lock);
1078                         }
1079                 }
1080         }
1081 
1082         setattr_copy(inode, attr);
1083         if (attr->ia_valid & ATTR_MODE)
1084                 error = posix_acl_chmod(inode, inode->i_mode);
1085         return error;
1086 }
1087 
1088 static void shmem_evict_inode(struct inode *inode)
1089 {
1090         struct shmem_inode_info *info = SHMEM_I(inode);
1091         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1092 
1093         if (inode->i_mapping->a_ops == &shmem_aops) {
1094                 shmem_unacct_size(info->flags, inode->i_size);
1095                 inode->i_size = 0;
1096                 shmem_truncate_range(inode, 0, (loff_t)-1);
1097                 if (!list_empty(&info->shrinklist)) {
1098                         spin_lock(&sbinfo->shrinklist_lock);
1099                         if (!list_empty(&info->shrinklist)) {
1100                                 list_del_init(&info->shrinklist);
1101                                 sbinfo->shrinklist_len--;
1102                         }
1103                         spin_unlock(&sbinfo->shrinklist_lock);
1104                 }
1105                 while (!list_empty(&info->swaplist)) {
1106                         /* Wait while shmem_unuse() is scanning this inode... */
1107                         wait_var_event(&info->stop_eviction,
1108                                        !atomic_read(&info->stop_eviction));
1109                         mutex_lock(&shmem_swaplist_mutex);
1110                         /* ...but beware of the race if we peeked too early */
1111                         if (!atomic_read(&info->stop_eviction))
1112                                 list_del_init(&info->swaplist);
1113                         mutex_unlock(&shmem_swaplist_mutex);
1114                 }
1115         }
1116 
1117         simple_xattrs_free(&info->xattrs);
1118         WARN_ON(inode->i_blocks);
1119         shmem_free_inode(inode->i_sb);
1120         clear_inode(inode);
1121 }
1122 
1123 extern struct swap_info_struct *swap_info[];
1124 
1125 static int shmem_find_swap_entries(struct address_space *mapping,
1126                                    pgoff_t start, unsigned int nr_entries,
1127                                    struct page **entries, pgoff_t *indices,
1128                                    unsigned int type, bool frontswap)
1129 {
1130         XA_STATE(xas, &mapping->i_pages, start);
1131         struct page *page;
1132         swp_entry_t entry;
1133         unsigned int ret = 0;
1134 
1135         if (!nr_entries)
1136                 return 0;
1137 
1138         rcu_read_lock();
1139         xas_for_each(&xas, page, ULONG_MAX) {
1140                 if (xas_retry(&xas, page))
1141                         continue;
1142 
1143                 if (!xa_is_value(page))
1144                         continue;
1145 
1146                 entry = radix_to_swp_entry(page);
1147                 if (swp_type(entry) != type)
1148                         continue;
1149                 if (frontswap &&
1150                     !frontswap_test(swap_info[type], swp_offset(entry)))
1151                         continue;
1152 
1153                 indices[ret] = xas.xa_index;
1154                 entries[ret] = page;
1155 
1156                 if (need_resched()) {
1157                         xas_pause(&xas);
1158                         cond_resched_rcu();
1159                 }
1160                 if (++ret == nr_entries)
1161                         break;
1162         }
1163         rcu_read_unlock();
1164 
1165         return ret;
1166 }
1167 
1168 /*
1169  * Move the swapped pages for an inode to page cache. Returns the count
1170  * of pages swapped in, or the error in case of failure.
1171  */
1172 static int shmem_unuse_swap_entries(struct inode *inode, struct pagevec pvec,
1173                                     pgoff_t *indices)
1174 {
1175         int i = 0;
1176         int ret = 0;
1177         int error = 0;
1178         struct address_space *mapping = inode->i_mapping;
1179 
1180         for (i = 0; i < pvec.nr; i++) {
1181                 struct page *page = pvec.pages[i];
1182 
1183                 if (!xa_is_value(page))
1184                         continue;
1185                 error = shmem_swapin_page(inode, indices[i],
1186                                           &page, SGP_CACHE,
1187                                           mapping_gfp_mask(mapping),
1188                                           NULL, NULL);
1189                 if (error == 0) {
1190                         unlock_page(page);
1191                         put_page(page);
1192                         ret++;
1193                 }
1194                 if (error == -ENOMEM)
1195                         break;
1196                 error = 0;
1197         }
1198         return error ? error : ret;
1199 }
1200 
1201 /*
1202  * If swap found in inode, free it and move page from swapcache to filecache.
1203  */
1204 static int shmem_unuse_inode(struct inode *inode, unsigned int type,
1205                              bool frontswap, unsigned long *fs_pages_to_unuse)
1206 {
1207         struct address_space *mapping = inode->i_mapping;
1208         pgoff_t start = 0;
1209         struct pagevec pvec;
1210         pgoff_t indices[PAGEVEC_SIZE];
1211         bool frontswap_partial = (frontswap && *fs_pages_to_unuse > 0);
1212         int ret = 0;
1213 
1214         pagevec_init(&pvec);
1215         do {
1216                 unsigned int nr_entries = PAGEVEC_SIZE;
1217 
1218                 if (frontswap_partial && *fs_pages_to_unuse < PAGEVEC_SIZE)
1219                         nr_entries = *fs_pages_to_unuse;
1220 
1221                 pvec.nr = shmem_find_swap_entries(mapping, start, nr_entries,
1222                                                   pvec.pages, indices,
1223                                                   type, frontswap);
1224                 if (pvec.nr == 0) {
1225                         ret = 0;
1226                         break;
1227                 }
1228 
1229                 ret = shmem_unuse_swap_entries(inode, pvec, indices);
1230                 if (ret < 0)
1231                         break;
1232 
1233                 if (frontswap_partial) {
1234                         *fs_pages_to_unuse -= ret;
1235                         if (*fs_pages_to_unuse == 0) {
1236                                 ret = FRONTSWAP_PAGES_UNUSED;
1237                                 break;
1238                         }
1239                 }
1240 
1241                 start = indices[pvec.nr - 1];
1242         } while (true);
1243 
1244         return ret;
1245 }
1246 
1247 /*
1248  * Read all the shared memory data that resides in the swap
1249  * device 'type' back into memory, so the swap device can be
1250  * unused.
1251  */
1252 int shmem_unuse(unsigned int type, bool frontswap,
1253                 unsigned long *fs_pages_to_unuse)
1254 {
1255         struct shmem_inode_info *info, *next;
1256         int error = 0;
1257 
1258         if (list_empty(&shmem_swaplist))
1259                 return 0;
1260 
1261         mutex_lock(&shmem_swaplist_mutex);
1262         list_for_each_entry_safe(info, next, &shmem_swaplist, swaplist) {
1263                 if (!info->swapped) {
1264                         list_del_init(&info->swaplist);
1265                         continue;
1266                 }
1267                 /*
1268                  * Drop the swaplist mutex while searching the inode for swap;
1269                  * but before doing so, make sure shmem_evict_inode() will not
1270                  * remove placeholder inode from swaplist, nor let it be freed
1271                  * (igrab() would protect from unlink, but not from unmount).
1272                  */
1273                 atomic_inc(&info->stop_eviction);
1274                 mutex_unlock(&shmem_swaplist_mutex);
1275 
1276                 error = shmem_unuse_inode(&info->vfs_inode, type, frontswap,
1277                                           fs_pages_to_unuse);
1278                 cond_resched();
1279 
1280                 mutex_lock(&shmem_swaplist_mutex);
1281                 next = list_next_entry(info, swaplist);
1282                 if (!info->swapped)
1283                         list_del_init(&info->swaplist);
1284                 if (atomic_dec_and_test(&info->stop_eviction))
1285                         wake_up_var(&info->stop_eviction);
1286                 if (error)
1287                         break;
1288         }
1289         mutex_unlock(&shmem_swaplist_mutex);
1290 
1291         return error;
1292 }
1293 
1294 /*
1295  * Move the page from the page cache to the swap cache.
1296  */
1297 static int shmem_writepage(struct page *page, struct writeback_control *wbc)
1298 {
1299         struct shmem_inode_info *info;
1300         struct address_space *mapping;
1301         struct inode *inode;
1302         swp_entry_t swap;
1303         pgoff_t index;
1304 
1305         VM_BUG_ON_PAGE(PageCompound(page), page);
1306         BUG_ON(!PageLocked(page));
1307         mapping = page->mapping;
1308         index = page->index;
1309         inode = mapping->host;
1310         info = SHMEM_I(inode);
1311         if (info->flags & VM_LOCKED)
1312                 goto redirty;
1313         if (!total_swap_pages)
1314                 goto redirty;
1315 
1316         /*
1317          * Our capabilities prevent regular writeback or sync from ever calling
1318          * shmem_writepage; but a stacking filesystem might use ->writepage of
1319          * its underlying filesystem, in which case tmpfs should write out to
1320          * swap only in response to memory pressure, and not for the writeback
1321          * threads or sync.
1322          */
1323         if (!wbc->for_reclaim) {
1324                 WARN_ON_ONCE(1);        /* Still happens? Tell us about it! */
1325                 goto redirty;
1326         }
1327 
1328         /*
1329          * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC
1330          * value into swapfile.c, the only way we can correctly account for a
1331          * fallocated page arriving here is now to initialize it and write it.
1332          *
1333          * That's okay for a page already fallocated earlier, but if we have
1334          * not yet completed the fallocation, then (a) we want to keep track
1335          * of this page in case we have to undo it, and (b) it may not be a
1336          * good idea to continue anyway, once we're pushing into swap.  So
1337          * reactivate the page, and let shmem_fallocate() quit when too many.
1338          */
1339         if (!PageUptodate(page)) {
1340                 if (inode->i_private) {
1341                         struct shmem_falloc *shmem_falloc;
1342                         spin_lock(&inode->i_lock);
1343                         shmem_falloc = inode->i_private;
1344                         if (shmem_falloc &&
1345                             !shmem_falloc->waitq &&
1346                             index >= shmem_falloc->start &&
1347                             index < shmem_falloc->next)
1348                                 shmem_falloc->nr_unswapped++;
1349                         else
1350                                 shmem_falloc = NULL;
1351                         spin_unlock(&inode->i_lock);
1352                         if (shmem_falloc)
1353                                 goto redirty;
1354                 }
1355                 clear_highpage(page);
1356                 flush_dcache_page(page);
1357                 SetPageUptodate(page);
1358         }
1359 
1360         swap = get_swap_page(page);
1361         if (!swap.val)
1362                 goto redirty;
1363 
1364         /*
1365          * Add inode to shmem_unuse()'s list of swapped-out inodes,
1366          * if it's not already there.  Do it now before the page is
1367          * moved to swap cache, when its pagelock no longer protects
1368          * the inode from eviction.  But don't unlock the mutex until
1369          * we've incremented swapped, because shmem_unuse_inode() will
1370          * prune a !swapped inode from the swaplist under this mutex.
1371          */
1372         mutex_lock(&shmem_swaplist_mutex);
1373         if (list_empty(&info->swaplist))
1374                 list_add(&info->swaplist, &shmem_swaplist);
1375 
1376         if (add_to_swap_cache(page, swap,
1377                         __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN) == 0) {
1378                 spin_lock_irq(&info->lock);
1379                 shmem_recalc_inode(inode);
1380                 info->swapped++;
1381                 spin_unlock_irq(&info->lock);
1382 
1383                 swap_shmem_alloc(swap);
1384                 shmem_delete_from_page_cache(page, swp_to_radix_entry(swap));
1385 
1386                 mutex_unlock(&shmem_swaplist_mutex);
1387                 BUG_ON(page_mapped(page));
1388                 swap_writepage(page, wbc);
1389                 return 0;
1390         }
1391 
1392         mutex_unlock(&shmem_swaplist_mutex);
1393         put_swap_page(page, swap);
1394 redirty:
1395         set_page_dirty(page);
1396         if (wbc->for_reclaim)
1397                 return AOP_WRITEPAGE_ACTIVATE;  /* Return with page locked */
1398         unlock_page(page);
1399         return 0;
1400 }
1401 
1402 #if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS)
1403 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1404 {
1405         char buffer[64];
1406 
1407         if (!mpol || mpol->mode == MPOL_DEFAULT)
1408                 return;         /* show nothing */
1409 
1410         mpol_to_str(buffer, sizeof(buffer), mpol);
1411 
1412         seq_printf(seq, ",mpol=%s", buffer);
1413 }
1414 
1415 static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1416 {
1417         struct mempolicy *mpol = NULL;
1418         if (sbinfo->mpol) {
1419                 spin_lock(&sbinfo->stat_lock);  /* prevent replace/use races */
1420                 mpol = sbinfo->mpol;
1421                 mpol_get(mpol);
1422                 spin_unlock(&sbinfo->stat_lock);
1423         }
1424         return mpol;
1425 }
1426 #else /* !CONFIG_NUMA || !CONFIG_TMPFS */
1427 static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1428 {
1429 }
1430 static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1431 {
1432         return NULL;
1433 }
1434 #endif /* CONFIG_NUMA && CONFIG_TMPFS */
1435 #ifndef CONFIG_NUMA
1436 #define vm_policy vm_private_data
1437 #endif
1438 
1439 static void shmem_pseudo_vma_init(struct vm_area_struct *vma,
1440                 struct shmem_inode_info *info, pgoff_t index)
1441 {
1442         /* Create a pseudo vma that just contains the policy */
1443         vma_init(vma, NULL);
1444         /* Bias interleave by inode number to distribute better across nodes */
1445         vma->vm_pgoff = index + info->vfs_inode.i_ino;
1446         vma->vm_policy = mpol_shared_policy_lookup(&info->policy, index);
1447 }
1448 
1449 static void shmem_pseudo_vma_destroy(struct vm_area_struct *vma)
1450 {
1451         /* Drop reference taken by mpol_shared_policy_lookup() */
1452         mpol_cond_put(vma->vm_policy);
1453 }
1454 
1455 static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
1456                         struct shmem_inode_info *info, pgoff_t index)
1457 {
1458         struct vm_area_struct pvma;
1459         struct page *page;
1460         struct vm_fault vmf;
1461 
1462         shmem_pseudo_vma_init(&pvma, info, index);
1463         vmf.vma = &pvma;
1464         vmf.address = 0;
1465         page = swap_cluster_readahead(swap, gfp, &vmf);
1466         shmem_pseudo_vma_destroy(&pvma);
1467 
1468         return page;
1469 }
1470 
1471 static struct page *shmem_alloc_hugepage(gfp_t gfp,
1472                 struct shmem_inode_info *info, pgoff_t index)
1473 {
1474         struct vm_area_struct pvma;
1475         struct address_space *mapping = info->vfs_inode.i_mapping;
1476         pgoff_t hindex;
1477         struct page *page;
1478 
1479         hindex = round_down(index, HPAGE_PMD_NR);
1480         if (xa_find(&mapping->i_pages, &hindex, hindex + HPAGE_PMD_NR - 1,
1481                                                                 XA_PRESENT))
1482                 return NULL;
1483 
1484         shmem_pseudo_vma_init(&pvma, info, hindex);
1485         page = alloc_pages_vma(gfp | __GFP_COMP | __GFP_NORETRY | __GFP_NOWARN,
1486                         HPAGE_PMD_ORDER, &pvma, 0, numa_node_id(), true);
1487         shmem_pseudo_vma_destroy(&pvma);
1488         if (page)
1489                 prep_transhuge_page(page);
1490         else
1491                 count_vm_event(THP_FILE_FALLBACK);
1492         return page;
1493 }
1494 
1495 static struct page *shmem_alloc_page(gfp_t gfp,
1496                         struct shmem_inode_info *info, pgoff_t index)
1497 {
1498         struct vm_area_struct pvma;
1499         struct page *page;
1500 
1501         shmem_pseudo_vma_init(&pvma, info, index);
1502         page = alloc_page_vma(gfp, &pvma, 0);
1503         shmem_pseudo_vma_destroy(&pvma);
1504 
1505         return page;
1506 }
1507 
1508 static struct page *shmem_alloc_and_acct_page(gfp_t gfp,
1509                 struct inode *inode,
1510                 pgoff_t index, bool huge)
1511 {
1512         struct shmem_inode_info *info = SHMEM_I(inode);
1513         struct page *page;
1514         int nr;
1515         int err = -ENOSPC;
1516 
1517         if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
1518                 huge = false;
1519         nr = huge ? HPAGE_PMD_NR : 1;
1520 
1521         if (!shmem_inode_acct_block(inode, nr))
1522                 goto failed;
1523 
1524         if (huge)
1525                 page = shmem_alloc_hugepage(gfp, info, index);
1526         else
1527                 page = shmem_alloc_page(gfp, info, index);
1528         if (page) {
1529                 __SetPageLocked(page);
1530                 __SetPageSwapBacked(page);
1531                 return page;
1532         }
1533 
1534         err = -ENOMEM;
1535         shmem_inode_unacct_blocks(inode, nr);
1536 failed:
1537         return ERR_PTR(err);
1538 }
1539 
1540 /*
1541  * When a page is moved from swapcache to shmem filecache (either by the
1542  * usual swapin of shmem_getpage_gfp(), or by the less common swapoff of
1543  * shmem_unuse_inode()), it may have been read in earlier from swap, in
1544  * ignorance of the mapping it belongs to.  If that mapping has special
1545  * constraints (like the gma500 GEM driver, which requires RAM below 4GB),
1546  * we may need to copy to a suitable page before moving to filecache.
1547  *
1548  * In a future release, this may well be extended to respect cpuset and
1549  * NUMA mempolicy, and applied also to anonymous pages in do_swap_page();
1550  * but for now it is a simple matter of zone.
1551  */
1552 static bool shmem_should_replace_page(struct page *page, gfp_t gfp)
1553 {
1554         return page_zonenum(page) > gfp_zone(gfp);
1555 }
1556 
1557 static int shmem_replace_page(struct page **pagep, gfp_t gfp,
1558                                 struct shmem_inode_info *info, pgoff_t index)
1559 {
1560         struct page *oldpage, *newpage;
1561         struct address_space *swap_mapping;
1562         swp_entry_t entry;
1563         pgoff_t swap_index;
1564         int error;
1565 
1566         oldpage = *pagep;
1567         entry.val = page_private(oldpage);
1568         swap_index = swp_offset(entry);
1569         swap_mapping = page_mapping(oldpage);
1570 
1571         /*
1572          * We have arrived here because our zones are constrained, so don't
1573          * limit chance of success by further cpuset and node constraints.
1574          */
1575         gfp &= ~GFP_CONSTRAINT_MASK;
1576         newpage = shmem_alloc_page(gfp, info, index);
1577         if (!newpage)
1578                 return -ENOMEM;
1579 
1580         get_page(newpage);
1581         copy_highpage(newpage, oldpage);
1582         flush_dcache_page(newpage);
1583 
1584         __SetPageLocked(newpage);
1585         __SetPageSwapBacked(newpage);
1586         SetPageUptodate(newpage);
1587         set_page_private(newpage, entry.val);
1588         SetPageSwapCache(newpage);
1589 
1590         /*
1591          * Our caller will very soon move newpage out of swapcache, but it's
1592          * a nice clean interface for us to replace oldpage by newpage there.
1593          */
1594         xa_lock_irq(&swap_mapping->i_pages);
1595         error = shmem_replace_entry(swap_mapping, swap_index, oldpage, newpage);
1596         if (!error) {
1597                 mem_cgroup_migrate(oldpage, newpage);
1598                 __inc_lruvec_page_state(newpage, NR_FILE_PAGES);
1599                 __dec_lruvec_page_state(oldpage, NR_FILE_PAGES);
1600         }
1601         xa_unlock_irq(&swap_mapping->i_pages);
1602 
1603         if (unlikely(error)) {
1604                 /*
1605                  * Is this possible?  I think not, now that our callers check
1606                  * both PageSwapCache and page_private after getting page lock;
1607                  * but be defensive.  Reverse old to newpage for clear and free.
1608                  */
1609                 oldpage = newpage;
1610         } else {
1611                 lru_cache_add(newpage);
1612                 *pagep = newpage;
1613         }
1614 
1615         ClearPageSwapCache(oldpage);
1616         set_page_private(oldpage, 0);
1617 
1618         unlock_page(oldpage);
1619         put_page(oldpage);
1620         put_page(oldpage);
1621         return error;
1622 }
1623 
1624 /*
1625  * Swap in the page pointed to by *pagep.
1626  * Caller has to make sure that *pagep contains a valid swapped page.
1627  * Returns 0 and the page in pagep if success. On failure, returns the
1628  * the error code and NULL in *pagep.
1629  */
1630 static int shmem_swapin_page(struct inode *inode, pgoff_t index,
1631                              struct page **pagep, enum sgp_type sgp,
1632                              gfp_t gfp, struct vm_area_struct *vma,
1633                              vm_fault_t *fault_type)
1634 {
1635         struct address_space *mapping = inode->i_mapping;
1636         struct shmem_inode_info *info = SHMEM_I(inode);
1637         struct mm_struct *charge_mm = vma ? vma->vm_mm : current->mm;
1638         struct page *page;
1639         swp_entry_t swap;
1640         int error;
1641 
1642         VM_BUG_ON(!*pagep || !xa_is_value(*pagep));
1643         swap = radix_to_swp_entry(*pagep);
1644         *pagep = NULL;
1645 
1646         /* Look it up and read it in.. */
1647         page = lookup_swap_cache(swap, NULL, 0);
1648         if (!page) {
1649                 /* Or update major stats only when swapin succeeds?? */
1650                 if (fault_type) {
1651                         *fault_type |= VM_FAULT_MAJOR;
1652                         count_vm_event(PGMAJFAULT);
1653                         count_memcg_event_mm(charge_mm, PGMAJFAULT);
1654                 }
1655                 /* Here we actually start the io */
1656                 page = shmem_swapin(swap, gfp, info, index);
1657                 if (!page) {
1658                         error = -ENOMEM;
1659                         goto failed;
1660                 }
1661         }
1662 
1663         /* We have to do this with page locked to prevent races */
1664         lock_page(page);
1665         if (!PageSwapCache(page) || page_private(page) != swap.val ||
1666             !shmem_confirm_swap(mapping, index, swap)) {
1667                 error = -EEXIST;
1668                 goto unlock;
1669         }
1670         if (!PageUptodate(page)) {
1671                 error = -EIO;
1672                 goto failed;
1673         }
1674         wait_on_page_writeback(page);
1675 
1676         if (shmem_should_replace_page(page, gfp)) {
1677                 error = shmem_replace_page(&page, gfp, info, index);
1678                 if (error)
1679                         goto failed;
1680         }
1681 
1682         error = shmem_add_to_page_cache(page, mapping, index,
1683                                         swp_to_radix_entry(swap), gfp,
1684                                         charge_mm);
1685         if (error)
1686                 goto failed;
1687 
1688         spin_lock_irq(&info->lock);
1689         info->swapped--;
1690         shmem_recalc_inode(inode);
1691         spin_unlock_irq(&info->lock);
1692 
1693         if (sgp == SGP_WRITE)
1694                 mark_page_accessed(page);
1695 
1696         delete_from_swap_cache(page);
1697         set_page_dirty(page);
1698         swap_free(swap);
1699 
1700         *pagep = page;
1701         return 0;
1702 failed:
1703         if (!shmem_confirm_swap(mapping, index, swap))
1704                 error = -EEXIST;
1705 unlock:
1706         if (page) {
1707                 unlock_page(page);
1708                 put_page(page);
1709         }
1710 
1711         return error;
1712 }
1713 
1714 /*
1715  * shmem_getpage_gfp - find page in cache, or get from swap, or allocate
1716  *
1717  * If we allocate a new one we do not mark it dirty. That's up to the
1718  * vm. If we swap it in we mark it dirty since we also free the swap
1719  * entry since a page cannot live in both the swap and page cache.
1720  *
1721  * vmf and fault_type are only supplied by shmem_fault:
1722  * otherwise they are NULL.
1723  */
1724 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
1725         struct page **pagep, enum sgp_type sgp, gfp_t gfp,
1726         struct vm_area_struct *vma, struct vm_fault *vmf,
1727                         vm_fault_t *fault_type)
1728 {
1729         struct address_space *mapping = inode->i_mapping;
1730         struct shmem_inode_info *info = SHMEM_I(inode);
1731         struct shmem_sb_info *sbinfo;
1732         struct mm_struct *charge_mm;
1733         struct page *page;
1734         enum sgp_type sgp_huge = sgp;
1735         pgoff_t hindex = index;
1736         int error;
1737         int once = 0;
1738         int alloced = 0;
1739 
1740         if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT))
1741                 return -EFBIG;
1742         if (sgp == SGP_NOHUGE || sgp == SGP_HUGE)
1743                 sgp = SGP_CACHE;
1744 repeat:
1745         if (sgp <= SGP_CACHE &&
1746             ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1747                 return -EINVAL;
1748         }
1749 
1750         sbinfo = SHMEM_SB(inode->i_sb);
1751         charge_mm = vma ? vma->vm_mm : current->mm;
1752 
1753         page = find_lock_entry(mapping, index);
1754         if (xa_is_value(page)) {
1755                 error = shmem_swapin_page(inode, index, &page,
1756                                           sgp, gfp, vma, fault_type);
1757                 if (error == -EEXIST)
1758                         goto repeat;
1759 
1760                 *pagep = page;
1761                 return error;
1762         }
1763 
1764         if (page && sgp == SGP_WRITE)
1765                 mark_page_accessed(page);
1766 
1767         /* fallocated page? */
1768         if (page && !PageUptodate(page)) {
1769                 if (sgp != SGP_READ)
1770                         goto clear;
1771                 unlock_page(page);
1772                 put_page(page);
1773                 page = NULL;
1774         }
1775         if (page || sgp == SGP_READ) {
1776                 *pagep = page;
1777                 return 0;
1778         }
1779 
1780         /*
1781          * Fast cache lookup did not find it:
1782          * bring it back from swap or allocate.
1783          */
1784 
1785         if (vma && userfaultfd_missing(vma)) {
1786                 *fault_type = handle_userfault(vmf, VM_UFFD_MISSING);
1787                 return 0;
1788         }
1789 
1790         /* shmem_symlink() */
1791         if (mapping->a_ops != &shmem_aops)
1792                 goto alloc_nohuge;
1793         if (shmem_huge == SHMEM_HUGE_DENY || sgp_huge == SGP_NOHUGE)
1794                 goto alloc_nohuge;
1795         if (shmem_huge == SHMEM_HUGE_FORCE)
1796                 goto alloc_huge;
1797         switch (sbinfo->huge) {
1798         case SHMEM_HUGE_NEVER:
1799                 goto alloc_nohuge;
1800         case SHMEM_HUGE_WITHIN_SIZE: {
1801                 loff_t i_size;
1802                 pgoff_t off;
1803 
1804                 off = round_up(index, HPAGE_PMD_NR);
1805                 i_size = round_up(i_size_read(inode), PAGE_SIZE);
1806                 if (i_size >= HPAGE_PMD_SIZE &&
1807                     i_size >> PAGE_SHIFT >= off)
1808                         goto alloc_huge;
1809 
1810                 fallthrough;
1811         }
1812         case SHMEM_HUGE_ADVISE:
1813                 if (sgp_huge == SGP_HUGE)
1814                         goto alloc_huge;
1815                 /* TODO: implement fadvise() hints */
1816                 goto alloc_nohuge;
1817         }
1818 
1819 alloc_huge:
1820         page = shmem_alloc_and_acct_page(gfp, inode, index, true);
1821         if (IS_ERR(page)) {
1822 alloc_nohuge:
1823                 page = shmem_alloc_and_acct_page(gfp, inode,
1824                                                  index, false);
1825         }
1826         if (IS_ERR(page)) {
1827                 int retry = 5;
1828 
1829                 error = PTR_ERR(page);
1830                 page = NULL;
1831                 if (error != -ENOSPC)
1832                         goto unlock;
1833                 /*
1834                  * Try to reclaim some space by splitting a huge page
1835                  * beyond i_size on the filesystem.
1836                  */
1837                 while (retry--) {
1838                         int ret;
1839 
1840                         ret = shmem_unused_huge_shrink(sbinfo, NULL, 1);
1841                         if (ret == SHRINK_STOP)
1842                                 break;
1843                         if (ret)
1844                                 goto alloc_nohuge;
1845                 }
1846                 goto unlock;
1847         }
1848 
1849         if (PageTransHuge(page))
1850                 hindex = round_down(index, HPAGE_PMD_NR);
1851         else
1852                 hindex = index;
1853 
1854         if (sgp == SGP_WRITE)
1855                 __SetPageReferenced(page);
1856 
1857         error = shmem_add_to_page_cache(page, mapping, hindex,
1858                                         NULL, gfp & GFP_RECLAIM_MASK,
1859                                         charge_mm);
1860         if (error)
1861                 goto unacct;
1862         lru_cache_add(page);
1863 
1864         spin_lock_irq(&info->lock);
1865         info->alloced += compound_nr(page);
1866         inode->i_blocks += BLOCKS_PER_PAGE << compound_order(page);
1867         shmem_recalc_inode(inode);
1868         spin_unlock_irq(&info->lock);
1869         alloced = true;
1870 
1871         if (PageTransHuge(page) &&
1872             DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) <
1873                         hindex + HPAGE_PMD_NR - 1) {
1874                 /*
1875                  * Part of the huge page is beyond i_size: subject
1876                  * to shrink under memory pressure.
1877                  */
1878                 spin_lock(&sbinfo->shrinklist_lock);
1879                 /*
1880                  * _careful to defend against unlocked access to
1881                  * ->shrink_list in shmem_unused_huge_shrink()
1882                  */
1883                 if (list_empty_careful(&info->shrinklist)) {
1884                         list_add_tail(&info->shrinklist,
1885                                       &sbinfo->shrinklist);
1886                         sbinfo->shrinklist_len++;
1887                 }
1888                 spin_unlock(&sbinfo->shrinklist_lock);
1889         }
1890 
1891         /*
1892          * Let SGP_FALLOC use the SGP_WRITE optimization on a new page.
1893          */
1894         if (sgp == SGP_FALLOC)
1895                 sgp = SGP_WRITE;
1896 clear:
1897         /*
1898          * Let SGP_WRITE caller clear ends if write does not fill page;
1899          * but SGP_FALLOC on a page fallocated earlier must initialize
1900          * it now, lest undo on failure cancel our earlier guarantee.
1901          */
1902         if (sgp != SGP_WRITE && !PageUptodate(page)) {
1903                 struct page *head = compound_head(page);
1904                 int i;
1905 
1906                 for (i = 0; i < compound_nr(head); i++) {
1907                         clear_highpage(head + i);
1908                         flush_dcache_page(head + i);
1909                 }
1910                 SetPageUptodate(head);
1911         }
1912 
1913         /* Perhaps the file has been truncated since we checked */
1914         if (sgp <= SGP_CACHE &&
1915             ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1916                 if (alloced) {
1917                         ClearPageDirty(page);
1918                         delete_from_page_cache(page);
1919                         spin_lock_irq(&info->lock);
1920                         shmem_recalc_inode(inode);
1921                         spin_unlock_irq(&info->lock);
1922                 }
1923                 error = -EINVAL;
1924                 goto unlock;
1925         }
1926         *pagep = page + index - hindex;
1927         return 0;
1928 
1929         /*
1930          * Error recovery.
1931          */
1932 unacct:
1933         shmem_inode_unacct_blocks(inode, compound_nr(page));
1934 
1935         if (PageTransHuge(page)) {
1936                 unlock_page(page);
1937                 put_page(page);
1938                 goto alloc_nohuge;
1939         }
1940 unlock:
1941         if (page) {
1942                 unlock_page(page);
1943                 put_page(page);
1944         }
1945         if (error == -ENOSPC && !once++) {
1946                 spin_lock_irq(&info->lock);
1947                 shmem_recalc_inode(inode);
1948                 spin_unlock_irq(&info->lock);
1949                 goto repeat;
1950         }
1951         if (error == -EEXIST)
1952                 goto repeat;
1953         return error;
1954 }
1955 
1956 /*
1957  * This is like autoremove_wake_function, but it removes the wait queue
1958  * entry unconditionally - even if something else had already woken the
1959  * target.
1960  */
1961 static int synchronous_wake_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
1962 {
1963         int ret = default_wake_function(wait, mode, sync, key);
1964         list_del_init(&wait->entry);
1965         return ret;
1966 }
1967 
1968 static vm_fault_t shmem_fault(struct vm_fault *vmf)
1969 {
1970         struct vm_area_struct *vma = vmf->vma;
1971         struct inode *inode = file_inode(vma->vm_file);
1972         gfp_t gfp = mapping_gfp_mask(inode->i_mapping);
1973         enum sgp_type sgp;
1974         int err;
1975         vm_fault_t ret = VM_FAULT_LOCKED;
1976 
1977         /*
1978          * Trinity finds that probing a hole which tmpfs is punching can
1979          * prevent the hole-punch from ever completing: which in turn
1980          * locks writers out with its hold on i_mutex.  So refrain from
1981          * faulting pages into the hole while it's being punched.  Although
1982          * shmem_undo_range() does remove the additions, it may be unable to
1983          * keep up, as each new page needs its own unmap_mapping_range() call,
1984          * and the i_mmap tree grows ever slower to scan if new vmas are added.
1985          *
1986          * It does not matter if we sometimes reach this check just before the
1987          * hole-punch begins, so that one fault then races with the punch:
1988          * we just need to make racing faults a rare case.
1989          *
1990          * The implementation below would be much simpler if we just used a
1991          * standard mutex or completion: but we cannot take i_mutex in fault,
1992          * and bloating every shmem inode for this unlikely case would be sad.
1993          */
1994         if (unlikely(inode->i_private)) {
1995                 struct shmem_falloc *shmem_falloc;
1996 
1997                 spin_lock(&inode->i_lock);
1998                 shmem_falloc = inode->i_private;
1999                 if (shmem_falloc &&
2000                     shmem_falloc->waitq &&
2001                     vmf->pgoff >= shmem_falloc->start &&
2002                     vmf->pgoff < shmem_falloc->next) {
2003                         struct file *fpin;
2004                         wait_queue_head_t *shmem_falloc_waitq;
2005                         DEFINE_WAIT_FUNC(shmem_fault_wait, synchronous_wake_function);
2006 
2007                         ret = VM_FAULT_NOPAGE;
2008                         fpin = maybe_unlock_mmap_for_io(vmf, NULL);
2009                         if (fpin)
2010                                 ret = VM_FAULT_RETRY;
2011 
2012                         shmem_falloc_waitq = shmem_falloc->waitq;
2013                         prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait,
2014                                         TASK_UNINTERRUPTIBLE);
2015                         spin_unlock(&inode->i_lock);
2016                         schedule();
2017 
2018                         /*
2019                          * shmem_falloc_waitq points into the shmem_fallocate()
2020                          * stack of the hole-punching task: shmem_falloc_waitq
2021                          * is usually invalid by the time we reach here, but
2022                          * finish_wait() does not dereference it in that case;
2023                          * though i_lock needed lest racing with wake_up_all().
2024                          */
2025                         spin_lock(&inode->i_lock);
2026                         finish_wait(shmem_falloc_waitq, &shmem_fault_wait);
2027                         spin_unlock(&inode->i_lock);
2028 
2029                         if (fpin)
2030                                 fput(fpin);
2031                         return ret;
2032                 }
2033                 spin_unlock(&inode->i_lock);
2034         }
2035 
2036         sgp = SGP_CACHE;
2037 
2038         if ((vma->vm_flags & VM_NOHUGEPAGE) ||
2039             test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))
2040                 sgp = SGP_NOHUGE;
2041         else if (vma->vm_flags & VM_HUGEPAGE)
2042                 sgp = SGP_HUGE;
2043 
2044         err = shmem_getpage_gfp(inode, vmf->pgoff, &vmf->page, sgp,
2045                                   gfp, vma, vmf, &ret);
2046         if (err)
2047                 return vmf_error(err);
2048         return ret;
2049 }
2050 
2051 unsigned long shmem_get_unmapped_area(struct file *file,
2052                                       unsigned long uaddr, unsigned long len,
2053                                       unsigned long pgoff, unsigned long flags)
2054 {
2055         unsigned long (*get_area)(struct file *,
2056                 unsigned long, unsigned long, unsigned long, unsigned long);
2057         unsigned long addr;
2058         unsigned long offset;
2059         unsigned long inflated_len;
2060         unsigned long inflated_addr;
2061         unsigned long inflated_offset;
2062 
2063         if (len > TASK_SIZE)
2064                 return -ENOMEM;
2065 
2066         get_area = current->mm->get_unmapped_area;
2067         addr = get_area(file, uaddr, len, pgoff, flags);
2068 
2069         if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
2070                 return addr;
2071         if (IS_ERR_VALUE(addr))
2072                 return addr;
2073         if (addr & ~PAGE_MASK)
2074                 return addr;
2075         if (addr > TASK_SIZE - len)
2076                 return addr;
2077 
2078         if (shmem_huge == SHMEM_HUGE_DENY)
2079                 return addr;
2080         if (len < HPAGE_PMD_SIZE)
2081                 return addr;
2082         if (flags & MAP_FIXED)
2083                 return addr;
2084         /*
2085          * Our priority is to support MAP_SHARED mapped hugely;
2086          * and support MAP_PRIVATE mapped hugely too, until it is COWed.
2087          * But if caller specified an address hint and we allocated area there
2088          * successfully, respect that as before.
2089          */
2090         if (uaddr == addr)
2091                 return addr;
2092 
2093         if (shmem_huge != SHMEM_HUGE_FORCE) {
2094                 struct super_block *sb;
2095 
2096                 if (file) {
2097                         VM_BUG_ON(file->f_op != &shmem_file_operations);
2098                         sb = file_inode(file)->i_sb;
2099                 } else {
2100                         /*
2101                          * Called directly from mm/mmap.c, or drivers/char/mem.c
2102                          * for "/dev/zero", to create a shared anonymous object.
2103                          */
2104                         if (IS_ERR(shm_mnt))
2105                                 return addr;
2106                         sb = shm_mnt->mnt_sb;
2107                 }
2108                 if (SHMEM_SB(sb)->huge == SHMEM_HUGE_NEVER)
2109                         return addr;
2110         }
2111 
2112         offset = (pgoff << PAGE_SHIFT) & (HPAGE_PMD_SIZE-1);
2113         if (offset && offset + len < 2 * HPAGE_PMD_SIZE)
2114                 return addr;
2115         if ((addr & (HPAGE_PMD_SIZE-1)) == offset)
2116                 return addr;
2117 
2118         inflated_len = len + HPAGE_PMD_SIZE - PAGE_SIZE;
2119         if (inflated_len > TASK_SIZE)
2120                 return addr;
2121         if (inflated_len < len)
2122                 return addr;
2123 
2124         inflated_addr = get_area(NULL, uaddr, inflated_len, 0, flags);
2125         if (IS_ERR_VALUE(inflated_addr))
2126                 return addr;
2127         if (inflated_addr & ~PAGE_MASK)
2128                 return addr;
2129 
2130         inflated_offset = inflated_addr & (HPAGE_PMD_SIZE-1);
2131         inflated_addr += offset - inflated_offset;
2132         if (inflated_offset > offset)
2133                 inflated_addr += HPAGE_PMD_SIZE;
2134 
2135         if (inflated_addr > TASK_SIZE - len)
2136                 return addr;
2137         return inflated_addr;
2138 }
2139 
2140 #ifdef CONFIG_NUMA
2141 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol)
2142 {
2143         struct inode *inode = file_inode(vma->vm_file);
2144         return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol);
2145 }
2146 
2147 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
2148                                           unsigned long addr)
2149 {
2150         struct inode *inode = file_inode(vma->vm_file);
2151         pgoff_t index;
2152 
2153         index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
2154         return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index);
2155 }
2156 #endif
2157 
2158 int shmem_lock(struct file *file, int lock, struct user_struct *user)
2159 {
2160         struct inode *inode = file_inode(file);
2161         struct shmem_inode_info *info = SHMEM_I(inode);
2162         int retval = -ENOMEM;
2163 
2164         /*
2165          * What serializes the accesses to info->flags?
2166          * ipc_lock_object() when called from shmctl_do_lock(),
2167          * no serialization needed when called from shm_destroy().
2168          */
2169         if (lock && !(info->flags & VM_LOCKED)) {
2170                 if (!user_shm_lock(inode->i_size, user))
2171                         goto out_nomem;
2172                 info->flags |= VM_LOCKED;
2173                 mapping_set_unevictable(file->f_mapping);
2174         }
2175         if (!lock && (info->flags & VM_LOCKED) && user) {
2176                 user_shm_unlock(inode->i_size, user);
2177                 info->flags &= ~VM_LOCKED;
2178                 mapping_clear_unevictable(file->f_mapping);
2179         }
2180         retval = 0;
2181 
2182 out_nomem:
2183         return retval;
2184 }
2185 
2186 static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
2187 {
2188         struct shmem_inode_info *info = SHMEM_I(file_inode(file));
2189 
2190         if (info->seals & F_SEAL_FUTURE_WRITE) {
2191                 /*
2192                  * New PROT_WRITE and MAP_SHARED mmaps are not allowed when
2193                  * "future write" seal active.
2194                  */
2195                 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_WRITE))
2196                         return -EPERM;
2197 
2198                 /*
2199                  * Since an F_SEAL_FUTURE_WRITE sealed memfd can be mapped as
2200                  * MAP_SHARED and read-only, take care to not allow mprotect to
2201                  * revert protections on such mappings. Do this only for shared
2202                  * mappings. For private mappings, don't need to mask
2203                  * VM_MAYWRITE as we still want them to be COW-writable.
2204                  */
2205                 if (vma->vm_flags & VM_SHARED)
2206                         vma->vm_flags &= ~(VM_MAYWRITE);
2207         }
2208 
2209         file_accessed(file);
2210         vma->vm_ops = &shmem_vm_ops;
2211         if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
2212                         ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) <
2213                         (vma->vm_end & HPAGE_PMD_MASK)) {
2214                 khugepaged_enter(vma, vma->vm_flags);
2215         }
2216         return 0;
2217 }
2218 
2219 static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir,
2220                                      umode_t mode, dev_t dev, unsigned long flags)
2221 {
2222         struct inode *inode;
2223         struct shmem_inode_info *info;
2224         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2225 
2226         if (shmem_reserve_inode(sb))
2227                 return NULL;
2228 
2229         inode = new_inode(sb);
2230         if (inode) {
2231                 inode->i_ino = get_next_ino();
2232                 inode_init_owner(inode, dir, mode);
2233                 inode->i_blocks = 0;
2234                 inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
2235                 inode->i_generation = prandom_u32();
2236                 info = SHMEM_I(inode);
2237                 memset(info, 0, (char *)inode - (char *)info);
2238                 spin_lock_init(&info->lock);
2239                 atomic_set(&info->stop_eviction, 0);
2240                 info->seals = F_SEAL_SEAL;
2241                 info->flags = flags & VM_NORESERVE;
2242                 INIT_LIST_HEAD(&info->shrinklist);
2243                 INIT_LIST_HEAD(&info->swaplist);
2244                 simple_xattrs_init(&info->xattrs);
2245                 cache_no_acl(inode);
2246 
2247                 switch (mode & S_IFMT) {
2248                 default:
2249                         inode->i_op = &shmem_special_inode_operations;
2250                         init_special_inode(inode, mode, dev);
2251                         break;
2252                 case S_IFREG:
2253                         inode->i_mapping->a_ops = &shmem_aops;
2254                         inode->i_op = &shmem_inode_operations;
2255                         inode->i_fop = &shmem_file_operations;
2256                         mpol_shared_policy_init(&info->policy,
2257                                                  shmem_get_sbmpol(sbinfo));
2258                         break;
2259                 case S_IFDIR:
2260                         inc_nlink(inode);
2261                         /* Some things misbehave if size == 0 on a directory */
2262                         inode->i_size = 2 * BOGO_DIRENT_SIZE;
2263                         inode->i_op = &shmem_dir_inode_operations;
2264                         inode->i_fop = &simple_dir_operations;
2265                         break;
2266                 case S_IFLNK:
2267                         /*
2268                          * Must not load anything in the rbtree,
2269                          * mpol_free_shared_policy will not be called.
2270                          */
2271                         mpol_shared_policy_init(&info->policy, NULL);
2272                         break;
2273                 }
2274 
2275                 lockdep_annotate_inode_mutex_key(inode);
2276         } else
2277                 shmem_free_inode(sb);
2278         return inode;
2279 }
2280 
2281 bool shmem_mapping(struct address_space *mapping)
2282 {
2283         return mapping->a_ops == &shmem_aops;
2284 }
2285 
2286 static int shmem_mfill_atomic_pte(struct mm_struct *dst_mm,
2287                                   pmd_t *dst_pmd,
2288                                   struct vm_area_struct *dst_vma,
2289                                   unsigned long dst_addr,
2290                                   unsigned long src_addr,
2291                                   bool zeropage,
2292                                   struct page **pagep)
2293 {
2294         struct inode *inode = file_inode(dst_vma->vm_file);
2295         struct shmem_inode_info *info = SHMEM_I(inode);
2296         struct address_space *mapping = inode->i_mapping;
2297         gfp_t gfp = mapping_gfp_mask(mapping);
2298         pgoff_t pgoff = linear_page_index(dst_vma, dst_addr);
2299         spinlock_t *ptl;
2300         void *page_kaddr;
2301         struct page *page;
2302         pte_t _dst_pte, *dst_pte;
2303         int ret;
2304         pgoff_t offset, max_off;
2305 
2306         ret = -ENOMEM;
2307         if (!shmem_inode_acct_block(inode, 1))
2308                 goto out;
2309 
2310         if (!*pagep) {
2311                 page = shmem_alloc_page(gfp, info, pgoff);
2312                 if (!page)
2313                         goto out_unacct_blocks;
2314 
2315                 if (!zeropage) {        /* mcopy_atomic */
2316                         page_kaddr = kmap_atomic(page);
2317                         ret = copy_from_user(page_kaddr,
2318                                              (const void __user *)src_addr,
2319                                              PAGE_SIZE);
2320                         kunmap_atomic(page_kaddr);
2321 
2322                         /* fallback to copy_from_user outside mmap_lock */
2323                         if (unlikely(ret)) {
2324                                 *pagep = page;
2325                                 shmem_inode_unacct_blocks(inode, 1);
2326                                 /* don't free the page */
2327                                 return -ENOENT;
2328                         }
2329                 } else {                /* mfill_zeropage_atomic */
2330                         clear_highpage(page);
2331                 }
2332         } else {
2333                 page = *pagep;
2334                 *pagep = NULL;
2335         }
2336 
2337         VM_BUG_ON(PageLocked(page) || PageSwapBacked(page));
2338         __SetPageLocked(page);
2339         __SetPageSwapBacked(page);
2340         __SetPageUptodate(page);
2341 
2342         ret = -EFAULT;
2343         offset = linear_page_index(dst_vma, dst_addr);
2344         max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
2345         if (unlikely(offset >= max_off))
2346                 goto out_release;
2347 
2348         ret = shmem_add_to_page_cache(page, mapping, pgoff, NULL,
2349                                       gfp & GFP_RECLAIM_MASK, dst_mm);
2350         if (ret)
2351                 goto out_release;
2352 
2353         _dst_pte = mk_pte(page, dst_vma->vm_page_prot);
2354         if (dst_vma->vm_flags & VM_WRITE)
2355                 _dst_pte = pte_mkwrite(pte_mkdirty(_dst_pte));
2356         else {
2357                 /*
2358                  * We don't set the pte dirty if the vma has no
2359                  * VM_WRITE permission, so mark the page dirty or it
2360                  * could be freed from under us. We could do it
2361                  * unconditionally before unlock_page(), but doing it
2362                  * only if VM_WRITE is not set is faster.
2363                  */
2364                 set_page_dirty(page);
2365         }
2366 
2367         dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);
2368 
2369         ret = -EFAULT;
2370         max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
2371         if (unlikely(offset >= max_off))
2372                 goto out_release_unlock;
2373 
2374         ret = -EEXIST;
2375         if (!pte_none(*dst_pte))
2376                 goto out_release_unlock;
2377 
2378         lru_cache_add(page);
2379 
2380         spin_lock_irq(&info->lock);
2381         info->alloced++;
2382         inode->i_blocks += BLOCKS_PER_PAGE;
2383         shmem_recalc_inode(inode);
2384         spin_unlock_irq(&info->lock);
2385 
2386         inc_mm_counter(dst_mm, mm_counter_file(page));
2387         page_add_file_rmap(page, false);
2388         set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
2389 
2390         /* No need to invalidate - it was non-present before */
2391         update_mmu_cache(dst_vma, dst_addr, dst_pte);
2392         pte_unmap_unlock(dst_pte, ptl);
2393         unlock_page(page);
2394         ret = 0;
2395 out:
2396         return ret;
2397 out_release_unlock:
2398         pte_unmap_unlock(dst_pte, ptl);
2399         ClearPageDirty(page);
2400         delete_from_page_cache(page);
2401 out_release:
2402         unlock_page(page);
2403         put_page(page);
2404 out_unacct_blocks:
2405         shmem_inode_unacct_blocks(inode, 1);
2406         goto out;
2407 }
2408 
2409 int shmem_mcopy_atomic_pte(struct mm_struct *dst_mm,
2410                            pmd_t *dst_pmd,
2411                            struct vm_area_struct *dst_vma,
2412                            unsigned long dst_addr,
2413                            unsigned long src_addr,
2414                            struct page **pagep)
2415 {
2416         return shmem_mfill_atomic_pte(dst_mm, dst_pmd, dst_vma,
2417                                       dst_addr, src_addr, false, pagep);
2418 }
2419 
2420 int shmem_mfill_zeropage_pte(struct mm_struct *dst_mm,
2421                              pmd_t *dst_pmd,
2422                              struct vm_area_struct *dst_vma,
2423                              unsigned long dst_addr)
2424 {
2425         struct page *page = NULL;
2426 
2427         return shmem_mfill_atomic_pte(dst_mm, dst_pmd, dst_vma,
2428                                       dst_addr, 0, true, &page);
2429 }
2430 
2431 #ifdef CONFIG_TMPFS
2432 static const struct inode_operations shmem_symlink_inode_operations;
2433 static const struct inode_operations shmem_short_symlink_operations;
2434 
2435 #ifdef CONFIG_TMPFS_XATTR
2436 static int shmem_initxattrs(struct inode *, const struct xattr *, void *);
2437 #else
2438 #define shmem_initxattrs NULL
2439 #endif
2440 
2441 static int
2442 shmem_write_begin(struct file *file, struct address_space *mapping,
2443                         loff_t pos, unsigned len, unsigned flags,
2444                         struct page **pagep, void **fsdata)
2445 {
2446         struct inode *inode = mapping->host;
2447         struct shmem_inode_info *info = SHMEM_I(inode);
2448         pgoff_t index = pos >> PAGE_SHIFT;
2449 
2450         /* i_mutex is held by caller */
2451         if (unlikely(info->seals & (F_SEAL_GROW |
2452                                    F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))) {
2453                 if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))
2454                         return -EPERM;
2455                 if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size)
2456                         return -EPERM;
2457         }
2458 
2459         return shmem_getpage(inode, index, pagep, SGP_WRITE);
2460 }
2461 
2462 static int
2463 shmem_write_end(struct file *file, struct address_space *mapping,
2464                         loff_t pos, unsigned len, unsigned copied,
2465                         struct page *page, void *fsdata)
2466 {
2467         struct inode *inode = mapping->host;
2468 
2469         if (pos + copied > inode->i_size)
2470                 i_size_write(inode, pos + copied);
2471 
2472         if (!PageUptodate(page)) {
2473                 struct page *head = compound_head(page);
2474                 if (PageTransCompound(page)) {
2475                         int i;
2476 
2477                         for (i = 0; i < HPAGE_PMD_NR; i++) {
2478                                 if (head + i == page)
2479                                         continue;
2480                                 clear_highpage(head + i);
2481                                 flush_dcache_page(head + i);
2482                         }
2483                 }
2484                 if (copied < PAGE_SIZE) {
2485                         unsigned from = pos & (PAGE_SIZE - 1);
2486                         zero_user_segments(page, 0, from,
2487                                         from + copied, PAGE_SIZE);
2488                 }
2489                 SetPageUptodate(head);
2490         }
2491         set_page_dirty(page);
2492         unlock_page(page);
2493         put_page(page);
2494 
2495         return copied;
2496 }
2497 
2498 static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
2499 {
2500         struct file *file = iocb->ki_filp;
2501         struct inode *inode = file_inode(file);
2502         struct address_space *mapping = inode->i_mapping;
2503         pgoff_t index;
2504         unsigned long offset;
2505         enum sgp_type sgp = SGP_READ;
2506         int error = 0;
2507         ssize_t retval = 0;
2508         loff_t *ppos = &iocb->ki_pos;
2509 
2510         /*
2511          * Might this read be for a stacking filesystem?  Then when reading
2512          * holes of a sparse file, we actually need to allocate those pages,
2513          * and even mark them dirty, so it cannot exceed the max_blocks limit.
2514          */
2515         if (!iter_is_iovec(to))
2516                 sgp = SGP_CACHE;
2517 
2518         index = *ppos >> PAGE_SHIFT;
2519         offset = *ppos & ~PAGE_MASK;
2520 
2521         for (;;) {
2522                 struct page *page = NULL;
2523                 pgoff_t end_index;
2524                 unsigned long nr, ret;
2525                 loff_t i_size = i_size_read(inode);
2526 
2527                 end_index = i_size >> PAGE_SHIFT;
2528                 if (index > end_index)
2529                         break;
2530                 if (index == end_index) {
2531                         nr = i_size & ~PAGE_MASK;
2532                         if (nr <= offset)
2533                                 break;
2534                 }
2535 
2536                 error = shmem_getpage(inode, index, &page, sgp);
2537                 if (error) {
2538                         if (error == -EINVAL)
2539                                 error = 0;
2540                         break;
2541                 }
2542                 if (page) {
2543                         if (sgp == SGP_CACHE)
2544                                 set_page_dirty(page);
2545                         unlock_page(page);
2546                 }
2547 
2548                 /*
2549                  * We must evaluate after, since reads (unlike writes)
2550                  * are called without i_mutex protection against truncate
2551                  */
2552                 nr = PAGE_SIZE;
2553                 i_size = i_size_read(inode);
2554                 end_index = i_size >> PAGE_SHIFT;
2555                 if (index == end_index) {
2556                         nr = i_size & ~PAGE_MASK;
2557                         if (nr <= offset) {
2558                                 if (page)
2559                                         put_page(page);
2560                                 break;
2561                         }
2562                 }
2563                 nr -= offset;
2564 
2565                 if (page) {
2566                         /*
2567                          * If users can be writing to this page using arbitrary
2568                          * virtual addresses, take care about potential aliasing
2569                          * before reading the page on the kernel side.
2570                          */
2571                         if (mapping_writably_mapped(mapping))
2572                                 flush_dcache_page(page);
2573                         /*
2574                          * Mark the page accessed if we read the beginning.
2575                          */
2576                         if (!offset)
2577                                 mark_page_accessed(page);
2578                 } else {
2579                         page = ZERO_PAGE(0);
2580                         get_page(page);
2581                 }
2582 
2583                 /*
2584                  * Ok, we have the page, and it's up-to-date, so
2585                  * now we can copy it to user space...
2586                  */
2587                 ret = copy_page_to_iter(page, offset, nr, to);
2588                 retval += ret;
2589                 offset += ret;
2590                 index += offset >> PAGE_SHIFT;
2591                 offset &= ~PAGE_MASK;
2592 
2593                 put_page(page);
2594                 if (!iov_iter_count(to))
2595                         break;
2596                 if (ret < nr) {
2597                         error = -EFAULT;
2598                         break;
2599                 }
2600                 cond_resched();
2601         }
2602 
2603         *ppos = ((loff_t) index << PAGE_SHIFT) + offset;
2604         file_accessed(file);
2605         return retval ? retval : error;
2606 }
2607 
2608 /*
2609  * llseek SEEK_DATA or SEEK_HOLE through the page cache.
2610  */
2611 static pgoff_t shmem_seek_hole_data(struct address_space *mapping,
2612                                     pgoff_t index, pgoff_t end, int whence)
2613 {
2614         struct page *page;
2615         struct pagevec pvec;
2616         pgoff_t indices[PAGEVEC_SIZE];
2617         bool done = false;
2618         int i;
2619 
2620         pagevec_init(&pvec);
2621         pvec.nr = 1;            /* start small: we may be there already */
2622         while (!done) {
2623                 pvec.nr = find_get_entries(mapping, index,
2624                                         pvec.nr, pvec.pages, indices);
2625                 if (!pvec.nr) {
2626                         if (whence == SEEK_DATA)
2627                                 index = end;
2628                         break;
2629                 }
2630                 for (i = 0; i < pvec.nr; i++, index++) {
2631                         if (index < indices[i]) {
2632                                 if (whence == SEEK_HOLE) {
2633                                         done = true;
2634                                         break;
2635                                 }
2636                                 index = indices[i];
2637                         }
2638                         page = pvec.pages[i];
2639                         if (page && !xa_is_value(page)) {
2640                                 if (!PageUptodate(page))
2641                                         page = NULL;
2642                         }
2643                         if (index >= end ||
2644                             (page && whence == SEEK_DATA) ||
2645                             (!page && whence == SEEK_HOLE)) {
2646                                 done = true;
2647                                 break;
2648                         }
2649                 }
2650                 pagevec_remove_exceptionals(&pvec);
2651                 pagevec_release(&pvec);
2652                 pvec.nr = PAGEVEC_SIZE;
2653                 cond_resched();
2654         }
2655         return index;
2656 }
2657 
2658 static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence)
2659 {
2660         struct address_space *mapping = file->f_mapping;
2661         struct inode *inode = mapping->host;
2662         pgoff_t start, end;
2663         loff_t new_offset;
2664 
2665         if (whence != SEEK_DATA && whence != SEEK_HOLE)
2666                 return generic_file_llseek_size(file, offset, whence,
2667                                         MAX_LFS_FILESIZE, i_size_read(inode));
2668         inode_lock(inode);
2669         /* We're holding i_mutex so we can access i_size directly */
2670 
2671         if (offset < 0 || offset >= inode->i_size)
2672                 offset = -ENXIO;
2673         else {
2674                 start = offset >> PAGE_SHIFT;
2675                 end = (inode->i_size + PAGE_SIZE - 1) >> PAGE_SHIFT;
2676                 new_offset = shmem_seek_hole_data(mapping, start, end, whence);
2677                 new_offset <<= PAGE_SHIFT;
2678                 if (new_offset > offset) {
2679                         if (new_offset < inode->i_size)
2680                                 offset = new_offset;
2681                         else if (whence == SEEK_DATA)
2682                                 offset = -ENXIO;
2683                         else
2684                                 offset = inode->i_size;
2685                 }
2686         }
2687 
2688         if (offset >= 0)
2689                 offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE);
2690         inode_unlock(inode);
2691         return offset;
2692 }
2693 
2694 static long shmem_fallocate(struct file *file, int mode, loff_t offset,
2695                                                          loff_t len)
2696 {
2697         struct inode *inode = file_inode(file);
2698         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
2699         struct shmem_inode_info *info = SHMEM_I(inode);
2700         struct shmem_falloc shmem_falloc;
2701         pgoff_t start, index, end;
2702         int error;
2703 
2704         if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
2705                 return -EOPNOTSUPP;
2706 
2707         inode_lock(inode);
2708 
2709         if (mode & FALLOC_FL_PUNCH_HOLE) {
2710                 struct address_space *mapping = file->f_mapping;
2711                 loff_t unmap_start = round_up(offset, PAGE_SIZE);
2712                 loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1;
2713                 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq);
2714 
2715                 /* protected by i_mutex */
2716                 if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) {
2717                         error = -EPERM;
2718                         goto out;
2719                 }
2720 
2721                 shmem_falloc.waitq = &shmem_falloc_waitq;
2722                 shmem_falloc.start = (u64)unmap_start >> PAGE_SHIFT;
2723                 shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT;
2724                 spin_lock(&inode->i_lock);
2725                 inode->i_private = &shmem_falloc;
2726                 spin_unlock(&inode->i_lock);
2727 
2728                 if ((u64)unmap_end > (u64)unmap_start)
2729                         unmap_mapping_range(mapping, unmap_start,
2730                                             1 + unmap_end - unmap_start, 0);
2731                 shmem_truncate_range(inode, offset, offset + len - 1);
2732                 /* No need to unmap again: hole-punching leaves COWed pages */
2733 
2734                 spin_lock(&inode->i_lock);
2735                 inode->i_private = NULL;
2736                 wake_up_all(&shmem_falloc_waitq);
2737                 WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.head));
2738                 spin_unlock(&inode->i_lock);
2739                 error = 0;
2740                 goto out;
2741         }
2742 
2743         /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
2744         error = inode_newsize_ok(inode, offset + len);
2745         if (error)
2746                 goto out;
2747 
2748         if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) {
2749                 error = -EPERM;
2750                 goto out;
2751         }
2752 
2753         start = offset >> PAGE_SHIFT;
2754         end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
2755         /* Try to avoid a swapstorm if len is impossible to satisfy */
2756         if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) {
2757                 error = -ENOSPC;
2758                 goto out;
2759         }
2760 
2761         shmem_falloc.waitq = NULL;
2762         shmem_falloc.start = start;
2763         shmem_falloc.next  = start;
2764         shmem_falloc.nr_falloced = 0;
2765         shmem_falloc.nr_unswapped = 0;
2766         spin_lock(&inode->i_lock);
2767         inode->i_private = &shmem_falloc;
2768         spin_unlock(&inode->i_lock);
2769 
2770         for (index = start; index < end; index++) {
2771                 struct page *page;
2772 
2773                 /*
2774                  * Good, the fallocate(2) manpage permits EINTR: we may have
2775                  * been interrupted because we are using up too much memory.
2776                  */
2777                 if (signal_pending(current))
2778                         error = -EINTR;
2779                 else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced)
2780                         error = -ENOMEM;
2781                 else
2782                         error = shmem_getpage(inode, index, &page, SGP_FALLOC);
2783                 if (error) {
2784                         /* Remove the !PageUptodate pages we added */
2785                         if (index > start) {
2786                                 shmem_undo_range(inode,
2787                                     (loff_t)start << PAGE_SHIFT,
2788                                     ((loff_t)index << PAGE_SHIFT) - 1, true);
2789                         }
2790                         goto undone;
2791                 }
2792 
2793                 /*
2794                  * Inform shmem_writepage() how far we have reached.
2795                  * No need for lock or barrier: we have the page lock.
2796                  */
2797                 shmem_falloc.next++;
2798                 if (!PageUptodate(page))
2799                         shmem_falloc.nr_falloced++;
2800 
2801                 /*
2802                  * If !PageUptodate, leave it that way so that freeable pages
2803                  * can be recognized if we need to rollback on error later.
2804                  * But set_page_dirty so that memory pressure will swap rather
2805                  * than free the pages we are allocating (and SGP_CACHE pages
2806                  * might still be clean: we now need to mark those dirty too).
2807                  */
2808                 set_page_dirty(page);
2809                 unlock_page(page);
2810                 put_page(page);
2811                 cond_resched();
2812         }
2813 
2814         if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
2815                 i_size_write(inode, offset + len);
2816         inode->i_ctime = current_time(inode);
2817 undone:
2818         spin_lock(&inode->i_lock);
2819         inode->i_private = NULL;
2820         spin_unlock(&inode->i_lock);
2821 out:
2822         inode_unlock(inode);
2823         return error;
2824 }
2825 
2826 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
2827 {
2828         struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
2829 
2830         buf->f_type = TMPFS_MAGIC;
2831         buf->f_bsize = PAGE_SIZE;
2832         buf->f_namelen = NAME_MAX;
2833         if (sbinfo->max_blocks) {
2834                 buf->f_blocks = sbinfo->max_blocks;
2835                 buf->f_bavail =
2836                 buf->f_bfree  = sbinfo->max_blocks -
2837                                 percpu_counter_sum(&sbinfo->used_blocks);
2838         }
2839         if (sbinfo->max_inodes) {
2840                 buf->f_files = sbinfo->max_inodes;
2841                 buf->f_ffree = sbinfo->free_inodes;
2842         }
2843         /* else leave those fields 0 like simple_statfs */
2844         return 0;
2845 }
2846 
2847 /*
2848  * File creation. Allocate an inode, and we're done..
2849  */
2850 static int
2851 shmem_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev)
2852 {
2853         struct inode *inode;
2854         int error = -ENOSPC;
2855 
2856         inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE);
2857         if (inode) {
2858                 error = simple_acl_create(dir, inode);
2859                 if (error)
2860                         goto out_iput;
2861                 error = security_inode_init_security(inode, dir,
2862                                                      &dentry->d_name,
2863                                                      shmem_initxattrs, NULL);
2864                 if (error && error != -EOPNOTSUPP)
2865                         goto out_iput;
2866 
2867                 error = 0;
2868                 dir->i_size += BOGO_DIRENT_SIZE;
2869                 dir->i_ctime = dir->i_mtime = current_time(dir);
2870                 d_instantiate(dentry, inode);
2871                 dget(dentry); /* Extra count - pin the dentry in core */
2872         }
2873         return error;
2874 out_iput:
2875         iput(inode);
2876         return error;
2877 }
2878 
2879 static int
2880 shmem_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode)
2881 {
2882         struct inode *inode;
2883         int error = -ENOSPC;
2884 
2885         inode = shmem_get_inode(dir->i_sb, dir, mode, 0, VM_NORESERVE);
2886         if (inode) {
2887                 error = security_inode_init_security(inode, dir,
2888                                                      NULL,
2889                                                      shmem_initxattrs, NULL);
2890                 if (error && error != -EOPNOTSUPP)
2891                         goto out_iput;
2892                 error = simple_acl_create(dir, inode);
2893                 if (error)
2894                         goto out_iput;
2895                 d_tmpfile(dentry, inode);
2896         }
2897         return error;
2898 out_iput:
2899         iput(inode);
2900         return error;
2901 }
2902 
2903 static int shmem_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
2904 {
2905         int error;
2906 
2907         if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0)))
2908                 return error;
2909         inc_nlink(dir);
2910         return 0;
2911 }
2912 
2913 static int shmem_create(struct inode *dir, struct dentry *dentry, umode_t mode,
2914                 bool excl)
2915 {
2916         return shmem_mknod(dir, dentry, mode | S_IFREG, 0);
2917 }
2918 
2919 /*
2920  * Link a file..
2921  */
2922 static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
2923 {
2924         struct inode *inode = d_inode(old_dentry);
2925         int ret = 0;
2926 
2927         /*
2928          * No ordinary (disk based) filesystem counts links as inodes;
2929          * but each new link needs a new dentry, pinning lowmem, and
2930          * tmpfs dentries cannot be pruned until they are unlinked.
2931          * But if an O_TMPFILE file is linked into the tmpfs, the
2932          * first link must skip that, to get the accounting right.
2933          */
2934         if (inode->i_nlink) {
2935                 ret = shmem_reserve_inode(inode->i_sb);
2936                 if (ret)
2937                         goto out;
2938         }
2939 
2940         dir->i_size += BOGO_DIRENT_SIZE;
2941         inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
2942         inc_nlink(inode);
2943         ihold(inode);   /* New dentry reference */
2944         dget(dentry);           /* Extra pinning count for the created dentry */
2945         d_instantiate(dentry, inode);
2946 out:
2947         return ret;
2948 }
2949 
2950 static int shmem_unlink(struct inode *dir, struct dentry *dentry)
2951 {
2952         struct inode *inode = d_inode(dentry);
2953 
2954         if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
2955                 shmem_free_inode(inode->i_sb);
2956 
2957         dir->i_size -= BOGO_DIRENT_SIZE;
2958         inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
2959         drop_nlink(inode);
2960         dput(dentry);   /* Undo the count from "create" - this does all the work */
2961         return 0;
2962 }
2963 
2964 static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
2965 {
2966         if (!simple_empty(dentry))
2967                 return -ENOTEMPTY;
2968 
2969         drop_nlink(d_inode(dentry));
2970         drop_nlink(dir);
2971         return shmem_unlink(dir, dentry);
2972 }
2973 
2974 static int shmem_exchange(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
2975 {
2976         bool old_is_dir = d_is_dir(old_dentry);
2977         bool new_is_dir = d_is_dir(new_dentry);
2978 
2979         if (old_dir != new_dir && old_is_dir != new_is_dir) {
2980                 if (old_is_dir) {
2981                         drop_nlink(old_dir);
2982                         inc_nlink(new_dir);
2983                 } else {
2984                         drop_nlink(new_dir);
2985                         inc_nlink(old_dir);
2986                 }
2987         }
2988         old_dir->i_ctime = old_dir->i_mtime =
2989         new_dir->i_ctime = new_dir->i_mtime =
2990         d_inode(old_dentry)->i_ctime =
2991         d_inode(new_dentry)->i_ctime = current_time(old_dir);
2992 
2993         return 0;
2994 }
2995 
2996 static int shmem_whiteout(struct inode *old_dir, struct dentry *old_dentry)
2997 {
2998         struct dentry *whiteout;
2999         int error;
3000 
3001         whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name);
3002         if (!whiteout)
3003                 return -ENOMEM;
3004 
3005         error = shmem_mknod(old_dir, whiteout,
3006                             S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV);
3007         dput(whiteout);
3008         if (error)
3009                 return error;
3010 
3011         /*
3012          * Cheat and hash the whiteout while the old dentry is still in
3013          * place, instead of playing games with FS_RENAME_DOES_D_MOVE.
3014          *
3015          * d_lookup() will consistently find one of them at this point,
3016          * not sure which one, but that isn't even important.
3017          */
3018         d_rehash(whiteout);
3019         return 0;
3020 }
3021 
3022 /*
3023  * The VFS layer already does all the dentry stuff for rename,
3024  * we just have to decrement the usage count for the target if
3025  * it exists so that the VFS layer correctly free's it when it
3026  * gets overwritten.
3027  */
3028 static int shmem_rename2(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags)
3029 {
3030         struct inode *inode = d_inode(old_dentry);
3031         int they_are_dirs = S_ISDIR(inode->i_mode);
3032 
3033         if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
3034                 return -EINVAL;
3035 
3036         if (flags & RENAME_EXCHANGE)
3037                 return shmem_exchange(old_dir, old_dentry, new_dir, new_dentry);
3038 
3039         if (!simple_empty(new_dentry))
3040                 return -ENOTEMPTY;
3041 
3042         if (flags & RENAME_WHITEOUT) {
3043                 int error;
3044 
3045                 error = shmem_whiteout(old_dir, old_dentry);
3046                 if (error)
3047                         return error;
3048         }
3049 
3050         if (d_really_is_positive(new_dentry)) {
3051                 (void) shmem_unlink(new_dir, new_dentry);
3052                 if (they_are_dirs) {
3053                         drop_nlink(d_inode(new_dentry));
3054                         drop_nlink(old_dir);
3055                 }
3056         } else if (they_are_dirs) {
3057                 drop_nlink(old_dir);
3058                 inc_nlink(new_dir);
3059         }
3060 
3061         old_dir->i_size -= BOGO_DIRENT_SIZE;
3062         new_dir->i_size += BOGO_DIRENT_SIZE;
3063         old_dir->i_ctime = old_dir->i_mtime =
3064         new_dir->i_ctime = new_dir->i_mtime =
3065         inode->i_ctime = current_time(old_dir);
3066         return 0;
3067 }
3068 
3069 static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
3070 {
3071         int error;
3072         int len;
3073         struct inode *inode;
3074         struct page *page;
3075 
3076         len = strlen(symname) + 1;
3077         if (len > PAGE_SIZE)
3078                 return -ENAMETOOLONG;
3079 
3080         inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK | 0777, 0,
3081                                 VM_NORESERVE);
3082         if (!inode)
3083                 return -ENOSPC;
3084 
3085         error = security_inode_init_security(inode, dir, &dentry->d_name,
3086                                              shmem_initxattrs, NULL);
3087         if (error && error != -EOPNOTSUPP) {
3088                 iput(inode);
3089                 return error;
3090         }
3091 
3092         inode->i_size = len-1;
3093         if (len <= SHORT_SYMLINK_LEN) {
3094                 inode->i_link = kmemdup(symname, len, GFP_KERNEL);
3095                 if (!inode->i_link) {
3096                         iput(inode);
3097                         return -ENOMEM;
3098                 }
3099                 inode->i_op = &shmem_short_symlink_operations;
3100         } else {
3101                 inode_nohighmem(inode);
3102                 error = shmem_getpage(inode, 0, &page, SGP_WRITE);
3103                 if (error) {
3104                         iput(inode);
3105                         return error;
3106                 }
3107                 inode->i_mapping->a_ops = &shmem_aops;
3108                 inode->i_op = &shmem_symlink_inode_operations;
3109                 memcpy(page_address(page), symname, len);
3110                 SetPageUptodate(page);
3111                 set_page_dirty(page);
3112                 unlock_page(page);
3113                 put_page(page);
3114         }
3115         dir->i_size += BOGO_DIRENT_SIZE;
3116         dir->i_ctime = dir->i_mtime = current_time(dir);
3117         d_instantiate(dentry, inode);
3118         dget(dentry);
3119         return 0;
3120 }
3121 
3122 static void shmem_put_link(void *arg)
3123 {
3124         mark_page_accessed(arg);
3125         put_page(arg);
3126 }
3127 
3128 static const char *shmem_get_link(struct dentry *dentry,
3129                                   struct inode *inode,
3130                                   struct delayed_call *done)
3131 {
3132         struct page *page = NULL;
3133         int error;
3134         if (!dentry) {
3135                 page = find_get_page(inode->i_mapping, 0);
3136                 if (!page)
3137                         return ERR_PTR(-ECHILD);
3138                 if (!PageUptodate(page)) {
3139                         put_page(page);
3140                         return ERR_PTR(-ECHILD);
3141                 }
3142         } else {
3143                 error = shmem_getpage(inode, 0, &page, SGP_READ);
3144                 if (error)
3145                         return ERR_PTR(error);
3146                 unlock_page(page);
3147         }
3148         set_delayed_call(done, shmem_put_link, page);
3149         return page_address(page);
3150 }
3151 
3152 #ifdef CONFIG_TMPFS_XATTR
3153 /*
3154  * Superblocks without xattr inode operations may get some security.* xattr
3155  * support from the LSM "for free". As soon as we have any other xattrs
3156  * like ACLs, we also need to implement the security.* handlers at
3157  * filesystem level, though.
3158  */
3159 
3160 /*
3161  * Callback for security_inode_init_security() for acquiring xattrs.
3162  */
3163 static int shmem_initxattrs(struct inode *inode,
3164                             const struct xattr *xattr_array,
3165                             void *fs_info)
3166 {
3167         struct shmem_inode_info *info = SHMEM_I(inode);
3168         const struct xattr *xattr;
3169         struct simple_xattr *new_xattr;
3170         size_t len;
3171 
3172         for (xattr = xattr_array; xattr->name != NULL; xattr++) {
3173                 new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len);
3174                 if (!new_xattr)
3175                         return -ENOMEM;
3176 
3177                 len = strlen(xattr->name) + 1;
3178                 new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len,
3179                                           GFP_KERNEL);
3180                 if (!new_xattr->name) {
3181                         kvfree(new_xattr);
3182                         return -ENOMEM;
3183                 }
3184 
3185                 memcpy(new_xattr->name, XATTR_SECURITY_PREFIX,
3186                        XATTR_SECURITY_PREFIX_LEN);
3187                 memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN,
3188                        xattr->name, len);
3189 
3190                 simple_xattr_list_add(&info->xattrs, new_xattr);
3191         }
3192 
3193         return 0;
3194 }
3195 
3196 static int shmem_xattr_handler_get(const struct xattr_handler *handler,
3197                                    struct dentry *unused, struct inode *inode,
3198                                    const char *name, void *buffer, size_t size)
3199 {
3200         struct shmem_inode_info *info = SHMEM_I(inode);
3201 
3202         name = xattr_full_name(handler, name);
3203         return simple_xattr_get(&info->xattrs, name, buffer, size);
3204 }
3205 
3206 static int shmem_xattr_handler_set(const struct xattr_handler *handler,
3207                                    struct dentry *unused, struct inode *inode,
3208                                    const char *name, const void *value,
3209                                    size_t size, int flags)
3210 {
3211         struct shmem_inode_info *info = SHMEM_I(inode);
3212 
3213         name = xattr_full_name(handler, name);
3214         return simple_xattr_set(&info->xattrs, name, value, size, flags, NULL);
3215 }
3216 
3217 static const struct xattr_handler shmem_security_xattr_handler = {
3218         .prefix = XATTR_SECURITY_PREFIX,
3219         .get = shmem_xattr_handler_get,
3220         .set = shmem_xattr_handler_set,
3221 };
3222 
3223 static const struct xattr_handler shmem_trusted_xattr_handler = {
3224         .prefix = XATTR_TRUSTED_PREFIX,
3225         .get = shmem_xattr_handler_get,
3226         .set = shmem_xattr_handler_set,
3227 };
3228 
3229 static const struct xattr_handler *shmem_xattr_handlers[] = {
3230 #ifdef CONFIG_TMPFS_POSIX_ACL
3231         &posix_acl_access_xattr_handler,
3232         &posix_acl_default_xattr_handler,
3233 #endif
3234         &shmem_security_xattr_handler,
3235         &shmem_trusted_xattr_handler,
3236         NULL
3237 };
3238 
3239 static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size)
3240 {
3241         struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
3242         return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size);
3243 }
3244 #endif /* CONFIG_TMPFS_XATTR */
3245 
3246 static const struct inode_operations shmem_short_symlink_operations = {
3247         .get_link       = simple_get_link,
3248 #ifdef CONFIG_TMPFS_XATTR
3249         .listxattr      = shmem_listxattr,
3250 #endif
3251 };
3252 
3253 static const struct inode_operations shmem_symlink_inode_operations = {
3254         .get_link       = shmem_get_link,
3255 #ifdef CONFIG_TMPFS_XATTR
3256         .listxattr      = shmem_listxattr,
3257 #endif
3258 };
3259 
3260 static struct dentry *shmem_get_parent(struct dentry *child)
3261 {
3262         return ERR_PTR(-ESTALE);
3263 }
3264 
3265 static int shmem_match(struct inode *ino, void *vfh)
3266 {
3267         __u32 *fh = vfh;
3268         __u64 inum = fh[2];
3269         inum = (inum << 32) | fh[1];
3270         return ino->i_ino == inum && fh[0] == ino->i_generation;
3271 }
3272 
3273 /* Find any alias of inode, but prefer a hashed alias */
3274 static struct dentry *shmem_find_alias(struct inode *inode)
3275 {
3276         struct dentry *alias = d_find_alias(inode);
3277 
3278         return alias ?: d_find_any_alias(inode);
3279 }
3280 
3281 
3282 static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
3283                 struct fid *fid, int fh_len, int fh_type)
3284 {
3285         struct inode *inode;
3286         struct dentry *dentry = NULL;
3287         u64 inum;
3288 
3289         if (fh_len < 3)
3290                 return NULL;
3291 
3292         inum = fid->raw[2];
3293         inum = (inum << 32) | fid->raw[1];
3294 
3295         inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
3296                         shmem_match, fid->raw);
3297         if (inode) {
3298                 dentry = shmem_find_alias(inode);
3299                 iput(inode);
3300         }
3301 
3302         return dentry;
3303 }
3304 
3305 static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len,
3306                                 struct inode *parent)
3307 {
3308         if (*len < 3) {
3309                 *len = 3;
3310                 return FILEID_INVALID;
3311         }
3312 
3313         if (inode_unhashed(inode)) {
3314                 /* Unfortunately insert_inode_hash is not idempotent,
3315                  * so as we hash inodes here rather than at creation
3316                  * time, we need a lock to ensure we only try
3317                  * to do it once
3318                  */
3319                 static DEFINE_SPINLOCK(lock);
3320                 spin_lock(&lock);
3321                 if (inode_unhashed(inode))
3322                         __insert_inode_hash(inode,
3323                                             inode->i_ino + inode->i_generation);
3324                 spin_unlock(&lock);
3325         }
3326 
3327         fh[0] = inode->i_generation;
3328         fh[1] = inode->i_ino;
3329         fh[2] = ((__u64)inode->i_ino) >> 32;
3330 
3331         *len = 3;
3332         return 1;
3333 }
3334 
3335 static const struct export_operations shmem_export_ops = {
3336         .get_parent     = shmem_get_parent,
3337         .encode_fh      = shmem_encode_fh,
3338         .fh_to_dentry   = shmem_fh_to_dentry,
3339 };
3340 
3341 enum shmem_param {
3342         Opt_gid,
3343         Opt_huge,
3344         Opt_mode,
3345         Opt_mpol,
3346         Opt_nr_blocks,
3347         Opt_nr_inodes,
3348         Opt_size,
3349         Opt_uid,
3350 };
3351 
3352 static const struct constant_table shmem_param_enums_huge[] = {
3353         {"never",       SHMEM_HUGE_NEVER },
3354         {"always",      SHMEM_HUGE_ALWAYS },
3355         {"within_size", SHMEM_HUGE_WITHIN_SIZE },
3356         {"advise",      SHMEM_HUGE_ADVISE },
3357         {}
3358 };
3359 
3360 const struct fs_parameter_spec shmem_fs_parameters[] = {
3361         fsparam_u32   ("gid",           Opt_gid),
3362         fsparam_enum  ("huge",          Opt_huge,  shmem_param_enums_huge),
3363         fsparam_u32oct("mode",          Opt_mode),
3364         fsparam_string("mpol",          Opt_mpol),
3365         fsparam_string("nr_blocks",     Opt_nr_blocks),
3366         fsparam_string("nr_inodes",     Opt_nr_inodes),
3367         fsparam_string("size",          Opt_size),
3368         fsparam_u32   ("uid",           Opt_uid),
3369         {}
3370 };
3371 
3372 static int shmem_parse_one(struct fs_context *fc, struct fs_parameter *param)
3373 {
3374         struct shmem_options *ctx = fc->fs_private;
3375         struct fs_parse_result result;
3376         unsigned long long size;
3377         char *rest;
3378         int opt;
3379 
3380         opt = fs_parse(fc, shmem_fs_parameters, param, &result);
3381         if (opt < 0)
3382                 return opt;
3383 
3384         switch (opt) {
3385         case Opt_size:
3386                 size = memparse(param->string, &rest);
3387                 if (*rest == '%') {
3388                         size <<= PAGE_SHIFT;
3389                         size *= totalram_pages();
3390                         do_div(size, 100);
3391                         rest++;
3392                 }
3393                 if (*rest)
3394                         goto bad_value;
3395                 ctx->blocks = DIV_ROUND_UP(size, PAGE_SIZE);
3396                 ctx->seen |= SHMEM_SEEN_BLOCKS;
3397                 break;
3398         case Opt_nr_blocks:
3399                 ctx->blocks = memparse(param->string, &rest);
3400                 if (*rest)
3401                         goto bad_value;
3402                 ctx->seen |= SHMEM_SEEN_BLOCKS;
3403                 break;
3404         case Opt_nr_inodes:
3405                 ctx->inodes = memparse(param->string, &rest);
3406                 if (*rest)
3407                         goto bad_value;
3408                 ctx->seen |= SHMEM_SEEN_INODES;
3409                 break;
3410         case Opt_mode:
3411                 ctx->mode = result.uint_32 & 07777;
3412                 break;
3413         case Opt_uid:
3414                 ctx->uid = make_kuid(current_user_ns(), result.uint_32);
3415                 if (!uid_valid(ctx->uid))
3416                         goto bad_value;
3417                 break;
3418         case Opt_gid:
3419                 ctx->gid = make_kgid(current_user_ns(), result.uint_32);
3420                 if (!gid_valid(ctx->gid))
3421                         goto bad_value;
3422                 break;
3423         case Opt_huge:
3424                 ctx->huge = result.uint_32;
3425                 if (ctx->huge != SHMEM_HUGE_NEVER &&
3426                     !(IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
3427                       has_transparent_hugepage()))
3428                         goto unsupported_parameter;
3429                 ctx->seen |= SHMEM_SEEN_HUGE;
3430                 break;
3431         case Opt_mpol:
3432                 if (IS_ENABLED(CONFIG_NUMA)) {
3433                         mpol_put(ctx->mpol);
3434                         ctx->mpol = NULL;
3435                         if (mpol_parse_str(param->string, &ctx->mpol))
3436                                 goto bad_value;
3437                         break;
3438                 }
3439                 goto unsupported_parameter;
3440         }
3441         return 0;
3442 
3443 unsupported_parameter:
3444         return invalfc(fc, "Unsupported parameter '%s'", param->key);
3445 bad_value:
3446         return invalfc(fc, "Bad value for '%s'", param->key);
3447 }
3448 
3449 static int shmem_parse_options(struct fs_context *fc, void *data)
3450 {
3451         char *options = data;
3452 
3453         if (options) {
3454                 int err = security_sb_eat_lsm_opts(options, &fc->security);
3455                 if (err)
3456                         return err;
3457         }
3458 
3459         while (options != NULL) {
3460                 char *this_char = options;
3461                 for (;;) {
3462                         /*
3463                          * NUL-terminate this option: unfortunately,
3464                          * mount options form a comma-separated list,
3465                          * but mpol's nodelist may also contain commas.
3466                          */
3467                         options = strchr(options, ',');
3468                         if (options == NULL)
3469                                 break;
3470                         options++;
3471                         if (!isdigit(*options)) {
3472                                 options[-1] = '\0';
3473                                 break;
3474                         }
3475                 }
3476                 if (*this_char) {
3477                         char *value = strchr(this_char,'=');
3478                         size_t len = 0;
3479                         int err;
3480 
3481                         if (value) {
3482                                 *value++ = '\0';
3483                                 len = strlen(value);
3484                         }
3485                         err = vfs_parse_fs_string(fc, this_char, value, len);
3486                         if (err < 0)
3487                                 return err;
3488                 }
3489         }
3490         return 0;
3491 }
3492 
3493 /*
3494  * Reconfigure a shmem filesystem.
3495  *
3496  * Note that we disallow change from limited->unlimited blocks/inodes while any
3497  * are in use; but we must separately disallow unlimited->limited, because in
3498  * that case we have no record of how much is already in use.
3499  */
3500 static int shmem_reconfigure(struct fs_context *fc)
3501 {
3502         struct shmem_options *ctx = fc->fs_private;
3503         struct shmem_sb_info *sbinfo = SHMEM_SB(fc->root->d_sb);
3504         unsigned long inodes;
3505         const char *err;
3506 
3507         spin_lock(&sbinfo->stat_lock);
3508         inodes = sbinfo->max_inodes - sbinfo->free_inodes;
3509         if ((ctx->seen & SHMEM_SEEN_BLOCKS) && ctx->blocks) {
3510                 if (!sbinfo->max_blocks) {
3511                         err = "Cannot retroactively limit size";
3512                         goto out;
3513                 }
3514                 if (percpu_counter_compare(&sbinfo->used_blocks,
3515                                            ctx->blocks) > 0) {
3516                         err = "Too small a size for current use";
3517                         goto out;
3518                 }
3519         }
3520         if ((ctx->seen & SHMEM_SEEN_INODES) && ctx->inodes) {
3521                 if (!sbinfo->max_inodes) {
3522                         err = "Cannot retroactively limit inodes";
3523                         goto out;
3524                 }
3525                 if (ctx->inodes < inodes) {
3526                         err = "Too few inodes for current use";
3527                         goto out;
3528                 }
3529         }
3530 
3531         if (ctx->seen & SHMEM_SEEN_HUGE)
3532                 sbinfo->huge = ctx->huge;
3533         if (ctx->seen & SHMEM_SEEN_BLOCKS)
3534                 sbinfo->max_blocks  = ctx->blocks;
3535         if (ctx->seen & SHMEM_SEEN_INODES) {
3536                 sbinfo->max_inodes  = ctx->inodes;
3537                 sbinfo->free_inodes = ctx->inodes - inodes;
3538         }
3539 
3540         /*
3541          * Preserve previous mempolicy unless mpol remount option was specified.
3542          */
3543         if (ctx->mpol) {
3544                 mpol_put(sbinfo->mpol);
3545                 sbinfo->mpol = ctx->mpol;       /* transfers initial ref */
3546                 ctx->mpol = NULL;
3547         }
3548         spin_unlock(&sbinfo->stat_lock);
3549         return 0;
3550 out:
3551         spin_unlock(&sbinfo->stat_lock);
3552         return invalfc(fc, "%s", err);
3553 }
3554 
3555 static int shmem_show_options(struct seq_file *seq, struct dentry *root)
3556 {
3557         struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb);
3558 
3559         if (sbinfo->max_blocks != shmem_default_max_blocks())
3560                 seq_printf(seq, ",size=%luk",
3561                         sbinfo->max_blocks << (PAGE_SHIFT - 10));
3562         if (sbinfo->max_inodes != shmem_default_max_inodes())
3563                 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
3564         if (sbinfo->mode != (0777 | S_ISVTX))
3565                 seq_printf(seq, ",mode=%03ho", sbinfo->mode);
3566         if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
3567                 seq_printf(seq, ",uid=%u",
3568                                 from_kuid_munged(&init_user_ns, sbinfo->uid));
3569         if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
3570                 seq_printf(seq, ",gid=%u",
3571                                 from_kgid_munged(&init_user_ns, sbinfo->gid));
3572 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
3573         /* Rightly or wrongly, show huge mount option unmasked by shmem_huge */
3574         if (sbinfo->huge)
3575                 seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge));
3576 #endif
3577         shmem_show_mpol(seq, sbinfo->mpol);
3578         return 0;
3579 }
3580 
3581 #endif /* CONFIG_TMPFS */
3582 
3583 static void shmem_put_super(struct super_block *sb)
3584 {
3585         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
3586 
3587         percpu_counter_destroy(&sbinfo->used_blocks);
3588         mpol_put(sbinfo->mpol);
3589         kfree(sbinfo);
3590         sb->s_fs_info = NULL;
3591 }
3592 
3593 static int shmem_fill_super(struct super_block *sb, struct fs_context *fc)
3594 {
3595         struct shmem_options *ctx = fc->fs_private;
3596         struct inode *inode;
3597         struct shmem_sb_info *sbinfo;
3598         int err = -ENOMEM;
3599 
3600         /* Round up to L1_CACHE_BYTES to resist false sharing */
3601         sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
3602                                 L1_CACHE_BYTES), GFP_KERNEL);
3603         if (!sbinfo)
3604                 return -ENOMEM;
3605 
3606         sb->s_fs_info = sbinfo;
3607 
3608 #ifdef CONFIG_TMPFS
3609         /*
3610          * Per default we only allow half of the physical ram per
3611          * tmpfs instance, limiting inodes to one per page of lowmem;
3612          * but the internal instance is left unlimited.
3613          */
3614         if (!(sb->s_flags & SB_KERNMOUNT)) {
3615                 if (!(ctx->seen & SHMEM_SEEN_BLOCKS))
3616                         ctx->blocks = shmem_default_max_blocks();
3617                 if (!(ctx->seen & SHMEM_SEEN_INODES))
3618                         ctx->inodes = shmem_default_max_inodes();
3619         } else {
3620                 sb->s_flags |= SB_NOUSER;
3621         }
3622         sb->s_export_op = &shmem_export_ops;
3623         sb->s_flags |= SB_NOSEC;
3624 #else
3625         sb->s_flags |= SB_NOUSER;
3626 #endif
3627         sbinfo->max_blocks = ctx->blocks;
3628         sbinfo->free_inodes = sbinfo->max_inodes = ctx->inodes;
3629         sbinfo->uid = ctx->uid;
3630         sbinfo->gid = ctx->gid;
3631         sbinfo->mode = ctx->mode;
3632         sbinfo->huge = ctx->huge;
3633         sbinfo->mpol = ctx->mpol;
3634         ctx->mpol = NULL;
3635 
3636         spin_lock_init(&sbinfo->stat_lock);
3637         if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL))
3638                 goto failed;
3639         spin_lock_init(&sbinfo->shrinklist_lock);
3640         INIT_LIST_HEAD(&sbinfo->shrinklist);
3641 
3642         sb->s_maxbytes = MAX_LFS_FILESIZE;
3643         sb->s_blocksize = PAGE_SIZE;
3644         sb->s_blocksize_bits = PAGE_SHIFT;
3645         sb->s_magic = TMPFS_MAGIC;
3646         sb->s_op = &shmem_ops;
3647         sb->s_time_gran = 1;
3648 #ifdef CONFIG_TMPFS_XATTR
3649         sb->s_xattr = shmem_xattr_handlers;
3650 #endif
3651 #ifdef CONFIG_TMPFS_POSIX_ACL
3652         sb->s_flags |= SB_POSIXACL;
3653 #endif
3654         uuid_gen(&sb->s_uuid);
3655 
3656         inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
3657         if (!inode)
3658                 goto failed;
3659         inode->i_uid = sbinfo->uid;
3660         inode->i_gid = sbinfo->gid;
3661         sb->s_root = d_make_root(inode);
3662         if (!sb->s_root)
3663                 goto failed;
3664         return 0;
3665 
3666 failed:
3667         shmem_put_super(sb);
3668         return err;
3669 }
3670 
3671 static int shmem_get_tree(struct fs_context *fc)
3672 {
3673         return get_tree_nodev(fc, shmem_fill_super);
3674 }
3675 
3676 static void shmem_free_fc(struct fs_context *fc)
3677 {
3678         struct shmem_options *ctx = fc->fs_private;
3679 
3680         if (ctx) {
3681                 mpol_put(ctx->mpol);
3682                 kfree(ctx);
3683         }
3684 }
3685 
3686 static const struct fs_context_operations shmem_fs_context_ops = {
3687         .free                   = shmem_free_fc,
3688         .get_tree               = shmem_get_tree,
3689 #ifdef CONFIG_TMPFS
3690         .parse_monolithic       = shmem_parse_options,
3691         .parse_param            = shmem_parse_one,
3692         .reconfigure            = shmem_reconfigure,
3693 #endif
3694 };
3695 
3696 static struct kmem_cache *shmem_inode_cachep;
3697 
3698 static struct inode *shmem_alloc_inode(struct super_block *sb)
3699 {
3700         struct shmem_inode_info *info;
3701         info = kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL);
3702         if (!info)
3703                 return NULL;
3704         return &info->vfs_inode;
3705 }
3706 
3707 static void shmem_free_in_core_inode(struct inode *inode)
3708 {
3709         if (S_ISLNK(inode->i_mode))
3710                 kfree(inode->i_link);
3711         kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
3712 }
3713 
3714 static void shmem_destroy_inode(struct inode *inode)
3715 {
3716         if (S_ISREG(inode->i_mode))
3717                 mpol_free_shared_policy(&SHMEM_I(inode)->policy);
3718 }
3719 
3720 static void shmem_init_inode(void *foo)
3721 {
3722         struct shmem_inode_info *info = foo;
3723         inode_init_once(&info->vfs_inode);
3724 }
3725 
3726 static void shmem_init_inodecache(void)
3727 {
3728         shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
3729                                 sizeof(struct shmem_inode_info),
3730                                 0, SLAB_PANIC|SLAB_ACCOUNT, shmem_init_inode);
3731 }
3732 
3733 static void shmem_destroy_inodecache(void)
3734 {
3735         kmem_cache_destroy(shmem_inode_cachep);
3736 }
3737 
3738 static const struct address_space_operations shmem_aops = {
3739         .writepage      = shmem_writepage,
3740         .set_page_dirty = __set_page_dirty_no_writeback,
3741 #ifdef CONFIG_TMPFS
3742         .write_begin    = shmem_write_begin,
3743         .write_end      = shmem_write_end,
3744 #endif
3745 #ifdef CONFIG_MIGRATION
3746         .migratepage    = migrate_page,
3747 #endif
3748         .error_remove_page = generic_error_remove_page,
3749 };
3750 
3751 static const struct file_operations shmem_file_operations = {
3752         .mmap           = shmem_mmap,
3753         .get_unmapped_area = shmem_get_unmapped_area,
3754 #ifdef CONFIG_TMPFS
3755         .llseek         = shmem_file_llseek,
3756         .read_iter      = shmem_file_read_iter,
3757         .write_iter     = generic_file_write_iter,
3758         .fsync          = noop_fsync,
3759         .splice_read    = generic_file_splice_read,
3760         .splice_write   = iter_file_splice_write,
3761         .fallocate      = shmem_fallocate,
3762 #endif
3763 };
3764 
3765 static const struct inode_operations shmem_inode_operations = {
3766         .getattr        = shmem_getattr,
3767         .setattr        = shmem_setattr,
3768 #ifdef CONFIG_TMPFS_XATTR
3769         .listxattr      = shmem_listxattr,
3770         .set_acl        = simple_set_acl,
3771 #endif
3772 };
3773 
3774 static const struct inode_operations shmem_dir_inode_operations = {
3775 #ifdef CONFIG_TMPFS
3776         .create         = shmem_create,
3777         .lookup         = simple_lookup,
3778         .link           = shmem_link,
3779         .unlink         = shmem_unlink,
3780         .symlink        = shmem_symlink,
3781         .mkdir          = shmem_mkdir,
3782         .rmdir          = shmem_rmdir,
3783         .mknod          = shmem_mknod,
3784         .rename         = shmem_rename2,
3785         .tmpfile        = shmem_tmpfile,
3786 #endif
3787 #ifdef CONFIG_TMPFS_XATTR
3788         .listxattr      = shmem_listxattr,
3789 #endif
3790 #ifdef CONFIG_TMPFS_POSIX_ACL
3791         .setattr        = shmem_setattr,
3792         .set_acl        = simple_set_acl,
3793 #endif
3794 };
3795 
3796 static const struct inode_operations shmem_special_inode_operations = {
3797 #ifdef CONFIG_TMPFS_XATTR
3798         .listxattr      = shmem_listxattr,
3799 #endif
3800 #ifdef CONFIG_TMPFS_POSIX_ACL
3801         .setattr        = shmem_setattr,
3802         .set_acl        = simple_set_acl,
3803 #endif
3804 };
3805 
3806 static const struct super_operations shmem_ops = {
3807         .alloc_inode    = shmem_alloc_inode,
3808         .free_inode     = shmem_free_in_core_inode,
3809         .destroy_inode  = shmem_destroy_inode,
3810 #ifdef CONFIG_TMPFS
3811         .statfs         = shmem_statfs,
3812         .show_options   = shmem_show_options,
3813 #endif
3814         .evict_inode    = shmem_evict_inode,
3815         .drop_inode     = generic_delete_inode,
3816         .put_super      = shmem_put_super,
3817 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
3818         .nr_cached_objects      = shmem_unused_huge_count,
3819         .free_cached_objects    = shmem_unused_huge_scan,
3820 #endif
3821 };
3822 
3823 static const struct vm_operations_struct shmem_vm_ops = {
3824         .fault          = shmem_fault,
3825         .map_pages      = filemap_map_pages,
3826 #ifdef CONFIG_NUMA
3827         .set_policy     = shmem_set_policy,
3828         .get_policy     = shmem_get_policy,
3829 #endif
3830 };
3831 
3832 int shmem_init_fs_context(struct fs_context *fc)
3833 {
3834         struct shmem_options *ctx;
3835 
3836         ctx = kzalloc(sizeof(struct shmem_options), GFP_KERNEL);
3837         if (!ctx)
3838                 return -ENOMEM;
3839 
3840         ctx->mode = 0777 | S_ISVTX;
3841         ctx->uid = current_fsuid();
3842         ctx->gid = current_fsgid();
3843 
3844         fc->fs_private = ctx;
3845         fc->ops = &shmem_fs_context_ops;
3846         return 0;
3847 }
3848 
3849 static struct file_system_type shmem_fs_type = {
3850         .owner          = THIS_MODULE,
3851         .name           = "tmpfs",
3852         .init_fs_context = shmem_init_fs_context,
3853 #ifdef CONFIG_TMPFS
3854         .parameters     = shmem_fs_parameters,
3855 #endif
3856         .kill_sb        = kill_litter_super,
3857         .fs_flags       = FS_USERNS_MOUNT,
3858 };
3859 
3860 int __init shmem_init(void)
3861 {
3862         int error;
3863 
3864         shmem_init_inodecache();
3865 
3866         error = register_filesystem(&shmem_fs_type);
3867         if (error) {
3868                 pr_err("Could not register tmpfs\n");
3869                 goto out2;
3870         }
3871 
3872         shm_mnt = kern_mount(&shmem_fs_type);
3873         if (IS_ERR(shm_mnt)) {
3874                 error = PTR_ERR(shm_mnt);
3875                 pr_err("Could not kern_mount tmpfs\n");
3876                 goto out1;
3877         }
3878 
3879 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
3880         if (has_transparent_hugepage() && shmem_huge > SHMEM_HUGE_DENY)
3881                 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
3882         else
3883                 shmem_huge = 0; /* just in case it was patched */
3884 #endif
3885         return 0;
3886 
3887 out1:
3888         unregister_filesystem(&shmem_fs_type);
3889 out2:
3890         shmem_destroy_inodecache();
3891         shm_mnt = ERR_PTR(error);
3892         return error;
3893 }
3894 
3895 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_SYSFS)
3896 static ssize_t shmem_enabled_show(struct kobject *kobj,
3897                 struct kobj_attribute *attr, char *buf)
3898 {
3899         static const int values[] = {
3900                 SHMEM_HUGE_ALWAYS,
3901                 SHMEM_HUGE_WITHIN_SIZE,
3902                 SHMEM_HUGE_ADVISE,
3903                 SHMEM_HUGE_NEVER,
3904                 SHMEM_HUGE_DENY,
3905                 SHMEM_HUGE_FORCE,
3906         };
3907         int i, count;
3908 
3909         for (i = 0, count = 0; i < ARRAY_SIZE(values); i++) {
3910                 const char *fmt = shmem_huge == values[i] ? "[%s] " : "%s ";
3911 
3912                 count += sprintf(buf + count, fmt,
3913                                 shmem_format_huge(values[i]));
3914         }
3915         buf[count - 1] = '\n';
3916         return count;
3917 }
3918 
3919 static ssize_t shmem_enabled_store(struct kobject *kobj,
3920                 struct kobj_attribute *attr, const char *buf, size_t count)
3921 {
3922         char tmp[16];
3923         int huge;
3924 
3925         if (count + 1 > sizeof(tmp))
3926                 return -EINVAL;
3927         memcpy(tmp, buf, count);
3928         tmp[count] = '\0';
3929         if (count && tmp[count - 1] == '\n')
3930                 tmp[count - 1] = '\0';
3931 
3932         huge = shmem_parse_huge(tmp);
3933         if (huge == -EINVAL)
3934                 return -EINVAL;
3935         if (!has_transparent_hugepage() &&
3936                         huge != SHMEM_HUGE_NEVER && huge != SHMEM_HUGE_DENY)
3937                 return -EINVAL;
3938 
3939         shmem_huge = huge;
3940         if (shmem_huge > SHMEM_HUGE_DENY)
3941                 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
3942         return count;
3943 }
3944 
3945 struct kobj_attribute shmem_enabled_attr =
3946         __ATTR(shmem_enabled, 0644, shmem_enabled_show, shmem_enabled_store);
3947 #endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONFIG_SYSFS */
3948 
3949 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
3950 bool shmem_huge_enabled(struct vm_area_struct *vma)
3951 {
3952         struct inode *inode = file_inode(vma->vm_file);
3953         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
3954         loff_t i_size;
3955         pgoff_t off;
3956 
3957         if ((vma->vm_flags & VM_NOHUGEPAGE) ||
3958             test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))
3959                 return false;
3960         if (shmem_huge == SHMEM_HUGE_FORCE)
3961                 return true;
3962         if (shmem_huge == SHMEM_HUGE_DENY)
3963                 return false;
3964         switch (sbinfo->huge) {
3965                 case SHMEM_HUGE_NEVER:
3966                         return false;
3967                 case SHMEM_HUGE_ALWAYS:
3968                         return true;
3969                 case SHMEM_HUGE_WITHIN_SIZE:
3970                         off = round_up(vma->vm_pgoff, HPAGE_PMD_NR);
3971                         i_size = round_up(i_size_read(inode), PAGE_SIZE);
3972                         if (i_size >= HPAGE_PMD_SIZE &&
3973                                         i_size >> PAGE_SHIFT >= off)
3974                                 return true;
3975                         fallthrough;
3976                 case SHMEM_HUGE_ADVISE:
3977                         /* TODO: implement fadvise() hints */
3978                         return (vma->vm_flags & VM_HUGEPAGE);
3979                 default:
3980                         VM_BUG_ON(1);
3981                         return false;
3982         }
3983 }
3984 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
3985 
3986 #else /* !CONFIG_SHMEM */
3987 
3988 /*
3989  * tiny-shmem: simple shmemfs and tmpfs using ramfs code
3990  *
3991  * This is intended for small system where the benefits of the full
3992  * shmem code (swap-backed and resource-limited) are outweighed by
3993  * their complexity. On systems without swap this code should be
3994  * effectively equivalent, but much lighter weight.
3995  */
3996 
3997 static struct file_system_type shmem_fs_type = {
3998         .name           = "tmpfs",
3999         .init_fs_context = ramfs_init_fs_context,
4000         .parameters     = ramfs_fs_parameters,
4001         .kill_sb        = kill_litter_super,
4002         .fs_flags       = FS_USERNS_MOUNT,
4003 };
4004 
4005 int __init shmem_init(void)
4006 {
4007         BUG_ON(register_filesystem(&shmem_fs_type) != 0);
4008 
4009         shm_mnt = kern_mount(&shmem_fs_type);
4010         BUG_ON(IS_ERR(shm_mnt));
4011 
4012         return 0;
4013 }
4014 
4015 int shmem_unuse(unsigned int type, bool frontswap,
4016                 unsigned long *fs_pages_to_unuse)
4017 {
4018         return 0;
4019 }
4020 
4021 int shmem_lock(struct file *file, int lock, struct user_struct *user)
4022 {
4023         return 0;
4024 }
4025 
4026 void shmem_unlock_mapping(struct address_space *mapping)
4027 {
4028 }
4029 
4030 #ifdef CONFIG_MMU
4031 unsigned long shmem_get_unmapped_area(struct file *file,
4032                                       unsigned long addr, unsigned long len,
4033                                       unsigned long pgoff, unsigned long flags)
4034 {
4035         return current->mm->get_unmapped_area(file, addr, len, pgoff, flags);
4036 }
4037 #endif
4038 
4039 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
4040 {
4041         truncate_inode_pages_range(inode->i_mapping, lstart, lend);
4042 }
4043 EXPORT_SYMBOL_GPL(shmem_truncate_range);
4044 
4045 #define shmem_vm_ops                            generic_file_vm_ops
4046 #define shmem_file_operations                   ramfs_file_operations
4047 #define shmem_get_inode(sb, dir, mode, dev, flags)      ramfs_get_inode(sb, dir, mode, dev)
4048 #define shmem_acct_size(flags, size)            0
4049 #define shmem_unacct_size(flags, size)          do {} while (0)
4050 
4051 #endif /* CONFIG_SHMEM */
4052 
4053 /* common code */
4054 
4055 static struct file *__shmem_file_setup(struct vfsmount *mnt, const char *name, loff_t size,
4056                                        unsigned long flags, unsigned int i_flags)
4057 {
4058         struct inode *inode;
4059         struct file *res;
4060 
4061         if (IS_ERR(mnt))
4062                 return ERR_CAST(mnt);
4063 
4064         if (size < 0 || size > MAX_LFS_FILESIZE)
4065                 return ERR_PTR(-EINVAL);
4066 
4067         if (shmem_acct_size(flags, size))
4068                 return ERR_PTR(-ENOMEM);
4069 
4070         inode = shmem_get_inode(mnt->mnt_sb, NULL, S_IFREG | S_IRWXUGO, 0,
4071                                 flags);
4072         if (unlikely(!inode)) {
4073                 shmem_unacct_size(flags, size);
4074                 return ERR_PTR(-ENOSPC);
4075         }
4076         inode->i_flags |= i_flags;
4077         inode->i_size = size;
4078         clear_nlink(inode);     /* It is unlinked */
4079         res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size));
4080         if (!IS_ERR(res))
4081                 res = alloc_file_pseudo(inode, mnt, name, O_RDWR,
4082                                 &shmem_file_operations);
4083         if (IS_ERR(res))
4084                 iput(inode);
4085         return res;
4086 }
4087 
4088 /**
4089  * shmem_kernel_file_setup - get an unlinked file living in tmpfs which must be
4090  *      kernel internal.  There will be NO LSM permission checks against the
4091  *      underlying inode.  So users of this interface must do LSM checks at a
4092  *      higher layer.  The users are the big_key and shm implementations.  LSM
4093  *      checks are provided at the key or shm level rather than the inode.
4094  * @name: name for dentry (to be seen in /proc/<pid>/maps
4095  * @size: size to be set for the file
4096  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
4097  */
4098 struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags)
4099 {
4100         return __shmem_file_setup(shm_mnt, name, size, flags, S_PRIVATE);
4101 }
4102 
4103 /**
4104  * shmem_file_setup - get an unlinked file living in tmpfs
4105  * @name: name for dentry (to be seen in /proc/<pid>/maps
4106  * @size: size to be set for the file
4107  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
4108  */
4109 struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
4110 {
4111         return __shmem_file_setup(shm_mnt, name, size, flags, 0);
4112 }
4113 EXPORT_SYMBOL_GPL(shmem_file_setup);
4114 
4115 /**
4116  * shmem_file_setup_with_mnt - get an unlinked file living in tmpfs
4117  * @mnt: the tmpfs mount where the file will be created
4118  * @name: name for dentry (to be seen in /proc/<pid>/maps
4119  * @size: size to be set for the file
4120  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
4121  */
4122 struct file *shmem_file_setup_with_mnt(struct vfsmount *mnt, const char *name,
4123                                        loff_t size, unsigned long flags)
4124 {
4125         return __shmem_file_setup(mnt, name, size, flags, 0);
4126 }
4127 EXPORT_SYMBOL_GPL(shmem_file_setup_with_mnt);
4128 
4129 /**
4130  * shmem_zero_setup - setup a shared anonymous mapping
4131  * @vma: the vma to be mmapped is prepared by do_mmap_pgoff
4132  */
4133 int shmem_zero_setup(struct vm_area_struct *vma)
4134 {
4135         struct file *file;
4136         loff_t size = vma->vm_end - vma->vm_start;
4137 
4138         /*
4139          * Cloning a new file under mmap_lock leads to a lock ordering conflict
4140          * between XFS directory reading and selinux: since this file is only
4141          * accessible to the user through its mapping, use S_PRIVATE flag to
4142          * bypass file security, in the same way as shmem_kernel_file_setup().
4143          */
4144         file = shmem_kernel_file_setup("dev/zero", size, vma->vm_flags);
4145         if (IS_ERR(file))
4146                 return PTR_ERR(file);
4147 
4148         if (vma->vm_file)
4149                 fput(vma->vm_file);
4150         vma->vm_file = file;
4151         vma->vm_ops = &shmem_vm_ops;
4152 
4153         if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
4154                         ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) <
4155                         (vma->vm_end & HPAGE_PMD_MASK)) {
4156                 khugepaged_enter(vma, vma->vm_flags);
4157         }
4158 
4159         return 0;
4160 }
4161 
4162 /**
4163  * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags.
4164  * @mapping:    the page's address_space
4165  * @index:      the page index
4166  * @gfp:        the page allocator flags to use if allocating
4167  *
4168  * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)",
4169  * with any new page allocations done using the specified allocation flags.
4170  * But read_cache_page_gfp() uses the ->readpage() method: which does not
4171  * suit tmpfs, since it may have pages in swapcache, and needs to find those
4172  * for itself; although drivers/gpu/drm i915 and ttm rely upon this support.
4173  *
4174  * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in
4175  * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily.
4176  */
4177 struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
4178                                          pgoff_t index, gfp_t gfp)
4179 {
4180 #ifdef CONFIG_SHMEM
4181         struct inode *inode = mapping->host;
4182         struct page *page;
4183         int error;
4184 
4185         BUG_ON(mapping->a_ops != &shmem_aops);
4186         error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE,
4187                                   gfp, NULL, NULL, NULL);
4188         if (error)
4189                 page = ERR_PTR(error);
4190         else
4191                 unlock_page(page);
4192         return page;
4193 #else
4194         /*
4195          * The tiny !SHMEM case uses ramfs without swap
4196          */
4197         return read_cache_page_gfp(mapping, index, gfp);
4198 #endif
4199 }
4200 EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp);
4201 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp