~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/mm/shmem.c

Version: ~ [ linux-5.4-rc3 ] ~ [ linux-5.3.6 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.79 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.149 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.196 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.196 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.75 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Resizable virtual memory filesystem for Linux.
  3  *
  4  * Copyright (C) 2000 Linus Torvalds.
  5  *               2000 Transmeta Corp.
  6  *               2000-2001 Christoph Rohland
  7  *               2000-2001 SAP AG
  8  *               2002 Red Hat Inc.
  9  * Copyright (C) 2002-2011 Hugh Dickins.
 10  * Copyright (C) 2011 Google Inc.
 11  * Copyright (C) 2002-2005 VERITAS Software Corporation.
 12  * Copyright (C) 2004 Andi Kleen, SuSE Labs
 13  *
 14  * Extended attribute support for tmpfs:
 15  * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
 16  * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
 17  *
 18  * tiny-shmem:
 19  * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
 20  *
 21  * This file is released under the GPL.
 22  */
 23 
 24 #include <linux/fs.h>
 25 #include <linux/init.h>
 26 #include <linux/vfs.h>
 27 #include <linux/mount.h>
 28 #include <linux/ramfs.h>
 29 #include <linux/pagemap.h>
 30 #include <linux/file.h>
 31 #include <linux/mm.h>
 32 #include <linux/export.h>
 33 #include <linux/swap.h>
 34 #include <linux/aio.h>
 35 
 36 static struct vfsmount *shm_mnt;
 37 
 38 #ifdef CONFIG_SHMEM
 39 /*
 40  * This virtual memory filesystem is heavily based on the ramfs. It
 41  * extends ramfs by the ability to use swap and honor resource limits
 42  * which makes it a completely usable filesystem.
 43  */
 44 
 45 #include <linux/xattr.h>
 46 #include <linux/exportfs.h>
 47 #include <linux/posix_acl.h>
 48 #include <linux/generic_acl.h>
 49 #include <linux/mman.h>
 50 #include <linux/string.h>
 51 #include <linux/slab.h>
 52 #include <linux/backing-dev.h>
 53 #include <linux/shmem_fs.h>
 54 #include <linux/writeback.h>
 55 #include <linux/blkdev.h>
 56 #include <linux/pagevec.h>
 57 #include <linux/percpu_counter.h>
 58 #include <linux/falloc.h>
 59 #include <linux/splice.h>
 60 #include <linux/security.h>
 61 #include <linux/swapops.h>
 62 #include <linux/mempolicy.h>
 63 #include <linux/namei.h>
 64 #include <linux/ctype.h>
 65 #include <linux/migrate.h>
 66 #include <linux/highmem.h>
 67 #include <linux/seq_file.h>
 68 #include <linux/magic.h>
 69 
 70 #include <asm/uaccess.h>
 71 #include <asm/pgtable.h>
 72 
 73 #define BLOCKS_PER_PAGE  (PAGE_CACHE_SIZE/512)
 74 #define VM_ACCT(size)    (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT)
 75 
 76 /* Pretend that each entry is of this size in directory's i_size */
 77 #define BOGO_DIRENT_SIZE 20
 78 
 79 /* Symlink up to this size is kmalloc'ed instead of using a swappable page */
 80 #define SHORT_SYMLINK_LEN 128
 81 
 82 /*
 83  * shmem_fallocate communicates with shmem_fault or shmem_writepage via
 84  * inode->i_private (with i_mutex making sure that it has only one user at
 85  * a time): we would prefer not to enlarge the shmem inode just for that.
 86  */
 87 struct shmem_falloc {
 88         wait_queue_head_t *waitq; /* faults into hole wait for punch to end */
 89         pgoff_t start;          /* start of range currently being fallocated */
 90         pgoff_t next;           /* the next page offset to be fallocated */
 91         pgoff_t nr_falloced;    /* how many new pages have been fallocated */
 92         pgoff_t nr_unswapped;   /* how often writepage refused to swap out */
 93 };
 94 
 95 /* Flag allocation requirements to shmem_getpage */
 96 enum sgp_type {
 97         SGP_READ,       /* don't exceed i_size, don't allocate page */
 98         SGP_CACHE,      /* don't exceed i_size, may allocate page */
 99         SGP_DIRTY,      /* like SGP_CACHE, but set new page dirty */
100         SGP_WRITE,      /* may exceed i_size, may allocate !Uptodate page */
101         SGP_FALLOC,     /* like SGP_WRITE, but make existing page Uptodate */
102 };
103 
104 #ifdef CONFIG_TMPFS
105 static unsigned long shmem_default_max_blocks(void)
106 {
107         return totalram_pages / 2;
108 }
109 
110 static unsigned long shmem_default_max_inodes(void)
111 {
112         return min(totalram_pages - totalhigh_pages, totalram_pages / 2);
113 }
114 #endif
115 
116 static bool shmem_should_replace_page(struct page *page, gfp_t gfp);
117 static int shmem_replace_page(struct page **pagep, gfp_t gfp,
118                                 struct shmem_inode_info *info, pgoff_t index);
119 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
120         struct page **pagep, enum sgp_type sgp, gfp_t gfp, int *fault_type);
121 
122 static inline int shmem_getpage(struct inode *inode, pgoff_t index,
123         struct page **pagep, enum sgp_type sgp, int *fault_type)
124 {
125         return shmem_getpage_gfp(inode, index, pagep, sgp,
126                         mapping_gfp_mask(inode->i_mapping), fault_type);
127 }
128 
129 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
130 {
131         return sb->s_fs_info;
132 }
133 
134 /*
135  * shmem_file_setup pre-accounts the whole fixed size of a VM object,
136  * for shared memory and for shared anonymous (/dev/zero) mappings
137  * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
138  * consistent with the pre-accounting of private mappings ...
139  */
140 static inline int shmem_acct_size(unsigned long flags, loff_t size)
141 {
142         return (flags & VM_NORESERVE) ?
143                 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size));
144 }
145 
146 static inline void shmem_unacct_size(unsigned long flags, loff_t size)
147 {
148         if (!(flags & VM_NORESERVE))
149                 vm_unacct_memory(VM_ACCT(size));
150 }
151 
152 /*
153  * ... whereas tmpfs objects are accounted incrementally as
154  * pages are allocated, in order to allow huge sparse files.
155  * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
156  * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
157  */
158 static inline int shmem_acct_block(unsigned long flags)
159 {
160         return (flags & VM_NORESERVE) ?
161                 security_vm_enough_memory_mm(current->mm, VM_ACCT(PAGE_CACHE_SIZE)) : 0;
162 }
163 
164 static inline void shmem_unacct_blocks(unsigned long flags, long pages)
165 {
166         if (flags & VM_NORESERVE)
167                 vm_unacct_memory(pages * VM_ACCT(PAGE_CACHE_SIZE));
168 }
169 
170 static const struct super_operations shmem_ops;
171 static const struct address_space_operations shmem_aops;
172 static const struct file_operations shmem_file_operations;
173 static const struct inode_operations shmem_inode_operations;
174 static const struct inode_operations shmem_dir_inode_operations;
175 static const struct inode_operations shmem_special_inode_operations;
176 static const struct vm_operations_struct shmem_vm_ops;
177 
178 static struct backing_dev_info shmem_backing_dev_info  __read_mostly = {
179         .ra_pages       = 0,    /* No readahead */
180         .capabilities   = BDI_CAP_NO_ACCT_AND_WRITEBACK | BDI_CAP_SWAP_BACKED,
181 };
182 
183 static LIST_HEAD(shmem_swaplist);
184 static DEFINE_MUTEX(shmem_swaplist_mutex);
185 
186 static int shmem_reserve_inode(struct super_block *sb)
187 {
188         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
189         if (sbinfo->max_inodes) {
190                 spin_lock(&sbinfo->stat_lock);
191                 if (!sbinfo->free_inodes) {
192                         spin_unlock(&sbinfo->stat_lock);
193                         return -ENOSPC;
194                 }
195                 sbinfo->free_inodes--;
196                 spin_unlock(&sbinfo->stat_lock);
197         }
198         return 0;
199 }
200 
201 static void shmem_free_inode(struct super_block *sb)
202 {
203         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
204         if (sbinfo->max_inodes) {
205                 spin_lock(&sbinfo->stat_lock);
206                 sbinfo->free_inodes++;
207                 spin_unlock(&sbinfo->stat_lock);
208         }
209 }
210 
211 /**
212  * shmem_recalc_inode - recalculate the block usage of an inode
213  * @inode: inode to recalc
214  *
215  * We have to calculate the free blocks since the mm can drop
216  * undirtied hole pages behind our back.
217  *
218  * But normally   info->alloced == inode->i_mapping->nrpages + info->swapped
219  * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
220  *
221  * It has to be called with the spinlock held.
222  */
223 static void shmem_recalc_inode(struct inode *inode)
224 {
225         struct shmem_inode_info *info = SHMEM_I(inode);
226         long freed;
227 
228         freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
229         if (freed > 0) {
230                 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
231                 if (sbinfo->max_blocks)
232                         percpu_counter_add(&sbinfo->used_blocks, -freed);
233                 info->alloced -= freed;
234                 inode->i_blocks -= freed * BLOCKS_PER_PAGE;
235                 shmem_unacct_blocks(info->flags, freed);
236         }
237 }
238 
239 /*
240  * Replace item expected in radix tree by a new item, while holding tree lock.
241  */
242 static int shmem_radix_tree_replace(struct address_space *mapping,
243                         pgoff_t index, void *expected, void *replacement)
244 {
245         void **pslot;
246         void *item;
247 
248         VM_BUG_ON(!expected);
249         VM_BUG_ON(!replacement);
250         pslot = radix_tree_lookup_slot(&mapping->page_tree, index);
251         if (!pslot)
252                 return -ENOENT;
253         item = radix_tree_deref_slot_protected(pslot, &mapping->tree_lock);
254         if (item != expected)
255                 return -ENOENT;
256         radix_tree_replace_slot(pslot, replacement);
257         return 0;
258 }
259 
260 /*
261  * Sometimes, before we decide whether to proceed or to fail, we must check
262  * that an entry was not already brought back from swap by a racing thread.
263  *
264  * Checking page is not enough: by the time a SwapCache page is locked, it
265  * might be reused, and again be SwapCache, using the same swap as before.
266  */
267 static bool shmem_confirm_swap(struct address_space *mapping,
268                                pgoff_t index, swp_entry_t swap)
269 {
270         void *item;
271 
272         rcu_read_lock();
273         item = radix_tree_lookup(&mapping->page_tree, index);
274         rcu_read_unlock();
275         return item == swp_to_radix_entry(swap);
276 }
277 
278 /*
279  * Like add_to_page_cache_locked, but error if expected item has gone.
280  */
281 static int shmem_add_to_page_cache(struct page *page,
282                                    struct address_space *mapping,
283                                    pgoff_t index, gfp_t gfp, void *expected)
284 {
285         int error;
286 
287         VM_BUG_ON(!PageLocked(page));
288         VM_BUG_ON(!PageSwapBacked(page));
289 
290         page_cache_get(page);
291         page->mapping = mapping;
292         page->index = index;
293 
294         spin_lock_irq(&mapping->tree_lock);
295         if (!expected)
296                 error = radix_tree_insert(&mapping->page_tree, index, page);
297         else
298                 error = shmem_radix_tree_replace(mapping, index, expected,
299                                                                  page);
300         if (!error) {
301                 mapping->nrpages++;
302                 __inc_zone_page_state(page, NR_FILE_PAGES);
303                 __inc_zone_page_state(page, NR_SHMEM);
304                 spin_unlock_irq(&mapping->tree_lock);
305         } else {
306                 page->mapping = NULL;
307                 spin_unlock_irq(&mapping->tree_lock);
308                 page_cache_release(page);
309         }
310         return error;
311 }
312 
313 /*
314  * Like delete_from_page_cache, but substitutes swap for page.
315  */
316 static void shmem_delete_from_page_cache(struct page *page, void *radswap)
317 {
318         struct address_space *mapping = page->mapping;
319         int error;
320 
321         spin_lock_irq(&mapping->tree_lock);
322         error = shmem_radix_tree_replace(mapping, page->index, page, radswap);
323         page->mapping = NULL;
324         mapping->nrpages--;
325         __dec_zone_page_state(page, NR_FILE_PAGES);
326         __dec_zone_page_state(page, NR_SHMEM);
327         spin_unlock_irq(&mapping->tree_lock);
328         page_cache_release(page);
329         BUG_ON(error);
330 }
331 
332 /*
333  * Remove swap entry from radix tree, free the swap and its page cache.
334  */
335 static int shmem_free_swap(struct address_space *mapping,
336                            pgoff_t index, void *radswap)
337 {
338         void *old;
339 
340         spin_lock_irq(&mapping->tree_lock);
341         old = radix_tree_delete_item(&mapping->page_tree, index, radswap);
342         spin_unlock_irq(&mapping->tree_lock);
343         if (old != radswap)
344                 return -ENOENT;
345         free_swap_and_cache(radix_to_swp_entry(radswap));
346         return 0;
347 }
348 
349 /*
350  * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists.
351  */
352 void shmem_unlock_mapping(struct address_space *mapping)
353 {
354         struct pagevec pvec;
355         pgoff_t indices[PAGEVEC_SIZE];
356         pgoff_t index = 0;
357 
358         pagevec_init(&pvec, 0);
359         /*
360          * Minor point, but we might as well stop if someone else SHM_LOCKs it.
361          */
362         while (!mapping_unevictable(mapping)) {
363                 /*
364                  * Avoid pagevec_lookup(): find_get_pages() returns 0 as if it
365                  * has finished, if it hits a row of PAGEVEC_SIZE swap entries.
366                  */
367                 pvec.nr = find_get_entries(mapping, index,
368                                            PAGEVEC_SIZE, pvec.pages, indices);
369                 if (!pvec.nr)
370                         break;
371                 index = indices[pvec.nr - 1] + 1;
372                 pagevec_remove_exceptionals(&pvec);
373                 check_move_unevictable_pages(pvec.pages, pvec.nr);
374                 pagevec_release(&pvec);
375                 cond_resched();
376         }
377 }
378 
379 /*
380  * Remove range of pages and swap entries from radix tree, and free them.
381  * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate.
382  */
383 static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
384                                                                  bool unfalloc)
385 {
386         struct address_space *mapping = inode->i_mapping;
387         struct shmem_inode_info *info = SHMEM_I(inode);
388         pgoff_t start = (lstart + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
389         pgoff_t end = (lend + 1) >> PAGE_CACHE_SHIFT;
390         unsigned int partial_start = lstart & (PAGE_CACHE_SIZE - 1);
391         unsigned int partial_end = (lend + 1) & (PAGE_CACHE_SIZE - 1);
392         struct pagevec pvec;
393         pgoff_t indices[PAGEVEC_SIZE];
394         long nr_swaps_freed = 0;
395         pgoff_t index;
396         int i;
397 
398         if (lend == -1)
399                 end = -1;       /* unsigned, so actually very big */
400 
401         pagevec_init(&pvec, 0);
402         index = start;
403         while (index < end) {
404                 pvec.nr = find_get_entries(mapping, index,
405                         min(end - index, (pgoff_t)PAGEVEC_SIZE),
406                         pvec.pages, indices);
407                 if (!pvec.nr)
408                         break;
409                 mem_cgroup_uncharge_start();
410                 for (i = 0; i < pagevec_count(&pvec); i++) {
411                         struct page *page = pvec.pages[i];
412 
413                         index = indices[i];
414                         if (index >= end)
415                                 break;
416 
417                         if (radix_tree_exceptional_entry(page)) {
418                                 if (unfalloc)
419                                         continue;
420                                 nr_swaps_freed += !shmem_free_swap(mapping,
421                                                                 index, page);
422                                 continue;
423                         }
424 
425                         if (!trylock_page(page))
426                                 continue;
427                         if (!unfalloc || !PageUptodate(page)) {
428                                 if (page->mapping == mapping) {
429                                         VM_BUG_ON(PageWriteback(page));
430                                         truncate_inode_page(mapping, page);
431                                 }
432                         }
433                         unlock_page(page);
434                 }
435                 pagevec_remove_exceptionals(&pvec);
436                 pagevec_release(&pvec);
437                 mem_cgroup_uncharge_end();
438                 cond_resched();
439                 index++;
440         }
441 
442         if (partial_start) {
443                 struct page *page = NULL;
444                 shmem_getpage(inode, start - 1, &page, SGP_READ, NULL);
445                 if (page) {
446                         unsigned int top = PAGE_CACHE_SIZE;
447                         if (start > end) {
448                                 top = partial_end;
449                                 partial_end = 0;
450                         }
451                         zero_user_segment(page, partial_start, top);
452                         set_page_dirty(page);
453                         unlock_page(page);
454                         page_cache_release(page);
455                 }
456         }
457         if (partial_end) {
458                 struct page *page = NULL;
459                 shmem_getpage(inode, end, &page, SGP_READ, NULL);
460                 if (page) {
461                         zero_user_segment(page, 0, partial_end);
462                         set_page_dirty(page);
463                         unlock_page(page);
464                         page_cache_release(page);
465                 }
466         }
467         if (start >= end)
468                 return;
469 
470         index = start;
471         while (index < end) {
472                 cond_resched();
473 
474                 pvec.nr = find_get_entries(mapping, index,
475                                 min(end - index, (pgoff_t)PAGEVEC_SIZE),
476                                 pvec.pages, indices);
477                 if (!pvec.nr) {
478                         /* If all gone or hole-punch or unfalloc, we're done */
479                         if (index == start || end != -1)
480                                 break;
481                         /* But if truncating, restart to make sure all gone */
482                         index = start;
483                         continue;
484                 }
485                 mem_cgroup_uncharge_start();
486                 for (i = 0; i < pagevec_count(&pvec); i++) {
487                         struct page *page = pvec.pages[i];
488 
489                         index = indices[i];
490                         if (index >= end)
491                                 break;
492 
493                         if (radix_tree_exceptional_entry(page)) {
494                                 if (unfalloc)
495                                         continue;
496                                 if (shmem_free_swap(mapping, index, page)) {
497                                         /* Swap was replaced by page: retry */
498                                         index--;
499                                         break;
500                                 }
501                                 nr_swaps_freed++;
502                                 continue;
503                         }
504 
505                         lock_page(page);
506                         if (!unfalloc || !PageUptodate(page)) {
507                                 if (page->mapping == mapping) {
508                                         VM_BUG_ON(PageWriteback(page));
509                                         truncate_inode_page(mapping, page);
510                                 } else {
511                                         /* Page was replaced by swap: retry */
512                                         unlock_page(page);
513                                         index--;
514                                         break;
515                                 }
516                         }
517                         unlock_page(page);
518                 }
519                 pagevec_remove_exceptionals(&pvec);
520                 pagevec_release(&pvec);
521                 mem_cgroup_uncharge_end();
522                 index++;
523         }
524 
525         spin_lock(&info->lock);
526         info->swapped -= nr_swaps_freed;
527         shmem_recalc_inode(inode);
528         spin_unlock(&info->lock);
529 }
530 
531 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
532 {
533         shmem_undo_range(inode, lstart, lend, false);
534         inode->i_ctime = inode->i_mtime = CURRENT_TIME;
535 }
536 EXPORT_SYMBOL_GPL(shmem_truncate_range);
537 
538 static int shmem_setattr(struct dentry *dentry, struct iattr *attr)
539 {
540         struct inode *inode = dentry->d_inode;
541         int error;
542 
543         error = inode_change_ok(inode, attr);
544         if (error)
545                 return error;
546 
547         if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
548                 loff_t oldsize = inode->i_size;
549                 loff_t newsize = attr->ia_size;
550 
551                 if (newsize != oldsize) {
552                         i_size_write(inode, newsize);
553                         inode->i_ctime = inode->i_mtime = CURRENT_TIME;
554                 }
555                 if (newsize < oldsize) {
556                         loff_t holebegin = round_up(newsize, PAGE_SIZE);
557                         unmap_mapping_range(inode->i_mapping, holebegin, 0, 1);
558                         shmem_truncate_range(inode, newsize, (loff_t)-1);
559                         /* unmap again to remove racily COWed private pages */
560                         unmap_mapping_range(inode->i_mapping, holebegin, 0, 1);
561                 }
562         }
563 
564         setattr_copy(inode, attr);
565 #ifdef CONFIG_TMPFS_POSIX_ACL
566         if (attr->ia_valid & ATTR_MODE)
567                 error = generic_acl_chmod(inode);
568 #endif
569         return error;
570 }
571 
572 static void shmem_evict_inode(struct inode *inode)
573 {
574         struct shmem_inode_info *info = SHMEM_I(inode);
575 
576         if (inode->i_mapping->a_ops == &shmem_aops) {
577                 shmem_unacct_size(info->flags, inode->i_size);
578                 inode->i_size = 0;
579                 shmem_truncate_range(inode, 0, (loff_t)-1);
580                 if (!list_empty(&info->swaplist)) {
581                         mutex_lock(&shmem_swaplist_mutex);
582                         list_del_init(&info->swaplist);
583                         mutex_unlock(&shmem_swaplist_mutex);
584                 }
585         } else
586                 kfree(info->symlink);
587 
588         simple_xattrs_free(&info->xattrs);
589         WARN_ON(inode->i_blocks);
590         shmem_free_inode(inode->i_sb);
591         clear_inode(inode);
592 }
593 
594 /*
595  * If swap found in inode, free it and move page from swapcache to filecache.
596  */
597 static int shmem_unuse_inode(struct shmem_inode_info *info,
598                              swp_entry_t swap, struct page **pagep)
599 {
600         struct address_space *mapping = info->vfs_inode.i_mapping;
601         void *radswap;
602         pgoff_t index;
603         gfp_t gfp;
604         int error = 0;
605 
606         radswap = swp_to_radix_entry(swap);
607         index = radix_tree_locate_item(&mapping->page_tree, radswap);
608         if (index == -1)
609                 return 0;
610 
611         /*
612          * Move _head_ to start search for next from here.
613          * But be careful: shmem_evict_inode checks list_empty without taking
614          * mutex, and there's an instant in list_move_tail when info->swaplist
615          * would appear empty, if it were the only one on shmem_swaplist.
616          */
617         if (shmem_swaplist.next != &info->swaplist)
618                 list_move_tail(&shmem_swaplist, &info->swaplist);
619 
620         gfp = mapping_gfp_mask(mapping);
621         if (shmem_should_replace_page(*pagep, gfp)) {
622                 mutex_unlock(&shmem_swaplist_mutex);
623                 error = shmem_replace_page(pagep, gfp, info, index);
624                 mutex_lock(&shmem_swaplist_mutex);
625                 /*
626                  * We needed to drop mutex to make that restrictive page
627                  * allocation, but the inode might have been freed while we
628                  * dropped it: although a racing shmem_evict_inode() cannot
629                  * complete without emptying the radix_tree, our page lock
630                  * on this swapcache page is not enough to prevent that -
631                  * free_swap_and_cache() of our swap entry will only
632                  * trylock_page(), removing swap from radix_tree whatever.
633                  *
634                  * We must not proceed to shmem_add_to_page_cache() if the
635                  * inode has been freed, but of course we cannot rely on
636                  * inode or mapping or info to check that.  However, we can
637                  * safely check if our swap entry is still in use (and here
638                  * it can't have got reused for another page): if it's still
639                  * in use, then the inode cannot have been freed yet, and we
640                  * can safely proceed (if it's no longer in use, that tells
641                  * nothing about the inode, but we don't need to unuse swap).
642                  */
643                 if (!page_swapcount(*pagep))
644                         error = -ENOENT;
645         }
646 
647         /*
648          * We rely on shmem_swaplist_mutex, not only to protect the swaplist,
649          * but also to hold up shmem_evict_inode(): so inode cannot be freed
650          * beneath us (pagelock doesn't help until the page is in pagecache).
651          */
652         if (!error)
653                 error = shmem_add_to_page_cache(*pagep, mapping, index,
654                                                 GFP_NOWAIT, radswap);
655         if (error != -ENOMEM) {
656                 /*
657                  * Truncation and eviction use free_swap_and_cache(), which
658                  * only does trylock page: if we raced, best clean up here.
659                  */
660                 delete_from_swap_cache(*pagep);
661                 set_page_dirty(*pagep);
662                 if (!error) {
663                         spin_lock(&info->lock);
664                         info->swapped--;
665                         spin_unlock(&info->lock);
666                         swap_free(swap);
667                 }
668                 error = 1;      /* not an error, but entry was found */
669         }
670         return error;
671 }
672 
673 /*
674  * Search through swapped inodes to find and replace swap by page.
675  */
676 int shmem_unuse(swp_entry_t swap, struct page *page)
677 {
678         struct list_head *this, *next;
679         struct shmem_inode_info *info;
680         int found = 0;
681         int error = 0;
682 
683         /*
684          * There's a faint possibility that swap page was replaced before
685          * caller locked it: caller will come back later with the right page.
686          */
687         if (unlikely(!PageSwapCache(page) || page_private(page) != swap.val))
688                 goto out;
689 
690         /*
691          * Charge page using GFP_KERNEL while we can wait, before taking
692          * the shmem_swaplist_mutex which might hold up shmem_writepage().
693          * Charged back to the user (not to caller) when swap account is used.
694          */
695         error = mem_cgroup_cache_charge(page, current->mm, GFP_KERNEL);
696         if (error)
697                 goto out;
698         /* No radix_tree_preload: swap entry keeps a place for page in tree */
699 
700         mutex_lock(&shmem_swaplist_mutex);
701         list_for_each_safe(this, next, &shmem_swaplist) {
702                 info = list_entry(this, struct shmem_inode_info, swaplist);
703                 if (info->swapped)
704                         found = shmem_unuse_inode(info, swap, &page);
705                 else
706                         list_del_init(&info->swaplist);
707                 cond_resched();
708                 if (found)
709                         break;
710         }
711         mutex_unlock(&shmem_swaplist_mutex);
712 
713         if (found < 0)
714                 error = found;
715 out:
716         unlock_page(page);
717         page_cache_release(page);
718         return error;
719 }
720 
721 /*
722  * Move the page from the page cache to the swap cache.
723  */
724 static int shmem_writepage(struct page *page, struct writeback_control *wbc)
725 {
726         struct shmem_inode_info *info;
727         struct address_space *mapping;
728         struct inode *inode;
729         swp_entry_t swap;
730         pgoff_t index;
731 
732         BUG_ON(!PageLocked(page));
733         mapping = page->mapping;
734         index = page->index;
735         inode = mapping->host;
736         info = SHMEM_I(inode);
737         if (info->flags & VM_LOCKED)
738                 goto redirty;
739         if (!total_swap_pages)
740                 goto redirty;
741 
742         /*
743          * shmem_backing_dev_info's capabilities prevent regular writeback or
744          * sync from ever calling shmem_writepage; but a stacking filesystem
745          * might use ->writepage of its underlying filesystem, in which case
746          * tmpfs should write out to swap only in response to memory pressure,
747          * and not for the writeback threads or sync.
748          */
749         if (!wbc->for_reclaim) {
750                 WARN_ON_ONCE(1);        /* Still happens? Tell us about it! */
751                 goto redirty;
752         }
753 
754         /*
755          * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC
756          * value into swapfile.c, the only way we can correctly account for a
757          * fallocated page arriving here is now to initialize it and write it.
758          *
759          * That's okay for a page already fallocated earlier, but if we have
760          * not yet completed the fallocation, then (a) we want to keep track
761          * of this page in case we have to undo it, and (b) it may not be a
762          * good idea to continue anyway, once we're pushing into swap.  So
763          * reactivate the page, and let shmem_fallocate() quit when too many.
764          */
765         if (!PageUptodate(page)) {
766                 if (inode->i_private) {
767                         struct shmem_falloc *shmem_falloc;
768                         spin_lock(&inode->i_lock);
769                         shmem_falloc = inode->i_private;
770                         if (shmem_falloc &&
771                             !shmem_falloc->waitq &&
772                             index >= shmem_falloc->start &&
773                             index < shmem_falloc->next)
774                                 shmem_falloc->nr_unswapped++;
775                         else
776                                 shmem_falloc = NULL;
777                         spin_unlock(&inode->i_lock);
778                         if (shmem_falloc)
779                                 goto redirty;
780                 }
781                 clear_highpage(page);
782                 flush_dcache_page(page);
783                 SetPageUptodate(page);
784         }
785 
786         swap = get_swap_page();
787         if (!swap.val)
788                 goto redirty;
789 
790         /*
791          * Add inode to shmem_unuse()'s list of swapped-out inodes,
792          * if it's not already there.  Do it now before the page is
793          * moved to swap cache, when its pagelock no longer protects
794          * the inode from eviction.  But don't unlock the mutex until
795          * we've incremented swapped, because shmem_unuse_inode() will
796          * prune a !swapped inode from the swaplist under this mutex.
797          */
798         mutex_lock(&shmem_swaplist_mutex);
799         if (list_empty(&info->swaplist))
800                 list_add_tail(&info->swaplist, &shmem_swaplist);
801 
802         if (add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) {
803                 swap_shmem_alloc(swap);
804                 shmem_delete_from_page_cache(page, swp_to_radix_entry(swap));
805 
806                 spin_lock(&info->lock);
807                 info->swapped++;
808                 shmem_recalc_inode(inode);
809                 spin_unlock(&info->lock);
810 
811                 mutex_unlock(&shmem_swaplist_mutex);
812                 BUG_ON(page_mapped(page));
813                 swap_writepage(page, wbc);
814                 return 0;
815         }
816 
817         mutex_unlock(&shmem_swaplist_mutex);
818         swapcache_free(swap, NULL);
819 redirty:
820         set_page_dirty(page);
821         if (wbc->for_reclaim)
822                 return AOP_WRITEPAGE_ACTIVATE;  /* Return with page locked */
823         unlock_page(page);
824         return 0;
825 }
826 
827 #ifdef CONFIG_NUMA
828 #ifdef CONFIG_TMPFS
829 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
830 {
831         char buffer[64];
832 
833         if (!mpol || mpol->mode == MPOL_DEFAULT)
834                 return;         /* show nothing */
835 
836         mpol_to_str(buffer, sizeof(buffer), mpol);
837 
838         seq_printf(seq, ",mpol=%s", buffer);
839 }
840 
841 static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
842 {
843         struct mempolicy *mpol = NULL;
844         if (sbinfo->mpol) {
845                 spin_lock(&sbinfo->stat_lock);  /* prevent replace/use races */
846                 mpol = sbinfo->mpol;
847                 mpol_get(mpol);
848                 spin_unlock(&sbinfo->stat_lock);
849         }
850         return mpol;
851 }
852 #endif /* CONFIG_TMPFS */
853 
854 static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
855                         struct shmem_inode_info *info, pgoff_t index)
856 {
857         struct vm_area_struct pvma;
858         struct page *page;
859 
860         /* Create a pseudo vma that just contains the policy */
861         pvma.vm_start = 0;
862         /* Bias interleave by inode number to distribute better across nodes */
863         pvma.vm_pgoff = index + info->vfs_inode.i_ino;
864         pvma.vm_ops = NULL;
865         pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, index);
866 
867         page = swapin_readahead(swap, gfp, &pvma, 0);
868 
869         /* Drop reference taken by mpol_shared_policy_lookup() */
870         mpol_cond_put(pvma.vm_policy);
871 
872         return page;
873 }
874 
875 static struct page *shmem_alloc_page(gfp_t gfp,
876                         struct shmem_inode_info *info, pgoff_t index)
877 {
878         struct vm_area_struct pvma;
879         struct page *page;
880 
881         /* Create a pseudo vma that just contains the policy */
882         pvma.vm_start = 0;
883         /* Bias interleave by inode number to distribute better across nodes */
884         pvma.vm_pgoff = index + info->vfs_inode.i_ino;
885         pvma.vm_ops = NULL;
886         pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, index);
887 
888         page = alloc_page_vma(gfp, &pvma, 0);
889 
890         /* Drop reference taken by mpol_shared_policy_lookup() */
891         mpol_cond_put(pvma.vm_policy);
892 
893         return page;
894 }
895 #else /* !CONFIG_NUMA */
896 #ifdef CONFIG_TMPFS
897 static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
898 {
899 }
900 #endif /* CONFIG_TMPFS */
901 
902 static inline struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
903                         struct shmem_inode_info *info, pgoff_t index)
904 {
905         return swapin_readahead(swap, gfp, NULL, 0);
906 }
907 
908 static inline struct page *shmem_alloc_page(gfp_t gfp,
909                         struct shmem_inode_info *info, pgoff_t index)
910 {
911         return alloc_page(gfp);
912 }
913 #endif /* CONFIG_NUMA */
914 
915 #if !defined(CONFIG_NUMA) || !defined(CONFIG_TMPFS)
916 static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
917 {
918         return NULL;
919 }
920 #endif
921 
922 /*
923  * When a page is moved from swapcache to shmem filecache (either by the
924  * usual swapin of shmem_getpage_gfp(), or by the less common swapoff of
925  * shmem_unuse_inode()), it may have been read in earlier from swap, in
926  * ignorance of the mapping it belongs to.  If that mapping has special
927  * constraints (like the gma500 GEM driver, which requires RAM below 4GB),
928  * we may need to copy to a suitable page before moving to filecache.
929  *
930  * In a future release, this may well be extended to respect cpuset and
931  * NUMA mempolicy, and applied also to anonymous pages in do_swap_page();
932  * but for now it is a simple matter of zone.
933  */
934 static bool shmem_should_replace_page(struct page *page, gfp_t gfp)
935 {
936         return page_zonenum(page) > gfp_zone(gfp);
937 }
938 
939 static int shmem_replace_page(struct page **pagep, gfp_t gfp,
940                                 struct shmem_inode_info *info, pgoff_t index)
941 {
942         struct page *oldpage, *newpage;
943         struct address_space *swap_mapping;
944         pgoff_t swap_index;
945         int error;
946 
947         oldpage = *pagep;
948         swap_index = page_private(oldpage);
949         swap_mapping = page_mapping(oldpage);
950 
951         /*
952          * We have arrived here because our zones are constrained, so don't
953          * limit chance of success by further cpuset and node constraints.
954          */
955         gfp &= ~GFP_CONSTRAINT_MASK;
956         newpage = shmem_alloc_page(gfp, info, index);
957         if (!newpage)
958                 return -ENOMEM;
959 
960         page_cache_get(newpage);
961         copy_highpage(newpage, oldpage);
962         flush_dcache_page(newpage);
963 
964         __set_page_locked(newpage);
965         SetPageUptodate(newpage);
966         SetPageSwapBacked(newpage);
967         set_page_private(newpage, swap_index);
968         SetPageSwapCache(newpage);
969 
970         /*
971          * Our caller will very soon move newpage out of swapcache, but it's
972          * a nice clean interface for us to replace oldpage by newpage there.
973          */
974         spin_lock_irq(&swap_mapping->tree_lock);
975         error = shmem_radix_tree_replace(swap_mapping, swap_index, oldpage,
976                                                                    newpage);
977         if (!error) {
978                 __inc_zone_page_state(newpage, NR_FILE_PAGES);
979                 __dec_zone_page_state(oldpage, NR_FILE_PAGES);
980         }
981         spin_unlock_irq(&swap_mapping->tree_lock);
982 
983         if (unlikely(error)) {
984                 /*
985                  * Is this possible?  I think not, now that our callers check
986                  * both PageSwapCache and page_private after getting page lock;
987                  * but be defensive.  Reverse old to newpage for clear and free.
988                  */
989                 oldpage = newpage;
990         } else {
991                 mem_cgroup_replace_page_cache(oldpage, newpage);
992                 lru_cache_add_anon(newpage);
993                 *pagep = newpage;
994         }
995 
996         ClearPageSwapCache(oldpage);
997         set_page_private(oldpage, 0);
998 
999         unlock_page(oldpage);
1000         page_cache_release(oldpage);
1001         page_cache_release(oldpage);
1002         return error;
1003 }
1004 
1005 /*
1006  * shmem_getpage_gfp - find page in cache, or get from swap, or allocate
1007  *
1008  * If we allocate a new one we do not mark it dirty. That's up to the
1009  * vm. If we swap it in we mark it dirty since we also free the swap
1010  * entry since a page cannot live in both the swap and page cache
1011  */
1012 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
1013         struct page **pagep, enum sgp_type sgp, gfp_t gfp, int *fault_type)
1014 {
1015         struct address_space *mapping = inode->i_mapping;
1016         struct shmem_inode_info *info;
1017         struct shmem_sb_info *sbinfo;
1018         struct page *page;
1019         swp_entry_t swap;
1020         int error;
1021         int once = 0;
1022         int alloced = 0;
1023 
1024         if (index > (MAX_LFS_FILESIZE >> PAGE_CACHE_SHIFT))
1025                 return -EFBIG;
1026 repeat:
1027         swap.val = 0;
1028         page = find_lock_entry(mapping, index);
1029         if (radix_tree_exceptional_entry(page)) {
1030                 swap = radix_to_swp_entry(page);
1031                 page = NULL;
1032         }
1033 
1034         if (sgp != SGP_WRITE && sgp != SGP_FALLOC &&
1035             ((loff_t)index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
1036                 error = -EINVAL;
1037                 goto failed;
1038         }
1039 
1040         if (page && sgp == SGP_WRITE)
1041                 mark_page_accessed(page);
1042 
1043         /* fallocated page? */
1044         if (page && !PageUptodate(page)) {
1045                 if (sgp != SGP_READ)
1046                         goto clear;
1047                 unlock_page(page);
1048                 page_cache_release(page);
1049                 page = NULL;
1050         }
1051         if (page || (sgp == SGP_READ && !swap.val)) {
1052                 *pagep = page;
1053                 return 0;
1054         }
1055 
1056         /*
1057          * Fast cache lookup did not find it:
1058          * bring it back from swap or allocate.
1059          */
1060         info = SHMEM_I(inode);
1061         sbinfo = SHMEM_SB(inode->i_sb);
1062 
1063         if (swap.val) {
1064                 /* Look it up and read it in.. */
1065                 page = lookup_swap_cache(swap);
1066                 if (!page) {
1067                         /* here we actually do the io */
1068                         if (fault_type)
1069                                 *fault_type |= VM_FAULT_MAJOR;
1070                         page = shmem_swapin(swap, gfp, info, index);
1071                         if (!page) {
1072                                 error = -ENOMEM;
1073                                 goto failed;
1074                         }
1075                 }
1076 
1077                 /* We have to do this with page locked to prevent races */
1078                 lock_page(page);
1079                 if (!PageSwapCache(page) || page_private(page) != swap.val ||
1080                     !shmem_confirm_swap(mapping, index, swap)) {
1081                         error = -EEXIST;        /* try again */
1082                         goto unlock;
1083                 }
1084                 if (!PageUptodate(page)) {
1085                         error = -EIO;
1086                         goto failed;
1087                 }
1088                 wait_on_page_writeback(page);
1089 
1090                 if (shmem_should_replace_page(page, gfp)) {
1091                         error = shmem_replace_page(&page, gfp, info, index);
1092                         if (error)
1093                                 goto failed;
1094                 }
1095 
1096                 error = mem_cgroup_cache_charge(page, current->mm,
1097                                                 gfp & GFP_RECLAIM_MASK);
1098                 if (!error) {
1099                         error = shmem_add_to_page_cache(page, mapping, index,
1100                                                 gfp, swp_to_radix_entry(swap));
1101                         /*
1102                          * We already confirmed swap under page lock, and make
1103                          * no memory allocation here, so usually no possibility
1104                          * of error; but free_swap_and_cache() only trylocks a
1105                          * page, so it is just possible that the entry has been
1106                          * truncated or holepunched since swap was confirmed.
1107                          * shmem_undo_range() will have done some of the
1108                          * unaccounting, now delete_from_swap_cache() will do
1109                          * the rest (including mem_cgroup_uncharge_swapcache).
1110                          * Reset swap.val? No, leave it so "failed" goes back to
1111                          * "repeat": reading a hole and writing should succeed.
1112                          */
1113                         if (error)
1114                                 delete_from_swap_cache(page);
1115                 }
1116                 if (error)
1117                         goto failed;
1118 
1119                 spin_lock(&info->lock);
1120                 info->swapped--;
1121                 shmem_recalc_inode(inode);
1122                 spin_unlock(&info->lock);
1123 
1124                 if (sgp == SGP_WRITE)
1125                         mark_page_accessed(page);
1126 
1127                 delete_from_swap_cache(page);
1128                 set_page_dirty(page);
1129                 swap_free(swap);
1130 
1131         } else {
1132                 if (shmem_acct_block(info->flags)) {
1133                         error = -ENOSPC;
1134                         goto failed;
1135                 }
1136                 if (sbinfo->max_blocks) {
1137                         if (percpu_counter_compare(&sbinfo->used_blocks,
1138                                                 sbinfo->max_blocks) >= 0) {
1139                                 error = -ENOSPC;
1140                                 goto unacct;
1141                         }
1142                         percpu_counter_inc(&sbinfo->used_blocks);
1143                 }
1144 
1145                 page = shmem_alloc_page(gfp, info, index);
1146                 if (!page) {
1147                         error = -ENOMEM;
1148                         goto decused;
1149                 }
1150 
1151                 __SetPageSwapBacked(page);
1152                 __set_page_locked(page);
1153                 if (sgp == SGP_WRITE)
1154                         init_page_accessed(page);
1155 
1156                 error = mem_cgroup_cache_charge(page, current->mm,
1157                                                 gfp & GFP_RECLAIM_MASK);
1158                 if (error)
1159                         goto decused;
1160                 error = radix_tree_maybe_preload(gfp & GFP_RECLAIM_MASK);
1161                 if (!error) {
1162                         error = shmem_add_to_page_cache(page, mapping, index,
1163                                                         gfp, NULL);
1164                         radix_tree_preload_end();
1165                 }
1166                 if (error) {
1167                         mem_cgroup_uncharge_cache_page(page);
1168                         goto decused;
1169                 }
1170                 lru_cache_add_anon(page);
1171 
1172                 spin_lock(&info->lock);
1173                 info->alloced++;
1174                 inode->i_blocks += BLOCKS_PER_PAGE;
1175                 shmem_recalc_inode(inode);
1176                 spin_unlock(&info->lock);
1177                 alloced = true;
1178 
1179                 /*
1180                  * Let SGP_FALLOC use the SGP_WRITE optimization on a new page.
1181                  */
1182                 if (sgp == SGP_FALLOC)
1183                         sgp = SGP_WRITE;
1184 clear:
1185                 /*
1186                  * Let SGP_WRITE caller clear ends if write does not fill page;
1187                  * but SGP_FALLOC on a page fallocated earlier must initialize
1188                  * it now, lest undo on failure cancel our earlier guarantee.
1189                  */
1190                 if (sgp != SGP_WRITE) {
1191                         clear_highpage(page);
1192                         flush_dcache_page(page);
1193                         SetPageUptodate(page);
1194                 }
1195                 if (sgp == SGP_DIRTY)
1196                         set_page_dirty(page);
1197         }
1198 
1199         /* Perhaps the file has been truncated since we checked */
1200         if (sgp != SGP_WRITE && sgp != SGP_FALLOC &&
1201             ((loff_t)index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
1202                 error = -EINVAL;
1203                 if (alloced)
1204                         goto trunc;
1205                 else
1206                         goto failed;
1207         }
1208         *pagep = page;
1209         return 0;
1210 
1211         /*
1212          * Error recovery.
1213          */
1214 trunc:
1215         info = SHMEM_I(inode);
1216         ClearPageDirty(page);
1217         delete_from_page_cache(page);
1218         spin_lock(&info->lock);
1219         info->alloced--;
1220         inode->i_blocks -= BLOCKS_PER_PAGE;
1221         spin_unlock(&info->lock);
1222 decused:
1223         sbinfo = SHMEM_SB(inode->i_sb);
1224         if (sbinfo->max_blocks)
1225                 percpu_counter_add(&sbinfo->used_blocks, -1);
1226 unacct:
1227         shmem_unacct_blocks(info->flags, 1);
1228 failed:
1229         if (swap.val && error != -EINVAL &&
1230             !shmem_confirm_swap(mapping, index, swap))
1231                 error = -EEXIST;
1232 unlock:
1233         if (page) {
1234                 unlock_page(page);
1235                 page_cache_release(page);
1236         }
1237         if (error == -ENOSPC && !once++) {
1238                 info = SHMEM_I(inode);
1239                 spin_lock(&info->lock);
1240                 shmem_recalc_inode(inode);
1241                 spin_unlock(&info->lock);
1242                 goto repeat;
1243         }
1244         if (error == -EEXIST)   /* from above or from radix_tree_insert */
1245                 goto repeat;
1246         return error;
1247 }
1248 
1249 static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1250 {
1251         struct inode *inode = file_inode(vma->vm_file);
1252         int error;
1253         int ret = VM_FAULT_LOCKED;
1254 
1255         /*
1256          * Trinity finds that probing a hole which tmpfs is punching can
1257          * prevent the hole-punch from ever completing: which in turn
1258          * locks writers out with its hold on i_mutex.  So refrain from
1259          * faulting pages into the hole while it's being punched.  Although
1260          * shmem_undo_range() does remove the additions, it may be unable to
1261          * keep up, as each new page needs its own unmap_mapping_range() call,
1262          * and the i_mmap tree grows ever slower to scan if new vmas are added.
1263          *
1264          * It does not matter if we sometimes reach this check just before the
1265          * hole-punch begins, so that one fault then races with the punch:
1266          * we just need to make racing faults a rare case.
1267          *
1268          * The implementation below would be much simpler if we just used a
1269          * standard mutex or completion: but we cannot take i_mutex in fault,
1270          * and bloating every shmem inode for this unlikely case would be sad.
1271          */
1272         if (unlikely(inode->i_private)) {
1273                 struct shmem_falloc *shmem_falloc;
1274 
1275                 spin_lock(&inode->i_lock);
1276                 shmem_falloc = inode->i_private;
1277                 if (shmem_falloc &&
1278                     shmem_falloc->waitq &&
1279                     vmf->pgoff >= shmem_falloc->start &&
1280                     vmf->pgoff < shmem_falloc->next) {
1281                         wait_queue_head_t *shmem_falloc_waitq;
1282                         DEFINE_WAIT(shmem_fault_wait);
1283 
1284                         ret = VM_FAULT_NOPAGE;
1285                         if ((vmf->flags & FAULT_FLAG_ALLOW_RETRY) &&
1286                            !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) {
1287                                 /* It's polite to up mmap_sem if we can */
1288                                 up_read(&vma->vm_mm->mmap_sem);
1289                                 ret = VM_FAULT_RETRY;
1290                         }
1291 
1292                         shmem_falloc_waitq = shmem_falloc->waitq;
1293                         prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait,
1294                                         TASK_UNINTERRUPTIBLE);
1295                         spin_unlock(&inode->i_lock);
1296                         schedule();
1297 
1298                         /*
1299                          * shmem_falloc_waitq points into the shmem_fallocate()
1300                          * stack of the hole-punching task: shmem_falloc_waitq
1301                          * is usually invalid by the time we reach here, but
1302                          * finish_wait() does not dereference it in that case;
1303                          * though i_lock needed lest racing with wake_up_all().
1304                          */
1305                         spin_lock(&inode->i_lock);
1306                         finish_wait(shmem_falloc_waitq, &shmem_fault_wait);
1307                         spin_unlock(&inode->i_lock);
1308                         return ret;
1309                 }
1310                 spin_unlock(&inode->i_lock);
1311         }
1312 
1313         error = shmem_getpage(inode, vmf->pgoff, &vmf->page, SGP_CACHE, &ret);
1314         if (error)
1315                 return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS);
1316 
1317         if (ret & VM_FAULT_MAJOR) {
1318                 count_vm_event(PGMAJFAULT);
1319                 mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT);
1320         }
1321         return ret;
1322 }
1323 
1324 #ifdef CONFIG_NUMA
1325 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol)
1326 {
1327         struct inode *inode = file_inode(vma->vm_file);
1328         return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol);
1329 }
1330 
1331 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
1332                                           unsigned long addr)
1333 {
1334         struct inode *inode = file_inode(vma->vm_file);
1335         pgoff_t index;
1336 
1337         index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
1338         return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index);
1339 }
1340 #endif
1341 
1342 int shmem_lock(struct file *file, int lock, struct user_struct *user)
1343 {
1344         struct inode *inode = file_inode(file);
1345         struct shmem_inode_info *info = SHMEM_I(inode);
1346         int retval = -ENOMEM;
1347 
1348         spin_lock(&info->lock);
1349         if (lock && !(info->flags & VM_LOCKED)) {
1350                 if (!user_shm_lock(inode->i_size, user))
1351                         goto out_nomem;
1352                 info->flags |= VM_LOCKED;
1353                 mapping_set_unevictable(file->f_mapping);
1354         }
1355         if (!lock && (info->flags & VM_LOCKED) && user) {
1356                 user_shm_unlock(inode->i_size, user);
1357                 info->flags &= ~VM_LOCKED;
1358                 mapping_clear_unevictable(file->f_mapping);
1359         }
1360         retval = 0;
1361 
1362 out_nomem:
1363         spin_unlock(&info->lock);
1364         return retval;
1365 }
1366 
1367 static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
1368 {
1369         file_accessed(file);
1370         vma->vm_ops = &shmem_vm_ops;
1371         return 0;
1372 }
1373 
1374 static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir,
1375                                      umode_t mode, dev_t dev, unsigned long flags)
1376 {
1377         struct inode *inode;
1378         struct shmem_inode_info *info;
1379         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
1380 
1381         if (shmem_reserve_inode(sb))
1382                 return NULL;
1383 
1384         inode = new_inode(sb);
1385         if (inode) {
1386                 inode->i_ino = get_next_ino();
1387                 inode_init_owner(inode, dir, mode);
1388                 inode->i_blocks = 0;
1389                 inode->i_mapping->backing_dev_info = &shmem_backing_dev_info;
1390                 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
1391                 inode->i_generation = get_seconds();
1392                 info = SHMEM_I(inode);
1393                 memset(info, 0, (char *)inode - (char *)info);
1394                 spin_lock_init(&info->lock);
1395                 info->flags = flags & VM_NORESERVE;
1396                 INIT_LIST_HEAD(&info->swaplist);
1397                 simple_xattrs_init(&info->xattrs);
1398                 cache_no_acl(inode);
1399 
1400                 switch (mode & S_IFMT) {
1401                 default:
1402                         inode->i_op = &shmem_special_inode_operations;
1403                         init_special_inode(inode, mode, dev);
1404                         break;
1405                 case S_IFREG:
1406                         inode->i_mapping->a_ops = &shmem_aops;
1407                         inode->i_op = &shmem_inode_operations;
1408                         inode->i_fop = &shmem_file_operations;
1409                         mpol_shared_policy_init(&info->policy,
1410                                                  shmem_get_sbmpol(sbinfo));
1411                         break;
1412                 case S_IFDIR:
1413                         inc_nlink(inode);
1414                         /* Some things misbehave if size == 0 on a directory */
1415                         inode->i_size = 2 * BOGO_DIRENT_SIZE;
1416                         inode->i_op = &shmem_dir_inode_operations;
1417                         inode->i_fop = &simple_dir_operations;
1418                         break;
1419                 case S_IFLNK:
1420                         /*
1421                          * Must not load anything in the rbtree,
1422                          * mpol_free_shared_policy will not be called.
1423                          */
1424                         mpol_shared_policy_init(&info->policy, NULL);
1425                         break;
1426                 }
1427         } else
1428                 shmem_free_inode(sb);
1429         return inode;
1430 }
1431 
1432 bool shmem_mapping(struct address_space *mapping)
1433 {
1434         return mapping->backing_dev_info == &shmem_backing_dev_info;
1435 }
1436 
1437 #ifdef CONFIG_TMPFS
1438 static const struct inode_operations shmem_symlink_inode_operations;
1439 static const struct inode_operations shmem_short_symlink_operations;
1440 
1441 #ifdef CONFIG_TMPFS_XATTR
1442 static int shmem_initxattrs(struct inode *, const struct xattr *, void *);
1443 #else
1444 #define shmem_initxattrs NULL
1445 #endif
1446 
1447 static int
1448 shmem_write_begin(struct file *file, struct address_space *mapping,
1449                         loff_t pos, unsigned len, unsigned flags,
1450                         struct page **pagep, void **fsdata)
1451 {
1452         struct inode *inode = mapping->host;
1453         pgoff_t index = pos >> PAGE_CACHE_SHIFT;
1454         return shmem_getpage(inode, index, pagep, SGP_WRITE, NULL);
1455 }
1456 
1457 static int
1458 shmem_write_end(struct file *file, struct address_space *mapping,
1459                         loff_t pos, unsigned len, unsigned copied,
1460                         struct page *page, void *fsdata)
1461 {
1462         struct inode *inode = mapping->host;
1463 
1464         if (pos + copied > inode->i_size)
1465                 i_size_write(inode, pos + copied);
1466 
1467         if (!PageUptodate(page)) {
1468                 if (copied < PAGE_CACHE_SIZE) {
1469                         unsigned from = pos & (PAGE_CACHE_SIZE - 1);
1470                         zero_user_segments(page, 0, from,
1471                                         from + copied, PAGE_CACHE_SIZE);
1472                 }
1473                 SetPageUptodate(page);
1474         }
1475         set_page_dirty(page);
1476         unlock_page(page);
1477         page_cache_release(page);
1478 
1479         return copied;
1480 }
1481 
1482 static void do_shmem_file_read(struct file *filp, loff_t *ppos, read_descriptor_t *desc, read_actor_t actor)
1483 {
1484         struct inode *inode = file_inode(filp);
1485         struct address_space *mapping = inode->i_mapping;
1486         pgoff_t index;
1487         unsigned long offset;
1488         enum sgp_type sgp = SGP_READ;
1489 
1490         /*
1491          * Might this read be for a stacking filesystem?  Then when reading
1492          * holes of a sparse file, we actually need to allocate those pages,
1493          * and even mark them dirty, so it cannot exceed the max_blocks limit.
1494          */
1495         if (segment_eq(get_fs(), KERNEL_DS))
1496                 sgp = SGP_DIRTY;
1497 
1498         index = *ppos >> PAGE_CACHE_SHIFT;
1499         offset = *ppos & ~PAGE_CACHE_MASK;
1500 
1501         for (;;) {
1502                 struct page *page = NULL;
1503                 pgoff_t end_index;
1504                 unsigned long nr, ret;
1505                 loff_t i_size = i_size_read(inode);
1506 
1507                 end_index = i_size >> PAGE_CACHE_SHIFT;
1508                 if (index > end_index)
1509                         break;
1510                 if (index == end_index) {
1511                         nr = i_size & ~PAGE_CACHE_MASK;
1512                         if (nr <= offset)
1513                                 break;
1514                 }
1515 
1516                 desc->error = shmem_getpage(inode, index, &page, sgp, NULL);
1517                 if (desc->error) {
1518                         if (desc->error == -EINVAL)
1519                                 desc->error = 0;
1520                         break;
1521                 }
1522                 if (page)
1523                         unlock_page(page);
1524 
1525                 /*
1526                  * We must evaluate after, since reads (unlike writes)
1527                  * are called without i_mutex protection against truncate
1528                  */
1529                 nr = PAGE_CACHE_SIZE;
1530                 i_size = i_size_read(inode);
1531                 end_index = i_size >> PAGE_CACHE_SHIFT;
1532                 if (index == end_index) {
1533                         nr = i_size & ~PAGE_CACHE_MASK;
1534                         if (nr <= offset) {
1535                                 if (page)
1536                                         page_cache_release(page);
1537                                 break;
1538                         }
1539                 }
1540                 nr -= offset;
1541 
1542                 if (page) {
1543                         /*
1544                          * If users can be writing to this page using arbitrary
1545                          * virtual addresses, take care about potential aliasing
1546                          * before reading the page on the kernel side.
1547                          */
1548                         if (mapping_writably_mapped(mapping))
1549                                 flush_dcache_page(page);
1550                         /*
1551                          * Mark the page accessed if we read the beginning.
1552                          */
1553                         if (!offset)
1554                                 mark_page_accessed(page);
1555                 } else {
1556                         page = ZERO_PAGE(0);
1557                         page_cache_get(page);
1558                 }
1559 
1560                 /*
1561                  * Ok, we have the page, and it's up-to-date, so
1562                  * now we can copy it to user space...
1563                  *
1564                  * The actor routine returns how many bytes were actually used..
1565                  * NOTE! This may not be the same as how much of a user buffer
1566                  * we filled up (we may be padding etc), so we can only update
1567                  * "pos" here (the actor routine has to update the user buffer
1568                  * pointers and the remaining count).
1569                  */
1570                 ret = actor(desc, page, offset, nr);
1571                 offset += ret;
1572                 index += offset >> PAGE_CACHE_SHIFT;
1573                 offset &= ~PAGE_CACHE_MASK;
1574 
1575                 page_cache_release(page);
1576                 if (ret != nr || !desc->count)
1577                         break;
1578 
1579                 cond_resched();
1580         }
1581 
1582         *ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset;
1583         file_accessed(filp);
1584 }
1585 
1586 static ssize_t shmem_file_aio_read(struct kiocb *iocb,
1587                 const struct iovec *iov, unsigned long nr_segs, loff_t pos)
1588 {
1589         struct file *filp = iocb->ki_filp;
1590         ssize_t retval;
1591         unsigned long seg;
1592         size_t count;
1593         loff_t *ppos = &iocb->ki_pos;
1594 
1595         retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE);
1596         if (retval)
1597                 return retval;
1598 
1599         for (seg = 0; seg < nr_segs; seg++) {
1600                 read_descriptor_t desc;
1601 
1602                 desc.written = 0;
1603                 desc.arg.buf = iov[seg].iov_base;
1604                 desc.count = iov[seg].iov_len;
1605                 if (desc.count == 0)
1606                         continue;
1607                 desc.error = 0;
1608                 do_shmem_file_read(filp, ppos, &desc, file_read_actor);
1609                 retval += desc.written;
1610                 if (desc.error) {
1611                         retval = retval ?: desc.error;
1612                         break;
1613                 }
1614                 if (desc.count > 0)
1615                         break;
1616         }
1617         return retval;
1618 }
1619 
1620 static ssize_t shmem_file_splice_read(struct file *in, loff_t *ppos,
1621                                 struct pipe_inode_info *pipe, size_t len,
1622                                 unsigned int flags)
1623 {
1624         struct address_space *mapping = in->f_mapping;
1625         struct inode *inode = mapping->host;
1626         unsigned int loff, nr_pages, req_pages;
1627         struct page *pages[PIPE_DEF_BUFFERS];
1628         struct partial_page partial[PIPE_DEF_BUFFERS];
1629         struct page *page;
1630         pgoff_t index, end_index;
1631         loff_t isize, left;
1632         int error, page_nr;
1633         struct splice_pipe_desc spd = {
1634                 .pages = pages,
1635                 .partial = partial,
1636                 .nr_pages_max = PIPE_DEF_BUFFERS,
1637                 .flags = flags,
1638                 .ops = &page_cache_pipe_buf_ops,
1639                 .spd_release = spd_release_page,
1640         };
1641 
1642         isize = i_size_read(inode);
1643         if (unlikely(*ppos >= isize))
1644                 return 0;
1645 
1646         left = isize - *ppos;
1647         if (unlikely(left < len))
1648                 len = left;
1649 
1650         if (splice_grow_spd(pipe, &spd))
1651                 return -ENOMEM;
1652 
1653         index = *ppos >> PAGE_CACHE_SHIFT;
1654         loff = *ppos & ~PAGE_CACHE_MASK;
1655         req_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
1656         nr_pages = min(req_pages, pipe->buffers);
1657 
1658         spd.nr_pages = find_get_pages_contig(mapping, index,
1659                                                 nr_pages, spd.pages);
1660         index += spd.nr_pages;
1661         error = 0;
1662 
1663         while (spd.nr_pages < nr_pages) {
1664                 error = shmem_getpage(inode, index, &page, SGP_CACHE, NULL);
1665                 if (error)
1666                         break;
1667                 unlock_page(page);
1668                 spd.pages[spd.nr_pages++] = page;
1669                 index++;
1670         }
1671 
1672         index = *ppos >> PAGE_CACHE_SHIFT;
1673         nr_pages = spd.nr_pages;
1674         spd.nr_pages = 0;
1675 
1676         for (page_nr = 0; page_nr < nr_pages; page_nr++) {
1677                 unsigned int this_len;
1678 
1679                 if (!len)
1680                         break;
1681 
1682                 this_len = min_t(unsigned long, len, PAGE_CACHE_SIZE - loff);
1683                 page = spd.pages[page_nr];
1684 
1685                 if (!PageUptodate(page) || page->mapping != mapping) {
1686                         error = shmem_getpage(inode, index, &page,
1687                                                         SGP_CACHE, NULL);
1688                         if (error)
1689                                 break;
1690                         unlock_page(page);
1691                         page_cache_release(spd.pages[page_nr]);
1692                         spd.pages[page_nr] = page;
1693                 }
1694 
1695                 isize = i_size_read(inode);
1696                 end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
1697                 if (unlikely(!isize || index > end_index))
1698                         break;
1699 
1700                 if (end_index == index) {
1701                         unsigned int plen;
1702 
1703                         plen = ((isize - 1) & ~PAGE_CACHE_MASK) + 1;
1704                         if (plen <= loff)
1705                                 break;
1706 
1707                         this_len = min(this_len, plen - loff);
1708                         len = this_len;
1709                 }
1710 
1711                 spd.partial[page_nr].offset = loff;
1712                 spd.partial[page_nr].len = this_len;
1713                 len -= this_len;
1714                 loff = 0;
1715                 spd.nr_pages++;
1716                 index++;
1717         }
1718 
1719         while (page_nr < nr_pages)
1720                 page_cache_release(spd.pages[page_nr++]);
1721 
1722         if (spd.nr_pages)
1723                 error = splice_to_pipe(pipe, &spd);
1724 
1725         splice_shrink_spd(&spd);
1726 
1727         if (error > 0) {
1728                 *ppos += error;
1729                 file_accessed(in);
1730         }
1731         return error;
1732 }
1733 
1734 /*
1735  * llseek SEEK_DATA or SEEK_HOLE through the radix_tree.
1736  */
1737 static pgoff_t shmem_seek_hole_data(struct address_space *mapping,
1738                                     pgoff_t index, pgoff_t end, int whence)
1739 {
1740         struct page *page;
1741         struct pagevec pvec;
1742         pgoff_t indices[PAGEVEC_SIZE];
1743         bool done = false;
1744         int i;
1745 
1746         pagevec_init(&pvec, 0);
1747         pvec.nr = 1;            /* start small: we may be there already */
1748         while (!done) {
1749                 pvec.nr = find_get_entries(mapping, index,
1750                                         pvec.nr, pvec.pages, indices);
1751                 if (!pvec.nr) {
1752                         if (whence == SEEK_DATA)
1753                                 index = end;
1754                         break;
1755                 }
1756                 for (i = 0; i < pvec.nr; i++, index++) {
1757                         if (index < indices[i]) {
1758                                 if (whence == SEEK_HOLE) {
1759                                         done = true;
1760                                         break;
1761                                 }
1762                                 index = indices[i];
1763                         }
1764                         page = pvec.pages[i];
1765                         if (page && !radix_tree_exceptional_entry(page)) {
1766                                 if (!PageUptodate(page))
1767                                         page = NULL;
1768                         }
1769                         if (index >= end ||
1770                             (page && whence == SEEK_DATA) ||
1771                             (!page && whence == SEEK_HOLE)) {
1772                                 done = true;
1773                                 break;
1774                         }
1775                 }
1776                 pagevec_remove_exceptionals(&pvec);
1777                 pagevec_release(&pvec);
1778                 pvec.nr = PAGEVEC_SIZE;
1779                 cond_resched();
1780         }
1781         return index;
1782 }
1783 
1784 static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence)
1785 {
1786         struct address_space *mapping = file->f_mapping;
1787         struct inode *inode = mapping->host;
1788         pgoff_t start, end;
1789         loff_t new_offset;
1790 
1791         if (whence != SEEK_DATA && whence != SEEK_HOLE)
1792                 return generic_file_llseek_size(file, offset, whence,
1793                                         MAX_LFS_FILESIZE, i_size_read(inode));
1794         mutex_lock(&inode->i_mutex);
1795         /* We're holding i_mutex so we can access i_size directly */
1796 
1797         if (offset < 0)
1798                 offset = -EINVAL;
1799         else if (offset >= inode->i_size)
1800                 offset = -ENXIO;
1801         else {
1802                 start = offset >> PAGE_CACHE_SHIFT;
1803                 end = (inode->i_size + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
1804                 new_offset = shmem_seek_hole_data(mapping, start, end, whence);
1805                 new_offset <<= PAGE_CACHE_SHIFT;
1806                 if (new_offset > offset) {
1807                         if (new_offset < inode->i_size)
1808                                 offset = new_offset;
1809                         else if (whence == SEEK_DATA)
1810                                 offset = -ENXIO;
1811                         else
1812                                 offset = inode->i_size;
1813                 }
1814         }
1815 
1816         if (offset >= 0)
1817                 offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE);
1818         mutex_unlock(&inode->i_mutex);
1819         return offset;
1820 }
1821 
1822 static long shmem_fallocate(struct file *file, int mode, loff_t offset,
1823                                                          loff_t len)
1824 {
1825         struct inode *inode = file_inode(file);
1826         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1827         struct shmem_falloc shmem_falloc;
1828         pgoff_t start, index, end;
1829         int error;
1830 
1831         mutex_lock(&inode->i_mutex);
1832 
1833         if (mode & FALLOC_FL_PUNCH_HOLE) {
1834                 struct address_space *mapping = file->f_mapping;
1835                 loff_t unmap_start = round_up(offset, PAGE_SIZE);
1836                 loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1;
1837                 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq);
1838 
1839                 shmem_falloc.waitq = &shmem_falloc_waitq;
1840                 shmem_falloc.start = unmap_start >> PAGE_SHIFT;
1841                 shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT;
1842                 spin_lock(&inode->i_lock);
1843                 inode->i_private = &shmem_falloc;
1844                 spin_unlock(&inode->i_lock);
1845 
1846                 if ((u64)unmap_end > (u64)unmap_start)
1847                         unmap_mapping_range(mapping, unmap_start,
1848                                             1 + unmap_end - unmap_start, 0);
1849                 shmem_truncate_range(inode, offset, offset + len - 1);
1850                 /* No need to unmap again: hole-punching leaves COWed pages */
1851 
1852                 spin_lock(&inode->i_lock);
1853                 inode->i_private = NULL;
1854                 wake_up_all(&shmem_falloc_waitq);
1855                 spin_unlock(&inode->i_lock);
1856                 error = 0;
1857                 goto out;
1858         }
1859 
1860         /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
1861         error = inode_newsize_ok(inode, offset + len);
1862         if (error)
1863                 goto out;
1864 
1865         start = offset >> PAGE_CACHE_SHIFT;
1866         end = (offset + len + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
1867         /* Try to avoid a swapstorm if len is impossible to satisfy */
1868         if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) {
1869                 error = -ENOSPC;
1870                 goto out;
1871         }
1872 
1873         shmem_falloc.waitq = NULL;
1874         shmem_falloc.start = start;
1875         shmem_falloc.next  = start;
1876         shmem_falloc.nr_falloced = 0;
1877         shmem_falloc.nr_unswapped = 0;
1878         spin_lock(&inode->i_lock);
1879         inode->i_private = &shmem_falloc;
1880         spin_unlock(&inode->i_lock);
1881 
1882         for (index = start; index < end; index++) {
1883                 struct page *page;
1884 
1885                 /*
1886                  * Good, the fallocate(2) manpage permits EINTR: we may have
1887                  * been interrupted because we are using up too much memory.
1888                  */
1889                 if (signal_pending(current))
1890                         error = -EINTR;
1891                 else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced)
1892                         error = -ENOMEM;
1893                 else
1894                         error = shmem_getpage(inode, index, &page, SGP_FALLOC,
1895                                                                         NULL);
1896                 if (error) {
1897                         /* Remove the !PageUptodate pages we added */
1898                         if (index > start) {
1899                                 shmem_undo_range(inode,
1900                                  (loff_t)start << PAGE_CACHE_SHIFT,
1901                                  ((loff_t)index << PAGE_CACHE_SHIFT) - 1, true);
1902                         }
1903                         goto undone;
1904                 }
1905 
1906                 /*
1907                  * Inform shmem_writepage() how far we have reached.
1908                  * No need for lock or barrier: we have the page lock.
1909                  */
1910                 shmem_falloc.next++;
1911                 if (!PageUptodate(page))
1912                         shmem_falloc.nr_falloced++;
1913 
1914                 /*
1915                  * If !PageUptodate, leave it that way so that freeable pages
1916                  * can be recognized if we need to rollback on error later.
1917                  * But set_page_dirty so that memory pressure will swap rather
1918                  * than free the pages we are allocating (and SGP_CACHE pages
1919                  * might still be clean: we now need to mark those dirty too).
1920                  */
1921                 set_page_dirty(page);
1922                 unlock_page(page);
1923                 page_cache_release(page);
1924                 cond_resched();
1925         }
1926 
1927         if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
1928                 i_size_write(inode, offset + len);
1929         inode->i_ctime = CURRENT_TIME;
1930 undone:
1931         spin_lock(&inode->i_lock);
1932         inode->i_private = NULL;
1933         spin_unlock(&inode->i_lock);
1934 out:
1935         mutex_unlock(&inode->i_mutex);
1936         return error;
1937 }
1938 
1939 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
1940 {
1941         struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
1942 
1943         buf->f_type = TMPFS_MAGIC;
1944         buf->f_bsize = PAGE_CACHE_SIZE;
1945         buf->f_namelen = NAME_MAX;
1946         if (sbinfo->max_blocks) {
1947                 buf->f_blocks = sbinfo->max_blocks;
1948                 buf->f_bavail =
1949                 buf->f_bfree  = sbinfo->max_blocks -
1950                                 percpu_counter_sum(&sbinfo->used_blocks);
1951         }
1952         if (sbinfo->max_inodes) {
1953                 buf->f_files = sbinfo->max_inodes;
1954                 buf->f_ffree = sbinfo->free_inodes;
1955         }
1956         /* else leave those fields 0 like simple_statfs */
1957         return 0;
1958 }
1959 
1960 /*
1961  * File creation. Allocate an inode, and we're done..
1962  */
1963 static int
1964 shmem_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev)
1965 {
1966         struct inode *inode;
1967         int error = -ENOSPC;
1968 
1969         inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE);
1970         if (inode) {
1971 #ifdef CONFIG_TMPFS_POSIX_ACL
1972                 error = generic_acl_init(inode, dir);
1973                 if (error) {
1974                         iput(inode);
1975                         return error;
1976                 }
1977 #endif
1978                 error = security_inode_init_security(inode, dir,
1979                                                      &dentry->d_name,
1980                                                      shmem_initxattrs, NULL);
1981                 if (error) {
1982                         if (error != -EOPNOTSUPP) {
1983                                 iput(inode);
1984                                 return error;
1985                         }
1986                 }
1987 
1988                 error = 0;
1989                 dir->i_size += BOGO_DIRENT_SIZE;
1990                 dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1991                 d_instantiate(dentry, inode);
1992                 dget(dentry); /* Extra count - pin the dentry in core */
1993         }
1994         return error;
1995 }
1996 
1997 static int
1998 shmem_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode)
1999 {
2000         struct inode *inode;
2001         int error = -ENOSPC;
2002 
2003         inode = shmem_get_inode(dir->i_sb, dir, mode, 0, VM_NORESERVE);
2004         if (inode) {
2005                 error = security_inode_init_security(inode, dir,
2006                                                      NULL,
2007                                                      shmem_initxattrs, NULL);
2008                 if (error) {
2009                         if (error != -EOPNOTSUPP) {
2010                                 iput(inode);
2011                                 return error;
2012                         }
2013                 }
2014 #ifdef CONFIG_TMPFS_POSIX_ACL
2015                 error = generic_acl_init(inode, dir);
2016                 if (error) {
2017                         iput(inode);
2018                         return error;
2019                 }
2020 #else
2021                 error = 0;
2022 #endif
2023                 d_tmpfile(dentry, inode);
2024         }
2025         return error;
2026 }
2027 
2028 static int shmem_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
2029 {
2030         int error;
2031 
2032         if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0)))
2033                 return error;
2034         inc_nlink(dir);
2035         return 0;
2036 }
2037 
2038 static int shmem_create(struct inode *dir, struct dentry *dentry, umode_t mode,
2039                 bool excl)
2040 {
2041         return shmem_mknod(dir, dentry, mode | S_IFREG, 0);
2042 }
2043 
2044 /*
2045  * Link a file..
2046  */
2047 static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
2048 {
2049         struct inode *inode = old_dentry->d_inode;
2050         int ret;
2051 
2052         /*
2053          * No ordinary (disk based) filesystem counts links as inodes;
2054          * but each new link needs a new dentry, pinning lowmem, and
2055          * tmpfs dentries cannot be pruned until they are unlinked.
2056          */
2057         ret = shmem_reserve_inode(inode->i_sb);
2058         if (ret)
2059                 goto out;
2060 
2061         dir->i_size += BOGO_DIRENT_SIZE;
2062         inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
2063         inc_nlink(inode);
2064         ihold(inode);   /* New dentry reference */
2065         dget(dentry);           /* Extra pinning count for the created dentry */
2066         d_instantiate(dentry, inode);
2067 out:
2068         return ret;
2069 }
2070 
2071 static int shmem_unlink(struct inode *dir, struct dentry *dentry)
2072 {
2073         struct inode *inode = dentry->d_inode;
2074 
2075         if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
2076                 shmem_free_inode(inode->i_sb);
2077 
2078         dir->i_size -= BOGO_DIRENT_SIZE;
2079         inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
2080         drop_nlink(inode);
2081         dput(dentry);   /* Undo the count from "create" - this does all the work */
2082         return 0;
2083 }
2084 
2085 static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
2086 {
2087         if (!simple_empty(dentry))
2088                 return -ENOTEMPTY;
2089 
2090         drop_nlink(dentry->d_inode);
2091         drop_nlink(dir);
2092         return shmem_unlink(dir, dentry);
2093 }
2094 
2095 /*
2096  * The VFS layer already does all the dentry stuff for rename,
2097  * we just have to decrement the usage count for the target if
2098  * it exists so that the VFS layer correctly free's it when it
2099  * gets overwritten.
2100  */
2101 static int shmem_rename(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
2102 {
2103         struct inode *inode = old_dentry->d_inode;
2104         int they_are_dirs = S_ISDIR(inode->i_mode);
2105 
2106         if (!simple_empty(new_dentry))
2107                 return -ENOTEMPTY;
2108 
2109         if (new_dentry->d_inode) {
2110                 (void) shmem_unlink(new_dir, new_dentry);
2111                 if (they_are_dirs) {
2112                         drop_nlink(new_dentry->d_inode);
2113                         drop_nlink(old_dir);
2114                 }
2115         } else if (they_are_dirs) {
2116                 drop_nlink(old_dir);
2117                 inc_nlink(new_dir);
2118         }
2119 
2120         old_dir->i_size -= BOGO_DIRENT_SIZE;
2121         new_dir->i_size += BOGO_DIRENT_SIZE;
2122         old_dir->i_ctime = old_dir->i_mtime =
2123         new_dir->i_ctime = new_dir->i_mtime =
2124         inode->i_ctime = CURRENT_TIME;
2125         return 0;
2126 }
2127 
2128 static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
2129 {
2130         int error;
2131         int len;
2132         struct inode *inode;
2133         struct page *page;
2134         char *kaddr;
2135         struct shmem_inode_info *info;
2136 
2137         len = strlen(symname) + 1;
2138         if (len > PAGE_CACHE_SIZE)
2139                 return -ENAMETOOLONG;
2140 
2141         inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0, VM_NORESERVE);
2142         if (!inode)
2143                 return -ENOSPC;
2144 
2145         error = security_inode_init_security(inode, dir, &dentry->d_name,
2146                                              shmem_initxattrs, NULL);
2147         if (error) {
2148                 if (error != -EOPNOTSUPP) {
2149                         iput(inode);
2150                         return error;
2151                 }
2152                 error = 0;
2153         }
2154 
2155         info = SHMEM_I(inode);
2156         inode->i_size = len-1;
2157         if (len <= SHORT_SYMLINK_LEN) {
2158                 info->symlink = kmemdup(symname, len, GFP_KERNEL);
2159                 if (!info->symlink) {
2160                         iput(inode);
2161                         return -ENOMEM;
2162                 }
2163                 inode->i_op = &shmem_short_symlink_operations;
2164         } else {
2165                 error = shmem_getpage(inode, 0, &page, SGP_WRITE, NULL);
2166                 if (error) {
2167                         iput(inode);
2168                         return error;
2169                 }
2170                 inode->i_mapping->a_ops = &shmem_aops;
2171                 inode->i_op = &shmem_symlink_inode_operations;
2172                 kaddr = kmap_atomic(page);
2173                 memcpy(kaddr, symname, len);
2174                 kunmap_atomic(kaddr);
2175                 SetPageUptodate(page);
2176                 set_page_dirty(page);
2177                 unlock_page(page);
2178                 page_cache_release(page);
2179         }
2180         dir->i_size += BOGO_DIRENT_SIZE;
2181         dir->i_ctime = dir->i_mtime = CURRENT_TIME;
2182         d_instantiate(dentry, inode);
2183         dget(dentry);
2184         return 0;
2185 }
2186 
2187 static void *shmem_follow_short_symlink(struct dentry *dentry, struct nameidata *nd)
2188 {
2189         nd_set_link(nd, SHMEM_I(dentry->d_inode)->symlink);
2190         return NULL;
2191 }
2192 
2193 static void *shmem_follow_link(struct dentry *dentry, struct nameidata *nd)
2194 {
2195         struct page *page = NULL;
2196         int error = shmem_getpage(dentry->d_inode, 0, &page, SGP_READ, NULL);
2197         nd_set_link(nd, error ? ERR_PTR(error) : kmap(page));
2198         if (page)
2199                 unlock_page(page);
2200         return page;
2201 }
2202 
2203 static void shmem_put_link(struct dentry *dentry, struct nameidata *nd, void *cookie)
2204 {
2205         if (!IS_ERR(nd_get_link(nd))) {
2206                 struct page *page = cookie;
2207                 kunmap(page);
2208                 mark_page_accessed(page);
2209                 page_cache_release(page);
2210         }
2211 }
2212 
2213 #ifdef CONFIG_TMPFS_XATTR
2214 /*
2215  * Superblocks without xattr inode operations may get some security.* xattr
2216  * support from the LSM "for free". As soon as we have any other xattrs
2217  * like ACLs, we also need to implement the security.* handlers at
2218  * filesystem level, though.
2219  */
2220 
2221 /*
2222  * Callback for security_inode_init_security() for acquiring xattrs.
2223  */
2224 static int shmem_initxattrs(struct inode *inode,
2225                             const struct xattr *xattr_array,
2226                             void *fs_info)
2227 {
2228         struct shmem_inode_info *info = SHMEM_I(inode);
2229         const struct xattr *xattr;
2230         struct simple_xattr *new_xattr;
2231         size_t len;
2232 
2233         for (xattr = xattr_array; xattr->name != NULL; xattr++) {
2234                 new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len);
2235                 if (!new_xattr)
2236                         return -ENOMEM;
2237 
2238                 len = strlen(xattr->name) + 1;
2239                 new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len,
2240                                           GFP_KERNEL);
2241                 if (!new_xattr->name) {
2242                         kfree(new_xattr);
2243                         return -ENOMEM;
2244                 }
2245 
2246                 memcpy(new_xattr->name, XATTR_SECURITY_PREFIX,
2247                        XATTR_SECURITY_PREFIX_LEN);
2248                 memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN,
2249                        xattr->name, len);
2250 
2251                 simple_xattr_list_add(&info->xattrs, new_xattr);
2252         }
2253 
2254         return 0;
2255 }
2256 
2257 static const struct xattr_handler *shmem_xattr_handlers[] = {
2258 #ifdef CONFIG_TMPFS_POSIX_ACL
2259         &generic_acl_access_handler,
2260         &generic_acl_default_handler,
2261 #endif
2262         NULL
2263 };
2264 
2265 static int shmem_xattr_validate(const char *name)
2266 {
2267         struct { const char *prefix; size_t len; } arr[] = {
2268                 { XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN },
2269                 { XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN }
2270         };
2271         int i;
2272 
2273         for (i = 0; i < ARRAY_SIZE(arr); i++) {
2274                 size_t preflen = arr[i].len;
2275                 if (strncmp(name, arr[i].prefix, preflen) == 0) {
2276                         if (!name[preflen])
2277                                 return -EINVAL;
2278                         return 0;
2279                 }
2280         }
2281         return -EOPNOTSUPP;
2282 }
2283 
2284 static ssize_t shmem_getxattr(struct dentry *dentry, const char *name,
2285                               void *buffer, size_t size)
2286 {
2287         struct shmem_inode_info *info = SHMEM_I(dentry->d_inode);
2288         int err;
2289 
2290         /*
2291          * If this is a request for a synthetic attribute in the system.*
2292          * namespace use the generic infrastructure to resolve a handler
2293          * for it via sb->s_xattr.
2294          */
2295         if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN))
2296                 return generic_getxattr(dentry, name, buffer, size);
2297 
2298         err = shmem_xattr_validate(name);
2299         if (err)
2300                 return err;
2301 
2302         return simple_xattr_get(&info->xattrs, name, buffer, size);
2303 }
2304 
2305 static int shmem_setxattr(struct dentry *dentry, const char *name,
2306                           const void *value, size_t size, int flags)
2307 {
2308         struct shmem_inode_info *info = SHMEM_I(dentry->d_inode);
2309         int err;
2310 
2311         /*
2312          * If this is a request for a synthetic attribute in the system.*
2313          * namespace use the generic infrastructure to resolve a handler
2314          * for it via sb->s_xattr.
2315          */
2316         if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN))
2317                 return generic_setxattr(dentry, name, value, size, flags);
2318 
2319         err = shmem_xattr_validate(name);
2320         if (err)
2321                 return err;
2322 
2323         return simple_xattr_set(&info->xattrs, name, value, size, flags);
2324 }
2325 
2326 static int shmem_removexattr(struct dentry *dentry, const char *name)
2327 {
2328         struct shmem_inode_info *info = SHMEM_I(dentry->d_inode);
2329         int err;
2330 
2331         /*
2332          * If this is a request for a synthetic attribute in the system.*
2333          * namespace use the generic infrastructure to resolve a handler
2334          * for it via sb->s_xattr.
2335          */
2336         if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN))
2337                 return generic_removexattr(dentry, name);
2338 
2339         err = shmem_xattr_validate(name);
2340         if (err)
2341                 return err;
2342 
2343         return simple_xattr_remove(&info->xattrs, name);
2344 }
2345 
2346 static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size)
2347 {
2348         struct shmem_inode_info *info = SHMEM_I(dentry->d_inode);
2349         return simple_xattr_list(&info->xattrs, buffer, size);
2350 }
2351 #endif /* CONFIG_TMPFS_XATTR */
2352 
2353 static const struct inode_operations shmem_short_symlink_operations = {
2354         .readlink       = generic_readlink,
2355         .follow_link    = shmem_follow_short_symlink,
2356 #ifdef CONFIG_TMPFS_XATTR
2357         .setxattr       = shmem_setxattr,
2358         .getxattr       = shmem_getxattr,
2359         .listxattr      = shmem_listxattr,
2360         .removexattr    = shmem_removexattr,
2361 #endif
2362 };
2363 
2364 static const struct inode_operations shmem_symlink_inode_operations = {
2365         .readlink       = generic_readlink,
2366         .follow_link    = shmem_follow_link,
2367         .put_link       = shmem_put_link,
2368 #ifdef CONFIG_TMPFS_XATTR
2369         .setxattr       = shmem_setxattr,
2370         .getxattr       = shmem_getxattr,
2371         .listxattr      = shmem_listxattr,
2372         .removexattr    = shmem_removexattr,
2373 #endif
2374 };
2375 
2376 static struct dentry *shmem_get_parent(struct dentry *child)
2377 {
2378         return ERR_PTR(-ESTALE);
2379 }
2380 
2381 static int shmem_match(struct inode *ino, void *vfh)
2382 {
2383         __u32 *fh = vfh;
2384         __u64 inum = fh[2];
2385         inum = (inum << 32) | fh[1];
2386         return ino->i_ino == inum && fh[0] == ino->i_generation;
2387 }
2388 
2389 static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
2390                 struct fid *fid, int fh_len, int fh_type)
2391 {
2392         struct inode *inode;
2393         struct dentry *dentry = NULL;
2394         u64 inum;
2395 
2396         if (fh_len < 3)
2397                 return NULL;
2398 
2399         inum = fid->raw[2];
2400         inum = (inum << 32) | fid->raw[1];
2401 
2402         inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
2403                         shmem_match, fid->raw);
2404         if (inode) {
2405                 dentry = d_find_alias(inode);
2406                 iput(inode);
2407         }
2408 
2409         return dentry;
2410 }
2411 
2412 static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len,
2413                                 struct inode *parent)
2414 {
2415         if (*len < 3) {
2416                 *len = 3;
2417                 return FILEID_INVALID;
2418         }
2419 
2420         if (inode_unhashed(inode)) {
2421                 /* Unfortunately insert_inode_hash is not idempotent,
2422                  * so as we hash inodes here rather than at creation
2423                  * time, we need a lock to ensure we only try
2424                  * to do it once
2425                  */
2426                 static DEFINE_SPINLOCK(lock);
2427                 spin_lock(&lock);
2428                 if (inode_unhashed(inode))
2429                         __insert_inode_hash(inode,
2430                                             inode->i_ino + inode->i_generation);
2431                 spin_unlock(&lock);
2432         }
2433 
2434         fh[0] = inode->i_generation;
2435         fh[1] = inode->i_ino;
2436         fh[2] = ((__u64)inode->i_ino) >> 32;
2437 
2438         *len = 3;
2439         return 1;
2440 }
2441 
2442 static const struct export_operations shmem_export_ops = {
2443         .get_parent     = shmem_get_parent,
2444         .encode_fh      = shmem_encode_fh,
2445         .fh_to_dentry   = shmem_fh_to_dentry,
2446 };
2447 
2448 static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo,
2449                                bool remount)
2450 {
2451         char *this_char, *value, *rest;
2452         struct mempolicy *mpol = NULL;
2453         uid_t uid;
2454         gid_t gid;
2455 
2456         while (options != NULL) {
2457                 this_char = options;
2458                 for (;;) {
2459                         /*
2460                          * NUL-terminate this option: unfortunately,
2461                          * mount options form a comma-separated list,
2462                          * but mpol's nodelist may also contain commas.
2463                          */
2464                         options = strchr(options, ',');
2465                         if (options == NULL)
2466                                 break;
2467                         options++;
2468                         if (!isdigit(*options)) {
2469                                 options[-1] = '\0';
2470                                 break;
2471                         }
2472                 }
2473                 if (!*this_char)
2474                         continue;
2475                 if ((value = strchr(this_char,'=')) != NULL) {
2476                         *value++ = 0;
2477                 } else {
2478                         printk(KERN_ERR
2479                             "tmpfs: No value for mount option '%s'\n",
2480                             this_char);
2481                         goto error;
2482                 }
2483 
2484                 if (!strcmp(this_char,"size")) {
2485                         unsigned long long size;
2486                         size = memparse(value,&rest);
2487                         if (*rest == '%') {
2488                                 size <<= PAGE_SHIFT;
2489                                 size *= totalram_pages;
2490                                 do_div(size, 100);
2491                                 rest++;
2492                         }
2493                         if (*rest)
2494                                 goto bad_val;
2495                         sbinfo->max_blocks =
2496                                 DIV_ROUND_UP(size, PAGE_CACHE_SIZE);
2497                 } else if (!strcmp(this_char,"nr_blocks")) {
2498                         sbinfo->max_blocks = memparse(value, &rest);
2499                         if (*rest)
2500                                 goto bad_val;
2501                 } else if (!strcmp(this_char,"nr_inodes")) {
2502                         sbinfo->max_inodes = memparse(value, &rest);
2503                         if (*rest)
2504                                 goto bad_val;
2505                 } else if (!strcmp(this_char,"mode")) {
2506                         if (remount)
2507                                 continue;
2508                         sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777;
2509                         if (*rest)
2510                                 goto bad_val;
2511                 } else if (!strcmp(this_char,"uid")) {
2512                         if (remount)
2513                                 continue;
2514                         uid = simple_strtoul(value, &rest, 0);
2515                         if (*rest)
2516                                 goto bad_val;
2517                         sbinfo->uid = make_kuid(current_user_ns(), uid);
2518                         if (!uid_valid(sbinfo->uid))
2519                                 goto bad_val;
2520                 } else if (!strcmp(this_char,"gid")) {
2521                         if (remount)
2522                                 continue;
2523                         gid = simple_strtoul(value, &rest, 0);
2524                         if (*rest)
2525                                 goto bad_val;
2526                         sbinfo->gid = make_kgid(current_user_ns(), gid);
2527                         if (!gid_valid(sbinfo->gid))
2528                                 goto bad_val;
2529                 } else if (!strcmp(this_char,"mpol")) {
2530                         mpol_put(mpol);
2531                         mpol = NULL;
2532                         if (mpol_parse_str(value, &mpol))
2533                                 goto bad_val;
2534                 } else {
2535                         printk(KERN_ERR "tmpfs: Bad mount option %s\n",
2536                                this_char);
2537                         goto error;
2538                 }
2539         }
2540         sbinfo->mpol = mpol;
2541         return 0;
2542 
2543 bad_val:
2544         printk(KERN_ERR "tmpfs: Bad value '%s' for mount option '%s'\n",
2545                value, this_char);
2546 error:
2547         mpol_put(mpol);
2548         return 1;
2549 
2550 }
2551 
2552 static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
2553 {
2554         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2555         struct shmem_sb_info config = *sbinfo;
2556         unsigned long inodes;
2557         int error = -EINVAL;
2558 
2559         config.mpol = NULL;
2560         if (shmem_parse_options(data, &config, true))
2561                 return error;
2562 
2563         spin_lock(&sbinfo->stat_lock);
2564         inodes = sbinfo->max_inodes - sbinfo->free_inodes;
2565         if (percpu_counter_compare(&sbinfo->used_blocks, config.max_blocks) > 0)
2566                 goto out;
2567         if (config.max_inodes < inodes)
2568                 goto out;
2569         /*
2570          * Those tests disallow limited->unlimited while any are in use;
2571          * but we must separately disallow unlimited->limited, because
2572          * in that case we have no record of how much is already in use.
2573          */
2574         if (config.max_blocks && !sbinfo->max_blocks)
2575                 goto out;
2576         if (config.max_inodes && !sbinfo->max_inodes)
2577                 goto out;
2578 
2579         error = 0;
2580         sbinfo->max_blocks  = config.max_blocks;
2581         sbinfo->max_inodes  = config.max_inodes;
2582         sbinfo->free_inodes = config.max_inodes - inodes;
2583 
2584         /*
2585          * Preserve previous mempolicy unless mpol remount option was specified.
2586          */
2587         if (config.mpol) {
2588                 mpol_put(sbinfo->mpol);
2589                 sbinfo->mpol = config.mpol;     /* transfers initial ref */
2590         }
2591 out:
2592         spin_unlock(&sbinfo->stat_lock);
2593         return error;
2594 }
2595 
2596 static int shmem_show_options(struct seq_file *seq, struct dentry *root)
2597 {
2598         struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb);
2599 
2600         if (sbinfo->max_blocks != shmem_default_max_blocks())
2601                 seq_printf(seq, ",size=%luk",
2602                         sbinfo->max_blocks << (PAGE_CACHE_SHIFT - 10));
2603         if (sbinfo->max_inodes != shmem_default_max_inodes())
2604                 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
2605         if (sbinfo->mode != (S_IRWXUGO | S_ISVTX))
2606                 seq_printf(seq, ",mode=%03ho", sbinfo->mode);
2607         if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
2608                 seq_printf(seq, ",uid=%u",
2609                                 from_kuid_munged(&init_user_ns, sbinfo->uid));
2610         if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
2611                 seq_printf(seq, ",gid=%u",
2612                                 from_kgid_munged(&init_user_ns, sbinfo->gid));
2613         shmem_show_mpol(seq, sbinfo->mpol);
2614         return 0;
2615 }
2616 #endif /* CONFIG_TMPFS */
2617 
2618 static void shmem_put_super(struct super_block *sb)
2619 {
2620         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2621 
2622         percpu_counter_destroy(&sbinfo->used_blocks);
2623         mpol_put(sbinfo->mpol);
2624         kfree(sbinfo);
2625         sb->s_fs_info = NULL;
2626 }
2627 
2628 int shmem_fill_super(struct super_block *sb, void *data, int silent)
2629 {
2630         struct inode *inode;
2631         struct shmem_sb_info *sbinfo;
2632         int err = -ENOMEM;
2633 
2634         /* Round up to L1_CACHE_BYTES to resist false sharing */
2635         sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
2636                                 L1_CACHE_BYTES), GFP_KERNEL);
2637         if (!sbinfo)
2638                 return -ENOMEM;
2639 
2640         sbinfo->mode = S_IRWXUGO | S_ISVTX;
2641         sbinfo->uid = current_fsuid();
2642         sbinfo->gid = current_fsgid();
2643         sb->s_fs_info = sbinfo;
2644 
2645 #ifdef CONFIG_TMPFS
2646         /*
2647          * Per default we only allow half of the physical ram per
2648          * tmpfs instance, limiting inodes to one per page of lowmem;
2649          * but the internal instance is left unlimited.
2650          */
2651         if (!(sb->s_flags & MS_KERNMOUNT)) {
2652                 sbinfo->max_blocks = shmem_default_max_blocks();
2653                 sbinfo->max_inodes = shmem_default_max_inodes();
2654                 if (shmem_parse_options(data, sbinfo, false)) {
2655                         err = -EINVAL;
2656                         goto failed;
2657                 }
2658         } else {
2659                 sb->s_flags |= MS_NOUSER;
2660         }
2661         sb->s_export_op = &shmem_export_ops;
2662         sb->s_flags |= MS_NOSEC;
2663 #else
2664         sb->s_flags |= MS_NOUSER;
2665 #endif
2666 
2667         spin_lock_init(&sbinfo->stat_lock);
2668         if (percpu_counter_init(&sbinfo->used_blocks, 0))
2669                 goto failed;
2670         sbinfo->free_inodes = sbinfo->max_inodes;
2671 
2672         sb->s_maxbytes = MAX_LFS_FILESIZE;
2673         sb->s_blocksize = PAGE_CACHE_SIZE;
2674         sb->s_blocksize_bits = PAGE_CACHE_SHIFT;
2675         sb->s_magic = TMPFS_MAGIC;
2676         sb->s_op = &shmem_ops;
2677         sb->s_time_gran = 1;
2678 #ifdef CONFIG_TMPFS_XATTR
2679         sb->s_xattr = shmem_xattr_handlers;
2680 #endif
2681 #ifdef CONFIG_TMPFS_POSIX_ACL
2682         sb->s_flags |= MS_POSIXACL;
2683 #endif
2684 
2685         inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
2686         if (!inode)
2687                 goto failed;
2688         inode->i_uid = sbinfo->uid;
2689         inode->i_gid = sbinfo->gid;
2690         sb->s_root = d_make_root(inode);
2691         if (!sb->s_root)
2692                 goto failed;
2693         return 0;
2694 
2695 failed:
2696         shmem_put_super(sb);
2697         return err;
2698 }
2699 
2700 static struct kmem_cache *shmem_inode_cachep;
2701 
2702 static struct inode *shmem_alloc_inode(struct super_block *sb)
2703 {
2704         struct shmem_inode_info *info;
2705         info = kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL);
2706         if (!info)
2707                 return NULL;
2708         return &info->vfs_inode;
2709 }
2710 
2711 static void shmem_destroy_callback(struct rcu_head *head)
2712 {
2713         struct inode *inode = container_of(head, struct inode, i_rcu);
2714         kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
2715 }
2716 
2717 static void shmem_destroy_inode(struct inode *inode)
2718 {
2719         if (S_ISREG(inode->i_mode))
2720                 mpol_free_shared_policy(&SHMEM_I(inode)->policy);
2721         call_rcu(&inode->i_rcu, shmem_destroy_callback);
2722 }
2723 
2724 static void shmem_init_inode(void *foo)
2725 {
2726         struct shmem_inode_info *info = foo;
2727         inode_init_once(&info->vfs_inode);
2728 }
2729 
2730 static int shmem_init_inodecache(void)
2731 {
2732         shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
2733                                 sizeof(struct shmem_inode_info),
2734                                 0, SLAB_PANIC, shmem_init_inode);
2735         return 0;
2736 }
2737 
2738 static void shmem_destroy_inodecache(void)
2739 {
2740         kmem_cache_destroy(shmem_inode_cachep);
2741 }
2742 
2743 static const struct address_space_operations shmem_aops = {
2744         .writepage      = shmem_writepage,
2745         .set_page_dirty = __set_page_dirty_no_writeback,
2746 #ifdef CONFIG_TMPFS
2747         .write_begin    = shmem_write_begin,
2748         .write_end      = shmem_write_end,
2749 #endif
2750         .migratepage    = migrate_page,
2751         .error_remove_page = generic_error_remove_page,
2752 };
2753 
2754 static const struct file_operations shmem_file_operations = {
2755         .mmap           = shmem_mmap,
2756 #ifdef CONFIG_TMPFS
2757         .llseek         = shmem_file_llseek,
2758         .read           = do_sync_read,
2759         .write          = do_sync_write,
2760         .aio_read       = shmem_file_aio_read,
2761         .aio_write      = generic_file_aio_write,
2762         .fsync          = noop_fsync,
2763         .splice_read    = shmem_file_splice_read,
2764         .splice_write   = generic_file_splice_write,
2765         .fallocate      = shmem_fallocate,
2766 #endif
2767 };
2768 
2769 static const struct inode_operations shmem_inode_operations = {
2770         .setattr        = shmem_setattr,
2771 #ifdef CONFIG_TMPFS_XATTR
2772         .setxattr       = shmem_setxattr,
2773         .getxattr       = shmem_getxattr,
2774         .listxattr      = shmem_listxattr,
2775         .removexattr    = shmem_removexattr,
2776 #endif
2777 };
2778 
2779 static const struct inode_operations shmem_dir_inode_operations = {
2780 #ifdef CONFIG_TMPFS
2781         .create         = shmem_create,
2782         .lookup         = simple_lookup,
2783         .link           = shmem_link,
2784         .unlink         = shmem_unlink,
2785         .symlink        = shmem_symlink,
2786         .mkdir          = shmem_mkdir,
2787         .rmdir          = shmem_rmdir,
2788         .mknod          = shmem_mknod,
2789         .rename         = shmem_rename,
2790         .tmpfile        = shmem_tmpfile,
2791 #endif
2792 #ifdef CONFIG_TMPFS_XATTR
2793         .setxattr       = shmem_setxattr,
2794         .getxattr       = shmem_getxattr,
2795         .listxattr      = shmem_listxattr,
2796         .removexattr    = shmem_removexattr,
2797 #endif
2798 #ifdef CONFIG_TMPFS_POSIX_ACL
2799         .setattr        = shmem_setattr,
2800 #endif
2801 };
2802 
2803 static const struct inode_operations shmem_special_inode_operations = {
2804 #ifdef CONFIG_TMPFS_XATTR
2805         .setxattr       = shmem_setxattr,
2806         .getxattr       = shmem_getxattr,
2807         .listxattr      = shmem_listxattr,
2808         .removexattr    = shmem_removexattr,
2809 #endif
2810 #ifdef CONFIG_TMPFS_POSIX_ACL
2811         .setattr        = shmem_setattr,
2812 #endif
2813 };
2814 
2815 static const struct super_operations shmem_ops = {
2816         .alloc_inode    = shmem_alloc_inode,
2817         .destroy_inode  = shmem_destroy_inode,
2818 #ifdef CONFIG_TMPFS
2819         .statfs         = shmem_statfs,
2820         .remount_fs     = shmem_remount_fs,
2821         .show_options   = shmem_show_options,
2822 #endif
2823         .evict_inode    = shmem_evict_inode,
2824         .drop_inode     = generic_delete_inode,
2825         .put_super      = shmem_put_super,
2826 };
2827 
2828 static const struct vm_operations_struct shmem_vm_ops = {
2829         .fault          = shmem_fault,
2830 #ifdef CONFIG_NUMA
2831         .set_policy     = shmem_set_policy,
2832         .get_policy     = shmem_get_policy,
2833 #endif
2834         .remap_pages    = generic_file_remap_pages,
2835 };
2836 
2837 static struct dentry *shmem_mount(struct file_system_type *fs_type,
2838         int flags, const char *dev_name, void *data)
2839 {
2840         return mount_nodev(fs_type, flags, data, shmem_fill_super);
2841 }
2842 
2843 static struct file_system_type shmem_fs_type = {
2844         .owner          = THIS_MODULE,
2845         .name           = "tmpfs",
2846         .mount          = shmem_mount,
2847         .kill_sb        = kill_litter_super,
2848         .fs_flags       = FS_USERNS_MOUNT,
2849 };
2850 
2851 int __init shmem_init(void)
2852 {
2853         int error;
2854 
2855         /* If rootfs called this, don't re-init */
2856         if (shmem_inode_cachep)
2857                 return 0;
2858 
2859         error = bdi_init(&shmem_backing_dev_info);
2860         if (error)
2861                 goto out4;
2862 
2863         error = shmem_init_inodecache();
2864         if (error)
2865                 goto out3;
2866 
2867         error = register_filesystem(&shmem_fs_type);
2868         if (error) {
2869                 printk(KERN_ERR "Could not register tmpfs\n");
2870                 goto out2;
2871         }
2872 
2873         shm_mnt = kern_mount(&shmem_fs_type);
2874         if (IS_ERR(shm_mnt)) {
2875                 error = PTR_ERR(shm_mnt);
2876                 printk(KERN_ERR "Could not kern_mount tmpfs\n");
2877                 goto out1;
2878         }
2879         return 0;
2880 
2881 out1:
2882         unregister_filesystem(&shmem_fs_type);
2883 out2:
2884         shmem_destroy_inodecache();
2885 out3:
2886         bdi_destroy(&shmem_backing_dev_info);
2887 out4:
2888         shm_mnt = ERR_PTR(error);
2889         return error;
2890 }
2891 
2892 #else /* !CONFIG_SHMEM */
2893 
2894 /*
2895  * tiny-shmem: simple shmemfs and tmpfs using ramfs code
2896  *
2897  * This is intended for small system where the benefits of the full
2898  * shmem code (swap-backed and resource-limited) are outweighed by
2899  * their complexity. On systems without swap this code should be
2900  * effectively equivalent, but much lighter weight.
2901  */
2902 
2903 static struct file_system_type shmem_fs_type = {
2904         .name           = "tmpfs",
2905         .mount          = ramfs_mount,
2906         .kill_sb        = kill_litter_super,
2907         .fs_flags       = FS_USERNS_MOUNT,
2908 };
2909 
2910 int __init shmem_init(void)
2911 {
2912         BUG_ON(register_filesystem(&shmem_fs_type) != 0);
2913 
2914         shm_mnt = kern_mount(&shmem_fs_type);
2915         BUG_ON(IS_ERR(shm_mnt));
2916 
2917         return 0;
2918 }
2919 
2920 int shmem_unuse(swp_entry_t swap, struct page *page)
2921 {
2922         return 0;
2923 }
2924 
2925 int shmem_lock(struct file *file, int lock, struct user_struct *user)
2926 {
2927         return 0;
2928 }
2929 
2930 void shmem_unlock_mapping(struct address_space *mapping)
2931 {
2932 }
2933 
2934 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
2935 {
2936         truncate_inode_pages_range(inode->i_mapping, lstart, lend);
2937 }
2938 EXPORT_SYMBOL_GPL(shmem_truncate_range);
2939 
2940 #define shmem_vm_ops                            generic_file_vm_ops
2941 #define shmem_file_operations                   ramfs_file_operations
2942 #define shmem_get_inode(sb, dir, mode, dev, flags)      ramfs_get_inode(sb, dir, mode, dev)
2943 #define shmem_acct_size(flags, size)            0
2944 #define shmem_unacct_size(flags, size)          do {} while (0)
2945 
2946 #endif /* CONFIG_SHMEM */
2947 
2948 /* common code */
2949 
2950 static struct dentry_operations anon_ops = {
2951         .d_dname = simple_dname
2952 };
2953 
2954 /**
2955  * shmem_file_setup - get an unlinked file living in tmpfs
2956  * @name: name for dentry (to be seen in /proc/<pid>/maps
2957  * @size: size to be set for the file
2958  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
2959  */
2960 struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
2961 {
2962         struct file *res;
2963         struct inode *inode;
2964         struct path path;
2965         struct super_block *sb;
2966         struct qstr this;
2967 
2968         if (IS_ERR(shm_mnt))
2969                 return ERR_CAST(shm_mnt);
2970 
2971         if (size < 0 || size > MAX_LFS_FILESIZE)
2972                 return ERR_PTR(-EINVAL);
2973 
2974         if (shmem_acct_size(flags, size))
2975                 return ERR_PTR(-ENOMEM);
2976 
2977         res = ERR_PTR(-ENOMEM);
2978         this.name = name;
2979         this.len = strlen(name);
2980         this.hash = 0; /* will go */
2981         sb = shm_mnt->mnt_sb;
2982         path.dentry = d_alloc_pseudo(sb, &this);
2983         if (!path.dentry)
2984                 goto put_memory;
2985         d_set_d_op(path.dentry, &anon_ops);
2986         path.mnt = mntget(shm_mnt);
2987 
2988         res = ERR_PTR(-ENOSPC);
2989         inode = shmem_get_inode(sb, NULL, S_IFREG | S_IRWXUGO, 0, flags);
2990         if (!inode)
2991                 goto put_dentry;
2992 
2993         d_instantiate(path.dentry, inode);
2994         inode->i_size = size;
2995         clear_nlink(inode);     /* It is unlinked */
2996         res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size));
2997         if (IS_ERR(res))
2998                 goto put_dentry;
2999 
3000         res = alloc_file(&path, FMODE_WRITE | FMODE_READ,
3001                   &shmem_file_operations);
3002         if (IS_ERR(res))
3003                 goto put_dentry;
3004 
3005         return res;
3006 
3007 put_dentry:
3008         path_put(&path);
3009 put_memory:
3010         shmem_unacct_size(flags, size);
3011         return res;
3012 }
3013 EXPORT_SYMBOL_GPL(shmem_file_setup);
3014 
3015 /**
3016  * shmem_zero_setup - setup a shared anonymous mapping
3017  * @vma: the vma to be mmapped is prepared by do_mmap_pgoff
3018  */
3019 int shmem_zero_setup(struct vm_area_struct *vma)
3020 {
3021         struct file *file;
3022         loff_t size = vma->vm_end - vma->vm_start;
3023 
3024         file = shmem_file_setup("dev/zero", size, vma->vm_flags);
3025         if (IS_ERR(file))
3026                 return PTR_ERR(file);
3027 
3028         if (vma->vm_file)
3029                 fput(vma->vm_file);
3030         vma->vm_file = file;
3031         vma->vm_ops = &shmem_vm_ops;
3032         return 0;
3033 }
3034 
3035 /**
3036  * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags.
3037  * @mapping:    the page's address_space
3038  * @index:      the page index
3039  * @gfp:        the page allocator flags to use if allocating
3040  *
3041  * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)",
3042  * with any new page allocations done using the specified allocation flags.
3043  * But read_cache_page_gfp() uses the ->readpage() method: which does not
3044  * suit tmpfs, since it may have pages in swapcache, and needs to find those
3045  * for itself; although drivers/gpu/drm i915 and ttm rely upon this support.
3046  *
3047  * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in
3048  * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily.
3049  */
3050 struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
3051                                          pgoff_t index, gfp_t gfp)
3052 {
3053 #ifdef CONFIG_SHMEM
3054         struct inode *inode = mapping->host;
3055         struct page *page;
3056         int error;
3057 
3058         BUG_ON(mapping->a_ops != &shmem_aops);
3059         error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE, gfp, NULL);
3060         if (error)
3061                 page = ERR_PTR(error);
3062         else
3063                 unlock_page(page);
3064         return page;
3065 #else
3066         /*
3067          * The tiny !SHMEM case uses ramfs without swap
3068          */
3069         return read_cache_page_gfp(mapping, index, gfp);
3070 #endif
3071 }
3072 EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp);
3073 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp