1 /* 2 * Resizable virtual memory filesystem for Linux. 3 * 4 * Copyright (C) 2000 Linus Torvalds. 5 * 2000 Transmeta Corp. 6 * 2000-2001 Christoph Rohland 7 * 2000-2001 SAP AG 8 * 2002 Red Hat Inc. 9 * Copyright (C) 2002-2011 Hugh Dickins. 10 * Copyright (C) 2011 Google Inc. 11 * Copyright (C) 2002-2005 VERITAS Software Corporation. 12 * Copyright (C) 2004 Andi Kleen, SuSE Labs 13 * 14 * Extended attribute support for tmpfs: 15 * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net> 16 * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com> 17 * 18 * tiny-shmem: 19 * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com> 20 * 21 * This file is released under the GPL. 22 */ 23 24 #include <linux/fs.h> 25 #include <linux/init.h> 26 #include <linux/vfs.h> 27 #include <linux/mount.h> 28 #include <linux/ramfs.h> 29 #include <linux/pagemap.h> 30 #include <linux/file.h> 31 #include <linux/mm.h> 32 #include <linux/export.h> 33 #include <linux/swap.h> 34 #include <linux/uio.h> 35 #include <linux/khugepaged.h> 36 37 static struct vfsmount *shm_mnt; 38 39 #ifdef CONFIG_SHMEM 40 /* 41 * This virtual memory filesystem is heavily based on the ramfs. It 42 * extends ramfs by the ability to use swap and honor resource limits 43 * which makes it a completely usable filesystem. 44 */ 45 46 #include <linux/xattr.h> 47 #include <linux/exportfs.h> 48 #include <linux/posix_acl.h> 49 #include <linux/posix_acl_xattr.h> 50 #include <linux/mman.h> 51 #include <linux/string.h> 52 #include <linux/slab.h> 53 #include <linux/backing-dev.h> 54 #include <linux/shmem_fs.h> 55 #include <linux/writeback.h> 56 #include <linux/blkdev.h> 57 #include <linux/pagevec.h> 58 #include <linux/percpu_counter.h> 59 #include <linux/falloc.h> 60 #include <linux/splice.h> 61 #include <linux/security.h> 62 #include <linux/swapops.h> 63 #include <linux/mempolicy.h> 64 #include <linux/namei.h> 65 #include <linux/ctype.h> 66 #include <linux/migrate.h> 67 #include <linux/highmem.h> 68 #include <linux/seq_file.h> 69 #include <linux/magic.h> 70 #include <linux/syscalls.h> 71 #include <linux/fcntl.h> 72 #include <uapi/linux/memfd.h> 73 74 #include <asm/uaccess.h> 75 #include <asm/pgtable.h> 76 77 #include "internal.h" 78 79 #define BLOCKS_PER_PAGE (PAGE_SIZE/512) 80 #define VM_ACCT(size) (PAGE_ALIGN(size) >> PAGE_SHIFT) 81 82 /* Pretend that each entry is of this size in directory's i_size */ 83 #define BOGO_DIRENT_SIZE 20 84 85 /* Symlink up to this size is kmalloc'ed instead of using a swappable page */ 86 #define SHORT_SYMLINK_LEN 128 87 88 /* 89 * shmem_fallocate communicates with shmem_fault or shmem_writepage via 90 * inode->i_private (with i_mutex making sure that it has only one user at 91 * a time): we would prefer not to enlarge the shmem inode just for that. 92 */ 93 struct shmem_falloc { 94 wait_queue_head_t *waitq; /* faults into hole wait for punch to end */ 95 pgoff_t start; /* start of range currently being fallocated */ 96 pgoff_t next; /* the next page offset to be fallocated */ 97 pgoff_t nr_falloced; /* how many new pages have been fallocated */ 98 pgoff_t nr_unswapped; /* how often writepage refused to swap out */ 99 }; 100 101 #ifdef CONFIG_TMPFS 102 static unsigned long shmem_default_max_blocks(void) 103 { 104 return totalram_pages / 2; 105 } 106 107 static unsigned long shmem_default_max_inodes(void) 108 { 109 return min(totalram_pages - totalhigh_pages, totalram_pages / 2); 110 } 111 #endif 112 113 static bool shmem_should_replace_page(struct page *page, gfp_t gfp); 114 static int shmem_replace_page(struct page **pagep, gfp_t gfp, 115 struct shmem_inode_info *info, pgoff_t index); 116 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, 117 struct page **pagep, enum sgp_type sgp, 118 gfp_t gfp, struct mm_struct *fault_mm, int *fault_type); 119 120 int shmem_getpage(struct inode *inode, pgoff_t index, 121 struct page **pagep, enum sgp_type sgp) 122 { 123 return shmem_getpage_gfp(inode, index, pagep, sgp, 124 mapping_gfp_mask(inode->i_mapping), NULL, NULL); 125 } 126 127 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb) 128 { 129 return sb->s_fs_info; 130 } 131 132 /* 133 * shmem_file_setup pre-accounts the whole fixed size of a VM object, 134 * for shared memory and for shared anonymous (/dev/zero) mappings 135 * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1), 136 * consistent with the pre-accounting of private mappings ... 137 */ 138 static inline int shmem_acct_size(unsigned long flags, loff_t size) 139 { 140 return (flags & VM_NORESERVE) ? 141 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size)); 142 } 143 144 static inline void shmem_unacct_size(unsigned long flags, loff_t size) 145 { 146 if (!(flags & VM_NORESERVE)) 147 vm_unacct_memory(VM_ACCT(size)); 148 } 149 150 static inline int shmem_reacct_size(unsigned long flags, 151 loff_t oldsize, loff_t newsize) 152 { 153 if (!(flags & VM_NORESERVE)) { 154 if (VM_ACCT(newsize) > VM_ACCT(oldsize)) 155 return security_vm_enough_memory_mm(current->mm, 156 VM_ACCT(newsize) - VM_ACCT(oldsize)); 157 else if (VM_ACCT(newsize) < VM_ACCT(oldsize)) 158 vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize)); 159 } 160 return 0; 161 } 162 163 /* 164 * ... whereas tmpfs objects are accounted incrementally as 165 * pages are allocated, in order to allow large sparse files. 166 * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM, 167 * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM. 168 */ 169 static inline int shmem_acct_block(unsigned long flags, long pages) 170 { 171 if (!(flags & VM_NORESERVE)) 172 return 0; 173 174 return security_vm_enough_memory_mm(current->mm, 175 pages * VM_ACCT(PAGE_SIZE)); 176 } 177 178 static inline void shmem_unacct_blocks(unsigned long flags, long pages) 179 { 180 if (flags & VM_NORESERVE) 181 vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE)); 182 } 183 184 static const struct super_operations shmem_ops; 185 static const struct address_space_operations shmem_aops; 186 static const struct file_operations shmem_file_operations; 187 static const struct inode_operations shmem_inode_operations; 188 static const struct inode_operations shmem_dir_inode_operations; 189 static const struct inode_operations shmem_special_inode_operations; 190 static const struct vm_operations_struct shmem_vm_ops; 191 static struct file_system_type shmem_fs_type; 192 193 static LIST_HEAD(shmem_swaplist); 194 static DEFINE_MUTEX(shmem_swaplist_mutex); 195 196 static int shmem_reserve_inode(struct super_block *sb) 197 { 198 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 199 if (sbinfo->max_inodes) { 200 spin_lock(&sbinfo->stat_lock); 201 if (!sbinfo->free_inodes) { 202 spin_unlock(&sbinfo->stat_lock); 203 return -ENOSPC; 204 } 205 sbinfo->free_inodes--; 206 spin_unlock(&sbinfo->stat_lock); 207 } 208 return 0; 209 } 210 211 static void shmem_free_inode(struct super_block *sb) 212 { 213 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 214 if (sbinfo->max_inodes) { 215 spin_lock(&sbinfo->stat_lock); 216 sbinfo->free_inodes++; 217 spin_unlock(&sbinfo->stat_lock); 218 } 219 } 220 221 /** 222 * shmem_recalc_inode - recalculate the block usage of an inode 223 * @inode: inode to recalc 224 * 225 * We have to calculate the free blocks since the mm can drop 226 * undirtied hole pages behind our back. 227 * 228 * But normally info->alloced == inode->i_mapping->nrpages + info->swapped 229 * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped) 230 * 231 * It has to be called with the spinlock held. 232 */ 233 static void shmem_recalc_inode(struct inode *inode) 234 { 235 struct shmem_inode_info *info = SHMEM_I(inode); 236 long freed; 237 238 freed = info->alloced - info->swapped - inode->i_mapping->nrpages; 239 if (freed > 0) { 240 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 241 if (sbinfo->max_blocks) 242 percpu_counter_add(&sbinfo->used_blocks, -freed); 243 info->alloced -= freed; 244 inode->i_blocks -= freed * BLOCKS_PER_PAGE; 245 shmem_unacct_blocks(info->flags, freed); 246 } 247 } 248 249 bool shmem_charge(struct inode *inode, long pages) 250 { 251 struct shmem_inode_info *info = SHMEM_I(inode); 252 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 253 unsigned long flags; 254 255 if (shmem_acct_block(info->flags, pages)) 256 return false; 257 spin_lock_irqsave(&info->lock, flags); 258 info->alloced += pages; 259 inode->i_blocks += pages * BLOCKS_PER_PAGE; 260 shmem_recalc_inode(inode); 261 spin_unlock_irqrestore(&info->lock, flags); 262 inode->i_mapping->nrpages += pages; 263 264 if (!sbinfo->max_blocks) 265 return true; 266 if (percpu_counter_compare(&sbinfo->used_blocks, 267 sbinfo->max_blocks - pages) > 0) { 268 inode->i_mapping->nrpages -= pages; 269 spin_lock_irqsave(&info->lock, flags); 270 info->alloced -= pages; 271 shmem_recalc_inode(inode); 272 spin_unlock_irqrestore(&info->lock, flags); 273 shmem_unacct_blocks(info->flags, pages); 274 return false; 275 } 276 percpu_counter_add(&sbinfo->used_blocks, pages); 277 return true; 278 } 279 280 void shmem_uncharge(struct inode *inode, long pages) 281 { 282 struct shmem_inode_info *info = SHMEM_I(inode); 283 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 284 unsigned long flags; 285 286 spin_lock_irqsave(&info->lock, flags); 287 info->alloced -= pages; 288 inode->i_blocks -= pages * BLOCKS_PER_PAGE; 289 shmem_recalc_inode(inode); 290 spin_unlock_irqrestore(&info->lock, flags); 291 292 if (sbinfo->max_blocks) 293 percpu_counter_sub(&sbinfo->used_blocks, pages); 294 shmem_unacct_blocks(info->flags, pages); 295 } 296 297 /* 298 * Replace item expected in radix tree by a new item, while holding tree lock. 299 */ 300 static int shmem_radix_tree_replace(struct address_space *mapping, 301 pgoff_t index, void *expected, void *replacement) 302 { 303 void **pslot; 304 void *item; 305 306 VM_BUG_ON(!expected); 307 VM_BUG_ON(!replacement); 308 pslot = radix_tree_lookup_slot(&mapping->page_tree, index); 309 if (!pslot) 310 return -ENOENT; 311 item = radix_tree_deref_slot_protected(pslot, &mapping->tree_lock); 312 if (item != expected) 313 return -ENOENT; 314 radix_tree_replace_slot(pslot, replacement); 315 return 0; 316 } 317 318 /* 319 * Sometimes, before we decide whether to proceed or to fail, we must check 320 * that an entry was not already brought back from swap by a racing thread. 321 * 322 * Checking page is not enough: by the time a SwapCache page is locked, it 323 * might be reused, and again be SwapCache, using the same swap as before. 324 */ 325 static bool shmem_confirm_swap(struct address_space *mapping, 326 pgoff_t index, swp_entry_t swap) 327 { 328 void *item; 329 330 rcu_read_lock(); 331 item = radix_tree_lookup(&mapping->page_tree, index); 332 rcu_read_unlock(); 333 return item == swp_to_radix_entry(swap); 334 } 335 336 /* 337 * Definitions for "huge tmpfs": tmpfs mounted with the huge= option 338 * 339 * SHMEM_HUGE_NEVER: 340 * disables huge pages for the mount; 341 * SHMEM_HUGE_ALWAYS: 342 * enables huge pages for the mount; 343 * SHMEM_HUGE_WITHIN_SIZE: 344 * only allocate huge pages if the page will be fully within i_size, 345 * also respect fadvise()/madvise() hints; 346 * SHMEM_HUGE_ADVISE: 347 * only allocate huge pages if requested with fadvise()/madvise(); 348 */ 349 350 #define SHMEM_HUGE_NEVER 0 351 #define SHMEM_HUGE_ALWAYS 1 352 #define SHMEM_HUGE_WITHIN_SIZE 2 353 #define SHMEM_HUGE_ADVISE 3 354 355 /* 356 * Special values. 357 * Only can be set via /sys/kernel/mm/transparent_hugepage/shmem_enabled: 358 * 359 * SHMEM_HUGE_DENY: 360 * disables huge on shm_mnt and all mounts, for emergency use; 361 * SHMEM_HUGE_FORCE: 362 * enables huge on shm_mnt and all mounts, w/o needing option, for testing; 363 * 364 */ 365 #define SHMEM_HUGE_DENY (-1) 366 #define SHMEM_HUGE_FORCE (-2) 367 368 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 369 /* ifdef here to avoid bloating shmem.o when not necessary */ 370 371 int shmem_huge __read_mostly; 372 373 static int shmem_parse_huge(const char *str) 374 { 375 if (!strcmp(str, "never")) 376 return SHMEM_HUGE_NEVER; 377 if (!strcmp(str, "always")) 378 return SHMEM_HUGE_ALWAYS; 379 if (!strcmp(str, "within_size")) 380 return SHMEM_HUGE_WITHIN_SIZE; 381 if (!strcmp(str, "advise")) 382 return SHMEM_HUGE_ADVISE; 383 if (!strcmp(str, "deny")) 384 return SHMEM_HUGE_DENY; 385 if (!strcmp(str, "force")) 386 return SHMEM_HUGE_FORCE; 387 return -EINVAL; 388 } 389 390 static const char *shmem_format_huge(int huge) 391 { 392 switch (huge) { 393 case SHMEM_HUGE_NEVER: 394 return "never"; 395 case SHMEM_HUGE_ALWAYS: 396 return "always"; 397 case SHMEM_HUGE_WITHIN_SIZE: 398 return "within_size"; 399 case SHMEM_HUGE_ADVISE: 400 return "advise"; 401 case SHMEM_HUGE_DENY: 402 return "deny"; 403 case SHMEM_HUGE_FORCE: 404 return "force"; 405 default: 406 VM_BUG_ON(1); 407 return "bad_val"; 408 } 409 } 410 411 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo, 412 struct shrink_control *sc, unsigned long nr_to_split) 413 { 414 LIST_HEAD(list), *pos, *next; 415 struct inode *inode; 416 struct shmem_inode_info *info; 417 struct page *page; 418 unsigned long batch = sc ? sc->nr_to_scan : 128; 419 int removed = 0, split = 0; 420 421 if (list_empty(&sbinfo->shrinklist)) 422 return SHRINK_STOP; 423 424 spin_lock(&sbinfo->shrinklist_lock); 425 list_for_each_safe(pos, next, &sbinfo->shrinklist) { 426 info = list_entry(pos, struct shmem_inode_info, shrinklist); 427 428 /* pin the inode */ 429 inode = igrab(&info->vfs_inode); 430 431 /* inode is about to be evicted */ 432 if (!inode) { 433 list_del_init(&info->shrinklist); 434 removed++; 435 goto next; 436 } 437 438 /* Check if there's anything to gain */ 439 if (round_up(inode->i_size, PAGE_SIZE) == 440 round_up(inode->i_size, HPAGE_PMD_SIZE)) { 441 list_del_init(&info->shrinklist); 442 removed++; 443 iput(inode); 444 goto next; 445 } 446 447 list_move(&info->shrinklist, &list); 448 next: 449 if (!--batch) 450 break; 451 } 452 spin_unlock(&sbinfo->shrinklist_lock); 453 454 list_for_each_safe(pos, next, &list) { 455 int ret; 456 457 info = list_entry(pos, struct shmem_inode_info, shrinklist); 458 inode = &info->vfs_inode; 459 460 if (nr_to_split && split >= nr_to_split) { 461 iput(inode); 462 continue; 463 } 464 465 page = find_lock_page(inode->i_mapping, 466 (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT); 467 if (!page) 468 goto drop; 469 470 if (!PageTransHuge(page)) { 471 unlock_page(page); 472 put_page(page); 473 goto drop; 474 } 475 476 ret = split_huge_page(page); 477 unlock_page(page); 478 put_page(page); 479 480 if (ret) { 481 /* split failed: leave it on the list */ 482 iput(inode); 483 continue; 484 } 485 486 split++; 487 drop: 488 list_del_init(&info->shrinklist); 489 removed++; 490 iput(inode); 491 } 492 493 spin_lock(&sbinfo->shrinklist_lock); 494 list_splice_tail(&list, &sbinfo->shrinklist); 495 sbinfo->shrinklist_len -= removed; 496 spin_unlock(&sbinfo->shrinklist_lock); 497 498 return split; 499 } 500 501 static long shmem_unused_huge_scan(struct super_block *sb, 502 struct shrink_control *sc) 503 { 504 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 505 506 if (!READ_ONCE(sbinfo->shrinklist_len)) 507 return SHRINK_STOP; 508 509 return shmem_unused_huge_shrink(sbinfo, sc, 0); 510 } 511 512 static long shmem_unused_huge_count(struct super_block *sb, 513 struct shrink_control *sc) 514 { 515 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 516 return READ_ONCE(sbinfo->shrinklist_len); 517 } 518 #else /* !CONFIG_TRANSPARENT_HUGE_PAGECACHE */ 519 520 #define shmem_huge SHMEM_HUGE_DENY 521 522 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo, 523 struct shrink_control *sc, unsigned long nr_to_split) 524 { 525 return 0; 526 } 527 #endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE */ 528 529 /* 530 * Like add_to_page_cache_locked, but error if expected item has gone. 531 */ 532 static int shmem_add_to_page_cache(struct page *page, 533 struct address_space *mapping, 534 pgoff_t index, void *expected) 535 { 536 int error, nr = hpage_nr_pages(page); 537 538 VM_BUG_ON_PAGE(PageTail(page), page); 539 VM_BUG_ON_PAGE(index != round_down(index, nr), page); 540 VM_BUG_ON_PAGE(!PageLocked(page), page); 541 VM_BUG_ON_PAGE(!PageSwapBacked(page), page); 542 VM_BUG_ON(expected && PageTransHuge(page)); 543 544 page_ref_add(page, nr); 545 page->mapping = mapping; 546 page->index = index; 547 548 spin_lock_irq(&mapping->tree_lock); 549 if (PageTransHuge(page)) { 550 void __rcu **results; 551 pgoff_t idx; 552 int i; 553 554 error = 0; 555 if (radix_tree_gang_lookup_slot(&mapping->page_tree, 556 &results, &idx, index, 1) && 557 idx < index + HPAGE_PMD_NR) { 558 error = -EEXIST; 559 } 560 561 if (!error) { 562 for (i = 0; i < HPAGE_PMD_NR; i++) { 563 error = radix_tree_insert(&mapping->page_tree, 564 index + i, page + i); 565 VM_BUG_ON(error); 566 } 567 count_vm_event(THP_FILE_ALLOC); 568 } 569 } else if (!expected) { 570 error = radix_tree_insert(&mapping->page_tree, index, page); 571 } else { 572 error = shmem_radix_tree_replace(mapping, index, expected, 573 page); 574 } 575 576 if (!error) { 577 mapping->nrpages += nr; 578 if (PageTransHuge(page)) 579 __inc_node_page_state(page, NR_SHMEM_THPS); 580 __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr); 581 __mod_node_page_state(page_pgdat(page), NR_SHMEM, nr); 582 spin_unlock_irq(&mapping->tree_lock); 583 } else { 584 page->mapping = NULL; 585 spin_unlock_irq(&mapping->tree_lock); 586 page_ref_sub(page, nr); 587 } 588 return error; 589 } 590 591 /* 592 * Like delete_from_page_cache, but substitutes swap for page. 593 */ 594 static void shmem_delete_from_page_cache(struct page *page, void *radswap) 595 { 596 struct address_space *mapping = page->mapping; 597 int error; 598 599 VM_BUG_ON_PAGE(PageCompound(page), page); 600 601 spin_lock_irq(&mapping->tree_lock); 602 error = shmem_radix_tree_replace(mapping, page->index, page, radswap); 603 page->mapping = NULL; 604 mapping->nrpages--; 605 __dec_node_page_state(page, NR_FILE_PAGES); 606 __dec_node_page_state(page, NR_SHMEM); 607 spin_unlock_irq(&mapping->tree_lock); 608 put_page(page); 609 BUG_ON(error); 610 } 611 612 /* 613 * Remove swap entry from radix tree, free the swap and its page cache. 614 */ 615 static int shmem_free_swap(struct address_space *mapping, 616 pgoff_t index, void *radswap) 617 { 618 void *old; 619 620 spin_lock_irq(&mapping->tree_lock); 621 old = radix_tree_delete_item(&mapping->page_tree, index, radswap); 622 spin_unlock_irq(&mapping->tree_lock); 623 if (old != radswap) 624 return -ENOENT; 625 free_swap_and_cache(radix_to_swp_entry(radswap)); 626 return 0; 627 } 628 629 /* 630 * Determine (in bytes) how many of the shmem object's pages mapped by the 631 * given offsets are swapped out. 632 * 633 * This is safe to call without i_mutex or mapping->tree_lock thanks to RCU, 634 * as long as the inode doesn't go away and racy results are not a problem. 635 */ 636 unsigned long shmem_partial_swap_usage(struct address_space *mapping, 637 pgoff_t start, pgoff_t end) 638 { 639 struct radix_tree_iter iter; 640 void **slot; 641 struct page *page; 642 unsigned long swapped = 0; 643 644 rcu_read_lock(); 645 646 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { 647 if (iter.index >= end) 648 break; 649 650 page = radix_tree_deref_slot(slot); 651 652 if (radix_tree_deref_retry(page)) { 653 slot = radix_tree_iter_retry(&iter); 654 continue; 655 } 656 657 if (radix_tree_exceptional_entry(page)) 658 swapped++; 659 660 if (need_resched()) { 661 cond_resched_rcu(); 662 slot = radix_tree_iter_next(&iter); 663 } 664 } 665 666 rcu_read_unlock(); 667 668 return swapped << PAGE_SHIFT; 669 } 670 671 /* 672 * Determine (in bytes) how many of the shmem object's pages mapped by the 673 * given vma is swapped out. 674 * 675 * This is safe to call without i_mutex or mapping->tree_lock thanks to RCU, 676 * as long as the inode doesn't go away and racy results are not a problem. 677 */ 678 unsigned long shmem_swap_usage(struct vm_area_struct *vma) 679 { 680 struct inode *inode = file_inode(vma->vm_file); 681 struct shmem_inode_info *info = SHMEM_I(inode); 682 struct address_space *mapping = inode->i_mapping; 683 unsigned long swapped; 684 685 /* Be careful as we don't hold info->lock */ 686 swapped = READ_ONCE(info->swapped); 687 688 /* 689 * The easier cases are when the shmem object has nothing in swap, or 690 * the vma maps it whole. Then we can simply use the stats that we 691 * already track. 692 */ 693 if (!swapped) 694 return 0; 695 696 if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size) 697 return swapped << PAGE_SHIFT; 698 699 /* Here comes the more involved part */ 700 return shmem_partial_swap_usage(mapping, 701 linear_page_index(vma, vma->vm_start), 702 linear_page_index(vma, vma->vm_end)); 703 } 704 705 /* 706 * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists. 707 */ 708 void shmem_unlock_mapping(struct address_space *mapping) 709 { 710 struct pagevec pvec; 711 pgoff_t indices[PAGEVEC_SIZE]; 712 pgoff_t index = 0; 713 714 pagevec_init(&pvec, 0); 715 /* 716 * Minor point, but we might as well stop if someone else SHM_LOCKs it. 717 */ 718 while (!mapping_unevictable(mapping)) { 719 /* 720 * Avoid pagevec_lookup(): find_get_pages() returns 0 as if it 721 * has finished, if it hits a row of PAGEVEC_SIZE swap entries. 722 */ 723 pvec.nr = find_get_entries(mapping, index, 724 PAGEVEC_SIZE, pvec.pages, indices); 725 if (!pvec.nr) 726 break; 727 index = indices[pvec.nr - 1] + 1; 728 pagevec_remove_exceptionals(&pvec); 729 check_move_unevictable_pages(pvec.pages, pvec.nr); 730 pagevec_release(&pvec); 731 cond_resched(); 732 } 733 } 734 735 /* 736 * Remove range of pages and swap entries from radix tree, and free them. 737 * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate. 738 */ 739 static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, 740 bool unfalloc) 741 { 742 struct address_space *mapping = inode->i_mapping; 743 struct shmem_inode_info *info = SHMEM_I(inode); 744 pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT; 745 pgoff_t end = (lend + 1) >> PAGE_SHIFT; 746 unsigned int partial_start = lstart & (PAGE_SIZE - 1); 747 unsigned int partial_end = (lend + 1) & (PAGE_SIZE - 1); 748 struct pagevec pvec; 749 pgoff_t indices[PAGEVEC_SIZE]; 750 long nr_swaps_freed = 0; 751 pgoff_t index; 752 int i; 753 754 if (lend == -1) 755 end = -1; /* unsigned, so actually very big */ 756 757 pagevec_init(&pvec, 0); 758 index = start; 759 while (index < end) { 760 pvec.nr = find_get_entries(mapping, index, 761 min(end - index, (pgoff_t)PAGEVEC_SIZE), 762 pvec.pages, indices); 763 if (!pvec.nr) 764 break; 765 for (i = 0; i < pagevec_count(&pvec); i++) { 766 struct page *page = pvec.pages[i]; 767 768 index = indices[i]; 769 if (index >= end) 770 break; 771 772 if (radix_tree_exceptional_entry(page)) { 773 if (unfalloc) 774 continue; 775 nr_swaps_freed += !shmem_free_swap(mapping, 776 index, page); 777 continue; 778 } 779 780 VM_BUG_ON_PAGE(page_to_pgoff(page) != index, page); 781 782 if (!trylock_page(page)) 783 continue; 784 785 if (PageTransTail(page)) { 786 /* Middle of THP: zero out the page */ 787 clear_highpage(page); 788 unlock_page(page); 789 continue; 790 } else if (PageTransHuge(page)) { 791 if (index == round_down(end, HPAGE_PMD_NR)) { 792 /* 793 * Range ends in the middle of THP: 794 * zero out the page 795 */ 796 clear_highpage(page); 797 unlock_page(page); 798 continue; 799 } 800 index += HPAGE_PMD_NR - 1; 801 i += HPAGE_PMD_NR - 1; 802 } 803 804 if (!unfalloc || !PageUptodate(page)) { 805 VM_BUG_ON_PAGE(PageTail(page), page); 806 if (page_mapping(page) == mapping) { 807 VM_BUG_ON_PAGE(PageWriteback(page), page); 808 truncate_inode_page(mapping, page); 809 } 810 } 811 unlock_page(page); 812 } 813 pagevec_remove_exceptionals(&pvec); 814 pagevec_release(&pvec); 815 cond_resched(); 816 index++; 817 } 818 819 if (partial_start) { 820 struct page *page = NULL; 821 shmem_getpage(inode, start - 1, &page, SGP_READ); 822 if (page) { 823 unsigned int top = PAGE_SIZE; 824 if (start > end) { 825 top = partial_end; 826 partial_end = 0; 827 } 828 zero_user_segment(page, partial_start, top); 829 set_page_dirty(page); 830 unlock_page(page); 831 put_page(page); 832 } 833 } 834 if (partial_end) { 835 struct page *page = NULL; 836 shmem_getpage(inode, end, &page, SGP_READ); 837 if (page) { 838 zero_user_segment(page, 0, partial_end); 839 set_page_dirty(page); 840 unlock_page(page); 841 put_page(page); 842 } 843 } 844 if (start >= end) 845 return; 846 847 index = start; 848 while (index < end) { 849 cond_resched(); 850 851 pvec.nr = find_get_entries(mapping, index, 852 min(end - index, (pgoff_t)PAGEVEC_SIZE), 853 pvec.pages, indices); 854 if (!pvec.nr) { 855 /* If all gone or hole-punch or unfalloc, we're done */ 856 if (index == start || end != -1) 857 break; 858 /* But if truncating, restart to make sure all gone */ 859 index = start; 860 continue; 861 } 862 for (i = 0; i < pagevec_count(&pvec); i++) { 863 struct page *page = pvec.pages[i]; 864 865 index = indices[i]; 866 if (index >= end) 867 break; 868 869 if (radix_tree_exceptional_entry(page)) { 870 if (unfalloc) 871 continue; 872 if (shmem_free_swap(mapping, index, page)) { 873 /* Swap was replaced by page: retry */ 874 index--; 875 break; 876 } 877 nr_swaps_freed++; 878 continue; 879 } 880 881 lock_page(page); 882 883 if (PageTransTail(page)) { 884 /* Middle of THP: zero out the page */ 885 clear_highpage(page); 886 unlock_page(page); 887 /* 888 * Partial thp truncate due 'start' in middle 889 * of THP: don't need to look on these pages 890 * again on !pvec.nr restart. 891 */ 892 if (index != round_down(end, HPAGE_PMD_NR)) 893 start++; 894 continue; 895 } else if (PageTransHuge(page)) { 896 if (index == round_down(end, HPAGE_PMD_NR)) { 897 /* 898 * Range ends in the middle of THP: 899 * zero out the page 900 */ 901 clear_highpage(page); 902 unlock_page(page); 903 continue; 904 } 905 index += HPAGE_PMD_NR - 1; 906 i += HPAGE_PMD_NR - 1; 907 } 908 909 if (!unfalloc || !PageUptodate(page)) { 910 VM_BUG_ON_PAGE(PageTail(page), page); 911 if (page_mapping(page) == mapping) { 912 VM_BUG_ON_PAGE(PageWriteback(page), page); 913 truncate_inode_page(mapping, page); 914 } else { 915 /* Page was replaced by swap: retry */ 916 unlock_page(page); 917 index--; 918 break; 919 } 920 } 921 unlock_page(page); 922 } 923 pagevec_remove_exceptionals(&pvec); 924 pagevec_release(&pvec); 925 index++; 926 } 927 928 spin_lock_irq(&info->lock); 929 info->swapped -= nr_swaps_freed; 930 shmem_recalc_inode(inode); 931 spin_unlock_irq(&info->lock); 932 } 933 934 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) 935 { 936 shmem_undo_range(inode, lstart, lend, false); 937 inode->i_ctime = inode->i_mtime = CURRENT_TIME; 938 } 939 EXPORT_SYMBOL_GPL(shmem_truncate_range); 940 941 static int shmem_getattr(struct vfsmount *mnt, struct dentry *dentry, 942 struct kstat *stat) 943 { 944 struct inode *inode = dentry->d_inode; 945 struct shmem_inode_info *info = SHMEM_I(inode); 946 947 if (info->alloced - info->swapped != inode->i_mapping->nrpages) { 948 spin_lock_irq(&info->lock); 949 shmem_recalc_inode(inode); 950 spin_unlock_irq(&info->lock); 951 } 952 generic_fillattr(inode, stat); 953 return 0; 954 } 955 956 static int shmem_setattr(struct dentry *dentry, struct iattr *attr) 957 { 958 struct inode *inode = d_inode(dentry); 959 struct shmem_inode_info *info = SHMEM_I(inode); 960 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 961 int error; 962 963 error = inode_change_ok(inode, attr); 964 if (error) 965 return error; 966 967 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) { 968 loff_t oldsize = inode->i_size; 969 loff_t newsize = attr->ia_size; 970 971 /* protected by i_mutex */ 972 if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) || 973 (newsize > oldsize && (info->seals & F_SEAL_GROW))) 974 return -EPERM; 975 976 if (newsize != oldsize) { 977 error = shmem_reacct_size(SHMEM_I(inode)->flags, 978 oldsize, newsize); 979 if (error) 980 return error; 981 i_size_write(inode, newsize); 982 inode->i_ctime = inode->i_mtime = CURRENT_TIME; 983 } 984 if (newsize <= oldsize) { 985 loff_t holebegin = round_up(newsize, PAGE_SIZE); 986 if (oldsize > holebegin) 987 unmap_mapping_range(inode->i_mapping, 988 holebegin, 0, 1); 989 if (info->alloced) 990 shmem_truncate_range(inode, 991 newsize, (loff_t)-1); 992 /* unmap again to remove racily COWed private pages */ 993 if (oldsize > holebegin) 994 unmap_mapping_range(inode->i_mapping, 995 holebegin, 0, 1); 996 997 /* 998 * Part of the huge page can be beyond i_size: subject 999 * to shrink under memory pressure. 1000 */ 1001 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) { 1002 spin_lock(&sbinfo->shrinklist_lock); 1003 if (list_empty(&info->shrinklist)) { 1004 list_add_tail(&info->shrinklist, 1005 &sbinfo->shrinklist); 1006 sbinfo->shrinklist_len++; 1007 } 1008 spin_unlock(&sbinfo->shrinklist_lock); 1009 } 1010 } 1011 } 1012 1013 setattr_copy(inode, attr); 1014 if (attr->ia_valid & ATTR_MODE) 1015 error = posix_acl_chmod(inode, inode->i_mode); 1016 return error; 1017 } 1018 1019 static void shmem_evict_inode(struct inode *inode) 1020 { 1021 struct shmem_inode_info *info = SHMEM_I(inode); 1022 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 1023 1024 if (inode->i_mapping->a_ops == &shmem_aops) { 1025 shmem_unacct_size(info->flags, inode->i_size); 1026 inode->i_size = 0; 1027 shmem_truncate_range(inode, 0, (loff_t)-1); 1028 if (!list_empty(&info->shrinklist)) { 1029 spin_lock(&sbinfo->shrinklist_lock); 1030 if (!list_empty(&info->shrinklist)) { 1031 list_del_init(&info->shrinklist); 1032 sbinfo->shrinklist_len--; 1033 } 1034 spin_unlock(&sbinfo->shrinklist_lock); 1035 } 1036 if (!list_empty(&info->swaplist)) { 1037 mutex_lock(&shmem_swaplist_mutex); 1038 list_del_init(&info->swaplist); 1039 mutex_unlock(&shmem_swaplist_mutex); 1040 } 1041 } 1042 1043 simple_xattrs_free(&info->xattrs); 1044 WARN_ON(inode->i_blocks); 1045 shmem_free_inode(inode->i_sb); 1046 clear_inode(inode); 1047 } 1048 1049 /* 1050 * If swap found in inode, free it and move page from swapcache to filecache. 1051 */ 1052 static int shmem_unuse_inode(struct shmem_inode_info *info, 1053 swp_entry_t swap, struct page **pagep) 1054 { 1055 struct address_space *mapping = info->vfs_inode.i_mapping; 1056 void *radswap; 1057 pgoff_t index; 1058 gfp_t gfp; 1059 int error = 0; 1060 1061 radswap = swp_to_radix_entry(swap); 1062 index = radix_tree_locate_item(&mapping->page_tree, radswap); 1063 if (index == -1) 1064 return -EAGAIN; /* tell shmem_unuse we found nothing */ 1065 1066 /* 1067 * Move _head_ to start search for next from here. 1068 * But be careful: shmem_evict_inode checks list_empty without taking 1069 * mutex, and there's an instant in list_move_tail when info->swaplist 1070 * would appear empty, if it were the only one on shmem_swaplist. 1071 */ 1072 if (shmem_swaplist.next != &info->swaplist) 1073 list_move_tail(&shmem_swaplist, &info->swaplist); 1074 1075 gfp = mapping_gfp_mask(mapping); 1076 if (shmem_should_replace_page(*pagep, gfp)) { 1077 mutex_unlock(&shmem_swaplist_mutex); 1078 error = shmem_replace_page(pagep, gfp, info, index); 1079 mutex_lock(&shmem_swaplist_mutex); 1080 /* 1081 * We needed to drop mutex to make that restrictive page 1082 * allocation, but the inode might have been freed while we 1083 * dropped it: although a racing shmem_evict_inode() cannot 1084 * complete without emptying the radix_tree, our page lock 1085 * on this swapcache page is not enough to prevent that - 1086 * free_swap_and_cache() of our swap entry will only 1087 * trylock_page(), removing swap from radix_tree whatever. 1088 * 1089 * We must not proceed to shmem_add_to_page_cache() if the 1090 * inode has been freed, but of course we cannot rely on 1091 * inode or mapping or info to check that. However, we can 1092 * safely check if our swap entry is still in use (and here 1093 * it can't have got reused for another page): if it's still 1094 * in use, then the inode cannot have been freed yet, and we 1095 * can safely proceed (if it's no longer in use, that tells 1096 * nothing about the inode, but we don't need to unuse swap). 1097 */ 1098 if (!page_swapcount(*pagep)) 1099 error = -ENOENT; 1100 } 1101 1102 /* 1103 * We rely on shmem_swaplist_mutex, not only to protect the swaplist, 1104 * but also to hold up shmem_evict_inode(): so inode cannot be freed 1105 * beneath us (pagelock doesn't help until the page is in pagecache). 1106 */ 1107 if (!error) 1108 error = shmem_add_to_page_cache(*pagep, mapping, index, 1109 radswap); 1110 if (error != -ENOMEM) { 1111 /* 1112 * Truncation and eviction use free_swap_and_cache(), which 1113 * only does trylock page: if we raced, best clean up here. 1114 */ 1115 delete_from_swap_cache(*pagep); 1116 set_page_dirty(*pagep); 1117 if (!error) { 1118 spin_lock_irq(&info->lock); 1119 info->swapped--; 1120 spin_unlock_irq(&info->lock); 1121 swap_free(swap); 1122 } 1123 } 1124 return error; 1125 } 1126 1127 /* 1128 * Search through swapped inodes to find and replace swap by page. 1129 */ 1130 int shmem_unuse(swp_entry_t swap, struct page *page) 1131 { 1132 struct list_head *this, *next; 1133 struct shmem_inode_info *info; 1134 struct mem_cgroup *memcg; 1135 int error = 0; 1136 1137 /* 1138 * There's a faint possibility that swap page was replaced before 1139 * caller locked it: caller will come back later with the right page. 1140 */ 1141 if (unlikely(!PageSwapCache(page) || page_private(page) != swap.val)) 1142 goto out; 1143 1144 /* 1145 * Charge page using GFP_KERNEL while we can wait, before taking 1146 * the shmem_swaplist_mutex which might hold up shmem_writepage(). 1147 * Charged back to the user (not to caller) when swap account is used. 1148 */ 1149 error = mem_cgroup_try_charge(page, current->mm, GFP_KERNEL, &memcg, 1150 false); 1151 if (error) 1152 goto out; 1153 /* No radix_tree_preload: swap entry keeps a place for page in tree */ 1154 error = -EAGAIN; 1155 1156 mutex_lock(&shmem_swaplist_mutex); 1157 list_for_each_safe(this, next, &shmem_swaplist) { 1158 info = list_entry(this, struct shmem_inode_info, swaplist); 1159 if (info->swapped) 1160 error = shmem_unuse_inode(info, swap, &page); 1161 else 1162 list_del_init(&info->swaplist); 1163 cond_resched(); 1164 if (error != -EAGAIN) 1165 break; 1166 /* found nothing in this: move on to search the next */ 1167 } 1168 mutex_unlock(&shmem_swaplist_mutex); 1169 1170 if (error) { 1171 if (error != -ENOMEM) 1172 error = 0; 1173 mem_cgroup_cancel_charge(page, memcg, false); 1174 } else 1175 mem_cgroup_commit_charge(page, memcg, true, false); 1176 out: 1177 unlock_page(page); 1178 put_page(page); 1179 return error; 1180 } 1181 1182 /* 1183 * Move the page from the page cache to the swap cache. 1184 */ 1185 static int shmem_writepage(struct page *page, struct writeback_control *wbc) 1186 { 1187 struct shmem_inode_info *info; 1188 struct address_space *mapping; 1189 struct inode *inode; 1190 swp_entry_t swap; 1191 pgoff_t index; 1192 1193 VM_BUG_ON_PAGE(PageCompound(page), page); 1194 BUG_ON(!PageLocked(page)); 1195 mapping = page->mapping; 1196 index = page->index; 1197 inode = mapping->host; 1198 info = SHMEM_I(inode); 1199 if (info->flags & VM_LOCKED) 1200 goto redirty; 1201 if (!total_swap_pages) 1202 goto redirty; 1203 1204 /* 1205 * Our capabilities prevent regular writeback or sync from ever calling 1206 * shmem_writepage; but a stacking filesystem might use ->writepage of 1207 * its underlying filesystem, in which case tmpfs should write out to 1208 * swap only in response to memory pressure, and not for the writeback 1209 * threads or sync. 1210 */ 1211 if (!wbc->for_reclaim) { 1212 WARN_ON_ONCE(1); /* Still happens? Tell us about it! */ 1213 goto redirty; 1214 } 1215 1216 /* 1217 * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC 1218 * value into swapfile.c, the only way we can correctly account for a 1219 * fallocated page arriving here is now to initialize it and write it. 1220 * 1221 * That's okay for a page already fallocated earlier, but if we have 1222 * not yet completed the fallocation, then (a) we want to keep track 1223 * of this page in case we have to undo it, and (b) it may not be a 1224 * good idea to continue anyway, once we're pushing into swap. So 1225 * reactivate the page, and let shmem_fallocate() quit when too many. 1226 */ 1227 if (!PageUptodate(page)) { 1228 if (inode->i_private) { 1229 struct shmem_falloc *shmem_falloc; 1230 spin_lock(&inode->i_lock); 1231 shmem_falloc = inode->i_private; 1232 if (shmem_falloc && 1233 !shmem_falloc->waitq && 1234 index >= shmem_falloc->start && 1235 index < shmem_falloc->next) 1236 shmem_falloc->nr_unswapped++; 1237 else 1238 shmem_falloc = NULL; 1239 spin_unlock(&inode->i_lock); 1240 if (shmem_falloc) 1241 goto redirty; 1242 } 1243 clear_highpage(page); 1244 flush_dcache_page(page); 1245 SetPageUptodate(page); 1246 } 1247 1248 swap = get_swap_page(); 1249 if (!swap.val) 1250 goto redirty; 1251 1252 if (mem_cgroup_try_charge_swap(page, swap)) 1253 goto free_swap; 1254 1255 /* 1256 * Add inode to shmem_unuse()'s list of swapped-out inodes, 1257 * if it's not already there. Do it now before the page is 1258 * moved to swap cache, when its pagelock no longer protects 1259 * the inode from eviction. But don't unlock the mutex until 1260 * we've incremented swapped, because shmem_unuse_inode() will 1261 * prune a !swapped inode from the swaplist under this mutex. 1262 */ 1263 mutex_lock(&shmem_swaplist_mutex); 1264 if (list_empty(&info->swaplist)) 1265 list_add_tail(&info->swaplist, &shmem_swaplist); 1266 1267 if (add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) { 1268 spin_lock_irq(&info->lock); 1269 shmem_recalc_inode(inode); 1270 info->swapped++; 1271 spin_unlock_irq(&info->lock); 1272 1273 swap_shmem_alloc(swap); 1274 shmem_delete_from_page_cache(page, swp_to_radix_entry(swap)); 1275 1276 mutex_unlock(&shmem_swaplist_mutex); 1277 BUG_ON(page_mapped(page)); 1278 swap_writepage(page, wbc); 1279 return 0; 1280 } 1281 1282 mutex_unlock(&shmem_swaplist_mutex); 1283 free_swap: 1284 swapcache_free(swap); 1285 redirty: 1286 set_page_dirty(page); 1287 if (wbc->for_reclaim) 1288 return AOP_WRITEPAGE_ACTIVATE; /* Return with page locked */ 1289 unlock_page(page); 1290 return 0; 1291 } 1292 1293 #if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS) 1294 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) 1295 { 1296 char buffer[64]; 1297 1298 if (!mpol || mpol->mode == MPOL_DEFAULT) 1299 return; /* show nothing */ 1300 1301 mpol_to_str(buffer, sizeof(buffer), mpol); 1302 1303 seq_printf(seq, ",mpol=%s", buffer); 1304 } 1305 1306 static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) 1307 { 1308 struct mempolicy *mpol = NULL; 1309 if (sbinfo->mpol) { 1310 spin_lock(&sbinfo->stat_lock); /* prevent replace/use races */ 1311 mpol = sbinfo->mpol; 1312 mpol_get(mpol); 1313 spin_unlock(&sbinfo->stat_lock); 1314 } 1315 return mpol; 1316 } 1317 #else /* !CONFIG_NUMA || !CONFIG_TMPFS */ 1318 static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) 1319 { 1320 } 1321 static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) 1322 { 1323 return NULL; 1324 } 1325 #endif /* CONFIG_NUMA && CONFIG_TMPFS */ 1326 #ifndef CONFIG_NUMA 1327 #define vm_policy vm_private_data 1328 #endif 1329 1330 static void shmem_pseudo_vma_init(struct vm_area_struct *vma, 1331 struct shmem_inode_info *info, pgoff_t index) 1332 { 1333 /* Create a pseudo vma that just contains the policy */ 1334 vma->vm_start = 0; 1335 /* Bias interleave by inode number to distribute better across nodes */ 1336 vma->vm_pgoff = index + info->vfs_inode.i_ino; 1337 vma->vm_ops = NULL; 1338 vma->vm_policy = mpol_shared_policy_lookup(&info->policy, index); 1339 } 1340 1341 static void shmem_pseudo_vma_destroy(struct vm_area_struct *vma) 1342 { 1343 /* Drop reference taken by mpol_shared_policy_lookup() */ 1344 mpol_cond_put(vma->vm_policy); 1345 } 1346 1347 static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp, 1348 struct shmem_inode_info *info, pgoff_t index) 1349 { 1350 struct vm_area_struct pvma; 1351 struct page *page; 1352 1353 shmem_pseudo_vma_init(&pvma, info, index); 1354 page = swapin_readahead(swap, gfp, &pvma, 0); 1355 shmem_pseudo_vma_destroy(&pvma); 1356 1357 return page; 1358 } 1359 1360 static struct page *shmem_alloc_hugepage(gfp_t gfp, 1361 struct shmem_inode_info *info, pgoff_t index) 1362 { 1363 struct vm_area_struct pvma; 1364 struct inode *inode = &info->vfs_inode; 1365 struct address_space *mapping = inode->i_mapping; 1366 pgoff_t idx, hindex; 1367 void __rcu **results; 1368 struct page *page; 1369 1370 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) 1371 return NULL; 1372 1373 hindex = round_down(index, HPAGE_PMD_NR); 1374 rcu_read_lock(); 1375 if (radix_tree_gang_lookup_slot(&mapping->page_tree, &results, &idx, 1376 hindex, 1) && idx < hindex + HPAGE_PMD_NR) { 1377 rcu_read_unlock(); 1378 return NULL; 1379 } 1380 rcu_read_unlock(); 1381 1382 shmem_pseudo_vma_init(&pvma, info, hindex); 1383 page = alloc_pages_vma(gfp | __GFP_COMP | __GFP_NORETRY | __GFP_NOWARN, 1384 HPAGE_PMD_ORDER, &pvma, 0, numa_node_id(), true); 1385 shmem_pseudo_vma_destroy(&pvma); 1386 if (page) 1387 prep_transhuge_page(page); 1388 return page; 1389 } 1390 1391 static struct page *shmem_alloc_page(gfp_t gfp, 1392 struct shmem_inode_info *info, pgoff_t index) 1393 { 1394 struct vm_area_struct pvma; 1395 struct page *page; 1396 1397 shmem_pseudo_vma_init(&pvma, info, index); 1398 page = alloc_page_vma(gfp, &pvma, 0); 1399 shmem_pseudo_vma_destroy(&pvma); 1400 1401 return page; 1402 } 1403 1404 static struct page *shmem_alloc_and_acct_page(gfp_t gfp, 1405 struct shmem_inode_info *info, struct shmem_sb_info *sbinfo, 1406 pgoff_t index, bool huge) 1407 { 1408 struct page *page; 1409 int nr; 1410 int err = -ENOSPC; 1411 1412 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) 1413 huge = false; 1414 nr = huge ? HPAGE_PMD_NR : 1; 1415 1416 if (shmem_acct_block(info->flags, nr)) 1417 goto failed; 1418 if (sbinfo->max_blocks) { 1419 if (percpu_counter_compare(&sbinfo->used_blocks, 1420 sbinfo->max_blocks - nr) > 0) 1421 goto unacct; 1422 percpu_counter_add(&sbinfo->used_blocks, nr); 1423 } 1424 1425 if (huge) 1426 page = shmem_alloc_hugepage(gfp, info, index); 1427 else 1428 page = shmem_alloc_page(gfp, info, index); 1429 if (page) { 1430 __SetPageLocked(page); 1431 __SetPageSwapBacked(page); 1432 return page; 1433 } 1434 1435 err = -ENOMEM; 1436 if (sbinfo->max_blocks) 1437 percpu_counter_add(&sbinfo->used_blocks, -nr); 1438 unacct: 1439 shmem_unacct_blocks(info->flags, nr); 1440 failed: 1441 return ERR_PTR(err); 1442 } 1443 1444 /* 1445 * When a page is moved from swapcache to shmem filecache (either by the 1446 * usual swapin of shmem_getpage_gfp(), or by the less common swapoff of 1447 * shmem_unuse_inode()), it may have been read in earlier from swap, in 1448 * ignorance of the mapping it belongs to. If that mapping has special 1449 * constraints (like the gma500 GEM driver, which requires RAM below 4GB), 1450 * we may need to copy to a suitable page before moving to filecache. 1451 * 1452 * In a future release, this may well be extended to respect cpuset and 1453 * NUMA mempolicy, and applied also to anonymous pages in do_swap_page(); 1454 * but for now it is a simple matter of zone. 1455 */ 1456 static bool shmem_should_replace_page(struct page *page, gfp_t gfp) 1457 { 1458 return page_zonenum(page) > gfp_zone(gfp); 1459 } 1460 1461 static int shmem_replace_page(struct page **pagep, gfp_t gfp, 1462 struct shmem_inode_info *info, pgoff_t index) 1463 { 1464 struct page *oldpage, *newpage; 1465 struct address_space *swap_mapping; 1466 pgoff_t swap_index; 1467 int error; 1468 1469 oldpage = *pagep; 1470 swap_index = page_private(oldpage); 1471 swap_mapping = page_mapping(oldpage); 1472 1473 /* 1474 * We have arrived here because our zones are constrained, so don't 1475 * limit chance of success by further cpuset and node constraints. 1476 */ 1477 gfp &= ~GFP_CONSTRAINT_MASK; 1478 newpage = shmem_alloc_page(gfp, info, index); 1479 if (!newpage) 1480 return -ENOMEM; 1481 1482 get_page(newpage); 1483 copy_highpage(newpage, oldpage); 1484 flush_dcache_page(newpage); 1485 1486 __SetPageLocked(newpage); 1487 __SetPageSwapBacked(newpage); 1488 SetPageUptodate(newpage); 1489 set_page_private(newpage, swap_index); 1490 SetPageSwapCache(newpage); 1491 1492 /* 1493 * Our caller will very soon move newpage out of swapcache, but it's 1494 * a nice clean interface for us to replace oldpage by newpage there. 1495 */ 1496 spin_lock_irq(&swap_mapping->tree_lock); 1497 error = shmem_radix_tree_replace(swap_mapping, swap_index, oldpage, 1498 newpage); 1499 if (!error) { 1500 __inc_node_page_state(newpage, NR_FILE_PAGES); 1501 __dec_node_page_state(oldpage, NR_FILE_PAGES); 1502 } 1503 spin_unlock_irq(&swap_mapping->tree_lock); 1504 1505 if (unlikely(error)) { 1506 /* 1507 * Is this possible? I think not, now that our callers check 1508 * both PageSwapCache and page_private after getting page lock; 1509 * but be defensive. Reverse old to newpage for clear and free. 1510 */ 1511 oldpage = newpage; 1512 } else { 1513 mem_cgroup_migrate(oldpage, newpage); 1514 lru_cache_add_anon(newpage); 1515 *pagep = newpage; 1516 } 1517 1518 ClearPageSwapCache(oldpage); 1519 set_page_private(oldpage, 0); 1520 1521 unlock_page(oldpage); 1522 put_page(oldpage); 1523 put_page(oldpage); 1524 return error; 1525 } 1526 1527 /* 1528 * shmem_getpage_gfp - find page in cache, or get from swap, or allocate 1529 * 1530 * If we allocate a new one we do not mark it dirty. That's up to the 1531 * vm. If we swap it in we mark it dirty since we also free the swap 1532 * entry since a page cannot live in both the swap and page cache. 1533 * 1534 * fault_mm and fault_type are only supplied by shmem_fault: 1535 * otherwise they are NULL. 1536 */ 1537 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, 1538 struct page **pagep, enum sgp_type sgp, gfp_t gfp, 1539 struct mm_struct *fault_mm, int *fault_type) 1540 { 1541 struct address_space *mapping = inode->i_mapping; 1542 struct shmem_inode_info *info; 1543 struct shmem_sb_info *sbinfo; 1544 struct mm_struct *charge_mm; 1545 struct mem_cgroup *memcg; 1546 struct page *page; 1547 swp_entry_t swap; 1548 enum sgp_type sgp_huge = sgp; 1549 pgoff_t hindex = index; 1550 int error; 1551 int once = 0; 1552 int alloced = 0; 1553 1554 if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT)) 1555 return -EFBIG; 1556 if (sgp == SGP_NOHUGE || sgp == SGP_HUGE) 1557 sgp = SGP_CACHE; 1558 repeat: 1559 swap.val = 0; 1560 page = find_lock_entry(mapping, index); 1561 if (radix_tree_exceptional_entry(page)) { 1562 swap = radix_to_swp_entry(page); 1563 page = NULL; 1564 } 1565 1566 if (sgp <= SGP_CACHE && 1567 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) { 1568 error = -EINVAL; 1569 goto unlock; 1570 } 1571 1572 if (page && sgp == SGP_WRITE) 1573 mark_page_accessed(page); 1574 1575 /* fallocated page? */ 1576 if (page && !PageUptodate(page)) { 1577 if (sgp != SGP_READ) 1578 goto clear; 1579 unlock_page(page); 1580 put_page(page); 1581 page = NULL; 1582 } 1583 if (page || (sgp == SGP_READ && !swap.val)) { 1584 *pagep = page; 1585 return 0; 1586 } 1587 1588 /* 1589 * Fast cache lookup did not find it: 1590 * bring it back from swap or allocate. 1591 */ 1592 info = SHMEM_I(inode); 1593 sbinfo = SHMEM_SB(inode->i_sb); 1594 charge_mm = fault_mm ? : current->mm; 1595 1596 if (swap.val) { 1597 /* Look it up and read it in.. */ 1598 page = lookup_swap_cache(swap); 1599 if (!page) { 1600 /* Or update major stats only when swapin succeeds?? */ 1601 if (fault_type) { 1602 *fault_type |= VM_FAULT_MAJOR; 1603 count_vm_event(PGMAJFAULT); 1604 mem_cgroup_count_vm_event(fault_mm, PGMAJFAULT); 1605 } 1606 /* Here we actually start the io */ 1607 page = shmem_swapin(swap, gfp, info, index); 1608 if (!page) { 1609 error = -ENOMEM; 1610 goto failed; 1611 } 1612 } 1613 1614 /* We have to do this with page locked to prevent races */ 1615 lock_page(page); 1616 if (!PageSwapCache(page) || page_private(page) != swap.val || 1617 !shmem_confirm_swap(mapping, index, swap)) { 1618 error = -EEXIST; /* try again */ 1619 goto unlock; 1620 } 1621 if (!PageUptodate(page)) { 1622 error = -EIO; 1623 goto failed; 1624 } 1625 wait_on_page_writeback(page); 1626 1627 if (shmem_should_replace_page(page, gfp)) { 1628 error = shmem_replace_page(&page, gfp, info, index); 1629 if (error) 1630 goto failed; 1631 } 1632 1633 error = mem_cgroup_try_charge(page, charge_mm, gfp, &memcg, 1634 false); 1635 if (!error) { 1636 error = shmem_add_to_page_cache(page, mapping, index, 1637 swp_to_radix_entry(swap)); 1638 /* 1639 * We already confirmed swap under page lock, and make 1640 * no memory allocation here, so usually no possibility 1641 * of error; but free_swap_and_cache() only trylocks a 1642 * page, so it is just possible that the entry has been 1643 * truncated or holepunched since swap was confirmed. 1644 * shmem_undo_range() will have done some of the 1645 * unaccounting, now delete_from_swap_cache() will do 1646 * the rest. 1647 * Reset swap.val? No, leave it so "failed" goes back to 1648 * "repeat": reading a hole and writing should succeed. 1649 */ 1650 if (error) { 1651 mem_cgroup_cancel_charge(page, memcg, false); 1652 delete_from_swap_cache(page); 1653 } 1654 } 1655 if (error) 1656 goto failed; 1657 1658 mem_cgroup_commit_charge(page, memcg, true, false); 1659 1660 spin_lock_irq(&info->lock); 1661 info->swapped--; 1662 shmem_recalc_inode(inode); 1663 spin_unlock_irq(&info->lock); 1664 1665 if (sgp == SGP_WRITE) 1666 mark_page_accessed(page); 1667 1668 delete_from_swap_cache(page); 1669 set_page_dirty(page); 1670 swap_free(swap); 1671 1672 } else { 1673 /* shmem_symlink() */ 1674 if (mapping->a_ops != &shmem_aops) 1675 goto alloc_nohuge; 1676 if (shmem_huge == SHMEM_HUGE_DENY || sgp_huge == SGP_NOHUGE) 1677 goto alloc_nohuge; 1678 if (shmem_huge == SHMEM_HUGE_FORCE) 1679 goto alloc_huge; 1680 switch (sbinfo->huge) { 1681 loff_t i_size; 1682 pgoff_t off; 1683 case SHMEM_HUGE_NEVER: 1684 goto alloc_nohuge; 1685 case SHMEM_HUGE_WITHIN_SIZE: 1686 off = round_up(index, HPAGE_PMD_NR); 1687 i_size = round_up(i_size_read(inode), PAGE_SIZE); 1688 if (i_size >= HPAGE_PMD_SIZE && 1689 i_size >> PAGE_SHIFT >= off) 1690 goto alloc_huge; 1691 /* fallthrough */ 1692 case SHMEM_HUGE_ADVISE: 1693 if (sgp_huge == SGP_HUGE) 1694 goto alloc_huge; 1695 /* TODO: implement fadvise() hints */ 1696 goto alloc_nohuge; 1697 } 1698 1699 alloc_huge: 1700 page = shmem_alloc_and_acct_page(gfp, info, sbinfo, 1701 index, true); 1702 if (IS_ERR(page)) { 1703 alloc_nohuge: page = shmem_alloc_and_acct_page(gfp, info, sbinfo, 1704 index, false); 1705 } 1706 if (IS_ERR(page)) { 1707 int retry = 5; 1708 error = PTR_ERR(page); 1709 page = NULL; 1710 if (error != -ENOSPC) 1711 goto failed; 1712 /* 1713 * Try to reclaim some spece by splitting a huge page 1714 * beyond i_size on the filesystem. 1715 */ 1716 while (retry--) { 1717 int ret; 1718 ret = shmem_unused_huge_shrink(sbinfo, NULL, 1); 1719 if (ret == SHRINK_STOP) 1720 break; 1721 if (ret) 1722 goto alloc_nohuge; 1723 } 1724 goto failed; 1725 } 1726 1727 if (PageTransHuge(page)) 1728 hindex = round_down(index, HPAGE_PMD_NR); 1729 else 1730 hindex = index; 1731 1732 if (sgp == SGP_WRITE) 1733 __SetPageReferenced(page); 1734 1735 error = mem_cgroup_try_charge(page, charge_mm, gfp, &memcg, 1736 PageTransHuge(page)); 1737 if (error) 1738 goto unacct; 1739 error = radix_tree_maybe_preload_order(gfp & GFP_RECLAIM_MASK, 1740 compound_order(page)); 1741 if (!error) { 1742 error = shmem_add_to_page_cache(page, mapping, hindex, 1743 NULL); 1744 radix_tree_preload_end(); 1745 } 1746 if (error) { 1747 mem_cgroup_cancel_charge(page, memcg, 1748 PageTransHuge(page)); 1749 goto unacct; 1750 } 1751 mem_cgroup_commit_charge(page, memcg, false, 1752 PageTransHuge(page)); 1753 lru_cache_add_anon(page); 1754 1755 spin_lock_irq(&info->lock); 1756 info->alloced += 1 << compound_order(page); 1757 inode->i_blocks += BLOCKS_PER_PAGE << compound_order(page); 1758 shmem_recalc_inode(inode); 1759 spin_unlock_irq(&info->lock); 1760 alloced = true; 1761 1762 if (PageTransHuge(page) && 1763 DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) < 1764 hindex + HPAGE_PMD_NR - 1) { 1765 /* 1766 * Part of the huge page is beyond i_size: subject 1767 * to shrink under memory pressure. 1768 */ 1769 spin_lock(&sbinfo->shrinklist_lock); 1770 if (list_empty(&info->shrinklist)) { 1771 list_add_tail(&info->shrinklist, 1772 &sbinfo->shrinklist); 1773 sbinfo->shrinklist_len++; 1774 } 1775 spin_unlock(&sbinfo->shrinklist_lock); 1776 } 1777 1778 /* 1779 * Let SGP_FALLOC use the SGP_WRITE optimization on a new page. 1780 */ 1781 if (sgp == SGP_FALLOC) 1782 sgp = SGP_WRITE; 1783 clear: 1784 /* 1785 * Let SGP_WRITE caller clear ends if write does not fill page; 1786 * but SGP_FALLOC on a page fallocated earlier must initialize 1787 * it now, lest undo on failure cancel our earlier guarantee. 1788 */ 1789 if (sgp != SGP_WRITE && !PageUptodate(page)) { 1790 struct page *head = compound_head(page); 1791 int i; 1792 1793 for (i = 0; i < (1 << compound_order(head)); i++) { 1794 clear_highpage(head + i); 1795 flush_dcache_page(head + i); 1796 } 1797 SetPageUptodate(head); 1798 } 1799 } 1800 1801 /* Perhaps the file has been truncated since we checked */ 1802 if (sgp <= SGP_CACHE && 1803 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) { 1804 if (alloced) { 1805 ClearPageDirty(page); 1806 delete_from_page_cache(page); 1807 spin_lock_irq(&info->lock); 1808 shmem_recalc_inode(inode); 1809 spin_unlock_irq(&info->lock); 1810 } 1811 error = -EINVAL; 1812 goto unlock; 1813 } 1814 *pagep = page + index - hindex; 1815 return 0; 1816 1817 /* 1818 * Error recovery. 1819 */ 1820 unacct: 1821 if (sbinfo->max_blocks) 1822 percpu_counter_sub(&sbinfo->used_blocks, 1823 1 << compound_order(page)); 1824 shmem_unacct_blocks(info->flags, 1 << compound_order(page)); 1825 1826 if (PageTransHuge(page)) { 1827 unlock_page(page); 1828 put_page(page); 1829 goto alloc_nohuge; 1830 } 1831 failed: 1832 if (swap.val && !shmem_confirm_swap(mapping, index, swap)) 1833 error = -EEXIST; 1834 unlock: 1835 if (page) { 1836 unlock_page(page); 1837 put_page(page); 1838 } 1839 if (error == -ENOSPC && !once++) { 1840 info = SHMEM_I(inode); 1841 spin_lock_irq(&info->lock); 1842 shmem_recalc_inode(inode); 1843 spin_unlock_irq(&info->lock); 1844 goto repeat; 1845 } 1846 if (error == -EEXIST) /* from above or from radix_tree_insert */ 1847 goto repeat; 1848 return error; 1849 } 1850 1851 static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 1852 { 1853 struct inode *inode = file_inode(vma->vm_file); 1854 gfp_t gfp = mapping_gfp_mask(inode->i_mapping); 1855 enum sgp_type sgp; 1856 int error; 1857 int ret = VM_FAULT_LOCKED; 1858 1859 /* 1860 * Trinity finds that probing a hole which tmpfs is punching can 1861 * prevent the hole-punch from ever completing: which in turn 1862 * locks writers out with its hold on i_mutex. So refrain from 1863 * faulting pages into the hole while it's being punched. Although 1864 * shmem_undo_range() does remove the additions, it may be unable to 1865 * keep up, as each new page needs its own unmap_mapping_range() call, 1866 * and the i_mmap tree grows ever slower to scan if new vmas are added. 1867 * 1868 * It does not matter if we sometimes reach this check just before the 1869 * hole-punch begins, so that one fault then races with the punch: 1870 * we just need to make racing faults a rare case. 1871 * 1872 * The implementation below would be much simpler if we just used a 1873 * standard mutex or completion: but we cannot take i_mutex in fault, 1874 * and bloating every shmem inode for this unlikely case would be sad. 1875 */ 1876 if (unlikely(inode->i_private)) { 1877 struct shmem_falloc *shmem_falloc; 1878 1879 spin_lock(&inode->i_lock); 1880 shmem_falloc = inode->i_private; 1881 if (shmem_falloc && 1882 shmem_falloc->waitq && 1883 vmf->pgoff >= shmem_falloc->start && 1884 vmf->pgoff < shmem_falloc->next) { 1885 wait_queue_head_t *shmem_falloc_waitq; 1886 DEFINE_WAIT(shmem_fault_wait); 1887 1888 ret = VM_FAULT_NOPAGE; 1889 if ((vmf->flags & FAULT_FLAG_ALLOW_RETRY) && 1890 !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) { 1891 /* It's polite to up mmap_sem if we can */ 1892 up_read(&vma->vm_mm->mmap_sem); 1893 ret = VM_FAULT_RETRY; 1894 } 1895 1896 shmem_falloc_waitq = shmem_falloc->waitq; 1897 prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait, 1898 TASK_UNINTERRUPTIBLE); 1899 spin_unlock(&inode->i_lock); 1900 schedule(); 1901 1902 /* 1903 * shmem_falloc_waitq points into the shmem_fallocate() 1904 * stack of the hole-punching task: shmem_falloc_waitq 1905 * is usually invalid by the time we reach here, but 1906 * finish_wait() does not dereference it in that case; 1907 * though i_lock needed lest racing with wake_up_all(). 1908 */ 1909 spin_lock(&inode->i_lock); 1910 finish_wait(shmem_falloc_waitq, &shmem_fault_wait); 1911 spin_unlock(&inode->i_lock); 1912 return ret; 1913 } 1914 spin_unlock(&inode->i_lock); 1915 } 1916 1917 sgp = SGP_CACHE; 1918 if (vma->vm_flags & VM_HUGEPAGE) 1919 sgp = SGP_HUGE; 1920 else if (vma->vm_flags & VM_NOHUGEPAGE) 1921 sgp = SGP_NOHUGE; 1922 1923 error = shmem_getpage_gfp(inode, vmf->pgoff, &vmf->page, sgp, 1924 gfp, vma->vm_mm, &ret); 1925 if (error) 1926 return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS); 1927 return ret; 1928 } 1929 1930 unsigned long shmem_get_unmapped_area(struct file *file, 1931 unsigned long uaddr, unsigned long len, 1932 unsigned long pgoff, unsigned long flags) 1933 { 1934 unsigned long (*get_area)(struct file *, 1935 unsigned long, unsigned long, unsigned long, unsigned long); 1936 unsigned long addr; 1937 unsigned long offset; 1938 unsigned long inflated_len; 1939 unsigned long inflated_addr; 1940 unsigned long inflated_offset; 1941 1942 if (len > TASK_SIZE) 1943 return -ENOMEM; 1944 1945 get_area = current->mm->get_unmapped_area; 1946 addr = get_area(file, uaddr, len, pgoff, flags); 1947 1948 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) 1949 return addr; 1950 if (IS_ERR_VALUE(addr)) 1951 return addr; 1952 if (addr & ~PAGE_MASK) 1953 return addr; 1954 if (addr > TASK_SIZE - len) 1955 return addr; 1956 1957 if (shmem_huge == SHMEM_HUGE_DENY) 1958 return addr; 1959 if (len < HPAGE_PMD_SIZE) 1960 return addr; 1961 if (flags & MAP_FIXED) 1962 return addr; 1963 /* 1964 * Our priority is to support MAP_SHARED mapped hugely; 1965 * and support MAP_PRIVATE mapped hugely too, until it is COWed. 1966 * But if caller specified an address hint, respect that as before. 1967 */ 1968 if (uaddr) 1969 return addr; 1970 1971 if (shmem_huge != SHMEM_HUGE_FORCE) { 1972 struct super_block *sb; 1973 1974 if (file) { 1975 VM_BUG_ON(file->f_op != &shmem_file_operations); 1976 sb = file_inode(file)->i_sb; 1977 } else { 1978 /* 1979 * Called directly from mm/mmap.c, or drivers/char/mem.c 1980 * for "/dev/zero", to create a shared anonymous object. 1981 */ 1982 if (IS_ERR(shm_mnt)) 1983 return addr; 1984 sb = shm_mnt->mnt_sb; 1985 } 1986 if (SHMEM_SB(sb)->huge == SHMEM_HUGE_NEVER) 1987 return addr; 1988 } 1989 1990 offset = (pgoff << PAGE_SHIFT) & (HPAGE_PMD_SIZE-1); 1991 if (offset && offset + len < 2 * HPAGE_PMD_SIZE) 1992 return addr; 1993 if ((addr & (HPAGE_PMD_SIZE-1)) == offset) 1994 return addr; 1995 1996 inflated_len = len + HPAGE_PMD_SIZE - PAGE_SIZE; 1997 if (inflated_len > TASK_SIZE) 1998 return addr; 1999 if (inflated_len < len) 2000 return addr; 2001 2002 inflated_addr = get_area(NULL, 0, inflated_len, 0, flags); 2003 if (IS_ERR_VALUE(inflated_addr)) 2004 return addr; 2005 if (inflated_addr & ~PAGE_MASK) 2006 return addr; 2007 2008 inflated_offset = inflated_addr & (HPAGE_PMD_SIZE-1); 2009 inflated_addr += offset - inflated_offset; 2010 if (inflated_offset > offset) 2011 inflated_addr += HPAGE_PMD_SIZE; 2012 2013 if (inflated_addr > TASK_SIZE - len) 2014 return addr; 2015 return inflated_addr; 2016 } 2017 2018 #ifdef CONFIG_NUMA 2019 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol) 2020 { 2021 struct inode *inode = file_inode(vma->vm_file); 2022 return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol); 2023 } 2024 2025 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma, 2026 unsigned long addr) 2027 { 2028 struct inode *inode = file_inode(vma->vm_file); 2029 pgoff_t index; 2030 2031 index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; 2032 return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index); 2033 } 2034 #endif 2035 2036 int shmem_lock(struct file *file, int lock, struct user_struct *user) 2037 { 2038 struct inode *inode = file_inode(file); 2039 struct shmem_inode_info *info = SHMEM_I(inode); 2040 int retval = -ENOMEM; 2041 2042 spin_lock_irq(&info->lock); 2043 if (lock && !(info->flags & VM_LOCKED)) { 2044 if (!user_shm_lock(inode->i_size, user)) 2045 goto out_nomem; 2046 info->flags |= VM_LOCKED; 2047 mapping_set_unevictable(file->f_mapping); 2048 } 2049 if (!lock && (info->flags & VM_LOCKED) && user) { 2050 user_shm_unlock(inode->i_size, user); 2051 info->flags &= ~VM_LOCKED; 2052 mapping_clear_unevictable(file->f_mapping); 2053 } 2054 retval = 0; 2055 2056 out_nomem: 2057 spin_unlock_irq(&info->lock); 2058 return retval; 2059 } 2060 2061 static int shmem_mmap(struct file *file, struct vm_area_struct *vma) 2062 { 2063 file_accessed(file); 2064 vma->vm_ops = &shmem_vm_ops; 2065 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && 2066 ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) < 2067 (vma->vm_end & HPAGE_PMD_MASK)) { 2068 khugepaged_enter(vma, vma->vm_flags); 2069 } 2070 return 0; 2071 } 2072 2073 static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir, 2074 umode_t mode, dev_t dev, unsigned long flags) 2075 { 2076 struct inode *inode; 2077 struct shmem_inode_info *info; 2078 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 2079 2080 if (shmem_reserve_inode(sb)) 2081 return NULL; 2082 2083 inode = new_inode(sb); 2084 if (inode) { 2085 inode->i_ino = get_next_ino(); 2086 inode_init_owner(inode, dir, mode); 2087 inode->i_blocks = 0; 2088 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME; 2089 inode->i_generation = get_seconds(); 2090 info = SHMEM_I(inode); 2091 memset(info, 0, (char *)inode - (char *)info); 2092 spin_lock_init(&info->lock); 2093 info->seals = F_SEAL_SEAL; 2094 info->flags = flags & VM_NORESERVE; 2095 INIT_LIST_HEAD(&info->shrinklist); 2096 INIT_LIST_HEAD(&info->swaplist); 2097 simple_xattrs_init(&info->xattrs); 2098 cache_no_acl(inode); 2099 2100 switch (mode & S_IFMT) { 2101 default: 2102 inode->i_op = &shmem_special_inode_operations; 2103 init_special_inode(inode, mode, dev); 2104 break; 2105 case S_IFREG: 2106 inode->i_mapping->a_ops = &shmem_aops; 2107 inode->i_op = &shmem_inode_operations; 2108 inode->i_fop = &shmem_file_operations; 2109 mpol_shared_policy_init(&info->policy, 2110 shmem_get_sbmpol(sbinfo)); 2111 break; 2112 case S_IFDIR: 2113 inc_nlink(inode); 2114 /* Some things misbehave if size == 0 on a directory */ 2115 inode->i_size = 2 * BOGO_DIRENT_SIZE; 2116 inode->i_op = &shmem_dir_inode_operations; 2117 inode->i_fop = &simple_dir_operations; 2118 break; 2119 case S_IFLNK: 2120 /* 2121 * Must not load anything in the rbtree, 2122 * mpol_free_shared_policy will not be called. 2123 */ 2124 mpol_shared_policy_init(&info->policy, NULL); 2125 break; 2126 } 2127 } else 2128 shmem_free_inode(sb); 2129 return inode; 2130 } 2131 2132 bool shmem_mapping(struct address_space *mapping) 2133 { 2134 if (!mapping->host) 2135 return false; 2136 2137 return mapping->host->i_sb->s_op == &shmem_ops; 2138 } 2139 2140 #ifdef CONFIG_TMPFS 2141 static const struct inode_operations shmem_symlink_inode_operations; 2142 static const struct inode_operations shmem_short_symlink_operations; 2143 2144 #ifdef CONFIG_TMPFS_XATTR 2145 static int shmem_initxattrs(struct inode *, const struct xattr *, void *); 2146 #else 2147 #define shmem_initxattrs NULL 2148 #endif 2149 2150 static int 2151 shmem_write_begin(struct file *file, struct address_space *mapping, 2152 loff_t pos, unsigned len, unsigned flags, 2153 struct page **pagep, void **fsdata) 2154 { 2155 struct inode *inode = mapping->host; 2156 struct shmem_inode_info *info = SHMEM_I(inode); 2157 pgoff_t index = pos >> PAGE_SHIFT; 2158 2159 /* i_mutex is held by caller */ 2160 if (unlikely(info->seals)) { 2161 if (info->seals & F_SEAL_WRITE) 2162 return -EPERM; 2163 if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size) 2164 return -EPERM; 2165 } 2166 2167 return shmem_getpage(inode, index, pagep, SGP_WRITE); 2168 } 2169 2170 static int 2171 shmem_write_end(struct file *file, struct address_space *mapping, 2172 loff_t pos, unsigned len, unsigned copied, 2173 struct page *page, void *fsdata) 2174 { 2175 struct inode *inode = mapping->host; 2176 2177 if (pos + copied > inode->i_size) 2178 i_size_write(inode, pos + copied); 2179 2180 if (!PageUptodate(page)) { 2181 struct page *head = compound_head(page); 2182 if (PageTransCompound(page)) { 2183 int i; 2184 2185 for (i = 0; i < HPAGE_PMD_NR; i++) { 2186 if (head + i == page) 2187 continue; 2188 clear_highpage(head + i); 2189 flush_dcache_page(head + i); 2190 } 2191 } 2192 if (copied < PAGE_SIZE) { 2193 unsigned from = pos & (PAGE_SIZE - 1); 2194 zero_user_segments(page, 0, from, 2195 from + copied, PAGE_SIZE); 2196 } 2197 SetPageUptodate(head); 2198 } 2199 set_page_dirty(page); 2200 unlock_page(page); 2201 put_page(page); 2202 2203 return copied; 2204 } 2205 2206 static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to) 2207 { 2208 struct file *file = iocb->ki_filp; 2209 struct inode *inode = file_inode(file); 2210 struct address_space *mapping = inode->i_mapping; 2211 pgoff_t index; 2212 unsigned long offset; 2213 enum sgp_type sgp = SGP_READ; 2214 int error = 0; 2215 ssize_t retval = 0; 2216 loff_t *ppos = &iocb->ki_pos; 2217 2218 /* 2219 * Might this read be for a stacking filesystem? Then when reading 2220 * holes of a sparse file, we actually need to allocate those pages, 2221 * and even mark them dirty, so it cannot exceed the max_blocks limit. 2222 */ 2223 if (!iter_is_iovec(to)) 2224 sgp = SGP_CACHE; 2225 2226 index = *ppos >> PAGE_SHIFT; 2227 offset = *ppos & ~PAGE_MASK; 2228 2229 for (;;) { 2230 struct page *page = NULL; 2231 pgoff_t end_index; 2232 unsigned long nr, ret; 2233 loff_t i_size = i_size_read(inode); 2234 2235 end_index = i_size >> PAGE_SHIFT; 2236 if (index > end_index) 2237 break; 2238 if (index == end_index) { 2239 nr = i_size & ~PAGE_MASK; 2240 if (nr <= offset) 2241 break; 2242 } 2243 2244 error = shmem_getpage(inode, index, &page, sgp); 2245 if (error) { 2246 if (error == -EINVAL) 2247 error = 0; 2248 break; 2249 } 2250 if (page) { 2251 if (sgp == SGP_CACHE) 2252 set_page_dirty(page); 2253 unlock_page(page); 2254 } 2255 2256 /* 2257 * We must evaluate after, since reads (unlike writes) 2258 * are called without i_mutex protection against truncate 2259 */ 2260 nr = PAGE_SIZE; 2261 i_size = i_size_read(inode); 2262 end_index = i_size >> PAGE_SHIFT; 2263 if (index == end_index) { 2264 nr = i_size & ~PAGE_MASK; 2265 if (nr <= offset) { 2266 if (page) 2267 put_page(page); 2268 break; 2269 } 2270 } 2271 nr -= offset; 2272 2273 if (page) { 2274 /* 2275 * If users can be writing to this page using arbitrary 2276 * virtual addresses, take care about potential aliasing 2277 * before reading the page on the kernel side. 2278 */ 2279 if (mapping_writably_mapped(mapping)) 2280 flush_dcache_page(page); 2281 /* 2282 * Mark the page accessed if we read the beginning. 2283 */ 2284 if (!offset) 2285 mark_page_accessed(page); 2286 } else { 2287 page = ZERO_PAGE(0); 2288 get_page(page); 2289 } 2290 2291 /* 2292 * Ok, we have the page, and it's up-to-date, so 2293 * now we can copy it to user space... 2294 */ 2295 ret = copy_page_to_iter(page, offset, nr, to); 2296 retval += ret; 2297 offset += ret; 2298 index += offset >> PAGE_SHIFT; 2299 offset &= ~PAGE_MASK; 2300 2301 put_page(page); 2302 if (!iov_iter_count(to)) 2303 break; 2304 if (ret < nr) { 2305 error = -EFAULT; 2306 break; 2307 } 2308 cond_resched(); 2309 } 2310 2311 *ppos = ((loff_t) index << PAGE_SHIFT) + offset; 2312 file_accessed(file); 2313 return retval ? retval : error; 2314 } 2315 2316 static ssize_t shmem_file_splice_read(struct file *in, loff_t *ppos, 2317 struct pipe_inode_info *pipe, size_t len, 2318 unsigned int flags) 2319 { 2320 struct address_space *mapping = in->f_mapping; 2321 struct inode *inode = mapping->host; 2322 unsigned int loff, nr_pages, req_pages; 2323 struct page *pages[PIPE_DEF_BUFFERS]; 2324 struct partial_page partial[PIPE_DEF_BUFFERS]; 2325 struct page *page; 2326 pgoff_t index, end_index; 2327 loff_t isize, left; 2328 int error, page_nr; 2329 struct splice_pipe_desc spd = { 2330 .pages = pages, 2331 .partial = partial, 2332 .nr_pages_max = PIPE_DEF_BUFFERS, 2333 .flags = flags, 2334 .ops = &page_cache_pipe_buf_ops, 2335 .spd_release = spd_release_page, 2336 }; 2337 2338 isize = i_size_read(inode); 2339 if (unlikely(*ppos >= isize)) 2340 return 0; 2341 2342 left = isize - *ppos; 2343 if (unlikely(left < len)) 2344 len = left; 2345 2346 if (splice_grow_spd(pipe, &spd)) 2347 return -ENOMEM; 2348 2349 index = *ppos >> PAGE_SHIFT; 2350 loff = *ppos & ~PAGE_MASK; 2351 req_pages = (len + loff + PAGE_SIZE - 1) >> PAGE_SHIFT; 2352 nr_pages = min(req_pages, spd.nr_pages_max); 2353 2354 spd.nr_pages = find_get_pages_contig(mapping, index, 2355 nr_pages, spd.pages); 2356 index += spd.nr_pages; 2357 error = 0; 2358 2359 while (spd.nr_pages < nr_pages) { 2360 error = shmem_getpage(inode, index, &page, SGP_CACHE); 2361 if (error) 2362 break; 2363 unlock_page(page); 2364 spd.pages[spd.nr_pages++] = page; 2365 index++; 2366 } 2367 2368 index = *ppos >> PAGE_SHIFT; 2369 nr_pages = spd.nr_pages; 2370 spd.nr_pages = 0; 2371 2372 for (page_nr = 0; page_nr < nr_pages; page_nr++) { 2373 unsigned int this_len; 2374 2375 if (!len) 2376 break; 2377 2378 this_len = min_t(unsigned long, len, PAGE_SIZE - loff); 2379 page = spd.pages[page_nr]; 2380 2381 if (!PageUptodate(page) || page->mapping != mapping) { 2382 error = shmem_getpage(inode, index, &page, SGP_CACHE); 2383 if (error) 2384 break; 2385 unlock_page(page); 2386 put_page(spd.pages[page_nr]); 2387 spd.pages[page_nr] = page; 2388 } 2389 2390 isize = i_size_read(inode); 2391 end_index = (isize - 1) >> PAGE_SHIFT; 2392 if (unlikely(!isize || index > end_index)) 2393 break; 2394 2395 if (end_index == index) { 2396 unsigned int plen; 2397 2398 plen = ((isize - 1) & ~PAGE_MASK) + 1; 2399 if (plen <= loff) 2400 break; 2401 2402 this_len = min(this_len, plen - loff); 2403 len = this_len; 2404 } 2405 2406 spd.partial[page_nr].offset = loff; 2407 spd.partial[page_nr].len = this_len; 2408 len -= this_len; 2409 loff = 0; 2410 spd.nr_pages++; 2411 index++; 2412 } 2413 2414 while (page_nr < nr_pages) 2415 put_page(spd.pages[page_nr++]); 2416 2417 if (spd.nr_pages) 2418 error = splice_to_pipe(pipe, &spd); 2419 2420 splice_shrink_spd(&spd); 2421 2422 if (error > 0) { 2423 *ppos += error; 2424 file_accessed(in); 2425 } 2426 return error; 2427 } 2428 2429 /* 2430 * llseek SEEK_DATA or SEEK_HOLE through the radix_tree. 2431 */ 2432 static pgoff_t shmem_seek_hole_data(struct address_space *mapping, 2433 pgoff_t index, pgoff_t end, int whence) 2434 { 2435 struct page *page; 2436 struct pagevec pvec; 2437 pgoff_t indices[PAGEVEC_SIZE]; 2438 bool done = false; 2439 int i; 2440 2441 pagevec_init(&pvec, 0); 2442 pvec.nr = 1; /* start small: we may be there already */ 2443 while (!done) { 2444 pvec.nr = find_get_entries(mapping, index, 2445 pvec.nr, pvec.pages, indices); 2446 if (!pvec.nr) { 2447 if (whence == SEEK_DATA) 2448 index = end; 2449 break; 2450 } 2451 for (i = 0; i < pvec.nr; i++, index++) { 2452 if (index < indices[i]) { 2453 if (whence == SEEK_HOLE) { 2454 done = true; 2455 break; 2456 } 2457 index = indices[i]; 2458 } 2459 page = pvec.pages[i]; 2460 if (page && !radix_tree_exceptional_entry(page)) { 2461 if (!PageUptodate(page)) 2462 page = NULL; 2463 } 2464 if (index >= end || 2465 (page && whence == SEEK_DATA) || 2466 (!page && whence == SEEK_HOLE)) { 2467 done = true; 2468 break; 2469 } 2470 } 2471 pagevec_remove_exceptionals(&pvec); 2472 pagevec_release(&pvec); 2473 pvec.nr = PAGEVEC_SIZE; 2474 cond_resched(); 2475 } 2476 return index; 2477 } 2478 2479 static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence) 2480 { 2481 struct address_space *mapping = file->f_mapping; 2482 struct inode *inode = mapping->host; 2483 pgoff_t start, end; 2484 loff_t new_offset; 2485 2486 if (whence != SEEK_DATA && whence != SEEK_HOLE) 2487 return generic_file_llseek_size(file, offset, whence, 2488 MAX_LFS_FILESIZE, i_size_read(inode)); 2489 inode_lock(inode); 2490 /* We're holding i_mutex so we can access i_size directly */ 2491 2492 if (offset < 0) 2493 offset = -EINVAL; 2494 else if (offset >= inode->i_size) 2495 offset = -ENXIO; 2496 else { 2497 start = offset >> PAGE_SHIFT; 2498 end = (inode->i_size + PAGE_SIZE - 1) >> PAGE_SHIFT; 2499 new_offset = shmem_seek_hole_data(mapping, start, end, whence); 2500 new_offset <<= PAGE_SHIFT; 2501 if (new_offset > offset) { 2502 if (new_offset < inode->i_size) 2503 offset = new_offset; 2504 else if (whence == SEEK_DATA) 2505 offset = -ENXIO; 2506 else 2507 offset = inode->i_size; 2508 } 2509 } 2510 2511 if (offset >= 0) 2512 offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE); 2513 inode_unlock(inode); 2514 return offset; 2515 } 2516 2517 /* 2518 * We need a tag: a new tag would expand every radix_tree_node by 8 bytes, 2519 * so reuse a tag which we firmly believe is never set or cleared on shmem. 2520 */ 2521 #define SHMEM_TAG_PINNED PAGECACHE_TAG_TOWRITE 2522 #define LAST_SCAN 4 /* about 150ms max */ 2523 2524 static void shmem_tag_pins(struct address_space *mapping) 2525 { 2526 struct radix_tree_iter iter; 2527 void **slot; 2528 pgoff_t start; 2529 struct page *page; 2530 2531 lru_add_drain(); 2532 start = 0; 2533 rcu_read_lock(); 2534 2535 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { 2536 page = radix_tree_deref_slot(slot); 2537 if (!page || radix_tree_exception(page)) { 2538 if (radix_tree_deref_retry(page)) { 2539 slot = radix_tree_iter_retry(&iter); 2540 continue; 2541 } 2542 } else if (page_count(page) - page_mapcount(page) > 1) { 2543 spin_lock_irq(&mapping->tree_lock); 2544 radix_tree_tag_set(&mapping->page_tree, iter.index, 2545 SHMEM_TAG_PINNED); 2546 spin_unlock_irq(&mapping->tree_lock); 2547 } 2548 2549 if (need_resched()) { 2550 cond_resched_rcu(); 2551 slot = radix_tree_iter_next(&iter); 2552 } 2553 } 2554 rcu_read_unlock(); 2555 } 2556 2557 /* 2558 * Setting SEAL_WRITE requires us to verify there's no pending writer. However, 2559 * via get_user_pages(), drivers might have some pending I/O without any active 2560 * user-space mappings (eg., direct-IO, AIO). Therefore, we look at all pages 2561 * and see whether it has an elevated ref-count. If so, we tag them and wait for 2562 * them to be dropped. 2563 * The caller must guarantee that no new user will acquire writable references 2564 * to those pages to avoid races. 2565 */ 2566 static int shmem_wait_for_pins(struct address_space *mapping) 2567 { 2568 struct radix_tree_iter iter; 2569 void **slot; 2570 pgoff_t start; 2571 struct page *page; 2572 int error, scan; 2573 2574 shmem_tag_pins(mapping); 2575 2576 error = 0; 2577 for (scan = 0; scan <= LAST_SCAN; scan++) { 2578 if (!radix_tree_tagged(&mapping->page_tree, SHMEM_TAG_PINNED)) 2579 break; 2580 2581 if (!scan) 2582 lru_add_drain_all(); 2583 else if (schedule_timeout_killable((HZ << scan) / 200)) 2584 scan = LAST_SCAN; 2585 2586 start = 0; 2587 rcu_read_lock(); 2588 radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter, 2589 start, SHMEM_TAG_PINNED) { 2590 2591 page = radix_tree_deref_slot(slot); 2592 if (radix_tree_exception(page)) { 2593 if (radix_tree_deref_retry(page)) { 2594 slot = radix_tree_iter_retry(&iter); 2595 continue; 2596 } 2597 2598 page = NULL; 2599 } 2600 2601 if (page && 2602 page_count(page) - page_mapcount(page) != 1) { 2603 if (scan < LAST_SCAN) 2604 goto continue_resched; 2605 2606 /* 2607 * On the last scan, we clean up all those tags 2608 * we inserted; but make a note that we still 2609 * found pages pinned. 2610 */ 2611 error = -EBUSY; 2612 } 2613 2614 spin_lock_irq(&mapping->tree_lock); 2615 radix_tree_tag_clear(&mapping->page_tree, 2616 iter.index, SHMEM_TAG_PINNED); 2617 spin_unlock_irq(&mapping->tree_lock); 2618 continue_resched: 2619 if (need_resched()) { 2620 cond_resched_rcu(); 2621 slot = radix_tree_iter_next(&iter); 2622 } 2623 } 2624 rcu_read_unlock(); 2625 } 2626 2627 return error; 2628 } 2629 2630 #define F_ALL_SEALS (F_SEAL_SEAL | \ 2631 F_SEAL_SHRINK | \ 2632 F_SEAL_GROW | \ 2633 F_SEAL_WRITE) 2634 2635 int shmem_add_seals(struct file *file, unsigned int seals) 2636 { 2637 struct inode *inode = file_inode(file); 2638 struct shmem_inode_info *info = SHMEM_I(inode); 2639 int error; 2640 2641 /* 2642 * SEALING 2643 * Sealing allows multiple parties to share a shmem-file but restrict 2644 * access to a specific subset of file operations. Seals can only be 2645 * added, but never removed. This way, mutually untrusted parties can 2646 * share common memory regions with a well-defined policy. A malicious 2647 * peer can thus never perform unwanted operations on a shared object. 2648 * 2649 * Seals are only supported on special shmem-files and always affect 2650 * the whole underlying inode. Once a seal is set, it may prevent some 2651 * kinds of access to the file. Currently, the following seals are 2652 * defined: 2653 * SEAL_SEAL: Prevent further seals from being set on this file 2654 * SEAL_SHRINK: Prevent the file from shrinking 2655 * SEAL_GROW: Prevent the file from growing 2656 * SEAL_WRITE: Prevent write access to the file 2657 * 2658 * As we don't require any trust relationship between two parties, we 2659 * must prevent seals from being removed. Therefore, sealing a file 2660 * only adds a given set of seals to the file, it never touches 2661 * existing seals. Furthermore, the "setting seals"-operation can be 2662 * sealed itself, which basically prevents any further seal from being 2663 * added. 2664 * 2665 * Semantics of sealing are only defined on volatile files. Only 2666 * anonymous shmem files support sealing. More importantly, seals are 2667 * never written to disk. Therefore, there's no plan to support it on 2668 * other file types. 2669 */ 2670 2671 if (file->f_op != &shmem_file_operations) 2672 return -EINVAL; 2673 if (!(file->f_mode & FMODE_WRITE)) 2674 return -EPERM; 2675 if (seals & ~(unsigned int)F_ALL_SEALS) 2676 return -EINVAL; 2677 2678 inode_lock(inode); 2679 2680 if (info->seals & F_SEAL_SEAL) { 2681 error = -EPERM; 2682 goto unlock; 2683 } 2684 2685 if ((seals & F_SEAL_WRITE) && !(info->seals & F_SEAL_WRITE)) { 2686 error = mapping_deny_writable(file->f_mapping); 2687 if (error) 2688 goto unlock; 2689 2690 error = shmem_wait_for_pins(file->f_mapping); 2691 if (error) { 2692 mapping_allow_writable(file->f_mapping); 2693 goto unlock; 2694 } 2695 } 2696 2697 info->seals |= seals; 2698 error = 0; 2699 2700 unlock: 2701 inode_unlock(inode); 2702 return error; 2703 } 2704 EXPORT_SYMBOL_GPL(shmem_add_seals); 2705 2706 int shmem_get_seals(struct file *file) 2707 { 2708 if (file->f_op != &shmem_file_operations) 2709 return -EINVAL; 2710 2711 return SHMEM_I(file_inode(file))->seals; 2712 } 2713 EXPORT_SYMBOL_GPL(shmem_get_seals); 2714 2715 long shmem_fcntl(struct file *file, unsigned int cmd, unsigned long arg) 2716 { 2717 long error; 2718 2719 switch (cmd) { 2720 case F_ADD_SEALS: 2721 /* disallow upper 32bit */ 2722 if (arg > UINT_MAX) 2723 return -EINVAL; 2724 2725 error = shmem_add_seals(file, arg); 2726 break; 2727 case F_GET_SEALS: 2728 error = shmem_get_seals(file); 2729 break; 2730 default: 2731 error = -EINVAL; 2732 break; 2733 } 2734 2735 return error; 2736 } 2737 2738 static long shmem_fallocate(struct file *file, int mode, loff_t offset, 2739 loff_t len) 2740 { 2741 struct inode *inode = file_inode(file); 2742 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 2743 struct shmem_inode_info *info = SHMEM_I(inode); 2744 struct shmem_falloc shmem_falloc; 2745 pgoff_t start, index, end; 2746 int error; 2747 2748 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE)) 2749 return -EOPNOTSUPP; 2750 2751 inode_lock(inode); 2752 2753 if (mode & FALLOC_FL_PUNCH_HOLE) { 2754 struct address_space *mapping = file->f_mapping; 2755 loff_t unmap_start = round_up(offset, PAGE_SIZE); 2756 loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1; 2757 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq); 2758 2759 /* protected by i_mutex */ 2760 if (info->seals & F_SEAL_WRITE) { 2761 error = -EPERM; 2762 goto out; 2763 } 2764 2765 shmem_falloc.waitq = &shmem_falloc_waitq; 2766 shmem_falloc.start = unmap_start >> PAGE_SHIFT; 2767 shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT; 2768 spin_lock(&inode->i_lock); 2769 inode->i_private = &shmem_falloc; 2770 spin_unlock(&inode->i_lock); 2771 2772 if ((u64)unmap_end > (u64)unmap_start) 2773 unmap_mapping_range(mapping, unmap_start, 2774 1 + unmap_end - unmap_start, 0); 2775 shmem_truncate_range(inode, offset, offset + len - 1); 2776 /* No need to unmap again: hole-punching leaves COWed pages */ 2777 2778 spin_lock(&inode->i_lock); 2779 inode->i_private = NULL; 2780 wake_up_all(&shmem_falloc_waitq); 2781 spin_unlock(&inode->i_lock); 2782 error = 0; 2783 goto out; 2784 } 2785 2786 /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */ 2787 error = inode_newsize_ok(inode, offset + len); 2788 if (error) 2789 goto out; 2790 2791 if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) { 2792 error = -EPERM; 2793 goto out; 2794 } 2795 2796 start = offset >> PAGE_SHIFT; 2797 end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 2798 /* Try to avoid a swapstorm if len is impossible to satisfy */ 2799 if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) { 2800 error = -ENOSPC; 2801 goto out; 2802 } 2803 2804 shmem_falloc.waitq = NULL; 2805 shmem_falloc.start = start; 2806 shmem_falloc.next = start; 2807 shmem_falloc.nr_falloced = 0; 2808 shmem_falloc.nr_unswapped = 0; 2809 spin_lock(&inode->i_lock); 2810 inode->i_private = &shmem_falloc; 2811 spin_unlock(&inode->i_lock); 2812 2813 for (index = start; index < end; index++) { 2814 struct page *page; 2815 2816 /* 2817 * Good, the fallocate(2) manpage permits EINTR: we may have 2818 * been interrupted because we are using up too much memory. 2819 */ 2820 if (signal_pending(current)) 2821 error = -EINTR; 2822 else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced) 2823 error = -ENOMEM; 2824 else 2825 error = shmem_getpage(inode, index, &page, SGP_FALLOC); 2826 if (error) { 2827 /* Remove the !PageUptodate pages we added */ 2828 if (index > start) { 2829 shmem_undo_range(inode, 2830 (loff_t)start << PAGE_SHIFT, 2831 ((loff_t)index << PAGE_SHIFT) - 1, true); 2832 } 2833 goto undone; 2834 } 2835 2836 /* 2837 * Inform shmem_writepage() how far we have reached. 2838 * No need for lock or barrier: we have the page lock. 2839 */ 2840 shmem_falloc.next++; 2841 if (!PageUptodate(page)) 2842 shmem_falloc.nr_falloced++; 2843 2844 /* 2845 * If !PageUptodate, leave it that way so that freeable pages 2846 * can be recognized if we need to rollback on error later. 2847 * But set_page_dirty so that memory pressure will swap rather 2848 * than free the pages we are allocating (and SGP_CACHE pages 2849 * might still be clean: we now need to mark those dirty too). 2850 */ 2851 set_page_dirty(page); 2852 unlock_page(page); 2853 put_page(page); 2854 cond_resched(); 2855 } 2856 2857 if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size) 2858 i_size_write(inode, offset + len); 2859 inode->i_ctime = CURRENT_TIME; 2860 undone: 2861 spin_lock(&inode->i_lock); 2862 inode->i_private = NULL; 2863 spin_unlock(&inode->i_lock); 2864 out: 2865 inode_unlock(inode); 2866 return error; 2867 } 2868 2869 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf) 2870 { 2871 struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb); 2872 2873 buf->f_type = TMPFS_MAGIC; 2874 buf->f_bsize = PAGE_SIZE; 2875 buf->f_namelen = NAME_MAX; 2876 if (sbinfo->max_blocks) { 2877 buf->f_blocks = sbinfo->max_blocks; 2878 buf->f_bavail = 2879 buf->f_bfree = sbinfo->max_blocks - 2880 percpu_counter_sum(&sbinfo->used_blocks); 2881 } 2882 if (sbinfo->max_inodes) { 2883 buf->f_files = sbinfo->max_inodes; 2884 buf->f_ffree = sbinfo->free_inodes; 2885 } 2886 /* else leave those fields 0 like simple_statfs */ 2887 return 0; 2888 } 2889 2890 /* 2891 * File creation. Allocate an inode, and we're done.. 2892 */ 2893 static int 2894 shmem_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev) 2895 { 2896 struct inode *inode; 2897 int error = -ENOSPC; 2898 2899 inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE); 2900 if (inode) { 2901 error = simple_acl_create(dir, inode); 2902 if (error) 2903 goto out_iput; 2904 error = security_inode_init_security(inode, dir, 2905 &dentry->d_name, 2906 shmem_initxattrs, NULL); 2907 if (error && error != -EOPNOTSUPP) 2908 goto out_iput; 2909 2910 error = 0; 2911 dir->i_size += BOGO_DIRENT_SIZE; 2912 dir->i_ctime = dir->i_mtime = CURRENT_TIME; 2913 d_instantiate(dentry, inode); 2914 dget(dentry); /* Extra count - pin the dentry in core */ 2915 } 2916 return error; 2917 out_iput: 2918 iput(inode); 2919 return error; 2920 } 2921 2922 static int 2923 shmem_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode) 2924 { 2925 struct inode *inode; 2926 int error = -ENOSPC; 2927 2928 inode = shmem_get_inode(dir->i_sb, dir, mode, 0, VM_NORESERVE); 2929 if (inode) { 2930 error = security_inode_init_security(inode, dir, 2931 NULL, 2932 shmem_initxattrs, NULL); 2933 if (error && error != -EOPNOTSUPP) 2934 goto out_iput; 2935 error = simple_acl_create(dir, inode); 2936 if (error) 2937 goto out_iput; 2938 d_tmpfile(dentry, inode); 2939 } 2940 return error; 2941 out_iput: 2942 iput(inode); 2943 return error; 2944 } 2945 2946 static int shmem_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) 2947 { 2948 int error; 2949 2950 if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0))) 2951 return error; 2952 inc_nlink(dir); 2953 return 0; 2954 } 2955 2956 static int shmem_create(struct inode *dir, struct dentry *dentry, umode_t mode, 2957 bool excl) 2958 { 2959 return shmem_mknod(dir, dentry, mode | S_IFREG, 0); 2960 } 2961 2962 /* 2963 * Link a file.. 2964 */ 2965 static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry) 2966 { 2967 struct inode *inode = d_inode(old_dentry); 2968 int ret; 2969 2970 /* 2971 * No ordinary (disk based) filesystem counts links as inodes; 2972 * but each new link needs a new dentry, pinning lowmem, and 2973 * tmpfs dentries cannot be pruned until they are unlinked. 2974 */ 2975 ret = shmem_reserve_inode(inode->i_sb); 2976 if (ret) 2977 goto out; 2978 2979 dir->i_size += BOGO_DIRENT_SIZE; 2980 inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME; 2981 inc_nlink(inode); 2982 ihold(inode); /* New dentry reference */ 2983 dget(dentry); /* Extra pinning count for the created dentry */ 2984 d_instantiate(dentry, inode); 2985 out: 2986 return ret; 2987 } 2988 2989 static int shmem_unlink(struct inode *dir, struct dentry *dentry) 2990 { 2991 struct inode *inode = d_inode(dentry); 2992 2993 if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode)) 2994 shmem_free_inode(inode->i_sb); 2995 2996 dir->i_size -= BOGO_DIRENT_SIZE; 2997 inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME; 2998 drop_nlink(inode); 2999 dput(dentry); /* Undo the count from "create" - this does all the work */ 3000 return 0; 3001 } 3002 3003 static int shmem_rmdir(struct inode *dir, struct dentry *dentry) 3004 { 3005 if (!simple_empty(dentry)) 3006 return -ENOTEMPTY; 3007 3008 drop_nlink(d_inode(dentry)); 3009 drop_nlink(dir); 3010 return shmem_unlink(dir, dentry); 3011 } 3012 3013 static int shmem_exchange(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry) 3014 { 3015 bool old_is_dir = d_is_dir(old_dentry); 3016 bool new_is_dir = d_is_dir(new_dentry); 3017 3018 if (old_dir != new_dir && old_is_dir != new_is_dir) { 3019 if (old_is_dir) { 3020 drop_nlink(old_dir); 3021 inc_nlink(new_dir); 3022 } else { 3023 drop_nlink(new_dir); 3024 inc_nlink(old_dir); 3025 } 3026 } 3027 old_dir->i_ctime = old_dir->i_mtime = 3028 new_dir->i_ctime = new_dir->i_mtime = 3029 d_inode(old_dentry)->i_ctime = 3030 d_inode(new_dentry)->i_ctime = CURRENT_TIME; 3031 3032 return 0; 3033 } 3034 3035 static int shmem_whiteout(struct inode *old_dir, struct dentry *old_dentry) 3036 { 3037 struct dentry *whiteout; 3038 int error; 3039 3040 whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name); 3041 if (!whiteout) 3042 return -ENOMEM; 3043 3044 error = shmem_mknod(old_dir, whiteout, 3045 S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV); 3046 dput(whiteout); 3047 if (error) 3048 return error; 3049 3050 /* 3051 * Cheat and hash the whiteout while the old dentry is still in 3052 * place, instead of playing games with FS_RENAME_DOES_D_MOVE. 3053 * 3054 * d_lookup() will consistently find one of them at this point, 3055 * not sure which one, but that isn't even important. 3056 */ 3057 d_rehash(whiteout); 3058 return 0; 3059 } 3060 3061 /* 3062 * The VFS layer already does all the dentry stuff for rename, 3063 * we just have to decrement the usage count for the target if 3064 * it exists so that the VFS layer correctly free's it when it 3065 * gets overwritten. 3066 */ 3067 static int shmem_rename2(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags) 3068 { 3069 struct inode *inode = d_inode(old_dentry); 3070 int they_are_dirs = S_ISDIR(inode->i_mode); 3071 3072 if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) 3073 return -EINVAL; 3074 3075 if (flags & RENAME_EXCHANGE) 3076 return shmem_exchange(old_dir, old_dentry, new_dir, new_dentry); 3077 3078 if (!simple_empty(new_dentry)) 3079 return -ENOTEMPTY; 3080 3081 if (flags & RENAME_WHITEOUT) { 3082 int error; 3083 3084 error = shmem_whiteout(old_dir, old_dentry); 3085 if (error) 3086 return error; 3087 } 3088 3089 if (d_really_is_positive(new_dentry)) { 3090 (void) shmem_unlink(new_dir, new_dentry); 3091 if (they_are_dirs) { 3092 drop_nlink(d_inode(new_dentry)); 3093 drop_nlink(old_dir); 3094 } 3095 } else if (they_are_dirs) { 3096 drop_nlink(old_dir); 3097 inc_nlink(new_dir); 3098 } 3099 3100 old_dir->i_size -= BOGO_DIRENT_SIZE; 3101 new_dir->i_size += BOGO_DIRENT_SIZE; 3102 old_dir->i_ctime = old_dir->i_mtime = 3103 new_dir->i_ctime = new_dir->i_mtime = 3104 inode->i_ctime = CURRENT_TIME; 3105 return 0; 3106 } 3107 3108 static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname) 3109 { 3110 int error; 3111 int len; 3112 struct inode *inode; 3113 struct page *page; 3114 struct shmem_inode_info *info; 3115 3116 len = strlen(symname) + 1; 3117 if (len > PAGE_SIZE) 3118 return -ENAMETOOLONG; 3119 3120 inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0, VM_NORESERVE); 3121 if (!inode) 3122 return -ENOSPC; 3123 3124 error = security_inode_init_security(inode, dir, &dentry->d_name, 3125 shmem_initxattrs, NULL); 3126 if (error) { 3127 if (error != -EOPNOTSUPP) { 3128 iput(inode); 3129 return error; 3130 } 3131 error = 0; 3132 } 3133 3134 info = SHMEM_I(inode); 3135 inode->i_size = len-1; 3136 if (len <= SHORT_SYMLINK_LEN) { 3137 inode->i_link = kmemdup(symname, len, GFP_KERNEL); 3138 if (!inode->i_link) { 3139 iput(inode); 3140 return -ENOMEM; 3141 } 3142 inode->i_op = &shmem_short_symlink_operations; 3143 } else { 3144 inode_nohighmem(inode); 3145 error = shmem_getpage(inode, 0, &page, SGP_WRITE); 3146 if (error) { 3147 iput(inode); 3148 return error; 3149 } 3150 inode->i_mapping->a_ops = &shmem_aops; 3151 inode->i_op = &shmem_symlink_inode_operations; 3152 memcpy(page_address(page), symname, len); 3153 SetPageUptodate(page); 3154 set_page_dirty(page); 3155 unlock_page(page); 3156 put_page(page); 3157 } 3158 dir->i_size += BOGO_DIRENT_SIZE; 3159 dir->i_ctime = dir->i_mtime = CURRENT_TIME; 3160 d_instantiate(dentry, inode); 3161 dget(dentry); 3162 return 0; 3163 } 3164 3165 static void shmem_put_link(void *arg) 3166 { 3167 mark_page_accessed(arg); 3168 put_page(arg); 3169 } 3170 3171 static const char *shmem_get_link(struct dentry *dentry, 3172 struct inode *inode, 3173 struct delayed_call *done) 3174 { 3175 struct page *page = NULL; 3176 int error; 3177 if (!dentry) { 3178 page = find_get_page(inode->i_mapping, 0); 3179 if (!page) 3180 return ERR_PTR(-ECHILD); 3181 if (!PageUptodate(page)) { 3182 put_page(page); 3183 return ERR_PTR(-ECHILD); 3184 } 3185 } else { 3186 error = shmem_getpage(inode, 0, &page, SGP_READ); 3187 if (error) 3188 return ERR_PTR(error); 3189 unlock_page(page); 3190 } 3191 set_delayed_call(done, shmem_put_link, page); 3192 return page_address(page); 3193 } 3194 3195 #ifdef CONFIG_TMPFS_XATTR 3196 /* 3197 * Superblocks without xattr inode operations may get some security.* xattr 3198 * support from the LSM "for free". As soon as we have any other xattrs 3199 * like ACLs, we also need to implement the security.* handlers at 3200 * filesystem level, though. 3201 */ 3202 3203 /* 3204 * Callback for security_inode_init_security() for acquiring xattrs. 3205 */ 3206 static int shmem_initxattrs(struct inode *inode, 3207 const struct xattr *xattr_array, 3208 void *fs_info) 3209 { 3210 struct shmem_inode_info *info = SHMEM_I(inode); 3211 const struct xattr *xattr; 3212 struct simple_xattr *new_xattr; 3213 size_t len; 3214 3215 for (xattr = xattr_array; xattr->name != NULL; xattr++) { 3216 new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len); 3217 if (!new_xattr) 3218 return -ENOMEM; 3219 3220 len = strlen(xattr->name) + 1; 3221 new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len, 3222 GFP_KERNEL); 3223 if (!new_xattr->name) { 3224 kfree(new_xattr); 3225 return -ENOMEM; 3226 } 3227 3228 memcpy(new_xattr->name, XATTR_SECURITY_PREFIX, 3229 XATTR_SECURITY_PREFIX_LEN); 3230 memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN, 3231 xattr->name, len); 3232 3233 simple_xattr_list_add(&info->xattrs, new_xattr); 3234 } 3235 3236 return 0; 3237 } 3238 3239 static int shmem_xattr_handler_get(const struct xattr_handler *handler, 3240 struct dentry *unused, struct inode *inode, 3241 const char *name, void *buffer, size_t size) 3242 { 3243 struct shmem_inode_info *info = SHMEM_I(inode); 3244 3245 name = xattr_full_name(handler, name); 3246 return simple_xattr_get(&info->xattrs, name, buffer, size); 3247 } 3248 3249 static int shmem_xattr_handler_set(const struct xattr_handler *handler, 3250 struct dentry *unused, struct inode *inode, 3251 const char *name, const void *value, 3252 size_t size, int flags) 3253 { 3254 struct shmem_inode_info *info = SHMEM_I(inode); 3255 3256 name = xattr_full_name(handler, name); 3257 return simple_xattr_set(&info->xattrs, name, value, size, flags); 3258 } 3259 3260 static const struct xattr_handler shmem_security_xattr_handler = { 3261 .prefix = XATTR_SECURITY_PREFIX, 3262 .get = shmem_xattr_handler_get, 3263 .set = shmem_xattr_handler_set, 3264 }; 3265 3266 static const struct xattr_handler shmem_trusted_xattr_handler = { 3267 .prefix = XATTR_TRUSTED_PREFIX, 3268 .get = shmem_xattr_handler_get, 3269 .set = shmem_xattr_handler_set, 3270 }; 3271 3272 static const struct xattr_handler *shmem_xattr_handlers[] = { 3273 #ifdef CONFIG_TMPFS_POSIX_ACL 3274 &posix_acl_access_xattr_handler, 3275 &posix_acl_default_xattr_handler, 3276 #endif 3277 &shmem_security_xattr_handler, 3278 &shmem_trusted_xattr_handler, 3279 NULL 3280 }; 3281 3282 static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size) 3283 { 3284 struct shmem_inode_info *info = SHMEM_I(d_inode(dentry)); 3285 return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size); 3286 } 3287 #endif /* CONFIG_TMPFS_XATTR */ 3288 3289 static const struct inode_operations shmem_short_symlink_operations = { 3290 .readlink = generic_readlink, 3291 .get_link = simple_get_link, 3292 #ifdef CONFIG_TMPFS_XATTR 3293 .setxattr = generic_setxattr, 3294 .getxattr = generic_getxattr, 3295 .listxattr = shmem_listxattr, 3296 .removexattr = generic_removexattr, 3297 #endif 3298 }; 3299 3300 static const struct inode_operations shmem_symlink_inode_operations = { 3301 .readlink = generic_readlink, 3302 .get_link = shmem_get_link, 3303 #ifdef CONFIG_TMPFS_XATTR 3304 .setxattr = generic_setxattr, 3305 .getxattr = generic_getxattr, 3306 .listxattr = shmem_listxattr, 3307 .removexattr = generic_removexattr, 3308 #endif 3309 }; 3310 3311 static struct dentry *shmem_get_parent(struct dentry *child) 3312 { 3313 return ERR_PTR(-ESTALE); 3314 } 3315 3316 static int shmem_match(struct inode *ino, void *vfh) 3317 { 3318 __u32 *fh = vfh; 3319 __u64 inum = fh[2]; 3320 inum = (inum << 32) | fh[1]; 3321 return ino->i_ino == inum && fh[0] == ino->i_generation; 3322 } 3323 3324 static struct dentry *shmem_fh_to_dentry(struct super_block *sb, 3325 struct fid *fid, int fh_len, int fh_type) 3326 { 3327 struct inode *inode; 3328 struct dentry *dentry = NULL; 3329 u64 inum; 3330 3331 if (fh_len < 3) 3332 return NULL; 3333 3334 inum = fid->raw[2]; 3335 inum = (inum << 32) | fid->raw[1]; 3336 3337 inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]), 3338 shmem_match, fid->raw); 3339 if (inode) { 3340 dentry = d_find_alias(inode); 3341 iput(inode); 3342 } 3343 3344 return dentry; 3345 } 3346 3347 static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len, 3348 struct inode *parent) 3349 { 3350 if (*len < 3) { 3351 *len = 3; 3352 return FILEID_INVALID; 3353 } 3354 3355 if (inode_unhashed(inode)) { 3356 /* Unfortunately insert_inode_hash is not idempotent, 3357 * so as we hash inodes here rather than at creation 3358 * time, we need a lock to ensure we only try 3359 * to do it once 3360 */ 3361 static DEFINE_SPINLOCK(lock); 3362 spin_lock(&lock); 3363 if (inode_unhashed(inode)) 3364 __insert_inode_hash(inode, 3365 inode->i_ino + inode->i_generation); 3366 spin_unlock(&lock); 3367 } 3368 3369 fh[0] = inode->i_generation; 3370 fh[1] = inode->i_ino; 3371 fh[2] = ((__u64)inode->i_ino) >> 32; 3372 3373 *len = 3; 3374 return 1; 3375 } 3376 3377 static const struct export_operations shmem_export_ops = { 3378 .get_parent = shmem_get_parent, 3379 .encode_fh = shmem_encode_fh, 3380 .fh_to_dentry = shmem_fh_to_dentry, 3381 }; 3382 3383 static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo, 3384 bool remount) 3385 { 3386 char *this_char, *value, *rest; 3387 struct mempolicy *mpol = NULL; 3388 uid_t uid; 3389 gid_t gid; 3390 3391 while (options != NULL) { 3392 this_char = options; 3393 for (;;) { 3394 /* 3395 * NUL-terminate this option: unfortunately, 3396 * mount options form a comma-separated list, 3397 * but mpol's nodelist may also contain commas. 3398 */ 3399 options = strchr(options, ','); 3400 if (options == NULL) 3401 break; 3402 options++; 3403 if (!isdigit(*options)) { 3404 options[-1] = '\0'; 3405 break; 3406 } 3407 } 3408 if (!*this_char) 3409 continue; 3410 if ((value = strchr(this_char,'=')) != NULL) { 3411 *value++ = 0; 3412 } else { 3413 pr_err("tmpfs: No value for mount option '%s'\n", 3414 this_char); 3415 goto error; 3416 } 3417 3418 if (!strcmp(this_char,"size")) { 3419 unsigned long long size; 3420 size = memparse(value,&rest); 3421 if (*rest == '%') { 3422 size <<= PAGE_SHIFT; 3423 size *= totalram_pages; 3424 do_div(size, 100); 3425 rest++; 3426 } 3427 if (*rest) 3428 goto bad_val; 3429 sbinfo->max_blocks = 3430 DIV_ROUND_UP(size, PAGE_SIZE); 3431 } else if (!strcmp(this_char,"nr_blocks")) { 3432 sbinfo->max_blocks = memparse(value, &rest); 3433 if (*rest) 3434 goto bad_val; 3435 } else if (!strcmp(this_char,"nr_inodes")) { 3436 sbinfo->max_inodes = memparse(value, &rest); 3437 if (*rest) 3438 goto bad_val; 3439 } else if (!strcmp(this_char,"mode")) { 3440 if (remount) 3441 continue; 3442 sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777; 3443 if (*rest) 3444 goto bad_val; 3445 } else if (!strcmp(this_char,"uid")) { 3446 if (remount) 3447 continue; 3448 uid = simple_strtoul(value, &rest, 0); 3449 if (*rest) 3450 goto bad_val; 3451 sbinfo->uid = make_kuid(current_user_ns(), uid); 3452 if (!uid_valid(sbinfo->uid)) 3453 goto bad_val; 3454 } else if (!strcmp(this_char,"gid")) { 3455 if (remount) 3456 continue; 3457 gid = simple_strtoul(value, &rest, 0); 3458 if (*rest) 3459 goto bad_val; 3460 sbinfo->gid = make_kgid(current_user_ns(), gid); 3461 if (!gid_valid(sbinfo->gid)) 3462 goto bad_val; 3463 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 3464 } else if (!strcmp(this_char, "huge")) { 3465 int huge; 3466 huge = shmem_parse_huge(value); 3467 if (huge < 0) 3468 goto bad_val; 3469 if (!has_transparent_hugepage() && 3470 huge != SHMEM_HUGE_NEVER) 3471 goto bad_val; 3472 sbinfo->huge = huge; 3473 #endif 3474 #ifdef CONFIG_NUMA 3475 } else if (!strcmp(this_char,"mpol")) { 3476 mpol_put(mpol); 3477 mpol = NULL; 3478 if (mpol_parse_str(value, &mpol)) 3479 goto bad_val; 3480 #endif 3481 } else { 3482 pr_err("tmpfs: Bad mount option %s\n", this_char); 3483 goto error; 3484 } 3485 } 3486 sbinfo->mpol = mpol; 3487 return 0; 3488 3489 bad_val: 3490 pr_err("tmpfs: Bad value '%s' for mount option '%s'\n", 3491 value, this_char); 3492 error: 3493 mpol_put(mpol); 3494 return 1; 3495 3496 } 3497 3498 static int shmem_remount_fs(struct super_block *sb, int *flags, char *data) 3499 { 3500 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 3501 struct shmem_sb_info config = *sbinfo; 3502 unsigned long inodes; 3503 int error = -EINVAL; 3504 3505 config.mpol = NULL; 3506 if (shmem_parse_options(data, &config, true)) 3507 return error; 3508 3509 spin_lock(&sbinfo->stat_lock); 3510 inodes = sbinfo->max_inodes - sbinfo->free_inodes; 3511 if (percpu_counter_compare(&sbinfo->used_blocks, config.max_blocks) > 0) 3512 goto out; 3513 if (config.max_inodes < inodes) 3514 goto out; 3515 /* 3516 * Those tests disallow limited->unlimited while any are in use; 3517 * but we must separately disallow unlimited->limited, because 3518 * in that case we have no record of how much is already in use. 3519 */ 3520 if (config.max_blocks && !sbinfo->max_blocks) 3521 goto out; 3522 if (config.max_inodes && !sbinfo->max_inodes) 3523 goto out; 3524 3525 error = 0; 3526 sbinfo->huge = config.huge; 3527 sbinfo->max_blocks = config.max_blocks; 3528 sbinfo->max_inodes = config.max_inodes; 3529 sbinfo->free_inodes = config.max_inodes - inodes; 3530 3531 /* 3532 * Preserve previous mempolicy unless mpol remount option was specified. 3533 */ 3534 if (config.mpol) { 3535 mpol_put(sbinfo->mpol); 3536 sbinfo->mpol = config.mpol; /* transfers initial ref */ 3537 } 3538 out: 3539 spin_unlock(&sbinfo->stat_lock); 3540 return error; 3541 } 3542 3543 static int shmem_show_options(struct seq_file *seq, struct dentry *root) 3544 { 3545 struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb); 3546 3547 if (sbinfo->max_blocks != shmem_default_max_blocks()) 3548 seq_printf(seq, ",size=%luk", 3549 sbinfo->max_blocks << (PAGE_SHIFT - 10)); 3550 if (sbinfo->max_inodes != shmem_default_max_inodes()) 3551 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes); 3552 if (sbinfo->mode != (S_IRWXUGO | S_ISVTX)) 3553 seq_printf(seq, ",mode=%03ho", sbinfo->mode); 3554 if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID)) 3555 seq_printf(seq, ",uid=%u", 3556 from_kuid_munged(&init_user_ns, sbinfo->uid)); 3557 if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID)) 3558 seq_printf(seq, ",gid=%u", 3559 from_kgid_munged(&init_user_ns, sbinfo->gid)); 3560 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 3561 /* Rightly or wrongly, show huge mount option unmasked by shmem_huge */ 3562 if (sbinfo->huge) 3563 seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge)); 3564 #endif 3565 shmem_show_mpol(seq, sbinfo->mpol); 3566 return 0; 3567 } 3568 3569 #define MFD_NAME_PREFIX "memfd:" 3570 #define MFD_NAME_PREFIX_LEN (sizeof(MFD_NAME_PREFIX) - 1) 3571 #define MFD_NAME_MAX_LEN (NAME_MAX - MFD_NAME_PREFIX_LEN) 3572 3573 #define MFD_ALL_FLAGS (MFD_CLOEXEC | MFD_ALLOW_SEALING) 3574 3575 SYSCALL_DEFINE2(memfd_create, 3576 const char __user *, uname, 3577 unsigned int, flags) 3578 { 3579 struct shmem_inode_info *info; 3580 struct file *file; 3581 int fd, error; 3582 char *name; 3583 long len; 3584 3585 if (flags & ~(unsigned int)MFD_ALL_FLAGS) 3586 return -EINVAL; 3587 3588 /* length includes terminating zero */ 3589 len = strnlen_user(uname, MFD_NAME_MAX_LEN + 1); 3590 if (len <= 0) 3591 return -EFAULT; 3592 if (len > MFD_NAME_MAX_LEN + 1) 3593 return -EINVAL; 3594 3595 name = kmalloc(len + MFD_NAME_PREFIX_LEN, GFP_TEMPORARY); 3596 if (!name) 3597 return -ENOMEM; 3598 3599 strcpy(name, MFD_NAME_PREFIX); 3600 if (copy_from_user(&name[MFD_NAME_PREFIX_LEN], uname, len)) { 3601 error = -EFAULT; 3602 goto err_name; 3603 } 3604 3605 /* terminating-zero may have changed after strnlen_user() returned */ 3606 if (name[len + MFD_NAME_PREFIX_LEN - 1]) { 3607 error = -EFAULT; 3608 goto err_name; 3609 } 3610 3611 fd = get_unused_fd_flags((flags & MFD_CLOEXEC) ? O_CLOEXEC : 0); 3612 if (fd < 0) { 3613 error = fd; 3614 goto err_name; 3615 } 3616 3617 file = shmem_file_setup(name, 0, VM_NORESERVE); 3618 if (IS_ERR(file)) { 3619 error = PTR_ERR(file); 3620 goto err_fd; 3621 } 3622 info = SHMEM_I(file_inode(file)); 3623 file->f_mode |= FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE; 3624 file->f_flags |= O_RDWR | O_LARGEFILE; 3625 if (flags & MFD_ALLOW_SEALING) 3626 info->seals &= ~F_SEAL_SEAL; 3627 3628 fd_install(fd, file); 3629 kfree(name); 3630 return fd; 3631 3632 err_fd: 3633 put_unused_fd(fd); 3634 err_name: 3635 kfree(name); 3636 return error; 3637 } 3638 3639 #endif /* CONFIG_TMPFS */ 3640 3641 static void shmem_put_super(struct super_block *sb) 3642 { 3643 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 3644 3645 percpu_counter_destroy(&sbinfo->used_blocks); 3646 mpol_put(sbinfo->mpol); 3647 kfree(sbinfo); 3648 sb->s_fs_info = NULL; 3649 } 3650 3651 int shmem_fill_super(struct super_block *sb, void *data, int silent) 3652 { 3653 struct inode *inode; 3654 struct shmem_sb_info *sbinfo; 3655 int err = -ENOMEM; 3656 3657 /* Round up to L1_CACHE_BYTES to resist false sharing */ 3658 sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info), 3659 L1_CACHE_BYTES), GFP_KERNEL); 3660 if (!sbinfo) 3661 return -ENOMEM; 3662 3663 sbinfo->mode = S_IRWXUGO | S_ISVTX; 3664 sbinfo->uid = current_fsuid(); 3665 sbinfo->gid = current_fsgid(); 3666 sb->s_fs_info = sbinfo; 3667 3668 #ifdef CONFIG_TMPFS 3669 /* 3670 * Per default we only allow half of the physical ram per 3671 * tmpfs instance, limiting inodes to one per page of lowmem; 3672 * but the internal instance is left unlimited. 3673 */ 3674 if (!(sb->s_flags & MS_KERNMOUNT)) { 3675 sbinfo->max_blocks = shmem_default_max_blocks(); 3676 sbinfo->max_inodes = shmem_default_max_inodes(); 3677 if (shmem_parse_options(data, sbinfo, false)) { 3678 err = -EINVAL; 3679 goto failed; 3680 } 3681 } else { 3682 sb->s_flags |= MS_NOUSER; 3683 } 3684 sb->s_export_op = &shmem_export_ops; 3685 sb->s_flags |= MS_NOSEC; 3686 #else 3687 sb->s_flags |= MS_NOUSER; 3688 #endif 3689 3690 spin_lock_init(&sbinfo->stat_lock); 3691 if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL)) 3692 goto failed; 3693 sbinfo->free_inodes = sbinfo->max_inodes; 3694 spin_lock_init(&sbinfo->shrinklist_lock); 3695 INIT_LIST_HEAD(&sbinfo->shrinklist); 3696 3697 sb->s_maxbytes = MAX_LFS_FILESIZE; 3698 sb->s_blocksize = PAGE_SIZE; 3699 sb->s_blocksize_bits = PAGE_SHIFT; 3700 sb->s_magic = TMPFS_MAGIC; 3701 sb->s_op = &shmem_ops; 3702 sb->s_time_gran = 1; 3703 #ifdef CONFIG_TMPFS_XATTR 3704 sb->s_xattr = shmem_xattr_handlers; 3705 #endif 3706 #ifdef CONFIG_TMPFS_POSIX_ACL 3707 sb->s_flags |= MS_POSIXACL; 3708 #endif 3709 3710 inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE); 3711 if (!inode) 3712 goto failed; 3713 inode->i_uid = sbinfo->uid; 3714 inode->i_gid = sbinfo->gid; 3715 sb->s_root = d_make_root(inode); 3716 if (!sb->s_root) 3717 goto failed; 3718 return 0; 3719 3720 failed: 3721 shmem_put_super(sb); 3722 return err; 3723 } 3724 3725 static struct kmem_cache *shmem_inode_cachep; 3726 3727 static struct inode *shmem_alloc_inode(struct super_block *sb) 3728 { 3729 struct shmem_inode_info *info; 3730 info = kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL); 3731 if (!info) 3732 return NULL; 3733 return &info->vfs_inode; 3734 } 3735 3736 static void shmem_destroy_callback(struct rcu_head *head) 3737 { 3738 struct inode *inode = container_of(head, struct inode, i_rcu); 3739 if (S_ISLNK(inode->i_mode)) 3740 kfree(inode->i_link); 3741 kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode)); 3742 } 3743 3744 static void shmem_destroy_inode(struct inode *inode) 3745 { 3746 if (S_ISREG(inode->i_mode)) 3747 mpol_free_shared_policy(&SHMEM_I(inode)->policy); 3748 call_rcu(&inode->i_rcu, shmem_destroy_callback); 3749 } 3750 3751 static void shmem_init_inode(void *foo) 3752 { 3753 struct shmem_inode_info *info = foo; 3754 inode_init_once(&info->vfs_inode); 3755 } 3756 3757 static int shmem_init_inodecache(void) 3758 { 3759 shmem_inode_cachep = kmem_cache_create("shmem_inode_cache", 3760 sizeof(struct shmem_inode_info), 3761 0, SLAB_PANIC|SLAB_ACCOUNT, shmem_init_inode); 3762 return 0; 3763 } 3764 3765 static void shmem_destroy_inodecache(void) 3766 { 3767 kmem_cache_destroy(shmem_inode_cachep); 3768 } 3769 3770 static const struct address_space_operations shmem_aops = { 3771 .writepage = shmem_writepage, 3772 .set_page_dirty = __set_page_dirty_no_writeback, 3773 #ifdef CONFIG_TMPFS 3774 .write_begin = shmem_write_begin, 3775 .write_end = shmem_write_end, 3776 #endif 3777 #ifdef CONFIG_MIGRATION 3778 .migratepage = migrate_page, 3779 #endif 3780 .error_remove_page = generic_error_remove_page, 3781 }; 3782 3783 static const struct file_operations shmem_file_operations = { 3784 .mmap = shmem_mmap, 3785 .get_unmapped_area = shmem_get_unmapped_area, 3786 #ifdef CONFIG_TMPFS 3787 .llseek = shmem_file_llseek, 3788 .read_iter = shmem_file_read_iter, 3789 .write_iter = generic_file_write_iter, 3790 .fsync = noop_fsync, 3791 .splice_read = shmem_file_splice_read, 3792 .splice_write = iter_file_splice_write, 3793 .fallocate = shmem_fallocate, 3794 #endif 3795 }; 3796 3797 static const struct inode_operations shmem_inode_operations = { 3798 .getattr = shmem_getattr, 3799 .setattr = shmem_setattr, 3800 #ifdef CONFIG_TMPFS_XATTR 3801 .setxattr = generic_setxattr, 3802 .getxattr = generic_getxattr, 3803 .listxattr = shmem_listxattr, 3804 .removexattr = generic_removexattr, 3805 .set_acl = simple_set_acl, 3806 #endif 3807 }; 3808 3809 static const struct inode_operations shmem_dir_inode_operations = { 3810 #ifdef CONFIG_TMPFS 3811 .create = shmem_create, 3812 .lookup = simple_lookup, 3813 .link = shmem_link, 3814 .unlink = shmem_unlink, 3815 .symlink = shmem_symlink, 3816 .mkdir = shmem_mkdir, 3817 .rmdir = shmem_rmdir, 3818 .mknod = shmem_mknod, 3819 .rename2 = shmem_rename2, 3820 .tmpfile = shmem_tmpfile, 3821 #endif 3822 #ifdef CONFIG_TMPFS_XATTR 3823 .setxattr = generic_setxattr, 3824 .getxattr = generic_getxattr, 3825 .listxattr = shmem_listxattr, 3826 .removexattr = generic_removexattr, 3827 #endif 3828 #ifdef CONFIG_TMPFS_POSIX_ACL 3829 .setattr = shmem_setattr, 3830 .set_acl = simple_set_acl, 3831 #endif 3832 }; 3833 3834 static const struct inode_operations shmem_special_inode_operations = { 3835 #ifdef CONFIG_TMPFS_XATTR 3836 .setxattr = generic_setxattr, 3837 .getxattr = generic_getxattr, 3838 .listxattr = shmem_listxattr, 3839 .removexattr = generic_removexattr, 3840 #endif 3841 #ifdef CONFIG_TMPFS_POSIX_ACL 3842 .setattr = shmem_setattr, 3843 .set_acl = simple_set_acl, 3844 #endif 3845 }; 3846 3847 static const struct super_operations shmem_ops = { 3848 .alloc_inode = shmem_alloc_inode, 3849 .destroy_inode = shmem_destroy_inode, 3850 #ifdef CONFIG_TMPFS 3851 .statfs = shmem_statfs, 3852 .remount_fs = shmem_remount_fs, 3853 .show_options = shmem_show_options, 3854 #endif 3855 .evict_inode = shmem_evict_inode, 3856 .drop_inode = generic_delete_inode, 3857 .put_super = shmem_put_super, 3858 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 3859 .nr_cached_objects = shmem_unused_huge_count, 3860 .free_cached_objects = shmem_unused_huge_scan, 3861 #endif 3862 }; 3863 3864 static const struct vm_operations_struct shmem_vm_ops = { 3865 .fault = shmem_fault, 3866 .map_pages = filemap_map_pages, 3867 #ifdef CONFIG_NUMA 3868 .set_policy = shmem_set_policy, 3869 .get_policy = shmem_get_policy, 3870 #endif 3871 }; 3872 3873 static struct dentry *shmem_mount(struct file_system_type *fs_type, 3874 int flags, const char *dev_name, void *data) 3875 { 3876 return mount_nodev(fs_type, flags, data, shmem_fill_super); 3877 } 3878 3879 static struct file_system_type shmem_fs_type = { 3880 .owner = THIS_MODULE, 3881 .name = "tmpfs", 3882 .mount = shmem_mount, 3883 .kill_sb = kill_litter_super, 3884 .fs_flags = FS_USERNS_MOUNT, 3885 }; 3886 3887 int __init shmem_init(void) 3888 { 3889 int error; 3890 3891 /* If rootfs called this, don't re-init */ 3892 if (shmem_inode_cachep) 3893 return 0; 3894 3895 error = shmem_init_inodecache(); 3896 if (error) 3897 goto out3; 3898 3899 error = register_filesystem(&shmem_fs_type); 3900 if (error) { 3901 pr_err("Could not register tmpfs\n"); 3902 goto out2; 3903 } 3904 3905 shm_mnt = kern_mount(&shmem_fs_type); 3906 if (IS_ERR(shm_mnt)) { 3907 error = PTR_ERR(shm_mnt); 3908 pr_err("Could not kern_mount tmpfs\n"); 3909 goto out1; 3910 } 3911 3912 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 3913 if (has_transparent_hugepage() && shmem_huge < SHMEM_HUGE_DENY) 3914 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge; 3915 else 3916 shmem_huge = 0; /* just in case it was patched */ 3917 #endif 3918 return 0; 3919 3920 out1: 3921 unregister_filesystem(&shmem_fs_type); 3922 out2: 3923 shmem_destroy_inodecache(); 3924 out3: 3925 shm_mnt = ERR_PTR(error); 3926 return error; 3927 } 3928 3929 #if defined(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && defined(CONFIG_SYSFS) 3930 static ssize_t shmem_enabled_show(struct kobject *kobj, 3931 struct kobj_attribute *attr, char *buf) 3932 { 3933 int values[] = { 3934 SHMEM_HUGE_ALWAYS, 3935 SHMEM_HUGE_WITHIN_SIZE, 3936 SHMEM_HUGE_ADVISE, 3937 SHMEM_HUGE_NEVER, 3938 SHMEM_HUGE_DENY, 3939 SHMEM_HUGE_FORCE, 3940 }; 3941 int i, count; 3942 3943 for (i = 0, count = 0; i < ARRAY_SIZE(values); i++) { 3944 const char *fmt = shmem_huge == values[i] ? "[%s] " : "%s "; 3945 3946 count += sprintf(buf + count, fmt, 3947 shmem_format_huge(values[i])); 3948 } 3949 buf[count - 1] = '\n'; 3950 return count; 3951 } 3952 3953 static ssize_t shmem_enabled_store(struct kobject *kobj, 3954 struct kobj_attribute *attr, const char *buf, size_t count) 3955 { 3956 char tmp[16]; 3957 int huge; 3958 3959 if (count + 1 > sizeof(tmp)) 3960 return -EINVAL; 3961 memcpy(tmp, buf, count); 3962 tmp[count] = '\0'; 3963 if (count && tmp[count - 1] == '\n') 3964 tmp[count - 1] = '\0'; 3965 3966 huge = shmem_parse_huge(tmp); 3967 if (huge == -EINVAL) 3968 return -EINVAL; 3969 if (!has_transparent_hugepage() && 3970 huge != SHMEM_HUGE_NEVER && huge != SHMEM_HUGE_DENY) 3971 return -EINVAL; 3972 3973 shmem_huge = huge; 3974 if (shmem_huge < SHMEM_HUGE_DENY) 3975 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge; 3976 return count; 3977 } 3978 3979 struct kobj_attribute shmem_enabled_attr = 3980 __ATTR(shmem_enabled, 0644, shmem_enabled_show, shmem_enabled_store); 3981 #endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE && CONFIG_SYSFS */ 3982 3983 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 3984 bool shmem_huge_enabled(struct vm_area_struct *vma) 3985 { 3986 struct inode *inode = file_inode(vma->vm_file); 3987 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 3988 loff_t i_size; 3989 pgoff_t off; 3990 3991 if (shmem_huge == SHMEM_HUGE_FORCE) 3992 return true; 3993 if (shmem_huge == SHMEM_HUGE_DENY) 3994 return false; 3995 switch (sbinfo->huge) { 3996 case SHMEM_HUGE_NEVER: 3997 return false; 3998 case SHMEM_HUGE_ALWAYS: 3999 return true; 4000 case SHMEM_HUGE_WITHIN_SIZE: 4001 off = round_up(vma->vm_pgoff, HPAGE_PMD_NR); 4002 i_size = round_up(i_size_read(inode), PAGE_SIZE); 4003 if (i_size >= HPAGE_PMD_SIZE && 4004 i_size >> PAGE_SHIFT >= off) 4005 return true; 4006 case SHMEM_HUGE_ADVISE: 4007 /* TODO: implement fadvise() hints */ 4008 return (vma->vm_flags & VM_HUGEPAGE); 4009 default: 4010 VM_BUG_ON(1); 4011 return false; 4012 } 4013 } 4014 #endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE */ 4015 4016 #else /* !CONFIG_SHMEM */ 4017 4018 /* 4019 * tiny-shmem: simple shmemfs and tmpfs using ramfs code 4020 * 4021 * This is intended for small system where the benefits of the full 4022 * shmem code (swap-backed and resource-limited) are outweighed by 4023 * their complexity. On systems without swap this code should be 4024 * effectively equivalent, but much lighter weight. 4025 */ 4026 4027 static struct file_system_type shmem_fs_type = { 4028 .name = "tmpfs", 4029 .mount = ramfs_mount, 4030 .kill_sb = kill_litter_super, 4031 .fs_flags = FS_USERNS_MOUNT, 4032 }; 4033 4034 int __init shmem_init(void) 4035 { 4036 BUG_ON(register_filesystem(&shmem_fs_type) != 0); 4037 4038 shm_mnt = kern_mount(&shmem_fs_type); 4039 BUG_ON(IS_ERR(shm_mnt)); 4040 4041 return 0; 4042 } 4043 4044 int shmem_unuse(swp_entry_t swap, struct page *page) 4045 { 4046 return 0; 4047 } 4048 4049 int shmem_lock(struct file *file, int lock, struct user_struct *user) 4050 { 4051 return 0; 4052 } 4053 4054 void shmem_unlock_mapping(struct address_space *mapping) 4055 { 4056 } 4057 4058 #ifdef CONFIG_MMU 4059 unsigned long shmem_get_unmapped_area(struct file *file, 4060 unsigned long addr, unsigned long len, 4061 unsigned long pgoff, unsigned long flags) 4062 { 4063 return current->mm->get_unmapped_area(file, addr, len, pgoff, flags); 4064 } 4065 #endif 4066 4067 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) 4068 { 4069 truncate_inode_pages_range(inode->i_mapping, lstart, lend); 4070 } 4071 EXPORT_SYMBOL_GPL(shmem_truncate_range); 4072 4073 #define shmem_vm_ops generic_file_vm_ops 4074 #define shmem_file_operations ramfs_file_operations 4075 #define shmem_get_inode(sb, dir, mode, dev, flags) ramfs_get_inode(sb, dir, mode, dev) 4076 #define shmem_acct_size(flags, size) 0 4077 #define shmem_unacct_size(flags, size) do {} while (0) 4078 4079 #endif /* CONFIG_SHMEM */ 4080 4081 /* common code */ 4082 4083 static struct dentry_operations anon_ops = { 4084 .d_dname = simple_dname 4085 }; 4086 4087 static struct file *__shmem_file_setup(const char *name, loff_t size, 4088 unsigned long flags, unsigned int i_flags) 4089 { 4090 struct file *res; 4091 struct inode *inode; 4092 struct path path; 4093 struct super_block *sb; 4094 struct qstr this; 4095 4096 if (IS_ERR(shm_mnt)) 4097 return ERR_CAST(shm_mnt); 4098 4099 if (size < 0 || size > MAX_LFS_FILESIZE) 4100 return ERR_PTR(-EINVAL); 4101 4102 if (shmem_acct_size(flags, size)) 4103 return ERR_PTR(-ENOMEM); 4104 4105 res = ERR_PTR(-ENOMEM); 4106 this.name = name; 4107 this.len = strlen(name); 4108 this.hash = 0; /* will go */ 4109 sb = shm_mnt->mnt_sb; 4110 path.mnt = mntget(shm_mnt); 4111 path.dentry = d_alloc_pseudo(sb, &this); 4112 if (!path.dentry) 4113 goto put_memory; 4114 d_set_d_op(path.dentry, &anon_ops); 4115 4116 res = ERR_PTR(-ENOSPC); 4117 inode = shmem_get_inode(sb, NULL, S_IFREG | S_IRWXUGO, 0, flags); 4118 if (!inode) 4119 goto put_memory; 4120 4121 inode->i_flags |= i_flags; 4122 d_instantiate(path.dentry, inode); 4123 inode->i_size = size; 4124 clear_nlink(inode); /* It is unlinked */ 4125 res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size)); 4126 if (IS_ERR(res)) 4127 goto put_path; 4128 4129 res = alloc_file(&path, FMODE_WRITE | FMODE_READ, 4130 &shmem_file_operations); 4131 if (IS_ERR(res)) 4132 goto put_path; 4133 4134 return res; 4135 4136 put_memory: 4137 shmem_unacct_size(flags, size); 4138 put_path: 4139 path_put(&path); 4140 return res; 4141 } 4142 4143 /** 4144 * shmem_kernel_file_setup - get an unlinked file living in tmpfs which must be 4145 * kernel internal. There will be NO LSM permission checks against the 4146 * underlying inode. So users of this interface must do LSM checks at a 4147 * higher layer. The users are the big_key and shm implementations. LSM 4148 * checks are provided at the key or shm level rather than the inode. 4149 * @name: name for dentry (to be seen in /proc/<pid>/maps 4150 * @size: size to be set for the file 4151 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 4152 */ 4153 struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags) 4154 { 4155 return __shmem_file_setup(name, size, flags, S_PRIVATE); 4156 } 4157 4158 /** 4159 * shmem_file_setup - get an unlinked file living in tmpfs 4160 * @name: name for dentry (to be seen in /proc/<pid>/maps 4161 * @size: size to be set for the file 4162 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 4163 */ 4164 struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags) 4165 { 4166 return __shmem_file_setup(name, size, flags, 0); 4167 } 4168 EXPORT_SYMBOL_GPL(shmem_file_setup); 4169 4170 /** 4171 * shmem_zero_setup - setup a shared anonymous mapping 4172 * @vma: the vma to be mmapped is prepared by do_mmap_pgoff 4173 */ 4174 int shmem_zero_setup(struct vm_area_struct *vma) 4175 { 4176 struct file *file; 4177 loff_t size = vma->vm_end - vma->vm_start; 4178 4179 /* 4180 * Cloning a new file under mmap_sem leads to a lock ordering conflict 4181 * between XFS directory reading and selinux: since this file is only 4182 * accessible to the user through its mapping, use S_PRIVATE flag to 4183 * bypass file security, in the same way as shmem_kernel_file_setup(). 4184 */ 4185 file = __shmem_file_setup("dev/zero", size, vma->vm_flags, S_PRIVATE); 4186 if (IS_ERR(file)) 4187 return PTR_ERR(file); 4188 4189 if (vma->vm_file) 4190 fput(vma->vm_file); 4191 vma->vm_file = file; 4192 vma->vm_ops = &shmem_vm_ops; 4193 4194 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && 4195 ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) < 4196 (vma->vm_end & HPAGE_PMD_MASK)) { 4197 khugepaged_enter(vma, vma->vm_flags); 4198 } 4199 4200 return 0; 4201 } 4202 4203 /** 4204 * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags. 4205 * @mapping: the page's address_space 4206 * @index: the page index 4207 * @gfp: the page allocator flags to use if allocating 4208 * 4209 * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)", 4210 * with any new page allocations done using the specified allocation flags. 4211 * But read_cache_page_gfp() uses the ->readpage() method: which does not 4212 * suit tmpfs, since it may have pages in swapcache, and needs to find those 4213 * for itself; although drivers/gpu/drm i915 and ttm rely upon this support. 4214 * 4215 * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in 4216 * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily. 4217 */ 4218 struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, 4219 pgoff_t index, gfp_t gfp) 4220 { 4221 #ifdef CONFIG_SHMEM 4222 struct inode *inode = mapping->host; 4223 struct page *page; 4224 int error; 4225 4226 BUG_ON(mapping->a_ops != &shmem_aops); 4227 error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE, 4228 gfp, NULL, NULL); 4229 if (error) 4230 page = ERR_PTR(error); 4231 else 4232 unlock_page(page); 4233 return page; 4234 #else 4235 /* 4236 * The tiny !SHMEM case uses ramfs without swap 4237 */ 4238 return read_cache_page_gfp(mapping, index, gfp); 4239 #endif 4240 } 4241 EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp); 4242
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.