1 /*
2 * hugetlbpage-backed filesystem. Based on ramfs.
3 *
4 * Nadia Yvette Chambers, 2002
5 *
6 * Copyright (C) 2002 Linus Torvalds.
7 * License: GPL
8 */
9
10 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
11
12 #include <linux/thread_info.h>
13 #include <asm/current.h>
14 #include <linux/sched/signal.h> /* remove ASAP */
15 #include <linux/falloc.h>
16 #include <linux/fs.h>
17 #include <linux/mount.h>
18 #include <linux/file.h>
19 #include <linux/kernel.h>
20 #include <linux/writeback.h>
21 #include <linux/pagemap.h>
22 #include <linux/highmem.h>
23 #include <linux/init.h>
24 #include <linux/string.h>
25 #include <linux/capability.h>
26 #include <linux/ctype.h>
27 #include <linux/backing-dev.h>
28 #include <linux/hugetlb.h>
29 #include <linux/pagevec.h>
30 #include <linux/parser.h>
31 #include <linux/mman.h>
32 #include <linux/slab.h>
33 #include <linux/dnotify.h>
34 #include <linux/statfs.h>
35 #include <linux/security.h>
36 #include <linux/magic.h>
37 #include <linux/migrate.h>
38 #include <linux/uio.h>
39
40 #include <linux/uaccess.h>
41
42 static const struct super_operations hugetlbfs_ops;
43 static const struct address_space_operations hugetlbfs_aops;
44 const struct file_operations hugetlbfs_file_operations;
45 static const struct inode_operations hugetlbfs_dir_inode_operations;
46 static const struct inode_operations hugetlbfs_inode_operations;
47
48 struct hugetlbfs_config {
49 struct hstate *hstate;
50 long max_hpages;
51 long nr_inodes;
52 long min_hpages;
53 kuid_t uid;
54 kgid_t gid;
55 umode_t mode;
56 };
57
58 int sysctl_hugetlb_shm_group;
59
60 enum {
61 Opt_size, Opt_nr_inodes,
62 Opt_mode, Opt_uid, Opt_gid,
63 Opt_pagesize, Opt_min_size,
64 Opt_err,
65 };
66
67 static const match_table_t tokens = {
68 {Opt_size, "size=%s"},
69 {Opt_nr_inodes, "nr_inodes=%s"},
70 {Opt_mode, "mode=%o"},
71 {Opt_uid, "uid=%u"},
72 {Opt_gid, "gid=%u"},
73 {Opt_pagesize, "pagesize=%s"},
74 {Opt_min_size, "min_size=%s"},
75 {Opt_err, NULL},
76 };
77
78 #ifdef CONFIG_NUMA
79 static inline void hugetlb_set_vma_policy(struct vm_area_struct *vma,
80 struct inode *inode, pgoff_t index)
81 {
82 vma->vm_policy = mpol_shared_policy_lookup(&HUGETLBFS_I(inode)->policy,
83 index);
84 }
85
86 static inline void hugetlb_drop_vma_policy(struct vm_area_struct *vma)
87 {
88 mpol_cond_put(vma->vm_policy);
89 }
90 #else
91 static inline void hugetlb_set_vma_policy(struct vm_area_struct *vma,
92 struct inode *inode, pgoff_t index)
93 {
94 }
95
96 static inline void hugetlb_drop_vma_policy(struct vm_area_struct *vma)
97 {
98 }
99 #endif
100
101 static void huge_pagevec_release(struct pagevec *pvec)
102 {
103 int i;
104
105 for (i = 0; i < pagevec_count(pvec); ++i)
106 put_page(pvec->pages[i]);
107
108 pagevec_reinit(pvec);
109 }
110
111 /*
112 * Mask used when checking the page offset value passed in via system
113 * calls. This value will be converted to a loff_t which is signed.
114 * Therefore, we want to check the upper PAGE_SHIFT + 1 bits of the
115 * value. The extra bit (- 1 in the shift value) is to take the sign
116 * bit into account.
117 */
118 #define PGOFF_LOFFT_MAX \
119 (((1UL << (PAGE_SHIFT + 1)) - 1) << (BITS_PER_LONG - (PAGE_SHIFT + 1)))
120
121 static int hugetlbfs_file_mmap(struct file *file, struct vm_area_struct *vma)
122 {
123 struct inode *inode = file_inode(file);
124 loff_t len, vma_len;
125 int ret;
126 struct hstate *h = hstate_file(file);
127
128 /*
129 * vma address alignment (but not the pgoff alignment) has
130 * already been checked by prepare_hugepage_range. If you add
131 * any error returns here, do so after setting VM_HUGETLB, so
132 * is_vm_hugetlb_page tests below unmap_region go the right
133 * way when do_mmap_pgoff unwinds (may be important on powerpc
134 * and ia64).
135 */
136 vma->vm_flags |= VM_HUGETLB | VM_DONTEXPAND;
137 vma->vm_ops = &hugetlb_vm_ops;
138
139 /*
140 * page based offset in vm_pgoff could be sufficiently large to
141 * overflow a loff_t when converted to byte offset. This can
142 * only happen on architectures where sizeof(loff_t) ==
143 * sizeof(unsigned long). So, only check in those instances.
144 */
145 if (sizeof(unsigned long) == sizeof(loff_t)) {
146 if (vma->vm_pgoff & PGOFF_LOFFT_MAX)
147 return -EINVAL;
148 }
149
150 /* must be huge page aligned */
151 if (vma->vm_pgoff & (~huge_page_mask(h) >> PAGE_SHIFT))
152 return -EINVAL;
153
154 vma_len = (loff_t)(vma->vm_end - vma->vm_start);
155 len = vma_len + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
156 /* check for overflow */
157 if (len < vma_len)
158 return -EINVAL;
159
160 inode_lock(inode);
161 file_accessed(file);
162
163 ret = -ENOMEM;
164 if (hugetlb_reserve_pages(inode,
165 vma->vm_pgoff >> huge_page_order(h),
166 len >> huge_page_shift(h), vma,
167 vma->vm_flags))
168 goto out;
169
170 ret = 0;
171 if (vma->vm_flags & VM_WRITE && inode->i_size < len)
172 i_size_write(inode, len);
173 out:
174 inode_unlock(inode);
175
176 return ret;
177 }
178
179 /*
180 * Called under down_write(mmap_sem).
181 */
182
183 #ifndef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
184 static unsigned long
185 hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
186 unsigned long len, unsigned long pgoff, unsigned long flags)
187 {
188 struct mm_struct *mm = current->mm;
189 struct vm_area_struct *vma;
190 struct hstate *h = hstate_file(file);
191 struct vm_unmapped_area_info info;
192
193 if (len & ~huge_page_mask(h))
194 return -EINVAL;
195 if (len > TASK_SIZE)
196 return -ENOMEM;
197
198 if (flags & MAP_FIXED) {
199 if (prepare_hugepage_range(file, addr, len))
200 return -EINVAL;
201 return addr;
202 }
203
204 if (addr) {
205 addr = ALIGN(addr, huge_page_size(h));
206 vma = find_vma(mm, addr);
207 if (TASK_SIZE - len >= addr &&
208 (!vma || addr + len <= vm_start_gap(vma)))
209 return addr;
210 }
211
212 info.flags = 0;
213 info.length = len;
214 info.low_limit = TASK_UNMAPPED_BASE;
215 info.high_limit = TASK_SIZE;
216 info.align_mask = PAGE_MASK & ~huge_page_mask(h);
217 info.align_offset = 0;
218 return vm_unmapped_area(&info);
219 }
220 #endif
221
222 static size_t
223 hugetlbfs_read_actor(struct page *page, unsigned long offset,
224 struct iov_iter *to, unsigned long size)
225 {
226 size_t copied = 0;
227 int i, chunksize;
228
229 /* Find which 4k chunk and offset with in that chunk */
230 i = offset >> PAGE_SHIFT;
231 offset = offset & ~PAGE_MASK;
232
233 while (size) {
234 size_t n;
235 chunksize = PAGE_SIZE;
236 if (offset)
237 chunksize -= offset;
238 if (chunksize > size)
239 chunksize = size;
240 n = copy_page_to_iter(&page[i], offset, chunksize, to);
241 copied += n;
242 if (n != chunksize)
243 return copied;
244 offset = 0;
245 size -= chunksize;
246 i++;
247 }
248 return copied;
249 }
250
251 /*
252 * Support for read() - Find the page attached to f_mapping and copy out the
253 * data. Its *very* similar to do_generic_mapping_read(), we can't use that
254 * since it has PAGE_SIZE assumptions.
255 */
256 static ssize_t hugetlbfs_read_iter(struct kiocb *iocb, struct iov_iter *to)
257 {
258 struct file *file = iocb->ki_filp;
259 struct hstate *h = hstate_file(file);
260 struct address_space *mapping = file->f_mapping;
261 struct inode *inode = mapping->host;
262 unsigned long index = iocb->ki_pos >> huge_page_shift(h);
263 unsigned long offset = iocb->ki_pos & ~huge_page_mask(h);
264 unsigned long end_index;
265 loff_t isize;
266 ssize_t retval = 0;
267
268 while (iov_iter_count(to)) {
269 struct page *page;
270 size_t nr, copied;
271
272 /* nr is the maximum number of bytes to copy from this page */
273 nr = huge_page_size(h);
274 isize = i_size_read(inode);
275 if (!isize)
276 break;
277 end_index = (isize - 1) >> huge_page_shift(h);
278 if (index > end_index)
279 break;
280 if (index == end_index) {
281 nr = ((isize - 1) & ~huge_page_mask(h)) + 1;
282 if (nr <= offset)
283 break;
284 }
285 nr = nr - offset;
286
287 /* Find the page */
288 page = find_lock_page(mapping, index);
289 if (unlikely(page == NULL)) {
290 /*
291 * We have a HOLE, zero out the user-buffer for the
292 * length of the hole or request.
293 */
294 copied = iov_iter_zero(nr, to);
295 } else {
296 unlock_page(page);
297
298 /*
299 * We have the page, copy it to user space buffer.
300 */
301 copied = hugetlbfs_read_actor(page, offset, to, nr);
302 put_page(page);
303 }
304 offset += copied;
305 retval += copied;
306 if (copied != nr && iov_iter_count(to)) {
307 if (!retval)
308 retval = -EFAULT;
309 break;
310 }
311 index += offset >> huge_page_shift(h);
312 offset &= ~huge_page_mask(h);
313 }
314 iocb->ki_pos = ((loff_t)index << huge_page_shift(h)) + offset;
315 return retval;
316 }
317
318 static int hugetlbfs_write_begin(struct file *file,
319 struct address_space *mapping,
320 loff_t pos, unsigned len, unsigned flags,
321 struct page **pagep, void **fsdata)
322 {
323 return -EINVAL;
324 }
325
326 static int hugetlbfs_write_end(struct file *file, struct address_space *mapping,
327 loff_t pos, unsigned len, unsigned copied,
328 struct page *page, void *fsdata)
329 {
330 BUG();
331 return -EINVAL;
332 }
333
334 static void remove_huge_page(struct page *page)
335 {
336 ClearPageDirty(page);
337 ClearPageUptodate(page);
338 delete_from_page_cache(page);
339 }
340
341 static void
342 hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end)
343 {
344 struct vm_area_struct *vma;
345
346 /*
347 * end == 0 indicates that the entire range after
348 * start should be unmapped.
349 */
350 vma_interval_tree_foreach(vma, root, start, end ? end : ULONG_MAX) {
351 unsigned long v_offset;
352 unsigned long v_end;
353
354 /*
355 * Can the expression below overflow on 32-bit arches?
356 * No, because the interval tree returns us only those vmas
357 * which overlap the truncated area starting at pgoff,
358 * and no vma on a 32-bit arch can span beyond the 4GB.
359 */
360 if (vma->vm_pgoff < start)
361 v_offset = (start - vma->vm_pgoff) << PAGE_SHIFT;
362 else
363 v_offset = 0;
364
365 if (!end)
366 v_end = vma->vm_end;
367 else {
368 v_end = ((end - vma->vm_pgoff) << PAGE_SHIFT)
369 + vma->vm_start;
370 if (v_end > vma->vm_end)
371 v_end = vma->vm_end;
372 }
373
374 unmap_hugepage_range(vma, vma->vm_start + v_offset, v_end,
375 NULL);
376 }
377 }
378
379 /*
380 * remove_inode_hugepages handles two distinct cases: truncation and hole
381 * punch. There are subtle differences in operation for each case.
382 *
383 * truncation is indicated by end of range being LLONG_MAX
384 * In this case, we first scan the range and release found pages.
385 * After releasing pages, hugetlb_unreserve_pages cleans up region/reserv
386 * maps and global counts. Page faults can not race with truncation
387 * in this routine. hugetlb_no_page() prevents page faults in the
388 * truncated range. It checks i_size before allocation, and again after
389 * with the page table lock for the page held. The same lock must be
390 * acquired to unmap a page.
391 * hole punch is indicated if end is not LLONG_MAX
392 * In the hole punch case we scan the range and release found pages.
393 * Only when releasing a page is the associated region/reserv map
394 * deleted. The region/reserv map for ranges without associated
395 * pages are not modified. Page faults can race with hole punch.
396 * This is indicated if we find a mapped page.
397 * Note: If the passed end of range value is beyond the end of file, but
398 * not LLONG_MAX this routine still performs a hole punch operation.
399 */
400 static void remove_inode_hugepages(struct inode *inode, loff_t lstart,
401 loff_t lend)
402 {
403 struct hstate *h = hstate_inode(inode);
404 struct address_space *mapping = &inode->i_data;
405 const pgoff_t start = lstart >> huge_page_shift(h);
406 const pgoff_t end = lend >> huge_page_shift(h);
407 struct vm_area_struct pseudo_vma;
408 struct pagevec pvec;
409 pgoff_t next, index;
410 int i, freed = 0;
411 bool truncate_op = (lend == LLONG_MAX);
412
413 vma_init(&pseudo_vma, current->mm);
414 pseudo_vma.vm_flags = (VM_HUGETLB | VM_MAYSHARE | VM_SHARED);
415 pagevec_init(&pvec);
416 next = start;
417 while (next < end) {
418 /*
419 * When no more pages are found, we are done.
420 */
421 if (!pagevec_lookup_range(&pvec, mapping, &next, end - 1))
422 break;
423
424 for (i = 0; i < pagevec_count(&pvec); ++i) {
425 struct page *page = pvec.pages[i];
426 u32 hash;
427
428 index = page->index;
429 hash = hugetlb_fault_mutex_hash(h, mapping, index, 0);
430 mutex_lock(&hugetlb_fault_mutex_table[hash]);
431
432 /*
433 * If page is mapped, it was faulted in after being
434 * unmapped in caller. Unmap (again) now after taking
435 * the fault mutex. The mutex will prevent faults
436 * until we finish removing the page.
437 *
438 * This race can only happen in the hole punch case.
439 * Getting here in a truncate operation is a bug.
440 */
441 if (unlikely(page_mapped(page))) {
442 BUG_ON(truncate_op);
443
444 i_mmap_lock_write(mapping);
445 hugetlb_vmdelete_list(&mapping->i_mmap,
446 index * pages_per_huge_page(h),
447 (index + 1) * pages_per_huge_page(h));
448 i_mmap_unlock_write(mapping);
449 }
450
451 lock_page(page);
452 /*
453 * We must free the huge page and remove from page
454 * cache (remove_huge_page) BEFORE removing the
455 * region/reserve map (hugetlb_unreserve_pages). In
456 * rare out of memory conditions, removal of the
457 * region/reserve map could fail. Correspondingly,
458 * the subpool and global reserve usage count can need
459 * to be adjusted.
460 */
461 VM_BUG_ON(PagePrivate(page));
462 remove_huge_page(page);
463 freed++;
464 if (!truncate_op) {
465 if (unlikely(hugetlb_unreserve_pages(inode,
466 index, index + 1, 1)))
467 hugetlb_fix_reserve_counts(inode);
468 }
469
470 unlock_page(page);
471 mutex_unlock(&hugetlb_fault_mutex_table[hash]);
472 }
473 huge_pagevec_release(&pvec);
474 cond_resched();
475 }
476
477 if (truncate_op)
478 (void)hugetlb_unreserve_pages(inode, start, LONG_MAX, freed);
479 }
480
481 static void hugetlbfs_evict_inode(struct inode *inode)
482 {
483 struct resv_map *resv_map;
484
485 remove_inode_hugepages(inode, 0, LLONG_MAX);
486 resv_map = (struct resv_map *)inode->i_mapping->private_data;
487 /* root inode doesn't have the resv_map, so we should check it */
488 if (resv_map)
489 resv_map_release(&resv_map->refs);
490 clear_inode(inode);
491 }
492
493 static int hugetlb_vmtruncate(struct inode *inode, loff_t offset)
494 {
495 pgoff_t pgoff;
496 struct address_space *mapping = inode->i_mapping;
497 struct hstate *h = hstate_inode(inode);
498
499 BUG_ON(offset & ~huge_page_mask(h));
500 pgoff = offset >> PAGE_SHIFT;
501
502 i_size_write(inode, offset);
503 i_mmap_lock_write(mapping);
504 if (!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root))
505 hugetlb_vmdelete_list(&mapping->i_mmap, pgoff, 0);
506 i_mmap_unlock_write(mapping);
507 remove_inode_hugepages(inode, offset, LLONG_MAX);
508 return 0;
509 }
510
511 static long hugetlbfs_punch_hole(struct inode *inode, loff_t offset, loff_t len)
512 {
513 struct hstate *h = hstate_inode(inode);
514 loff_t hpage_size = huge_page_size(h);
515 loff_t hole_start, hole_end;
516
517 /*
518 * For hole punch round up the beginning offset of the hole and
519 * round down the end.
520 */
521 hole_start = round_up(offset, hpage_size);
522 hole_end = round_down(offset + len, hpage_size);
523
524 if (hole_end > hole_start) {
525 struct address_space *mapping = inode->i_mapping;
526 struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
527
528 inode_lock(inode);
529
530 /* protected by i_mutex */
531 if (info->seals & F_SEAL_WRITE) {
532 inode_unlock(inode);
533 return -EPERM;
534 }
535
536 i_mmap_lock_write(mapping);
537 if (!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root))
538 hugetlb_vmdelete_list(&mapping->i_mmap,
539 hole_start >> PAGE_SHIFT,
540 hole_end >> PAGE_SHIFT);
541 i_mmap_unlock_write(mapping);
542 remove_inode_hugepages(inode, hole_start, hole_end);
543 inode_unlock(inode);
544 }
545
546 return 0;
547 }
548
549 static long hugetlbfs_fallocate(struct file *file, int mode, loff_t offset,
550 loff_t len)
551 {
552 struct inode *inode = file_inode(file);
553 struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
554 struct address_space *mapping = inode->i_mapping;
555 struct hstate *h = hstate_inode(inode);
556 struct vm_area_struct pseudo_vma;
557 struct mm_struct *mm = current->mm;
558 loff_t hpage_size = huge_page_size(h);
559 unsigned long hpage_shift = huge_page_shift(h);
560 pgoff_t start, index, end;
561 int error;
562 u32 hash;
563
564 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
565 return -EOPNOTSUPP;
566
567 if (mode & FALLOC_FL_PUNCH_HOLE)
568 return hugetlbfs_punch_hole(inode, offset, len);
569
570 /*
571 * Default preallocate case.
572 * For this range, start is rounded down and end is rounded up
573 * as well as being converted to page offsets.
574 */
575 start = offset >> hpage_shift;
576 end = (offset + len + hpage_size - 1) >> hpage_shift;
577
578 inode_lock(inode);
579
580 /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
581 error = inode_newsize_ok(inode, offset + len);
582 if (error)
583 goto out;
584
585 if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) {
586 error = -EPERM;
587 goto out;
588 }
589
590 /*
591 * Initialize a pseudo vma as this is required by the huge page
592 * allocation routines. If NUMA is configured, use page index
593 * as input to create an allocation policy.
594 */
595 vma_init(&pseudo_vma, mm);
596 pseudo_vma.vm_flags = (VM_HUGETLB | VM_MAYSHARE | VM_SHARED);
597 pseudo_vma.vm_file = file;
598
599 for (index = start; index < end; index++) {
600 /*
601 * This is supposed to be the vaddr where the page is being
602 * faulted in, but we have no vaddr here.
603 */
604 struct page *page;
605 unsigned long addr;
606 int avoid_reserve = 0;
607
608 cond_resched();
609
610 /*
611 * fallocate(2) manpage permits EINTR; we may have been
612 * interrupted because we are using up too much memory.
613 */
614 if (signal_pending(current)) {
615 error = -EINTR;
616 break;
617 }
618
619 /* Set numa allocation policy based on index */
620 hugetlb_set_vma_policy(&pseudo_vma, inode, index);
621
622 /* addr is the offset within the file (zero based) */
623 addr = index * hpage_size;
624
625 /* mutex taken here, fault path and hole punch */
626 hash = hugetlb_fault_mutex_hash(h, mapping, index, addr);
627 mutex_lock(&hugetlb_fault_mutex_table[hash]);
628
629 /* See if already present in mapping to avoid alloc/free */
630 page = find_get_page(mapping, index);
631 if (page) {
632 put_page(page);
633 mutex_unlock(&hugetlb_fault_mutex_table[hash]);
634 hugetlb_drop_vma_policy(&pseudo_vma);
635 continue;
636 }
637
638 /* Allocate page and add to page cache */
639 page = alloc_huge_page(&pseudo_vma, addr, avoid_reserve);
640 hugetlb_drop_vma_policy(&pseudo_vma);
641 if (IS_ERR(page)) {
642 mutex_unlock(&hugetlb_fault_mutex_table[hash]);
643 error = PTR_ERR(page);
644 goto out;
645 }
646 clear_huge_page(page, addr, pages_per_huge_page(h));
647 __SetPageUptodate(page);
648 error = huge_add_to_page_cache(page, mapping, index);
649 if (unlikely(error)) {
650 put_page(page);
651 mutex_unlock(&hugetlb_fault_mutex_table[hash]);
652 goto out;
653 }
654
655 mutex_unlock(&hugetlb_fault_mutex_table[hash]);
656
657 /*
658 * unlock_page because locked by add_to_page_cache()
659 * page_put due to reference from alloc_huge_page()
660 */
661 unlock_page(page);
662 put_page(page);
663 }
664
665 if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
666 i_size_write(inode, offset + len);
667 inode->i_ctime = current_time(inode);
668 out:
669 inode_unlock(inode);
670 return error;
671 }
672
673 static int hugetlbfs_setattr(struct dentry *dentry, struct iattr *attr)
674 {
675 struct inode *inode = d_inode(dentry);
676 struct hstate *h = hstate_inode(inode);
677 int error;
678 unsigned int ia_valid = attr->ia_valid;
679 struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
680
681 BUG_ON(!inode);
682
683 error = setattr_prepare(dentry, attr);
684 if (error)
685 return error;
686
687 if (ia_valid & ATTR_SIZE) {
688 loff_t oldsize = inode->i_size;
689 loff_t newsize = attr->ia_size;
690
691 if (newsize & ~huge_page_mask(h))
692 return -EINVAL;
693 /* protected by i_mutex */
694 if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) ||
695 (newsize > oldsize && (info->seals & F_SEAL_GROW)))
696 return -EPERM;
697 error = hugetlb_vmtruncate(inode, newsize);
698 if (error)
699 return error;
700 }
701
702 setattr_copy(inode, attr);
703 mark_inode_dirty(inode);
704 return 0;
705 }
706
707 static struct inode *hugetlbfs_get_root(struct super_block *sb,
708 struct hugetlbfs_config *config)
709 {
710 struct inode *inode;
711
712 inode = new_inode(sb);
713 if (inode) {
714 inode->i_ino = get_next_ino();
715 inode->i_mode = S_IFDIR | config->mode;
716 inode->i_uid = config->uid;
717 inode->i_gid = config->gid;
718 inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
719 inode->i_op = &hugetlbfs_dir_inode_operations;
720 inode->i_fop = &simple_dir_operations;
721 /* directory inodes start off with i_nlink == 2 (for "." entry) */
722 inc_nlink(inode);
723 lockdep_annotate_inode_mutex_key(inode);
724 }
725 return inode;
726 }
727
728 /*
729 * Hugetlbfs is not reclaimable; therefore its i_mmap_rwsem will never
730 * be taken from reclaim -- unlike regular filesystems. This needs an
731 * annotation because huge_pmd_share() does an allocation under hugetlb's
732 * i_mmap_rwsem.
733 */
734 static struct lock_class_key hugetlbfs_i_mmap_rwsem_key;
735
736 static struct inode *hugetlbfs_get_inode(struct super_block *sb,
737 struct inode *dir,
738 umode_t mode, dev_t dev)
739 {
740 struct inode *inode;
741 struct resv_map *resv_map = NULL;
742
743 /*
744 * Reserve maps are only needed for inodes that can have associated
745 * page allocations.
746 */
747 if (S_ISREG(mode) || S_ISLNK(mode)) {
748 resv_map = resv_map_alloc();
749 if (!resv_map)
750 return NULL;
751 }
752
753 inode = new_inode(sb);
754 if (inode) {
755 struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
756
757 inode->i_ino = get_next_ino();
758 inode_init_owner(inode, dir, mode);
759 lockdep_set_class(&inode->i_mapping->i_mmap_rwsem,
760 &hugetlbfs_i_mmap_rwsem_key);
761 inode->i_mapping->a_ops = &hugetlbfs_aops;
762 inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
763 inode->i_mapping->private_data = resv_map;
764 info->seals = F_SEAL_SEAL;
765 switch (mode & S_IFMT) {
766 default:
767 init_special_inode(inode, mode, dev);
768 break;
769 case S_IFREG:
770 inode->i_op = &hugetlbfs_inode_operations;
771 inode->i_fop = &hugetlbfs_file_operations;
772 break;
773 case S_IFDIR:
774 inode->i_op = &hugetlbfs_dir_inode_operations;
775 inode->i_fop = &simple_dir_operations;
776
777 /* directory inodes start off with i_nlink == 2 (for "." entry) */
778 inc_nlink(inode);
779 break;
780 case S_IFLNK:
781 inode->i_op = &page_symlink_inode_operations;
782 inode_nohighmem(inode);
783 break;
784 }
785 lockdep_annotate_inode_mutex_key(inode);
786 } else {
787 if (resv_map)
788 kref_put(&resv_map->refs, resv_map_release);
789 }
790
791 return inode;
792 }
793
794 /*
795 * File creation. Allocate an inode, and we're done..
796 */
797 static int hugetlbfs_mknod(struct inode *dir,
798 struct dentry *dentry, umode_t mode, dev_t dev)
799 {
800 struct inode *inode;
801 int error = -ENOSPC;
802
803 inode = hugetlbfs_get_inode(dir->i_sb, dir, mode, dev);
804 if (inode) {
805 dir->i_ctime = dir->i_mtime = current_time(dir);
806 d_instantiate(dentry, inode);
807 dget(dentry); /* Extra count - pin the dentry in core */
808 error = 0;
809 }
810 return error;
811 }
812
813 static int hugetlbfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
814 {
815 int retval = hugetlbfs_mknod(dir, dentry, mode | S_IFDIR, 0);
816 if (!retval)
817 inc_nlink(dir);
818 return retval;
819 }
820
821 static int hugetlbfs_create(struct inode *dir, struct dentry *dentry, umode_t mode, bool excl)
822 {
823 return hugetlbfs_mknod(dir, dentry, mode | S_IFREG, 0);
824 }
825
826 static int hugetlbfs_symlink(struct inode *dir,
827 struct dentry *dentry, const char *symname)
828 {
829 struct inode *inode;
830 int error = -ENOSPC;
831
832 inode = hugetlbfs_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0);
833 if (inode) {
834 int l = strlen(symname)+1;
835 error = page_symlink(inode, symname, l);
836 if (!error) {
837 d_instantiate(dentry, inode);
838 dget(dentry);
839 } else
840 iput(inode);
841 }
842 dir->i_ctime = dir->i_mtime = current_time(dir);
843
844 return error;
845 }
846
847 /*
848 * mark the head page dirty
849 */
850 static int hugetlbfs_set_page_dirty(struct page *page)
851 {
852 struct page *head = compound_head(page);
853
854 SetPageDirty(head);
855 return 0;
856 }
857
858 static int hugetlbfs_migrate_page(struct address_space *mapping,
859 struct page *newpage, struct page *page,
860 enum migrate_mode mode)
861 {
862 int rc;
863
864 rc = migrate_huge_page_move_mapping(mapping, newpage, page);
865 if (rc != MIGRATEPAGE_SUCCESS)
866 return rc;
867
868 /*
869 * page_private is subpool pointer in hugetlb pages. Transfer to
870 * new page. PagePrivate is not associated with page_private for
871 * hugetlb pages and can not be set here as only page_huge_active
872 * pages can be migrated.
873 */
874 if (page_private(page)) {
875 set_page_private(newpage, page_private(page));
876 set_page_private(page, 0);
877 }
878
879 if (mode != MIGRATE_SYNC_NO_COPY)
880 migrate_page_copy(newpage, page);
881 else
882 migrate_page_states(newpage, page);
883
884 return MIGRATEPAGE_SUCCESS;
885 }
886
887 static int hugetlbfs_error_remove_page(struct address_space *mapping,
888 struct page *page)
889 {
890 struct inode *inode = mapping->host;
891 pgoff_t index = page->index;
892
893 remove_huge_page(page);
894 if (unlikely(hugetlb_unreserve_pages(inode, index, index + 1, 1)))
895 hugetlb_fix_reserve_counts(inode);
896
897 return 0;
898 }
899
900 /*
901 * Display the mount options in /proc/mounts.
902 */
903 static int hugetlbfs_show_options(struct seq_file *m, struct dentry *root)
904 {
905 struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(root->d_sb);
906 struct hugepage_subpool *spool = sbinfo->spool;
907 unsigned long hpage_size = huge_page_size(sbinfo->hstate);
908 unsigned hpage_shift = huge_page_shift(sbinfo->hstate);
909 char mod;
910
911 if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
912 seq_printf(m, ",uid=%u",
913 from_kuid_munged(&init_user_ns, sbinfo->uid));
914 if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
915 seq_printf(m, ",gid=%u",
916 from_kgid_munged(&init_user_ns, sbinfo->gid));
917 if (sbinfo->mode != 0755)
918 seq_printf(m, ",mode=%o", sbinfo->mode);
919 if (sbinfo->max_inodes != -1)
920 seq_printf(m, ",nr_inodes=%lu", sbinfo->max_inodes);
921
922 hpage_size /= 1024;
923 mod = 'K';
924 if (hpage_size >= 1024) {
925 hpage_size /= 1024;
926 mod = 'M';
927 }
928 seq_printf(m, ",pagesize=%lu%c", hpage_size, mod);
929 if (spool) {
930 if (spool->max_hpages != -1)
931 seq_printf(m, ",size=%llu",
932 (unsigned long long)spool->max_hpages << hpage_shift);
933 if (spool->min_hpages != -1)
934 seq_printf(m, ",min_size=%llu",
935 (unsigned long long)spool->min_hpages << hpage_shift);
936 }
937 return 0;
938 }
939
940 static int hugetlbfs_statfs(struct dentry *dentry, struct kstatfs *buf)
941 {
942 struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(dentry->d_sb);
943 struct hstate *h = hstate_inode(d_inode(dentry));
944
945 buf->f_type = HUGETLBFS_MAGIC;
946 buf->f_bsize = huge_page_size(h);
947 if (sbinfo) {
948 spin_lock(&sbinfo->stat_lock);
949 /* If no limits set, just report 0 for max/free/used
950 * blocks, like simple_statfs() */
951 if (sbinfo->spool) {
952 long free_pages;
953
954 spin_lock(&sbinfo->spool->lock);
955 buf->f_blocks = sbinfo->spool->max_hpages;
956 free_pages = sbinfo->spool->max_hpages
957 - sbinfo->spool->used_hpages;
958 buf->f_bavail = buf->f_bfree = free_pages;
959 spin_unlock(&sbinfo->spool->lock);
960 buf->f_files = sbinfo->max_inodes;
961 buf->f_ffree = sbinfo->free_inodes;
962 }
963 spin_unlock(&sbinfo->stat_lock);
964 }
965 buf->f_namelen = NAME_MAX;
966 return 0;
967 }
968
969 static void hugetlbfs_put_super(struct super_block *sb)
970 {
971 struct hugetlbfs_sb_info *sbi = HUGETLBFS_SB(sb);
972
973 if (sbi) {
974 sb->s_fs_info = NULL;
975
976 if (sbi->spool)
977 hugepage_put_subpool(sbi->spool);
978
979 kfree(sbi);
980 }
981 }
982
983 static inline int hugetlbfs_dec_free_inodes(struct hugetlbfs_sb_info *sbinfo)
984 {
985 if (sbinfo->free_inodes >= 0) {
986 spin_lock(&sbinfo->stat_lock);
987 if (unlikely(!sbinfo->free_inodes)) {
988 spin_unlock(&sbinfo->stat_lock);
989 return 0;
990 }
991 sbinfo->free_inodes--;
992 spin_unlock(&sbinfo->stat_lock);
993 }
994
995 return 1;
996 }
997
998 static void hugetlbfs_inc_free_inodes(struct hugetlbfs_sb_info *sbinfo)
999 {
1000 if (sbinfo->free_inodes >= 0) {
1001 spin_lock(&sbinfo->stat_lock);
1002 sbinfo->free_inodes++;
1003 spin_unlock(&sbinfo->stat_lock);
1004 }
1005 }
1006
1007
1008 static struct kmem_cache *hugetlbfs_inode_cachep;
1009
1010 static struct inode *hugetlbfs_alloc_inode(struct super_block *sb)
1011 {
1012 struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(sb);
1013 struct hugetlbfs_inode_info *p;
1014
1015 if (unlikely(!hugetlbfs_dec_free_inodes(sbinfo)))
1016 return NULL;
1017 p = kmem_cache_alloc(hugetlbfs_inode_cachep, GFP_KERNEL);
1018 if (unlikely(!p)) {
1019 hugetlbfs_inc_free_inodes(sbinfo);
1020 return NULL;
1021 }
1022
1023 /*
1024 * Any time after allocation, hugetlbfs_destroy_inode can be called
1025 * for the inode. mpol_free_shared_policy is unconditionally called
1026 * as part of hugetlbfs_destroy_inode. So, initialize policy here
1027 * in case of a quick call to destroy.
1028 *
1029 * Note that the policy is initialized even if we are creating a
1030 * private inode. This simplifies hugetlbfs_destroy_inode.
1031 */
1032 mpol_shared_policy_init(&p->policy, NULL);
1033
1034 return &p->vfs_inode;
1035 }
1036
1037 static void hugetlbfs_i_callback(struct rcu_head *head)
1038 {
1039 struct inode *inode = container_of(head, struct inode, i_rcu);
1040 kmem_cache_free(hugetlbfs_inode_cachep, HUGETLBFS_I(inode));
1041 }
1042
1043 static void hugetlbfs_destroy_inode(struct inode *inode)
1044 {
1045 hugetlbfs_inc_free_inodes(HUGETLBFS_SB(inode->i_sb));
1046 mpol_free_shared_policy(&HUGETLBFS_I(inode)->policy);
1047 call_rcu(&inode->i_rcu, hugetlbfs_i_callback);
1048 }
1049
1050 static const struct address_space_operations hugetlbfs_aops = {
1051 .write_begin = hugetlbfs_write_begin,
1052 .write_end = hugetlbfs_write_end,
1053 .set_page_dirty = hugetlbfs_set_page_dirty,
1054 .migratepage = hugetlbfs_migrate_page,
1055 .error_remove_page = hugetlbfs_error_remove_page,
1056 };
1057
1058
1059 static void init_once(void *foo)
1060 {
1061 struct hugetlbfs_inode_info *ei = (struct hugetlbfs_inode_info *)foo;
1062
1063 inode_init_once(&ei->vfs_inode);
1064 }
1065
1066 const struct file_operations hugetlbfs_file_operations = {
1067 .read_iter = hugetlbfs_read_iter,
1068 .mmap = hugetlbfs_file_mmap,
1069 .fsync = noop_fsync,
1070 .get_unmapped_area = hugetlb_get_unmapped_area,
1071 .llseek = default_llseek,
1072 .fallocate = hugetlbfs_fallocate,
1073 };
1074
1075 static const struct inode_operations hugetlbfs_dir_inode_operations = {
1076 .create = hugetlbfs_create,
1077 .lookup = simple_lookup,
1078 .link = simple_link,
1079 .unlink = simple_unlink,
1080 .symlink = hugetlbfs_symlink,
1081 .mkdir = hugetlbfs_mkdir,
1082 .rmdir = simple_rmdir,
1083 .mknod = hugetlbfs_mknod,
1084 .rename = simple_rename,
1085 .setattr = hugetlbfs_setattr,
1086 };
1087
1088 static const struct inode_operations hugetlbfs_inode_operations = {
1089 .setattr = hugetlbfs_setattr,
1090 };
1091
1092 static const struct super_operations hugetlbfs_ops = {
1093 .alloc_inode = hugetlbfs_alloc_inode,
1094 .destroy_inode = hugetlbfs_destroy_inode,
1095 .evict_inode = hugetlbfs_evict_inode,
1096 .statfs = hugetlbfs_statfs,
1097 .put_super = hugetlbfs_put_super,
1098 .show_options = hugetlbfs_show_options,
1099 };
1100
1101 enum hugetlbfs_size_type { NO_SIZE, SIZE_STD, SIZE_PERCENT };
1102
1103 /*
1104 * Convert size option passed from command line to number of huge pages
1105 * in the pool specified by hstate. Size option could be in bytes
1106 * (val_type == SIZE_STD) or percentage of the pool (val_type == SIZE_PERCENT).
1107 */
1108 static long
1109 hugetlbfs_size_to_hpages(struct hstate *h, unsigned long long size_opt,
1110 enum hugetlbfs_size_type val_type)
1111 {
1112 if (val_type == NO_SIZE)
1113 return -1;
1114
1115 if (val_type == SIZE_PERCENT) {
1116 size_opt <<= huge_page_shift(h);
1117 size_opt *= h->max_huge_pages;
1118 do_div(size_opt, 100);
1119 }
1120
1121 size_opt >>= huge_page_shift(h);
1122 return size_opt;
1123 }
1124
1125 static int
1126 hugetlbfs_parse_options(char *options, struct hugetlbfs_config *pconfig)
1127 {
1128 char *p, *rest;
1129 substring_t args[MAX_OPT_ARGS];
1130 int option;
1131 unsigned long long max_size_opt = 0, min_size_opt = 0;
1132 enum hugetlbfs_size_type max_val_type = NO_SIZE, min_val_type = NO_SIZE;
1133
1134 if (!options)
1135 return 0;
1136
1137 while ((p = strsep(&options, ",")) != NULL) {
1138 int token;
1139 if (!*p)
1140 continue;
1141
1142 token = match_token(p, tokens, args);
1143 switch (token) {
1144 case Opt_uid:
1145 if (match_int(&args[0], &option))
1146 goto bad_val;
1147 pconfig->uid = make_kuid(current_user_ns(), option);
1148 if (!uid_valid(pconfig->uid))
1149 goto bad_val;
1150 break;
1151
1152 case Opt_gid:
1153 if (match_int(&args[0], &option))
1154 goto bad_val;
1155 pconfig->gid = make_kgid(current_user_ns(), option);
1156 if (!gid_valid(pconfig->gid))
1157 goto bad_val;
1158 break;
1159
1160 case Opt_mode:
1161 if (match_octal(&args[0], &option))
1162 goto bad_val;
1163 pconfig->mode = option & 01777U;
1164 break;
1165
1166 case Opt_size: {
1167 /* memparse() will accept a K/M/G without a digit */
1168 if (!isdigit(*args[0].from))
1169 goto bad_val;
1170 max_size_opt = memparse(args[0].from, &rest);
1171 max_val_type = SIZE_STD;
1172 if (*rest == '%')
1173 max_val_type = SIZE_PERCENT;
1174 break;
1175 }
1176
1177 case Opt_nr_inodes:
1178 /* memparse() will accept a K/M/G without a digit */
1179 if (!isdigit(*args[0].from))
1180 goto bad_val;
1181 pconfig->nr_inodes = memparse(args[0].from, &rest);
1182 break;
1183
1184 case Opt_pagesize: {
1185 unsigned long ps;
1186 ps = memparse(args[0].from, &rest);
1187 pconfig->hstate = size_to_hstate(ps);
1188 if (!pconfig->hstate) {
1189 pr_err("Unsupported page size %lu MB\n",
1190 ps >> 20);
1191 return -EINVAL;
1192 }
1193 break;
1194 }
1195
1196 case Opt_min_size: {
1197 /* memparse() will accept a K/M/G without a digit */
1198 if (!isdigit(*args[0].from))
1199 goto bad_val;
1200 min_size_opt = memparse(args[0].from, &rest);
1201 min_val_type = SIZE_STD;
1202 if (*rest == '%')
1203 min_val_type = SIZE_PERCENT;
1204 break;
1205 }
1206
1207 default:
1208 pr_err("Bad mount option: \"%s\"\n", p);
1209 return -EINVAL;
1210 break;
1211 }
1212 }
1213
1214 /*
1215 * Use huge page pool size (in hstate) to convert the size
1216 * options to number of huge pages. If NO_SIZE, -1 is returned.
1217 */
1218 pconfig->max_hpages = hugetlbfs_size_to_hpages(pconfig->hstate,
1219 max_size_opt, max_val_type);
1220 pconfig->min_hpages = hugetlbfs_size_to_hpages(pconfig->hstate,
1221 min_size_opt, min_val_type);
1222
1223 /*
1224 * If max_size was specified, then min_size must be smaller
1225 */
1226 if (max_val_type > NO_SIZE &&
1227 pconfig->min_hpages > pconfig->max_hpages) {
1228 pr_err("minimum size can not be greater than maximum size\n");
1229 return -EINVAL;
1230 }
1231
1232 return 0;
1233
1234 bad_val:
1235 pr_err("Bad value '%s' for mount option '%s'\n", args[0].from, p);
1236 return -EINVAL;
1237 }
1238
1239 static int
1240 hugetlbfs_fill_super(struct super_block *sb, void *data, int silent)
1241 {
1242 int ret;
1243 struct hugetlbfs_config config;
1244 struct hugetlbfs_sb_info *sbinfo;
1245
1246 config.max_hpages = -1; /* No limit on size by default */
1247 config.nr_inodes = -1; /* No limit on number of inodes by default */
1248 config.uid = current_fsuid();
1249 config.gid = current_fsgid();
1250 config.mode = 0755;
1251 config.hstate = &default_hstate;
1252 config.min_hpages = -1; /* No default minimum size */
1253 ret = hugetlbfs_parse_options(data, &config);
1254 if (ret)
1255 return ret;
1256
1257 sbinfo = kmalloc(sizeof(struct hugetlbfs_sb_info), GFP_KERNEL);
1258 if (!sbinfo)
1259 return -ENOMEM;
1260 sb->s_fs_info = sbinfo;
1261 sbinfo->hstate = config.hstate;
1262 spin_lock_init(&sbinfo->stat_lock);
1263 sbinfo->max_inodes = config.nr_inodes;
1264 sbinfo->free_inodes = config.nr_inodes;
1265 sbinfo->spool = NULL;
1266 sbinfo->uid = config.uid;
1267 sbinfo->gid = config.gid;
1268 sbinfo->mode = config.mode;
1269
1270 /*
1271 * Allocate and initialize subpool if maximum or minimum size is
1272 * specified. Any needed reservations (for minimim size) are taken
1273 * taken when the subpool is created.
1274 */
1275 if (config.max_hpages != -1 || config.min_hpages != -1) {
1276 sbinfo->spool = hugepage_new_subpool(config.hstate,
1277 config.max_hpages,
1278 config.min_hpages);
1279 if (!sbinfo->spool)
1280 goto out_free;
1281 }
1282 sb->s_maxbytes = MAX_LFS_FILESIZE;
1283 sb->s_blocksize = huge_page_size(config.hstate);
1284 sb->s_blocksize_bits = huge_page_shift(config.hstate);
1285 sb->s_magic = HUGETLBFS_MAGIC;
1286 sb->s_op = &hugetlbfs_ops;
1287 sb->s_time_gran = 1;
1288 sb->s_root = d_make_root(hugetlbfs_get_root(sb, &config));
1289 if (!sb->s_root)
1290 goto out_free;
1291 return 0;
1292 out_free:
1293 kfree(sbinfo->spool);
1294 kfree(sbinfo);
1295 return -ENOMEM;
1296 }
1297
1298 static struct dentry *hugetlbfs_mount(struct file_system_type *fs_type,
1299 int flags, const char *dev_name, void *data)
1300 {
1301 return mount_nodev(fs_type, flags, data, hugetlbfs_fill_super);
1302 }
1303
1304 static struct file_system_type hugetlbfs_fs_type = {
1305 .name = "hugetlbfs",
1306 .mount = hugetlbfs_mount,
1307 .kill_sb = kill_litter_super,
1308 };
1309
1310 static struct vfsmount *hugetlbfs_vfsmount[HUGE_MAX_HSTATE];
1311
1312 static int can_do_hugetlb_shm(void)
1313 {
1314 kgid_t shm_group;
1315 shm_group = make_kgid(&init_user_ns, sysctl_hugetlb_shm_group);
1316 return capable(CAP_IPC_LOCK) || in_group_p(shm_group);
1317 }
1318
1319 static int get_hstate_idx(int page_size_log)
1320 {
1321 struct hstate *h = hstate_sizelog(page_size_log);
1322
1323 if (!h)
1324 return -1;
1325 return h - hstates;
1326 }
1327
1328 /*
1329 * Note that size should be aligned to proper hugepage size in caller side,
1330 * otherwise hugetlb_reserve_pages reserves one less hugepages than intended.
1331 */
1332 struct file *hugetlb_file_setup(const char *name, size_t size,
1333 vm_flags_t acctflag, struct user_struct **user,
1334 int creat_flags, int page_size_log)
1335 {
1336 struct inode *inode;
1337 struct vfsmount *mnt;
1338 int hstate_idx;
1339 struct file *file;
1340
1341 hstate_idx = get_hstate_idx(page_size_log);
1342 if (hstate_idx < 0)
1343 return ERR_PTR(-ENODEV);
1344
1345 *user = NULL;
1346 mnt = hugetlbfs_vfsmount[hstate_idx];
1347 if (!mnt)
1348 return ERR_PTR(-ENOENT);
1349
1350 if (creat_flags == HUGETLB_SHMFS_INODE && !can_do_hugetlb_shm()) {
1351 *user = current_user();
1352 if (user_shm_lock(size, *user)) {
1353 task_lock(current);
1354 pr_warn_once("%s (%d): Using mlock ulimits for SHM_HUGETLB is deprecated\n",
1355 current->comm, current->pid);
1356 task_unlock(current);
1357 } else {
1358 *user = NULL;
1359 return ERR_PTR(-EPERM);
1360 }
1361 }
1362
1363 file = ERR_PTR(-ENOSPC);
1364 inode = hugetlbfs_get_inode(mnt->mnt_sb, NULL, S_IFREG | S_IRWXUGO, 0);
1365 if (!inode)
1366 goto out;
1367 if (creat_flags == HUGETLB_SHMFS_INODE)
1368 inode->i_flags |= S_PRIVATE;
1369
1370 inode->i_size = size;
1371 clear_nlink(inode);
1372
1373 if (hugetlb_reserve_pages(inode, 0,
1374 size >> huge_page_shift(hstate_inode(inode)), NULL,
1375 acctflag))
1376 file = ERR_PTR(-ENOMEM);
1377 else
1378 file = alloc_file_pseudo(inode, mnt, name, O_RDWR,
1379 &hugetlbfs_file_operations);
1380 if (!IS_ERR(file))
1381 return file;
1382
1383 iput(inode);
1384 out:
1385 if (*user) {
1386 user_shm_unlock(size, *user);
1387 *user = NULL;
1388 }
1389 return file;
1390 }
1391
1392 static int __init init_hugetlbfs_fs(void)
1393 {
1394 struct hstate *h;
1395 int error;
1396 int i;
1397
1398 if (!hugepages_supported()) {
1399 pr_info("disabling because there are no supported hugepage sizes\n");
1400 return -ENOTSUPP;
1401 }
1402
1403 error = -ENOMEM;
1404 hugetlbfs_inode_cachep = kmem_cache_create("hugetlbfs_inode_cache",
1405 sizeof(struct hugetlbfs_inode_info),
1406 0, SLAB_ACCOUNT, init_once);
1407 if (hugetlbfs_inode_cachep == NULL)
1408 goto out2;
1409
1410 error = register_filesystem(&hugetlbfs_fs_type);
1411 if (error)
1412 goto out;
1413
1414 i = 0;
1415 for_each_hstate(h) {
1416 char buf[50];
1417 unsigned ps_kb = 1U << (h->order + PAGE_SHIFT - 10);
1418
1419 snprintf(buf, sizeof(buf), "pagesize=%uK", ps_kb);
1420 hugetlbfs_vfsmount[i] = kern_mount_data(&hugetlbfs_fs_type,
1421 buf);
1422
1423 if (IS_ERR(hugetlbfs_vfsmount[i])) {
1424 pr_err("Cannot mount internal hugetlbfs for "
1425 "page size %uK", ps_kb);
1426 error = PTR_ERR(hugetlbfs_vfsmount[i]);
1427 hugetlbfs_vfsmount[i] = NULL;
1428 }
1429 i++;
1430 }
1431 /* Non default hstates are optional */
1432 if (!IS_ERR_OR_NULL(hugetlbfs_vfsmount[default_hstate_idx]))
1433 return 0;
1434
1435 out:
1436 kmem_cache_destroy(hugetlbfs_inode_cachep);
1437 out2:
1438 return error;
1439 }
1440 fs_initcall(init_hugetlbfs_fs)
1441
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.