1 /* 2 * linux/fs/block_dev.c 3 * 4 * Copyright (C) 1991, 1992 Linus Torvalds 5 * Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE 6 */ 7 8 #include <linux/init.h> 9 #include <linux/mm.h> 10 #include <linux/fcntl.h> 11 #include <linux/slab.h> 12 #include <linux/kmod.h> 13 #include <linux/major.h> 14 #include <linux/device_cgroup.h> 15 #include <linux/highmem.h> 16 #include <linux/blkdev.h> 17 #include <linux/backing-dev.h> 18 #include <linux/module.h> 19 #include <linux/blkpg.h> 20 #include <linux/magic.h> 21 #include <linux/dax.h> 22 #include <linux/buffer_head.h> 23 #include <linux/swap.h> 24 #include <linux/pagevec.h> 25 #include <linux/writeback.h> 26 #include <linux/mpage.h> 27 #include <linux/mount.h> 28 #include <linux/uio.h> 29 #include <linux/namei.h> 30 #include <linux/log2.h> 31 #include <linux/cleancache.h> 32 #include <linux/dax.h> 33 #include <linux/badblocks.h> 34 #include <linux/task_io_accounting_ops.h> 35 #include <linux/falloc.h> 36 #include <linux/uaccess.h> 37 #include "internal.h" 38 39 struct bdev_inode { 40 struct block_device bdev; 41 struct inode vfs_inode; 42 }; 43 44 static const struct address_space_operations def_blk_aops; 45 46 static inline struct bdev_inode *BDEV_I(struct inode *inode) 47 { 48 return container_of(inode, struct bdev_inode, vfs_inode); 49 } 50 51 struct block_device *I_BDEV(struct inode *inode) 52 { 53 return &BDEV_I(inode)->bdev; 54 } 55 EXPORT_SYMBOL(I_BDEV); 56 57 static void bdev_write_inode(struct block_device *bdev) 58 { 59 struct inode *inode = bdev->bd_inode; 60 int ret; 61 62 spin_lock(&inode->i_lock); 63 while (inode->i_state & I_DIRTY) { 64 spin_unlock(&inode->i_lock); 65 ret = write_inode_now(inode, true); 66 if (ret) { 67 char name[BDEVNAME_SIZE]; 68 pr_warn_ratelimited("VFS: Dirty inode writeback failed " 69 "for block device %s (err=%d).\n", 70 bdevname(bdev, name), ret); 71 } 72 spin_lock(&inode->i_lock); 73 } 74 spin_unlock(&inode->i_lock); 75 } 76 77 /* Kill _all_ buffers and pagecache , dirty or not.. */ 78 void kill_bdev(struct block_device *bdev) 79 { 80 struct address_space *mapping = bdev->bd_inode->i_mapping; 81 82 if (mapping->nrpages == 0 && mapping->nrexceptional == 0) 83 return; 84 85 invalidate_bh_lrus(); 86 truncate_inode_pages(mapping, 0); 87 } 88 EXPORT_SYMBOL(kill_bdev); 89 90 /* Invalidate clean unused buffers and pagecache. */ 91 void invalidate_bdev(struct block_device *bdev) 92 { 93 struct address_space *mapping = bdev->bd_inode->i_mapping; 94 95 if (mapping->nrpages) { 96 invalidate_bh_lrus(); 97 lru_add_drain_all(); /* make sure all lru add caches are flushed */ 98 invalidate_mapping_pages(mapping, 0, -1); 99 } 100 /* 99% of the time, we don't need to flush the cleancache on the bdev. 101 * But, for the strange corners, lets be cautious 102 */ 103 cleancache_invalidate_inode(mapping); 104 } 105 EXPORT_SYMBOL(invalidate_bdev); 106 107 int set_blocksize(struct block_device *bdev, int size) 108 { 109 /* Size must be a power of two, and between 512 and PAGE_SIZE */ 110 if (size > PAGE_SIZE || size < 512 || !is_power_of_2(size)) 111 return -EINVAL; 112 113 /* Size cannot be smaller than the size supported by the device */ 114 if (size < bdev_logical_block_size(bdev)) 115 return -EINVAL; 116 117 /* Don't change the size if it is same as current */ 118 if (bdev->bd_block_size != size) { 119 sync_blockdev(bdev); 120 bdev->bd_block_size = size; 121 bdev->bd_inode->i_blkbits = blksize_bits(size); 122 kill_bdev(bdev); 123 } 124 return 0; 125 } 126 127 EXPORT_SYMBOL(set_blocksize); 128 129 int sb_set_blocksize(struct super_block *sb, int size) 130 { 131 if (set_blocksize(sb->s_bdev, size)) 132 return 0; 133 /* If we get here, we know size is power of two 134 * and it's value is between 512 and PAGE_SIZE */ 135 sb->s_blocksize = size; 136 sb->s_blocksize_bits = blksize_bits(size); 137 return sb->s_blocksize; 138 } 139 140 EXPORT_SYMBOL(sb_set_blocksize); 141 142 int sb_min_blocksize(struct super_block *sb, int size) 143 { 144 int minsize = bdev_logical_block_size(sb->s_bdev); 145 if (size < minsize) 146 size = minsize; 147 return sb_set_blocksize(sb, size); 148 } 149 150 EXPORT_SYMBOL(sb_min_blocksize); 151 152 static int 153 blkdev_get_block(struct inode *inode, sector_t iblock, 154 struct buffer_head *bh, int create) 155 { 156 bh->b_bdev = I_BDEV(inode); 157 bh->b_blocknr = iblock; 158 set_buffer_mapped(bh); 159 return 0; 160 } 161 162 static struct inode *bdev_file_inode(struct file *file) 163 { 164 return file->f_mapping->host; 165 } 166 167 static unsigned int dio_bio_write_op(struct kiocb *iocb) 168 { 169 unsigned int op = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE; 170 171 /* avoid the need for a I/O completion work item */ 172 if (iocb->ki_flags & IOCB_DSYNC) 173 op |= REQ_FUA; 174 return op; 175 } 176 177 #define DIO_INLINE_BIO_VECS 4 178 179 static void blkdev_bio_end_io_simple(struct bio *bio) 180 { 181 struct task_struct *waiter = bio->bi_private; 182 183 WRITE_ONCE(bio->bi_private, NULL); 184 wake_up_process(waiter); 185 } 186 187 static ssize_t 188 __blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter, 189 int nr_pages) 190 { 191 struct file *file = iocb->ki_filp; 192 struct block_device *bdev = I_BDEV(bdev_file_inode(file)); 193 struct bio_vec inline_vecs[DIO_INLINE_BIO_VECS], *vecs, *bvec; 194 loff_t pos = iocb->ki_pos; 195 bool should_dirty = false; 196 struct bio bio; 197 ssize_t ret; 198 blk_qc_t qc; 199 int i; 200 201 if ((pos | iov_iter_alignment(iter)) & 202 (bdev_logical_block_size(bdev) - 1)) 203 return -EINVAL; 204 205 if (nr_pages <= DIO_INLINE_BIO_VECS) 206 vecs = inline_vecs; 207 else { 208 vecs = kmalloc(nr_pages * sizeof(struct bio_vec), GFP_KERNEL); 209 if (!vecs) 210 return -ENOMEM; 211 } 212 213 bio_init(&bio, vecs, nr_pages); 214 bio_set_dev(&bio, bdev); 215 bio.bi_iter.bi_sector = pos >> 9; 216 bio.bi_write_hint = iocb->ki_hint; 217 bio.bi_private = current; 218 bio.bi_end_io = blkdev_bio_end_io_simple; 219 220 ret = bio_iov_iter_get_pages(&bio, iter); 221 if (unlikely(ret)) 222 return ret; 223 ret = bio.bi_iter.bi_size; 224 225 if (iov_iter_rw(iter) == READ) { 226 bio.bi_opf = REQ_OP_READ; 227 if (iter_is_iovec(iter)) 228 should_dirty = true; 229 } else { 230 bio.bi_opf = dio_bio_write_op(iocb); 231 task_io_account_write(ret); 232 } 233 234 qc = submit_bio(&bio); 235 for (;;) { 236 set_current_state(TASK_UNINTERRUPTIBLE); 237 if (!READ_ONCE(bio.bi_private)) 238 break; 239 if (!(iocb->ki_flags & IOCB_HIPRI) || 240 !blk_poll(bdev_get_queue(bdev), qc)) 241 io_schedule(); 242 } 243 __set_current_state(TASK_RUNNING); 244 245 bio_for_each_segment_all(bvec, &bio, i) { 246 if (should_dirty && !PageCompound(bvec->bv_page)) 247 set_page_dirty_lock(bvec->bv_page); 248 put_page(bvec->bv_page); 249 } 250 251 if (vecs != inline_vecs) 252 kfree(vecs); 253 254 if (unlikely(bio.bi_status)) 255 ret = blk_status_to_errno(bio.bi_status); 256 257 bio_uninit(&bio); 258 259 return ret; 260 } 261 262 struct blkdev_dio { 263 union { 264 struct kiocb *iocb; 265 struct task_struct *waiter; 266 }; 267 size_t size; 268 atomic_t ref; 269 bool multi_bio : 1; 270 bool should_dirty : 1; 271 bool is_sync : 1; 272 struct bio bio; 273 }; 274 275 static struct bio_set *blkdev_dio_pool __read_mostly; 276 277 static void blkdev_bio_end_io(struct bio *bio) 278 { 279 struct blkdev_dio *dio = bio->bi_private; 280 bool should_dirty = dio->should_dirty; 281 282 if (dio->multi_bio && !atomic_dec_and_test(&dio->ref)) { 283 if (bio->bi_status && !dio->bio.bi_status) 284 dio->bio.bi_status = bio->bi_status; 285 } else { 286 if (!dio->is_sync) { 287 struct kiocb *iocb = dio->iocb; 288 ssize_t ret; 289 290 if (likely(!dio->bio.bi_status)) { 291 ret = dio->size; 292 iocb->ki_pos += ret; 293 } else { 294 ret = blk_status_to_errno(dio->bio.bi_status); 295 } 296 297 dio->iocb->ki_complete(iocb, ret, 0); 298 bio_put(&dio->bio); 299 } else { 300 struct task_struct *waiter = dio->waiter; 301 302 WRITE_ONCE(dio->waiter, NULL); 303 wake_up_process(waiter); 304 } 305 } 306 307 if (should_dirty) { 308 bio_check_pages_dirty(bio); 309 } else { 310 struct bio_vec *bvec; 311 int i; 312 313 bio_for_each_segment_all(bvec, bio, i) 314 put_page(bvec->bv_page); 315 bio_put(bio); 316 } 317 } 318 319 static ssize_t 320 __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages) 321 { 322 struct file *file = iocb->ki_filp; 323 struct inode *inode = bdev_file_inode(file); 324 struct block_device *bdev = I_BDEV(inode); 325 struct blk_plug plug; 326 struct blkdev_dio *dio; 327 struct bio *bio; 328 bool is_read = (iov_iter_rw(iter) == READ), is_sync; 329 loff_t pos = iocb->ki_pos; 330 blk_qc_t qc = BLK_QC_T_NONE; 331 int ret = 0; 332 333 if ((pos | iov_iter_alignment(iter)) & 334 (bdev_logical_block_size(bdev) - 1)) 335 return -EINVAL; 336 337 bio = bio_alloc_bioset(GFP_KERNEL, nr_pages, blkdev_dio_pool); 338 bio_get(bio); /* extra ref for the completion handler */ 339 340 dio = container_of(bio, struct blkdev_dio, bio); 341 dio->is_sync = is_sync = is_sync_kiocb(iocb); 342 if (dio->is_sync) 343 dio->waiter = current; 344 else 345 dio->iocb = iocb; 346 347 dio->size = 0; 348 dio->multi_bio = false; 349 dio->should_dirty = is_read && (iter->type == ITER_IOVEC); 350 351 blk_start_plug(&plug); 352 for (;;) { 353 bio_set_dev(bio, bdev); 354 bio->bi_iter.bi_sector = pos >> 9; 355 bio->bi_write_hint = iocb->ki_hint; 356 bio->bi_private = dio; 357 bio->bi_end_io = blkdev_bio_end_io; 358 359 ret = bio_iov_iter_get_pages(bio, iter); 360 if (unlikely(ret)) { 361 bio->bi_status = BLK_STS_IOERR; 362 bio_endio(bio); 363 break; 364 } 365 366 if (is_read) { 367 bio->bi_opf = REQ_OP_READ; 368 if (dio->should_dirty) 369 bio_set_pages_dirty(bio); 370 } else { 371 bio->bi_opf = dio_bio_write_op(iocb); 372 task_io_account_write(bio->bi_iter.bi_size); 373 } 374 375 dio->size += bio->bi_iter.bi_size; 376 pos += bio->bi_iter.bi_size; 377 378 nr_pages = iov_iter_npages(iter, BIO_MAX_PAGES); 379 if (!nr_pages) { 380 qc = submit_bio(bio); 381 break; 382 } 383 384 if (!dio->multi_bio) { 385 dio->multi_bio = true; 386 atomic_set(&dio->ref, 2); 387 } else { 388 atomic_inc(&dio->ref); 389 } 390 391 submit_bio(bio); 392 bio = bio_alloc(GFP_KERNEL, nr_pages); 393 } 394 blk_finish_plug(&plug); 395 396 if (!is_sync) 397 return -EIOCBQUEUED; 398 399 for (;;) { 400 set_current_state(TASK_UNINTERRUPTIBLE); 401 if (!READ_ONCE(dio->waiter)) 402 break; 403 404 if (!(iocb->ki_flags & IOCB_HIPRI) || 405 !blk_poll(bdev_get_queue(bdev), qc)) 406 io_schedule(); 407 } 408 __set_current_state(TASK_RUNNING); 409 410 if (!ret) 411 ret = blk_status_to_errno(dio->bio.bi_status); 412 if (likely(!ret)) 413 ret = dio->size; 414 415 bio_put(&dio->bio); 416 return ret; 417 } 418 419 static ssize_t 420 blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter) 421 { 422 int nr_pages; 423 424 nr_pages = iov_iter_npages(iter, BIO_MAX_PAGES + 1); 425 if (!nr_pages) 426 return 0; 427 if (is_sync_kiocb(iocb) && nr_pages <= BIO_MAX_PAGES) 428 return __blkdev_direct_IO_simple(iocb, iter, nr_pages); 429 430 return __blkdev_direct_IO(iocb, iter, min(nr_pages, BIO_MAX_PAGES)); 431 } 432 433 static __init int blkdev_init(void) 434 { 435 blkdev_dio_pool = bioset_create(4, offsetof(struct blkdev_dio, bio), BIOSET_NEED_BVECS); 436 if (!blkdev_dio_pool) 437 return -ENOMEM; 438 return 0; 439 } 440 module_init(blkdev_init); 441 442 int __sync_blockdev(struct block_device *bdev, int wait) 443 { 444 if (!bdev) 445 return 0; 446 if (!wait) 447 return filemap_flush(bdev->bd_inode->i_mapping); 448 return filemap_write_and_wait(bdev->bd_inode->i_mapping); 449 } 450 451 /* 452 * Write out and wait upon all the dirty data associated with a block 453 * device via its mapping. Does not take the superblock lock. 454 */ 455 int sync_blockdev(struct block_device *bdev) 456 { 457 return __sync_blockdev(bdev, 1); 458 } 459 EXPORT_SYMBOL(sync_blockdev); 460 461 /* 462 * Write out and wait upon all dirty data associated with this 463 * device. Filesystem data as well as the underlying block 464 * device. Takes the superblock lock. 465 */ 466 int fsync_bdev(struct block_device *bdev) 467 { 468 struct super_block *sb = get_super(bdev); 469 if (sb) { 470 int res = sync_filesystem(sb); 471 drop_super(sb); 472 return res; 473 } 474 return sync_blockdev(bdev); 475 } 476 EXPORT_SYMBOL(fsync_bdev); 477 478 /** 479 * freeze_bdev -- lock a filesystem and force it into a consistent state 480 * @bdev: blockdevice to lock 481 * 482 * If a superblock is found on this device, we take the s_umount semaphore 483 * on it to make sure nobody unmounts until the snapshot creation is done. 484 * The reference counter (bd_fsfreeze_count) guarantees that only the last 485 * unfreeze process can unfreeze the frozen filesystem actually when multiple 486 * freeze requests arrive simultaneously. It counts up in freeze_bdev() and 487 * count down in thaw_bdev(). When it becomes 0, thaw_bdev() will unfreeze 488 * actually. 489 */ 490 struct super_block *freeze_bdev(struct block_device *bdev) 491 { 492 struct super_block *sb; 493 int error = 0; 494 495 mutex_lock(&bdev->bd_fsfreeze_mutex); 496 if (++bdev->bd_fsfreeze_count > 1) { 497 /* 498 * We don't even need to grab a reference - the first call 499 * to freeze_bdev grab an active reference and only the last 500 * thaw_bdev drops it. 501 */ 502 sb = get_super(bdev); 503 if (sb) 504 drop_super(sb); 505 mutex_unlock(&bdev->bd_fsfreeze_mutex); 506 return sb; 507 } 508 509 sb = get_active_super(bdev); 510 if (!sb) 511 goto out; 512 if (sb->s_op->freeze_super) 513 error = sb->s_op->freeze_super(sb); 514 else 515 error = freeze_super(sb); 516 if (error) { 517 deactivate_super(sb); 518 bdev->bd_fsfreeze_count--; 519 mutex_unlock(&bdev->bd_fsfreeze_mutex); 520 return ERR_PTR(error); 521 } 522 deactivate_super(sb); 523 out: 524 sync_blockdev(bdev); 525 mutex_unlock(&bdev->bd_fsfreeze_mutex); 526 return sb; /* thaw_bdev releases s->s_umount */ 527 } 528 EXPORT_SYMBOL(freeze_bdev); 529 530 /** 531 * thaw_bdev -- unlock filesystem 532 * @bdev: blockdevice to unlock 533 * @sb: associated superblock 534 * 535 * Unlocks the filesystem and marks it writeable again after freeze_bdev(). 536 */ 537 int thaw_bdev(struct block_device *bdev, struct super_block *sb) 538 { 539 int error = -EINVAL; 540 541 mutex_lock(&bdev->bd_fsfreeze_mutex); 542 if (!bdev->bd_fsfreeze_count) 543 goto out; 544 545 error = 0; 546 if (--bdev->bd_fsfreeze_count > 0) 547 goto out; 548 549 if (!sb) 550 goto out; 551 552 if (sb->s_op->thaw_super) 553 error = sb->s_op->thaw_super(sb); 554 else 555 error = thaw_super(sb); 556 if (error) 557 bdev->bd_fsfreeze_count++; 558 out: 559 mutex_unlock(&bdev->bd_fsfreeze_mutex); 560 return error; 561 } 562 EXPORT_SYMBOL(thaw_bdev); 563 564 static int blkdev_writepage(struct page *page, struct writeback_control *wbc) 565 { 566 return block_write_full_page(page, blkdev_get_block, wbc); 567 } 568 569 static int blkdev_readpage(struct file * file, struct page * page) 570 { 571 return block_read_full_page(page, blkdev_get_block); 572 } 573 574 static int blkdev_readpages(struct file *file, struct address_space *mapping, 575 struct list_head *pages, unsigned nr_pages) 576 { 577 return mpage_readpages(mapping, pages, nr_pages, blkdev_get_block); 578 } 579 580 static int blkdev_write_begin(struct file *file, struct address_space *mapping, 581 loff_t pos, unsigned len, unsigned flags, 582 struct page **pagep, void **fsdata) 583 { 584 return block_write_begin(mapping, pos, len, flags, pagep, 585 blkdev_get_block); 586 } 587 588 static int blkdev_write_end(struct file *file, struct address_space *mapping, 589 loff_t pos, unsigned len, unsigned copied, 590 struct page *page, void *fsdata) 591 { 592 int ret; 593 ret = block_write_end(file, mapping, pos, len, copied, page, fsdata); 594 595 unlock_page(page); 596 put_page(page); 597 598 return ret; 599 } 600 601 /* 602 * private llseek: 603 * for a block special file file_inode(file)->i_size is zero 604 * so we compute the size by hand (just as in block_read/write above) 605 */ 606 static loff_t block_llseek(struct file *file, loff_t offset, int whence) 607 { 608 struct inode *bd_inode = bdev_file_inode(file); 609 loff_t retval; 610 611 inode_lock(bd_inode); 612 retval = fixed_size_llseek(file, offset, whence, i_size_read(bd_inode)); 613 inode_unlock(bd_inode); 614 return retval; 615 } 616 617 int blkdev_fsync(struct file *filp, loff_t start, loff_t end, int datasync) 618 { 619 struct inode *bd_inode = bdev_file_inode(filp); 620 struct block_device *bdev = I_BDEV(bd_inode); 621 int error; 622 623 error = file_write_and_wait_range(filp, start, end); 624 if (error) 625 return error; 626 627 /* 628 * There is no need to serialise calls to blkdev_issue_flush with 629 * i_mutex and doing so causes performance issues with concurrent 630 * O_SYNC writers to a block device. 631 */ 632 error = blkdev_issue_flush(bdev, GFP_KERNEL, NULL); 633 if (error == -EOPNOTSUPP) 634 error = 0; 635 636 return error; 637 } 638 EXPORT_SYMBOL(blkdev_fsync); 639 640 /** 641 * bdev_read_page() - Start reading a page from a block device 642 * @bdev: The device to read the page from 643 * @sector: The offset on the device to read the page to (need not be aligned) 644 * @page: The page to read 645 * 646 * On entry, the page should be locked. It will be unlocked when the page 647 * has been read. If the block driver implements rw_page synchronously, 648 * that will be true on exit from this function, but it need not be. 649 * 650 * Errors returned by this function are usually "soft", eg out of memory, or 651 * queue full; callers should try a different route to read this page rather 652 * than propagate an error back up the stack. 653 * 654 * Return: negative errno if an error occurs, 0 if submission was successful. 655 */ 656 int bdev_read_page(struct block_device *bdev, sector_t sector, 657 struct page *page) 658 { 659 const struct block_device_operations *ops = bdev->bd_disk->fops; 660 int result = -EOPNOTSUPP; 661 662 if (!ops->rw_page || bdev_get_integrity(bdev)) 663 return result; 664 665 result = blk_queue_enter(bdev->bd_queue, 0); 666 if (result) 667 return result; 668 result = ops->rw_page(bdev, sector + get_start_sect(bdev), page, false); 669 blk_queue_exit(bdev->bd_queue); 670 return result; 671 } 672 EXPORT_SYMBOL_GPL(bdev_read_page); 673 674 /** 675 * bdev_write_page() - Start writing a page to a block device 676 * @bdev: The device to write the page to 677 * @sector: The offset on the device to write the page to (need not be aligned) 678 * @page: The page to write 679 * @wbc: The writeback_control for the write 680 * 681 * On entry, the page should be locked and not currently under writeback. 682 * On exit, if the write started successfully, the page will be unlocked and 683 * under writeback. If the write failed already (eg the driver failed to 684 * queue the page to the device), the page will still be locked. If the 685 * caller is a ->writepage implementation, it will need to unlock the page. 686 * 687 * Errors returned by this function are usually "soft", eg out of memory, or 688 * queue full; callers should try a different route to write this page rather 689 * than propagate an error back up the stack. 690 * 691 * Return: negative errno if an error occurs, 0 if submission was successful. 692 */ 693 int bdev_write_page(struct block_device *bdev, sector_t sector, 694 struct page *page, struct writeback_control *wbc) 695 { 696 int result; 697 const struct block_device_operations *ops = bdev->bd_disk->fops; 698 699 if (!ops->rw_page || bdev_get_integrity(bdev)) 700 return -EOPNOTSUPP; 701 result = blk_queue_enter(bdev->bd_queue, 0); 702 if (result) 703 return result; 704 705 set_page_writeback(page); 706 result = ops->rw_page(bdev, sector + get_start_sect(bdev), page, true); 707 if (result) { 708 end_page_writeback(page); 709 } else { 710 clean_page_buffers(page); 711 unlock_page(page); 712 } 713 blk_queue_exit(bdev->bd_queue); 714 return result; 715 } 716 EXPORT_SYMBOL_GPL(bdev_write_page); 717 718 /* 719 * pseudo-fs 720 */ 721 722 static __cacheline_aligned_in_smp DEFINE_SPINLOCK(bdev_lock); 723 static struct kmem_cache * bdev_cachep __read_mostly; 724 725 static struct inode *bdev_alloc_inode(struct super_block *sb) 726 { 727 struct bdev_inode *ei = kmem_cache_alloc(bdev_cachep, GFP_KERNEL); 728 if (!ei) 729 return NULL; 730 return &ei->vfs_inode; 731 } 732 733 static void bdev_i_callback(struct rcu_head *head) 734 { 735 struct inode *inode = container_of(head, struct inode, i_rcu); 736 struct bdev_inode *bdi = BDEV_I(inode); 737 738 kmem_cache_free(bdev_cachep, bdi); 739 } 740 741 static void bdev_destroy_inode(struct inode *inode) 742 { 743 call_rcu(&inode->i_rcu, bdev_i_callback); 744 } 745 746 static void init_once(void *foo) 747 { 748 struct bdev_inode *ei = (struct bdev_inode *) foo; 749 struct block_device *bdev = &ei->bdev; 750 751 memset(bdev, 0, sizeof(*bdev)); 752 mutex_init(&bdev->bd_mutex); 753 INIT_LIST_HEAD(&bdev->bd_list); 754 #ifdef CONFIG_SYSFS 755 INIT_LIST_HEAD(&bdev->bd_holder_disks); 756 #endif 757 bdev->bd_bdi = &noop_backing_dev_info; 758 inode_init_once(&ei->vfs_inode); 759 /* Initialize mutex for freeze. */ 760 mutex_init(&bdev->bd_fsfreeze_mutex); 761 } 762 763 static void bdev_evict_inode(struct inode *inode) 764 { 765 struct block_device *bdev = &BDEV_I(inode)->bdev; 766 truncate_inode_pages_final(&inode->i_data); 767 invalidate_inode_buffers(inode); /* is it needed here? */ 768 clear_inode(inode); 769 spin_lock(&bdev_lock); 770 list_del_init(&bdev->bd_list); 771 spin_unlock(&bdev_lock); 772 /* Detach inode from wb early as bdi_put() may free bdi->wb */ 773 inode_detach_wb(inode); 774 if (bdev->bd_bdi != &noop_backing_dev_info) { 775 bdi_put(bdev->bd_bdi); 776 bdev->bd_bdi = &noop_backing_dev_info; 777 } 778 } 779 780 static const struct super_operations bdev_sops = { 781 .statfs = simple_statfs, 782 .alloc_inode = bdev_alloc_inode, 783 .destroy_inode = bdev_destroy_inode, 784 .drop_inode = generic_delete_inode, 785 .evict_inode = bdev_evict_inode, 786 }; 787 788 static struct dentry *bd_mount(struct file_system_type *fs_type, 789 int flags, const char *dev_name, void *data) 790 { 791 struct dentry *dent; 792 dent = mount_pseudo(fs_type, "bdev:", &bdev_sops, NULL, BDEVFS_MAGIC); 793 if (!IS_ERR(dent)) 794 dent->d_sb->s_iflags |= SB_I_CGROUPWB; 795 return dent; 796 } 797 798 static struct file_system_type bd_type = { 799 .name = "bdev", 800 .mount = bd_mount, 801 .kill_sb = kill_anon_super, 802 }; 803 804 struct super_block *blockdev_superblock __read_mostly; 805 EXPORT_SYMBOL_GPL(blockdev_superblock); 806 807 void __init bdev_cache_init(void) 808 { 809 int err; 810 static struct vfsmount *bd_mnt; 811 812 bdev_cachep = kmem_cache_create("bdev_cache", sizeof(struct bdev_inode), 813 0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| 814 SLAB_MEM_SPREAD|SLAB_ACCOUNT|SLAB_PANIC), 815 init_once); 816 err = register_filesystem(&bd_type); 817 if (err) 818 panic("Cannot register bdev pseudo-fs"); 819 bd_mnt = kern_mount(&bd_type); 820 if (IS_ERR(bd_mnt)) 821 panic("Cannot create bdev pseudo-fs"); 822 blockdev_superblock = bd_mnt->mnt_sb; /* For writeback */ 823 } 824 825 /* 826 * Most likely _very_ bad one - but then it's hardly critical for small 827 * /dev and can be fixed when somebody will need really large one. 828 * Keep in mind that it will be fed through icache hash function too. 829 */ 830 static inline unsigned long hash(dev_t dev) 831 { 832 return MAJOR(dev)+MINOR(dev); 833 } 834 835 static int bdev_test(struct inode *inode, void *data) 836 { 837 return BDEV_I(inode)->bdev.bd_dev == *(dev_t *)data; 838 } 839 840 static int bdev_set(struct inode *inode, void *data) 841 { 842 BDEV_I(inode)->bdev.bd_dev = *(dev_t *)data; 843 return 0; 844 } 845 846 static LIST_HEAD(all_bdevs); 847 848 /* 849 * If there is a bdev inode for this device, unhash it so that it gets evicted 850 * as soon as last inode reference is dropped. 851 */ 852 void bdev_unhash_inode(dev_t dev) 853 { 854 struct inode *inode; 855 856 inode = ilookup5(blockdev_superblock, hash(dev), bdev_test, &dev); 857 if (inode) { 858 remove_inode_hash(inode); 859 iput(inode); 860 } 861 } 862 863 struct block_device *bdget(dev_t dev) 864 { 865 struct block_device *bdev; 866 struct inode *inode; 867 868 inode = iget5_locked(blockdev_superblock, hash(dev), 869 bdev_test, bdev_set, &dev); 870 871 if (!inode) 872 return NULL; 873 874 bdev = &BDEV_I(inode)->bdev; 875 876 if (inode->i_state & I_NEW) { 877 bdev->bd_contains = NULL; 878 bdev->bd_super = NULL; 879 bdev->bd_inode = inode; 880 bdev->bd_block_size = i_blocksize(inode); 881 bdev->bd_part_count = 0; 882 bdev->bd_invalidated = 0; 883 inode->i_mode = S_IFBLK; 884 inode->i_rdev = dev; 885 inode->i_bdev = bdev; 886 inode->i_data.a_ops = &def_blk_aops; 887 mapping_set_gfp_mask(&inode->i_data, GFP_USER); 888 spin_lock(&bdev_lock); 889 list_add(&bdev->bd_list, &all_bdevs); 890 spin_unlock(&bdev_lock); 891 unlock_new_inode(inode); 892 } 893 return bdev; 894 } 895 896 EXPORT_SYMBOL(bdget); 897 898 /** 899 * bdgrab -- Grab a reference to an already referenced block device 900 * @bdev: Block device to grab a reference to. 901 */ 902 struct block_device *bdgrab(struct block_device *bdev) 903 { 904 ihold(bdev->bd_inode); 905 return bdev; 906 } 907 EXPORT_SYMBOL(bdgrab); 908 909 long nr_blockdev_pages(void) 910 { 911 struct block_device *bdev; 912 long ret = 0; 913 spin_lock(&bdev_lock); 914 list_for_each_entry(bdev, &all_bdevs, bd_list) { 915 ret += bdev->bd_inode->i_mapping->nrpages; 916 } 917 spin_unlock(&bdev_lock); 918 return ret; 919 } 920 921 void bdput(struct block_device *bdev) 922 { 923 iput(bdev->bd_inode); 924 } 925 926 EXPORT_SYMBOL(bdput); 927 928 static struct block_device *bd_acquire(struct inode *inode) 929 { 930 struct block_device *bdev; 931 932 spin_lock(&bdev_lock); 933 bdev = inode->i_bdev; 934 if (bdev && !inode_unhashed(bdev->bd_inode)) { 935 bdgrab(bdev); 936 spin_unlock(&bdev_lock); 937 return bdev; 938 } 939 spin_unlock(&bdev_lock); 940 941 /* 942 * i_bdev references block device inode that was already shut down 943 * (corresponding device got removed). Remove the reference and look 944 * up block device inode again just in case new device got 945 * reestablished under the same device number. 946 */ 947 if (bdev) 948 bd_forget(inode); 949 950 bdev = bdget(inode->i_rdev); 951 if (bdev) { 952 spin_lock(&bdev_lock); 953 if (!inode->i_bdev) { 954 /* 955 * We take an additional reference to bd_inode, 956 * and it's released in clear_inode() of inode. 957 * So, we can access it via ->i_mapping always 958 * without igrab(). 959 */ 960 bdgrab(bdev); 961 inode->i_bdev = bdev; 962 inode->i_mapping = bdev->bd_inode->i_mapping; 963 } 964 spin_unlock(&bdev_lock); 965 } 966 return bdev; 967 } 968 969 /* Call when you free inode */ 970 971 void bd_forget(struct inode *inode) 972 { 973 struct block_device *bdev = NULL; 974 975 spin_lock(&bdev_lock); 976 if (!sb_is_blkdev_sb(inode->i_sb)) 977 bdev = inode->i_bdev; 978 inode->i_bdev = NULL; 979 inode->i_mapping = &inode->i_data; 980 spin_unlock(&bdev_lock); 981 982 if (bdev) 983 bdput(bdev); 984 } 985 986 /** 987 * bd_may_claim - test whether a block device can be claimed 988 * @bdev: block device of interest 989 * @whole: whole block device containing @bdev, may equal @bdev 990 * @holder: holder trying to claim @bdev 991 * 992 * Test whether @bdev can be claimed by @holder. 993 * 994 * CONTEXT: 995 * spin_lock(&bdev_lock). 996 * 997 * RETURNS: 998 * %true if @bdev can be claimed, %false otherwise. 999 */ 1000 static bool bd_may_claim(struct block_device *bdev, struct block_device *whole, 1001 void *holder) 1002 { 1003 if (bdev->bd_holder == holder) 1004 return true; /* already a holder */ 1005 else if (bdev->bd_holder != NULL) 1006 return false; /* held by someone else */ 1007 else if (whole == bdev) 1008 return true; /* is a whole device which isn't held */ 1009 1010 else if (whole->bd_holder == bd_may_claim) 1011 return true; /* is a partition of a device that is being partitioned */ 1012 else if (whole->bd_holder != NULL) 1013 return false; /* is a partition of a held device */ 1014 else 1015 return true; /* is a partition of an un-held device */ 1016 } 1017 1018 /** 1019 * bd_prepare_to_claim - prepare to claim a block device 1020 * @bdev: block device of interest 1021 * @whole: the whole device containing @bdev, may equal @bdev 1022 * @holder: holder trying to claim @bdev 1023 * 1024 * Prepare to claim @bdev. This function fails if @bdev is already 1025 * claimed by another holder and waits if another claiming is in 1026 * progress. This function doesn't actually claim. On successful 1027 * return, the caller has ownership of bd_claiming and bd_holder[s]. 1028 * 1029 * CONTEXT: 1030 * spin_lock(&bdev_lock). Might release bdev_lock, sleep and regrab 1031 * it multiple times. 1032 * 1033 * RETURNS: 1034 * 0 if @bdev can be claimed, -EBUSY otherwise. 1035 */ 1036 static int bd_prepare_to_claim(struct block_device *bdev, 1037 struct block_device *whole, void *holder) 1038 { 1039 retry: 1040 /* if someone else claimed, fail */ 1041 if (!bd_may_claim(bdev, whole, holder)) 1042 return -EBUSY; 1043 1044 /* if claiming is already in progress, wait for it to finish */ 1045 if (whole->bd_claiming) { 1046 wait_queue_head_t *wq = bit_waitqueue(&whole->bd_claiming, 0); 1047 DEFINE_WAIT(wait); 1048 1049 prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE); 1050 spin_unlock(&bdev_lock); 1051 schedule(); 1052 finish_wait(wq, &wait); 1053 spin_lock(&bdev_lock); 1054 goto retry; 1055 } 1056 1057 /* yay, all mine */ 1058 return 0; 1059 } 1060 1061 static struct gendisk *bdev_get_gendisk(struct block_device *bdev, int *partno) 1062 { 1063 struct gendisk *disk = get_gendisk(bdev->bd_dev, partno); 1064 1065 if (!disk) 1066 return NULL; 1067 /* 1068 * Now that we hold gendisk reference we make sure bdev we looked up is 1069 * not stale. If it is, it means device got removed and created before 1070 * we looked up gendisk and we fail open in such case. Associating 1071 * unhashed bdev with newly created gendisk could lead to two bdevs 1072 * (and thus two independent caches) being associated with one device 1073 * which is bad. 1074 */ 1075 if (inode_unhashed(bdev->bd_inode)) { 1076 put_disk_and_module(disk); 1077 return NULL; 1078 } 1079 return disk; 1080 } 1081 1082 /** 1083 * bd_start_claiming - start claiming a block device 1084 * @bdev: block device of interest 1085 * @holder: holder trying to claim @bdev 1086 * 1087 * @bdev is about to be opened exclusively. Check @bdev can be opened 1088 * exclusively and mark that an exclusive open is in progress. Each 1089 * successful call to this function must be matched with a call to 1090 * either bd_finish_claiming() or bd_abort_claiming() (which do not 1091 * fail). 1092 * 1093 * This function is used to gain exclusive access to the block device 1094 * without actually causing other exclusive open attempts to fail. It 1095 * should be used when the open sequence itself requires exclusive 1096 * access but may subsequently fail. 1097 * 1098 * CONTEXT: 1099 * Might sleep. 1100 * 1101 * RETURNS: 1102 * Pointer to the block device containing @bdev on success, ERR_PTR() 1103 * value on failure. 1104 */ 1105 static struct block_device *bd_start_claiming(struct block_device *bdev, 1106 void *holder) 1107 { 1108 struct gendisk *disk; 1109 struct block_device *whole; 1110 int partno, err; 1111 1112 might_sleep(); 1113 1114 /* 1115 * @bdev might not have been initialized properly yet, look up 1116 * and grab the outer block device the hard way. 1117 */ 1118 disk = bdev_get_gendisk(bdev, &partno); 1119 if (!disk) 1120 return ERR_PTR(-ENXIO); 1121 1122 /* 1123 * Normally, @bdev should equal what's returned from bdget_disk() 1124 * if partno is 0; however, some drivers (floppy) use multiple 1125 * bdev's for the same physical device and @bdev may be one of the 1126 * aliases. Keep @bdev if partno is 0. This means claimer 1127 * tracking is broken for those devices but it has always been that 1128 * way. 1129 */ 1130 if (partno) 1131 whole = bdget_disk(disk, 0); 1132 else 1133 whole = bdgrab(bdev); 1134 1135 put_disk_and_module(disk); 1136 if (!whole) 1137 return ERR_PTR(-ENOMEM); 1138 1139 /* prepare to claim, if successful, mark claiming in progress */ 1140 spin_lock(&bdev_lock); 1141 1142 err = bd_prepare_to_claim(bdev, whole, holder); 1143 if (err == 0) { 1144 whole->bd_claiming = holder; 1145 spin_unlock(&bdev_lock); 1146 return whole; 1147 } else { 1148 spin_unlock(&bdev_lock); 1149 bdput(whole); 1150 return ERR_PTR(err); 1151 } 1152 } 1153 1154 #ifdef CONFIG_SYSFS 1155 struct bd_holder_disk { 1156 struct list_head list; 1157 struct gendisk *disk; 1158 int refcnt; 1159 }; 1160 1161 static struct bd_holder_disk *bd_find_holder_disk(struct block_device *bdev, 1162 struct gendisk *disk) 1163 { 1164 struct bd_holder_disk *holder; 1165 1166 list_for_each_entry(holder, &bdev->bd_holder_disks, list) 1167 if (holder->disk == disk) 1168 return holder; 1169 return NULL; 1170 } 1171 1172 static int add_symlink(struct kobject *from, struct kobject *to) 1173 { 1174 return sysfs_create_link(from, to, kobject_name(to)); 1175 } 1176 1177 static void del_symlink(struct kobject *from, struct kobject *to) 1178 { 1179 sysfs_remove_link(from, kobject_name(to)); 1180 } 1181 1182 /** 1183 * bd_link_disk_holder - create symlinks between holding disk and slave bdev 1184 * @bdev: the claimed slave bdev 1185 * @disk: the holding disk 1186 * 1187 * DON'T USE THIS UNLESS YOU'RE ALREADY USING IT. 1188 * 1189 * This functions creates the following sysfs symlinks. 1190 * 1191 * - from "slaves" directory of the holder @disk to the claimed @bdev 1192 * - from "holders" directory of the @bdev to the holder @disk 1193 * 1194 * For example, if /dev/dm-0 maps to /dev/sda and disk for dm-0 is 1195 * passed to bd_link_disk_holder(), then: 1196 * 1197 * /sys/block/dm-0/slaves/sda --> /sys/block/sda 1198 * /sys/block/sda/holders/dm-0 --> /sys/block/dm-0 1199 * 1200 * The caller must have claimed @bdev before calling this function and 1201 * ensure that both @bdev and @disk are valid during the creation and 1202 * lifetime of these symlinks. 1203 * 1204 * CONTEXT: 1205 * Might sleep. 1206 * 1207 * RETURNS: 1208 * 0 on success, -errno on failure. 1209 */ 1210 int bd_link_disk_holder(struct block_device *bdev, struct gendisk *disk) 1211 { 1212 struct bd_holder_disk *holder; 1213 int ret = 0; 1214 1215 mutex_lock(&bdev->bd_mutex); 1216 1217 WARN_ON_ONCE(!bdev->bd_holder); 1218 1219 /* FIXME: remove the following once add_disk() handles errors */ 1220 if (WARN_ON(!disk->slave_dir || !bdev->bd_part->holder_dir)) 1221 goto out_unlock; 1222 1223 holder = bd_find_holder_disk(bdev, disk); 1224 if (holder) { 1225 holder->refcnt++; 1226 goto out_unlock; 1227 } 1228 1229 holder = kzalloc(sizeof(*holder), GFP_KERNEL); 1230 if (!holder) { 1231 ret = -ENOMEM; 1232 goto out_unlock; 1233 } 1234 1235 INIT_LIST_HEAD(&holder->list); 1236 holder->disk = disk; 1237 holder->refcnt = 1; 1238 1239 ret = add_symlink(disk->slave_dir, &part_to_dev(bdev->bd_part)->kobj); 1240 if (ret) 1241 goto out_free; 1242 1243 ret = add_symlink(bdev->bd_part->holder_dir, &disk_to_dev(disk)->kobj); 1244 if (ret) 1245 goto out_del; 1246 /* 1247 * bdev could be deleted beneath us which would implicitly destroy 1248 * the holder directory. Hold on to it. 1249 */ 1250 kobject_get(bdev->bd_part->holder_dir); 1251 1252 list_add(&holder->list, &bdev->bd_holder_disks); 1253 goto out_unlock; 1254 1255 out_del: 1256 del_symlink(disk->slave_dir, &part_to_dev(bdev->bd_part)->kobj); 1257 out_free: 1258 kfree(holder); 1259 out_unlock: 1260 mutex_unlock(&bdev->bd_mutex); 1261 return ret; 1262 } 1263 EXPORT_SYMBOL_GPL(bd_link_disk_holder); 1264 1265 /** 1266 * bd_unlink_disk_holder - destroy symlinks created by bd_link_disk_holder() 1267 * @bdev: the calimed slave bdev 1268 * @disk: the holding disk 1269 * 1270 * DON'T USE THIS UNLESS YOU'RE ALREADY USING IT. 1271 * 1272 * CONTEXT: 1273 * Might sleep. 1274 */ 1275 void bd_unlink_disk_holder(struct block_device *bdev, struct gendisk *disk) 1276 { 1277 struct bd_holder_disk *holder; 1278 1279 mutex_lock(&bdev->bd_mutex); 1280 1281 holder = bd_find_holder_disk(bdev, disk); 1282 1283 if (!WARN_ON_ONCE(holder == NULL) && !--holder->refcnt) { 1284 del_symlink(disk->slave_dir, &part_to_dev(bdev->bd_part)->kobj); 1285 del_symlink(bdev->bd_part->holder_dir, 1286 &disk_to_dev(disk)->kobj); 1287 kobject_put(bdev->bd_part->holder_dir); 1288 list_del_init(&holder->list); 1289 kfree(holder); 1290 } 1291 1292 mutex_unlock(&bdev->bd_mutex); 1293 } 1294 EXPORT_SYMBOL_GPL(bd_unlink_disk_holder); 1295 #endif 1296 1297 /** 1298 * flush_disk - invalidates all buffer-cache entries on a disk 1299 * 1300 * @bdev: struct block device to be flushed 1301 * @kill_dirty: flag to guide handling of dirty inodes 1302 * 1303 * Invalidates all buffer-cache entries on a disk. It should be called 1304 * when a disk has been changed -- either by a media change or online 1305 * resize. 1306 */ 1307 static void flush_disk(struct block_device *bdev, bool kill_dirty) 1308 { 1309 if (__invalidate_device(bdev, kill_dirty)) { 1310 printk(KERN_WARNING "VFS: busy inodes on changed media or " 1311 "resized disk %s\n", 1312 bdev->bd_disk ? bdev->bd_disk->disk_name : ""); 1313 } 1314 1315 if (!bdev->bd_disk) 1316 return; 1317 if (disk_part_scan_enabled(bdev->bd_disk)) 1318 bdev->bd_invalidated = 1; 1319 } 1320 1321 /** 1322 * check_disk_size_change - checks for disk size change and adjusts bdev size. 1323 * @disk: struct gendisk to check 1324 * @bdev: struct bdev to adjust. 1325 * 1326 * This routine checks to see if the bdev size does not match the disk size 1327 * and adjusts it if it differs. When shrinking the bdev size, its all caches 1328 * are freed. 1329 */ 1330 void check_disk_size_change(struct gendisk *disk, struct block_device *bdev) 1331 { 1332 loff_t disk_size, bdev_size; 1333 1334 disk_size = (loff_t)get_capacity(disk) << 9; 1335 bdev_size = i_size_read(bdev->bd_inode); 1336 if (disk_size != bdev_size) { 1337 printk(KERN_INFO 1338 "%s: detected capacity change from %lld to %lld\n", 1339 disk->disk_name, bdev_size, disk_size); 1340 i_size_write(bdev->bd_inode, disk_size); 1341 if (bdev_size > disk_size) 1342 flush_disk(bdev, false); 1343 } 1344 } 1345 EXPORT_SYMBOL(check_disk_size_change); 1346 1347 /** 1348 * revalidate_disk - wrapper for lower-level driver's revalidate_disk call-back 1349 * @disk: struct gendisk to be revalidated 1350 * 1351 * This routine is a wrapper for lower-level driver's revalidate_disk 1352 * call-backs. It is used to do common pre and post operations needed 1353 * for all revalidate_disk operations. 1354 */ 1355 int revalidate_disk(struct gendisk *disk) 1356 { 1357 struct block_device *bdev; 1358 int ret = 0; 1359 1360 if (disk->fops->revalidate_disk) 1361 ret = disk->fops->revalidate_disk(disk); 1362 bdev = bdget_disk(disk, 0); 1363 if (!bdev) 1364 return ret; 1365 1366 mutex_lock(&bdev->bd_mutex); 1367 check_disk_size_change(disk, bdev); 1368 bdev->bd_invalidated = 0; 1369 mutex_unlock(&bdev->bd_mutex); 1370 bdput(bdev); 1371 return ret; 1372 } 1373 EXPORT_SYMBOL(revalidate_disk); 1374 1375 /* 1376 * This routine checks whether a removable media has been changed, 1377 * and invalidates all buffer-cache-entries in that case. This 1378 * is a relatively slow routine, so we have to try to minimize using 1379 * it. Thus it is called only upon a 'mount' or 'open'. This 1380 * is the best way of combining speed and utility, I think. 1381 * People changing diskettes in the middle of an operation deserve 1382 * to lose :-) 1383 */ 1384 int check_disk_change(struct block_device *bdev) 1385 { 1386 struct gendisk *disk = bdev->bd_disk; 1387 const struct block_device_operations *bdops = disk->fops; 1388 unsigned int events; 1389 1390 events = disk_clear_events(disk, DISK_EVENT_MEDIA_CHANGE | 1391 DISK_EVENT_EJECT_REQUEST); 1392 if (!(events & DISK_EVENT_MEDIA_CHANGE)) 1393 return 0; 1394 1395 flush_disk(bdev, true); 1396 if (bdops->revalidate_disk) 1397 bdops->revalidate_disk(bdev->bd_disk); 1398 return 1; 1399 } 1400 1401 EXPORT_SYMBOL(check_disk_change); 1402 1403 void bd_set_size(struct block_device *bdev, loff_t size) 1404 { 1405 unsigned bsize = bdev_logical_block_size(bdev); 1406 1407 inode_lock(bdev->bd_inode); 1408 i_size_write(bdev->bd_inode, size); 1409 inode_unlock(bdev->bd_inode); 1410 while (bsize < PAGE_SIZE) { 1411 if (size & bsize) 1412 break; 1413 bsize <<= 1; 1414 } 1415 bdev->bd_block_size = bsize; 1416 bdev->bd_inode->i_blkbits = blksize_bits(bsize); 1417 } 1418 EXPORT_SYMBOL(bd_set_size); 1419 1420 static void __blkdev_put(struct block_device *bdev, fmode_t mode, int for_part); 1421 1422 /* 1423 * bd_mutex locking: 1424 * 1425 * mutex_lock(part->bd_mutex) 1426 * mutex_lock_nested(whole->bd_mutex, 1) 1427 */ 1428 1429 static int __blkdev_get(struct block_device *bdev, fmode_t mode, int for_part) 1430 { 1431 struct gendisk *disk; 1432 int ret; 1433 int partno; 1434 int perm = 0; 1435 bool first_open = false; 1436 1437 if (mode & FMODE_READ) 1438 perm |= MAY_READ; 1439 if (mode & FMODE_WRITE) 1440 perm |= MAY_WRITE; 1441 /* 1442 * hooks: /n/, see "layering violations". 1443 */ 1444 if (!for_part) { 1445 ret = devcgroup_inode_permission(bdev->bd_inode, perm); 1446 if (ret != 0) { 1447 bdput(bdev); 1448 return ret; 1449 } 1450 } 1451 1452 restart: 1453 1454 ret = -ENXIO; 1455 disk = bdev_get_gendisk(bdev, &partno); 1456 if (!disk) 1457 goto out; 1458 1459 disk_block_events(disk); 1460 mutex_lock_nested(&bdev->bd_mutex, for_part); 1461 if (!bdev->bd_openers) { 1462 first_open = true; 1463 bdev->bd_disk = disk; 1464 bdev->bd_queue = disk->queue; 1465 bdev->bd_contains = bdev; 1466 bdev->bd_partno = partno; 1467 1468 if (!partno) { 1469 ret = -ENXIO; 1470 bdev->bd_part = disk_get_part(disk, partno); 1471 if (!bdev->bd_part) 1472 goto out_clear; 1473 1474 ret = 0; 1475 if (disk->fops->open) { 1476 ret = disk->fops->open(bdev, mode); 1477 if (ret == -ERESTARTSYS) { 1478 /* Lost a race with 'disk' being 1479 * deleted, try again. 1480 * See md.c 1481 */ 1482 disk_put_part(bdev->bd_part); 1483 bdev->bd_part = NULL; 1484 bdev->bd_disk = NULL; 1485 bdev->bd_queue = NULL; 1486 mutex_unlock(&bdev->bd_mutex); 1487 disk_unblock_events(disk); 1488 put_disk_and_module(disk); 1489 goto restart; 1490 } 1491 } 1492 1493 if (!ret) 1494 bd_set_size(bdev,(loff_t)get_capacity(disk)<<9); 1495 1496 /* 1497 * If the device is invalidated, rescan partition 1498 * if open succeeded or failed with -ENOMEDIUM. 1499 * The latter is necessary to prevent ghost 1500 * partitions on a removed medium. 1501 */ 1502 if (bdev->bd_invalidated) { 1503 if (!ret) 1504 rescan_partitions(disk, bdev); 1505 else if (ret == -ENOMEDIUM) 1506 invalidate_partitions(disk, bdev); 1507 } 1508 1509 if (ret) 1510 goto out_clear; 1511 } else { 1512 struct block_device *whole; 1513 whole = bdget_disk(disk, 0); 1514 ret = -ENOMEM; 1515 if (!whole) 1516 goto out_clear; 1517 BUG_ON(for_part); 1518 ret = __blkdev_get(whole, mode, 1); 1519 if (ret) 1520 goto out_clear; 1521 bdev->bd_contains = whole; 1522 bdev->bd_part = disk_get_part(disk, partno); 1523 if (!(disk->flags & GENHD_FL_UP) || 1524 !bdev->bd_part || !bdev->bd_part->nr_sects) { 1525 ret = -ENXIO; 1526 goto out_clear; 1527 } 1528 bd_set_size(bdev, (loff_t)bdev->bd_part->nr_sects << 9); 1529 } 1530 1531 if (bdev->bd_bdi == &noop_backing_dev_info) 1532 bdev->bd_bdi = bdi_get(disk->queue->backing_dev_info); 1533 } else { 1534 if (bdev->bd_contains == bdev) { 1535 ret = 0; 1536 if (bdev->bd_disk->fops->open) 1537 ret = bdev->bd_disk->fops->open(bdev, mode); 1538 /* the same as first opener case, read comment there */ 1539 if (bdev->bd_invalidated) { 1540 if (!ret) 1541 rescan_partitions(bdev->bd_disk, bdev); 1542 else if (ret == -ENOMEDIUM) 1543 invalidate_partitions(bdev->bd_disk, bdev); 1544 } 1545 if (ret) 1546 goto out_unlock_bdev; 1547 } 1548 } 1549 bdev->bd_openers++; 1550 if (for_part) 1551 bdev->bd_part_count++; 1552 mutex_unlock(&bdev->bd_mutex); 1553 disk_unblock_events(disk); 1554 /* only one opener holds refs to the module and disk */ 1555 if (!first_open) 1556 put_disk_and_module(disk); 1557 return 0; 1558 1559 out_clear: 1560 disk_put_part(bdev->bd_part); 1561 bdev->bd_disk = NULL; 1562 bdev->bd_part = NULL; 1563 bdev->bd_queue = NULL; 1564 if (bdev != bdev->bd_contains) 1565 __blkdev_put(bdev->bd_contains, mode, 1); 1566 bdev->bd_contains = NULL; 1567 out_unlock_bdev: 1568 mutex_unlock(&bdev->bd_mutex); 1569 disk_unblock_events(disk); 1570 put_disk_and_module(disk); 1571 out: 1572 bdput(bdev); 1573 1574 return ret; 1575 } 1576 1577 /** 1578 * blkdev_get - open a block device 1579 * @bdev: block_device to open 1580 * @mode: FMODE_* mask 1581 * @holder: exclusive holder identifier 1582 * 1583 * Open @bdev with @mode. If @mode includes %FMODE_EXCL, @bdev is 1584 * open with exclusive access. Specifying %FMODE_EXCL with %NULL 1585 * @holder is invalid. Exclusive opens may nest for the same @holder. 1586 * 1587 * On success, the reference count of @bdev is unchanged. On failure, 1588 * @bdev is put. 1589 * 1590 * CONTEXT: 1591 * Might sleep. 1592 * 1593 * RETURNS: 1594 * 0 on success, -errno on failure. 1595 */ 1596 int blkdev_get(struct block_device *bdev, fmode_t mode, void *holder) 1597 { 1598 struct block_device *whole = NULL; 1599 int res; 1600 1601 WARN_ON_ONCE((mode & FMODE_EXCL) && !holder); 1602 1603 if ((mode & FMODE_EXCL) && holder) { 1604 whole = bd_start_claiming(bdev, holder); 1605 if (IS_ERR(whole)) { 1606 bdput(bdev); 1607 return PTR_ERR(whole); 1608 } 1609 } 1610 1611 res = __blkdev_get(bdev, mode, 0); 1612 1613 if (whole) { 1614 struct gendisk *disk = whole->bd_disk; 1615 1616 /* finish claiming */ 1617 mutex_lock(&bdev->bd_mutex); 1618 spin_lock(&bdev_lock); 1619 1620 if (!res) { 1621 BUG_ON(!bd_may_claim(bdev, whole, holder)); 1622 /* 1623 * Note that for a whole device bd_holders 1624 * will be incremented twice, and bd_holder 1625 * will be set to bd_may_claim before being 1626 * set to holder 1627 */ 1628 whole->bd_holders++; 1629 whole->bd_holder = bd_may_claim; 1630 bdev->bd_holders++; 1631 bdev->bd_holder = holder; 1632 } 1633 1634 /* tell others that we're done */ 1635 BUG_ON(whole->bd_claiming != holder); 1636 whole->bd_claiming = NULL; 1637 wake_up_bit(&whole->bd_claiming, 0); 1638 1639 spin_unlock(&bdev_lock); 1640 1641 /* 1642 * Block event polling for write claims if requested. Any 1643 * write holder makes the write_holder state stick until 1644 * all are released. This is good enough and tracking 1645 * individual writeable reference is too fragile given the 1646 * way @mode is used in blkdev_get/put(). 1647 */ 1648 if (!res && (mode & FMODE_WRITE) && !bdev->bd_write_holder && 1649 (disk->flags & GENHD_FL_BLOCK_EVENTS_ON_EXCL_WRITE)) { 1650 bdev->bd_write_holder = true; 1651 disk_block_events(disk); 1652 } 1653 1654 mutex_unlock(&bdev->bd_mutex); 1655 bdput(whole); 1656 } 1657 1658 return res; 1659 } 1660 EXPORT_SYMBOL(blkdev_get); 1661 1662 /** 1663 * blkdev_get_by_path - open a block device by name 1664 * @path: path to the block device to open 1665 * @mode: FMODE_* mask 1666 * @holder: exclusive holder identifier 1667 * 1668 * Open the blockdevice described by the device file at @path. @mode 1669 * and @holder are identical to blkdev_get(). 1670 * 1671 * On success, the returned block_device has reference count of one. 1672 * 1673 * CONTEXT: 1674 * Might sleep. 1675 * 1676 * RETURNS: 1677 * Pointer to block_device on success, ERR_PTR(-errno) on failure. 1678 */ 1679 struct block_device *blkdev_get_by_path(const char *path, fmode_t mode, 1680 void *holder) 1681 { 1682 struct block_device *bdev; 1683 int err; 1684 1685 bdev = lookup_bdev(path); 1686 if (IS_ERR(bdev)) 1687 return bdev; 1688 1689 err = blkdev_get(bdev, mode, holder); 1690 if (err) 1691 return ERR_PTR(err); 1692 1693 if ((mode & FMODE_WRITE) && bdev_read_only(bdev)) { 1694 blkdev_put(bdev, mode); 1695 return ERR_PTR(-EACCES); 1696 } 1697 1698 return bdev; 1699 } 1700 EXPORT_SYMBOL(blkdev_get_by_path); 1701 1702 /** 1703 * blkdev_get_by_dev - open a block device by device number 1704 * @dev: device number of block device to open 1705 * @mode: FMODE_* mask 1706 * @holder: exclusive holder identifier 1707 * 1708 * Open the blockdevice described by device number @dev. @mode and 1709 * @holder are identical to blkdev_get(). 1710 * 1711 * Use it ONLY if you really do not have anything better - i.e. when 1712 * you are behind a truly sucky interface and all you are given is a 1713 * device number. _Never_ to be used for internal purposes. If you 1714 * ever need it - reconsider your API. 1715 * 1716 * On success, the returned block_device has reference count of one. 1717 * 1718 * CONTEXT: 1719 * Might sleep. 1720 * 1721 * RETURNS: 1722 * Pointer to block_device on success, ERR_PTR(-errno) on failure. 1723 */ 1724 struct block_device *blkdev_get_by_dev(dev_t dev, fmode_t mode, void *holder) 1725 { 1726 struct block_device *bdev; 1727 int err; 1728 1729 bdev = bdget(dev); 1730 if (!bdev) 1731 return ERR_PTR(-ENOMEM); 1732 1733 err = blkdev_get(bdev, mode, holder); 1734 if (err) 1735 return ERR_PTR(err); 1736 1737 return bdev; 1738 } 1739 EXPORT_SYMBOL(blkdev_get_by_dev); 1740 1741 static int blkdev_open(struct inode * inode, struct file * filp) 1742 { 1743 struct block_device *bdev; 1744 1745 /* 1746 * Preserve backwards compatibility and allow large file access 1747 * even if userspace doesn't ask for it explicitly. Some mkfs 1748 * binary needs it. We might want to drop this workaround 1749 * during an unstable branch. 1750 */ 1751 filp->f_flags |= O_LARGEFILE; 1752 1753 filp->f_mode |= FMODE_NOWAIT; 1754 1755 if (filp->f_flags & O_NDELAY) 1756 filp->f_mode |= FMODE_NDELAY; 1757 if (filp->f_flags & O_EXCL) 1758 filp->f_mode |= FMODE_EXCL; 1759 if ((filp->f_flags & O_ACCMODE) == 3) 1760 filp->f_mode |= FMODE_WRITE_IOCTL; 1761 1762 bdev = bd_acquire(inode); 1763 if (bdev == NULL) 1764 return -ENOMEM; 1765 1766 filp->f_mapping = bdev->bd_inode->i_mapping; 1767 filp->f_wb_err = filemap_sample_wb_err(filp->f_mapping); 1768 1769 return blkdev_get(bdev, filp->f_mode, filp); 1770 } 1771 1772 static void __blkdev_put(struct block_device *bdev, fmode_t mode, int for_part) 1773 { 1774 struct gendisk *disk = bdev->bd_disk; 1775 struct block_device *victim = NULL; 1776 1777 mutex_lock_nested(&bdev->bd_mutex, for_part); 1778 if (for_part) 1779 bdev->bd_part_count--; 1780 1781 if (!--bdev->bd_openers) { 1782 WARN_ON_ONCE(bdev->bd_holders); 1783 sync_blockdev(bdev); 1784 kill_bdev(bdev); 1785 1786 bdev_write_inode(bdev); 1787 } 1788 if (bdev->bd_contains == bdev) { 1789 if (disk->fops->release) 1790 disk->fops->release(disk, mode); 1791 } 1792 if (!bdev->bd_openers) { 1793 disk_put_part(bdev->bd_part); 1794 bdev->bd_part = NULL; 1795 bdev->bd_disk = NULL; 1796 if (bdev != bdev->bd_contains) 1797 victim = bdev->bd_contains; 1798 bdev->bd_contains = NULL; 1799 1800 put_disk_and_module(disk); 1801 } 1802 mutex_unlock(&bdev->bd_mutex); 1803 bdput(bdev); 1804 if (victim) 1805 __blkdev_put(victim, mode, 1); 1806 } 1807 1808 void blkdev_put(struct block_device *bdev, fmode_t mode) 1809 { 1810 mutex_lock(&bdev->bd_mutex); 1811 1812 if (mode & FMODE_EXCL) { 1813 bool bdev_free; 1814 1815 /* 1816 * Release a claim on the device. The holder fields 1817 * are protected with bdev_lock. bd_mutex is to 1818 * synchronize disk_holder unlinking. 1819 */ 1820 spin_lock(&bdev_lock); 1821 1822 WARN_ON_ONCE(--bdev->bd_holders < 0); 1823 WARN_ON_ONCE(--bdev->bd_contains->bd_holders < 0); 1824 1825 /* bd_contains might point to self, check in a separate step */ 1826 if ((bdev_free = !bdev->bd_holders)) 1827 bdev->bd_holder = NULL; 1828 if (!bdev->bd_contains->bd_holders) 1829 bdev->bd_contains->bd_holder = NULL; 1830 1831 spin_unlock(&bdev_lock); 1832 1833 /* 1834 * If this was the last claim, remove holder link and 1835 * unblock evpoll if it was a write holder. 1836 */ 1837 if (bdev_free && bdev->bd_write_holder) { 1838 disk_unblock_events(bdev->bd_disk); 1839 bdev->bd_write_holder = false; 1840 } 1841 } 1842 1843 /* 1844 * Trigger event checking and tell drivers to flush MEDIA_CHANGE 1845 * event. This is to ensure detection of media removal commanded 1846 * from userland - e.g. eject(1). 1847 */ 1848 disk_flush_events(bdev->bd_disk, DISK_EVENT_MEDIA_CHANGE); 1849 1850 mutex_unlock(&bdev->bd_mutex); 1851 1852 __blkdev_put(bdev, mode, 0); 1853 } 1854 EXPORT_SYMBOL(blkdev_put); 1855 1856 static int blkdev_close(struct inode * inode, struct file * filp) 1857 { 1858 struct block_device *bdev = I_BDEV(bdev_file_inode(filp)); 1859 blkdev_put(bdev, filp->f_mode); 1860 return 0; 1861 } 1862 1863 static long block_ioctl(struct file *file, unsigned cmd, unsigned long arg) 1864 { 1865 struct block_device *bdev = I_BDEV(bdev_file_inode(file)); 1866 fmode_t mode = file->f_mode; 1867 1868 /* 1869 * O_NDELAY can be altered using fcntl(.., F_SETFL, ..), so we have 1870 * to updated it before every ioctl. 1871 */ 1872 if (file->f_flags & O_NDELAY) 1873 mode |= FMODE_NDELAY; 1874 else 1875 mode &= ~FMODE_NDELAY; 1876 1877 return blkdev_ioctl(bdev, mode, cmd, arg); 1878 } 1879 1880 /* 1881 * Write data to the block device. Only intended for the block device itself 1882 * and the raw driver which basically is a fake block device. 1883 * 1884 * Does not take i_mutex for the write and thus is not for general purpose 1885 * use. 1886 */ 1887 ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from) 1888 { 1889 struct file *file = iocb->ki_filp; 1890 struct inode *bd_inode = bdev_file_inode(file); 1891 loff_t size = i_size_read(bd_inode); 1892 struct blk_plug plug; 1893 ssize_t ret; 1894 1895 if (bdev_read_only(I_BDEV(bd_inode))) 1896 return -EPERM; 1897 1898 if (!iov_iter_count(from)) 1899 return 0; 1900 1901 if (iocb->ki_pos >= size) 1902 return -ENOSPC; 1903 1904 if ((iocb->ki_flags & (IOCB_NOWAIT | IOCB_DIRECT)) == IOCB_NOWAIT) 1905 return -EOPNOTSUPP; 1906 1907 iov_iter_truncate(from, size - iocb->ki_pos); 1908 1909 blk_start_plug(&plug); 1910 ret = __generic_file_write_iter(iocb, from); 1911 if (ret > 0) 1912 ret = generic_write_sync(iocb, ret); 1913 blk_finish_plug(&plug); 1914 return ret; 1915 } 1916 EXPORT_SYMBOL_GPL(blkdev_write_iter); 1917 1918 ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to) 1919 { 1920 struct file *file = iocb->ki_filp; 1921 struct inode *bd_inode = bdev_file_inode(file); 1922 loff_t size = i_size_read(bd_inode); 1923 loff_t pos = iocb->ki_pos; 1924 1925 if (pos >= size) 1926 return 0; 1927 1928 size -= pos; 1929 iov_iter_truncate(to, size); 1930 return generic_file_read_iter(iocb, to); 1931 } 1932 EXPORT_SYMBOL_GPL(blkdev_read_iter); 1933 1934 /* 1935 * Try to release a page associated with block device when the system 1936 * is under memory pressure. 1937 */ 1938 static int blkdev_releasepage(struct page *page, gfp_t wait) 1939 { 1940 struct super_block *super = BDEV_I(page->mapping->host)->bdev.bd_super; 1941 1942 if (super && super->s_op->bdev_try_to_free_page) 1943 return super->s_op->bdev_try_to_free_page(super, page, wait); 1944 1945 return try_to_free_buffers(page); 1946 } 1947 1948 static int blkdev_writepages(struct address_space *mapping, 1949 struct writeback_control *wbc) 1950 { 1951 return generic_writepages(mapping, wbc); 1952 } 1953 1954 static const struct address_space_operations def_blk_aops = { 1955 .readpage = blkdev_readpage, 1956 .readpages = blkdev_readpages, 1957 .writepage = blkdev_writepage, 1958 .write_begin = blkdev_write_begin, 1959 .write_end = blkdev_write_end, 1960 .writepages = blkdev_writepages, 1961 .releasepage = blkdev_releasepage, 1962 .direct_IO = blkdev_direct_IO, 1963 .is_dirty_writeback = buffer_check_dirty_writeback, 1964 }; 1965 1966 #define BLKDEV_FALLOC_FL_SUPPORTED \ 1967 (FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE | \ 1968 FALLOC_FL_ZERO_RANGE | FALLOC_FL_NO_HIDE_STALE) 1969 1970 static long blkdev_fallocate(struct file *file, int mode, loff_t start, 1971 loff_t len) 1972 { 1973 struct block_device *bdev = I_BDEV(bdev_file_inode(file)); 1974 struct address_space *mapping; 1975 loff_t end = start + len - 1; 1976 loff_t isize; 1977 int error; 1978 1979 /* Fail if we don't recognize the flags. */ 1980 if (mode & ~BLKDEV_FALLOC_FL_SUPPORTED) 1981 return -EOPNOTSUPP; 1982 1983 /* Don't go off the end of the device. */ 1984 isize = i_size_read(bdev->bd_inode); 1985 if (start >= isize) 1986 return -EINVAL; 1987 if (end >= isize) { 1988 if (mode & FALLOC_FL_KEEP_SIZE) { 1989 len = isize - start; 1990 end = start + len - 1; 1991 } else 1992 return -EINVAL; 1993 } 1994 1995 /* 1996 * Don't allow IO that isn't aligned to logical block size. 1997 */ 1998 if ((start | len) & (bdev_logical_block_size(bdev) - 1)) 1999 return -EINVAL; 2000 2001 /* Invalidate the page cache, including dirty pages. */ 2002 mapping = bdev->bd_inode->i_mapping; 2003 truncate_inode_pages_range(mapping, start, end); 2004 2005 switch (mode) { 2006 case FALLOC_FL_ZERO_RANGE: 2007 case FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE: 2008 error = blkdev_issue_zeroout(bdev, start >> 9, len >> 9, 2009 GFP_KERNEL, BLKDEV_ZERO_NOUNMAP); 2010 break; 2011 case FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE: 2012 error = blkdev_issue_zeroout(bdev, start >> 9, len >> 9, 2013 GFP_KERNEL, BLKDEV_ZERO_NOFALLBACK); 2014 break; 2015 case FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE | FALLOC_FL_NO_HIDE_STALE: 2016 error = blkdev_issue_discard(bdev, start >> 9, len >> 9, 2017 GFP_KERNEL, 0); 2018 break; 2019 default: 2020 return -EOPNOTSUPP; 2021 } 2022 if (error) 2023 return error; 2024 2025 /* 2026 * Invalidate again; if someone wandered in and dirtied a page, 2027 * the caller will be given -EBUSY. The third argument is 2028 * inclusive, so the rounding here is safe. 2029 */ 2030 return invalidate_inode_pages2_range(mapping, 2031 start >> PAGE_SHIFT, 2032 end >> PAGE_SHIFT); 2033 } 2034 2035 const struct file_operations def_blk_fops = { 2036 .open = blkdev_open, 2037 .release = blkdev_close, 2038 .llseek = block_llseek, 2039 .read_iter = blkdev_read_iter, 2040 .write_iter = blkdev_write_iter, 2041 .mmap = generic_file_mmap, 2042 .fsync = blkdev_fsync, 2043 .unlocked_ioctl = block_ioctl, 2044 #ifdef CONFIG_COMPAT 2045 .compat_ioctl = compat_blkdev_ioctl, 2046 #endif 2047 .splice_read = generic_file_splice_read, 2048 .splice_write = iter_file_splice_write, 2049 .fallocate = blkdev_fallocate, 2050 }; 2051 2052 int ioctl_by_bdev(struct block_device *bdev, unsigned cmd, unsigned long arg) 2053 { 2054 int res; 2055 mm_segment_t old_fs = get_fs(); 2056 set_fs(KERNEL_DS); 2057 res = blkdev_ioctl(bdev, 0, cmd, arg); 2058 set_fs(old_fs); 2059 return res; 2060 } 2061 2062 EXPORT_SYMBOL(ioctl_by_bdev); 2063 2064 /** 2065 * lookup_bdev - lookup a struct block_device by name 2066 * @pathname: special file representing the block device 2067 * 2068 * Get a reference to the blockdevice at @pathname in the current 2069 * namespace if possible and return it. Return ERR_PTR(error) 2070 * otherwise. 2071 */ 2072 struct block_device *lookup_bdev(const char *pathname) 2073 { 2074 struct block_device *bdev; 2075 struct inode *inode; 2076 struct path path; 2077 int error; 2078 2079 if (!pathname || !*pathname) 2080 return ERR_PTR(-EINVAL); 2081 2082 error = kern_path(pathname, LOOKUP_FOLLOW, &path); 2083 if (error) 2084 return ERR_PTR(error); 2085 2086 inode = d_backing_inode(path.dentry); 2087 error = -ENOTBLK; 2088 if (!S_ISBLK(inode->i_mode)) 2089 goto fail; 2090 error = -EACCES; 2091 if (!may_open_dev(&path)) 2092 goto fail; 2093 error = -ENOMEM; 2094 bdev = bd_acquire(inode); 2095 if (!bdev) 2096 goto fail; 2097 out: 2098 path_put(&path); 2099 return bdev; 2100 fail: 2101 bdev = ERR_PTR(error); 2102 goto out; 2103 } 2104 EXPORT_SYMBOL(lookup_bdev); 2105 2106 int __invalidate_device(struct block_device *bdev, bool kill_dirty) 2107 { 2108 struct super_block *sb = get_super(bdev); 2109 int res = 0; 2110 2111 if (sb) { 2112 /* 2113 * no need to lock the super, get_super holds the 2114 * read mutex so the filesystem cannot go away 2115 * under us (->put_super runs with the write lock 2116 * hold). 2117 */ 2118 shrink_dcache_sb(sb); 2119 res = invalidate_inodes(sb, kill_dirty); 2120 drop_super(sb); 2121 } 2122 invalidate_bdev(bdev); 2123 return res; 2124 } 2125 EXPORT_SYMBOL(__invalidate_device); 2126 2127 void iterate_bdevs(void (*func)(struct block_device *, void *), void *arg) 2128 { 2129 struct inode *inode, *old_inode = NULL; 2130 2131 spin_lock(&blockdev_superblock->s_inode_list_lock); 2132 list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) { 2133 struct address_space *mapping = inode->i_mapping; 2134 struct block_device *bdev; 2135 2136 spin_lock(&inode->i_lock); 2137 if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW) || 2138 mapping->nrpages == 0) { 2139 spin_unlock(&inode->i_lock); 2140 continue; 2141 } 2142 __iget(inode); 2143 spin_unlock(&inode->i_lock); 2144 spin_unlock(&blockdev_superblock->s_inode_list_lock); 2145 /* 2146 * We hold a reference to 'inode' so it couldn't have been 2147 * removed from s_inodes list while we dropped the 2148 * s_inode_list_lock We cannot iput the inode now as we can 2149 * be holding the last reference and we cannot iput it under 2150 * s_inode_list_lock. So we keep the reference and iput it 2151 * later. 2152 */ 2153 iput(old_inode); 2154 old_inode = inode; 2155 bdev = I_BDEV(inode); 2156 2157 mutex_lock(&bdev->bd_mutex); 2158 if (bdev->bd_openers) 2159 func(bdev, arg); 2160 mutex_unlock(&bdev->bd_mutex); 2161 2162 spin_lock(&blockdev_superblock->s_inode_list_lock); 2163 } 2164 spin_unlock(&blockdev_superblock->s_inode_list_lock); 2165 iput(old_inode); 2166 } 2167
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.