~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/fs/block_dev.c

Version: ~ [ linux-5.15-rc5 ] ~ [ linux-5.14.11 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.72 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.152 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.210 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.250 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.286 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.288 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0-only
  2 /*
  3  *  linux/fs/block_dev.c
  4  *
  5  *  Copyright (C) 1991, 1992  Linus Torvalds
  6  *  Copyright (C) 2001  Andrea Arcangeli <andrea@suse.de> SuSE
  7  */
  8 
  9 #include <linux/init.h>
 10 #include <linux/mm.h>
 11 #include <linux/fcntl.h>
 12 #include <linux/slab.h>
 13 #include <linux/kmod.h>
 14 #include <linux/major.h>
 15 #include <linux/device_cgroup.h>
 16 #include <linux/highmem.h>
 17 #include <linux/blkdev.h>
 18 #include <linux/backing-dev.h>
 19 #include <linux/module.h>
 20 #include <linux/blkpg.h>
 21 #include <linux/magic.h>
 22 #include <linux/dax.h>
 23 #include <linux/buffer_head.h>
 24 #include <linux/swap.h>
 25 #include <linux/pagevec.h>
 26 #include <linux/writeback.h>
 27 #include <linux/mpage.h>
 28 #include <linux/mount.h>
 29 #include <linux/pseudo_fs.h>
 30 #include <linux/uio.h>
 31 #include <linux/namei.h>
 32 #include <linux/log2.h>
 33 #include <linux/cleancache.h>
 34 #include <linux/task_io_accounting_ops.h>
 35 #include <linux/falloc.h>
 36 #include <linux/uaccess.h>
 37 #include "internal.h"
 38 
 39 struct bdev_inode {
 40         struct block_device bdev;
 41         struct inode vfs_inode;
 42 };
 43 
 44 static const struct address_space_operations def_blk_aops;
 45 
 46 static inline struct bdev_inode *BDEV_I(struct inode *inode)
 47 {
 48         return container_of(inode, struct bdev_inode, vfs_inode);
 49 }
 50 
 51 struct block_device *I_BDEV(struct inode *inode)
 52 {
 53         return &BDEV_I(inode)->bdev;
 54 }
 55 EXPORT_SYMBOL(I_BDEV);
 56 
 57 static void bdev_write_inode(struct block_device *bdev)
 58 {
 59         struct inode *inode = bdev->bd_inode;
 60         int ret;
 61 
 62         spin_lock(&inode->i_lock);
 63         while (inode->i_state & I_DIRTY) {
 64                 spin_unlock(&inode->i_lock);
 65                 ret = write_inode_now(inode, true);
 66                 if (ret) {
 67                         char name[BDEVNAME_SIZE];
 68                         pr_warn_ratelimited("VFS: Dirty inode writeback failed "
 69                                             "for block device %s (err=%d).\n",
 70                                             bdevname(bdev, name), ret);
 71                 }
 72                 spin_lock(&inode->i_lock);
 73         }
 74         spin_unlock(&inode->i_lock);
 75 }
 76 
 77 /* Kill _all_ buffers and pagecache , dirty or not.. */
 78 void kill_bdev(struct block_device *bdev)
 79 {
 80         struct address_space *mapping = bdev->bd_inode->i_mapping;
 81 
 82         if (mapping->nrpages == 0 && mapping->nrexceptional == 0)
 83                 return;
 84 
 85         invalidate_bh_lrus();
 86         truncate_inode_pages(mapping, 0);
 87 }       
 88 EXPORT_SYMBOL(kill_bdev);
 89 
 90 /* Invalidate clean unused buffers and pagecache. */
 91 void invalidate_bdev(struct block_device *bdev)
 92 {
 93         struct address_space *mapping = bdev->bd_inode->i_mapping;
 94 
 95         if (mapping->nrpages) {
 96                 invalidate_bh_lrus();
 97                 lru_add_drain_all();    /* make sure all lru add caches are flushed */
 98                 invalidate_mapping_pages(mapping, 0, -1);
 99         }
100         /* 99% of the time, we don't need to flush the cleancache on the bdev.
101          * But, for the strange corners, lets be cautious
102          */
103         cleancache_invalidate_inode(mapping);
104 }
105 EXPORT_SYMBOL(invalidate_bdev);
106 
107 static void set_init_blocksize(struct block_device *bdev)
108 {
109         unsigned bsize = bdev_logical_block_size(bdev);
110         loff_t size = i_size_read(bdev->bd_inode);
111 
112         while (bsize < PAGE_SIZE) {
113                 if (size & bsize)
114                         break;
115                 bsize <<= 1;
116         }
117         bdev->bd_block_size = bsize;
118         bdev->bd_inode->i_blkbits = blksize_bits(bsize);
119 }
120 
121 int set_blocksize(struct block_device *bdev, int size)
122 {
123         /* Size must be a power of two, and between 512 and PAGE_SIZE */
124         if (size > PAGE_SIZE || size < 512 || !is_power_of_2(size))
125                 return -EINVAL;
126 
127         /* Size cannot be smaller than the size supported by the device */
128         if (size < bdev_logical_block_size(bdev))
129                 return -EINVAL;
130 
131         /* Don't change the size if it is same as current */
132         if (bdev->bd_block_size != size) {
133                 sync_blockdev(bdev);
134                 bdev->bd_block_size = size;
135                 bdev->bd_inode->i_blkbits = blksize_bits(size);
136                 kill_bdev(bdev);
137         }
138         return 0;
139 }
140 
141 EXPORT_SYMBOL(set_blocksize);
142 
143 int sb_set_blocksize(struct super_block *sb, int size)
144 {
145         if (set_blocksize(sb->s_bdev, size))
146                 return 0;
147         /* If we get here, we know size is power of two
148          * and it's value is between 512 and PAGE_SIZE */
149         sb->s_blocksize = size;
150         sb->s_blocksize_bits = blksize_bits(size);
151         return sb->s_blocksize;
152 }
153 
154 EXPORT_SYMBOL(sb_set_blocksize);
155 
156 int sb_min_blocksize(struct super_block *sb, int size)
157 {
158         int minsize = bdev_logical_block_size(sb->s_bdev);
159         if (size < minsize)
160                 size = minsize;
161         return sb_set_blocksize(sb, size);
162 }
163 
164 EXPORT_SYMBOL(sb_min_blocksize);
165 
166 static int
167 blkdev_get_block(struct inode *inode, sector_t iblock,
168                 struct buffer_head *bh, int create)
169 {
170         bh->b_bdev = I_BDEV(inode);
171         bh->b_blocknr = iblock;
172         set_buffer_mapped(bh);
173         return 0;
174 }
175 
176 static struct inode *bdev_file_inode(struct file *file)
177 {
178         return file->f_mapping->host;
179 }
180 
181 static unsigned int dio_bio_write_op(struct kiocb *iocb)
182 {
183         unsigned int op = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE;
184 
185         /* avoid the need for a I/O completion work item */
186         if (iocb->ki_flags & IOCB_DSYNC)
187                 op |= REQ_FUA;
188         return op;
189 }
190 
191 #define DIO_INLINE_BIO_VECS 4
192 
193 static void blkdev_bio_end_io_simple(struct bio *bio)
194 {
195         struct task_struct *waiter = bio->bi_private;
196 
197         WRITE_ONCE(bio->bi_private, NULL);
198         blk_wake_io_task(waiter);
199 }
200 
201 static ssize_t
202 __blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter,
203                 int nr_pages)
204 {
205         struct file *file = iocb->ki_filp;
206         struct block_device *bdev = I_BDEV(bdev_file_inode(file));
207         struct bio_vec inline_vecs[DIO_INLINE_BIO_VECS], *vecs;
208         loff_t pos = iocb->ki_pos;
209         bool should_dirty = false;
210         struct bio bio;
211         ssize_t ret;
212         blk_qc_t qc;
213 
214         if ((pos | iov_iter_alignment(iter)) &
215             (bdev_logical_block_size(bdev) - 1))
216                 return -EINVAL;
217 
218         if (nr_pages <= DIO_INLINE_BIO_VECS)
219                 vecs = inline_vecs;
220         else {
221                 vecs = kmalloc_array(nr_pages, sizeof(struct bio_vec),
222                                      GFP_KERNEL);
223                 if (!vecs)
224                         return -ENOMEM;
225         }
226 
227         bio_init(&bio, vecs, nr_pages);
228         bio_set_dev(&bio, bdev);
229         bio.bi_iter.bi_sector = pos >> 9;
230         bio.bi_write_hint = iocb->ki_hint;
231         bio.bi_private = current;
232         bio.bi_end_io = blkdev_bio_end_io_simple;
233         bio.bi_ioprio = iocb->ki_ioprio;
234 
235         ret = bio_iov_iter_get_pages(&bio, iter);
236         if (unlikely(ret))
237                 goto out;
238         ret = bio.bi_iter.bi_size;
239 
240         if (iov_iter_rw(iter) == READ) {
241                 bio.bi_opf = REQ_OP_READ;
242                 if (iter_is_iovec(iter))
243                         should_dirty = true;
244         } else {
245                 bio.bi_opf = dio_bio_write_op(iocb);
246                 task_io_account_write(ret);
247         }
248         if (iocb->ki_flags & IOCB_HIPRI)
249                 bio_set_polled(&bio, iocb);
250 
251         qc = submit_bio(&bio);
252         for (;;) {
253                 set_current_state(TASK_UNINTERRUPTIBLE);
254                 if (!READ_ONCE(bio.bi_private))
255                         break;
256                 if (!(iocb->ki_flags & IOCB_HIPRI) ||
257                     !blk_poll(bdev_get_queue(bdev), qc, true))
258                         io_schedule();
259         }
260         __set_current_state(TASK_RUNNING);
261 
262         bio_release_pages(&bio, should_dirty);
263         if (unlikely(bio.bi_status))
264                 ret = blk_status_to_errno(bio.bi_status);
265 
266 out:
267         if (vecs != inline_vecs)
268                 kfree(vecs);
269 
270         bio_uninit(&bio);
271 
272         return ret;
273 }
274 
275 struct blkdev_dio {
276         union {
277                 struct kiocb            *iocb;
278                 struct task_struct      *waiter;
279         };
280         size_t                  size;
281         atomic_t                ref;
282         bool                    multi_bio : 1;
283         bool                    should_dirty : 1;
284         bool                    is_sync : 1;
285         struct bio              bio;
286 };
287 
288 static struct bio_set blkdev_dio_pool;
289 
290 static int blkdev_iopoll(struct kiocb *kiocb, bool wait)
291 {
292         struct block_device *bdev = I_BDEV(kiocb->ki_filp->f_mapping->host);
293         struct request_queue *q = bdev_get_queue(bdev);
294 
295         return blk_poll(q, READ_ONCE(kiocb->ki_cookie), wait);
296 }
297 
298 static void blkdev_bio_end_io(struct bio *bio)
299 {
300         struct blkdev_dio *dio = bio->bi_private;
301         bool should_dirty = dio->should_dirty;
302 
303         if (bio->bi_status && !dio->bio.bi_status)
304                 dio->bio.bi_status = bio->bi_status;
305 
306         if (!dio->multi_bio || atomic_dec_and_test(&dio->ref)) {
307                 if (!dio->is_sync) {
308                         struct kiocb *iocb = dio->iocb;
309                         ssize_t ret;
310 
311                         if (likely(!dio->bio.bi_status)) {
312                                 ret = dio->size;
313                                 iocb->ki_pos += ret;
314                         } else {
315                                 ret = blk_status_to_errno(dio->bio.bi_status);
316                         }
317 
318                         dio->iocb->ki_complete(iocb, ret, 0);
319                         if (dio->multi_bio)
320                                 bio_put(&dio->bio);
321                 } else {
322                         struct task_struct *waiter = dio->waiter;
323 
324                         WRITE_ONCE(dio->waiter, NULL);
325                         blk_wake_io_task(waiter);
326                 }
327         }
328 
329         if (should_dirty) {
330                 bio_check_pages_dirty(bio);
331         } else {
332                 bio_release_pages(bio, false);
333                 bio_put(bio);
334         }
335 }
336 
337 static ssize_t
338 __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages)
339 {
340         struct file *file = iocb->ki_filp;
341         struct inode *inode = bdev_file_inode(file);
342         struct block_device *bdev = I_BDEV(inode);
343         struct blk_plug plug;
344         struct blkdev_dio *dio;
345         struct bio *bio;
346         bool is_poll = (iocb->ki_flags & IOCB_HIPRI) != 0;
347         bool is_read = (iov_iter_rw(iter) == READ), is_sync;
348         loff_t pos = iocb->ki_pos;
349         blk_qc_t qc = BLK_QC_T_NONE;
350         int ret = 0;
351 
352         if ((pos | iov_iter_alignment(iter)) &
353             (bdev_logical_block_size(bdev) - 1))
354                 return -EINVAL;
355 
356         bio = bio_alloc_bioset(GFP_KERNEL, nr_pages, &blkdev_dio_pool);
357 
358         dio = container_of(bio, struct blkdev_dio, bio);
359         dio->is_sync = is_sync = is_sync_kiocb(iocb);
360         if (dio->is_sync) {
361                 dio->waiter = current;
362                 bio_get(bio);
363         } else {
364                 dio->iocb = iocb;
365         }
366 
367         dio->size = 0;
368         dio->multi_bio = false;
369         dio->should_dirty = is_read && iter_is_iovec(iter);
370 
371         /*
372          * Don't plug for HIPRI/polled IO, as those should go straight
373          * to issue
374          */
375         if (!is_poll)
376                 blk_start_plug(&plug);
377 
378         for (;;) {
379                 bio_set_dev(bio, bdev);
380                 bio->bi_iter.bi_sector = pos >> 9;
381                 bio->bi_write_hint = iocb->ki_hint;
382                 bio->bi_private = dio;
383                 bio->bi_end_io = blkdev_bio_end_io;
384                 bio->bi_ioprio = iocb->ki_ioprio;
385 
386                 ret = bio_iov_iter_get_pages(bio, iter);
387                 if (unlikely(ret)) {
388                         bio->bi_status = BLK_STS_IOERR;
389                         bio_endio(bio);
390                         break;
391                 }
392 
393                 if (is_read) {
394                         bio->bi_opf = REQ_OP_READ;
395                         if (dio->should_dirty)
396                                 bio_set_pages_dirty(bio);
397                 } else {
398                         bio->bi_opf = dio_bio_write_op(iocb);
399                         task_io_account_write(bio->bi_iter.bi_size);
400                 }
401 
402                 dio->size += bio->bi_iter.bi_size;
403                 pos += bio->bi_iter.bi_size;
404 
405                 nr_pages = iov_iter_npages(iter, BIO_MAX_PAGES);
406                 if (!nr_pages) {
407                         bool polled = false;
408 
409                         if (iocb->ki_flags & IOCB_HIPRI) {
410                                 bio_set_polled(bio, iocb);
411                                 polled = true;
412                         }
413 
414                         qc = submit_bio(bio);
415 
416                         if (polled)
417                                 WRITE_ONCE(iocb->ki_cookie, qc);
418                         break;
419                 }
420 
421                 if (!dio->multi_bio) {
422                         /*
423                          * AIO needs an extra reference to ensure the dio
424                          * structure which is embedded into the first bio
425                          * stays around.
426                          */
427                         if (!is_sync)
428                                 bio_get(bio);
429                         dio->multi_bio = true;
430                         atomic_set(&dio->ref, 2);
431                 } else {
432                         atomic_inc(&dio->ref);
433                 }
434 
435                 submit_bio(bio);
436                 bio = bio_alloc(GFP_KERNEL, nr_pages);
437         }
438 
439         if (!is_poll)
440                 blk_finish_plug(&plug);
441 
442         if (!is_sync)
443                 return -EIOCBQUEUED;
444 
445         for (;;) {
446                 set_current_state(TASK_UNINTERRUPTIBLE);
447                 if (!READ_ONCE(dio->waiter))
448                         break;
449 
450                 if (!(iocb->ki_flags & IOCB_HIPRI) ||
451                     !blk_poll(bdev_get_queue(bdev), qc, true))
452                         io_schedule();
453         }
454         __set_current_state(TASK_RUNNING);
455 
456         if (!ret)
457                 ret = blk_status_to_errno(dio->bio.bi_status);
458         if (likely(!ret))
459                 ret = dio->size;
460 
461         bio_put(&dio->bio);
462         return ret;
463 }
464 
465 static ssize_t
466 blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
467 {
468         int nr_pages;
469 
470         nr_pages = iov_iter_npages(iter, BIO_MAX_PAGES + 1);
471         if (!nr_pages)
472                 return 0;
473         if (is_sync_kiocb(iocb) && nr_pages <= BIO_MAX_PAGES)
474                 return __blkdev_direct_IO_simple(iocb, iter, nr_pages);
475 
476         return __blkdev_direct_IO(iocb, iter, min(nr_pages, BIO_MAX_PAGES));
477 }
478 
479 static __init int blkdev_init(void)
480 {
481         return bioset_init(&blkdev_dio_pool, 4, offsetof(struct blkdev_dio, bio), BIOSET_NEED_BVECS);
482 }
483 module_init(blkdev_init);
484 
485 int __sync_blockdev(struct block_device *bdev, int wait)
486 {
487         if (!bdev)
488                 return 0;
489         if (!wait)
490                 return filemap_flush(bdev->bd_inode->i_mapping);
491         return filemap_write_and_wait(bdev->bd_inode->i_mapping);
492 }
493 
494 /*
495  * Write out and wait upon all the dirty data associated with a block
496  * device via its mapping.  Does not take the superblock lock.
497  */
498 int sync_blockdev(struct block_device *bdev)
499 {
500         return __sync_blockdev(bdev, 1);
501 }
502 EXPORT_SYMBOL(sync_blockdev);
503 
504 /*
505  * Write out and wait upon all dirty data associated with this
506  * device.   Filesystem data as well as the underlying block
507  * device.  Takes the superblock lock.
508  */
509 int fsync_bdev(struct block_device *bdev)
510 {
511         struct super_block *sb = get_super(bdev);
512         if (sb) {
513                 int res = sync_filesystem(sb);
514                 drop_super(sb);
515                 return res;
516         }
517         return sync_blockdev(bdev);
518 }
519 EXPORT_SYMBOL(fsync_bdev);
520 
521 /**
522  * freeze_bdev  --  lock a filesystem and force it into a consistent state
523  * @bdev:       blockdevice to lock
524  *
525  * If a superblock is found on this device, we take the s_umount semaphore
526  * on it to make sure nobody unmounts until the snapshot creation is done.
527  * The reference counter (bd_fsfreeze_count) guarantees that only the last
528  * unfreeze process can unfreeze the frozen filesystem actually when multiple
529  * freeze requests arrive simultaneously. It counts up in freeze_bdev() and
530  * count down in thaw_bdev(). When it becomes 0, thaw_bdev() will unfreeze
531  * actually.
532  */
533 struct super_block *freeze_bdev(struct block_device *bdev)
534 {
535         struct super_block *sb;
536         int error = 0;
537 
538         mutex_lock(&bdev->bd_fsfreeze_mutex);
539         if (++bdev->bd_fsfreeze_count > 1) {
540                 /*
541                  * We don't even need to grab a reference - the first call
542                  * to freeze_bdev grab an active reference and only the last
543                  * thaw_bdev drops it.
544                  */
545                 sb = get_super(bdev);
546                 if (sb)
547                         drop_super(sb);
548                 mutex_unlock(&bdev->bd_fsfreeze_mutex);
549                 return sb;
550         }
551 
552         sb = get_active_super(bdev);
553         if (!sb)
554                 goto out;
555         if (sb->s_op->freeze_super)
556                 error = sb->s_op->freeze_super(sb);
557         else
558                 error = freeze_super(sb);
559         if (error) {
560                 deactivate_super(sb);
561                 bdev->bd_fsfreeze_count--;
562                 mutex_unlock(&bdev->bd_fsfreeze_mutex);
563                 return ERR_PTR(error);
564         }
565         deactivate_super(sb);
566  out:
567         sync_blockdev(bdev);
568         mutex_unlock(&bdev->bd_fsfreeze_mutex);
569         return sb;      /* thaw_bdev releases s->s_umount */
570 }
571 EXPORT_SYMBOL(freeze_bdev);
572 
573 /**
574  * thaw_bdev  -- unlock filesystem
575  * @bdev:       blockdevice to unlock
576  * @sb:         associated superblock
577  *
578  * Unlocks the filesystem and marks it writeable again after freeze_bdev().
579  */
580 int thaw_bdev(struct block_device *bdev, struct super_block *sb)
581 {
582         int error = -EINVAL;
583 
584         mutex_lock(&bdev->bd_fsfreeze_mutex);
585         if (!bdev->bd_fsfreeze_count)
586                 goto out;
587 
588         error = 0;
589         if (--bdev->bd_fsfreeze_count > 0)
590                 goto out;
591 
592         if (!sb)
593                 goto out;
594 
595         if (sb->s_op->thaw_super)
596                 error = sb->s_op->thaw_super(sb);
597         else
598                 error = thaw_super(sb);
599         if (error)
600                 bdev->bd_fsfreeze_count++;
601 out:
602         mutex_unlock(&bdev->bd_fsfreeze_mutex);
603         return error;
604 }
605 EXPORT_SYMBOL(thaw_bdev);
606 
607 static int blkdev_writepage(struct page *page, struct writeback_control *wbc)
608 {
609         return block_write_full_page(page, blkdev_get_block, wbc);
610 }
611 
612 static int blkdev_readpage(struct file * file, struct page * page)
613 {
614         return block_read_full_page(page, blkdev_get_block);
615 }
616 
617 static int blkdev_readpages(struct file *file, struct address_space *mapping,
618                         struct list_head *pages, unsigned nr_pages)
619 {
620         return mpage_readpages(mapping, pages, nr_pages, blkdev_get_block);
621 }
622 
623 static int blkdev_write_begin(struct file *file, struct address_space *mapping,
624                         loff_t pos, unsigned len, unsigned flags,
625                         struct page **pagep, void **fsdata)
626 {
627         return block_write_begin(mapping, pos, len, flags, pagep,
628                                  blkdev_get_block);
629 }
630 
631 static int blkdev_write_end(struct file *file, struct address_space *mapping,
632                         loff_t pos, unsigned len, unsigned copied,
633                         struct page *page, void *fsdata)
634 {
635         int ret;
636         ret = block_write_end(file, mapping, pos, len, copied, page, fsdata);
637 
638         unlock_page(page);
639         put_page(page);
640 
641         return ret;
642 }
643 
644 /*
645  * private llseek:
646  * for a block special file file_inode(file)->i_size is zero
647  * so we compute the size by hand (just as in block_read/write above)
648  */
649 static loff_t block_llseek(struct file *file, loff_t offset, int whence)
650 {
651         struct inode *bd_inode = bdev_file_inode(file);
652         loff_t retval;
653 
654         inode_lock(bd_inode);
655         retval = fixed_size_llseek(file, offset, whence, i_size_read(bd_inode));
656         inode_unlock(bd_inode);
657         return retval;
658 }
659         
660 int blkdev_fsync(struct file *filp, loff_t start, loff_t end, int datasync)
661 {
662         struct inode *bd_inode = bdev_file_inode(filp);
663         struct block_device *bdev = I_BDEV(bd_inode);
664         int error;
665         
666         error = file_write_and_wait_range(filp, start, end);
667         if (error)
668                 return error;
669 
670         /*
671          * There is no need to serialise calls to blkdev_issue_flush with
672          * i_mutex and doing so causes performance issues with concurrent
673          * O_SYNC writers to a block device.
674          */
675         error = blkdev_issue_flush(bdev, GFP_KERNEL, NULL);
676         if (error == -EOPNOTSUPP)
677                 error = 0;
678 
679         return error;
680 }
681 EXPORT_SYMBOL(blkdev_fsync);
682 
683 /**
684  * bdev_read_page() - Start reading a page from a block device
685  * @bdev: The device to read the page from
686  * @sector: The offset on the device to read the page to (need not be aligned)
687  * @page: The page to read
688  *
689  * On entry, the page should be locked.  It will be unlocked when the page
690  * has been read.  If the block driver implements rw_page synchronously,
691  * that will be true on exit from this function, but it need not be.
692  *
693  * Errors returned by this function are usually "soft", eg out of memory, or
694  * queue full; callers should try a different route to read this page rather
695  * than propagate an error back up the stack.
696  *
697  * Return: negative errno if an error occurs, 0 if submission was successful.
698  */
699 int bdev_read_page(struct block_device *bdev, sector_t sector,
700                         struct page *page)
701 {
702         const struct block_device_operations *ops = bdev->bd_disk->fops;
703         int result = -EOPNOTSUPP;
704 
705         if (!ops->rw_page || bdev_get_integrity(bdev))
706                 return result;
707 
708         result = blk_queue_enter(bdev->bd_queue, 0);
709         if (result)
710                 return result;
711         result = ops->rw_page(bdev, sector + get_start_sect(bdev), page,
712                               REQ_OP_READ);
713         blk_queue_exit(bdev->bd_queue);
714         return result;
715 }
716 EXPORT_SYMBOL_GPL(bdev_read_page);
717 
718 /**
719  * bdev_write_page() - Start writing a page to a block device
720  * @bdev: The device to write the page to
721  * @sector: The offset on the device to write the page to (need not be aligned)
722  * @page: The page to write
723  * @wbc: The writeback_control for the write
724  *
725  * On entry, the page should be locked and not currently under writeback.
726  * On exit, if the write started successfully, the page will be unlocked and
727  * under writeback.  If the write failed already (eg the driver failed to
728  * queue the page to the device), the page will still be locked.  If the
729  * caller is a ->writepage implementation, it will need to unlock the page.
730  *
731  * Errors returned by this function are usually "soft", eg out of memory, or
732  * queue full; callers should try a different route to write this page rather
733  * than propagate an error back up the stack.
734  *
735  * Return: negative errno if an error occurs, 0 if submission was successful.
736  */
737 int bdev_write_page(struct block_device *bdev, sector_t sector,
738                         struct page *page, struct writeback_control *wbc)
739 {
740         int result;
741         const struct block_device_operations *ops = bdev->bd_disk->fops;
742 
743         if (!ops->rw_page || bdev_get_integrity(bdev))
744                 return -EOPNOTSUPP;
745         result = blk_queue_enter(bdev->bd_queue, 0);
746         if (result)
747                 return result;
748 
749         set_page_writeback(page);
750         result = ops->rw_page(bdev, sector + get_start_sect(bdev), page,
751                               REQ_OP_WRITE);
752         if (result) {
753                 end_page_writeback(page);
754         } else {
755                 clean_page_buffers(page);
756                 unlock_page(page);
757         }
758         blk_queue_exit(bdev->bd_queue);
759         return result;
760 }
761 EXPORT_SYMBOL_GPL(bdev_write_page);
762 
763 /*
764  * pseudo-fs
765  */
766 
767 static  __cacheline_aligned_in_smp DEFINE_SPINLOCK(bdev_lock);
768 static struct kmem_cache * bdev_cachep __read_mostly;
769 
770 static struct inode *bdev_alloc_inode(struct super_block *sb)
771 {
772         struct bdev_inode *ei = kmem_cache_alloc(bdev_cachep, GFP_KERNEL);
773         if (!ei)
774                 return NULL;
775         return &ei->vfs_inode;
776 }
777 
778 static void bdev_free_inode(struct inode *inode)
779 {
780         kmem_cache_free(bdev_cachep, BDEV_I(inode));
781 }
782 
783 static void init_once(void *foo)
784 {
785         struct bdev_inode *ei = (struct bdev_inode *) foo;
786         struct block_device *bdev = &ei->bdev;
787 
788         memset(bdev, 0, sizeof(*bdev));
789         mutex_init(&bdev->bd_mutex);
790         INIT_LIST_HEAD(&bdev->bd_list);
791 #ifdef CONFIG_SYSFS
792         INIT_LIST_HEAD(&bdev->bd_holder_disks);
793 #endif
794         bdev->bd_bdi = &noop_backing_dev_info;
795         inode_init_once(&ei->vfs_inode);
796         /* Initialize mutex for freeze. */
797         mutex_init(&bdev->bd_fsfreeze_mutex);
798 }
799 
800 static void bdev_evict_inode(struct inode *inode)
801 {
802         struct block_device *bdev = &BDEV_I(inode)->bdev;
803         truncate_inode_pages_final(&inode->i_data);
804         invalidate_inode_buffers(inode); /* is it needed here? */
805         clear_inode(inode);
806         spin_lock(&bdev_lock);
807         list_del_init(&bdev->bd_list);
808         spin_unlock(&bdev_lock);
809         /* Detach inode from wb early as bdi_put() may free bdi->wb */
810         inode_detach_wb(inode);
811         if (bdev->bd_bdi != &noop_backing_dev_info) {
812                 bdi_put(bdev->bd_bdi);
813                 bdev->bd_bdi = &noop_backing_dev_info;
814         }
815 }
816 
817 static const struct super_operations bdev_sops = {
818         .statfs = simple_statfs,
819         .alloc_inode = bdev_alloc_inode,
820         .free_inode = bdev_free_inode,
821         .drop_inode = generic_delete_inode,
822         .evict_inode = bdev_evict_inode,
823 };
824 
825 static int bd_init_fs_context(struct fs_context *fc)
826 {
827         struct pseudo_fs_context *ctx = init_pseudo(fc, BDEVFS_MAGIC);
828         if (!ctx)
829                 return -ENOMEM;
830         fc->s_iflags |= SB_I_CGROUPWB;
831         ctx->ops = &bdev_sops;
832         return 0;
833 }
834 
835 static struct file_system_type bd_type = {
836         .name           = "bdev",
837         .init_fs_context = bd_init_fs_context,
838         .kill_sb        = kill_anon_super,
839 };
840 
841 struct super_block *blockdev_superblock __read_mostly;
842 EXPORT_SYMBOL_GPL(blockdev_superblock);
843 
844 void __init bdev_cache_init(void)
845 {
846         int err;
847         static struct vfsmount *bd_mnt;
848 
849         bdev_cachep = kmem_cache_create("bdev_cache", sizeof(struct bdev_inode),
850                         0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT|
851                                 SLAB_MEM_SPREAD|SLAB_ACCOUNT|SLAB_PANIC),
852                         init_once);
853         err = register_filesystem(&bd_type);
854         if (err)
855                 panic("Cannot register bdev pseudo-fs");
856         bd_mnt = kern_mount(&bd_type);
857         if (IS_ERR(bd_mnt))
858                 panic("Cannot create bdev pseudo-fs");
859         blockdev_superblock = bd_mnt->mnt_sb;   /* For writeback */
860 }
861 
862 /*
863  * Most likely _very_ bad one - but then it's hardly critical for small
864  * /dev and can be fixed when somebody will need really large one.
865  * Keep in mind that it will be fed through icache hash function too.
866  */
867 static inline unsigned long hash(dev_t dev)
868 {
869         return MAJOR(dev)+MINOR(dev);
870 }
871 
872 static int bdev_test(struct inode *inode, void *data)
873 {
874         return BDEV_I(inode)->bdev.bd_dev == *(dev_t *)data;
875 }
876 
877 static int bdev_set(struct inode *inode, void *data)
878 {
879         BDEV_I(inode)->bdev.bd_dev = *(dev_t *)data;
880         return 0;
881 }
882 
883 static LIST_HEAD(all_bdevs);
884 
885 /*
886  * If there is a bdev inode for this device, unhash it so that it gets evicted
887  * as soon as last inode reference is dropped.
888  */
889 void bdev_unhash_inode(dev_t dev)
890 {
891         struct inode *inode;
892 
893         inode = ilookup5(blockdev_superblock, hash(dev), bdev_test, &dev);
894         if (inode) {
895                 remove_inode_hash(inode);
896                 iput(inode);
897         }
898 }
899 
900 struct block_device *bdget(dev_t dev)
901 {
902         struct block_device *bdev;
903         struct inode *inode;
904 
905         inode = iget5_locked(blockdev_superblock, hash(dev),
906                         bdev_test, bdev_set, &dev);
907 
908         if (!inode)
909                 return NULL;
910 
911         bdev = &BDEV_I(inode)->bdev;
912 
913         if (inode->i_state & I_NEW) {
914                 bdev->bd_contains = NULL;
915                 bdev->bd_super = NULL;
916                 bdev->bd_inode = inode;
917                 bdev->bd_block_size = i_blocksize(inode);
918                 bdev->bd_part_count = 0;
919                 bdev->bd_invalidated = 0;
920                 inode->i_mode = S_IFBLK;
921                 inode->i_rdev = dev;
922                 inode->i_bdev = bdev;
923                 inode->i_data.a_ops = &def_blk_aops;
924                 mapping_set_gfp_mask(&inode->i_data, GFP_USER);
925                 spin_lock(&bdev_lock);
926                 list_add(&bdev->bd_list, &all_bdevs);
927                 spin_unlock(&bdev_lock);
928                 unlock_new_inode(inode);
929         }
930         return bdev;
931 }
932 
933 EXPORT_SYMBOL(bdget);
934 
935 /**
936  * bdgrab -- Grab a reference to an already referenced block device
937  * @bdev:       Block device to grab a reference to.
938  */
939 struct block_device *bdgrab(struct block_device *bdev)
940 {
941         ihold(bdev->bd_inode);
942         return bdev;
943 }
944 EXPORT_SYMBOL(bdgrab);
945 
946 long nr_blockdev_pages(void)
947 {
948         struct block_device *bdev;
949         long ret = 0;
950         spin_lock(&bdev_lock);
951         list_for_each_entry(bdev, &all_bdevs, bd_list) {
952                 ret += bdev->bd_inode->i_mapping->nrpages;
953         }
954         spin_unlock(&bdev_lock);
955         return ret;
956 }
957 
958 void bdput(struct block_device *bdev)
959 {
960         iput(bdev->bd_inode);
961 }
962 
963 EXPORT_SYMBOL(bdput);
964  
965 static struct block_device *bd_acquire(struct inode *inode)
966 {
967         struct block_device *bdev;
968 
969         spin_lock(&bdev_lock);
970         bdev = inode->i_bdev;
971         if (bdev && !inode_unhashed(bdev->bd_inode)) {
972                 bdgrab(bdev);
973                 spin_unlock(&bdev_lock);
974                 return bdev;
975         }
976         spin_unlock(&bdev_lock);
977 
978         /*
979          * i_bdev references block device inode that was already shut down
980          * (corresponding device got removed).  Remove the reference and look
981          * up block device inode again just in case new device got
982          * reestablished under the same device number.
983          */
984         if (bdev)
985                 bd_forget(inode);
986 
987         bdev = bdget(inode->i_rdev);
988         if (bdev) {
989                 spin_lock(&bdev_lock);
990                 if (!inode->i_bdev) {
991                         /*
992                          * We take an additional reference to bd_inode,
993                          * and it's released in clear_inode() of inode.
994                          * So, we can access it via ->i_mapping always
995                          * without igrab().
996                          */
997                         bdgrab(bdev);
998                         inode->i_bdev = bdev;
999                         inode->i_mapping = bdev->bd_inode->i_mapping;
1000                 }
1001                 spin_unlock(&bdev_lock);
1002         }
1003         return bdev;
1004 }
1005 
1006 /* Call when you free inode */
1007 
1008 void bd_forget(struct inode *inode)
1009 {
1010         struct block_device *bdev = NULL;
1011 
1012         spin_lock(&bdev_lock);
1013         if (!sb_is_blkdev_sb(inode->i_sb))
1014                 bdev = inode->i_bdev;
1015         inode->i_bdev = NULL;
1016         inode->i_mapping = &inode->i_data;
1017         spin_unlock(&bdev_lock);
1018 
1019         if (bdev)
1020                 bdput(bdev);
1021 }
1022 
1023 /**
1024  * bd_may_claim - test whether a block device can be claimed
1025  * @bdev: block device of interest
1026  * @whole: whole block device containing @bdev, may equal @bdev
1027  * @holder: holder trying to claim @bdev
1028  *
1029  * Test whether @bdev can be claimed by @holder.
1030  *
1031  * CONTEXT:
1032  * spin_lock(&bdev_lock).
1033  *
1034  * RETURNS:
1035  * %true if @bdev can be claimed, %false otherwise.
1036  */
1037 static bool bd_may_claim(struct block_device *bdev, struct block_device *whole,
1038                          void *holder)
1039 {
1040         if (bdev->bd_holder == holder)
1041                 return true;     /* already a holder */
1042         else if (bdev->bd_holder != NULL)
1043                 return false;    /* held by someone else */
1044         else if (whole == bdev)
1045                 return true;     /* is a whole device which isn't held */
1046 
1047         else if (whole->bd_holder == bd_may_claim)
1048                 return true;     /* is a partition of a device that is being partitioned */
1049         else if (whole->bd_holder != NULL)
1050                 return false;    /* is a partition of a held device */
1051         else
1052                 return true;     /* is a partition of an un-held device */
1053 }
1054 
1055 /**
1056  * bd_prepare_to_claim - prepare to claim a block device
1057  * @bdev: block device of interest
1058  * @whole: the whole device containing @bdev, may equal @bdev
1059  * @holder: holder trying to claim @bdev
1060  *
1061  * Prepare to claim @bdev.  This function fails if @bdev is already
1062  * claimed by another holder and waits if another claiming is in
1063  * progress.  This function doesn't actually claim.  On successful
1064  * return, the caller has ownership of bd_claiming and bd_holder[s].
1065  *
1066  * CONTEXT:
1067  * spin_lock(&bdev_lock).  Might release bdev_lock, sleep and regrab
1068  * it multiple times.
1069  *
1070  * RETURNS:
1071  * 0 if @bdev can be claimed, -EBUSY otherwise.
1072  */
1073 static int bd_prepare_to_claim(struct block_device *bdev,
1074                                struct block_device *whole, void *holder)
1075 {
1076 retry:
1077         /* if someone else claimed, fail */
1078         if (!bd_may_claim(bdev, whole, holder))
1079                 return -EBUSY;
1080 
1081         /* if claiming is already in progress, wait for it to finish */
1082         if (whole->bd_claiming) {
1083                 wait_queue_head_t *wq = bit_waitqueue(&whole->bd_claiming, 0);
1084                 DEFINE_WAIT(wait);
1085 
1086                 prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE);
1087                 spin_unlock(&bdev_lock);
1088                 schedule();
1089                 finish_wait(wq, &wait);
1090                 spin_lock(&bdev_lock);
1091                 goto retry;
1092         }
1093 
1094         /* yay, all mine */
1095         return 0;
1096 }
1097 
1098 static struct gendisk *bdev_get_gendisk(struct block_device *bdev, int *partno)
1099 {
1100         struct gendisk *disk = get_gendisk(bdev->bd_dev, partno);
1101 
1102         if (!disk)
1103                 return NULL;
1104         /*
1105          * Now that we hold gendisk reference we make sure bdev we looked up is
1106          * not stale. If it is, it means device got removed and created before
1107          * we looked up gendisk and we fail open in such case. Associating
1108          * unhashed bdev with newly created gendisk could lead to two bdevs
1109          * (and thus two independent caches) being associated with one device
1110          * which is bad.
1111          */
1112         if (inode_unhashed(bdev->bd_inode)) {
1113                 put_disk_and_module(disk);
1114                 return NULL;
1115         }
1116         return disk;
1117 }
1118 
1119 /**
1120  * bd_start_claiming - start claiming a block device
1121  * @bdev: block device of interest
1122  * @holder: holder trying to claim @bdev
1123  *
1124  * @bdev is about to be opened exclusively.  Check @bdev can be opened
1125  * exclusively and mark that an exclusive open is in progress.  Each
1126  * successful call to this function must be matched with a call to
1127  * either bd_finish_claiming() or bd_abort_claiming() (which do not
1128  * fail).
1129  *
1130  * This function is used to gain exclusive access to the block device
1131  * without actually causing other exclusive open attempts to fail. It
1132  * should be used when the open sequence itself requires exclusive
1133  * access but may subsequently fail.
1134  *
1135  * CONTEXT:
1136  * Might sleep.
1137  *
1138  * RETURNS:
1139  * Pointer to the block device containing @bdev on success, ERR_PTR()
1140  * value on failure.
1141  */
1142 struct block_device *bd_start_claiming(struct block_device *bdev, void *holder)
1143 {
1144         struct gendisk *disk;
1145         struct block_device *whole;
1146         int partno, err;
1147 
1148         might_sleep();
1149 
1150         /*
1151          * @bdev might not have been initialized properly yet, look up
1152          * and grab the outer block device the hard way.
1153          */
1154         disk = bdev_get_gendisk(bdev, &partno);
1155         if (!disk)
1156                 return ERR_PTR(-ENXIO);
1157 
1158         /*
1159          * Normally, @bdev should equal what's returned from bdget_disk()
1160          * if partno is 0; however, some drivers (floppy) use multiple
1161          * bdev's for the same physical device and @bdev may be one of the
1162          * aliases.  Keep @bdev if partno is 0.  This means claimer
1163          * tracking is broken for those devices but it has always been that
1164          * way.
1165          */
1166         if (partno)
1167                 whole = bdget_disk(disk, 0);
1168         else
1169                 whole = bdgrab(bdev);
1170 
1171         put_disk_and_module(disk);
1172         if (!whole)
1173                 return ERR_PTR(-ENOMEM);
1174 
1175         /* prepare to claim, if successful, mark claiming in progress */
1176         spin_lock(&bdev_lock);
1177 
1178         err = bd_prepare_to_claim(bdev, whole, holder);
1179         if (err == 0) {
1180                 whole->bd_claiming = holder;
1181                 spin_unlock(&bdev_lock);
1182                 return whole;
1183         } else {
1184                 spin_unlock(&bdev_lock);
1185                 bdput(whole);
1186                 return ERR_PTR(err);
1187         }
1188 }
1189 EXPORT_SYMBOL(bd_start_claiming);
1190 
1191 static void bd_clear_claiming(struct block_device *whole, void *holder)
1192 {
1193         lockdep_assert_held(&bdev_lock);
1194         /* tell others that we're done */
1195         BUG_ON(whole->bd_claiming != holder);
1196         whole->bd_claiming = NULL;
1197         wake_up_bit(&whole->bd_claiming, 0);
1198 }
1199 
1200 /**
1201  * bd_finish_claiming - finish claiming of a block device
1202  * @bdev: block device of interest
1203  * @whole: whole block device (returned from bd_start_claiming())
1204  * @holder: holder that has claimed @bdev
1205  *
1206  * Finish exclusive open of a block device. Mark the device as exlusively
1207  * open by the holder and wake up all waiters for exclusive open to finish.
1208  */
1209 void bd_finish_claiming(struct block_device *bdev, struct block_device *whole,
1210                         void *holder)
1211 {
1212         spin_lock(&bdev_lock);
1213         BUG_ON(!bd_may_claim(bdev, whole, holder));
1214         /*
1215          * Note that for a whole device bd_holders will be incremented twice,
1216          * and bd_holder will be set to bd_may_claim before being set to holder
1217          */
1218         whole->bd_holders++;
1219         whole->bd_holder = bd_may_claim;
1220         bdev->bd_holders++;
1221         bdev->bd_holder = holder;
1222         bd_clear_claiming(whole, holder);
1223         spin_unlock(&bdev_lock);
1224 }
1225 EXPORT_SYMBOL(bd_finish_claiming);
1226 
1227 /**
1228  * bd_abort_claiming - abort claiming of a block device
1229  * @bdev: block device of interest
1230  * @whole: whole block device (returned from bd_start_claiming())
1231  * @holder: holder that has claimed @bdev
1232  *
1233  * Abort claiming of a block device when the exclusive open failed. This can be
1234  * also used when exclusive open is not actually desired and we just needed
1235  * to block other exclusive openers for a while.
1236  */
1237 void bd_abort_claiming(struct block_device *bdev, struct block_device *whole,
1238                        void *holder)
1239 {
1240         spin_lock(&bdev_lock);
1241         bd_clear_claiming(whole, holder);
1242         spin_unlock(&bdev_lock);
1243 }
1244 EXPORT_SYMBOL(bd_abort_claiming);
1245 
1246 #ifdef CONFIG_SYSFS
1247 struct bd_holder_disk {
1248         struct list_head        list;
1249         struct gendisk          *disk;
1250         int                     refcnt;
1251 };
1252 
1253 static struct bd_holder_disk *bd_find_holder_disk(struct block_device *bdev,
1254                                                   struct gendisk *disk)
1255 {
1256         struct bd_holder_disk *holder;
1257 
1258         list_for_each_entry(holder, &bdev->bd_holder_disks, list)
1259                 if (holder->disk == disk)
1260                         return holder;
1261         return NULL;
1262 }
1263 
1264 static int add_symlink(struct kobject *from, struct kobject *to)
1265 {
1266         return sysfs_create_link(from, to, kobject_name(to));
1267 }
1268 
1269 static void del_symlink(struct kobject *from, struct kobject *to)
1270 {
1271         sysfs_remove_link(from, kobject_name(to));
1272 }
1273 
1274 /**
1275  * bd_link_disk_holder - create symlinks between holding disk and slave bdev
1276  * @bdev: the claimed slave bdev
1277  * @disk: the holding disk
1278  *
1279  * DON'T USE THIS UNLESS YOU'RE ALREADY USING IT.
1280  *
1281  * This functions creates the following sysfs symlinks.
1282  *
1283  * - from "slaves" directory of the holder @disk to the claimed @bdev
1284  * - from "holders" directory of the @bdev to the holder @disk
1285  *
1286  * For example, if /dev/dm-0 maps to /dev/sda and disk for dm-0 is
1287  * passed to bd_link_disk_holder(), then:
1288  *
1289  *   /sys/block/dm-0/slaves/sda --> /sys/block/sda
1290  *   /sys/block/sda/holders/dm-0 --> /sys/block/dm-0
1291  *
1292  * The caller must have claimed @bdev before calling this function and
1293  * ensure that both @bdev and @disk are valid during the creation and
1294  * lifetime of these symlinks.
1295  *
1296  * CONTEXT:
1297  * Might sleep.
1298  *
1299  * RETURNS:
1300  * 0 on success, -errno on failure.
1301  */
1302 int bd_link_disk_holder(struct block_device *bdev, struct gendisk *disk)
1303 {
1304         struct bd_holder_disk *holder;
1305         int ret = 0;
1306 
1307         mutex_lock(&bdev->bd_mutex);
1308 
1309         WARN_ON_ONCE(!bdev->bd_holder);
1310 
1311         /* FIXME: remove the following once add_disk() handles errors */
1312         if (WARN_ON(!disk->slave_dir || !bdev->bd_part->holder_dir))
1313                 goto out_unlock;
1314 
1315         holder = bd_find_holder_disk(bdev, disk);
1316         if (holder) {
1317                 holder->refcnt++;
1318                 goto out_unlock;
1319         }
1320 
1321         holder = kzalloc(sizeof(*holder), GFP_KERNEL);
1322         if (!holder) {
1323                 ret = -ENOMEM;
1324                 goto out_unlock;
1325         }
1326 
1327         INIT_LIST_HEAD(&holder->list);
1328         holder->disk = disk;
1329         holder->refcnt = 1;
1330 
1331         ret = add_symlink(disk->slave_dir, &part_to_dev(bdev->bd_part)->kobj);
1332         if (ret)
1333                 goto out_free;
1334 
1335         ret = add_symlink(bdev->bd_part->holder_dir, &disk_to_dev(disk)->kobj);
1336         if (ret)
1337                 goto out_del;
1338         /*
1339          * bdev could be deleted beneath us which would implicitly destroy
1340          * the holder directory.  Hold on to it.
1341          */
1342         kobject_get(bdev->bd_part->holder_dir);
1343 
1344         list_add(&holder->list, &bdev->bd_holder_disks);
1345         goto out_unlock;
1346 
1347 out_del:
1348         del_symlink(disk->slave_dir, &part_to_dev(bdev->bd_part)->kobj);
1349 out_free:
1350         kfree(holder);
1351 out_unlock:
1352         mutex_unlock(&bdev->bd_mutex);
1353         return ret;
1354 }
1355 EXPORT_SYMBOL_GPL(bd_link_disk_holder);
1356 
1357 /**
1358  * bd_unlink_disk_holder - destroy symlinks created by bd_link_disk_holder()
1359  * @bdev: the calimed slave bdev
1360  * @disk: the holding disk
1361  *
1362  * DON'T USE THIS UNLESS YOU'RE ALREADY USING IT.
1363  *
1364  * CONTEXT:
1365  * Might sleep.
1366  */
1367 void bd_unlink_disk_holder(struct block_device *bdev, struct gendisk *disk)
1368 {
1369         struct bd_holder_disk *holder;
1370 
1371         mutex_lock(&bdev->bd_mutex);
1372 
1373         holder = bd_find_holder_disk(bdev, disk);
1374 
1375         if (!WARN_ON_ONCE(holder == NULL) && !--holder->refcnt) {
1376                 del_symlink(disk->slave_dir, &part_to_dev(bdev->bd_part)->kobj);
1377                 del_symlink(bdev->bd_part->holder_dir,
1378                             &disk_to_dev(disk)->kobj);
1379                 kobject_put(bdev->bd_part->holder_dir);
1380                 list_del_init(&holder->list);
1381                 kfree(holder);
1382         }
1383 
1384         mutex_unlock(&bdev->bd_mutex);
1385 }
1386 EXPORT_SYMBOL_GPL(bd_unlink_disk_holder);
1387 #endif
1388 
1389 /**
1390  * flush_disk - invalidates all buffer-cache entries on a disk
1391  *
1392  * @bdev:      struct block device to be flushed
1393  * @kill_dirty: flag to guide handling of dirty inodes
1394  *
1395  * Invalidates all buffer-cache entries on a disk. It should be called
1396  * when a disk has been changed -- either by a media change or online
1397  * resize.
1398  */
1399 static void flush_disk(struct block_device *bdev, bool kill_dirty)
1400 {
1401         if (__invalidate_device(bdev, kill_dirty)) {
1402                 printk(KERN_WARNING "VFS: busy inodes on changed media or "
1403                        "resized disk %s\n",
1404                        bdev->bd_disk ? bdev->bd_disk->disk_name : "");
1405         }
1406         bdev->bd_invalidated = 1;
1407 }
1408 
1409 /**
1410  * check_disk_size_change - checks for disk size change and adjusts bdev size.
1411  * @disk: struct gendisk to check
1412  * @bdev: struct bdev to adjust.
1413  * @verbose: if %true log a message about a size change if there is any
1414  *
1415  * This routine checks to see if the bdev size does not match the disk size
1416  * and adjusts it if it differs. When shrinking the bdev size, its all caches
1417  * are freed.
1418  */
1419 static void check_disk_size_change(struct gendisk *disk,
1420                 struct block_device *bdev, bool verbose)
1421 {
1422         loff_t disk_size, bdev_size;
1423 
1424         disk_size = (loff_t)get_capacity(disk) << 9;
1425         bdev_size = i_size_read(bdev->bd_inode);
1426         if (disk_size != bdev_size) {
1427                 if (verbose) {
1428                         printk(KERN_INFO
1429                                "%s: detected capacity change from %lld to %lld\n",
1430                                disk->disk_name, bdev_size, disk_size);
1431                 }
1432                 i_size_write(bdev->bd_inode, disk_size);
1433                 if (bdev_size > disk_size)
1434                         flush_disk(bdev, false);
1435         }
1436         bdev->bd_invalidated = 0;
1437 }
1438 
1439 /**
1440  * revalidate_disk - wrapper for lower-level driver's revalidate_disk call-back
1441  * @disk: struct gendisk to be revalidated
1442  *
1443  * This routine is a wrapper for lower-level driver's revalidate_disk
1444  * call-backs.  It is used to do common pre and post operations needed
1445  * for all revalidate_disk operations.
1446  */
1447 int revalidate_disk(struct gendisk *disk)
1448 {
1449         int ret = 0;
1450 
1451         if (disk->fops->revalidate_disk)
1452                 ret = disk->fops->revalidate_disk(disk);
1453 
1454         /*
1455          * Hidden disks don't have associated bdev so there's no point in
1456          * revalidating it.
1457          */
1458         if (!(disk->flags & GENHD_FL_HIDDEN)) {
1459                 struct block_device *bdev = bdget_disk(disk, 0);
1460 
1461                 if (!bdev)
1462                         return ret;
1463 
1464                 mutex_lock(&bdev->bd_mutex);
1465                 check_disk_size_change(disk, bdev, ret == 0);
1466                 mutex_unlock(&bdev->bd_mutex);
1467                 bdput(bdev);
1468         }
1469         return ret;
1470 }
1471 EXPORT_SYMBOL(revalidate_disk);
1472 
1473 /*
1474  * This routine checks whether a removable media has been changed,
1475  * and invalidates all buffer-cache-entries in that case. This
1476  * is a relatively slow routine, so we have to try to minimize using
1477  * it. Thus it is called only upon a 'mount' or 'open'. This
1478  * is the best way of combining speed and utility, I think.
1479  * People changing diskettes in the middle of an operation deserve
1480  * to lose :-)
1481  */
1482 int check_disk_change(struct block_device *bdev)
1483 {
1484         struct gendisk *disk = bdev->bd_disk;
1485         const struct block_device_operations *bdops = disk->fops;
1486         unsigned int events;
1487 
1488         events = disk_clear_events(disk, DISK_EVENT_MEDIA_CHANGE |
1489                                    DISK_EVENT_EJECT_REQUEST);
1490         if (!(events & DISK_EVENT_MEDIA_CHANGE))
1491                 return 0;
1492 
1493         flush_disk(bdev, true);
1494         if (bdops->revalidate_disk)
1495                 bdops->revalidate_disk(bdev->bd_disk);
1496         return 1;
1497 }
1498 
1499 EXPORT_SYMBOL(check_disk_change);
1500 
1501 void bd_set_size(struct block_device *bdev, loff_t size)
1502 {
1503         inode_lock(bdev->bd_inode);
1504         i_size_write(bdev->bd_inode, size);
1505         inode_unlock(bdev->bd_inode);
1506 }
1507 EXPORT_SYMBOL(bd_set_size);
1508 
1509 static void __blkdev_put(struct block_device *bdev, fmode_t mode, int for_part);
1510 
1511 int bdev_disk_changed(struct block_device *bdev, bool invalidate)
1512 {
1513         struct gendisk *disk = bdev->bd_disk;
1514         int ret;
1515 
1516         lockdep_assert_held(&bdev->bd_mutex);
1517 
1518 rescan:
1519         ret = blk_drop_partitions(disk, bdev);
1520         if (ret)
1521                 return ret;
1522 
1523         if (invalidate)
1524                 set_capacity(disk, 0);
1525         else if (disk->fops->revalidate_disk)
1526                 disk->fops->revalidate_disk(disk);
1527 
1528         check_disk_size_change(disk, bdev, !invalidate);
1529 
1530         if (get_capacity(disk)) {
1531                 ret = blk_add_partitions(disk, bdev);
1532                 if (ret == -EAGAIN)
1533                         goto rescan;
1534         } else if (invalidate) {
1535                 /*
1536                  * Tell userspace that the media / partition table may have
1537                  * changed.
1538                  */
1539                 kobject_uevent(&disk_to_dev(disk)->kobj, KOBJ_CHANGE);
1540         }
1541 
1542         return ret;
1543 }
1544 /*
1545  * Only exported for for loop and dasd for historic reasons.  Don't use in new
1546  * code!
1547  */
1548 EXPORT_SYMBOL_GPL(bdev_disk_changed);
1549 
1550 /*
1551  * bd_mutex locking:
1552  *
1553  *  mutex_lock(part->bd_mutex)
1554  *    mutex_lock_nested(whole->bd_mutex, 1)
1555  */
1556 
1557 static int __blkdev_get(struct block_device *bdev, fmode_t mode, int for_part)
1558 {
1559         struct gendisk *disk;
1560         int ret;
1561         int partno;
1562         int perm = 0;
1563         bool first_open = false;
1564 
1565         if (mode & FMODE_READ)
1566                 perm |= MAY_READ;
1567         if (mode & FMODE_WRITE)
1568                 perm |= MAY_WRITE;
1569         /*
1570          * hooks: /n/, see "layering violations".
1571          */
1572         if (!for_part) {
1573                 ret = devcgroup_inode_permission(bdev->bd_inode, perm);
1574                 if (ret != 0) {
1575                         bdput(bdev);
1576                         return ret;
1577                 }
1578         }
1579 
1580  restart:
1581 
1582         ret = -ENXIO;
1583         disk = bdev_get_gendisk(bdev, &partno);
1584         if (!disk)
1585                 goto out;
1586 
1587         disk_block_events(disk);
1588         mutex_lock_nested(&bdev->bd_mutex, for_part);
1589         if (!bdev->bd_openers) {
1590                 first_open = true;
1591                 bdev->bd_disk = disk;
1592                 bdev->bd_queue = disk->queue;
1593                 bdev->bd_contains = bdev;
1594                 bdev->bd_partno = partno;
1595 
1596                 if (!partno) {
1597                         ret = -ENXIO;
1598                         bdev->bd_part = disk_get_part(disk, partno);
1599                         if (!bdev->bd_part)
1600                                 goto out_clear;
1601 
1602                         ret = 0;
1603                         if (disk->fops->open) {
1604                                 ret = disk->fops->open(bdev, mode);
1605                                 if (ret == -ERESTARTSYS) {
1606                                         /* Lost a race with 'disk' being
1607                                          * deleted, try again.
1608                                          * See md.c
1609                                          */
1610                                         disk_put_part(bdev->bd_part);
1611                                         bdev->bd_part = NULL;
1612                                         bdev->bd_disk = NULL;
1613                                         bdev->bd_queue = NULL;
1614                                         mutex_unlock(&bdev->bd_mutex);
1615                                         disk_unblock_events(disk);
1616                                         put_disk_and_module(disk);
1617                                         goto restart;
1618                                 }
1619                         }
1620 
1621                         if (!ret) {
1622                                 bd_set_size(bdev,(loff_t)get_capacity(disk)<<9);
1623                                 set_init_blocksize(bdev);
1624                         }
1625 
1626                         /*
1627                          * If the device is invalidated, rescan partition
1628                          * if open succeeded or failed with -ENOMEDIUM.
1629                          * The latter is necessary to prevent ghost
1630                          * partitions on a removed medium.
1631                          */
1632                         if (bdev->bd_invalidated &&
1633                             (!ret || ret == -ENOMEDIUM))
1634                                 bdev_disk_changed(bdev, ret == -ENOMEDIUM);
1635 
1636                         if (ret)
1637                                 goto out_clear;
1638                 } else {
1639                         struct block_device *whole;
1640                         whole = bdget_disk(disk, 0);
1641                         ret = -ENOMEM;
1642                         if (!whole)
1643                                 goto out_clear;
1644                         BUG_ON(for_part);
1645                         ret = __blkdev_get(whole, mode, 1);
1646                         if (ret)
1647                                 goto out_clear;
1648                         bdev->bd_contains = whole;
1649                         bdev->bd_part = disk_get_part(disk, partno);
1650                         if (!(disk->flags & GENHD_FL_UP) ||
1651                             !bdev->bd_part || !bdev->bd_part->nr_sects) {
1652                                 ret = -ENXIO;
1653                                 goto out_clear;
1654                         }
1655                         bd_set_size(bdev, (loff_t)bdev->bd_part->nr_sects << 9);
1656                         set_init_blocksize(bdev);
1657                 }
1658 
1659                 if (bdev->bd_bdi == &noop_backing_dev_info)
1660                         bdev->bd_bdi = bdi_get(disk->queue->backing_dev_info);
1661         } else {
1662                 if (bdev->bd_contains == bdev) {
1663                         ret = 0;
1664                         if (bdev->bd_disk->fops->open)
1665                                 ret = bdev->bd_disk->fops->open(bdev, mode);
1666                         /* the same as first opener case, read comment there */
1667                         if (bdev->bd_invalidated &&
1668                             (!ret || ret == -ENOMEDIUM))
1669                                 bdev_disk_changed(bdev, ret == -ENOMEDIUM);
1670                         if (ret)
1671                                 goto out_unlock_bdev;
1672                 }
1673         }
1674         bdev->bd_openers++;
1675         if (for_part)
1676                 bdev->bd_part_count++;
1677         mutex_unlock(&bdev->bd_mutex);
1678         disk_unblock_events(disk);
1679         /* only one opener holds refs to the module and disk */
1680         if (!first_open)
1681                 put_disk_and_module(disk);
1682         return 0;
1683 
1684  out_clear:
1685         disk_put_part(bdev->bd_part);
1686         bdev->bd_disk = NULL;
1687         bdev->bd_part = NULL;
1688         bdev->bd_queue = NULL;
1689         if (bdev != bdev->bd_contains)
1690                 __blkdev_put(bdev->bd_contains, mode, 1);
1691         bdev->bd_contains = NULL;
1692  out_unlock_bdev:
1693         mutex_unlock(&bdev->bd_mutex);
1694         disk_unblock_events(disk);
1695         put_disk_and_module(disk);
1696  out:
1697         bdput(bdev);
1698 
1699         return ret;
1700 }
1701 
1702 /**
1703  * blkdev_get - open a block device
1704  * @bdev: block_device to open
1705  * @mode: FMODE_* mask
1706  * @holder: exclusive holder identifier
1707  *
1708  * Open @bdev with @mode.  If @mode includes %FMODE_EXCL, @bdev is
1709  * open with exclusive access.  Specifying %FMODE_EXCL with %NULL
1710  * @holder is invalid.  Exclusive opens may nest for the same @holder.
1711  *
1712  * On success, the reference count of @bdev is unchanged.  On failure,
1713  * @bdev is put.
1714  *
1715  * CONTEXT:
1716  * Might sleep.
1717  *
1718  * RETURNS:
1719  * 0 on success, -errno on failure.
1720  */
1721 int blkdev_get(struct block_device *bdev, fmode_t mode, void *holder)
1722 {
1723         struct block_device *whole = NULL;
1724         int res;
1725 
1726         WARN_ON_ONCE((mode & FMODE_EXCL) && !holder);
1727 
1728         if ((mode & FMODE_EXCL) && holder) {
1729                 whole = bd_start_claiming(bdev, holder);
1730                 if (IS_ERR(whole)) {
1731                         bdput(bdev);
1732                         return PTR_ERR(whole);
1733                 }
1734         }
1735 
1736         res = __blkdev_get(bdev, mode, 0);
1737 
1738         if (whole) {
1739                 struct gendisk *disk = whole->bd_disk;
1740 
1741                 /* finish claiming */
1742                 mutex_lock(&bdev->bd_mutex);
1743                 if (!res)
1744                         bd_finish_claiming(bdev, whole, holder);
1745                 else
1746                         bd_abort_claiming(bdev, whole, holder);
1747                 /*
1748                  * Block event polling for write claims if requested.  Any
1749                  * write holder makes the write_holder state stick until
1750                  * all are released.  This is good enough and tracking
1751                  * individual writeable reference is too fragile given the
1752                  * way @mode is used in blkdev_get/put().
1753                  */
1754                 if (!res && (mode & FMODE_WRITE) && !bdev->bd_write_holder &&
1755                     (disk->flags & GENHD_FL_BLOCK_EVENTS_ON_EXCL_WRITE)) {
1756                         bdev->bd_write_holder = true;
1757                         disk_block_events(disk);
1758                 }
1759 
1760                 mutex_unlock(&bdev->bd_mutex);
1761                 bdput(whole);
1762         }
1763 
1764         return res;
1765 }
1766 EXPORT_SYMBOL(blkdev_get);
1767 
1768 /**
1769  * blkdev_get_by_path - open a block device by name
1770  * @path: path to the block device to open
1771  * @mode: FMODE_* mask
1772  * @holder: exclusive holder identifier
1773  *
1774  * Open the blockdevice described by the device file at @path.  @mode
1775  * and @holder are identical to blkdev_get().
1776  *
1777  * On success, the returned block_device has reference count of one.
1778  *
1779  * CONTEXT:
1780  * Might sleep.
1781  *
1782  * RETURNS:
1783  * Pointer to block_device on success, ERR_PTR(-errno) on failure.
1784  */
1785 struct block_device *blkdev_get_by_path(const char *path, fmode_t mode,
1786                                         void *holder)
1787 {
1788         struct block_device *bdev;
1789         int err;
1790 
1791         bdev = lookup_bdev(path);
1792         if (IS_ERR(bdev))
1793                 return bdev;
1794 
1795         err = blkdev_get(bdev, mode, holder);
1796         if (err)
1797                 return ERR_PTR(err);
1798 
1799         if ((mode & FMODE_WRITE) && bdev_read_only(bdev)) {
1800                 blkdev_put(bdev, mode);
1801                 return ERR_PTR(-EACCES);
1802         }
1803 
1804         return bdev;
1805 }
1806 EXPORT_SYMBOL(blkdev_get_by_path);
1807 
1808 /**
1809  * blkdev_get_by_dev - open a block device by device number
1810  * @dev: device number of block device to open
1811  * @mode: FMODE_* mask
1812  * @holder: exclusive holder identifier
1813  *
1814  * Open the blockdevice described by device number @dev.  @mode and
1815  * @holder are identical to blkdev_get().
1816  *
1817  * Use it ONLY if you really do not have anything better - i.e. when
1818  * you are behind a truly sucky interface and all you are given is a
1819  * device number.  _Never_ to be used for internal purposes.  If you
1820  * ever need it - reconsider your API.
1821  *
1822  * On success, the returned block_device has reference count of one.
1823  *
1824  * CONTEXT:
1825  * Might sleep.
1826  *
1827  * RETURNS:
1828  * Pointer to block_device on success, ERR_PTR(-errno) on failure.
1829  */
1830 struct block_device *blkdev_get_by_dev(dev_t dev, fmode_t mode, void *holder)
1831 {
1832         struct block_device *bdev;
1833         int err;
1834 
1835         bdev = bdget(dev);
1836         if (!bdev)
1837                 return ERR_PTR(-ENOMEM);
1838 
1839         err = blkdev_get(bdev, mode, holder);
1840         if (err)
1841                 return ERR_PTR(err);
1842 
1843         return bdev;
1844 }
1845 EXPORT_SYMBOL(blkdev_get_by_dev);
1846 
1847 static int blkdev_open(struct inode * inode, struct file * filp)
1848 {
1849         struct block_device *bdev;
1850 
1851         /*
1852          * Preserve backwards compatibility and allow large file access
1853          * even if userspace doesn't ask for it explicitly. Some mkfs
1854          * binary needs it. We might want to drop this workaround
1855          * during an unstable branch.
1856          */
1857         filp->f_flags |= O_LARGEFILE;
1858 
1859         filp->f_mode |= FMODE_NOWAIT;
1860 
1861         if (filp->f_flags & O_NDELAY)
1862                 filp->f_mode |= FMODE_NDELAY;
1863         if (filp->f_flags & O_EXCL)
1864                 filp->f_mode |= FMODE_EXCL;
1865         if ((filp->f_flags & O_ACCMODE) == 3)
1866                 filp->f_mode |= FMODE_WRITE_IOCTL;
1867 
1868         bdev = bd_acquire(inode);
1869         if (bdev == NULL)
1870                 return -ENOMEM;
1871 
1872         filp->f_mapping = bdev->bd_inode->i_mapping;
1873         filp->f_wb_err = filemap_sample_wb_err(filp->f_mapping);
1874 
1875         return blkdev_get(bdev, filp->f_mode, filp);
1876 }
1877 
1878 static void __blkdev_put(struct block_device *bdev, fmode_t mode, int for_part)
1879 {
1880         struct gendisk *disk = bdev->bd_disk;
1881         struct block_device *victim = NULL;
1882 
1883         mutex_lock_nested(&bdev->bd_mutex, for_part);
1884         if (for_part)
1885                 bdev->bd_part_count--;
1886 
1887         if (!--bdev->bd_openers) {
1888                 WARN_ON_ONCE(bdev->bd_holders);
1889                 sync_blockdev(bdev);
1890                 kill_bdev(bdev);
1891 
1892                 bdev_write_inode(bdev);
1893         }
1894         if (bdev->bd_contains == bdev) {
1895                 if (disk->fops->release)
1896                         disk->fops->release(disk, mode);
1897         }
1898         if (!bdev->bd_openers) {
1899                 disk_put_part(bdev->bd_part);
1900                 bdev->bd_part = NULL;
1901                 bdev->bd_disk = NULL;
1902                 if (bdev != bdev->bd_contains)
1903                         victim = bdev->bd_contains;
1904                 bdev->bd_contains = NULL;
1905 
1906                 put_disk_and_module(disk);
1907         }
1908         mutex_unlock(&bdev->bd_mutex);
1909         bdput(bdev);
1910         if (victim)
1911                 __blkdev_put(victim, mode, 1);
1912 }
1913 
1914 void blkdev_put(struct block_device *bdev, fmode_t mode)
1915 {
1916         mutex_lock(&bdev->bd_mutex);
1917 
1918         if (mode & FMODE_EXCL) {
1919                 bool bdev_free;
1920 
1921                 /*
1922                  * Release a claim on the device.  The holder fields
1923                  * are protected with bdev_lock.  bd_mutex is to
1924                  * synchronize disk_holder unlinking.
1925                  */
1926                 spin_lock(&bdev_lock);
1927 
1928                 WARN_ON_ONCE(--bdev->bd_holders < 0);
1929                 WARN_ON_ONCE(--bdev->bd_contains->bd_holders < 0);
1930 
1931                 /* bd_contains might point to self, check in a separate step */
1932                 if ((bdev_free = !bdev->bd_holders))
1933                         bdev->bd_holder = NULL;
1934                 if (!bdev->bd_contains->bd_holders)
1935                         bdev->bd_contains->bd_holder = NULL;
1936 
1937                 spin_unlock(&bdev_lock);
1938 
1939                 /*
1940                  * If this was the last claim, remove holder link and
1941                  * unblock evpoll if it was a write holder.
1942                  */
1943                 if (bdev_free && bdev->bd_write_holder) {
1944                         disk_unblock_events(bdev->bd_disk);
1945                         bdev->bd_write_holder = false;
1946                 }
1947         }
1948 
1949         /*
1950          * Trigger event checking and tell drivers to flush MEDIA_CHANGE
1951          * event.  This is to ensure detection of media removal commanded
1952          * from userland - e.g. eject(1).
1953          */
1954         disk_flush_events(bdev->bd_disk, DISK_EVENT_MEDIA_CHANGE);
1955 
1956         mutex_unlock(&bdev->bd_mutex);
1957 
1958         __blkdev_put(bdev, mode, 0);
1959 }
1960 EXPORT_SYMBOL(blkdev_put);
1961 
1962 static int blkdev_close(struct inode * inode, struct file * filp)
1963 {
1964         struct block_device *bdev = I_BDEV(bdev_file_inode(filp));
1965         blkdev_put(bdev, filp->f_mode);
1966         return 0;
1967 }
1968 
1969 static long block_ioctl(struct file *file, unsigned cmd, unsigned long arg)
1970 {
1971         struct block_device *bdev = I_BDEV(bdev_file_inode(file));
1972         fmode_t mode = file->f_mode;
1973 
1974         /*
1975          * O_NDELAY can be altered using fcntl(.., F_SETFL, ..), so we have
1976          * to updated it before every ioctl.
1977          */
1978         if (file->f_flags & O_NDELAY)
1979                 mode |= FMODE_NDELAY;
1980         else
1981                 mode &= ~FMODE_NDELAY;
1982 
1983         return blkdev_ioctl(bdev, mode, cmd, arg);
1984 }
1985 
1986 /*
1987  * Write data to the block device.  Only intended for the block device itself
1988  * and the raw driver which basically is a fake block device.
1989  *
1990  * Does not take i_mutex for the write and thus is not for general purpose
1991  * use.
1992  */
1993 ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from)
1994 {
1995         struct file *file = iocb->ki_filp;
1996         struct inode *bd_inode = bdev_file_inode(file);
1997         loff_t size = i_size_read(bd_inode);
1998         struct blk_plug plug;
1999         ssize_t ret;
2000 
2001         if (bdev_read_only(I_BDEV(bd_inode)))
2002                 return -EPERM;
2003 
2004         if (IS_SWAPFILE(bd_inode))
2005                 return -ETXTBSY;
2006 
2007         if (!iov_iter_count(from))
2008                 return 0;
2009 
2010         if (iocb->ki_pos >= size)
2011                 return -ENOSPC;
2012 
2013         if ((iocb->ki_flags & (IOCB_NOWAIT | IOCB_DIRECT)) == IOCB_NOWAIT)
2014                 return -EOPNOTSUPP;
2015 
2016         iov_iter_truncate(from, size - iocb->ki_pos);
2017 
2018         blk_start_plug(&plug);
2019         ret = __generic_file_write_iter(iocb, from);
2020         if (ret > 0)
2021                 ret = generic_write_sync(iocb, ret);
2022         blk_finish_plug(&plug);
2023         return ret;
2024 }
2025 EXPORT_SYMBOL_GPL(blkdev_write_iter);
2026 
2027 ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to)
2028 {
2029         struct file *file = iocb->ki_filp;
2030         struct inode *bd_inode = bdev_file_inode(file);
2031         loff_t size = i_size_read(bd_inode);
2032         loff_t pos = iocb->ki_pos;
2033 
2034         if (pos >= size)
2035                 return 0;
2036 
2037         size -= pos;
2038         iov_iter_truncate(to, size);
2039         return generic_file_read_iter(iocb, to);
2040 }
2041 EXPORT_SYMBOL_GPL(blkdev_read_iter);
2042 
2043 /*
2044  * Try to release a page associated with block device when the system
2045  * is under memory pressure.
2046  */
2047 static int blkdev_releasepage(struct page *page, gfp_t wait)
2048 {
2049         struct super_block *super = BDEV_I(page->mapping->host)->bdev.bd_super;
2050 
2051         if (super && super->s_op->bdev_try_to_free_page)
2052                 return super->s_op->bdev_try_to_free_page(super, page, wait);
2053 
2054         return try_to_free_buffers(page);
2055 }
2056 
2057 static int blkdev_writepages(struct address_space *mapping,
2058                              struct writeback_control *wbc)
2059 {
2060         return generic_writepages(mapping, wbc);
2061 }
2062 
2063 static const struct address_space_operations def_blk_aops = {
2064         .readpage       = blkdev_readpage,
2065         .readpages      = blkdev_readpages,
2066         .writepage      = blkdev_writepage,
2067         .write_begin    = blkdev_write_begin,
2068         .write_end      = blkdev_write_end,
2069         .writepages     = blkdev_writepages,
2070         .releasepage    = blkdev_releasepage,
2071         .direct_IO      = blkdev_direct_IO,
2072         .migratepage    = buffer_migrate_page_norefs,
2073         .is_dirty_writeback = buffer_check_dirty_writeback,
2074 };
2075 
2076 #define BLKDEV_FALLOC_FL_SUPPORTED                                      \
2077                 (FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE |           \
2078                  FALLOC_FL_ZERO_RANGE | FALLOC_FL_NO_HIDE_STALE)
2079 
2080 static long blkdev_fallocate(struct file *file, int mode, loff_t start,
2081                              loff_t len)
2082 {
2083         struct block_device *bdev = I_BDEV(bdev_file_inode(file));
2084         struct address_space *mapping;
2085         loff_t end = start + len - 1;
2086         loff_t isize;
2087         int error;
2088 
2089         /* Fail if we don't recognize the flags. */
2090         if (mode & ~BLKDEV_FALLOC_FL_SUPPORTED)
2091                 return -EOPNOTSUPP;
2092 
2093         /* Don't go off the end of the device. */
2094         isize = i_size_read(bdev->bd_inode);
2095         if (start >= isize)
2096                 return -EINVAL;
2097         if (end >= isize) {
2098                 if (mode & FALLOC_FL_KEEP_SIZE) {
2099                         len = isize - start;
2100                         end = start + len - 1;
2101                 } else
2102                         return -EINVAL;
2103         }
2104 
2105         /*
2106          * Don't allow IO that isn't aligned to logical block size.
2107          */
2108         if ((start | len) & (bdev_logical_block_size(bdev) - 1))
2109                 return -EINVAL;
2110 
2111         /* Invalidate the page cache, including dirty pages. */
2112         mapping = bdev->bd_inode->i_mapping;
2113         truncate_inode_pages_range(mapping, start, end);
2114 
2115         switch (mode) {
2116         case FALLOC_FL_ZERO_RANGE:
2117         case FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE:
2118                 error = blkdev_issue_zeroout(bdev, start >> 9, len >> 9,
2119                                             GFP_KERNEL, BLKDEV_ZERO_NOUNMAP);
2120                 break;
2121         case FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE:
2122                 error = blkdev_issue_zeroout(bdev, start >> 9, len >> 9,
2123                                              GFP_KERNEL, BLKDEV_ZERO_NOFALLBACK);
2124                 break;
2125         case FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE | FALLOC_FL_NO_HIDE_STALE:
2126                 error = blkdev_issue_discard(bdev, start >> 9, len >> 9,
2127                                              GFP_KERNEL, 0);
2128                 break;
2129         default:
2130                 return -EOPNOTSUPP;
2131         }
2132         if (error)
2133                 return error;
2134 
2135         /*
2136          * Invalidate again; if someone wandered in and dirtied a page,
2137          * the caller will be given -EBUSY.  The third argument is
2138          * inclusive, so the rounding here is safe.
2139          */
2140         return invalidate_inode_pages2_range(mapping,
2141                                              start >> PAGE_SHIFT,
2142                                              end >> PAGE_SHIFT);
2143 }
2144 
2145 const struct file_operations def_blk_fops = {
2146         .open           = blkdev_open,
2147         .release        = blkdev_close,
2148         .llseek         = block_llseek,
2149         .read_iter      = blkdev_read_iter,
2150         .write_iter     = blkdev_write_iter,
2151         .iopoll         = blkdev_iopoll,
2152         .mmap           = generic_file_mmap,
2153         .fsync          = blkdev_fsync,
2154         .unlocked_ioctl = block_ioctl,
2155 #ifdef CONFIG_COMPAT
2156         .compat_ioctl   = compat_blkdev_ioctl,
2157 #endif
2158         .splice_read    = generic_file_splice_read,
2159         .splice_write   = iter_file_splice_write,
2160         .fallocate      = blkdev_fallocate,
2161 };
2162 
2163 int ioctl_by_bdev(struct block_device *bdev, unsigned cmd, unsigned long arg)
2164 {
2165         int res;
2166         mm_segment_t old_fs = get_fs();
2167         set_fs(KERNEL_DS);
2168         res = blkdev_ioctl(bdev, 0, cmd, arg);
2169         set_fs(old_fs);
2170         return res;
2171 }
2172 
2173 EXPORT_SYMBOL(ioctl_by_bdev);
2174 
2175 /**
2176  * lookup_bdev  - lookup a struct block_device by name
2177  * @pathname:   special file representing the block device
2178  *
2179  * Get a reference to the blockdevice at @pathname in the current
2180  * namespace if possible and return it.  Return ERR_PTR(error)
2181  * otherwise.
2182  */
2183 struct block_device *lookup_bdev(const char *pathname)
2184 {
2185         struct block_device *bdev;
2186         struct inode *inode;
2187         struct path path;
2188         int error;
2189 
2190         if (!pathname || !*pathname)
2191                 return ERR_PTR(-EINVAL);
2192 
2193         error = kern_path(pathname, LOOKUP_FOLLOW, &path);
2194         if (error)
2195                 return ERR_PTR(error);
2196 
2197         inode = d_backing_inode(path.dentry);
2198         error = -ENOTBLK;
2199         if (!S_ISBLK(inode->i_mode))
2200                 goto fail;
2201         error = -EACCES;
2202         if (!may_open_dev(&path))
2203                 goto fail;
2204         error = -ENOMEM;
2205         bdev = bd_acquire(inode);
2206         if (!bdev)
2207                 goto fail;
2208 out:
2209         path_put(&path);
2210         return bdev;
2211 fail:
2212         bdev = ERR_PTR(error);
2213         goto out;
2214 }
2215 EXPORT_SYMBOL(lookup_bdev);
2216 
2217 int __invalidate_device(struct block_device *bdev, bool kill_dirty)
2218 {
2219         struct super_block *sb = get_super(bdev);
2220         int res = 0;
2221 
2222         if (sb) {
2223                 /*
2224                  * no need to lock the super, get_super holds the
2225                  * read mutex so the filesystem cannot go away
2226                  * under us (->put_super runs with the write lock
2227                  * hold).
2228                  */
2229                 shrink_dcache_sb(sb);
2230                 res = invalidate_inodes(sb, kill_dirty);
2231                 drop_super(sb);
2232         }
2233         invalidate_bdev(bdev);
2234         return res;
2235 }
2236 EXPORT_SYMBOL(__invalidate_device);
2237 
2238 void iterate_bdevs(void (*func)(struct block_device *, void *), void *arg)
2239 {
2240         struct inode *inode, *old_inode = NULL;
2241 
2242         spin_lock(&blockdev_superblock->s_inode_list_lock);
2243         list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) {
2244                 struct address_space *mapping = inode->i_mapping;
2245                 struct block_device *bdev;
2246 
2247                 spin_lock(&inode->i_lock);
2248                 if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW) ||
2249                     mapping->nrpages == 0) {
2250                         spin_unlock(&inode->i_lock);
2251                         continue;
2252                 }
2253                 __iget(inode);
2254                 spin_unlock(&inode->i_lock);
2255                 spin_unlock(&blockdev_superblock->s_inode_list_lock);
2256                 /*
2257                  * We hold a reference to 'inode' so it couldn't have been
2258                  * removed from s_inodes list while we dropped the
2259                  * s_inode_list_lock  We cannot iput the inode now as we can
2260                  * be holding the last reference and we cannot iput it under
2261                  * s_inode_list_lock. So we keep the reference and iput it
2262                  * later.
2263                  */
2264                 iput(old_inode);
2265                 old_inode = inode;
2266                 bdev = I_BDEV(inode);
2267 
2268                 mutex_lock(&bdev->bd_mutex);
2269                 if (bdev->bd_openers)
2270                         func(bdev, arg);
2271                 mutex_unlock(&bdev->bd_mutex);
2272 
2273                 spin_lock(&blockdev_superblock->s_inode_list_lock);
2274         }
2275         spin_unlock(&blockdev_superblock->s_inode_list_lock);
2276         iput(old_inode);
2277 }
2278 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp