~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/fs/buffer.c

Version: ~ [ linux-5.15-rc5 ] ~ [ linux-5.14.11 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.72 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.152 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.210 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.250 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.286 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.288 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0-only
  2 /*
  3  *  linux/fs/buffer.c
  4  *
  5  *  Copyright (C) 1991, 1992, 2002  Linus Torvalds
  6  */
  7 
  8 /*
  9  * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
 10  *
 11  * Removed a lot of unnecessary code and simplified things now that
 12  * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
 13  *
 14  * Speed up hash, lru, and free list operations.  Use gfp() for allocating
 15  * hash table, use SLAB cache for buffer heads. SMP threading.  -DaveM
 16  *
 17  * Added 32k buffer block sizes - these are required older ARM systems. - RMK
 18  *
 19  * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
 20  */
 21 
 22 #include <linux/kernel.h>
 23 #include <linux/sched/signal.h>
 24 #include <linux/syscalls.h>
 25 #include <linux/fs.h>
 26 #include <linux/iomap.h>
 27 #include <linux/mm.h>
 28 #include <linux/percpu.h>
 29 #include <linux/slab.h>
 30 #include <linux/capability.h>
 31 #include <linux/blkdev.h>
 32 #include <linux/file.h>
 33 #include <linux/quotaops.h>
 34 #include <linux/highmem.h>
 35 #include <linux/export.h>
 36 #include <linux/backing-dev.h>
 37 #include <linux/writeback.h>
 38 #include <linux/hash.h>
 39 #include <linux/suspend.h>
 40 #include <linux/buffer_head.h>
 41 #include <linux/task_io_accounting_ops.h>
 42 #include <linux/bio.h>
 43 #include <linux/cpu.h>
 44 #include <linux/bitops.h>
 45 #include <linux/mpage.h>
 46 #include <linux/bit_spinlock.h>
 47 #include <linux/pagevec.h>
 48 #include <linux/sched/mm.h>
 49 #include <trace/events/block.h>
 50 
 51 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
 52 static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh,
 53                          enum rw_hint hint, struct writeback_control *wbc);
 54 
 55 #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
 56 
 57 inline void touch_buffer(struct buffer_head *bh)
 58 {
 59         trace_block_touch_buffer(bh);
 60         mark_page_accessed(bh->b_page);
 61 }
 62 EXPORT_SYMBOL(touch_buffer);
 63 
 64 void __lock_buffer(struct buffer_head *bh)
 65 {
 66         wait_on_bit_lock_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
 67 }
 68 EXPORT_SYMBOL(__lock_buffer);
 69 
 70 void unlock_buffer(struct buffer_head *bh)
 71 {
 72         clear_bit_unlock(BH_Lock, &bh->b_state);
 73         smp_mb__after_atomic();
 74         wake_up_bit(&bh->b_state, BH_Lock);
 75 }
 76 EXPORT_SYMBOL(unlock_buffer);
 77 
 78 /*
 79  * Returns if the page has dirty or writeback buffers. If all the buffers
 80  * are unlocked and clean then the PageDirty information is stale. If
 81  * any of the pages are locked, it is assumed they are locked for IO.
 82  */
 83 void buffer_check_dirty_writeback(struct page *page,
 84                                      bool *dirty, bool *writeback)
 85 {
 86         struct buffer_head *head, *bh;
 87         *dirty = false;
 88         *writeback = false;
 89 
 90         BUG_ON(!PageLocked(page));
 91 
 92         if (!page_has_buffers(page))
 93                 return;
 94 
 95         if (PageWriteback(page))
 96                 *writeback = true;
 97 
 98         head = page_buffers(page);
 99         bh = head;
100         do {
101                 if (buffer_locked(bh))
102                         *writeback = true;
103 
104                 if (buffer_dirty(bh))
105                         *dirty = true;
106 
107                 bh = bh->b_this_page;
108         } while (bh != head);
109 }
110 EXPORT_SYMBOL(buffer_check_dirty_writeback);
111 
112 /*
113  * Block until a buffer comes unlocked.  This doesn't stop it
114  * from becoming locked again - you have to lock it yourself
115  * if you want to preserve its state.
116  */
117 void __wait_on_buffer(struct buffer_head * bh)
118 {
119         wait_on_bit_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
120 }
121 EXPORT_SYMBOL(__wait_on_buffer);
122 
123 static void
124 __clear_page_buffers(struct page *page)
125 {
126         ClearPagePrivate(page);
127         set_page_private(page, 0);
128         put_page(page);
129 }
130 
131 static void buffer_io_error(struct buffer_head *bh, char *msg)
132 {
133         if (!test_bit(BH_Quiet, &bh->b_state))
134                 printk_ratelimited(KERN_ERR
135                         "Buffer I/O error on dev %pg, logical block %llu%s\n",
136                         bh->b_bdev, (unsigned long long)bh->b_blocknr, msg);
137 }
138 
139 /*
140  * End-of-IO handler helper function which does not touch the bh after
141  * unlocking it.
142  * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
143  * a race there is benign: unlock_buffer() only use the bh's address for
144  * hashing after unlocking the buffer, so it doesn't actually touch the bh
145  * itself.
146  */
147 static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate)
148 {
149         if (uptodate) {
150                 set_buffer_uptodate(bh);
151         } else {
152                 /* This happens, due to failed read-ahead attempts. */
153                 clear_buffer_uptodate(bh);
154         }
155         unlock_buffer(bh);
156 }
157 
158 /*
159  * Default synchronous end-of-IO handler..  Just mark it up-to-date and
160  * unlock the buffer. This is what ll_rw_block uses too.
161  */
162 void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
163 {
164         __end_buffer_read_notouch(bh, uptodate);
165         put_bh(bh);
166 }
167 EXPORT_SYMBOL(end_buffer_read_sync);
168 
169 void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
170 {
171         if (uptodate) {
172                 set_buffer_uptodate(bh);
173         } else {
174                 buffer_io_error(bh, ", lost sync page write");
175                 mark_buffer_write_io_error(bh);
176                 clear_buffer_uptodate(bh);
177         }
178         unlock_buffer(bh);
179         put_bh(bh);
180 }
181 EXPORT_SYMBOL(end_buffer_write_sync);
182 
183 /*
184  * Various filesystems appear to want __find_get_block to be non-blocking.
185  * But it's the page lock which protects the buffers.  To get around this,
186  * we get exclusion from try_to_free_buffers with the blockdev mapping's
187  * private_lock.
188  *
189  * Hack idea: for the blockdev mapping, private_lock contention
190  * may be quite high.  This code could TryLock the page, and if that
191  * succeeds, there is no need to take private_lock.
192  */
193 static struct buffer_head *
194 __find_get_block_slow(struct block_device *bdev, sector_t block)
195 {
196         struct inode *bd_inode = bdev->bd_inode;
197         struct address_space *bd_mapping = bd_inode->i_mapping;
198         struct buffer_head *ret = NULL;
199         pgoff_t index;
200         struct buffer_head *bh;
201         struct buffer_head *head;
202         struct page *page;
203         int all_mapped = 1;
204         static DEFINE_RATELIMIT_STATE(last_warned, HZ, 1);
205 
206         index = block >> (PAGE_SHIFT - bd_inode->i_blkbits);
207         page = find_get_page_flags(bd_mapping, index, FGP_ACCESSED);
208         if (!page)
209                 goto out;
210 
211         spin_lock(&bd_mapping->private_lock);
212         if (!page_has_buffers(page))
213                 goto out_unlock;
214         head = page_buffers(page);
215         bh = head;
216         do {
217                 if (!buffer_mapped(bh))
218                         all_mapped = 0;
219                 else if (bh->b_blocknr == block) {
220                         ret = bh;
221                         get_bh(bh);
222                         goto out_unlock;
223                 }
224                 bh = bh->b_this_page;
225         } while (bh != head);
226 
227         /* we might be here because some of the buffers on this page are
228          * not mapped.  This is due to various races between
229          * file io on the block device and getblk.  It gets dealt with
230          * elsewhere, don't buffer_error if we had some unmapped buffers
231          */
232         ratelimit_set_flags(&last_warned, RATELIMIT_MSG_ON_RELEASE);
233         if (all_mapped && __ratelimit(&last_warned)) {
234                 printk("__find_get_block_slow() failed. block=%llu, "
235                        "b_blocknr=%llu, b_state=0x%08lx, b_size=%zu, "
236                        "device %pg blocksize: %d\n",
237                        (unsigned long long)block,
238                        (unsigned long long)bh->b_blocknr,
239                        bh->b_state, bh->b_size, bdev,
240                        1 << bd_inode->i_blkbits);
241         }
242 out_unlock:
243         spin_unlock(&bd_mapping->private_lock);
244         put_page(page);
245 out:
246         return ret;
247 }
248 
249 /*
250  * I/O completion handler for block_read_full_page() - pages
251  * which come unlocked at the end of I/O.
252  */
253 static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
254 {
255         unsigned long flags;
256         struct buffer_head *first;
257         struct buffer_head *tmp;
258         struct page *page;
259         int page_uptodate = 1;
260 
261         BUG_ON(!buffer_async_read(bh));
262 
263         page = bh->b_page;
264         if (uptodate) {
265                 set_buffer_uptodate(bh);
266         } else {
267                 clear_buffer_uptodate(bh);
268                 buffer_io_error(bh, ", async page read");
269                 SetPageError(page);
270         }
271 
272         /*
273          * Be _very_ careful from here on. Bad things can happen if
274          * two buffer heads end IO at almost the same time and both
275          * decide that the page is now completely done.
276          */
277         first = page_buffers(page);
278         local_irq_save(flags);
279         bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
280         clear_buffer_async_read(bh);
281         unlock_buffer(bh);
282         tmp = bh;
283         do {
284                 if (!buffer_uptodate(tmp))
285                         page_uptodate = 0;
286                 if (buffer_async_read(tmp)) {
287                         BUG_ON(!buffer_locked(tmp));
288                         goto still_busy;
289                 }
290                 tmp = tmp->b_this_page;
291         } while (tmp != bh);
292         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
293         local_irq_restore(flags);
294 
295         /*
296          * If none of the buffers had errors and they are all
297          * uptodate then we can set the page uptodate.
298          */
299         if (page_uptodate && !PageError(page))
300                 SetPageUptodate(page);
301         unlock_page(page);
302         return;
303 
304 still_busy:
305         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
306         local_irq_restore(flags);
307         return;
308 }
309 
310 /*
311  * Completion handler for block_write_full_page() - pages which are unlocked
312  * during I/O, and which have PageWriteback cleared upon I/O completion.
313  */
314 void end_buffer_async_write(struct buffer_head *bh, int uptodate)
315 {
316         unsigned long flags;
317         struct buffer_head *first;
318         struct buffer_head *tmp;
319         struct page *page;
320 
321         BUG_ON(!buffer_async_write(bh));
322 
323         page = bh->b_page;
324         if (uptodate) {
325                 set_buffer_uptodate(bh);
326         } else {
327                 buffer_io_error(bh, ", lost async page write");
328                 mark_buffer_write_io_error(bh);
329                 clear_buffer_uptodate(bh);
330                 SetPageError(page);
331         }
332 
333         first = page_buffers(page);
334         local_irq_save(flags);
335         bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
336 
337         clear_buffer_async_write(bh);
338         unlock_buffer(bh);
339         tmp = bh->b_this_page;
340         while (tmp != bh) {
341                 if (buffer_async_write(tmp)) {
342                         BUG_ON(!buffer_locked(tmp));
343                         goto still_busy;
344                 }
345                 tmp = tmp->b_this_page;
346         }
347         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
348         local_irq_restore(flags);
349         end_page_writeback(page);
350         return;
351 
352 still_busy:
353         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
354         local_irq_restore(flags);
355         return;
356 }
357 EXPORT_SYMBOL(end_buffer_async_write);
358 
359 /*
360  * If a page's buffers are under async readin (end_buffer_async_read
361  * completion) then there is a possibility that another thread of
362  * control could lock one of the buffers after it has completed
363  * but while some of the other buffers have not completed.  This
364  * locked buffer would confuse end_buffer_async_read() into not unlocking
365  * the page.  So the absence of BH_Async_Read tells end_buffer_async_read()
366  * that this buffer is not under async I/O.
367  *
368  * The page comes unlocked when it has no locked buffer_async buffers
369  * left.
370  *
371  * PageLocked prevents anyone starting new async I/O reads any of
372  * the buffers.
373  *
374  * PageWriteback is used to prevent simultaneous writeout of the same
375  * page.
376  *
377  * PageLocked prevents anyone from starting writeback of a page which is
378  * under read I/O (PageWriteback is only ever set against a locked page).
379  */
380 static void mark_buffer_async_read(struct buffer_head *bh)
381 {
382         bh->b_end_io = end_buffer_async_read;
383         set_buffer_async_read(bh);
384 }
385 
386 static void mark_buffer_async_write_endio(struct buffer_head *bh,
387                                           bh_end_io_t *handler)
388 {
389         bh->b_end_io = handler;
390         set_buffer_async_write(bh);
391 }
392 
393 void mark_buffer_async_write(struct buffer_head *bh)
394 {
395         mark_buffer_async_write_endio(bh, end_buffer_async_write);
396 }
397 EXPORT_SYMBOL(mark_buffer_async_write);
398 
399 
400 /*
401  * fs/buffer.c contains helper functions for buffer-backed address space's
402  * fsync functions.  A common requirement for buffer-based filesystems is
403  * that certain data from the backing blockdev needs to be written out for
404  * a successful fsync().  For example, ext2 indirect blocks need to be
405  * written back and waited upon before fsync() returns.
406  *
407  * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
408  * inode_has_buffers() and invalidate_inode_buffers() are provided for the
409  * management of a list of dependent buffers at ->i_mapping->private_list.
410  *
411  * Locking is a little subtle: try_to_free_buffers() will remove buffers
412  * from their controlling inode's queue when they are being freed.  But
413  * try_to_free_buffers() will be operating against the *blockdev* mapping
414  * at the time, not against the S_ISREG file which depends on those buffers.
415  * So the locking for private_list is via the private_lock in the address_space
416  * which backs the buffers.  Which is different from the address_space 
417  * against which the buffers are listed.  So for a particular address_space,
418  * mapping->private_lock does *not* protect mapping->private_list!  In fact,
419  * mapping->private_list will always be protected by the backing blockdev's
420  * ->private_lock.
421  *
422  * Which introduces a requirement: all buffers on an address_space's
423  * ->private_list must be from the same address_space: the blockdev's.
424  *
425  * address_spaces which do not place buffers at ->private_list via these
426  * utility functions are free to use private_lock and private_list for
427  * whatever they want.  The only requirement is that list_empty(private_list)
428  * be true at clear_inode() time.
429  *
430  * FIXME: clear_inode should not call invalidate_inode_buffers().  The
431  * filesystems should do that.  invalidate_inode_buffers() should just go
432  * BUG_ON(!list_empty).
433  *
434  * FIXME: mark_buffer_dirty_inode() is a data-plane operation.  It should
435  * take an address_space, not an inode.  And it should be called
436  * mark_buffer_dirty_fsync() to clearly define why those buffers are being
437  * queued up.
438  *
439  * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
440  * list if it is already on a list.  Because if the buffer is on a list,
441  * it *must* already be on the right one.  If not, the filesystem is being
442  * silly.  This will save a ton of locking.  But first we have to ensure
443  * that buffers are taken *off* the old inode's list when they are freed
444  * (presumably in truncate).  That requires careful auditing of all
445  * filesystems (do it inside bforget()).  It could also be done by bringing
446  * b_inode back.
447  */
448 
449 /*
450  * The buffer's backing address_space's private_lock must be held
451  */
452 static void __remove_assoc_queue(struct buffer_head *bh)
453 {
454         list_del_init(&bh->b_assoc_buffers);
455         WARN_ON(!bh->b_assoc_map);
456         bh->b_assoc_map = NULL;
457 }
458 
459 int inode_has_buffers(struct inode *inode)
460 {
461         return !list_empty(&inode->i_data.private_list);
462 }
463 
464 /*
465  * osync is designed to support O_SYNC io.  It waits synchronously for
466  * all already-submitted IO to complete, but does not queue any new
467  * writes to the disk.
468  *
469  * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
470  * you dirty the buffers, and then use osync_inode_buffers to wait for
471  * completion.  Any other dirty buffers which are not yet queued for
472  * write will not be flushed to disk by the osync.
473  */
474 static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
475 {
476         struct buffer_head *bh;
477         struct list_head *p;
478         int err = 0;
479 
480         spin_lock(lock);
481 repeat:
482         list_for_each_prev(p, list) {
483                 bh = BH_ENTRY(p);
484                 if (buffer_locked(bh)) {
485                         get_bh(bh);
486                         spin_unlock(lock);
487                         wait_on_buffer(bh);
488                         if (!buffer_uptodate(bh))
489                                 err = -EIO;
490                         brelse(bh);
491                         spin_lock(lock);
492                         goto repeat;
493                 }
494         }
495         spin_unlock(lock);
496         return err;
497 }
498 
499 void emergency_thaw_bdev(struct super_block *sb)
500 {
501         while (sb->s_bdev && !thaw_bdev(sb->s_bdev, sb))
502                 printk(KERN_WARNING "Emergency Thaw on %pg\n", sb->s_bdev);
503 }
504 
505 /**
506  * sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers
507  * @mapping: the mapping which wants those buffers written
508  *
509  * Starts I/O against the buffers at mapping->private_list, and waits upon
510  * that I/O.
511  *
512  * Basically, this is a convenience function for fsync().
513  * @mapping is a file or directory which needs those buffers to be written for
514  * a successful fsync().
515  */
516 int sync_mapping_buffers(struct address_space *mapping)
517 {
518         struct address_space *buffer_mapping = mapping->private_data;
519 
520         if (buffer_mapping == NULL || list_empty(&mapping->private_list))
521                 return 0;
522 
523         return fsync_buffers_list(&buffer_mapping->private_lock,
524                                         &mapping->private_list);
525 }
526 EXPORT_SYMBOL(sync_mapping_buffers);
527 
528 /*
529  * Called when we've recently written block `bblock', and it is known that
530  * `bblock' was for a buffer_boundary() buffer.  This means that the block at
531  * `bblock + 1' is probably a dirty indirect block.  Hunt it down and, if it's
532  * dirty, schedule it for IO.  So that indirects merge nicely with their data.
533  */
534 void write_boundary_block(struct block_device *bdev,
535                         sector_t bblock, unsigned blocksize)
536 {
537         struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
538         if (bh) {
539                 if (buffer_dirty(bh))
540                         ll_rw_block(REQ_OP_WRITE, 0, 1, &bh);
541                 put_bh(bh);
542         }
543 }
544 
545 void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
546 {
547         struct address_space *mapping = inode->i_mapping;
548         struct address_space *buffer_mapping = bh->b_page->mapping;
549 
550         mark_buffer_dirty(bh);
551         if (!mapping->private_data) {
552                 mapping->private_data = buffer_mapping;
553         } else {
554                 BUG_ON(mapping->private_data != buffer_mapping);
555         }
556         if (!bh->b_assoc_map) {
557                 spin_lock(&buffer_mapping->private_lock);
558                 list_move_tail(&bh->b_assoc_buffers,
559                                 &mapping->private_list);
560                 bh->b_assoc_map = mapping;
561                 spin_unlock(&buffer_mapping->private_lock);
562         }
563 }
564 EXPORT_SYMBOL(mark_buffer_dirty_inode);
565 
566 /*
567  * Mark the page dirty, and set it dirty in the page cache, and mark the inode
568  * dirty.
569  *
570  * If warn is true, then emit a warning if the page is not uptodate and has
571  * not been truncated.
572  *
573  * The caller must hold lock_page_memcg().
574  */
575 void __set_page_dirty(struct page *page, struct address_space *mapping,
576                              int warn)
577 {
578         unsigned long flags;
579 
580         xa_lock_irqsave(&mapping->i_pages, flags);
581         if (page->mapping) {    /* Race with truncate? */
582                 WARN_ON_ONCE(warn && !PageUptodate(page));
583                 account_page_dirtied(page, mapping);
584                 __xa_set_mark(&mapping->i_pages, page_index(page),
585                                 PAGECACHE_TAG_DIRTY);
586         }
587         xa_unlock_irqrestore(&mapping->i_pages, flags);
588 }
589 EXPORT_SYMBOL_GPL(__set_page_dirty);
590 
591 /*
592  * Add a page to the dirty page list.
593  *
594  * It is a sad fact of life that this function is called from several places
595  * deeply under spinlocking.  It may not sleep.
596  *
597  * If the page has buffers, the uptodate buffers are set dirty, to preserve
598  * dirty-state coherency between the page and the buffers.  It the page does
599  * not have buffers then when they are later attached they will all be set
600  * dirty.
601  *
602  * The buffers are dirtied before the page is dirtied.  There's a small race
603  * window in which a writepage caller may see the page cleanness but not the
604  * buffer dirtiness.  That's fine.  If this code were to set the page dirty
605  * before the buffers, a concurrent writepage caller could clear the page dirty
606  * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
607  * page on the dirty page list.
608  *
609  * We use private_lock to lock against try_to_free_buffers while using the
610  * page's buffer list.  Also use this to protect against clean buffers being
611  * added to the page after it was set dirty.
612  *
613  * FIXME: may need to call ->reservepage here as well.  That's rather up to the
614  * address_space though.
615  */
616 int __set_page_dirty_buffers(struct page *page)
617 {
618         int newly_dirty;
619         struct address_space *mapping = page_mapping(page);
620 
621         if (unlikely(!mapping))
622                 return !TestSetPageDirty(page);
623 
624         spin_lock(&mapping->private_lock);
625         if (page_has_buffers(page)) {
626                 struct buffer_head *head = page_buffers(page);
627                 struct buffer_head *bh = head;
628 
629                 do {
630                         set_buffer_dirty(bh);
631                         bh = bh->b_this_page;
632                 } while (bh != head);
633         }
634         /*
635          * Lock out page->mem_cgroup migration to keep PageDirty
636          * synchronized with per-memcg dirty page counters.
637          */
638         lock_page_memcg(page);
639         newly_dirty = !TestSetPageDirty(page);
640         spin_unlock(&mapping->private_lock);
641 
642         if (newly_dirty)
643                 __set_page_dirty(page, mapping, 1);
644 
645         unlock_page_memcg(page);
646 
647         if (newly_dirty)
648                 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
649 
650         return newly_dirty;
651 }
652 EXPORT_SYMBOL(__set_page_dirty_buffers);
653 
654 /*
655  * Write out and wait upon a list of buffers.
656  *
657  * We have conflicting pressures: we want to make sure that all
658  * initially dirty buffers get waited on, but that any subsequently
659  * dirtied buffers don't.  After all, we don't want fsync to last
660  * forever if somebody is actively writing to the file.
661  *
662  * Do this in two main stages: first we copy dirty buffers to a
663  * temporary inode list, queueing the writes as we go.  Then we clean
664  * up, waiting for those writes to complete.
665  * 
666  * During this second stage, any subsequent updates to the file may end
667  * up refiling the buffer on the original inode's dirty list again, so
668  * there is a chance we will end up with a buffer queued for write but
669  * not yet completed on that list.  So, as a final cleanup we go through
670  * the osync code to catch these locked, dirty buffers without requeuing
671  * any newly dirty buffers for write.
672  */
673 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
674 {
675         struct buffer_head *bh;
676         struct list_head tmp;
677         struct address_space *mapping;
678         int err = 0, err2;
679         struct blk_plug plug;
680 
681         INIT_LIST_HEAD(&tmp);
682         blk_start_plug(&plug);
683 
684         spin_lock(lock);
685         while (!list_empty(list)) {
686                 bh = BH_ENTRY(list->next);
687                 mapping = bh->b_assoc_map;
688                 __remove_assoc_queue(bh);
689                 /* Avoid race with mark_buffer_dirty_inode() which does
690                  * a lockless check and we rely on seeing the dirty bit */
691                 smp_mb();
692                 if (buffer_dirty(bh) || buffer_locked(bh)) {
693                         list_add(&bh->b_assoc_buffers, &tmp);
694                         bh->b_assoc_map = mapping;
695                         if (buffer_dirty(bh)) {
696                                 get_bh(bh);
697                                 spin_unlock(lock);
698                                 /*
699                                  * Ensure any pending I/O completes so that
700                                  * write_dirty_buffer() actually writes the
701                                  * current contents - it is a noop if I/O is
702                                  * still in flight on potentially older
703                                  * contents.
704                                  */
705                                 write_dirty_buffer(bh, REQ_SYNC);
706 
707                                 /*
708                                  * Kick off IO for the previous mapping. Note
709                                  * that we will not run the very last mapping,
710                                  * wait_on_buffer() will do that for us
711                                  * through sync_buffer().
712                                  */
713                                 brelse(bh);
714                                 spin_lock(lock);
715                         }
716                 }
717         }
718 
719         spin_unlock(lock);
720         blk_finish_plug(&plug);
721         spin_lock(lock);
722 
723         while (!list_empty(&tmp)) {
724                 bh = BH_ENTRY(tmp.prev);
725                 get_bh(bh);
726                 mapping = bh->b_assoc_map;
727                 __remove_assoc_queue(bh);
728                 /* Avoid race with mark_buffer_dirty_inode() which does
729                  * a lockless check and we rely on seeing the dirty bit */
730                 smp_mb();
731                 if (buffer_dirty(bh)) {
732                         list_add(&bh->b_assoc_buffers,
733                                  &mapping->private_list);
734                         bh->b_assoc_map = mapping;
735                 }
736                 spin_unlock(lock);
737                 wait_on_buffer(bh);
738                 if (!buffer_uptodate(bh))
739                         err = -EIO;
740                 brelse(bh);
741                 spin_lock(lock);
742         }
743         
744         spin_unlock(lock);
745         err2 = osync_buffers_list(lock, list);
746         if (err)
747                 return err;
748         else
749                 return err2;
750 }
751 
752 /*
753  * Invalidate any and all dirty buffers on a given inode.  We are
754  * probably unmounting the fs, but that doesn't mean we have already
755  * done a sync().  Just drop the buffers from the inode list.
756  *
757  * NOTE: we take the inode's blockdev's mapping's private_lock.  Which
758  * assumes that all the buffers are against the blockdev.  Not true
759  * for reiserfs.
760  */
761 void invalidate_inode_buffers(struct inode *inode)
762 {
763         if (inode_has_buffers(inode)) {
764                 struct address_space *mapping = &inode->i_data;
765                 struct list_head *list = &mapping->private_list;
766                 struct address_space *buffer_mapping = mapping->private_data;
767 
768                 spin_lock(&buffer_mapping->private_lock);
769                 while (!list_empty(list))
770                         __remove_assoc_queue(BH_ENTRY(list->next));
771                 spin_unlock(&buffer_mapping->private_lock);
772         }
773 }
774 EXPORT_SYMBOL(invalidate_inode_buffers);
775 
776 /*
777  * Remove any clean buffers from the inode's buffer list.  This is called
778  * when we're trying to free the inode itself.  Those buffers can pin it.
779  *
780  * Returns true if all buffers were removed.
781  */
782 int remove_inode_buffers(struct inode *inode)
783 {
784         int ret = 1;
785 
786         if (inode_has_buffers(inode)) {
787                 struct address_space *mapping = &inode->i_data;
788                 struct list_head *list = &mapping->private_list;
789                 struct address_space *buffer_mapping = mapping->private_data;
790 
791                 spin_lock(&buffer_mapping->private_lock);
792                 while (!list_empty(list)) {
793                         struct buffer_head *bh = BH_ENTRY(list->next);
794                         if (buffer_dirty(bh)) {
795                                 ret = 0;
796                                 break;
797                         }
798                         __remove_assoc_queue(bh);
799                 }
800                 spin_unlock(&buffer_mapping->private_lock);
801         }
802         return ret;
803 }
804 
805 /*
806  * Create the appropriate buffers when given a page for data area and
807  * the size of each buffer.. Use the bh->b_this_page linked list to
808  * follow the buffers created.  Return NULL if unable to create more
809  * buffers.
810  *
811  * The retry flag is used to differentiate async IO (paging, swapping)
812  * which may not fail from ordinary buffer allocations.
813  */
814 struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
815                 bool retry)
816 {
817         struct buffer_head *bh, *head;
818         gfp_t gfp = GFP_NOFS | __GFP_ACCOUNT;
819         long offset;
820         struct mem_cgroup *memcg;
821 
822         if (retry)
823                 gfp |= __GFP_NOFAIL;
824 
825         memcg = get_mem_cgroup_from_page(page);
826         memalloc_use_memcg(memcg);
827 
828         head = NULL;
829         offset = PAGE_SIZE;
830         while ((offset -= size) >= 0) {
831                 bh = alloc_buffer_head(gfp);
832                 if (!bh)
833                         goto no_grow;
834 
835                 bh->b_this_page = head;
836                 bh->b_blocknr = -1;
837                 head = bh;
838 
839                 bh->b_size = size;
840 
841                 /* Link the buffer to its page */
842                 set_bh_page(bh, page, offset);
843         }
844 out:
845         memalloc_unuse_memcg();
846         mem_cgroup_put(memcg);
847         return head;
848 /*
849  * In case anything failed, we just free everything we got.
850  */
851 no_grow:
852         if (head) {
853                 do {
854                         bh = head;
855                         head = head->b_this_page;
856                         free_buffer_head(bh);
857                 } while (head);
858         }
859 
860         goto out;
861 }
862 EXPORT_SYMBOL_GPL(alloc_page_buffers);
863 
864 static inline void
865 link_dev_buffers(struct page *page, struct buffer_head *head)
866 {
867         struct buffer_head *bh, *tail;
868 
869         bh = head;
870         do {
871                 tail = bh;
872                 bh = bh->b_this_page;
873         } while (bh);
874         tail->b_this_page = head;
875         attach_page_buffers(page, head);
876 }
877 
878 static sector_t blkdev_max_block(struct block_device *bdev, unsigned int size)
879 {
880         sector_t retval = ~((sector_t)0);
881         loff_t sz = i_size_read(bdev->bd_inode);
882 
883         if (sz) {
884                 unsigned int sizebits = blksize_bits(size);
885                 retval = (sz >> sizebits);
886         }
887         return retval;
888 }
889 
890 /*
891  * Initialise the state of a blockdev page's buffers.
892  */ 
893 static sector_t
894 init_page_buffers(struct page *page, struct block_device *bdev,
895                         sector_t block, int size)
896 {
897         struct buffer_head *head = page_buffers(page);
898         struct buffer_head *bh = head;
899         int uptodate = PageUptodate(page);
900         sector_t end_block = blkdev_max_block(I_BDEV(bdev->bd_inode), size);
901 
902         do {
903                 if (!buffer_mapped(bh)) {
904                         bh->b_end_io = NULL;
905                         bh->b_private = NULL;
906                         bh->b_bdev = bdev;
907                         bh->b_blocknr = block;
908                         if (uptodate)
909                                 set_buffer_uptodate(bh);
910                         if (block < end_block)
911                                 set_buffer_mapped(bh);
912                 }
913                 block++;
914                 bh = bh->b_this_page;
915         } while (bh != head);
916 
917         /*
918          * Caller needs to validate requested block against end of device.
919          */
920         return end_block;
921 }
922 
923 /*
924  * Create the page-cache page that contains the requested block.
925  *
926  * This is used purely for blockdev mappings.
927  */
928 static int
929 grow_dev_page(struct block_device *bdev, sector_t block,
930               pgoff_t index, int size, int sizebits, gfp_t gfp)
931 {
932         struct inode *inode = bdev->bd_inode;
933         struct page *page;
934         struct buffer_head *bh;
935         sector_t end_block;
936         int ret = 0;            /* Will call free_more_memory() */
937         gfp_t gfp_mask;
938 
939         gfp_mask = mapping_gfp_constraint(inode->i_mapping, ~__GFP_FS) | gfp;
940 
941         /*
942          * XXX: __getblk_slow() can not really deal with failure and
943          * will endlessly loop on improvised global reclaim.  Prefer
944          * looping in the allocator rather than here, at least that
945          * code knows what it's doing.
946          */
947         gfp_mask |= __GFP_NOFAIL;
948 
949         page = find_or_create_page(inode->i_mapping, index, gfp_mask);
950 
951         BUG_ON(!PageLocked(page));
952 
953         if (page_has_buffers(page)) {
954                 bh = page_buffers(page);
955                 if (bh->b_size == size) {
956                         end_block = init_page_buffers(page, bdev,
957                                                 (sector_t)index << sizebits,
958                                                 size);
959                         goto done;
960                 }
961                 if (!try_to_free_buffers(page))
962                         goto failed;
963         }
964 
965         /*
966          * Allocate some buffers for this page
967          */
968         bh = alloc_page_buffers(page, size, true);
969 
970         /*
971          * Link the page to the buffers and initialise them.  Take the
972          * lock to be atomic wrt __find_get_block(), which does not
973          * run under the page lock.
974          */
975         spin_lock(&inode->i_mapping->private_lock);
976         link_dev_buffers(page, bh);
977         end_block = init_page_buffers(page, bdev, (sector_t)index << sizebits,
978                         size);
979         spin_unlock(&inode->i_mapping->private_lock);
980 done:
981         ret = (block < end_block) ? 1 : -ENXIO;
982 failed:
983         unlock_page(page);
984         put_page(page);
985         return ret;
986 }
987 
988 /*
989  * Create buffers for the specified block device block's page.  If
990  * that page was dirty, the buffers are set dirty also.
991  */
992 static int
993 grow_buffers(struct block_device *bdev, sector_t block, int size, gfp_t gfp)
994 {
995         pgoff_t index;
996         int sizebits;
997 
998         sizebits = -1;
999         do {
1000                 sizebits++;
1001         } while ((size << sizebits) < PAGE_SIZE);
1002 
1003         index = block >> sizebits;
1004 
1005         /*
1006          * Check for a block which wants to lie outside our maximum possible
1007          * pagecache index.  (this comparison is done using sector_t types).
1008          */
1009         if (unlikely(index != block >> sizebits)) {
1010                 printk(KERN_ERR "%s: requested out-of-range block %llu for "
1011                         "device %pg\n",
1012                         __func__, (unsigned long long)block,
1013                         bdev);
1014                 return -EIO;
1015         }
1016 
1017         /* Create a page with the proper size buffers.. */
1018         return grow_dev_page(bdev, block, index, size, sizebits, gfp);
1019 }
1020 
1021 static struct buffer_head *
1022 __getblk_slow(struct block_device *bdev, sector_t block,
1023              unsigned size, gfp_t gfp)
1024 {
1025         /* Size must be multiple of hard sectorsize */
1026         if (unlikely(size & (bdev_logical_block_size(bdev)-1) ||
1027                         (size < 512 || size > PAGE_SIZE))) {
1028                 printk(KERN_ERR "getblk(): invalid block size %d requested\n",
1029                                         size);
1030                 printk(KERN_ERR "logical block size: %d\n",
1031                                         bdev_logical_block_size(bdev));
1032 
1033                 dump_stack();
1034                 return NULL;
1035         }
1036 
1037         for (;;) {
1038                 struct buffer_head *bh;
1039                 int ret;
1040 
1041                 bh = __find_get_block(bdev, block, size);
1042                 if (bh)
1043                         return bh;
1044 
1045                 ret = grow_buffers(bdev, block, size, gfp);
1046                 if (ret < 0)
1047                         return NULL;
1048         }
1049 }
1050 
1051 /*
1052  * The relationship between dirty buffers and dirty pages:
1053  *
1054  * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1055  * the page is tagged dirty in the page cache.
1056  *
1057  * At all times, the dirtiness of the buffers represents the dirtiness of
1058  * subsections of the page.  If the page has buffers, the page dirty bit is
1059  * merely a hint about the true dirty state.
1060  *
1061  * When a page is set dirty in its entirety, all its buffers are marked dirty
1062  * (if the page has buffers).
1063  *
1064  * When a buffer is marked dirty, its page is dirtied, but the page's other
1065  * buffers are not.
1066  *
1067  * Also.  When blockdev buffers are explicitly read with bread(), they
1068  * individually become uptodate.  But their backing page remains not
1069  * uptodate - even if all of its buffers are uptodate.  A subsequent
1070  * block_read_full_page() against that page will discover all the uptodate
1071  * buffers, will set the page uptodate and will perform no I/O.
1072  */
1073 
1074 /**
1075  * mark_buffer_dirty - mark a buffer_head as needing writeout
1076  * @bh: the buffer_head to mark dirty
1077  *
1078  * mark_buffer_dirty() will set the dirty bit against the buffer, then set
1079  * its backing page dirty, then tag the page as dirty in the page cache
1080  * and then attach the address_space's inode to its superblock's dirty
1081  * inode list.
1082  *
1083  * mark_buffer_dirty() is atomic.  It takes bh->b_page->mapping->private_lock,
1084  * i_pages lock and mapping->host->i_lock.
1085  */
1086 void mark_buffer_dirty(struct buffer_head *bh)
1087 {
1088         WARN_ON_ONCE(!buffer_uptodate(bh));
1089 
1090         trace_block_dirty_buffer(bh);
1091 
1092         /*
1093          * Very *carefully* optimize the it-is-already-dirty case.
1094          *
1095          * Don't let the final "is it dirty" escape to before we
1096          * perhaps modified the buffer.
1097          */
1098         if (buffer_dirty(bh)) {
1099                 smp_mb();
1100                 if (buffer_dirty(bh))
1101                         return;
1102         }
1103 
1104         if (!test_set_buffer_dirty(bh)) {
1105                 struct page *page = bh->b_page;
1106                 struct address_space *mapping = NULL;
1107 
1108                 lock_page_memcg(page);
1109                 if (!TestSetPageDirty(page)) {
1110                         mapping = page_mapping(page);
1111                         if (mapping)
1112                                 __set_page_dirty(page, mapping, 0);
1113                 }
1114                 unlock_page_memcg(page);
1115                 if (mapping)
1116                         __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
1117         }
1118 }
1119 EXPORT_SYMBOL(mark_buffer_dirty);
1120 
1121 void mark_buffer_write_io_error(struct buffer_head *bh)
1122 {
1123         set_buffer_write_io_error(bh);
1124         /* FIXME: do we need to set this in both places? */
1125         if (bh->b_page && bh->b_page->mapping)
1126                 mapping_set_error(bh->b_page->mapping, -EIO);
1127         if (bh->b_assoc_map)
1128                 mapping_set_error(bh->b_assoc_map, -EIO);
1129 }
1130 EXPORT_SYMBOL(mark_buffer_write_io_error);
1131 
1132 /*
1133  * Decrement a buffer_head's reference count.  If all buffers against a page
1134  * have zero reference count, are clean and unlocked, and if the page is clean
1135  * and unlocked then try_to_free_buffers() may strip the buffers from the page
1136  * in preparation for freeing it (sometimes, rarely, buffers are removed from
1137  * a page but it ends up not being freed, and buffers may later be reattached).
1138  */
1139 void __brelse(struct buffer_head * buf)
1140 {
1141         if (atomic_read(&buf->b_count)) {
1142                 put_bh(buf);
1143                 return;
1144         }
1145         WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n");
1146 }
1147 EXPORT_SYMBOL(__brelse);
1148 
1149 /*
1150  * bforget() is like brelse(), except it discards any
1151  * potentially dirty data.
1152  */
1153 void __bforget(struct buffer_head *bh)
1154 {
1155         clear_buffer_dirty(bh);
1156         if (bh->b_assoc_map) {
1157                 struct address_space *buffer_mapping = bh->b_page->mapping;
1158 
1159                 spin_lock(&buffer_mapping->private_lock);
1160                 list_del_init(&bh->b_assoc_buffers);
1161                 bh->b_assoc_map = NULL;
1162                 spin_unlock(&buffer_mapping->private_lock);
1163         }
1164         __brelse(bh);
1165 }
1166 EXPORT_SYMBOL(__bforget);
1167 
1168 static struct buffer_head *__bread_slow(struct buffer_head *bh)
1169 {
1170         lock_buffer(bh);
1171         if (buffer_uptodate(bh)) {
1172                 unlock_buffer(bh);
1173                 return bh;
1174         } else {
1175                 get_bh(bh);
1176                 bh->b_end_io = end_buffer_read_sync;
1177                 submit_bh(REQ_OP_READ, 0, bh);
1178                 wait_on_buffer(bh);
1179                 if (buffer_uptodate(bh))
1180                         return bh;
1181         }
1182         brelse(bh);
1183         return NULL;
1184 }
1185 
1186 /*
1187  * Per-cpu buffer LRU implementation.  To reduce the cost of __find_get_block().
1188  * The bhs[] array is sorted - newest buffer is at bhs[0].  Buffers have their
1189  * refcount elevated by one when they're in an LRU.  A buffer can only appear
1190  * once in a particular CPU's LRU.  A single buffer can be present in multiple
1191  * CPU's LRUs at the same time.
1192  *
1193  * This is a transparent caching front-end to sb_bread(), sb_getblk() and
1194  * sb_find_get_block().
1195  *
1196  * The LRUs themselves only need locking against invalidate_bh_lrus.  We use
1197  * a local interrupt disable for that.
1198  */
1199 
1200 #define BH_LRU_SIZE     16
1201 
1202 struct bh_lru {
1203         struct buffer_head *bhs[BH_LRU_SIZE];
1204 };
1205 
1206 static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
1207 
1208 #ifdef CONFIG_SMP
1209 #define bh_lru_lock()   local_irq_disable()
1210 #define bh_lru_unlock() local_irq_enable()
1211 #else
1212 #define bh_lru_lock()   preempt_disable()
1213 #define bh_lru_unlock() preempt_enable()
1214 #endif
1215 
1216 static inline void check_irqs_on(void)
1217 {
1218 #ifdef irqs_disabled
1219         BUG_ON(irqs_disabled());
1220 #endif
1221 }
1222 
1223 /*
1224  * Install a buffer_head into this cpu's LRU.  If not already in the LRU, it is
1225  * inserted at the front, and the buffer_head at the back if any is evicted.
1226  * Or, if already in the LRU it is moved to the front.
1227  */
1228 static void bh_lru_install(struct buffer_head *bh)
1229 {
1230         struct buffer_head *evictee = bh;
1231         struct bh_lru *b;
1232         int i;
1233 
1234         check_irqs_on();
1235         bh_lru_lock();
1236 
1237         b = this_cpu_ptr(&bh_lrus);
1238         for (i = 0; i < BH_LRU_SIZE; i++) {
1239                 swap(evictee, b->bhs[i]);
1240                 if (evictee == bh) {
1241                         bh_lru_unlock();
1242                         return;
1243                 }
1244         }
1245 
1246         get_bh(bh);
1247         bh_lru_unlock();
1248         brelse(evictee);
1249 }
1250 
1251 /*
1252  * Look up the bh in this cpu's LRU.  If it's there, move it to the head.
1253  */
1254 static struct buffer_head *
1255 lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size)
1256 {
1257         struct buffer_head *ret = NULL;
1258         unsigned int i;
1259 
1260         check_irqs_on();
1261         bh_lru_lock();
1262         for (i = 0; i < BH_LRU_SIZE; i++) {
1263                 struct buffer_head *bh = __this_cpu_read(bh_lrus.bhs[i]);
1264 
1265                 if (bh && bh->b_blocknr == block && bh->b_bdev == bdev &&
1266                     bh->b_size == size) {
1267                         if (i) {
1268                                 while (i) {
1269                                         __this_cpu_write(bh_lrus.bhs[i],
1270                                                 __this_cpu_read(bh_lrus.bhs[i - 1]));
1271                                         i--;
1272                                 }
1273                                 __this_cpu_write(bh_lrus.bhs[0], bh);
1274                         }
1275                         get_bh(bh);
1276                         ret = bh;
1277                         break;
1278                 }
1279         }
1280         bh_lru_unlock();
1281         return ret;
1282 }
1283 
1284 /*
1285  * Perform a pagecache lookup for the matching buffer.  If it's there, refresh
1286  * it in the LRU and mark it as accessed.  If it is not present then return
1287  * NULL
1288  */
1289 struct buffer_head *
1290 __find_get_block(struct block_device *bdev, sector_t block, unsigned size)
1291 {
1292         struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
1293 
1294         if (bh == NULL) {
1295                 /* __find_get_block_slow will mark the page accessed */
1296                 bh = __find_get_block_slow(bdev, block);
1297                 if (bh)
1298                         bh_lru_install(bh);
1299         } else
1300                 touch_buffer(bh);
1301 
1302         return bh;
1303 }
1304 EXPORT_SYMBOL(__find_get_block);
1305 
1306 /*
1307  * __getblk_gfp() will locate (and, if necessary, create) the buffer_head
1308  * which corresponds to the passed block_device, block and size. The
1309  * returned buffer has its reference count incremented.
1310  *
1311  * __getblk_gfp() will lock up the machine if grow_dev_page's
1312  * try_to_free_buffers() attempt is failing.  FIXME, perhaps?
1313  */
1314 struct buffer_head *
1315 __getblk_gfp(struct block_device *bdev, sector_t block,
1316              unsigned size, gfp_t gfp)
1317 {
1318         struct buffer_head *bh = __find_get_block(bdev, block, size);
1319 
1320         might_sleep();
1321         if (bh == NULL)
1322                 bh = __getblk_slow(bdev, block, size, gfp);
1323         return bh;
1324 }
1325 EXPORT_SYMBOL(__getblk_gfp);
1326 
1327 /*
1328  * Do async read-ahead on a buffer..
1329  */
1330 void __breadahead(struct block_device *bdev, sector_t block, unsigned size)
1331 {
1332         struct buffer_head *bh = __getblk(bdev, block, size);
1333         if (likely(bh)) {
1334                 ll_rw_block(REQ_OP_READ, REQ_RAHEAD, 1, &bh);
1335                 brelse(bh);
1336         }
1337 }
1338 EXPORT_SYMBOL(__breadahead);
1339 
1340 /**
1341  *  __bread_gfp() - reads a specified block and returns the bh
1342  *  @bdev: the block_device to read from
1343  *  @block: number of block
1344  *  @size: size (in bytes) to read
1345  *  @gfp: page allocation flag
1346  *
1347  *  Reads a specified block, and returns buffer head that contains it.
1348  *  The page cache can be allocated from non-movable area
1349  *  not to prevent page migration if you set gfp to zero.
1350  *  It returns NULL if the block was unreadable.
1351  */
1352 struct buffer_head *
1353 __bread_gfp(struct block_device *bdev, sector_t block,
1354                    unsigned size, gfp_t gfp)
1355 {
1356         struct buffer_head *bh = __getblk_gfp(bdev, block, size, gfp);
1357 
1358         if (likely(bh) && !buffer_uptodate(bh))
1359                 bh = __bread_slow(bh);
1360         return bh;
1361 }
1362 EXPORT_SYMBOL(__bread_gfp);
1363 
1364 /*
1365  * invalidate_bh_lrus() is called rarely - but not only at unmount.
1366  * This doesn't race because it runs in each cpu either in irq
1367  * or with preempt disabled.
1368  */
1369 static void invalidate_bh_lru(void *arg)
1370 {
1371         struct bh_lru *b = &get_cpu_var(bh_lrus);
1372         int i;
1373 
1374         for (i = 0; i < BH_LRU_SIZE; i++) {
1375                 brelse(b->bhs[i]);
1376                 b->bhs[i] = NULL;
1377         }
1378         put_cpu_var(bh_lrus);
1379 }
1380 
1381 static bool has_bh_in_lru(int cpu, void *dummy)
1382 {
1383         struct bh_lru *b = per_cpu_ptr(&bh_lrus, cpu);
1384         int i;
1385         
1386         for (i = 0; i < BH_LRU_SIZE; i++) {
1387                 if (b->bhs[i])
1388                         return 1;
1389         }
1390 
1391         return 0;
1392 }
1393 
1394 void invalidate_bh_lrus(void)
1395 {
1396         on_each_cpu_cond(has_bh_in_lru, invalidate_bh_lru, NULL, 1, GFP_KERNEL);
1397 }
1398 EXPORT_SYMBOL_GPL(invalidate_bh_lrus);
1399 
1400 void set_bh_page(struct buffer_head *bh,
1401                 struct page *page, unsigned long offset)
1402 {
1403         bh->b_page = page;
1404         BUG_ON(offset >= PAGE_SIZE);
1405         if (PageHighMem(page))
1406                 /*
1407                  * This catches illegal uses and preserves the offset:
1408                  */
1409                 bh->b_data = (char *)(0 + offset);
1410         else
1411                 bh->b_data = page_address(page) + offset;
1412 }
1413 EXPORT_SYMBOL(set_bh_page);
1414 
1415 /*
1416  * Called when truncating a buffer on a page completely.
1417  */
1418 
1419 /* Bits that are cleared during an invalidate */
1420 #define BUFFER_FLAGS_DISCARD \
1421         (1 << BH_Mapped | 1 << BH_New | 1 << BH_Req | \
1422          1 << BH_Delay | 1 << BH_Unwritten)
1423 
1424 static void discard_buffer(struct buffer_head * bh)
1425 {
1426         unsigned long b_state, b_state_old;
1427 
1428         lock_buffer(bh);
1429         clear_buffer_dirty(bh);
1430         bh->b_bdev = NULL;
1431         b_state = bh->b_state;
1432         for (;;) {
1433                 b_state_old = cmpxchg(&bh->b_state, b_state,
1434                                       (b_state & ~BUFFER_FLAGS_DISCARD));
1435                 if (b_state_old == b_state)
1436                         break;
1437                 b_state = b_state_old;
1438         }
1439         unlock_buffer(bh);
1440 }
1441 
1442 /**
1443  * block_invalidatepage - invalidate part or all of a buffer-backed page
1444  *
1445  * @page: the page which is affected
1446  * @offset: start of the range to invalidate
1447  * @length: length of the range to invalidate
1448  *
1449  * block_invalidatepage() is called when all or part of the page has become
1450  * invalidated by a truncate operation.
1451  *
1452  * block_invalidatepage() does not have to release all buffers, but it must
1453  * ensure that no dirty buffer is left outside @offset and that no I/O
1454  * is underway against any of the blocks which are outside the truncation
1455  * point.  Because the caller is about to free (and possibly reuse) those
1456  * blocks on-disk.
1457  */
1458 void block_invalidatepage(struct page *page, unsigned int offset,
1459                           unsigned int length)
1460 {
1461         struct buffer_head *head, *bh, *next;
1462         unsigned int curr_off = 0;
1463         unsigned int stop = length + offset;
1464 
1465         BUG_ON(!PageLocked(page));
1466         if (!page_has_buffers(page))
1467                 goto out;
1468 
1469         /*
1470          * Check for overflow
1471          */
1472         BUG_ON(stop > PAGE_SIZE || stop < length);
1473 
1474         head = page_buffers(page);
1475         bh = head;
1476         do {
1477                 unsigned int next_off = curr_off + bh->b_size;
1478                 next = bh->b_this_page;
1479 
1480                 /*
1481                  * Are we still fully in range ?
1482                  */
1483                 if (next_off > stop)
1484                         goto out;
1485 
1486                 /*
1487                  * is this block fully invalidated?
1488                  */
1489                 if (offset <= curr_off)
1490                         discard_buffer(bh);
1491                 curr_off = next_off;
1492                 bh = next;
1493         } while (bh != head);
1494 
1495         /*
1496          * We release buffers only if the entire page is being invalidated.
1497          * The get_block cached value has been unconditionally invalidated,
1498          * so real IO is not possible anymore.
1499          */
1500         if (length == PAGE_SIZE)
1501                 try_to_release_page(page, 0);
1502 out:
1503         return;
1504 }
1505 EXPORT_SYMBOL(block_invalidatepage);
1506 
1507 
1508 /*
1509  * We attach and possibly dirty the buffers atomically wrt
1510  * __set_page_dirty_buffers() via private_lock.  try_to_free_buffers
1511  * is already excluded via the page lock.
1512  */
1513 void create_empty_buffers(struct page *page,
1514                         unsigned long blocksize, unsigned long b_state)
1515 {
1516         struct buffer_head *bh, *head, *tail;
1517 
1518         head = alloc_page_buffers(page, blocksize, true);
1519         bh = head;
1520         do {
1521                 bh->b_state |= b_state;
1522                 tail = bh;
1523                 bh = bh->b_this_page;
1524         } while (bh);
1525         tail->b_this_page = head;
1526 
1527         spin_lock(&page->mapping->private_lock);
1528         if (PageUptodate(page) || PageDirty(page)) {
1529                 bh = head;
1530                 do {
1531                         if (PageDirty(page))
1532                                 set_buffer_dirty(bh);
1533                         if (PageUptodate(page))
1534                                 set_buffer_uptodate(bh);
1535                         bh = bh->b_this_page;
1536                 } while (bh != head);
1537         }
1538         attach_page_buffers(page, head);
1539         spin_unlock(&page->mapping->private_lock);
1540 }
1541 EXPORT_SYMBOL(create_empty_buffers);
1542 
1543 /**
1544  * clean_bdev_aliases: clean a range of buffers in block device
1545  * @bdev: Block device to clean buffers in
1546  * @block: Start of a range of blocks to clean
1547  * @len: Number of blocks to clean
1548  *
1549  * We are taking a range of blocks for data and we don't want writeback of any
1550  * buffer-cache aliases starting from return from this function and until the
1551  * moment when something will explicitly mark the buffer dirty (hopefully that
1552  * will not happen until we will free that block ;-) We don't even need to mark
1553  * it not-uptodate - nobody can expect anything from a newly allocated buffer
1554  * anyway. We used to use unmap_buffer() for such invalidation, but that was
1555  * wrong. We definitely don't want to mark the alias unmapped, for example - it
1556  * would confuse anyone who might pick it with bread() afterwards...
1557  *
1558  * Also..  Note that bforget() doesn't lock the buffer.  So there can be
1559  * writeout I/O going on against recently-freed buffers.  We don't wait on that
1560  * I/O in bforget() - it's more efficient to wait on the I/O only if we really
1561  * need to.  That happens here.
1562  */
1563 void clean_bdev_aliases(struct block_device *bdev, sector_t block, sector_t len)
1564 {
1565         struct inode *bd_inode = bdev->bd_inode;
1566         struct address_space *bd_mapping = bd_inode->i_mapping;
1567         struct pagevec pvec;
1568         pgoff_t index = block >> (PAGE_SHIFT - bd_inode->i_blkbits);
1569         pgoff_t end;
1570         int i, count;
1571         struct buffer_head *bh;
1572         struct buffer_head *head;
1573 
1574         end = (block + len - 1) >> (PAGE_SHIFT - bd_inode->i_blkbits);
1575         pagevec_init(&pvec);
1576         while (pagevec_lookup_range(&pvec, bd_mapping, &index, end)) {
1577                 count = pagevec_count(&pvec);
1578                 for (i = 0; i < count; i++) {
1579                         struct page *page = pvec.pages[i];
1580 
1581                         if (!page_has_buffers(page))
1582                                 continue;
1583                         /*
1584                          * We use page lock instead of bd_mapping->private_lock
1585                          * to pin buffers here since we can afford to sleep and
1586                          * it scales better than a global spinlock lock.
1587                          */
1588                         lock_page(page);
1589                         /* Recheck when the page is locked which pins bhs */
1590                         if (!page_has_buffers(page))
1591                                 goto unlock_page;
1592                         head = page_buffers(page);
1593                         bh = head;
1594                         do {
1595                                 if (!buffer_mapped(bh) || (bh->b_blocknr < block))
1596                                         goto next;
1597                                 if (bh->b_blocknr >= block + len)
1598                                         break;
1599                                 clear_buffer_dirty(bh);
1600                                 wait_on_buffer(bh);
1601                                 clear_buffer_req(bh);
1602 next:
1603                                 bh = bh->b_this_page;
1604                         } while (bh != head);
1605 unlock_page:
1606                         unlock_page(page);
1607                 }
1608                 pagevec_release(&pvec);
1609                 cond_resched();
1610                 /* End of range already reached? */
1611                 if (index > end || !index)
1612                         break;
1613         }
1614 }
1615 EXPORT_SYMBOL(clean_bdev_aliases);
1616 
1617 /*
1618  * Size is a power-of-two in the range 512..PAGE_SIZE,
1619  * and the case we care about most is PAGE_SIZE.
1620  *
1621  * So this *could* possibly be written with those
1622  * constraints in mind (relevant mostly if some
1623  * architecture has a slow bit-scan instruction)
1624  */
1625 static inline int block_size_bits(unsigned int blocksize)
1626 {
1627         return ilog2(blocksize);
1628 }
1629 
1630 static struct buffer_head *create_page_buffers(struct page *page, struct inode *inode, unsigned int b_state)
1631 {
1632         BUG_ON(!PageLocked(page));
1633 
1634         if (!page_has_buffers(page))
1635                 create_empty_buffers(page, 1 << READ_ONCE(inode->i_blkbits),
1636                                      b_state);
1637         return page_buffers(page);
1638 }
1639 
1640 /*
1641  * NOTE! All mapped/uptodate combinations are valid:
1642  *
1643  *      Mapped  Uptodate        Meaning
1644  *
1645  *      No      No              "unknown" - must do get_block()
1646  *      No      Yes             "hole" - zero-filled
1647  *      Yes     No              "allocated" - allocated on disk, not read in
1648  *      Yes     Yes             "valid" - allocated and up-to-date in memory.
1649  *
1650  * "Dirty" is valid only with the last case (mapped+uptodate).
1651  */
1652 
1653 /*
1654  * While block_write_full_page is writing back the dirty buffers under
1655  * the page lock, whoever dirtied the buffers may decide to clean them
1656  * again at any time.  We handle that by only looking at the buffer
1657  * state inside lock_buffer().
1658  *
1659  * If block_write_full_page() is called for regular writeback
1660  * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1661  * locked buffer.   This only can happen if someone has written the buffer
1662  * directly, with submit_bh().  At the address_space level PageWriteback
1663  * prevents this contention from occurring.
1664  *
1665  * If block_write_full_page() is called with wbc->sync_mode ==
1666  * WB_SYNC_ALL, the writes are posted using REQ_SYNC; this
1667  * causes the writes to be flagged as synchronous writes.
1668  */
1669 int __block_write_full_page(struct inode *inode, struct page *page,
1670                         get_block_t *get_block, struct writeback_control *wbc,
1671                         bh_end_io_t *handler)
1672 {
1673         int err;
1674         sector_t block;
1675         sector_t last_block;
1676         struct buffer_head *bh, *head;
1677         unsigned int blocksize, bbits;
1678         int nr_underway = 0;
1679         int write_flags = wbc_to_write_flags(wbc);
1680 
1681         head = create_page_buffers(page, inode,
1682                                         (1 << BH_Dirty)|(1 << BH_Uptodate));
1683 
1684         /*
1685          * Be very careful.  We have no exclusion from __set_page_dirty_buffers
1686          * here, and the (potentially unmapped) buffers may become dirty at
1687          * any time.  If a buffer becomes dirty here after we've inspected it
1688          * then we just miss that fact, and the page stays dirty.
1689          *
1690          * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
1691          * handle that here by just cleaning them.
1692          */
1693 
1694         bh = head;
1695         blocksize = bh->b_size;
1696         bbits = block_size_bits(blocksize);
1697 
1698         block = (sector_t)page->index << (PAGE_SHIFT - bbits);
1699         last_block = (i_size_read(inode) - 1) >> bbits;
1700 
1701         /*
1702          * Get all the dirty buffers mapped to disk addresses and
1703          * handle any aliases from the underlying blockdev's mapping.
1704          */
1705         do {
1706                 if (block > last_block) {
1707                         /*
1708                          * mapped buffers outside i_size will occur, because
1709                          * this page can be outside i_size when there is a
1710                          * truncate in progress.
1711                          */
1712                         /*
1713                          * The buffer was zeroed by block_write_full_page()
1714                          */
1715                         clear_buffer_dirty(bh);
1716                         set_buffer_uptodate(bh);
1717                 } else if ((!buffer_mapped(bh) || buffer_delay(bh)) &&
1718                            buffer_dirty(bh)) {
1719                         WARN_ON(bh->b_size != blocksize);
1720                         err = get_block(inode, block, bh, 1);
1721                         if (err)
1722                                 goto recover;
1723                         clear_buffer_delay(bh);
1724                         if (buffer_new(bh)) {
1725                                 /* blockdev mappings never come here */
1726                                 clear_buffer_new(bh);
1727                                 clean_bdev_bh_alias(bh);
1728                         }
1729                 }
1730                 bh = bh->b_this_page;
1731                 block++;
1732         } while (bh != head);
1733 
1734         do {
1735                 if (!buffer_mapped(bh))
1736                         continue;
1737                 /*
1738                  * If it's a fully non-blocking write attempt and we cannot
1739                  * lock the buffer then redirty the page.  Note that this can
1740                  * potentially cause a busy-wait loop from writeback threads
1741                  * and kswapd activity, but those code paths have their own
1742                  * higher-level throttling.
1743                  */
1744                 if (wbc->sync_mode != WB_SYNC_NONE) {
1745                         lock_buffer(bh);
1746                 } else if (!trylock_buffer(bh)) {
1747                         redirty_page_for_writepage(wbc, page);
1748                         continue;
1749                 }
1750                 if (test_clear_buffer_dirty(bh)) {
1751                         mark_buffer_async_write_endio(bh, handler);
1752                 } else {
1753                         unlock_buffer(bh);
1754                 }
1755         } while ((bh = bh->b_this_page) != head);
1756 
1757         /*
1758          * The page and its buffers are protected by PageWriteback(), so we can
1759          * drop the bh refcounts early.
1760          */
1761         BUG_ON(PageWriteback(page));
1762         set_page_writeback(page);
1763 
1764         do {
1765                 struct buffer_head *next = bh->b_this_page;
1766                 if (buffer_async_write(bh)) {
1767                         submit_bh_wbc(REQ_OP_WRITE, write_flags, bh,
1768                                         inode->i_write_hint, wbc);
1769                         nr_underway++;
1770                 }
1771                 bh = next;
1772         } while (bh != head);
1773         unlock_page(page);
1774 
1775         err = 0;
1776 done:
1777         if (nr_underway == 0) {
1778                 /*
1779                  * The page was marked dirty, but the buffers were
1780                  * clean.  Someone wrote them back by hand with
1781                  * ll_rw_block/submit_bh.  A rare case.
1782                  */
1783                 end_page_writeback(page);
1784 
1785                 /*
1786                  * The page and buffer_heads can be released at any time from
1787                  * here on.
1788                  */
1789         }
1790         return err;
1791 
1792 recover:
1793         /*
1794          * ENOSPC, or some other error.  We may already have added some
1795          * blocks to the file, so we need to write these out to avoid
1796          * exposing stale data.
1797          * The page is currently locked and not marked for writeback
1798          */
1799         bh = head;
1800         /* Recovery: lock and submit the mapped buffers */
1801         do {
1802                 if (buffer_mapped(bh) && buffer_dirty(bh) &&
1803                     !buffer_delay(bh)) {
1804                         lock_buffer(bh);
1805                         mark_buffer_async_write_endio(bh, handler);
1806                 } else {
1807                         /*
1808                          * The buffer may have been set dirty during
1809                          * attachment to a dirty page.
1810                          */
1811                         clear_buffer_dirty(bh);
1812                 }
1813         } while ((bh = bh->b_this_page) != head);
1814         SetPageError(page);
1815         BUG_ON(PageWriteback(page));
1816         mapping_set_error(page->mapping, err);
1817         set_page_writeback(page);
1818         do {
1819                 struct buffer_head *next = bh->b_this_page;
1820                 if (buffer_async_write(bh)) {
1821                         clear_buffer_dirty(bh);
1822                         submit_bh_wbc(REQ_OP_WRITE, write_flags, bh,
1823                                         inode->i_write_hint, wbc);
1824                         nr_underway++;
1825                 }
1826                 bh = next;
1827         } while (bh != head);
1828         unlock_page(page);
1829         goto done;
1830 }
1831 EXPORT_SYMBOL(__block_write_full_page);
1832 
1833 /*
1834  * If a page has any new buffers, zero them out here, and mark them uptodate
1835  * and dirty so they'll be written out (in order to prevent uninitialised
1836  * block data from leaking). And clear the new bit.
1837  */
1838 void page_zero_new_buffers(struct page *page, unsigned from, unsigned to)
1839 {
1840         unsigned int block_start, block_end;
1841         struct buffer_head *head, *bh;
1842 
1843         BUG_ON(!PageLocked(page));
1844         if (!page_has_buffers(page))
1845                 return;
1846 
1847         bh = head = page_buffers(page);
1848         block_start = 0;
1849         do {
1850                 block_end = block_start + bh->b_size;
1851 
1852                 if (buffer_new(bh)) {
1853                         if (block_end > from && block_start < to) {
1854                                 if (!PageUptodate(page)) {
1855                                         unsigned start, size;
1856 
1857                                         start = max(from, block_start);
1858                                         size = min(to, block_end) - start;
1859 
1860                                         zero_user(page, start, size);
1861                                         set_buffer_uptodate(bh);
1862                                 }
1863 
1864                                 clear_buffer_new(bh);
1865                                 mark_buffer_dirty(bh);
1866                         }
1867                 }
1868 
1869                 block_start = block_end;
1870                 bh = bh->b_this_page;
1871         } while (bh != head);
1872 }
1873 EXPORT_SYMBOL(page_zero_new_buffers);
1874 
1875 static void
1876 iomap_to_bh(struct inode *inode, sector_t block, struct buffer_head *bh,
1877                 struct iomap *iomap)
1878 {
1879         loff_t offset = block << inode->i_blkbits;
1880 
1881         bh->b_bdev = iomap->bdev;
1882 
1883         /*
1884          * Block points to offset in file we need to map, iomap contains
1885          * the offset at which the map starts. If the map ends before the
1886          * current block, then do not map the buffer and let the caller
1887          * handle it.
1888          */
1889         BUG_ON(offset >= iomap->offset + iomap->length);
1890 
1891         switch (iomap->type) {
1892         case IOMAP_HOLE:
1893                 /*
1894                  * If the buffer is not up to date or beyond the current EOF,
1895                  * we need to mark it as new to ensure sub-block zeroing is
1896                  * executed if necessary.
1897                  */
1898                 if (!buffer_uptodate(bh) ||
1899                     (offset >= i_size_read(inode)))
1900                         set_buffer_new(bh);
1901                 break;
1902         case IOMAP_DELALLOC:
1903                 if (!buffer_uptodate(bh) ||
1904                     (offset >= i_size_read(inode)))
1905                         set_buffer_new(bh);
1906                 set_buffer_uptodate(bh);
1907                 set_buffer_mapped(bh);
1908                 set_buffer_delay(bh);
1909                 break;
1910         case IOMAP_UNWRITTEN:
1911                 /*
1912                  * For unwritten regions, we always need to ensure that regions
1913                  * in the block we are not writing to are zeroed. Mark the
1914                  * buffer as new to ensure this.
1915                  */
1916                 set_buffer_new(bh);
1917                 set_buffer_unwritten(bh);
1918                 /* FALLTHRU */
1919         case IOMAP_MAPPED:
1920                 if ((iomap->flags & IOMAP_F_NEW) ||
1921                     offset >= i_size_read(inode))
1922                         set_buffer_new(bh);
1923                 bh->b_blocknr = (iomap->addr + offset - iomap->offset) >>
1924                                 inode->i_blkbits;
1925                 set_buffer_mapped(bh);
1926                 break;
1927         }
1928 }
1929 
1930 int __block_write_begin_int(struct page *page, loff_t pos, unsigned len,
1931                 get_block_t *get_block, struct iomap *iomap)
1932 {
1933         unsigned from = pos & (PAGE_SIZE - 1);
1934         unsigned to = from + len;
1935         struct inode *inode = page->mapping->host;
1936         unsigned block_start, block_end;
1937         sector_t block;
1938         int err = 0;
1939         unsigned blocksize, bbits;
1940         struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
1941 
1942         BUG_ON(!PageLocked(page));
1943         BUG_ON(from > PAGE_SIZE);
1944         BUG_ON(to > PAGE_SIZE);
1945         BUG_ON(from > to);
1946 
1947         head = create_page_buffers(page, inode, 0);
1948         blocksize = head->b_size;
1949         bbits = block_size_bits(blocksize);
1950 
1951         block = (sector_t)page->index << (PAGE_SHIFT - bbits);
1952 
1953         for(bh = head, block_start = 0; bh != head || !block_start;
1954             block++, block_start=block_end, bh = bh->b_this_page) {
1955                 block_end = block_start + blocksize;
1956                 if (block_end <= from || block_start >= to) {
1957                         if (PageUptodate(page)) {
1958                                 if (!buffer_uptodate(bh))
1959                                         set_buffer_uptodate(bh);
1960                         }
1961                         continue;
1962                 }
1963                 if (buffer_new(bh))
1964                         clear_buffer_new(bh);
1965                 if (!buffer_mapped(bh)) {
1966                         WARN_ON(bh->b_size != blocksize);
1967                         if (get_block) {
1968                                 err = get_block(inode, block, bh, 1);
1969                                 if (err)
1970                                         break;
1971                         } else {
1972                                 iomap_to_bh(inode, block, bh, iomap);
1973                         }
1974 
1975                         if (buffer_new(bh)) {
1976                                 clean_bdev_bh_alias(bh);
1977                                 if (PageUptodate(page)) {
1978                                         clear_buffer_new(bh);
1979                                         set_buffer_uptodate(bh);
1980                                         mark_buffer_dirty(bh);
1981                                         continue;
1982                                 }
1983                                 if (block_end > to || block_start < from)
1984                                         zero_user_segments(page,
1985                                                 to, block_end,
1986                                                 block_start, from);
1987                                 continue;
1988                         }
1989                 }
1990                 if (PageUptodate(page)) {
1991                         if (!buffer_uptodate(bh))
1992                                 set_buffer_uptodate(bh);
1993                         continue; 
1994                 }
1995                 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
1996                     !buffer_unwritten(bh) &&
1997                      (block_start < from || block_end > to)) {
1998                         ll_rw_block(REQ_OP_READ, 0, 1, &bh);
1999                         *wait_bh++=bh;
2000                 }
2001         }
2002         /*
2003          * If we issued read requests - let them complete.
2004          */
2005         while(wait_bh > wait) {
2006                 wait_on_buffer(*--wait_bh);
2007                 if (!buffer_uptodate(*wait_bh))
2008                         err = -EIO;
2009         }
2010         if (unlikely(err))
2011                 page_zero_new_buffers(page, from, to);
2012         return err;
2013 }
2014 
2015 int __block_write_begin(struct page *page, loff_t pos, unsigned len,
2016                 get_block_t *get_block)
2017 {
2018         return __block_write_begin_int(page, pos, len, get_block, NULL);
2019 }
2020 EXPORT_SYMBOL(__block_write_begin);
2021 
2022 static int __block_commit_write(struct inode *inode, struct page *page,
2023                 unsigned from, unsigned to)
2024 {
2025         unsigned block_start, block_end;
2026         int partial = 0;
2027         unsigned blocksize;
2028         struct buffer_head *bh, *head;
2029 
2030         bh = head = page_buffers(page);
2031         blocksize = bh->b_size;
2032 
2033         block_start = 0;
2034         do {
2035                 block_end = block_start + blocksize;
2036                 if (block_end <= from || block_start >= to) {
2037                         if (!buffer_uptodate(bh))
2038                                 partial = 1;
2039                 } else {
2040                         set_buffer_uptodate(bh);
2041                         mark_buffer_dirty(bh);
2042                 }
2043                 clear_buffer_new(bh);
2044 
2045                 block_start = block_end;
2046                 bh = bh->b_this_page;
2047         } while (bh != head);
2048 
2049         /*
2050          * If this is a partial write which happened to make all buffers
2051          * uptodate then we can optimize away a bogus readpage() for
2052          * the next read(). Here we 'discover' whether the page went
2053          * uptodate as a result of this (potentially partial) write.
2054          */
2055         if (!partial)
2056                 SetPageUptodate(page);
2057         return 0;
2058 }
2059 
2060 /*
2061  * block_write_begin takes care of the basic task of block allocation and
2062  * bringing partial write blocks uptodate first.
2063  *
2064  * The filesystem needs to handle block truncation upon failure.
2065  */
2066 int block_write_begin(struct address_space *mapping, loff_t pos, unsigned len,
2067                 unsigned flags, struct page **pagep, get_block_t *get_block)
2068 {
2069         pgoff_t index = pos >> PAGE_SHIFT;
2070         struct page *page;
2071         int status;
2072 
2073         page = grab_cache_page_write_begin(mapping, index, flags);
2074         if (!page)
2075                 return -ENOMEM;
2076 
2077         status = __block_write_begin(page, pos, len, get_block);
2078         if (unlikely(status)) {
2079                 unlock_page(page);
2080                 put_page(page);
2081                 page = NULL;
2082         }
2083 
2084         *pagep = page;
2085         return status;
2086 }
2087 EXPORT_SYMBOL(block_write_begin);
2088 
2089 void __generic_write_end(struct inode *inode, loff_t pos, unsigned copied,
2090                 struct page *page)
2091 {
2092         loff_t old_size = inode->i_size;
2093         bool i_size_changed = false;
2094 
2095         /*
2096          * No need to use i_size_read() here, the i_size cannot change under us
2097          * because we hold i_rwsem.
2098          *
2099          * But it's important to update i_size while still holding page lock:
2100          * page writeout could otherwise come in and zero beyond i_size.
2101          */
2102         if (pos + copied > inode->i_size) {
2103                 i_size_write(inode, pos + copied);
2104                 i_size_changed = true;
2105         }
2106 
2107         unlock_page(page);
2108 
2109         if (old_size < pos)
2110                 pagecache_isize_extended(inode, old_size, pos);
2111         /*
2112          * Don't mark the inode dirty under page lock. First, it unnecessarily
2113          * makes the holding time of page lock longer. Second, it forces lock
2114          * ordering of page lock and transaction start for journaling
2115          * filesystems.
2116          */
2117         if (i_size_changed)
2118                 mark_inode_dirty(inode);
2119 }
2120 
2121 int block_write_end(struct file *file, struct address_space *mapping,
2122                         loff_t pos, unsigned len, unsigned copied,
2123                         struct page *page, void *fsdata)
2124 {
2125         struct inode *inode = mapping->host;
2126         unsigned start;
2127 
2128         start = pos & (PAGE_SIZE - 1);
2129 
2130         if (unlikely(copied < len)) {
2131                 /*
2132                  * The buffers that were written will now be uptodate, so we
2133                  * don't have to worry about a readpage reading them and
2134                  * overwriting a partial write. However if we have encountered
2135                  * a short write and only partially written into a buffer, it
2136                  * will not be marked uptodate, so a readpage might come in and
2137                  * destroy our partial write.
2138                  *
2139                  * Do the simplest thing, and just treat any short write to a
2140                  * non uptodate page as a zero-length write, and force the
2141                  * caller to redo the whole thing.
2142                  */
2143                 if (!PageUptodate(page))
2144                         copied = 0;
2145 
2146                 page_zero_new_buffers(page, start+copied, start+len);
2147         }
2148         flush_dcache_page(page);
2149 
2150         /* This could be a short (even 0-length) commit */
2151         __block_commit_write(inode, page, start, start+copied);
2152 
2153         return copied;
2154 }
2155 EXPORT_SYMBOL(block_write_end);
2156 
2157 int generic_write_end(struct file *file, struct address_space *mapping,
2158                         loff_t pos, unsigned len, unsigned copied,
2159                         struct page *page, void *fsdata)
2160 {
2161         copied = block_write_end(file, mapping, pos, len, copied, page, fsdata);
2162         __generic_write_end(mapping->host, pos, copied, page);
2163         put_page(page);
2164         return copied;
2165 }
2166 EXPORT_SYMBOL(generic_write_end);
2167 
2168 /*
2169  * block_is_partially_uptodate checks whether buffers within a page are
2170  * uptodate or not.
2171  *
2172  * Returns true if all buffers which correspond to a file portion
2173  * we want to read are uptodate.
2174  */
2175 int block_is_partially_uptodate(struct page *page, unsigned long from,
2176                                         unsigned long count)
2177 {
2178         unsigned block_start, block_end, blocksize;
2179         unsigned to;
2180         struct buffer_head *bh, *head;
2181         int ret = 1;
2182 
2183         if (!page_has_buffers(page))
2184                 return 0;
2185 
2186         head = page_buffers(page);
2187         blocksize = head->b_size;
2188         to = min_t(unsigned, PAGE_SIZE - from, count);
2189         to = from + to;
2190         if (from < blocksize && to > PAGE_SIZE - blocksize)
2191                 return 0;
2192 
2193         bh = head;
2194         block_start = 0;
2195         do {
2196                 block_end = block_start + blocksize;
2197                 if (block_end > from && block_start < to) {
2198                         if (!buffer_uptodate(bh)) {
2199                                 ret = 0;
2200                                 break;
2201                         }
2202                         if (block_end >= to)
2203                                 break;
2204                 }
2205                 block_start = block_end;
2206                 bh = bh->b_this_page;
2207         } while (bh != head);
2208 
2209         return ret;
2210 }
2211 EXPORT_SYMBOL(block_is_partially_uptodate);
2212 
2213 /*
2214  * Generic "read page" function for block devices that have the normal
2215  * get_block functionality. This is most of the block device filesystems.
2216  * Reads the page asynchronously --- the unlock_buffer() and
2217  * set/clear_buffer_uptodate() functions propagate buffer state into the
2218  * page struct once IO has completed.
2219  */
2220 int block_read_full_page(struct page *page, get_block_t *get_block)
2221 {
2222         struct inode *inode = page->mapping->host;
2223         sector_t iblock, lblock;
2224         struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
2225         unsigned int blocksize, bbits;
2226         int nr, i;
2227         int fully_mapped = 1;
2228 
2229         head = create_page_buffers(page, inode, 0);
2230         blocksize = head->b_size;
2231         bbits = block_size_bits(blocksize);
2232 
2233         iblock = (sector_t)page->index << (PAGE_SHIFT - bbits);
2234         lblock = (i_size_read(inode)+blocksize-1) >> bbits;
2235         bh = head;
2236         nr = 0;
2237         i = 0;
2238 
2239         do {
2240                 if (buffer_uptodate(bh))
2241                         continue;
2242 
2243                 if (!buffer_mapped(bh)) {
2244                         int err = 0;
2245 
2246                         fully_mapped = 0;
2247                         if (iblock < lblock) {
2248                                 WARN_ON(bh->b_size != blocksize);
2249                                 err = get_block(inode, iblock, bh, 0);
2250                                 if (err)
2251                                         SetPageError(page);
2252                         }
2253                         if (!buffer_mapped(bh)) {
2254                                 zero_user(page, i * blocksize, blocksize);
2255                                 if (!err)
2256                                         set_buffer_uptodate(bh);
2257                                 continue;
2258                         }
2259                         /*
2260                          * get_block() might have updated the buffer
2261                          * synchronously
2262                          */
2263                         if (buffer_uptodate(bh))
2264                                 continue;
2265                 }
2266                 arr[nr++] = bh;
2267         } while (i++, iblock++, (bh = bh->b_this_page) != head);
2268 
2269         if (fully_mapped)
2270                 SetPageMappedToDisk(page);
2271 
2272         if (!nr) {
2273                 /*
2274                  * All buffers are uptodate - we can set the page uptodate
2275                  * as well. But not if get_block() returned an error.
2276                  */
2277                 if (!PageError(page))
2278                         SetPageUptodate(page);
2279                 unlock_page(page);
2280                 return 0;
2281         }
2282 
2283         /* Stage two: lock the buffers */
2284         for (i = 0; i < nr; i++) {
2285                 bh = arr[i];
2286                 lock_buffer(bh);
2287                 mark_buffer_async_read(bh);
2288         }
2289 
2290         /*
2291          * Stage 3: start the IO.  Check for uptodateness
2292          * inside the buffer lock in case another process reading
2293          * the underlying blockdev brought it uptodate (the sct fix).
2294          */
2295         for (i = 0; i < nr; i++) {
2296                 bh = arr[i];
2297                 if (buffer_uptodate(bh))
2298                         end_buffer_async_read(bh, 1);
2299                 else
2300                         submit_bh(REQ_OP_READ, 0, bh);
2301         }
2302         return 0;
2303 }
2304 EXPORT_SYMBOL(block_read_full_page);
2305 
2306 /* utility function for filesystems that need to do work on expanding
2307  * truncates.  Uses filesystem pagecache writes to allow the filesystem to
2308  * deal with the hole.  
2309  */
2310 int generic_cont_expand_simple(struct inode *inode, loff_t size)
2311 {
2312         struct address_space *mapping = inode->i_mapping;
2313         struct page *page;
2314         void *fsdata;
2315         int err;
2316 
2317         err = inode_newsize_ok(inode, size);
2318         if (err)
2319                 goto out;
2320 
2321         err = pagecache_write_begin(NULL, mapping, size, 0,
2322                                     AOP_FLAG_CONT_EXPAND, &page, &fsdata);
2323         if (err)
2324                 goto out;
2325 
2326         err = pagecache_write_end(NULL, mapping, size, 0, 0, page, fsdata);
2327         BUG_ON(err > 0);
2328 
2329 out:
2330         return err;
2331 }
2332 EXPORT_SYMBOL(generic_cont_expand_simple);
2333 
2334 static int cont_expand_zero(struct file *file, struct address_space *mapping,
2335                             loff_t pos, loff_t *bytes)
2336 {
2337         struct inode *inode = mapping->host;
2338         unsigned int blocksize = i_blocksize(inode);
2339         struct page *page;
2340         void *fsdata;
2341         pgoff_t index, curidx;
2342         loff_t curpos;
2343         unsigned zerofrom, offset, len;
2344         int err = 0;
2345 
2346         index = pos >> PAGE_SHIFT;
2347         offset = pos & ~PAGE_MASK;
2348 
2349         while (index > (curidx = (curpos = *bytes)>>PAGE_SHIFT)) {
2350                 zerofrom = curpos & ~PAGE_MASK;
2351                 if (zerofrom & (blocksize-1)) {
2352                         *bytes |= (blocksize-1);
2353                         (*bytes)++;
2354                 }
2355                 len = PAGE_SIZE - zerofrom;
2356 
2357                 err = pagecache_write_begin(file, mapping, curpos, len, 0,
2358                                             &page, &fsdata);
2359                 if (err)
2360                         goto out;
2361                 zero_user(page, zerofrom, len);
2362                 err = pagecache_write_end(file, mapping, curpos, len, len,
2363                                                 page, fsdata);
2364                 if (err < 0)
2365                         goto out;
2366                 BUG_ON(err != len);
2367                 err = 0;
2368 
2369                 balance_dirty_pages_ratelimited(mapping);
2370 
2371                 if (fatal_signal_pending(current)) {
2372                         err = -EINTR;
2373                         goto out;
2374                 }
2375         }
2376 
2377         /* page covers the boundary, find the boundary offset */
2378         if (index == curidx) {
2379                 zerofrom = curpos & ~PAGE_MASK;
2380                 /* if we will expand the thing last block will be filled */
2381                 if (offset <= zerofrom) {
2382                         goto out;
2383                 }
2384                 if (zerofrom & (blocksize-1)) {
2385                         *bytes |= (blocksize-1);
2386                         (*bytes)++;
2387                 }
2388                 len = offset - zerofrom;
2389 
2390                 err = pagecache_write_begin(file, mapping, curpos, len, 0,
2391                                             &page, &fsdata);
2392                 if (err)
2393                         goto out;
2394                 zero_user(page, zerofrom, len);
2395                 err = pagecache_write_end(file, mapping, curpos, len, len,
2396                                                 page, fsdata);
2397                 if (err < 0)
2398                         goto out;
2399                 BUG_ON(err != len);
2400                 err = 0;
2401         }
2402 out:
2403         return err;
2404 }
2405 
2406 /*
2407  * For moronic filesystems that do not allow holes in file.
2408  * We may have to extend the file.
2409  */
2410 int cont_write_begin(struct file *file, struct address_space *mapping,
2411                         loff_t pos, unsigned len, unsigned flags,
2412                         struct page **pagep, void **fsdata,
2413                         get_block_t *get_block, loff_t *bytes)
2414 {
2415         struct inode *inode = mapping->host;
2416         unsigned int blocksize = i_blocksize(inode);
2417         unsigned int zerofrom;
2418         int err;
2419 
2420         err = cont_expand_zero(file, mapping, pos, bytes);
2421         if (err)
2422                 return err;
2423 
2424         zerofrom = *bytes & ~PAGE_MASK;
2425         if (pos+len > *bytes && zerofrom & (blocksize-1)) {
2426                 *bytes |= (blocksize-1);
2427                 (*bytes)++;
2428         }
2429 
2430         return block_write_begin(mapping, pos, len, flags, pagep, get_block);
2431 }
2432 EXPORT_SYMBOL(cont_write_begin);
2433 
2434 int block_commit_write(struct page *page, unsigned from, unsigned to)
2435 {
2436         struct inode *inode = page->mapping->host;
2437         __block_commit_write(inode,page,from,to);
2438         return 0;
2439 }
2440 EXPORT_SYMBOL(block_commit_write);
2441 
2442 /*
2443  * block_page_mkwrite() is not allowed to change the file size as it gets
2444  * called from a page fault handler when a page is first dirtied. Hence we must
2445  * be careful to check for EOF conditions here. We set the page up correctly
2446  * for a written page which means we get ENOSPC checking when writing into
2447  * holes and correct delalloc and unwritten extent mapping on filesystems that
2448  * support these features.
2449  *
2450  * We are not allowed to take the i_mutex here so we have to play games to
2451  * protect against truncate races as the page could now be beyond EOF.  Because
2452  * truncate writes the inode size before removing pages, once we have the
2453  * page lock we can determine safely if the page is beyond EOF. If it is not
2454  * beyond EOF, then the page is guaranteed safe against truncation until we
2455  * unlock the page.
2456  *
2457  * Direct callers of this function should protect against filesystem freezing
2458  * using sb_start_pagefault() - sb_end_pagefault() functions.
2459  */
2460 int block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
2461                          get_block_t get_block)
2462 {
2463         struct page *page = vmf->page;
2464         struct inode *inode = file_inode(vma->vm_file);
2465         unsigned long end;
2466         loff_t size;
2467         int ret;
2468 
2469         lock_page(page);
2470         size = i_size_read(inode);
2471         if ((page->mapping != inode->i_mapping) ||
2472             (page_offset(page) > size)) {
2473                 /* We overload EFAULT to mean page got truncated */
2474                 ret = -EFAULT;
2475                 goto out_unlock;
2476         }
2477 
2478         /* page is wholly or partially inside EOF */
2479         if (((page->index + 1) << PAGE_SHIFT) > size)
2480                 end = size & ~PAGE_MASK;
2481         else
2482                 end = PAGE_SIZE;
2483 
2484         ret = __block_write_begin(page, 0, end, get_block);
2485         if (!ret)
2486                 ret = block_commit_write(page, 0, end);
2487 
2488         if (unlikely(ret < 0))
2489                 goto out_unlock;
2490         set_page_dirty(page);
2491         wait_for_stable_page(page);
2492         return 0;
2493 out_unlock:
2494         unlock_page(page);
2495         return ret;
2496 }
2497 EXPORT_SYMBOL(block_page_mkwrite);
2498 
2499 /*
2500  * nobh_write_begin()'s prereads are special: the buffer_heads are freed
2501  * immediately, while under the page lock.  So it needs a special end_io
2502  * handler which does not touch the bh after unlocking it.
2503  */
2504 static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
2505 {
2506         __end_buffer_read_notouch(bh, uptodate);
2507 }
2508 
2509 /*
2510  * Attach the singly-linked list of buffers created by nobh_write_begin, to
2511  * the page (converting it to circular linked list and taking care of page
2512  * dirty races).
2513  */
2514 static void attach_nobh_buffers(struct page *page, struct buffer_head *head)
2515 {
2516         struct buffer_head *bh;
2517 
2518         BUG_ON(!PageLocked(page));
2519 
2520         spin_lock(&page->mapping->private_lock);
2521         bh = head;
2522         do {
2523                 if (PageDirty(page))
2524                         set_buffer_dirty(bh);
2525                 if (!bh->b_this_page)
2526                         bh->b_this_page = head;
2527                 bh = bh->b_this_page;
2528         } while (bh != head);
2529         attach_page_buffers(page, head);
2530         spin_unlock(&page->mapping->private_lock);
2531 }
2532 
2533 /*
2534  * On entry, the page is fully not uptodate.
2535  * On exit the page is fully uptodate in the areas outside (from,to)
2536  * The filesystem needs to handle block truncation upon failure.
2537  */
2538 int nobh_write_begin(struct address_space *mapping,
2539                         loff_t pos, unsigned len, unsigned flags,
2540                         struct page **pagep, void **fsdata,
2541                         get_block_t *get_block)
2542 {
2543         struct inode *inode = mapping->host;
2544         const unsigned blkbits = inode->i_blkbits;
2545         const unsigned blocksize = 1 << blkbits;
2546         struct buffer_head *head, *bh;
2547         struct page *page;
2548         pgoff_t index;
2549         unsigned from, to;
2550         unsigned block_in_page;
2551         unsigned block_start, block_end;
2552         sector_t block_in_file;
2553         int nr_reads = 0;
2554         int ret = 0;
2555         int is_mapped_to_disk = 1;
2556 
2557         index = pos >> PAGE_SHIFT;
2558         from = pos & (PAGE_SIZE - 1);
2559         to = from + len;
2560 
2561         page = grab_cache_page_write_begin(mapping, index, flags);
2562         if (!page)
2563                 return -ENOMEM;
2564         *pagep = page;
2565         *fsdata = NULL;
2566 
2567         if (page_has_buffers(page)) {
2568                 ret = __block_write_begin(page, pos, len, get_block);
2569                 if (unlikely(ret))
2570                         goto out_release;
2571                 return ret;
2572         }
2573 
2574         if (PageMappedToDisk(page))
2575                 return 0;
2576 
2577         /*
2578          * Allocate buffers so that we can keep track of state, and potentially
2579          * attach them to the page if an error occurs. In the common case of
2580          * no error, they will just be freed again without ever being attached
2581          * to the page (which is all OK, because we're under the page lock).
2582          *
2583          * Be careful: the buffer linked list is a NULL terminated one, rather
2584          * than the circular one we're used to.
2585          */
2586         head = alloc_page_buffers(page, blocksize, false);
2587         if (!head) {
2588                 ret = -ENOMEM;
2589                 goto out_release;
2590         }
2591 
2592         block_in_file = (sector_t)page->index << (PAGE_SHIFT - blkbits);
2593 
2594         /*
2595          * We loop across all blocks in the page, whether or not they are
2596          * part of the affected region.  This is so we can discover if the
2597          * page is fully mapped-to-disk.
2598          */
2599         for (block_start = 0, block_in_page = 0, bh = head;
2600                   block_start < PAGE_SIZE;
2601                   block_in_page++, block_start += blocksize, bh = bh->b_this_page) {
2602                 int create;
2603 
2604                 block_end = block_start + blocksize;
2605                 bh->b_state = 0;
2606                 create = 1;
2607                 if (block_start >= to)
2608                         create = 0;
2609                 ret = get_block(inode, block_in_file + block_in_page,
2610                                         bh, create);
2611                 if (ret)
2612                         goto failed;
2613                 if (!buffer_mapped(bh))
2614                         is_mapped_to_disk = 0;
2615                 if (buffer_new(bh))
2616                         clean_bdev_bh_alias(bh);
2617                 if (PageUptodate(page)) {
2618                         set_buffer_uptodate(bh);
2619                         continue;
2620                 }
2621                 if (buffer_new(bh) || !buffer_mapped(bh)) {
2622                         zero_user_segments(page, block_start, from,
2623                                                         to, block_end);
2624                         continue;
2625                 }
2626                 if (buffer_uptodate(bh))
2627                         continue;       /* reiserfs does this */
2628                 if (block_start < from || block_end > to) {
2629                         lock_buffer(bh);
2630                         bh->b_end_io = end_buffer_read_nobh;
2631                         submit_bh(REQ_OP_READ, 0, bh);
2632                         nr_reads++;
2633                 }
2634         }
2635 
2636         if (nr_reads) {
2637                 /*
2638                  * The page is locked, so these buffers are protected from
2639                  * any VM or truncate activity.  Hence we don't need to care
2640                  * for the buffer_head refcounts.
2641                  */
2642                 for (bh = head; bh; bh = bh->b_this_page) {
2643                         wait_on_buffer(bh);
2644                         if (!buffer_uptodate(bh))
2645                                 ret = -EIO;
2646                 }
2647                 if (ret)
2648                         goto failed;
2649         }
2650 
2651         if (is_mapped_to_disk)
2652                 SetPageMappedToDisk(page);
2653 
2654         *fsdata = head; /* to be released by nobh_write_end */
2655 
2656         return 0;
2657 
2658 failed:
2659         BUG_ON(!ret);
2660         /*
2661          * Error recovery is a bit difficult. We need to zero out blocks that
2662          * were newly allocated, and dirty them to ensure they get written out.
2663          * Buffers need to be attached to the page at this point, otherwise
2664          * the handling of potential IO errors during writeout would be hard
2665          * (could try doing synchronous writeout, but what if that fails too?)
2666          */
2667         attach_nobh_buffers(page, head);
2668         page_zero_new_buffers(page, from, to);
2669 
2670 out_release:
2671         unlock_page(page);
2672         put_page(page);
2673         *pagep = NULL;
2674 
2675         return ret;
2676 }
2677 EXPORT_SYMBOL(nobh_write_begin);
2678 
2679 int nobh_write_end(struct file *file, struct address_space *mapping,
2680                         loff_t pos, unsigned len, unsigned copied,
2681                         struct page *page, void *fsdata)
2682 {
2683         struct inode *inode = page->mapping->host;
2684         struct buffer_head *head = fsdata;
2685         struct buffer_head *bh;
2686         BUG_ON(fsdata != NULL && page_has_buffers(page));
2687 
2688         if (unlikely(copied < len) && head)
2689                 attach_nobh_buffers(page, head);
2690         if (page_has_buffers(page))
2691                 return generic_write_end(file, mapping, pos, len,
2692                                         copied, page, fsdata);
2693 
2694         SetPageUptodate(page);
2695         set_page_dirty(page);
2696         if (pos+copied > inode->i_size) {
2697                 i_size_write(inode, pos+copied);
2698                 mark_inode_dirty(inode);
2699         }
2700 
2701         unlock_page(page);
2702         put_page(page);
2703 
2704         while (head) {
2705                 bh = head;
2706                 head = head->b_this_page;
2707                 free_buffer_head(bh);
2708         }
2709 
2710         return copied;
2711 }
2712 EXPORT_SYMBOL(nobh_write_end);
2713 
2714 /*
2715  * nobh_writepage() - based on block_full_write_page() except
2716  * that it tries to operate without attaching bufferheads to
2717  * the page.
2718  */
2719 int nobh_writepage(struct page *page, get_block_t *get_block,
2720                         struct writeback_control *wbc)
2721 {
2722         struct inode * const inode = page->mapping->host;
2723         loff_t i_size = i_size_read(inode);
2724         const pgoff_t end_index = i_size >> PAGE_SHIFT;
2725         unsigned offset;
2726         int ret;
2727 
2728         /* Is the page fully inside i_size? */
2729         if (page->index < end_index)
2730                 goto out;
2731 
2732         /* Is the page fully outside i_size? (truncate in progress) */
2733         offset = i_size & (PAGE_SIZE-1);
2734         if (page->index >= end_index+1 || !offset) {
2735                 /*
2736                  * The page may have dirty, unmapped buffers.  For example,
2737                  * they may have been added in ext3_writepage().  Make them
2738                  * freeable here, so the page does not leak.
2739                  */
2740 #if 0
2741                 /* Not really sure about this  - do we need this ? */
2742                 if (page->mapping->a_ops->invalidatepage)
2743                         page->mapping->a_ops->invalidatepage(page, offset);
2744 #endif
2745                 unlock_page(page);
2746                 return 0; /* don't care */
2747         }
2748 
2749         /*
2750          * The page straddles i_size.  It must be zeroed out on each and every
2751          * writepage invocation because it may be mmapped.  "A file is mapped
2752          * in multiples of the page size.  For a file that is not a multiple of
2753          * the  page size, the remaining memory is zeroed when mapped, and
2754          * writes to that region are not written out to the file."
2755          */
2756         zero_user_segment(page, offset, PAGE_SIZE);
2757 out:
2758         ret = mpage_writepage(page, get_block, wbc);
2759         if (ret == -EAGAIN)
2760                 ret = __block_write_full_page(inode, page, get_block, wbc,
2761                                               end_buffer_async_write);
2762         return ret;
2763 }
2764 EXPORT_SYMBOL(nobh_writepage);
2765 
2766 int nobh_truncate_page(struct address_space *mapping,
2767                         loff_t from, get_block_t *get_block)
2768 {
2769         pgoff_t index = from >> PAGE_SHIFT;
2770         unsigned offset = from & (PAGE_SIZE-1);
2771         unsigned blocksize;
2772         sector_t iblock;
2773         unsigned length, pos;
2774         struct inode *inode = mapping->host;
2775         struct page *page;
2776         struct buffer_head map_bh;
2777         int err;
2778 
2779         blocksize = i_blocksize(inode);
2780         length = offset & (blocksize - 1);
2781 
2782         /* Block boundary? Nothing to do */
2783         if (!length)
2784                 return 0;
2785 
2786         length = blocksize - length;
2787         iblock = (sector_t)index << (PAGE_SHIFT - inode->i_blkbits);
2788 
2789         page = grab_cache_page(mapping, index);
2790         err = -ENOMEM;
2791         if (!page)
2792                 goto out;
2793 
2794         if (page_has_buffers(page)) {
2795 has_buffers:
2796                 unlock_page(page);
2797                 put_page(page);
2798                 return block_truncate_page(mapping, from, get_block);
2799         }
2800 
2801         /* Find the buffer that contains "offset" */
2802         pos = blocksize;
2803         while (offset >= pos) {
2804                 iblock++;
2805                 pos += blocksize;
2806         }
2807 
2808         map_bh.b_size = blocksize;
2809         map_bh.b_state = 0;
2810         err = get_block(inode, iblock, &map_bh, 0);
2811         if (err)
2812                 goto unlock;
2813         /* unmapped? It's a hole - nothing to do */
2814         if (!buffer_mapped(&map_bh))
2815                 goto unlock;
2816 
2817         /* Ok, it's mapped. Make sure it's up-to-date */
2818         if (!PageUptodate(page)) {
2819                 err = mapping->a_ops->readpage(NULL, page);
2820                 if (err) {
2821                         put_page(page);
2822                         goto out;
2823                 }
2824                 lock_page(page);
2825                 if (!PageUptodate(page)) {
2826                         err = -EIO;
2827                         goto unlock;
2828                 }
2829                 if (page_has_buffers(page))
2830                         goto has_buffers;
2831         }
2832         zero_user(page, offset, length);
2833         set_page_dirty(page);
2834         err = 0;
2835 
2836 unlock:
2837         unlock_page(page);
2838         put_page(page);
2839 out:
2840         return err;
2841 }
2842 EXPORT_SYMBOL(nobh_truncate_page);
2843 
2844 int block_truncate_page(struct address_space *mapping,
2845                         loff_t from, get_block_t *get_block)
2846 {
2847         pgoff_t index = from >> PAGE_SHIFT;
2848         unsigned offset = from & (PAGE_SIZE-1);
2849         unsigned blocksize;
2850         sector_t iblock;
2851         unsigned length, pos;
2852         struct inode *inode = mapping->host;
2853         struct page *page;
2854         struct buffer_head *bh;
2855         int err;
2856 
2857         blocksize = i_blocksize(inode);
2858         length = offset & (blocksize - 1);
2859 
2860         /* Block boundary? Nothing to do */
2861         if (!length)
2862                 return 0;
2863 
2864         length = blocksize - length;
2865         iblock = (sector_t)index << (PAGE_SHIFT - inode->i_blkbits);
2866         
2867         page = grab_cache_page(mapping, index);
2868         err = -ENOMEM;
2869         if (!page)
2870                 goto out;
2871 
2872         if (!page_has_buffers(page))
2873                 create_empty_buffers(page, blocksize, 0);
2874 
2875         /* Find the buffer that contains "offset" */
2876         bh = page_buffers(page);
2877         pos = blocksize;
2878         while (offset >= pos) {
2879                 bh = bh->b_this_page;
2880                 iblock++;
2881                 pos += blocksize;
2882         }
2883 
2884         err = 0;
2885         if (!buffer_mapped(bh)) {
2886                 WARN_ON(bh->b_size != blocksize);
2887                 err = get_block(inode, iblock, bh, 0);
2888                 if (err)
2889                         goto unlock;
2890                 /* unmapped? It's a hole - nothing to do */
2891                 if (!buffer_mapped(bh))
2892                         goto unlock;
2893         }
2894 
2895         /* Ok, it's mapped. Make sure it's up-to-date */
2896         if (PageUptodate(page))
2897                 set_buffer_uptodate(bh);
2898 
2899         if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) {
2900                 err = -EIO;
2901                 ll_rw_block(REQ_OP_READ, 0, 1, &bh);
2902                 wait_on_buffer(bh);
2903                 /* Uhhuh. Read error. Complain and punt. */
2904                 if (!buffer_uptodate(bh))
2905                         goto unlock;
2906         }
2907 
2908         zero_user(page, offset, length);
2909         mark_buffer_dirty(bh);
2910         err = 0;
2911 
2912 unlock:
2913         unlock_page(page);
2914         put_page(page);
2915 out:
2916         return err;
2917 }
2918 EXPORT_SYMBOL(block_truncate_page);
2919 
2920 /*
2921  * The generic ->writepage function for buffer-backed address_spaces
2922  */
2923 int block_write_full_page(struct page *page, get_block_t *get_block,
2924                         struct writeback_control *wbc)
2925 {
2926         struct inode * const inode = page->mapping->host;
2927         loff_t i_size = i_size_read(inode);
2928         const pgoff_t end_index = i_size >> PAGE_SHIFT;
2929         unsigned offset;
2930 
2931         /* Is the page fully inside i_size? */
2932         if (page->index < end_index)
2933                 return __block_write_full_page(inode, page, get_block, wbc,
2934                                                end_buffer_async_write);
2935 
2936         /* Is the page fully outside i_size? (truncate in progress) */
2937         offset = i_size & (PAGE_SIZE-1);
2938         if (page->index >= end_index+1 || !offset) {
2939                 /*
2940                  * The page may have dirty, unmapped buffers.  For example,
2941                  * they may have been added in ext3_writepage().  Make them
2942                  * freeable here, so the page does not leak.
2943                  */
2944                 do_invalidatepage(page, 0, PAGE_SIZE);
2945                 unlock_page(page);
2946                 return 0; /* don't care */
2947         }
2948 
2949         /*
2950          * The page straddles i_size.  It must be zeroed out on each and every
2951          * writepage invocation because it may be mmapped.  "A file is mapped
2952          * in multiples of the page size.  For a file that is not a multiple of
2953          * the  page size, the remaining memory is zeroed when mapped, and
2954          * writes to that region are not written out to the file."
2955          */
2956         zero_user_segment(page, offset, PAGE_SIZE);
2957         return __block_write_full_page(inode, page, get_block, wbc,
2958                                                         end_buffer_async_write);
2959 }
2960 EXPORT_SYMBOL(block_write_full_page);
2961 
2962 sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
2963                             get_block_t *get_block)
2964 {
2965         struct inode *inode = mapping->host;
2966         struct buffer_head tmp = {
2967                 .b_size = i_blocksize(inode),
2968         };
2969 
2970         get_block(inode, block, &tmp, 0);
2971         return tmp.b_blocknr;
2972 }
2973 EXPORT_SYMBOL(generic_block_bmap);
2974 
2975 static void end_bio_bh_io_sync(struct bio *bio)
2976 {
2977         struct buffer_head *bh = bio->bi_private;
2978 
2979         if (unlikely(bio_flagged(bio, BIO_QUIET)))
2980                 set_bit(BH_Quiet, &bh->b_state);
2981 
2982         bh->b_end_io(bh, !bio->bi_status);
2983         bio_put(bio);
2984 }
2985 
2986 /*
2987  * This allows us to do IO even on the odd last sectors
2988  * of a device, even if the block size is some multiple
2989  * of the physical sector size.
2990  *
2991  * We'll just truncate the bio to the size of the device,
2992  * and clear the end of the buffer head manually.
2993  *
2994  * Truly out-of-range accesses will turn into actual IO
2995  * errors, this only handles the "we need to be able to
2996  * do IO at the final sector" case.
2997  */
2998 void guard_bio_eod(int op, struct bio *bio)
2999 {
3000         sector_t maxsector;
3001         struct bio_vec *bvec = bio_last_bvec_all(bio);
3002         unsigned truncated_bytes;
3003         struct hd_struct *part;
3004 
3005         rcu_read_lock();
3006         part = __disk_get_part(bio->bi_disk, bio->bi_partno);
3007         if (part)
3008                 maxsector = part_nr_sects_read(part);
3009         else
3010                 maxsector = get_capacity(bio->bi_disk);
3011         rcu_read_unlock();
3012 
3013         if (!maxsector)
3014                 return;
3015 
3016         /*
3017          * If the *whole* IO is past the end of the device,
3018          * let it through, and the IO layer will turn it into
3019          * an EIO.
3020          */
3021         if (unlikely(bio->bi_iter.bi_sector >= maxsector))
3022                 return;
3023 
3024         maxsector -= bio->bi_iter.bi_sector;
3025         if (likely((bio->bi_iter.bi_size >> 9) <= maxsector))
3026                 return;
3027 
3028         /* Uhhuh. We've got a bio that straddles the device size! */
3029         truncated_bytes = bio->bi_iter.bi_size - (maxsector << 9);
3030 
3031         /*
3032          * The bio contains more than one segment which spans EOD, just return
3033          * and let IO layer turn it into an EIO
3034          */
3035         if (truncated_bytes > bvec->bv_len)
3036                 return;
3037 
3038         /* Truncate the bio.. */
3039         bio->bi_iter.bi_size -= truncated_bytes;
3040         bvec->bv_len -= truncated_bytes;
3041 
3042         /* ..and clear the end of the buffer for reads */
3043         if (op == REQ_OP_READ) {
3044                 struct bio_vec bv;
3045 
3046                 mp_bvec_last_segment(bvec, &bv);
3047                 zero_user(bv.bv_page, bv.bv_offset + bv.bv_len,
3048                                 truncated_bytes);
3049         }
3050 }
3051 
3052 static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh,
3053                          enum rw_hint write_hint, struct writeback_control *wbc)
3054 {
3055         struct bio *bio;
3056 
3057         BUG_ON(!buffer_locked(bh));
3058         BUG_ON(!buffer_mapped(bh));
3059         BUG_ON(!bh->b_end_io);
3060         BUG_ON(buffer_delay(bh));
3061         BUG_ON(buffer_unwritten(bh));
3062 
3063         /*
3064          * Only clear out a write error when rewriting
3065          */
3066         if (test_set_buffer_req(bh) && (op == REQ_OP_WRITE))
3067                 clear_buffer_write_io_error(bh);
3068 
3069         /*
3070          * from here on down, it's all bio -- do the initial mapping,
3071          * submit_bio -> generic_make_request may further map this bio around
3072          */
3073         bio = bio_alloc(GFP_NOIO, 1);
3074 
3075         bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
3076         bio_set_dev(bio, bh->b_bdev);
3077         bio->bi_write_hint = write_hint;
3078 
3079         bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh));
3080         BUG_ON(bio->bi_iter.bi_size != bh->b_size);
3081 
3082         bio->bi_end_io = end_bio_bh_io_sync;
3083         bio->bi_private = bh;
3084 
3085         /* Take care of bh's that straddle the end of the device */
3086         guard_bio_eod(op, bio);
3087 
3088         if (buffer_meta(bh))
3089                 op_flags |= REQ_META;
3090         if (buffer_prio(bh))
3091                 op_flags |= REQ_PRIO;
3092         bio_set_op_attrs(bio, op, op_flags);
3093 
3094         if (wbc) {
3095                 wbc_init_bio(wbc, bio);
3096                 wbc_account_io(wbc, bh->b_page, bh->b_size);
3097         }
3098 
3099         submit_bio(bio);
3100         return 0;
3101 }
3102 
3103 int submit_bh(int op, int op_flags, struct buffer_head *bh)
3104 {
3105         return submit_bh_wbc(op, op_flags, bh, 0, NULL);
3106 }
3107 EXPORT_SYMBOL(submit_bh);
3108 
3109 /**
3110  * ll_rw_block: low-level access to block devices (DEPRECATED)
3111  * @op: whether to %READ or %WRITE
3112  * @op_flags: req_flag_bits
3113  * @nr: number of &struct buffer_heads in the array
3114  * @bhs: array of pointers to &struct buffer_head
3115  *
3116  * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
3117  * requests an I/O operation on them, either a %REQ_OP_READ or a %REQ_OP_WRITE.
3118  * @op_flags contains flags modifying the detailed I/O behavior, most notably
3119  * %REQ_RAHEAD.
3120  *
3121  * This function drops any buffer that it cannot get a lock on (with the
3122  * BH_Lock state bit), any buffer that appears to be clean when doing a write
3123  * request, and any buffer that appears to be up-to-date when doing read
3124  * request.  Further it marks as clean buffers that are processed for
3125  * writing (the buffer cache won't assume that they are actually clean
3126  * until the buffer gets unlocked).
3127  *
3128  * ll_rw_block sets b_end_io to simple completion handler that marks
3129  * the buffer up-to-date (if appropriate), unlocks the buffer and wakes
3130  * any waiters. 
3131  *
3132  * All of the buffers must be for the same device, and must also be a
3133  * multiple of the current approved size for the device.
3134  */
3135 void ll_rw_block(int op, int op_flags,  int nr, struct buffer_head *bhs[])
3136 {
3137         int i;
3138 
3139         for (i = 0; i < nr; i++) {
3140                 struct buffer_head *bh = bhs[i];
3141 
3142                 if (!trylock_buffer(bh))
3143                         continue;
3144                 if (op == WRITE) {
3145                         if (test_clear_buffer_dirty(bh)) {
3146                                 bh->b_end_io = end_buffer_write_sync;
3147                                 get_bh(bh);
3148                                 submit_bh(op, op_flags, bh);
3149                                 continue;
3150                         }
3151                 } else {
3152                         if (!buffer_uptodate(bh)) {
3153                                 bh->b_end_io = end_buffer_read_sync;
3154                                 get_bh(bh);
3155                                 submit_bh(op, op_flags, bh);
3156                                 continue;
3157                         }
3158                 }
3159                 unlock_buffer(bh);
3160         }
3161 }
3162 EXPORT_SYMBOL(ll_rw_block);
3163 
3164 void write_dirty_buffer(struct buffer_head *bh, int op_flags)
3165 {
3166         lock_buffer(bh);
3167         if (!test_clear_buffer_dirty(bh)) {
3168                 unlock_buffer(bh);
3169                 return;
3170         }
3171         bh->b_end_io = end_buffer_write_sync;
3172         get_bh(bh);
3173         submit_bh(REQ_OP_WRITE, op_flags, bh);
3174 }
3175 EXPORT_SYMBOL(write_dirty_buffer);
3176 
3177 /*
3178  * For a data-integrity writeout, we need to wait upon any in-progress I/O
3179  * and then start new I/O and then wait upon it.  The caller must have a ref on
3180  * the buffer_head.
3181  */
3182 int __sync_dirty_buffer(struct buffer_head *bh, int op_flags)
3183 {
3184         int ret = 0;
3185 
3186         WARN_ON(atomic_read(&bh->b_count) < 1);
3187         lock_buffer(bh);
3188         if (test_clear_buffer_dirty(bh)) {
3189                 get_bh(bh);
3190                 bh->b_end_io = end_buffer_write_sync;
3191                 ret = submit_bh(REQ_OP_WRITE, op_flags, bh);
3192                 wait_on_buffer(bh);
3193                 if (!ret && !buffer_uptodate(bh))
3194                         ret = -EIO;
3195         } else {
3196                 unlock_buffer(bh);
3197         }
3198         return ret;
3199 }
3200 EXPORT_SYMBOL(__sync_dirty_buffer);
3201 
3202 int sync_dirty_buffer(struct buffer_head *bh)
3203 {
3204         return __sync_dirty_buffer(bh, REQ_SYNC);
3205 }
3206 EXPORT_SYMBOL(sync_dirty_buffer);
3207 
3208 /*
3209  * try_to_free_buffers() checks if all the buffers on this particular page
3210  * are unused, and releases them if so.
3211  *
3212  * Exclusion against try_to_free_buffers may be obtained by either
3213  * locking the page or by holding its mapping's private_lock.
3214  *
3215  * If the page is dirty but all the buffers are clean then we need to
3216  * be sure to mark the page clean as well.  This is because the page
3217  * may be against a block device, and a later reattachment of buffers
3218  * to a dirty page will set *all* buffers dirty.  Which would corrupt
3219  * filesystem data on the same device.
3220  *
3221  * The same applies to regular filesystem pages: if all the buffers are
3222  * clean then we set the page clean and proceed.  To do that, we require
3223  * total exclusion from __set_page_dirty_buffers().  That is obtained with
3224  * private_lock.
3225  *
3226  * try_to_free_buffers() is non-blocking.
3227  */
3228 static inline int buffer_busy(struct buffer_head *bh)
3229 {
3230         return atomic_read(&bh->b_count) |
3231                 (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
3232 }
3233 
3234 static int
3235 drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
3236 {
3237         struct buffer_head *head = page_buffers(page);
3238         struct buffer_head *bh;
3239 
3240         bh = head;
3241         do {
3242                 if (buffer_busy(bh))
3243                         goto failed;
3244                 bh = bh->b_this_page;
3245         } while (bh != head);
3246 
3247         do {
3248                 struct buffer_head *next = bh->b_this_page;
3249 
3250                 if (bh->b_assoc_map)
3251                         __remove_assoc_queue(bh);
3252                 bh = next;
3253         } while (bh != head);
3254         *buffers_to_free = head;
3255         __clear_page_buffers(page);
3256         return 1;
3257 failed:
3258         return 0;
3259 }
3260 
3261 int try_to_free_buffers(struct page *page)
3262 {
3263         struct address_space * const mapping = page->mapping;
3264         struct buffer_head *buffers_to_free = NULL;
3265         int ret = 0;
3266 
3267         BUG_ON(!PageLocked(page));
3268         if (PageWriteback(page))
3269                 return 0;
3270 
3271         if (mapping == NULL) {          /* can this still happen? */
3272                 ret = drop_buffers(page, &buffers_to_free);
3273                 goto out;
3274         }
3275 
3276         spin_lock(&mapping->private_lock);
3277         ret = drop_buffers(page, &buffers_to_free);
3278 
3279         /*
3280          * If the filesystem writes its buffers by hand (eg ext3)
3281          * then we can have clean buffers against a dirty page.  We
3282          * clean the page here; otherwise the VM will never notice
3283          * that the filesystem did any IO at all.
3284          *
3285          * Also, during truncate, discard_buffer will have marked all
3286          * the page's buffers clean.  We discover that here and clean
3287          * the page also.
3288          *
3289          * private_lock must be held over this entire operation in order
3290          * to synchronise against __set_page_dirty_buffers and prevent the
3291          * dirty bit from being lost.
3292          */
3293         if (ret)
3294                 cancel_dirty_page(page);
3295         spin_unlock(&mapping->private_lock);
3296 out:
3297         if (buffers_to_free) {
3298                 struct buffer_head *bh = buffers_to_free;
3299 
3300                 do {
3301                         struct buffer_head *next = bh->b_this_page;
3302                         free_buffer_head(bh);
3303                         bh = next;
3304                 } while (bh != buffers_to_free);
3305         }
3306         return ret;
3307 }
3308 EXPORT_SYMBOL(try_to_free_buffers);
3309 
3310 /*
3311  * There are no bdflush tunables left.  But distributions are
3312  * still running obsolete flush daemons, so we terminate them here.
3313  *
3314  * Use of bdflush() is deprecated and will be removed in a future kernel.
3315  * The `flush-X' kernel threads fully replace bdflush daemons and this call.
3316  */
3317 SYSCALL_DEFINE2(bdflush, int, func, long, data)
3318 {
3319         static int msg_count;
3320 
3321         if (!capable(CAP_SYS_ADMIN))
3322                 return -EPERM;
3323 
3324         if (msg_count < 5) {
3325                 msg_count++;
3326                 printk(KERN_INFO
3327                         "warning: process `%s' used the obsolete bdflush"
3328                         " system call\n", current->comm);
3329                 printk(KERN_INFO "Fix your initscripts?\n");
3330         }
3331 
3332         if (func == 1)
3333                 do_exit(0);
3334         return 0;
3335 }
3336 
3337 /*
3338  * Buffer-head allocation
3339  */
3340 static struct kmem_cache *bh_cachep __read_mostly;
3341 
3342 /*
3343  * Once the number of bh's in the machine exceeds this level, we start
3344  * stripping them in writeback.
3345  */
3346 static unsigned long max_buffer_heads;
3347 
3348 int buffer_heads_over_limit;
3349 
3350 struct bh_accounting {
3351         int nr;                 /* Number of live bh's */
3352         int ratelimit;          /* Limit cacheline bouncing */
3353 };
3354 
3355 static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
3356 
3357 static void recalc_bh_state(void)
3358 {
3359         int i;
3360         int tot = 0;
3361 
3362         if (__this_cpu_inc_return(bh_accounting.ratelimit) - 1 < 4096)
3363                 return;
3364         __this_cpu_write(bh_accounting.ratelimit, 0);
3365         for_each_online_cpu(i)
3366                 tot += per_cpu(bh_accounting, i).nr;
3367         buffer_heads_over_limit = (tot > max_buffer_heads);
3368 }
3369 
3370 struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
3371 {
3372         struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags);
3373         if (ret) {
3374                 INIT_LIST_HEAD(&ret->b_assoc_buffers);
3375                 preempt_disable();
3376                 __this_cpu_inc(bh_accounting.nr);
3377                 recalc_bh_state();
3378                 preempt_enable();
3379         }
3380         return ret;
3381 }
3382 EXPORT_SYMBOL(alloc_buffer_head);
3383 
3384 void free_buffer_head(struct buffer_head *bh)
3385 {
3386         BUG_ON(!list_empty(&bh->b_assoc_buffers));
3387         kmem_cache_free(bh_cachep, bh);
3388         preempt_disable();
3389         __this_cpu_dec(bh_accounting.nr);
3390         recalc_bh_state();
3391         preempt_enable();
3392 }
3393 EXPORT_SYMBOL(free_buffer_head);
3394 
3395 static int buffer_exit_cpu_dead(unsigned int cpu)
3396 {
3397         int i;
3398         struct bh_lru *b = &per_cpu(bh_lrus, cpu);
3399 
3400         for (i = 0; i < BH_LRU_SIZE; i++) {
3401                 brelse(b->bhs[i]);
3402                 b->bhs[i] = NULL;
3403         }
3404         this_cpu_add(bh_accounting.nr, per_cpu(bh_accounting, cpu).nr);
3405         per_cpu(bh_accounting, cpu).nr = 0;
3406         return 0;
3407 }
3408 
3409 /**
3410  * bh_uptodate_or_lock - Test whether the buffer is uptodate
3411  * @bh: struct buffer_head
3412  *
3413  * Return true if the buffer is up-to-date and false,
3414  * with the buffer locked, if not.
3415  */
3416 int bh_uptodate_or_lock(struct buffer_head *bh)
3417 {
3418         if (!buffer_uptodate(bh)) {
3419                 lock_buffer(bh);
3420                 if (!buffer_uptodate(bh))
3421                         return 0;
3422                 unlock_buffer(bh);
3423         }
3424         return 1;
3425 }
3426 EXPORT_SYMBOL(bh_uptodate_or_lock);
3427 
3428 /**
3429  * bh_submit_read - Submit a locked buffer for reading
3430  * @bh: struct buffer_head
3431  *
3432  * Returns zero on success and -EIO on error.
3433  */
3434 int bh_submit_read(struct buffer_head *bh)
3435 {
3436         BUG_ON(!buffer_locked(bh));
3437 
3438         if (buffer_uptodate(bh)) {
3439                 unlock_buffer(bh);
3440                 return 0;
3441         }
3442 
3443         get_bh(bh);
3444         bh->b_end_io = end_buffer_read_sync;
3445         submit_bh(REQ_OP_READ, 0, bh);
3446         wait_on_buffer(bh);
3447         if (buffer_uptodate(bh))
3448                 return 0;
3449         return -EIO;
3450 }
3451 EXPORT_SYMBOL(bh_submit_read);
3452 
3453 void __init buffer_init(void)
3454 {
3455         unsigned long nrpages;
3456         int ret;
3457 
3458         bh_cachep = kmem_cache_create("buffer_head",
3459                         sizeof(struct buffer_head), 0,
3460                                 (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
3461                                 SLAB_MEM_SPREAD),
3462                                 NULL);
3463 
3464         /*
3465          * Limit the bh occupancy to 10% of ZONE_NORMAL
3466          */
3467         nrpages = (nr_free_buffer_pages() * 10) / 100;
3468         max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
3469         ret = cpuhp_setup_state_nocalls(CPUHP_FS_BUFF_DEAD, "fs/buffer:dead",
3470                                         NULL, buffer_exit_cpu_dead);
3471         WARN_ON(ret < 0);
3472 }
3473 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp