~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/fs/buffer.c

Version: ~ [ linux-4.18-rc5 ] ~ [ linux-4.17.6 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.55 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.112 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.140 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.115 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.57 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.27.62 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  *  linux/fs/buffer.c
  3  *
  4  *  Copyright (C) 1991, 1992, 2002  Linus Torvalds
  5  */
  6 
  7 /*
  8  * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
  9  *
 10  * Removed a lot of unnecessary code and simplified things now that
 11  * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
 12  *
 13  * Speed up hash, lru, and free list operations.  Use gfp() for allocating
 14  * hash table, use SLAB cache for buffer heads. SMP threading.  -DaveM
 15  *
 16  * Added 32k buffer block sizes - these are required older ARM systems. - RMK
 17  *
 18  * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
 19  */
 20 
 21 #include <linux/kernel.h>
 22 #include <linux/sched/signal.h>
 23 #include <linux/syscalls.h>
 24 #include <linux/fs.h>
 25 #include <linux/iomap.h>
 26 #include <linux/mm.h>
 27 #include <linux/percpu.h>
 28 #include <linux/slab.h>
 29 #include <linux/capability.h>
 30 #include <linux/blkdev.h>
 31 #include <linux/file.h>
 32 #include <linux/quotaops.h>
 33 #include <linux/highmem.h>
 34 #include <linux/export.h>
 35 #include <linux/backing-dev.h>
 36 #include <linux/writeback.h>
 37 #include <linux/hash.h>
 38 #include <linux/suspend.h>
 39 #include <linux/buffer_head.h>
 40 #include <linux/task_io_accounting_ops.h>
 41 #include <linux/bio.h>
 42 #include <linux/notifier.h>
 43 #include <linux/cpu.h>
 44 #include <linux/bitops.h>
 45 #include <linux/mpage.h>
 46 #include <linux/bit_spinlock.h>
 47 #include <linux/pagevec.h>
 48 #include <trace/events/block.h>
 49 
 50 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
 51 static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh,
 52                          enum rw_hint hint, struct writeback_control *wbc);
 53 
 54 #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
 55 
 56 inline void touch_buffer(struct buffer_head *bh)
 57 {
 58         trace_block_touch_buffer(bh);
 59         mark_page_accessed(bh->b_page);
 60 }
 61 EXPORT_SYMBOL(touch_buffer);
 62 
 63 void __lock_buffer(struct buffer_head *bh)
 64 {
 65         wait_on_bit_lock_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
 66 }
 67 EXPORT_SYMBOL(__lock_buffer);
 68 
 69 void unlock_buffer(struct buffer_head *bh)
 70 {
 71         clear_bit_unlock(BH_Lock, &bh->b_state);
 72         smp_mb__after_atomic();
 73         wake_up_bit(&bh->b_state, BH_Lock);
 74 }
 75 EXPORT_SYMBOL(unlock_buffer);
 76 
 77 /*
 78  * Returns if the page has dirty or writeback buffers. If all the buffers
 79  * are unlocked and clean then the PageDirty information is stale. If
 80  * any of the pages are locked, it is assumed they are locked for IO.
 81  */
 82 void buffer_check_dirty_writeback(struct page *page,
 83                                      bool *dirty, bool *writeback)
 84 {
 85         struct buffer_head *head, *bh;
 86         *dirty = false;
 87         *writeback = false;
 88 
 89         BUG_ON(!PageLocked(page));
 90 
 91         if (!page_has_buffers(page))
 92                 return;
 93 
 94         if (PageWriteback(page))
 95                 *writeback = true;
 96 
 97         head = page_buffers(page);
 98         bh = head;
 99         do {
100                 if (buffer_locked(bh))
101                         *writeback = true;
102 
103                 if (buffer_dirty(bh))
104                         *dirty = true;
105 
106                 bh = bh->b_this_page;
107         } while (bh != head);
108 }
109 EXPORT_SYMBOL(buffer_check_dirty_writeback);
110 
111 /*
112  * Block until a buffer comes unlocked.  This doesn't stop it
113  * from becoming locked again - you have to lock it yourself
114  * if you want to preserve its state.
115  */
116 void __wait_on_buffer(struct buffer_head * bh)
117 {
118         wait_on_bit_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
119 }
120 EXPORT_SYMBOL(__wait_on_buffer);
121 
122 static void
123 __clear_page_buffers(struct page *page)
124 {
125         ClearPagePrivate(page);
126         set_page_private(page, 0);
127         put_page(page);
128 }
129 
130 static void buffer_io_error(struct buffer_head *bh, char *msg)
131 {
132         if (!test_bit(BH_Quiet, &bh->b_state))
133                 printk_ratelimited(KERN_ERR
134                         "Buffer I/O error on dev %pg, logical block %llu%s\n",
135                         bh->b_bdev, (unsigned long long)bh->b_blocknr, msg);
136 }
137 
138 /*
139  * End-of-IO handler helper function which does not touch the bh after
140  * unlocking it.
141  * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
142  * a race there is benign: unlock_buffer() only use the bh's address for
143  * hashing after unlocking the buffer, so it doesn't actually touch the bh
144  * itself.
145  */
146 static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate)
147 {
148         if (uptodate) {
149                 set_buffer_uptodate(bh);
150         } else {
151                 /* This happens, due to failed read-ahead attempts. */
152                 clear_buffer_uptodate(bh);
153         }
154         unlock_buffer(bh);
155 }
156 
157 /*
158  * Default synchronous end-of-IO handler..  Just mark it up-to-date and
159  * unlock the buffer. This is what ll_rw_block uses too.
160  */
161 void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
162 {
163         __end_buffer_read_notouch(bh, uptodate);
164         put_bh(bh);
165 }
166 EXPORT_SYMBOL(end_buffer_read_sync);
167 
168 void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
169 {
170         if (uptodate) {
171                 set_buffer_uptodate(bh);
172         } else {
173                 buffer_io_error(bh, ", lost sync page write");
174                 mark_buffer_write_io_error(bh);
175                 clear_buffer_uptodate(bh);
176         }
177         unlock_buffer(bh);
178         put_bh(bh);
179 }
180 EXPORT_SYMBOL(end_buffer_write_sync);
181 
182 /*
183  * Various filesystems appear to want __find_get_block to be non-blocking.
184  * But it's the page lock which protects the buffers.  To get around this,
185  * we get exclusion from try_to_free_buffers with the blockdev mapping's
186  * private_lock.
187  *
188  * Hack idea: for the blockdev mapping, private_lock contention
189  * may be quite high.  This code could TryLock the page, and if that
190  * succeeds, there is no need to take private_lock.
191  */
192 static struct buffer_head *
193 __find_get_block_slow(struct block_device *bdev, sector_t block)
194 {
195         struct inode *bd_inode = bdev->bd_inode;
196         struct address_space *bd_mapping = bd_inode->i_mapping;
197         struct buffer_head *ret = NULL;
198         pgoff_t index;
199         struct buffer_head *bh;
200         struct buffer_head *head;
201         struct page *page;
202         int all_mapped = 1;
203 
204         index = block >> (PAGE_SHIFT - bd_inode->i_blkbits);
205         page = find_get_page_flags(bd_mapping, index, FGP_ACCESSED);
206         if (!page)
207                 goto out;
208 
209         spin_lock(&bd_mapping->private_lock);
210         if (!page_has_buffers(page))
211                 goto out_unlock;
212         head = page_buffers(page);
213         bh = head;
214         do {
215                 if (!buffer_mapped(bh))
216                         all_mapped = 0;
217                 else if (bh->b_blocknr == block) {
218                         ret = bh;
219                         get_bh(bh);
220                         goto out_unlock;
221                 }
222                 bh = bh->b_this_page;
223         } while (bh != head);
224 
225         /* we might be here because some of the buffers on this page are
226          * not mapped.  This is due to various races between
227          * file io on the block device and getblk.  It gets dealt with
228          * elsewhere, don't buffer_error if we had some unmapped buffers
229          */
230         if (all_mapped) {
231                 printk("__find_get_block_slow() failed. "
232                         "block=%llu, b_blocknr=%llu\n",
233                         (unsigned long long)block,
234                         (unsigned long long)bh->b_blocknr);
235                 printk("b_state=0x%08lx, b_size=%zu\n",
236                         bh->b_state, bh->b_size);
237                 printk("device %pg blocksize: %d\n", bdev,
238                         1 << bd_inode->i_blkbits);
239         }
240 out_unlock:
241         spin_unlock(&bd_mapping->private_lock);
242         put_page(page);
243 out:
244         return ret;
245 }
246 
247 /*
248  * I/O completion handler for block_read_full_page() - pages
249  * which come unlocked at the end of I/O.
250  */
251 static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
252 {
253         unsigned long flags;
254         struct buffer_head *first;
255         struct buffer_head *tmp;
256         struct page *page;
257         int page_uptodate = 1;
258 
259         BUG_ON(!buffer_async_read(bh));
260 
261         page = bh->b_page;
262         if (uptodate) {
263                 set_buffer_uptodate(bh);
264         } else {
265                 clear_buffer_uptodate(bh);
266                 buffer_io_error(bh, ", async page read");
267                 SetPageError(page);
268         }
269 
270         /*
271          * Be _very_ careful from here on. Bad things can happen if
272          * two buffer heads end IO at almost the same time and both
273          * decide that the page is now completely done.
274          */
275         first = page_buffers(page);
276         local_irq_save(flags);
277         bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
278         clear_buffer_async_read(bh);
279         unlock_buffer(bh);
280         tmp = bh;
281         do {
282                 if (!buffer_uptodate(tmp))
283                         page_uptodate = 0;
284                 if (buffer_async_read(tmp)) {
285                         BUG_ON(!buffer_locked(tmp));
286                         goto still_busy;
287                 }
288                 tmp = tmp->b_this_page;
289         } while (tmp != bh);
290         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
291         local_irq_restore(flags);
292 
293         /*
294          * If none of the buffers had errors and they are all
295          * uptodate then we can set the page uptodate.
296          */
297         if (page_uptodate && !PageError(page))
298                 SetPageUptodate(page);
299         unlock_page(page);
300         return;
301 
302 still_busy:
303         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
304         local_irq_restore(flags);
305         return;
306 }
307 
308 /*
309  * Completion handler for block_write_full_page() - pages which are unlocked
310  * during I/O, and which have PageWriteback cleared upon I/O completion.
311  */
312 void end_buffer_async_write(struct buffer_head *bh, int uptodate)
313 {
314         unsigned long flags;
315         struct buffer_head *first;
316         struct buffer_head *tmp;
317         struct page *page;
318 
319         BUG_ON(!buffer_async_write(bh));
320 
321         page = bh->b_page;
322         if (uptodate) {
323                 set_buffer_uptodate(bh);
324         } else {
325                 buffer_io_error(bh, ", lost async page write");
326                 mark_buffer_write_io_error(bh);
327                 clear_buffer_uptodate(bh);
328                 SetPageError(page);
329         }
330 
331         first = page_buffers(page);
332         local_irq_save(flags);
333         bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
334 
335         clear_buffer_async_write(bh);
336         unlock_buffer(bh);
337         tmp = bh->b_this_page;
338         while (tmp != bh) {
339                 if (buffer_async_write(tmp)) {
340                         BUG_ON(!buffer_locked(tmp));
341                         goto still_busy;
342                 }
343                 tmp = tmp->b_this_page;
344         }
345         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
346         local_irq_restore(flags);
347         end_page_writeback(page);
348         return;
349 
350 still_busy:
351         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
352         local_irq_restore(flags);
353         return;
354 }
355 EXPORT_SYMBOL(end_buffer_async_write);
356 
357 /*
358  * If a page's buffers are under async readin (end_buffer_async_read
359  * completion) then there is a possibility that another thread of
360  * control could lock one of the buffers after it has completed
361  * but while some of the other buffers have not completed.  This
362  * locked buffer would confuse end_buffer_async_read() into not unlocking
363  * the page.  So the absence of BH_Async_Read tells end_buffer_async_read()
364  * that this buffer is not under async I/O.
365  *
366  * The page comes unlocked when it has no locked buffer_async buffers
367  * left.
368  *
369  * PageLocked prevents anyone starting new async I/O reads any of
370  * the buffers.
371  *
372  * PageWriteback is used to prevent simultaneous writeout of the same
373  * page.
374  *
375  * PageLocked prevents anyone from starting writeback of a page which is
376  * under read I/O (PageWriteback is only ever set against a locked page).
377  */
378 static void mark_buffer_async_read(struct buffer_head *bh)
379 {
380         bh->b_end_io = end_buffer_async_read;
381         set_buffer_async_read(bh);
382 }
383 
384 static void mark_buffer_async_write_endio(struct buffer_head *bh,
385                                           bh_end_io_t *handler)
386 {
387         bh->b_end_io = handler;
388         set_buffer_async_write(bh);
389 }
390 
391 void mark_buffer_async_write(struct buffer_head *bh)
392 {
393         mark_buffer_async_write_endio(bh, end_buffer_async_write);
394 }
395 EXPORT_SYMBOL(mark_buffer_async_write);
396 
397 
398 /*
399  * fs/buffer.c contains helper functions for buffer-backed address space's
400  * fsync functions.  A common requirement for buffer-based filesystems is
401  * that certain data from the backing blockdev needs to be written out for
402  * a successful fsync().  For example, ext2 indirect blocks need to be
403  * written back and waited upon before fsync() returns.
404  *
405  * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
406  * inode_has_buffers() and invalidate_inode_buffers() are provided for the
407  * management of a list of dependent buffers at ->i_mapping->private_list.
408  *
409  * Locking is a little subtle: try_to_free_buffers() will remove buffers
410  * from their controlling inode's queue when they are being freed.  But
411  * try_to_free_buffers() will be operating against the *blockdev* mapping
412  * at the time, not against the S_ISREG file which depends on those buffers.
413  * So the locking for private_list is via the private_lock in the address_space
414  * which backs the buffers.  Which is different from the address_space 
415  * against which the buffers are listed.  So for a particular address_space,
416  * mapping->private_lock does *not* protect mapping->private_list!  In fact,
417  * mapping->private_list will always be protected by the backing blockdev's
418  * ->private_lock.
419  *
420  * Which introduces a requirement: all buffers on an address_space's
421  * ->private_list must be from the same address_space: the blockdev's.
422  *
423  * address_spaces which do not place buffers at ->private_list via these
424  * utility functions are free to use private_lock and private_list for
425  * whatever they want.  The only requirement is that list_empty(private_list)
426  * be true at clear_inode() time.
427  *
428  * FIXME: clear_inode should not call invalidate_inode_buffers().  The
429  * filesystems should do that.  invalidate_inode_buffers() should just go
430  * BUG_ON(!list_empty).
431  *
432  * FIXME: mark_buffer_dirty_inode() is a data-plane operation.  It should
433  * take an address_space, not an inode.  And it should be called
434  * mark_buffer_dirty_fsync() to clearly define why those buffers are being
435  * queued up.
436  *
437  * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
438  * list if it is already on a list.  Because if the buffer is on a list,
439  * it *must* already be on the right one.  If not, the filesystem is being
440  * silly.  This will save a ton of locking.  But first we have to ensure
441  * that buffers are taken *off* the old inode's list when they are freed
442  * (presumably in truncate).  That requires careful auditing of all
443  * filesystems (do it inside bforget()).  It could also be done by bringing
444  * b_inode back.
445  */
446 
447 /*
448  * The buffer's backing address_space's private_lock must be held
449  */
450 static void __remove_assoc_queue(struct buffer_head *bh)
451 {
452         list_del_init(&bh->b_assoc_buffers);
453         WARN_ON(!bh->b_assoc_map);
454         bh->b_assoc_map = NULL;
455 }
456 
457 int inode_has_buffers(struct inode *inode)
458 {
459         return !list_empty(&inode->i_data.private_list);
460 }
461 
462 /*
463  * osync is designed to support O_SYNC io.  It waits synchronously for
464  * all already-submitted IO to complete, but does not queue any new
465  * writes to the disk.
466  *
467  * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
468  * you dirty the buffers, and then use osync_inode_buffers to wait for
469  * completion.  Any other dirty buffers which are not yet queued for
470  * write will not be flushed to disk by the osync.
471  */
472 static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
473 {
474         struct buffer_head *bh;
475         struct list_head *p;
476         int err = 0;
477 
478         spin_lock(lock);
479 repeat:
480         list_for_each_prev(p, list) {
481                 bh = BH_ENTRY(p);
482                 if (buffer_locked(bh)) {
483                         get_bh(bh);
484                         spin_unlock(lock);
485                         wait_on_buffer(bh);
486                         if (!buffer_uptodate(bh))
487                                 err = -EIO;
488                         brelse(bh);
489                         spin_lock(lock);
490                         goto repeat;
491                 }
492         }
493         spin_unlock(lock);
494         return err;
495 }
496 
497 void emergency_thaw_bdev(struct super_block *sb)
498 {
499         while (sb->s_bdev && !thaw_bdev(sb->s_bdev, sb))
500                 printk(KERN_WARNING "Emergency Thaw on %pg\n", sb->s_bdev);
501 }
502 
503 /**
504  * sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers
505  * @mapping: the mapping which wants those buffers written
506  *
507  * Starts I/O against the buffers at mapping->private_list, and waits upon
508  * that I/O.
509  *
510  * Basically, this is a convenience function for fsync().
511  * @mapping is a file or directory which needs those buffers to be written for
512  * a successful fsync().
513  */
514 int sync_mapping_buffers(struct address_space *mapping)
515 {
516         struct address_space *buffer_mapping = mapping->private_data;
517 
518         if (buffer_mapping == NULL || list_empty(&mapping->private_list))
519                 return 0;
520 
521         return fsync_buffers_list(&buffer_mapping->private_lock,
522                                         &mapping->private_list);
523 }
524 EXPORT_SYMBOL(sync_mapping_buffers);
525 
526 /*
527  * Called when we've recently written block `bblock', and it is known that
528  * `bblock' was for a buffer_boundary() buffer.  This means that the block at
529  * `bblock + 1' is probably a dirty indirect block.  Hunt it down and, if it's
530  * dirty, schedule it for IO.  So that indirects merge nicely with their data.
531  */
532 void write_boundary_block(struct block_device *bdev,
533                         sector_t bblock, unsigned blocksize)
534 {
535         struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
536         if (bh) {
537                 if (buffer_dirty(bh))
538                         ll_rw_block(REQ_OP_WRITE, 0, 1, &bh);
539                 put_bh(bh);
540         }
541 }
542 
543 void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
544 {
545         struct address_space *mapping = inode->i_mapping;
546         struct address_space *buffer_mapping = bh->b_page->mapping;
547 
548         mark_buffer_dirty(bh);
549         if (!mapping->private_data) {
550                 mapping->private_data = buffer_mapping;
551         } else {
552                 BUG_ON(mapping->private_data != buffer_mapping);
553         }
554         if (!bh->b_assoc_map) {
555                 spin_lock(&buffer_mapping->private_lock);
556                 list_move_tail(&bh->b_assoc_buffers,
557                                 &mapping->private_list);
558                 bh->b_assoc_map = mapping;
559                 spin_unlock(&buffer_mapping->private_lock);
560         }
561 }
562 EXPORT_SYMBOL(mark_buffer_dirty_inode);
563 
564 /*
565  * Mark the page dirty, and set it dirty in the radix tree, and mark the inode
566  * dirty.
567  *
568  * If warn is true, then emit a warning if the page is not uptodate and has
569  * not been truncated.
570  *
571  * The caller must hold lock_page_memcg().
572  */
573 void __set_page_dirty(struct page *page, struct address_space *mapping,
574                              int warn)
575 {
576         unsigned long flags;
577 
578         xa_lock_irqsave(&mapping->i_pages, flags);
579         if (page->mapping) {    /* Race with truncate? */
580                 WARN_ON_ONCE(warn && !PageUptodate(page));
581                 account_page_dirtied(page, mapping);
582                 radix_tree_tag_set(&mapping->i_pages,
583                                 page_index(page), PAGECACHE_TAG_DIRTY);
584         }
585         xa_unlock_irqrestore(&mapping->i_pages, flags);
586 }
587 EXPORT_SYMBOL_GPL(__set_page_dirty);
588 
589 /*
590  * Add a page to the dirty page list.
591  *
592  * It is a sad fact of life that this function is called from several places
593  * deeply under spinlocking.  It may not sleep.
594  *
595  * If the page has buffers, the uptodate buffers are set dirty, to preserve
596  * dirty-state coherency between the page and the buffers.  It the page does
597  * not have buffers then when they are later attached they will all be set
598  * dirty.
599  *
600  * The buffers are dirtied before the page is dirtied.  There's a small race
601  * window in which a writepage caller may see the page cleanness but not the
602  * buffer dirtiness.  That's fine.  If this code were to set the page dirty
603  * before the buffers, a concurrent writepage caller could clear the page dirty
604  * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
605  * page on the dirty page list.
606  *
607  * We use private_lock to lock against try_to_free_buffers while using the
608  * page's buffer list.  Also use this to protect against clean buffers being
609  * added to the page after it was set dirty.
610  *
611  * FIXME: may need to call ->reservepage here as well.  That's rather up to the
612  * address_space though.
613  */
614 int __set_page_dirty_buffers(struct page *page)
615 {
616         int newly_dirty;
617         struct address_space *mapping = page_mapping(page);
618 
619         if (unlikely(!mapping))
620                 return !TestSetPageDirty(page);
621 
622         spin_lock(&mapping->private_lock);
623         if (page_has_buffers(page)) {
624                 struct buffer_head *head = page_buffers(page);
625                 struct buffer_head *bh = head;
626 
627                 do {
628                         set_buffer_dirty(bh);
629                         bh = bh->b_this_page;
630                 } while (bh != head);
631         }
632         /*
633          * Lock out page->mem_cgroup migration to keep PageDirty
634          * synchronized with per-memcg dirty page counters.
635          */
636         lock_page_memcg(page);
637         newly_dirty = !TestSetPageDirty(page);
638         spin_unlock(&mapping->private_lock);
639 
640         if (newly_dirty)
641                 __set_page_dirty(page, mapping, 1);
642 
643         unlock_page_memcg(page);
644 
645         if (newly_dirty)
646                 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
647 
648         return newly_dirty;
649 }
650 EXPORT_SYMBOL(__set_page_dirty_buffers);
651 
652 /*
653  * Write out and wait upon a list of buffers.
654  *
655  * We have conflicting pressures: we want to make sure that all
656  * initially dirty buffers get waited on, but that any subsequently
657  * dirtied buffers don't.  After all, we don't want fsync to last
658  * forever if somebody is actively writing to the file.
659  *
660  * Do this in two main stages: first we copy dirty buffers to a
661  * temporary inode list, queueing the writes as we go.  Then we clean
662  * up, waiting for those writes to complete.
663  * 
664  * During this second stage, any subsequent updates to the file may end
665  * up refiling the buffer on the original inode's dirty list again, so
666  * there is a chance we will end up with a buffer queued for write but
667  * not yet completed on that list.  So, as a final cleanup we go through
668  * the osync code to catch these locked, dirty buffers without requeuing
669  * any newly dirty buffers for write.
670  */
671 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
672 {
673         struct buffer_head *bh;
674         struct list_head tmp;
675         struct address_space *mapping;
676         int err = 0, err2;
677         struct blk_plug plug;
678 
679         INIT_LIST_HEAD(&tmp);
680         blk_start_plug(&plug);
681 
682         spin_lock(lock);
683         while (!list_empty(list)) {
684                 bh = BH_ENTRY(list->next);
685                 mapping = bh->b_assoc_map;
686                 __remove_assoc_queue(bh);
687                 /* Avoid race with mark_buffer_dirty_inode() which does
688                  * a lockless check and we rely on seeing the dirty bit */
689                 smp_mb();
690                 if (buffer_dirty(bh) || buffer_locked(bh)) {
691                         list_add(&bh->b_assoc_buffers, &tmp);
692                         bh->b_assoc_map = mapping;
693                         if (buffer_dirty(bh)) {
694                                 get_bh(bh);
695                                 spin_unlock(lock);
696                                 /*
697                                  * Ensure any pending I/O completes so that
698                                  * write_dirty_buffer() actually writes the
699                                  * current contents - it is a noop if I/O is
700                                  * still in flight on potentially older
701                                  * contents.
702                                  */
703                                 write_dirty_buffer(bh, REQ_SYNC);
704 
705                                 /*
706                                  * Kick off IO for the previous mapping. Note
707                                  * that we will not run the very last mapping,
708                                  * wait_on_buffer() will do that for us
709                                  * through sync_buffer().
710                                  */
711                                 brelse(bh);
712                                 spin_lock(lock);
713                         }
714                 }
715         }
716 
717         spin_unlock(lock);
718         blk_finish_plug(&plug);
719         spin_lock(lock);
720 
721         while (!list_empty(&tmp)) {
722                 bh = BH_ENTRY(tmp.prev);
723                 get_bh(bh);
724                 mapping = bh->b_assoc_map;
725                 __remove_assoc_queue(bh);
726                 /* Avoid race with mark_buffer_dirty_inode() which does
727                  * a lockless check and we rely on seeing the dirty bit */
728                 smp_mb();
729                 if (buffer_dirty(bh)) {
730                         list_add(&bh->b_assoc_buffers,
731                                  &mapping->private_list);
732                         bh->b_assoc_map = mapping;
733                 }
734                 spin_unlock(lock);
735                 wait_on_buffer(bh);
736                 if (!buffer_uptodate(bh))
737                         err = -EIO;
738                 brelse(bh);
739                 spin_lock(lock);
740         }
741         
742         spin_unlock(lock);
743         err2 = osync_buffers_list(lock, list);
744         if (err)
745                 return err;
746         else
747                 return err2;
748 }
749 
750 /*
751  * Invalidate any and all dirty buffers on a given inode.  We are
752  * probably unmounting the fs, but that doesn't mean we have already
753  * done a sync().  Just drop the buffers from the inode list.
754  *
755  * NOTE: we take the inode's blockdev's mapping's private_lock.  Which
756  * assumes that all the buffers are against the blockdev.  Not true
757  * for reiserfs.
758  */
759 void invalidate_inode_buffers(struct inode *inode)
760 {
761         if (inode_has_buffers(inode)) {
762                 struct address_space *mapping = &inode->i_data;
763                 struct list_head *list = &mapping->private_list;
764                 struct address_space *buffer_mapping = mapping->private_data;
765 
766                 spin_lock(&buffer_mapping->private_lock);
767                 while (!list_empty(list))
768                         __remove_assoc_queue(BH_ENTRY(list->next));
769                 spin_unlock(&buffer_mapping->private_lock);
770         }
771 }
772 EXPORT_SYMBOL(invalidate_inode_buffers);
773 
774 /*
775  * Remove any clean buffers from the inode's buffer list.  This is called
776  * when we're trying to free the inode itself.  Those buffers can pin it.
777  *
778  * Returns true if all buffers were removed.
779  */
780 int remove_inode_buffers(struct inode *inode)
781 {
782         int ret = 1;
783 
784         if (inode_has_buffers(inode)) {
785                 struct address_space *mapping = &inode->i_data;
786                 struct list_head *list = &mapping->private_list;
787                 struct address_space *buffer_mapping = mapping->private_data;
788 
789                 spin_lock(&buffer_mapping->private_lock);
790                 while (!list_empty(list)) {
791                         struct buffer_head *bh = BH_ENTRY(list->next);
792                         if (buffer_dirty(bh)) {
793                                 ret = 0;
794                                 break;
795                         }
796                         __remove_assoc_queue(bh);
797                 }
798                 spin_unlock(&buffer_mapping->private_lock);
799         }
800         return ret;
801 }
802 
803 /*
804  * Create the appropriate buffers when given a page for data area and
805  * the size of each buffer.. Use the bh->b_this_page linked list to
806  * follow the buffers created.  Return NULL if unable to create more
807  * buffers.
808  *
809  * The retry flag is used to differentiate async IO (paging, swapping)
810  * which may not fail from ordinary buffer allocations.
811  */
812 struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
813                 bool retry)
814 {
815         struct buffer_head *bh, *head;
816         gfp_t gfp = GFP_NOFS;
817         long offset;
818 
819         if (retry)
820                 gfp |= __GFP_NOFAIL;
821 
822         head = NULL;
823         offset = PAGE_SIZE;
824         while ((offset -= size) >= 0) {
825                 bh = alloc_buffer_head(gfp);
826                 if (!bh)
827                         goto no_grow;
828 
829                 bh->b_this_page = head;
830                 bh->b_blocknr = -1;
831                 head = bh;
832 
833                 bh->b_size = size;
834 
835                 /* Link the buffer to its page */
836                 set_bh_page(bh, page, offset);
837         }
838         return head;
839 /*
840  * In case anything failed, we just free everything we got.
841  */
842 no_grow:
843         if (head) {
844                 do {
845                         bh = head;
846                         head = head->b_this_page;
847                         free_buffer_head(bh);
848                 } while (head);
849         }
850 
851         return NULL;
852 }
853 EXPORT_SYMBOL_GPL(alloc_page_buffers);
854 
855 static inline void
856 link_dev_buffers(struct page *page, struct buffer_head *head)
857 {
858         struct buffer_head *bh, *tail;
859 
860         bh = head;
861         do {
862                 tail = bh;
863                 bh = bh->b_this_page;
864         } while (bh);
865         tail->b_this_page = head;
866         attach_page_buffers(page, head);
867 }
868 
869 static sector_t blkdev_max_block(struct block_device *bdev, unsigned int size)
870 {
871         sector_t retval = ~((sector_t)0);
872         loff_t sz = i_size_read(bdev->bd_inode);
873 
874         if (sz) {
875                 unsigned int sizebits = blksize_bits(size);
876                 retval = (sz >> sizebits);
877         }
878         return retval;
879 }
880 
881 /*
882  * Initialise the state of a blockdev page's buffers.
883  */ 
884 static sector_t
885 init_page_buffers(struct page *page, struct block_device *bdev,
886                         sector_t block, int size)
887 {
888         struct buffer_head *head = page_buffers(page);
889         struct buffer_head *bh = head;
890         int uptodate = PageUptodate(page);
891         sector_t end_block = blkdev_max_block(I_BDEV(bdev->bd_inode), size);
892 
893         do {
894                 if (!buffer_mapped(bh)) {
895                         bh->b_end_io = NULL;
896                         bh->b_private = NULL;
897                         bh->b_bdev = bdev;
898                         bh->b_blocknr = block;
899                         if (uptodate)
900                                 set_buffer_uptodate(bh);
901                         if (block < end_block)
902                                 set_buffer_mapped(bh);
903                 }
904                 block++;
905                 bh = bh->b_this_page;
906         } while (bh != head);
907 
908         /*
909          * Caller needs to validate requested block against end of device.
910          */
911         return end_block;
912 }
913 
914 /*
915  * Create the page-cache page that contains the requested block.
916  *
917  * This is used purely for blockdev mappings.
918  */
919 static int
920 grow_dev_page(struct block_device *bdev, sector_t block,
921               pgoff_t index, int size, int sizebits, gfp_t gfp)
922 {
923         struct inode *inode = bdev->bd_inode;
924         struct page *page;
925         struct buffer_head *bh;
926         sector_t end_block;
927         int ret = 0;            /* Will call free_more_memory() */
928         gfp_t gfp_mask;
929 
930         gfp_mask = mapping_gfp_constraint(inode->i_mapping, ~__GFP_FS) | gfp;
931 
932         /*
933          * XXX: __getblk_slow() can not really deal with failure and
934          * will endlessly loop on improvised global reclaim.  Prefer
935          * looping in the allocator rather than here, at least that
936          * code knows what it's doing.
937          */
938         gfp_mask |= __GFP_NOFAIL;
939 
940         page = find_or_create_page(inode->i_mapping, index, gfp_mask);
941 
942         BUG_ON(!PageLocked(page));
943 
944         if (page_has_buffers(page)) {
945                 bh = page_buffers(page);
946                 if (bh->b_size == size) {
947                         end_block = init_page_buffers(page, bdev,
948                                                 (sector_t)index << sizebits,
949                                                 size);
950                         goto done;
951                 }
952                 if (!try_to_free_buffers(page))
953                         goto failed;
954         }
955 
956         /*
957          * Allocate some buffers for this page
958          */
959         bh = alloc_page_buffers(page, size, true);
960 
961         /*
962          * Link the page to the buffers and initialise them.  Take the
963          * lock to be atomic wrt __find_get_block(), which does not
964          * run under the page lock.
965          */
966         spin_lock(&inode->i_mapping->private_lock);
967         link_dev_buffers(page, bh);
968         end_block = init_page_buffers(page, bdev, (sector_t)index << sizebits,
969                         size);
970         spin_unlock(&inode->i_mapping->private_lock);
971 done:
972         ret = (block < end_block) ? 1 : -ENXIO;
973 failed:
974         unlock_page(page);
975         put_page(page);
976         return ret;
977 }
978 
979 /*
980  * Create buffers for the specified block device block's page.  If
981  * that page was dirty, the buffers are set dirty also.
982  */
983 static int
984 grow_buffers(struct block_device *bdev, sector_t block, int size, gfp_t gfp)
985 {
986         pgoff_t index;
987         int sizebits;
988 
989         sizebits = -1;
990         do {
991                 sizebits++;
992         } while ((size << sizebits) < PAGE_SIZE);
993 
994         index = block >> sizebits;
995 
996         /*
997          * Check for a block which wants to lie outside our maximum possible
998          * pagecache index.  (this comparison is done using sector_t types).
999          */
1000         if (unlikely(index != block >> sizebits)) {
1001                 printk(KERN_ERR "%s: requested out-of-range block %llu for "
1002                         "device %pg\n",
1003                         __func__, (unsigned long long)block,
1004                         bdev);
1005                 return -EIO;
1006         }
1007 
1008         /* Create a page with the proper size buffers.. */
1009         return grow_dev_page(bdev, block, index, size, sizebits, gfp);
1010 }
1011 
1012 static struct buffer_head *
1013 __getblk_slow(struct block_device *bdev, sector_t block,
1014              unsigned size, gfp_t gfp)
1015 {
1016         /* Size must be multiple of hard sectorsize */
1017         if (unlikely(size & (bdev_logical_block_size(bdev)-1) ||
1018                         (size < 512 || size > PAGE_SIZE))) {
1019                 printk(KERN_ERR "getblk(): invalid block size %d requested\n",
1020                                         size);
1021                 printk(KERN_ERR "logical block size: %d\n",
1022                                         bdev_logical_block_size(bdev));
1023 
1024                 dump_stack();
1025                 return NULL;
1026         }
1027 
1028         for (;;) {
1029                 struct buffer_head *bh;
1030                 int ret;
1031 
1032                 bh = __find_get_block(bdev, block, size);
1033                 if (bh)
1034                         return bh;
1035 
1036                 ret = grow_buffers(bdev, block, size, gfp);
1037                 if (ret < 0)
1038                         return NULL;
1039         }
1040 }
1041 
1042 /*
1043  * The relationship between dirty buffers and dirty pages:
1044  *
1045  * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1046  * the page is tagged dirty in its radix tree.
1047  *
1048  * At all times, the dirtiness of the buffers represents the dirtiness of
1049  * subsections of the page.  If the page has buffers, the page dirty bit is
1050  * merely a hint about the true dirty state.
1051  *
1052  * When a page is set dirty in its entirety, all its buffers are marked dirty
1053  * (if the page has buffers).
1054  *
1055  * When a buffer is marked dirty, its page is dirtied, but the page's other
1056  * buffers are not.
1057  *
1058  * Also.  When blockdev buffers are explicitly read with bread(), they
1059  * individually become uptodate.  But their backing page remains not
1060  * uptodate - even if all of its buffers are uptodate.  A subsequent
1061  * block_read_full_page() against that page will discover all the uptodate
1062  * buffers, will set the page uptodate and will perform no I/O.
1063  */
1064 
1065 /**
1066  * mark_buffer_dirty - mark a buffer_head as needing writeout
1067  * @bh: the buffer_head to mark dirty
1068  *
1069  * mark_buffer_dirty() will set the dirty bit against the buffer, then set its
1070  * backing page dirty, then tag the page as dirty in its address_space's radix
1071  * tree and then attach the address_space's inode to its superblock's dirty
1072  * inode list.
1073  *
1074  * mark_buffer_dirty() is atomic.  It takes bh->b_page->mapping->private_lock,
1075  * i_pages lock and mapping->host->i_lock.
1076  */
1077 void mark_buffer_dirty(struct buffer_head *bh)
1078 {
1079         WARN_ON_ONCE(!buffer_uptodate(bh));
1080 
1081         trace_block_dirty_buffer(bh);
1082 
1083         /*
1084          * Very *carefully* optimize the it-is-already-dirty case.
1085          *
1086          * Don't let the final "is it dirty" escape to before we
1087          * perhaps modified the buffer.
1088          */
1089         if (buffer_dirty(bh)) {
1090                 smp_mb();
1091                 if (buffer_dirty(bh))
1092                         return;
1093         }
1094 
1095         if (!test_set_buffer_dirty(bh)) {
1096                 struct page *page = bh->b_page;
1097                 struct address_space *mapping = NULL;
1098 
1099                 lock_page_memcg(page);
1100                 if (!TestSetPageDirty(page)) {
1101                         mapping = page_mapping(page);
1102                         if (mapping)
1103                                 __set_page_dirty(page, mapping, 0);
1104                 }
1105                 unlock_page_memcg(page);
1106                 if (mapping)
1107                         __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
1108         }
1109 }
1110 EXPORT_SYMBOL(mark_buffer_dirty);
1111 
1112 void mark_buffer_write_io_error(struct buffer_head *bh)
1113 {
1114         set_buffer_write_io_error(bh);
1115         /* FIXME: do we need to set this in both places? */
1116         if (bh->b_page && bh->b_page->mapping)
1117                 mapping_set_error(bh->b_page->mapping, -EIO);
1118         if (bh->b_assoc_map)
1119                 mapping_set_error(bh->b_assoc_map, -EIO);
1120 }
1121 EXPORT_SYMBOL(mark_buffer_write_io_error);
1122 
1123 /*
1124  * Decrement a buffer_head's reference count.  If all buffers against a page
1125  * have zero reference count, are clean and unlocked, and if the page is clean
1126  * and unlocked then try_to_free_buffers() may strip the buffers from the page
1127  * in preparation for freeing it (sometimes, rarely, buffers are removed from
1128  * a page but it ends up not being freed, and buffers may later be reattached).
1129  */
1130 void __brelse(struct buffer_head * buf)
1131 {
1132         if (atomic_read(&buf->b_count)) {
1133                 put_bh(buf);
1134                 return;
1135         }
1136         WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n");
1137 }
1138 EXPORT_SYMBOL(__brelse);
1139 
1140 /*
1141  * bforget() is like brelse(), except it discards any
1142  * potentially dirty data.
1143  */
1144 void __bforget(struct buffer_head *bh)
1145 {
1146         clear_buffer_dirty(bh);
1147         if (bh->b_assoc_map) {
1148                 struct address_space *buffer_mapping = bh->b_page->mapping;
1149 
1150                 spin_lock(&buffer_mapping->private_lock);
1151                 list_del_init(&bh->b_assoc_buffers);
1152                 bh->b_assoc_map = NULL;
1153                 spin_unlock(&buffer_mapping->private_lock);
1154         }
1155         __brelse(bh);
1156 }
1157 EXPORT_SYMBOL(__bforget);
1158 
1159 static struct buffer_head *__bread_slow(struct buffer_head *bh)
1160 {
1161         lock_buffer(bh);
1162         if (buffer_uptodate(bh)) {
1163                 unlock_buffer(bh);
1164                 return bh;
1165         } else {
1166                 get_bh(bh);
1167                 bh->b_end_io = end_buffer_read_sync;
1168                 submit_bh(REQ_OP_READ, 0, bh);
1169                 wait_on_buffer(bh);
1170                 if (buffer_uptodate(bh))
1171                         return bh;
1172         }
1173         brelse(bh);
1174         return NULL;
1175 }
1176 
1177 /*
1178  * Per-cpu buffer LRU implementation.  To reduce the cost of __find_get_block().
1179  * The bhs[] array is sorted - newest buffer is at bhs[0].  Buffers have their
1180  * refcount elevated by one when they're in an LRU.  A buffer can only appear
1181  * once in a particular CPU's LRU.  A single buffer can be present in multiple
1182  * CPU's LRUs at the same time.
1183  *
1184  * This is a transparent caching front-end to sb_bread(), sb_getblk() and
1185  * sb_find_get_block().
1186  *
1187  * The LRUs themselves only need locking against invalidate_bh_lrus.  We use
1188  * a local interrupt disable for that.
1189  */
1190 
1191 #define BH_LRU_SIZE     16
1192 
1193 struct bh_lru {
1194         struct buffer_head *bhs[BH_LRU_SIZE];
1195 };
1196 
1197 static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
1198 
1199 #ifdef CONFIG_SMP
1200 #define bh_lru_lock()   local_irq_disable()
1201 #define bh_lru_unlock() local_irq_enable()
1202 #else
1203 #define bh_lru_lock()   preempt_disable()
1204 #define bh_lru_unlock() preempt_enable()
1205 #endif
1206 
1207 static inline void check_irqs_on(void)
1208 {
1209 #ifdef irqs_disabled
1210         BUG_ON(irqs_disabled());
1211 #endif
1212 }
1213 
1214 /*
1215  * Install a buffer_head into this cpu's LRU.  If not already in the LRU, it is
1216  * inserted at the front, and the buffer_head at the back if any is evicted.
1217  * Or, if already in the LRU it is moved to the front.
1218  */
1219 static void bh_lru_install(struct buffer_head *bh)
1220 {
1221         struct buffer_head *evictee = bh;
1222         struct bh_lru *b;
1223         int i;
1224 
1225         check_irqs_on();
1226         bh_lru_lock();
1227 
1228         b = this_cpu_ptr(&bh_lrus);
1229         for (i = 0; i < BH_LRU_SIZE; i++) {
1230                 swap(evictee, b->bhs[i]);
1231                 if (evictee == bh) {
1232                         bh_lru_unlock();
1233                         return;
1234                 }
1235         }
1236 
1237         get_bh(bh);
1238         bh_lru_unlock();
1239         brelse(evictee);
1240 }
1241 
1242 /*
1243  * Look up the bh in this cpu's LRU.  If it's there, move it to the head.
1244  */
1245 static struct buffer_head *
1246 lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size)
1247 {
1248         struct buffer_head *ret = NULL;
1249         unsigned int i;
1250 
1251         check_irqs_on();
1252         bh_lru_lock();
1253         for (i = 0; i < BH_LRU_SIZE; i++) {
1254                 struct buffer_head *bh = __this_cpu_read(bh_lrus.bhs[i]);
1255 
1256                 if (bh && bh->b_blocknr == block && bh->b_bdev == bdev &&
1257                     bh->b_size == size) {
1258                         if (i) {
1259                                 while (i) {
1260                                         __this_cpu_write(bh_lrus.bhs[i],
1261                                                 __this_cpu_read(bh_lrus.bhs[i - 1]));
1262                                         i--;
1263                                 }
1264                                 __this_cpu_write(bh_lrus.bhs[0], bh);
1265                         }
1266                         get_bh(bh);
1267                         ret = bh;
1268                         break;
1269                 }
1270         }
1271         bh_lru_unlock();
1272         return ret;
1273 }
1274 
1275 /*
1276  * Perform a pagecache lookup for the matching buffer.  If it's there, refresh
1277  * it in the LRU and mark it as accessed.  If it is not present then return
1278  * NULL
1279  */
1280 struct buffer_head *
1281 __find_get_block(struct block_device *bdev, sector_t block, unsigned size)
1282 {
1283         struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
1284 
1285         if (bh == NULL) {
1286                 /* __find_get_block_slow will mark the page accessed */
1287                 bh = __find_get_block_slow(bdev, block);
1288                 if (bh)
1289                         bh_lru_install(bh);
1290         } else
1291                 touch_buffer(bh);
1292 
1293         return bh;
1294 }
1295 EXPORT_SYMBOL(__find_get_block);
1296 
1297 /*
1298  * __getblk_gfp() will locate (and, if necessary, create) the buffer_head
1299  * which corresponds to the passed block_device, block and size. The
1300  * returned buffer has its reference count incremented.
1301  *
1302  * __getblk_gfp() will lock up the machine if grow_dev_page's
1303  * try_to_free_buffers() attempt is failing.  FIXME, perhaps?
1304  */
1305 struct buffer_head *
1306 __getblk_gfp(struct block_device *bdev, sector_t block,
1307              unsigned size, gfp_t gfp)
1308 {
1309         struct buffer_head *bh = __find_get_block(bdev, block, size);
1310 
1311         might_sleep();
1312         if (bh == NULL)
1313                 bh = __getblk_slow(bdev, block, size, gfp);
1314         return bh;
1315 }
1316 EXPORT_SYMBOL(__getblk_gfp);
1317 
1318 /*
1319  * Do async read-ahead on a buffer..
1320  */
1321 void __breadahead(struct block_device *bdev, sector_t block, unsigned size)
1322 {
1323         struct buffer_head *bh = __getblk(bdev, block, size);
1324         if (likely(bh)) {
1325                 ll_rw_block(REQ_OP_READ, REQ_RAHEAD, 1, &bh);
1326                 brelse(bh);
1327         }
1328 }
1329 EXPORT_SYMBOL(__breadahead);
1330 
1331 /**
1332  *  __bread_gfp() - reads a specified block and returns the bh
1333  *  @bdev: the block_device to read from
1334  *  @block: number of block
1335  *  @size: size (in bytes) to read
1336  *  @gfp: page allocation flag
1337  *
1338  *  Reads a specified block, and returns buffer head that contains it.
1339  *  The page cache can be allocated from non-movable area
1340  *  not to prevent page migration if you set gfp to zero.
1341  *  It returns NULL if the block was unreadable.
1342  */
1343 struct buffer_head *
1344 __bread_gfp(struct block_device *bdev, sector_t block,
1345                    unsigned size, gfp_t gfp)
1346 {
1347         struct buffer_head *bh = __getblk_gfp(bdev, block, size, gfp);
1348 
1349         if (likely(bh) && !buffer_uptodate(bh))
1350                 bh = __bread_slow(bh);
1351         return bh;
1352 }
1353 EXPORT_SYMBOL(__bread_gfp);
1354 
1355 /*
1356  * invalidate_bh_lrus() is called rarely - but not only at unmount.
1357  * This doesn't race because it runs in each cpu either in irq
1358  * or with preempt disabled.
1359  */
1360 static void invalidate_bh_lru(void *arg)
1361 {
1362         struct bh_lru *b = &get_cpu_var(bh_lrus);
1363         int i;
1364 
1365         for (i = 0; i < BH_LRU_SIZE; i++) {
1366                 brelse(b->bhs[i]);
1367                 b->bhs[i] = NULL;
1368         }
1369         put_cpu_var(bh_lrus);
1370 }
1371 
1372 static bool has_bh_in_lru(int cpu, void *dummy)
1373 {
1374         struct bh_lru *b = per_cpu_ptr(&bh_lrus, cpu);
1375         int i;
1376         
1377         for (i = 0; i < BH_LRU_SIZE; i++) {
1378                 if (b->bhs[i])
1379                         return 1;
1380         }
1381 
1382         return 0;
1383 }
1384 
1385 void invalidate_bh_lrus(void)
1386 {
1387         on_each_cpu_cond(has_bh_in_lru, invalidate_bh_lru, NULL, 1, GFP_KERNEL);
1388 }
1389 EXPORT_SYMBOL_GPL(invalidate_bh_lrus);
1390 
1391 void set_bh_page(struct buffer_head *bh,
1392                 struct page *page, unsigned long offset)
1393 {
1394         bh->b_page = page;
1395         BUG_ON(offset >= PAGE_SIZE);
1396         if (PageHighMem(page))
1397                 /*
1398                  * This catches illegal uses and preserves the offset:
1399                  */
1400                 bh->b_data = (char *)(0 + offset);
1401         else
1402                 bh->b_data = page_address(page) + offset;
1403 }
1404 EXPORT_SYMBOL(set_bh_page);
1405 
1406 /*
1407  * Called when truncating a buffer on a page completely.
1408  */
1409 
1410 /* Bits that are cleared during an invalidate */
1411 #define BUFFER_FLAGS_DISCARD \
1412         (1 << BH_Mapped | 1 << BH_New | 1 << BH_Req | \
1413          1 << BH_Delay | 1 << BH_Unwritten)
1414 
1415 static void discard_buffer(struct buffer_head * bh)
1416 {
1417         unsigned long b_state, b_state_old;
1418 
1419         lock_buffer(bh);
1420         clear_buffer_dirty(bh);
1421         bh->b_bdev = NULL;
1422         b_state = bh->b_state;
1423         for (;;) {
1424                 b_state_old = cmpxchg(&bh->b_state, b_state,
1425                                       (b_state & ~BUFFER_FLAGS_DISCARD));
1426                 if (b_state_old == b_state)
1427                         break;
1428                 b_state = b_state_old;
1429         }
1430         unlock_buffer(bh);
1431 }
1432 
1433 /**
1434  * block_invalidatepage - invalidate part or all of a buffer-backed page
1435  *
1436  * @page: the page which is affected
1437  * @offset: start of the range to invalidate
1438  * @length: length of the range to invalidate
1439  *
1440  * block_invalidatepage() is called when all or part of the page has become
1441  * invalidated by a truncate operation.
1442  *
1443  * block_invalidatepage() does not have to release all buffers, but it must
1444  * ensure that no dirty buffer is left outside @offset and that no I/O
1445  * is underway against any of the blocks which are outside the truncation
1446  * point.  Because the caller is about to free (and possibly reuse) those
1447  * blocks on-disk.
1448  */
1449 void block_invalidatepage(struct page *page, unsigned int offset,
1450                           unsigned int length)
1451 {
1452         struct buffer_head *head, *bh, *next;
1453         unsigned int curr_off = 0;
1454         unsigned int stop = length + offset;
1455 
1456         BUG_ON(!PageLocked(page));
1457         if (!page_has_buffers(page))
1458                 goto out;
1459 
1460         /*
1461          * Check for overflow
1462          */
1463         BUG_ON(stop > PAGE_SIZE || stop < length);
1464 
1465         head = page_buffers(page);
1466         bh = head;
1467         do {
1468                 unsigned int next_off = curr_off + bh->b_size;
1469                 next = bh->b_this_page;
1470 
1471                 /*
1472                  * Are we still fully in range ?
1473                  */
1474                 if (next_off > stop)
1475                         goto out;
1476 
1477                 /*
1478                  * is this block fully invalidated?
1479                  */
1480                 if (offset <= curr_off)
1481                         discard_buffer(bh);
1482                 curr_off = next_off;
1483                 bh = next;
1484         } while (bh != head);
1485 
1486         /*
1487          * We release buffers only if the entire page is being invalidated.
1488          * The get_block cached value has been unconditionally invalidated,
1489          * so real IO is not possible anymore.
1490          */
1491         if (length == PAGE_SIZE)
1492                 try_to_release_page(page, 0);
1493 out:
1494         return;
1495 }
1496 EXPORT_SYMBOL(block_invalidatepage);
1497 
1498 
1499 /*
1500  * We attach and possibly dirty the buffers atomically wrt
1501  * __set_page_dirty_buffers() via private_lock.  try_to_free_buffers
1502  * is already excluded via the page lock.
1503  */
1504 void create_empty_buffers(struct page *page,
1505                         unsigned long blocksize, unsigned long b_state)
1506 {
1507         struct buffer_head *bh, *head, *tail;
1508 
1509         head = alloc_page_buffers(page, blocksize, true);
1510         bh = head;
1511         do {
1512                 bh->b_state |= b_state;
1513                 tail = bh;
1514                 bh = bh->b_this_page;
1515         } while (bh);
1516         tail->b_this_page = head;
1517 
1518         spin_lock(&page->mapping->private_lock);
1519         if (PageUptodate(page) || PageDirty(page)) {
1520                 bh = head;
1521                 do {
1522                         if (PageDirty(page))
1523                                 set_buffer_dirty(bh);
1524                         if (PageUptodate(page))
1525                                 set_buffer_uptodate(bh);
1526                         bh = bh->b_this_page;
1527                 } while (bh != head);
1528         }
1529         attach_page_buffers(page, head);
1530         spin_unlock(&page->mapping->private_lock);
1531 }
1532 EXPORT_SYMBOL(create_empty_buffers);
1533 
1534 /**
1535  * clean_bdev_aliases: clean a range of buffers in block device
1536  * @bdev: Block device to clean buffers in
1537  * @block: Start of a range of blocks to clean
1538  * @len: Number of blocks to clean
1539  *
1540  * We are taking a range of blocks for data and we don't want writeback of any
1541  * buffer-cache aliases starting from return from this function and until the
1542  * moment when something will explicitly mark the buffer dirty (hopefully that
1543  * will not happen until we will free that block ;-) We don't even need to mark
1544  * it not-uptodate - nobody can expect anything from a newly allocated buffer
1545  * anyway. We used to use unmap_buffer() for such invalidation, but that was
1546  * wrong. We definitely don't want to mark the alias unmapped, for example - it
1547  * would confuse anyone who might pick it with bread() afterwards...
1548  *
1549  * Also..  Note that bforget() doesn't lock the buffer.  So there can be
1550  * writeout I/O going on against recently-freed buffers.  We don't wait on that
1551  * I/O in bforget() - it's more efficient to wait on the I/O only if we really
1552  * need to.  That happens here.
1553  */
1554 void clean_bdev_aliases(struct block_device *bdev, sector_t block, sector_t len)
1555 {
1556         struct inode *bd_inode = bdev->bd_inode;
1557         struct address_space *bd_mapping = bd_inode->i_mapping;
1558         struct pagevec pvec;
1559         pgoff_t index = block >> (PAGE_SHIFT - bd_inode->i_blkbits);
1560         pgoff_t end;
1561         int i, count;
1562         struct buffer_head *bh;
1563         struct buffer_head *head;
1564 
1565         end = (block + len - 1) >> (PAGE_SHIFT - bd_inode->i_blkbits);
1566         pagevec_init(&pvec);
1567         while (pagevec_lookup_range(&pvec, bd_mapping, &index, end)) {
1568                 count = pagevec_count(&pvec);
1569                 for (i = 0; i < count; i++) {
1570                         struct page *page = pvec.pages[i];
1571 
1572                         if (!page_has_buffers(page))
1573                                 continue;
1574                         /*
1575                          * We use page lock instead of bd_mapping->private_lock
1576                          * to pin buffers here since we can afford to sleep and
1577                          * it scales better than a global spinlock lock.
1578                          */
1579                         lock_page(page);
1580                         /* Recheck when the page is locked which pins bhs */
1581                         if (!page_has_buffers(page))
1582                                 goto unlock_page;
1583                         head = page_buffers(page);
1584                         bh = head;
1585                         do {
1586                                 if (!buffer_mapped(bh) || (bh->b_blocknr < block))
1587                                         goto next;
1588                                 if (bh->b_blocknr >= block + len)
1589                                         break;
1590                                 clear_buffer_dirty(bh);
1591                                 wait_on_buffer(bh);
1592                                 clear_buffer_req(bh);
1593 next:
1594                                 bh = bh->b_this_page;
1595                         } while (bh != head);
1596 unlock_page:
1597                         unlock_page(page);
1598                 }
1599                 pagevec_release(&pvec);
1600                 cond_resched();
1601                 /* End of range already reached? */
1602                 if (index > end || !index)
1603                         break;
1604         }
1605 }
1606 EXPORT_SYMBOL(clean_bdev_aliases);
1607 
1608 /*
1609  * Size is a power-of-two in the range 512..PAGE_SIZE,
1610  * and the case we care about most is PAGE_SIZE.
1611  *
1612  * So this *could* possibly be written with those
1613  * constraints in mind (relevant mostly if some
1614  * architecture has a slow bit-scan instruction)
1615  */
1616 static inline int block_size_bits(unsigned int blocksize)
1617 {
1618         return ilog2(blocksize);
1619 }
1620 
1621 static struct buffer_head *create_page_buffers(struct page *page, struct inode *inode, unsigned int b_state)
1622 {
1623         BUG_ON(!PageLocked(page));
1624 
1625         if (!page_has_buffers(page))
1626                 create_empty_buffers(page, 1 << READ_ONCE(inode->i_blkbits),
1627                                      b_state);
1628         return page_buffers(page);
1629 }
1630 
1631 /*
1632  * NOTE! All mapped/uptodate combinations are valid:
1633  *
1634  *      Mapped  Uptodate        Meaning
1635  *
1636  *      No      No              "unknown" - must do get_block()
1637  *      No      Yes             "hole" - zero-filled
1638  *      Yes     No              "allocated" - allocated on disk, not read in
1639  *      Yes     Yes             "valid" - allocated and up-to-date in memory.
1640  *
1641  * "Dirty" is valid only with the last case (mapped+uptodate).
1642  */
1643 
1644 /*
1645  * While block_write_full_page is writing back the dirty buffers under
1646  * the page lock, whoever dirtied the buffers may decide to clean them
1647  * again at any time.  We handle that by only looking at the buffer
1648  * state inside lock_buffer().
1649  *
1650  * If block_write_full_page() is called for regular writeback
1651  * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1652  * locked buffer.   This only can happen if someone has written the buffer
1653  * directly, with submit_bh().  At the address_space level PageWriteback
1654  * prevents this contention from occurring.
1655  *
1656  * If block_write_full_page() is called with wbc->sync_mode ==
1657  * WB_SYNC_ALL, the writes are posted using REQ_SYNC; this
1658  * causes the writes to be flagged as synchronous writes.
1659  */
1660 int __block_write_full_page(struct inode *inode, struct page *page,
1661                         get_block_t *get_block, struct writeback_control *wbc,
1662                         bh_end_io_t *handler)
1663 {
1664         int err;
1665         sector_t block;
1666         sector_t last_block;
1667         struct buffer_head *bh, *head;
1668         unsigned int blocksize, bbits;
1669         int nr_underway = 0;
1670         int write_flags = wbc_to_write_flags(wbc);
1671 
1672         head = create_page_buffers(page, inode,
1673                                         (1 << BH_Dirty)|(1 << BH_Uptodate));
1674 
1675         /*
1676          * Be very careful.  We have no exclusion from __set_page_dirty_buffers
1677          * here, and the (potentially unmapped) buffers may become dirty at
1678          * any time.  If a buffer becomes dirty here after we've inspected it
1679          * then we just miss that fact, and the page stays dirty.
1680          *
1681          * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
1682          * handle that here by just cleaning them.
1683          */
1684 
1685         bh = head;
1686         blocksize = bh->b_size;
1687         bbits = block_size_bits(blocksize);
1688 
1689         block = (sector_t)page->index << (PAGE_SHIFT - bbits);
1690         last_block = (i_size_read(inode) - 1) >> bbits;
1691 
1692         /*
1693          * Get all the dirty buffers mapped to disk addresses and
1694          * handle any aliases from the underlying blockdev's mapping.
1695          */
1696         do {
1697                 if (block > last_block) {
1698                         /*
1699                          * mapped buffers outside i_size will occur, because
1700                          * this page can be outside i_size when there is a
1701                          * truncate in progress.
1702                          */
1703                         /*
1704                          * The buffer was zeroed by block_write_full_page()
1705                          */
1706                         clear_buffer_dirty(bh);
1707                         set_buffer_uptodate(bh);
1708                 } else if ((!buffer_mapped(bh) || buffer_delay(bh)) &&
1709                            buffer_dirty(bh)) {
1710                         WARN_ON(bh->b_size != blocksize);
1711                         err = get_block(inode, block, bh, 1);
1712                         if (err)
1713                                 goto recover;
1714                         clear_buffer_delay(bh);
1715                         if (buffer_new(bh)) {
1716                                 /* blockdev mappings never come here */
1717                                 clear_buffer_new(bh);
1718                                 clean_bdev_bh_alias(bh);
1719                         }
1720                 }
1721                 bh = bh->b_this_page;
1722                 block++;
1723         } while (bh != head);
1724 
1725         do {
1726                 if (!buffer_mapped(bh))
1727                         continue;
1728                 /*
1729                  * If it's a fully non-blocking write attempt and we cannot
1730                  * lock the buffer then redirty the page.  Note that this can
1731                  * potentially cause a busy-wait loop from writeback threads
1732                  * and kswapd activity, but those code paths have their own
1733                  * higher-level throttling.
1734                  */
1735                 if (wbc->sync_mode != WB_SYNC_NONE) {
1736                         lock_buffer(bh);
1737                 } else if (!trylock_buffer(bh)) {
1738                         redirty_page_for_writepage(wbc, page);
1739                         continue;
1740                 }
1741                 if (test_clear_buffer_dirty(bh)) {
1742                         mark_buffer_async_write_endio(bh, handler);
1743                 } else {
1744                         unlock_buffer(bh);
1745                 }
1746         } while ((bh = bh->b_this_page) != head);
1747 
1748         /*
1749          * The page and its buffers are protected by PageWriteback(), so we can
1750          * drop the bh refcounts early.
1751          */
1752         BUG_ON(PageWriteback(page));
1753         set_page_writeback(page);
1754 
1755         do {
1756                 struct buffer_head *next = bh->b_this_page;
1757                 if (buffer_async_write(bh)) {
1758                         submit_bh_wbc(REQ_OP_WRITE, write_flags, bh,
1759                                         inode->i_write_hint, wbc);
1760                         nr_underway++;
1761                 }
1762                 bh = next;
1763         } while (bh != head);
1764         unlock_page(page);
1765 
1766         err = 0;
1767 done:
1768         if (nr_underway == 0) {
1769                 /*
1770                  * The page was marked dirty, but the buffers were
1771                  * clean.  Someone wrote them back by hand with
1772                  * ll_rw_block/submit_bh.  A rare case.
1773                  */
1774                 end_page_writeback(page);
1775 
1776                 /*
1777                  * The page and buffer_heads can be released at any time from
1778                  * here on.
1779                  */
1780         }
1781         return err;
1782 
1783 recover:
1784         /*
1785          * ENOSPC, or some other error.  We may already have added some
1786          * blocks to the file, so we need to write these out to avoid
1787          * exposing stale data.
1788          * The page is currently locked and not marked for writeback
1789          */
1790         bh = head;
1791         /* Recovery: lock and submit the mapped buffers */
1792         do {
1793                 if (buffer_mapped(bh) && buffer_dirty(bh) &&
1794                     !buffer_delay(bh)) {
1795                         lock_buffer(bh);
1796                         mark_buffer_async_write_endio(bh, handler);
1797                 } else {
1798                         /*
1799                          * The buffer may have been set dirty during
1800                          * attachment to a dirty page.
1801                          */
1802                         clear_buffer_dirty(bh);
1803                 }
1804         } while ((bh = bh->b_this_page) != head);
1805         SetPageError(page);
1806         BUG_ON(PageWriteback(page));
1807         mapping_set_error(page->mapping, err);
1808         set_page_writeback(page);
1809         do {
1810                 struct buffer_head *next = bh->b_this_page;
1811                 if (buffer_async_write(bh)) {
1812                         clear_buffer_dirty(bh);
1813                         submit_bh_wbc(REQ_OP_WRITE, write_flags, bh,
1814                                         inode->i_write_hint, wbc);
1815                         nr_underway++;
1816                 }
1817                 bh = next;
1818         } while (bh != head);
1819         unlock_page(page);
1820         goto done;
1821 }
1822 EXPORT_SYMBOL(__block_write_full_page);
1823 
1824 /*
1825  * If a page has any new buffers, zero them out here, and mark them uptodate
1826  * and dirty so they'll be written out (in order to prevent uninitialised
1827  * block data from leaking). And clear the new bit.
1828  */
1829 void page_zero_new_buffers(struct page *page, unsigned from, unsigned to)
1830 {
1831         unsigned int block_start, block_end;
1832         struct buffer_head *head, *bh;
1833 
1834         BUG_ON(!PageLocked(page));
1835         if (!page_has_buffers(page))
1836                 return;
1837 
1838         bh = head = page_buffers(page);
1839         block_start = 0;
1840         do {
1841                 block_end = block_start + bh->b_size;
1842 
1843                 if (buffer_new(bh)) {
1844                         if (block_end > from && block_start < to) {
1845                                 if (!PageUptodate(page)) {
1846                                         unsigned start, size;
1847 
1848                                         start = max(from, block_start);
1849                                         size = min(to, block_end) - start;
1850 
1851                                         zero_user(page, start, size);
1852                                         set_buffer_uptodate(bh);
1853                                 }
1854 
1855                                 clear_buffer_new(bh);
1856                                 mark_buffer_dirty(bh);
1857                         }
1858                 }
1859 
1860                 block_start = block_end;
1861                 bh = bh->b_this_page;
1862         } while (bh != head);
1863 }
1864 EXPORT_SYMBOL(page_zero_new_buffers);
1865 
1866 static void
1867 iomap_to_bh(struct inode *inode, sector_t block, struct buffer_head *bh,
1868                 struct iomap *iomap)
1869 {
1870         loff_t offset = block << inode->i_blkbits;
1871 
1872         bh->b_bdev = iomap->bdev;
1873 
1874         /*
1875          * Block points to offset in file we need to map, iomap contains
1876          * the offset at which the map starts. If the map ends before the
1877          * current block, then do not map the buffer and let the caller
1878          * handle it.
1879          */
1880         BUG_ON(offset >= iomap->offset + iomap->length);
1881 
1882         switch (iomap->type) {
1883         case IOMAP_HOLE:
1884                 /*
1885                  * If the buffer is not up to date or beyond the current EOF,
1886                  * we need to mark it as new to ensure sub-block zeroing is
1887                  * executed if necessary.
1888                  */
1889                 if (!buffer_uptodate(bh) ||
1890                     (offset >= i_size_read(inode)))
1891                         set_buffer_new(bh);
1892                 break;
1893         case IOMAP_DELALLOC:
1894                 if (!buffer_uptodate(bh) ||
1895                     (offset >= i_size_read(inode)))
1896                         set_buffer_new(bh);
1897                 set_buffer_uptodate(bh);
1898                 set_buffer_mapped(bh);
1899                 set_buffer_delay(bh);
1900                 break;
1901         case IOMAP_UNWRITTEN:
1902                 /*
1903                  * For unwritten regions, we always need to ensure that
1904                  * sub-block writes cause the regions in the block we are not
1905                  * writing to are zeroed. Set the buffer as new to ensure this.
1906                  */
1907                 set_buffer_new(bh);
1908                 set_buffer_unwritten(bh);
1909                 /* FALLTHRU */
1910         case IOMAP_MAPPED:
1911                 if (offset >= i_size_read(inode))
1912                         set_buffer_new(bh);
1913                 bh->b_blocknr = (iomap->addr + offset - iomap->offset) >>
1914                                 inode->i_blkbits;
1915                 set_buffer_mapped(bh);
1916                 break;
1917         }
1918 }
1919 
1920 int __block_write_begin_int(struct page *page, loff_t pos, unsigned len,
1921                 get_block_t *get_block, struct iomap *iomap)
1922 {
1923         unsigned from = pos & (PAGE_SIZE - 1);
1924         unsigned to = from + len;
1925         struct inode *inode = page->mapping->host;
1926         unsigned block_start, block_end;
1927         sector_t block;
1928         int err = 0;
1929         unsigned blocksize, bbits;
1930         struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
1931 
1932         BUG_ON(!PageLocked(page));
1933         BUG_ON(from > PAGE_SIZE);
1934         BUG_ON(to > PAGE_SIZE);
1935         BUG_ON(from > to);
1936 
1937         head = create_page_buffers(page, inode, 0);
1938         blocksize = head->b_size;
1939         bbits = block_size_bits(blocksize);
1940 
1941         block = (sector_t)page->index << (PAGE_SHIFT - bbits);
1942 
1943         for(bh = head, block_start = 0; bh != head || !block_start;
1944             block++, block_start=block_end, bh = bh->b_this_page) {
1945                 block_end = block_start + blocksize;
1946                 if (block_end <= from || block_start >= to) {
1947                         if (PageUptodate(page)) {
1948                                 if (!buffer_uptodate(bh))
1949                                         set_buffer_uptodate(bh);
1950                         }
1951                         continue;
1952                 }
1953                 if (buffer_new(bh))
1954                         clear_buffer_new(bh);
1955                 if (!buffer_mapped(bh)) {
1956                         WARN_ON(bh->b_size != blocksize);
1957                         if (get_block) {
1958                                 err = get_block(inode, block, bh, 1);
1959                                 if (err)
1960                                         break;
1961                         } else {
1962                                 iomap_to_bh(inode, block, bh, iomap);
1963                         }
1964 
1965                         if (buffer_new(bh)) {
1966                                 clean_bdev_bh_alias(bh);
1967                                 if (PageUptodate(page)) {
1968                                         clear_buffer_new(bh);
1969                                         set_buffer_uptodate(bh);
1970                                         mark_buffer_dirty(bh);
1971                                         continue;
1972                                 }
1973                                 if (block_end > to || block_start < from)
1974                                         zero_user_segments(page,
1975                                                 to, block_end,
1976                                                 block_start, from);
1977                                 continue;
1978                         }
1979                 }
1980                 if (PageUptodate(page)) {
1981                         if (!buffer_uptodate(bh))
1982                                 set_buffer_uptodate(bh);
1983                         continue; 
1984                 }
1985                 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
1986                     !buffer_unwritten(bh) &&
1987                      (block_start < from || block_end > to)) {
1988                         ll_rw_block(REQ_OP_READ, 0, 1, &bh);
1989                         *wait_bh++=bh;
1990                 }
1991         }
1992         /*
1993          * If we issued read requests - let them complete.
1994          */
1995         while(wait_bh > wait) {
1996                 wait_on_buffer(*--wait_bh);
1997                 if (!buffer_uptodate(*wait_bh))
1998                         err = -EIO;
1999         }
2000         if (unlikely(err))
2001                 page_zero_new_buffers(page, from, to);
2002         return err;
2003 }
2004 
2005 int __block_write_begin(struct page *page, loff_t pos, unsigned len,
2006                 get_block_t *get_block)
2007 {
2008         return __block_write_begin_int(page, pos, len, get_block, NULL);
2009 }
2010 EXPORT_SYMBOL(__block_write_begin);
2011 
2012 static int __block_commit_write(struct inode *inode, struct page *page,
2013                 unsigned from, unsigned to)
2014 {
2015         unsigned block_start, block_end;
2016         int partial = 0;
2017         unsigned blocksize;
2018         struct buffer_head *bh, *head;
2019 
2020         bh = head = page_buffers(page);
2021         blocksize = bh->b_size;
2022 
2023         block_start = 0;
2024         do {
2025                 block_end = block_start + blocksize;
2026                 if (block_end <= from || block_start >= to) {
2027                         if (!buffer_uptodate(bh))
2028                                 partial = 1;
2029                 } else {
2030                         set_buffer_uptodate(bh);
2031                         mark_buffer_dirty(bh);
2032                 }
2033                 clear_buffer_new(bh);
2034 
2035                 block_start = block_end;
2036                 bh = bh->b_this_page;
2037         } while (bh != head);
2038 
2039         /*
2040          * If this is a partial write which happened to make all buffers
2041          * uptodate then we can optimize away a bogus readpage() for
2042          * the next read(). Here we 'discover' whether the page went
2043          * uptodate as a result of this (potentially partial) write.
2044          */
2045         if (!partial)
2046                 SetPageUptodate(page);
2047         return 0;
2048 }
2049 
2050 /*
2051  * block_write_begin takes care of the basic task of block allocation and
2052  * bringing partial write blocks uptodate first.
2053  *
2054  * The filesystem needs to handle block truncation upon failure.
2055  */
2056 int block_write_begin(struct address_space *mapping, loff_t pos, unsigned len,
2057                 unsigned flags, struct page **pagep, get_block_t *get_block)
2058 {
2059         pgoff_t index = pos >> PAGE_SHIFT;
2060         struct page *page;
2061         int status;
2062 
2063         page = grab_cache_page_write_begin(mapping, index, flags);
2064         if (!page)
2065                 return -ENOMEM;
2066 
2067         status = __block_write_begin(page, pos, len, get_block);
2068         if (unlikely(status)) {
2069                 unlock_page(page);
2070                 put_page(page);
2071                 page = NULL;
2072         }
2073 
2074         *pagep = page;
2075         return status;
2076 }
2077 EXPORT_SYMBOL(block_write_begin);
2078 
2079 int block_write_end(struct file *file, struct address_space *mapping,
2080                         loff_t pos, unsigned len, unsigned copied,
2081                         struct page *page, void *fsdata)
2082 {
2083         struct inode *inode = mapping->host;
2084         unsigned start;
2085 
2086         start = pos & (PAGE_SIZE - 1);
2087 
2088         if (unlikely(copied < len)) {
2089                 /*
2090                  * The buffers that were written will now be uptodate, so we
2091                  * don't have to worry about a readpage reading them and
2092                  * overwriting a partial write. However if we have encountered
2093                  * a short write and only partially written into a buffer, it
2094                  * will not be marked uptodate, so a readpage might come in and
2095                  * destroy our partial write.
2096                  *
2097                  * Do the simplest thing, and just treat any short write to a
2098                  * non uptodate page as a zero-length write, and force the
2099                  * caller to redo the whole thing.
2100                  */
2101                 if (!PageUptodate(page))
2102                         copied = 0;
2103 
2104                 page_zero_new_buffers(page, start+copied, start+len);
2105         }
2106         flush_dcache_page(page);
2107 
2108         /* This could be a short (even 0-length) commit */
2109         __block_commit_write(inode, page, start, start+copied);
2110 
2111         return copied;
2112 }
2113 EXPORT_SYMBOL(block_write_end);
2114 
2115 int generic_write_end(struct file *file, struct address_space *mapping,
2116                         loff_t pos, unsigned len, unsigned copied,
2117                         struct page *page, void *fsdata)
2118 {
2119         struct inode *inode = mapping->host;
2120         loff_t old_size = inode->i_size;
2121         int i_size_changed = 0;
2122 
2123         copied = block_write_end(file, mapping, pos, len, copied, page, fsdata);
2124 
2125         /*
2126          * No need to use i_size_read() here, the i_size
2127          * cannot change under us because we hold i_mutex.
2128          *
2129          * But it's important to update i_size while still holding page lock:
2130          * page writeout could otherwise come in and zero beyond i_size.
2131          */
2132         if (pos+copied > inode->i_size) {
2133                 i_size_write(inode, pos+copied);
2134                 i_size_changed = 1;
2135         }
2136 
2137         unlock_page(page);
2138         put_page(page);
2139 
2140         if (old_size < pos)
2141                 pagecache_isize_extended(inode, old_size, pos);
2142         /*
2143          * Don't mark the inode dirty under page lock. First, it unnecessarily
2144          * makes the holding time of page lock longer. Second, it forces lock
2145          * ordering of page lock and transaction start for journaling
2146          * filesystems.
2147          */
2148         if (i_size_changed)
2149                 mark_inode_dirty(inode);
2150 
2151         return copied;
2152 }
2153 EXPORT_SYMBOL(generic_write_end);
2154 
2155 /*
2156  * block_is_partially_uptodate checks whether buffers within a page are
2157  * uptodate or not.
2158  *
2159  * Returns true if all buffers which correspond to a file portion
2160  * we want to read are uptodate.
2161  */
2162 int block_is_partially_uptodate(struct page *page, unsigned long from,
2163                                         unsigned long count)
2164 {
2165         unsigned block_start, block_end, blocksize;
2166         unsigned to;
2167         struct buffer_head *bh, *head;
2168         int ret = 1;
2169 
2170         if (!page_has_buffers(page))
2171                 return 0;
2172 
2173         head = page_buffers(page);
2174         blocksize = head->b_size;
2175         to = min_t(unsigned, PAGE_SIZE - from, count);
2176         to = from + to;
2177         if (from < blocksize && to > PAGE_SIZE - blocksize)
2178                 return 0;
2179 
2180         bh = head;
2181         block_start = 0;
2182         do {
2183                 block_end = block_start + blocksize;
2184                 if (block_end > from && block_start < to) {
2185                         if (!buffer_uptodate(bh)) {
2186                                 ret = 0;
2187                                 break;
2188                         }
2189                         if (block_end >= to)
2190                                 break;
2191                 }
2192                 block_start = block_end;
2193                 bh = bh->b_this_page;
2194         } while (bh != head);
2195 
2196         return ret;
2197 }
2198 EXPORT_SYMBOL(block_is_partially_uptodate);
2199 
2200 /*
2201  * Generic "read page" function for block devices that have the normal
2202  * get_block functionality. This is most of the block device filesystems.
2203  * Reads the page asynchronously --- the unlock_buffer() and
2204  * set/clear_buffer_uptodate() functions propagate buffer state into the
2205  * page struct once IO has completed.
2206  */
2207 int block_read_full_page(struct page *page, get_block_t *get_block)
2208 {
2209         struct inode *inode = page->mapping->host;
2210         sector_t iblock, lblock;
2211         struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
2212         unsigned int blocksize, bbits;
2213         int nr, i;
2214         int fully_mapped = 1;
2215 
2216         head = create_page_buffers(page, inode, 0);
2217         blocksize = head->b_size;
2218         bbits = block_size_bits(blocksize);
2219 
2220         iblock = (sector_t)page->index << (PAGE_SHIFT - bbits);
2221         lblock = (i_size_read(inode)+blocksize-1) >> bbits;
2222         bh = head;
2223         nr = 0;
2224         i = 0;
2225 
2226         do {
2227                 if (buffer_uptodate(bh))
2228                         continue;
2229 
2230                 if (!buffer_mapped(bh)) {
2231                         int err = 0;
2232 
2233                         fully_mapped = 0;
2234                         if (iblock < lblock) {
2235                                 WARN_ON(bh->b_size != blocksize);
2236                                 err = get_block(inode, iblock, bh, 0);
2237                                 if (err)
2238                                         SetPageError(page);
2239                         }
2240                         if (!buffer_mapped(bh)) {
2241                                 zero_user(page, i * blocksize, blocksize);
2242                                 if (!err)
2243                                         set_buffer_uptodate(bh);
2244                                 continue;
2245                         }
2246                         /*
2247                          * get_block() might have updated the buffer
2248                          * synchronously
2249                          */
2250                         if (buffer_uptodate(bh))
2251                                 continue;
2252                 }
2253                 arr[nr++] = bh;
2254         } while (i++, iblock++, (bh = bh->b_this_page) != head);
2255 
2256         if (fully_mapped)
2257                 SetPageMappedToDisk(page);
2258 
2259         if (!nr) {
2260                 /*
2261                  * All buffers are uptodate - we can set the page uptodate
2262                  * as well. But not if get_block() returned an error.
2263                  */
2264                 if (!PageError(page))
2265                         SetPageUptodate(page);
2266                 unlock_page(page);
2267                 return 0;
2268         }
2269 
2270         /* Stage two: lock the buffers */
2271         for (i = 0; i < nr; i++) {
2272                 bh = arr[i];
2273                 lock_buffer(bh);
2274                 mark_buffer_async_read(bh);
2275         }
2276 
2277         /*
2278          * Stage 3: start the IO.  Check for uptodateness
2279          * inside the buffer lock in case another process reading
2280          * the underlying blockdev brought it uptodate (the sct fix).
2281          */
2282         for (i = 0; i < nr; i++) {
2283                 bh = arr[i];
2284                 if (buffer_uptodate(bh))
2285                         end_buffer_async_read(bh, 1);
2286                 else
2287                         submit_bh(REQ_OP_READ, 0, bh);
2288         }
2289         return 0;
2290 }
2291 EXPORT_SYMBOL(block_read_full_page);
2292 
2293 /* utility function for filesystems that need to do work on expanding
2294  * truncates.  Uses filesystem pagecache writes to allow the filesystem to
2295  * deal with the hole.  
2296  */
2297 int generic_cont_expand_simple(struct inode *inode, loff_t size)
2298 {
2299         struct address_space *mapping = inode->i_mapping;
2300         struct page *page;
2301         void *fsdata;
2302         int err;
2303 
2304         err = inode_newsize_ok(inode, size);
2305         if (err)
2306                 goto out;
2307 
2308         err = pagecache_write_begin(NULL, mapping, size, 0,
2309                                     AOP_FLAG_CONT_EXPAND, &page, &fsdata);
2310         if (err)
2311                 goto out;
2312 
2313         err = pagecache_write_end(NULL, mapping, size, 0, 0, page, fsdata);
2314         BUG_ON(err > 0);
2315 
2316 out:
2317         return err;
2318 }
2319 EXPORT_SYMBOL(generic_cont_expand_simple);
2320 
2321 static int cont_expand_zero(struct file *file, struct address_space *mapping,
2322                             loff_t pos, loff_t *bytes)
2323 {
2324         struct inode *inode = mapping->host;
2325         unsigned int blocksize = i_blocksize(inode);
2326         struct page *page;
2327         void *fsdata;
2328         pgoff_t index, curidx;
2329         loff_t curpos;
2330         unsigned zerofrom, offset, len;
2331         int err = 0;
2332 
2333         index = pos >> PAGE_SHIFT;
2334         offset = pos & ~PAGE_MASK;
2335 
2336         while (index > (curidx = (curpos = *bytes)>>PAGE_SHIFT)) {
2337                 zerofrom = curpos & ~PAGE_MASK;
2338                 if (zerofrom & (blocksize-1)) {
2339                         *bytes |= (blocksize-1);
2340                         (*bytes)++;
2341                 }
2342                 len = PAGE_SIZE - zerofrom;
2343 
2344                 err = pagecache_write_begin(file, mapping, curpos, len, 0,
2345                                             &page, &fsdata);
2346                 if (err)
2347                         goto out;
2348                 zero_user(page, zerofrom, len);
2349                 err = pagecache_write_end(file, mapping, curpos, len, len,
2350                                                 page, fsdata);
2351                 if (err < 0)
2352                         goto out;
2353                 BUG_ON(err != len);
2354                 err = 0;
2355 
2356                 balance_dirty_pages_ratelimited(mapping);
2357 
2358                 if (unlikely(fatal_signal_pending(current))) {
2359                         err = -EINTR;
2360                         goto out;
2361                 }
2362         }
2363 
2364         /* page covers the boundary, find the boundary offset */
2365         if (index == curidx) {
2366                 zerofrom = curpos & ~PAGE_MASK;
2367                 /* if we will expand the thing last block will be filled */
2368                 if (offset <= zerofrom) {
2369                         goto out;
2370                 }
2371                 if (zerofrom & (blocksize-1)) {
2372                         *bytes |= (blocksize-1);
2373                         (*bytes)++;
2374                 }
2375                 len = offset - zerofrom;
2376 
2377                 err = pagecache_write_begin(file, mapping, curpos, len, 0,
2378                                             &page, &fsdata);
2379                 if (err)
2380                         goto out;
2381                 zero_user(page, zerofrom, len);
2382                 err = pagecache_write_end(file, mapping, curpos, len, len,
2383                                                 page, fsdata);
2384                 if (err < 0)
2385                         goto out;
2386                 BUG_ON(err != len);
2387                 err = 0;
2388         }
2389 out:
2390         return err;
2391 }
2392 
2393 /*
2394  * For moronic filesystems that do not allow holes in file.
2395  * We may have to extend the file.
2396  */
2397 int cont_write_begin(struct file *file, struct address_space *mapping,
2398                         loff_t pos, unsigned len, unsigned flags,
2399                         struct page **pagep, void **fsdata,
2400                         get_block_t *get_block, loff_t *bytes)
2401 {
2402         struct inode *inode = mapping->host;
2403         unsigned int blocksize = i_blocksize(inode);
2404         unsigned int zerofrom;
2405         int err;
2406 
2407         err = cont_expand_zero(file, mapping, pos, bytes);
2408         if (err)
2409                 return err;
2410 
2411         zerofrom = *bytes & ~PAGE_MASK;
2412         if (pos+len > *bytes && zerofrom & (blocksize-1)) {
2413                 *bytes |= (blocksize-1);
2414                 (*bytes)++;
2415         }
2416 
2417         return block_write_begin(mapping, pos, len, flags, pagep, get_block);
2418 }
2419 EXPORT_SYMBOL(cont_write_begin);
2420 
2421 int block_commit_write(struct page *page, unsigned from, unsigned to)
2422 {
2423         struct inode *inode = page->mapping->host;
2424         __block_commit_write(inode,page,from,to);
2425         return 0;
2426 }
2427 EXPORT_SYMBOL(block_commit_write);
2428 
2429 /*
2430  * block_page_mkwrite() is not allowed to change the file size as it gets
2431  * called from a page fault handler when a page is first dirtied. Hence we must
2432  * be careful to check for EOF conditions here. We set the page up correctly
2433  * for a written page which means we get ENOSPC checking when writing into
2434  * holes and correct delalloc and unwritten extent mapping on filesystems that
2435  * support these features.
2436  *
2437  * We are not allowed to take the i_mutex here so we have to play games to
2438  * protect against truncate races as the page could now be beyond EOF.  Because
2439  * truncate writes the inode size before removing pages, once we have the
2440  * page lock we can determine safely if the page is beyond EOF. If it is not
2441  * beyond EOF, then the page is guaranteed safe against truncation until we
2442  * unlock the page.
2443  *
2444  * Direct callers of this function should protect against filesystem freezing
2445  * using sb_start_pagefault() - sb_end_pagefault() functions.
2446  */
2447 int block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
2448                          get_block_t get_block)
2449 {
2450         struct page *page = vmf->page;
2451         struct inode *inode = file_inode(vma->vm_file);
2452         unsigned long end;
2453         loff_t size;
2454         int ret;
2455 
2456         lock_page(page);
2457         size = i_size_read(inode);
2458         if ((page->mapping != inode->i_mapping) ||
2459             (page_offset(page) > size)) {
2460                 /* We overload EFAULT to mean page got truncated */
2461                 ret = -EFAULT;
2462                 goto out_unlock;
2463         }
2464 
2465         /* page is wholly or partially inside EOF */
2466         if (((page->index + 1) << PAGE_SHIFT) > size)
2467                 end = size & ~PAGE_MASK;
2468         else
2469                 end = PAGE_SIZE;
2470 
2471         ret = __block_write_begin(page, 0, end, get_block);
2472         if (!ret)
2473                 ret = block_commit_write(page, 0, end);
2474 
2475         if (unlikely(ret < 0))
2476                 goto out_unlock;
2477         set_page_dirty(page);
2478         wait_for_stable_page(page);
2479         return 0;
2480 out_unlock:
2481         unlock_page(page);
2482         return ret;
2483 }
2484 EXPORT_SYMBOL(block_page_mkwrite);
2485 
2486 /*
2487  * nobh_write_begin()'s prereads are special: the buffer_heads are freed
2488  * immediately, while under the page lock.  So it needs a special end_io
2489  * handler which does not touch the bh after unlocking it.
2490  */
2491 static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
2492 {
2493         __end_buffer_read_notouch(bh, uptodate);
2494 }
2495 
2496 /*
2497  * Attach the singly-linked list of buffers created by nobh_write_begin, to
2498  * the page (converting it to circular linked list and taking care of page
2499  * dirty races).
2500  */
2501 static void attach_nobh_buffers(struct page *page, struct buffer_head *head)
2502 {
2503         struct buffer_head *bh;
2504 
2505         BUG_ON(!PageLocked(page));
2506 
2507         spin_lock(&page->mapping->private_lock);
2508         bh = head;
2509         do {
2510                 if (PageDirty(page))
2511                         set_buffer_dirty(bh);
2512                 if (!bh->b_this_page)
2513                         bh->b_this_page = head;
2514                 bh = bh->b_this_page;
2515         } while (bh != head);
2516         attach_page_buffers(page, head);
2517         spin_unlock(&page->mapping->private_lock);
2518 }
2519 
2520 /*
2521  * On entry, the page is fully not uptodate.
2522  * On exit the page is fully uptodate in the areas outside (from,to)
2523  * The filesystem needs to handle block truncation upon failure.
2524  */
2525 int nobh_write_begin(struct address_space *mapping,
2526                         loff_t pos, unsigned len, unsigned flags,
2527                         struct page **pagep, void **fsdata,
2528                         get_block_t *get_block)
2529 {
2530         struct inode *inode = mapping->host;
2531         const unsigned blkbits = inode->i_blkbits;
2532         const unsigned blocksize = 1 << blkbits;
2533         struct buffer_head *head, *bh;
2534         struct page *page;
2535         pgoff_t index;
2536         unsigned from, to;
2537         unsigned block_in_page;
2538         unsigned block_start, block_end;
2539         sector_t block_in_file;
2540         int nr_reads = 0;
2541         int ret = 0;
2542         int is_mapped_to_disk = 1;
2543 
2544         index = pos >> PAGE_SHIFT;
2545         from = pos & (PAGE_SIZE - 1);
2546         to = from + len;
2547 
2548         page = grab_cache_page_write_begin(mapping, index, flags);
2549         if (!page)
2550                 return -ENOMEM;
2551         *pagep = page;
2552         *fsdata = NULL;
2553 
2554         if (page_has_buffers(page)) {
2555                 ret = __block_write_begin(page, pos, len, get_block);
2556                 if (unlikely(ret))
2557                         goto out_release;
2558                 return ret;
2559         }
2560 
2561         if (PageMappedToDisk(page))
2562                 return 0;
2563 
2564         /*
2565          * Allocate buffers so that we can keep track of state, and potentially
2566          * attach them to the page if an error occurs. In the common case of
2567          * no error, they will just be freed again without ever being attached
2568          * to the page (which is all OK, because we're under the page lock).
2569          *
2570          * Be careful: the buffer linked list is a NULL terminated one, rather
2571          * than the circular one we're used to.
2572          */
2573         head = alloc_page_buffers(page, blocksize, false);
2574         if (!head) {
2575                 ret = -ENOMEM;
2576                 goto out_release;
2577         }
2578 
2579         block_in_file = (sector_t)page->index << (PAGE_SHIFT - blkbits);
2580 
2581         /*
2582          * We loop across all blocks in the page, whether or not they are
2583          * part of the affected region.  This is so we can discover if the
2584          * page is fully mapped-to-disk.
2585          */
2586         for (block_start = 0, block_in_page = 0, bh = head;
2587                   block_start < PAGE_SIZE;
2588                   block_in_page++, block_start += blocksize, bh = bh->b_this_page) {
2589                 int create;
2590 
2591                 block_end = block_start + blocksize;
2592                 bh->b_state = 0;
2593                 create = 1;
2594                 if (block_start >= to)
2595                         create = 0;
2596                 ret = get_block(inode, block_in_file + block_in_page,
2597                                         bh, create);
2598                 if (ret)
2599                         goto failed;
2600                 if (!buffer_mapped(bh))
2601                         is_mapped_to_disk = 0;
2602                 if (buffer_new(bh))
2603                         clean_bdev_bh_alias(bh);
2604                 if (PageUptodate(page)) {
2605                         set_buffer_uptodate(bh);
2606                         continue;
2607                 }
2608                 if (buffer_new(bh) || !buffer_mapped(bh)) {
2609                         zero_user_segments(page, block_start, from,
2610                                                         to, block_end);
2611                         continue;
2612                 }
2613                 if (buffer_uptodate(bh))
2614                         continue;       /* reiserfs does this */
2615                 if (block_start < from || block_end > to) {
2616                         lock_buffer(bh);
2617                         bh->b_end_io = end_buffer_read_nobh;
2618                         submit_bh(REQ_OP_READ, 0, bh);
2619                         nr_reads++;
2620                 }
2621         }
2622 
2623         if (nr_reads) {
2624                 /*
2625                  * The page is locked, so these buffers are protected from
2626                  * any VM or truncate activity.  Hence we don't need to care
2627                  * for the buffer_head refcounts.
2628                  */
2629                 for (bh = head; bh; bh = bh->b_this_page) {
2630                         wait_on_buffer(bh);
2631                         if (!buffer_uptodate(bh))
2632                                 ret = -EIO;
2633                 }
2634                 if (ret)
2635                         goto failed;
2636         }
2637 
2638         if (is_mapped_to_disk)
2639                 SetPageMappedToDisk(page);
2640 
2641         *fsdata = head; /* to be released by nobh_write_end */
2642 
2643         return 0;
2644 
2645 failed:
2646         BUG_ON(!ret);
2647         /*
2648          * Error recovery is a bit difficult. We need to zero out blocks that
2649          * were newly allocated, and dirty them to ensure they get written out.
2650          * Buffers need to be attached to the page at this point, otherwise
2651          * the handling of potential IO errors during writeout would be hard
2652          * (could try doing synchronous writeout, but what if that fails too?)
2653          */
2654         attach_nobh_buffers(page, head);
2655         page_zero_new_buffers(page, from, to);
2656 
2657 out_release:
2658         unlock_page(page);
2659         put_page(page);
2660         *pagep = NULL;
2661 
2662         return ret;
2663 }
2664 EXPORT_SYMBOL(nobh_write_begin);
2665 
2666 int nobh_write_end(struct file *file, struct address_space *mapping,
2667                         loff_t pos, unsigned len, unsigned copied,
2668                         struct page *page, void *fsdata)
2669 {
2670         struct inode *inode = page->mapping->host;
2671         struct buffer_head *head = fsdata;
2672         struct buffer_head *bh;
2673         BUG_ON(fsdata != NULL && page_has_buffers(page));
2674 
2675         if (unlikely(copied < len) && head)
2676                 attach_nobh_buffers(page, head);
2677         if (page_has_buffers(page))
2678                 return generic_write_end(file, mapping, pos, len,
2679                                         copied, page, fsdata);
2680 
2681         SetPageUptodate(page);
2682         set_page_dirty(page);
2683         if (pos+copied > inode->i_size) {
2684                 i_size_write(inode, pos+copied);
2685                 mark_inode_dirty(inode);
2686         }
2687 
2688         unlock_page(page);
2689         put_page(page);
2690 
2691         while (head) {
2692                 bh = head;
2693                 head = head->b_this_page;
2694                 free_buffer_head(bh);
2695         }
2696 
2697         return copied;
2698 }
2699 EXPORT_SYMBOL(nobh_write_end);
2700 
2701 /*
2702  * nobh_writepage() - based on block_full_write_page() except
2703  * that it tries to operate without attaching bufferheads to
2704  * the page.
2705  */
2706 int nobh_writepage(struct page *page, get_block_t *get_block,
2707                         struct writeback_control *wbc)
2708 {
2709         struct inode * const inode = page->mapping->host;
2710         loff_t i_size = i_size_read(inode);
2711         const pgoff_t end_index = i_size >> PAGE_SHIFT;
2712         unsigned offset;
2713         int ret;
2714 
2715         /* Is the page fully inside i_size? */
2716         if (page->index < end_index)
2717                 goto out;
2718 
2719         /* Is the page fully outside i_size? (truncate in progress) */
2720         offset = i_size & (PAGE_SIZE-1);
2721         if (page->index >= end_index+1 || !offset) {
2722                 /*
2723                  * The page may have dirty, unmapped buffers.  For example,
2724                  * they may have been added in ext3_writepage().  Make them
2725                  * freeable here, so the page does not leak.
2726                  */
2727 #if 0
2728                 /* Not really sure about this  - do we need this ? */
2729                 if (page->mapping->a_ops->invalidatepage)
2730                         page->mapping->a_ops->invalidatepage(page, offset);
2731 #endif
2732                 unlock_page(page);
2733                 return 0; /* don't care */
2734         }
2735 
2736         /*
2737          * The page straddles i_size.  It must be zeroed out on each and every
2738          * writepage invocation because it may be mmapped.  "A file is mapped
2739          * in multiples of the page size.  For a file that is not a multiple of
2740          * the  page size, the remaining memory is zeroed when mapped, and
2741          * writes to that region are not written out to the file."
2742          */
2743         zero_user_segment(page, offset, PAGE_SIZE);
2744 out:
2745         ret = mpage_writepage(page, get_block, wbc);
2746         if (ret == -EAGAIN)
2747                 ret = __block_write_full_page(inode, page, get_block, wbc,
2748                                               end_buffer_async_write);
2749         return ret;
2750 }
2751 EXPORT_SYMBOL(nobh_writepage);
2752 
2753 int nobh_truncate_page(struct address_space *mapping,
2754                         loff_t from, get_block_t *get_block)
2755 {
2756         pgoff_t index = from >> PAGE_SHIFT;
2757         unsigned offset = from & (PAGE_SIZE-1);
2758         unsigned blocksize;
2759         sector_t iblock;
2760         unsigned length, pos;
2761         struct inode *inode = mapping->host;
2762         struct page *page;
2763         struct buffer_head map_bh;
2764         int err;
2765 
2766         blocksize = i_blocksize(inode);
2767         length = offset & (blocksize - 1);
2768 
2769         /* Block boundary? Nothing to do */
2770         if (!length)
2771                 return 0;
2772 
2773         length = blocksize - length;
2774         iblock = (sector_t)index << (PAGE_SHIFT - inode->i_blkbits);
2775 
2776         page = grab_cache_page(mapping, index);
2777         err = -ENOMEM;
2778         if (!page)
2779                 goto out;
2780 
2781         if (page_has_buffers(page)) {
2782 has_buffers:
2783                 unlock_page(page);
2784                 put_page(page);
2785                 return block_truncate_page(mapping, from, get_block);
2786         }
2787 
2788         /* Find the buffer that contains "offset" */
2789         pos = blocksize;
2790         while (offset >= pos) {
2791                 iblock++;
2792                 pos += blocksize;
2793         }
2794 
2795         map_bh.b_size = blocksize;
2796         map_bh.b_state = 0;
2797         err = get_block(inode, iblock, &map_bh, 0);
2798         if (err)
2799                 goto unlock;
2800         /* unmapped? It's a hole - nothing to do */
2801         if (!buffer_mapped(&map_bh))
2802                 goto unlock;
2803 
2804         /* Ok, it's mapped. Make sure it's up-to-date */
2805         if (!PageUptodate(page)) {
2806                 err = mapping->a_ops->readpage(NULL, page);
2807                 if (err) {
2808                         put_page(page);
2809                         goto out;
2810                 }
2811                 lock_page(page);
2812                 if (!PageUptodate(page)) {
2813                         err = -EIO;
2814                         goto unlock;
2815                 }
2816                 if (page_has_buffers(page))
2817                         goto has_buffers;
2818         }
2819         zero_user(page, offset, length);
2820         set_page_dirty(page);
2821         err = 0;
2822 
2823 unlock:
2824         unlock_page(page);
2825         put_page(page);
2826 out:
2827         return err;
2828 }
2829 EXPORT_SYMBOL(nobh_truncate_page);
2830 
2831 int block_truncate_page(struct address_space *mapping,
2832                         loff_t from, get_block_t *get_block)
2833 {
2834         pgoff_t index = from >> PAGE_SHIFT;
2835         unsigned offset = from & (PAGE_SIZE-1);
2836         unsigned blocksize;
2837         sector_t iblock;
2838         unsigned length, pos;
2839         struct inode *inode = mapping->host;
2840         struct page *page;
2841         struct buffer_head *bh;
2842         int err;
2843 
2844         blocksize = i_blocksize(inode);
2845         length = offset & (blocksize - 1);
2846 
2847         /* Block boundary? Nothing to do */
2848         if (!length)
2849                 return 0;
2850 
2851         length = blocksize - length;
2852         iblock = (sector_t)index << (PAGE_SHIFT - inode->i_blkbits);
2853         
2854         page = grab_cache_page(mapping, index);
2855         err = -ENOMEM;
2856         if (!page)
2857                 goto out;
2858 
2859         if (!page_has_buffers(page))
2860                 create_empty_buffers(page, blocksize, 0);
2861 
2862         /* Find the buffer that contains "offset" */
2863         bh = page_buffers(page);
2864         pos = blocksize;
2865         while (offset >= pos) {
2866                 bh = bh->b_this_page;
2867                 iblock++;
2868                 pos += blocksize;
2869         }
2870 
2871         err = 0;
2872         if (!buffer_mapped(bh)) {
2873                 WARN_ON(bh->b_size != blocksize);
2874                 err = get_block(inode, iblock, bh, 0);
2875                 if (err)
2876                         goto unlock;
2877                 /* unmapped? It's a hole - nothing to do */
2878                 if (!buffer_mapped(bh))
2879                         goto unlock;
2880         }
2881 
2882         /* Ok, it's mapped. Make sure it's up-to-date */
2883         if (PageUptodate(page))
2884                 set_buffer_uptodate(bh);
2885 
2886         if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) {
2887                 err = -EIO;
2888                 ll_rw_block(REQ_OP_READ, 0, 1, &bh);
2889                 wait_on_buffer(bh);
2890                 /* Uhhuh. Read error. Complain and punt. */
2891                 if (!buffer_uptodate(bh))
2892                         goto unlock;
2893         }
2894 
2895         zero_user(page, offset, length);
2896         mark_buffer_dirty(bh);
2897         err = 0;
2898 
2899 unlock:
2900         unlock_page(page);
2901         put_page(page);
2902 out:
2903         return err;
2904 }
2905 EXPORT_SYMBOL(block_truncate_page);
2906 
2907 /*
2908  * The generic ->writepage function for buffer-backed address_spaces
2909  */
2910 int block_write_full_page(struct page *page, get_block_t *get_block,
2911                         struct writeback_control *wbc)
2912 {
2913         struct inode * const inode = page->mapping->host;
2914         loff_t i_size = i_size_read(inode);
2915         const pgoff_t end_index = i_size >> PAGE_SHIFT;
2916         unsigned offset;
2917 
2918         /* Is the page fully inside i_size? */
2919         if (page->index < end_index)
2920                 return __block_write_full_page(inode, page, get_block, wbc,
2921                                                end_buffer_async_write);
2922 
2923         /* Is the page fully outside i_size? (truncate in progress) */
2924         offset = i_size & (PAGE_SIZE-1);
2925         if (page->index >= end_index+1 || !offset) {
2926                 /*
2927                  * The page may have dirty, unmapped buffers.  For example,
2928                  * they may have been added in ext3_writepage().  Make them
2929                  * freeable here, so the page does not leak.
2930                  */
2931                 do_invalidatepage(page, 0, PAGE_SIZE);
2932                 unlock_page(page);
2933                 return 0; /* don't care */
2934         }
2935 
2936         /*
2937          * The page straddles i_size.  It must be zeroed out on each and every
2938          * writepage invocation because it may be mmapped.  "A file is mapped
2939          * in multiples of the page size.  For a file that is not a multiple of
2940          * the  page size, the remaining memory is zeroed when mapped, and
2941          * writes to that region are not written out to the file."
2942          */
2943         zero_user_segment(page, offset, PAGE_SIZE);
2944         return __block_write_full_page(inode, page, get_block, wbc,
2945                                                         end_buffer_async_write);
2946 }
2947 EXPORT_SYMBOL(block_write_full_page);
2948 
2949 sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
2950                             get_block_t *get_block)
2951 {
2952         struct inode *inode = mapping->host;
2953         struct buffer_head tmp = {
2954                 .b_size = i_blocksize(inode),
2955         };
2956 
2957         get_block(inode, block, &tmp, 0);
2958         return tmp.b_blocknr;
2959 }
2960 EXPORT_SYMBOL(generic_block_bmap);
2961 
2962 static void end_bio_bh_io_sync(struct bio *bio)
2963 {
2964         struct buffer_head *bh = bio->bi_private;
2965 
2966         if (unlikely(bio_flagged(bio, BIO_QUIET)))
2967                 set_bit(BH_Quiet, &bh->b_state);
2968 
2969         bh->b_end_io(bh, !bio->bi_status);
2970         bio_put(bio);
2971 }
2972 
2973 /*
2974  * This allows us to do IO even on the odd last sectors
2975  * of a device, even if the block size is some multiple
2976  * of the physical sector size.
2977  *
2978  * We'll just truncate the bio to the size of the device,
2979  * and clear the end of the buffer head manually.
2980  *
2981  * Truly out-of-range accesses will turn into actual IO
2982  * errors, this only handles the "we need to be able to
2983  * do IO at the final sector" case.
2984  */
2985 void guard_bio_eod(int op, struct bio *bio)
2986 {
2987         sector_t maxsector;
2988         struct bio_vec *bvec = bio_last_bvec_all(bio);
2989         unsigned truncated_bytes;
2990         struct hd_struct *part;
2991 
2992         rcu_read_lock();
2993         part = __disk_get_part(bio->bi_disk, bio->bi_partno);
2994         if (part)
2995                 maxsector = part_nr_sects_read(part);
2996         else
2997                 maxsector = get_capacity(bio->bi_disk);
2998         rcu_read_unlock();
2999 
3000         if (!maxsector)
3001                 return;
3002 
3003         /*
3004          * If the *whole* IO is past the end of the device,
3005          * let it through, and the IO layer will turn it into
3006          * an EIO.
3007          */
3008         if (unlikely(bio->bi_iter.bi_sector >= maxsector))
3009                 return;
3010 
3011         maxsector -= bio->bi_iter.bi_sector;
3012         if (likely((bio->bi_iter.bi_size >> 9) <= maxsector))
3013                 return;
3014 
3015         /* Uhhuh. We've got a bio that straddles the device size! */
3016         truncated_bytes = bio->bi_iter.bi_size - (maxsector << 9);
3017 
3018         /* Truncate the bio.. */
3019         bio->bi_iter.bi_size -= truncated_bytes;
3020         bvec->bv_len -= truncated_bytes;
3021 
3022         /* ..and clear the end of the buffer for reads */
3023         if (op == REQ_OP_READ) {
3024                 zero_user(bvec->bv_page, bvec->bv_offset + bvec->bv_len,
3025                                 truncated_bytes);
3026         }
3027 }
3028 
3029 static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh,
3030                          enum rw_hint write_hint, struct writeback_control *wbc)
3031 {
3032         struct bio *bio;
3033 
3034         BUG_ON(!buffer_locked(bh));
3035         BUG_ON(!buffer_mapped(bh));
3036         BUG_ON(!bh->b_end_io);
3037         BUG_ON(buffer_delay(bh));
3038         BUG_ON(buffer_unwritten(bh));
3039 
3040         /*
3041          * Only clear out a write error when rewriting
3042          */
3043         if (test_set_buffer_req(bh) && (op == REQ_OP_WRITE))
3044                 clear_buffer_write_io_error(bh);
3045 
3046         /*
3047          * from here on down, it's all bio -- do the initial mapping,
3048          * submit_bio -> generic_make_request may further map this bio around
3049          */
3050         bio = bio_alloc(GFP_NOIO, 1);
3051 
3052         if (wbc) {
3053                 wbc_init_bio(wbc, bio);
3054                 wbc_account_io(wbc, bh->b_page, bh->b_size);
3055         }
3056 
3057         bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
3058         bio_set_dev(bio, bh->b_bdev);
3059         bio->bi_write_hint = write_hint;
3060 
3061         bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh));
3062         BUG_ON(bio->bi_iter.bi_size != bh->b_size);
3063 
3064         bio->bi_end_io = end_bio_bh_io_sync;
3065         bio->bi_private = bh;
3066 
3067         /* Take care of bh's that straddle the end of the device */
3068         guard_bio_eod(op, bio);
3069 
3070         if (buffer_meta(bh))
3071                 op_flags |= REQ_META;
3072         if (buffer_prio(bh))
3073                 op_flags |= REQ_PRIO;
3074         bio_set_op_attrs(bio, op, op_flags);
3075 
3076         submit_bio(bio);
3077         return 0;
3078 }
3079 
3080 int submit_bh(int op, int op_flags, struct buffer_head *bh)
3081 {
3082         return submit_bh_wbc(op, op_flags, bh, 0, NULL);
3083 }
3084 EXPORT_SYMBOL(submit_bh);
3085 
3086 /**
3087  * ll_rw_block: low-level access to block devices (DEPRECATED)
3088  * @op: whether to %READ or %WRITE
3089  * @op_flags: req_flag_bits
3090  * @nr: number of &struct buffer_heads in the array
3091  * @bhs: array of pointers to &struct buffer_head
3092  *
3093  * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
3094  * requests an I/O operation on them, either a %REQ_OP_READ or a %REQ_OP_WRITE.
3095  * @op_flags contains flags modifying the detailed I/O behavior, most notably
3096  * %REQ_RAHEAD.
3097  *
3098  * This function drops any buffer that it cannot get a lock on (with the
3099  * BH_Lock state bit), any buffer that appears to be clean when doing a write
3100  * request, and any buffer that appears to be up-to-date when doing read
3101  * request.  Further it marks as clean buffers that are processed for
3102  * writing (the buffer cache won't assume that they are actually clean
3103  * until the buffer gets unlocked).
3104  *
3105  * ll_rw_block sets b_end_io to simple completion handler that marks
3106  * the buffer up-to-date (if appropriate), unlocks the buffer and wakes
3107  * any waiters. 
3108  *
3109  * All of the buffers must be for the same device, and must also be a
3110  * multiple of the current approved size for the device.
3111  */
3112 void ll_rw_block(int op, int op_flags,  int nr, struct buffer_head *bhs[])
3113 {
3114         int i;
3115 
3116         for (i = 0; i < nr; i++) {
3117                 struct buffer_head *bh = bhs[i];
3118 
3119                 if (!trylock_buffer(bh))
3120                         continue;
3121                 if (op == WRITE) {
3122                         if (test_clear_buffer_dirty(bh)) {
3123                                 bh->b_end_io = end_buffer_write_sync;
3124                                 get_bh(bh);
3125                                 submit_bh(op, op_flags, bh);
3126                                 continue;
3127                         }
3128                 } else {
3129                         if (!buffer_uptodate(bh)) {
3130                                 bh->b_end_io = end_buffer_read_sync;
3131                                 get_bh(bh);
3132                                 submit_bh(op, op_flags, bh);
3133                                 continue;
3134                         }
3135                 }
3136                 unlock_buffer(bh);
3137         }
3138 }
3139 EXPORT_SYMBOL(ll_rw_block);
3140 
3141 void write_dirty_buffer(struct buffer_head *bh, int op_flags)
3142 {
3143         lock_buffer(bh);
3144         if (!test_clear_buffer_dirty(bh)) {
3145                 unlock_buffer(bh);
3146                 return;
3147         }
3148         bh->b_end_io = end_buffer_write_sync;
3149         get_bh(bh);
3150         submit_bh(REQ_OP_WRITE, op_flags, bh);
3151 }
3152 EXPORT_SYMBOL(write_dirty_buffer);
3153 
3154 /*
3155  * For a data-integrity writeout, we need to wait upon any in-progress I/O
3156  * and then start new I/O and then wait upon it.  The caller must have a ref on
3157  * the buffer_head.
3158  */
3159 int __sync_dirty_buffer(struct buffer_head *bh, int op_flags)
3160 {
3161         int ret = 0;
3162 
3163         WARN_ON(atomic_read(&bh->b_count) < 1);
3164         lock_buffer(bh);
3165         if (test_clear_buffer_dirty(bh)) {
3166                 get_bh(bh);
3167                 bh->b_end_io = end_buffer_write_sync;
3168                 ret = submit_bh(REQ_OP_WRITE, op_flags, bh);
3169                 wait_on_buffer(bh);
3170                 if (!ret && !buffer_uptodate(bh))
3171                         ret = -EIO;
3172         } else {
3173                 unlock_buffer(bh);
3174         }
3175         return ret;
3176 }
3177 EXPORT_SYMBOL(__sync_dirty_buffer);
3178 
3179 int sync_dirty_buffer(struct buffer_head *bh)
3180 {
3181         return __sync_dirty_buffer(bh, REQ_SYNC);
3182 }
3183 EXPORT_SYMBOL(sync_dirty_buffer);
3184 
3185 /*
3186  * try_to_free_buffers() checks if all the buffers on this particular page
3187  * are unused, and releases them if so.
3188  *
3189  * Exclusion against try_to_free_buffers may be obtained by either
3190  * locking the page or by holding its mapping's private_lock.
3191  *
3192  * If the page is dirty but all the buffers are clean then we need to
3193  * be sure to mark the page clean as well.  This is because the page
3194  * may be against a block device, and a later reattachment of buffers
3195  * to a dirty page will set *all* buffers dirty.  Which would corrupt
3196  * filesystem data on the same device.
3197  *
3198  * The same applies to regular filesystem pages: if all the buffers are
3199  * clean then we set the page clean and proceed.  To do that, we require
3200  * total exclusion from __set_page_dirty_buffers().  That is obtained with
3201  * private_lock.
3202  *
3203  * try_to_free_buffers() is non-blocking.
3204  */
3205 static inline int buffer_busy(struct buffer_head *bh)
3206 {
3207         return atomic_read(&bh->b_count) |
3208                 (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
3209 }
3210 
3211 static int
3212 drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
3213 {
3214         struct buffer_head *head = page_buffers(page);
3215         struct buffer_head *bh;
3216 
3217         bh = head;
3218         do {
3219                 if (buffer_busy(bh))
3220                         goto failed;
3221                 bh = bh->b_this_page;
3222         } while (bh != head);
3223 
3224         do {
3225                 struct buffer_head *next = bh->b_this_page;
3226 
3227                 if (bh->b_assoc_map)
3228                         __remove_assoc_queue(bh);
3229                 bh = next;
3230         } while (bh != head);
3231         *buffers_to_free = head;
3232         __clear_page_buffers(page);
3233         return 1;
3234 failed:
3235         return 0;
3236 }
3237 
3238 int try_to_free_buffers(struct page *page)
3239 {
3240         struct address_space * const mapping = page->mapping;
3241         struct buffer_head *buffers_to_free = NULL;
3242         int ret = 0;
3243 
3244         BUG_ON(!PageLocked(page));
3245         if (PageWriteback(page))
3246                 return 0;
3247 
3248         if (mapping == NULL) {          /* can this still happen? */
3249                 ret = drop_buffers(page, &buffers_to_free);
3250                 goto out;
3251         }
3252 
3253         spin_lock(&mapping->private_lock);
3254         ret = drop_buffers(page, &buffers_to_free);
3255 
3256         /*
3257          * If the filesystem writes its buffers by hand (eg ext3)
3258          * then we can have clean buffers against a dirty page.  We
3259          * clean the page here; otherwise the VM will never notice
3260          * that the filesystem did any IO at all.
3261          *
3262          * Also, during truncate, discard_buffer will have marked all
3263          * the page's buffers clean.  We discover that here and clean
3264          * the page also.
3265          *
3266          * private_lock must be held over this entire operation in order
3267          * to synchronise against __set_page_dirty_buffers and prevent the
3268          * dirty bit from being lost.
3269          */
3270         if (ret)
3271                 cancel_dirty_page(page);
3272         spin_unlock(&mapping->private_lock);
3273 out:
3274         if (buffers_to_free) {
3275                 struct buffer_head *bh = buffers_to_free;
3276 
3277                 do {
3278                         struct buffer_head *next = bh->b_this_page;
3279                         free_buffer_head(bh);
3280                         bh = next;
3281                 } while (bh != buffers_to_free);
3282         }
3283         return ret;
3284 }
3285 EXPORT_SYMBOL(try_to_free_buffers);
3286 
3287 /*
3288  * There are no bdflush tunables left.  But distributions are
3289  * still running obsolete flush daemons, so we terminate them here.
3290  *
3291  * Use of bdflush() is deprecated and will be removed in a future kernel.
3292  * The `flush-X' kernel threads fully replace bdflush daemons and this call.
3293  */
3294 SYSCALL_DEFINE2(bdflush, int, func, long, data)
3295 {
3296         static int msg_count;
3297 
3298         if (!capable(CAP_SYS_ADMIN))
3299                 return -EPERM;
3300 
3301         if (msg_count < 5) {
3302                 msg_count++;
3303                 printk(KERN_INFO
3304                         "warning: process `%s' used the obsolete bdflush"
3305                         " system call\n", current->comm);
3306                 printk(KERN_INFO "Fix your initscripts?\n");
3307         }
3308 
3309         if (func == 1)
3310                 do_exit(0);
3311         return 0;
3312 }
3313 
3314 /*
3315  * Buffer-head allocation
3316  */
3317 static struct kmem_cache *bh_cachep __read_mostly;
3318 
3319 /*
3320  * Once the number of bh's in the machine exceeds this level, we start
3321  * stripping them in writeback.
3322  */
3323 static unsigned long max_buffer_heads;
3324 
3325 int buffer_heads_over_limit;
3326 
3327 struct bh_accounting {
3328         int nr;                 /* Number of live bh's */
3329         int ratelimit;          /* Limit cacheline bouncing */
3330 };
3331 
3332 static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
3333 
3334 static void recalc_bh_state(void)
3335 {
3336         int i;
3337         int tot = 0;
3338 
3339         if (__this_cpu_inc_return(bh_accounting.ratelimit) - 1 < 4096)
3340                 return;
3341         __this_cpu_write(bh_accounting.ratelimit, 0);
3342         for_each_online_cpu(i)
3343                 tot += per_cpu(bh_accounting, i).nr;
3344         buffer_heads_over_limit = (tot > max_buffer_heads);
3345 }
3346 
3347 struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
3348 {
3349         struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags);
3350         if (ret) {
3351                 INIT_LIST_HEAD(&ret->b_assoc_buffers);
3352                 preempt_disable();
3353                 __this_cpu_inc(bh_accounting.nr);
3354                 recalc_bh_state();
3355                 preempt_enable();
3356         }
3357         return ret;
3358 }
3359 EXPORT_SYMBOL(alloc_buffer_head);
3360 
3361 void free_buffer_head(struct buffer_head *bh)
3362 {
3363         BUG_ON(!list_empty(&bh->b_assoc_buffers));
3364         kmem_cache_free(bh_cachep, bh);
3365         preempt_disable();
3366         __this_cpu_dec(bh_accounting.nr);
3367         recalc_bh_state();
3368         preempt_enable();
3369 }
3370 EXPORT_SYMBOL(free_buffer_head);
3371 
3372 static int buffer_exit_cpu_dead(unsigned int cpu)
3373 {
3374         int i;
3375         struct bh_lru *b = &per_cpu(bh_lrus, cpu);
3376 
3377         for (i = 0; i < BH_LRU_SIZE; i++) {
3378                 brelse(b->bhs[i]);
3379                 b->bhs[i] = NULL;
3380         }
3381         this_cpu_add(bh_accounting.nr, per_cpu(bh_accounting, cpu).nr);
3382         per_cpu(bh_accounting, cpu).nr = 0;
3383         return 0;
3384 }
3385 
3386 /**
3387  * bh_uptodate_or_lock - Test whether the buffer is uptodate
3388  * @bh: struct buffer_head
3389  *
3390  * Return true if the buffer is up-to-date and false,
3391  * with the buffer locked, if not.
3392  */
3393 int bh_uptodate_or_lock(struct buffer_head *bh)
3394 {
3395         if (!buffer_uptodate(bh)) {
3396                 lock_buffer(bh);
3397                 if (!buffer_uptodate(bh))
3398                         return 0;
3399                 unlock_buffer(bh);
3400         }
3401         return 1;
3402 }
3403 EXPORT_SYMBOL(bh_uptodate_or_lock);
3404 
3405 /**
3406  * bh_submit_read - Submit a locked buffer for reading
3407  * @bh: struct buffer_head
3408  *
3409  * Returns zero on success and -EIO on error.
3410  */
3411 int bh_submit_read(struct buffer_head *bh)
3412 {
3413         BUG_ON(!buffer_locked(bh));
3414 
3415         if (buffer_uptodate(bh)) {
3416                 unlock_buffer(bh);
3417                 return 0;
3418         }
3419 
3420         get_bh(bh);
3421         bh->b_end_io = end_buffer_read_sync;
3422         submit_bh(REQ_OP_READ, 0, bh);
3423         wait_on_buffer(bh);
3424         if (buffer_uptodate(bh))
3425                 return 0;
3426         return -EIO;
3427 }
3428 EXPORT_SYMBOL(bh_submit_read);
3429 
3430 void __init buffer_init(void)
3431 {
3432         unsigned long nrpages;
3433         int ret;
3434 
3435         bh_cachep = kmem_cache_create("buffer_head",
3436                         sizeof(struct buffer_head), 0,
3437                                 (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
3438                                 SLAB_MEM_SPREAD),
3439                                 NULL);
3440 
3441         /*
3442          * Limit the bh occupancy to 10% of ZONE_NORMAL
3443          */
3444         nrpages = (nr_free_buffer_pages() * 10) / 100;
3445         max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
3446         ret = cpuhp_setup_state_nocalls(CPUHP_FS_BUFF_DEAD, "fs/buffer:dead",
3447                                         NULL, buffer_exit_cpu_dead);
3448         WARN_ON(ret < 0);
3449 }
3450 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp