~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/fs/buffer.c

Version: ~ [ linux-5.13-rc5 ] ~ [ linux-5.12.9 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.42 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.124 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.193 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.235 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.271 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.271 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  *  linux/fs/buffer.c
  3  *
  4  *  Copyright (C) 1991, 1992, 2002  Linus Torvalds
  5  */
  6 
  7 /*
  8  * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
  9  *
 10  * Removed a lot of unnecessary code and simplified things now that
 11  * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
 12  *
 13  * Speed up hash, lru, and free list operations.  Use gfp() for allocating
 14  * hash table, use SLAB cache for buffer heads. SMP threading.  -DaveM
 15  *
 16  * Added 32k buffer block sizes - these are required older ARM systems. - RMK
 17  *
 18  * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
 19  */
 20 
 21 #include <linux/kernel.h>
 22 #include <linux/sched/signal.h>
 23 #include <linux/syscalls.h>
 24 #include <linux/fs.h>
 25 #include <linux/iomap.h>
 26 #include <linux/mm.h>
 27 #include <linux/percpu.h>
 28 #include <linux/slab.h>
 29 #include <linux/capability.h>
 30 #include <linux/blkdev.h>
 31 #include <linux/file.h>
 32 #include <linux/quotaops.h>
 33 #include <linux/highmem.h>
 34 #include <linux/export.h>
 35 #include <linux/backing-dev.h>
 36 #include <linux/writeback.h>
 37 #include <linux/hash.h>
 38 #include <linux/suspend.h>
 39 #include <linux/buffer_head.h>
 40 #include <linux/task_io_accounting_ops.h>
 41 #include <linux/bio.h>
 42 #include <linux/notifier.h>
 43 #include <linux/cpu.h>
 44 #include <linux/bitops.h>
 45 #include <linux/mpage.h>
 46 #include <linux/bit_spinlock.h>
 47 #include <linux/pagevec.h>
 48 #include <trace/events/block.h>
 49 
 50 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
 51 static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh,
 52                          enum rw_hint hint, struct writeback_control *wbc);
 53 
 54 #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
 55 
 56 void init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private)
 57 {
 58         bh->b_end_io = handler;
 59         bh->b_private = private;
 60 }
 61 EXPORT_SYMBOL(init_buffer);
 62 
 63 inline void touch_buffer(struct buffer_head *bh)
 64 {
 65         trace_block_touch_buffer(bh);
 66         mark_page_accessed(bh->b_page);
 67 }
 68 EXPORT_SYMBOL(touch_buffer);
 69 
 70 void __lock_buffer(struct buffer_head *bh)
 71 {
 72         wait_on_bit_lock_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
 73 }
 74 EXPORT_SYMBOL(__lock_buffer);
 75 
 76 void unlock_buffer(struct buffer_head *bh)
 77 {
 78         clear_bit_unlock(BH_Lock, &bh->b_state);
 79         smp_mb__after_atomic();
 80         wake_up_bit(&bh->b_state, BH_Lock);
 81 }
 82 EXPORT_SYMBOL(unlock_buffer);
 83 
 84 /*
 85  * Returns if the page has dirty or writeback buffers. If all the buffers
 86  * are unlocked and clean then the PageDirty information is stale. If
 87  * any of the pages are locked, it is assumed they are locked for IO.
 88  */
 89 void buffer_check_dirty_writeback(struct page *page,
 90                                      bool *dirty, bool *writeback)
 91 {
 92         struct buffer_head *head, *bh;
 93         *dirty = false;
 94         *writeback = false;
 95 
 96         BUG_ON(!PageLocked(page));
 97 
 98         if (!page_has_buffers(page))
 99                 return;
100 
101         if (PageWriteback(page))
102                 *writeback = true;
103 
104         head = page_buffers(page);
105         bh = head;
106         do {
107                 if (buffer_locked(bh))
108                         *writeback = true;
109 
110                 if (buffer_dirty(bh))
111                         *dirty = true;
112 
113                 bh = bh->b_this_page;
114         } while (bh != head);
115 }
116 EXPORT_SYMBOL(buffer_check_dirty_writeback);
117 
118 /*
119  * Block until a buffer comes unlocked.  This doesn't stop it
120  * from becoming locked again - you have to lock it yourself
121  * if you want to preserve its state.
122  */
123 void __wait_on_buffer(struct buffer_head * bh)
124 {
125         wait_on_bit_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
126 }
127 EXPORT_SYMBOL(__wait_on_buffer);
128 
129 static void
130 __clear_page_buffers(struct page *page)
131 {
132         ClearPagePrivate(page);
133         set_page_private(page, 0);
134         put_page(page);
135 }
136 
137 static void buffer_io_error(struct buffer_head *bh, char *msg)
138 {
139         if (!test_bit(BH_Quiet, &bh->b_state))
140                 printk_ratelimited(KERN_ERR
141                         "Buffer I/O error on dev %pg, logical block %llu%s\n",
142                         bh->b_bdev, (unsigned long long)bh->b_blocknr, msg);
143 }
144 
145 /*
146  * End-of-IO handler helper function which does not touch the bh after
147  * unlocking it.
148  * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
149  * a race there is benign: unlock_buffer() only use the bh's address for
150  * hashing after unlocking the buffer, so it doesn't actually touch the bh
151  * itself.
152  */
153 static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate)
154 {
155         if (uptodate) {
156                 set_buffer_uptodate(bh);
157         } else {
158                 /* This happens, due to failed read-ahead attempts. */
159                 clear_buffer_uptodate(bh);
160         }
161         unlock_buffer(bh);
162 }
163 
164 /*
165  * Default synchronous end-of-IO handler..  Just mark it up-to-date and
166  * unlock the buffer. This is what ll_rw_block uses too.
167  */
168 void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
169 {
170         __end_buffer_read_notouch(bh, uptodate);
171         put_bh(bh);
172 }
173 EXPORT_SYMBOL(end_buffer_read_sync);
174 
175 void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
176 {
177         if (uptodate) {
178                 set_buffer_uptodate(bh);
179         } else {
180                 buffer_io_error(bh, ", lost sync page write");
181                 mark_buffer_write_io_error(bh);
182                 clear_buffer_uptodate(bh);
183         }
184         unlock_buffer(bh);
185         put_bh(bh);
186 }
187 EXPORT_SYMBOL(end_buffer_write_sync);
188 
189 /*
190  * Various filesystems appear to want __find_get_block to be non-blocking.
191  * But it's the page lock which protects the buffers.  To get around this,
192  * we get exclusion from try_to_free_buffers with the blockdev mapping's
193  * private_lock.
194  *
195  * Hack idea: for the blockdev mapping, i_bufferlist_lock contention
196  * may be quite high.  This code could TryLock the page, and if that
197  * succeeds, there is no need to take private_lock. (But if
198  * private_lock is contended then so is mapping->tree_lock).
199  */
200 static struct buffer_head *
201 __find_get_block_slow(struct block_device *bdev, sector_t block)
202 {
203         struct inode *bd_inode = bdev->bd_inode;
204         struct address_space *bd_mapping = bd_inode->i_mapping;
205         struct buffer_head *ret = NULL;
206         pgoff_t index;
207         struct buffer_head *bh;
208         struct buffer_head *head;
209         struct page *page;
210         int all_mapped = 1;
211 
212         index = block >> (PAGE_SHIFT - bd_inode->i_blkbits);
213         page = find_get_page_flags(bd_mapping, index, FGP_ACCESSED);
214         if (!page)
215                 goto out;
216 
217         spin_lock(&bd_mapping->private_lock);
218         if (!page_has_buffers(page))
219                 goto out_unlock;
220         head = page_buffers(page);
221         bh = head;
222         do {
223                 if (!buffer_mapped(bh))
224                         all_mapped = 0;
225                 else if (bh->b_blocknr == block) {
226                         ret = bh;
227                         get_bh(bh);
228                         goto out_unlock;
229                 }
230                 bh = bh->b_this_page;
231         } while (bh != head);
232 
233         /* we might be here because some of the buffers on this page are
234          * not mapped.  This is due to various races between
235          * file io on the block device and getblk.  It gets dealt with
236          * elsewhere, don't buffer_error if we had some unmapped buffers
237          */
238         if (all_mapped) {
239                 printk("__find_get_block_slow() failed. "
240                         "block=%llu, b_blocknr=%llu\n",
241                         (unsigned long long)block,
242                         (unsigned long long)bh->b_blocknr);
243                 printk("b_state=0x%08lx, b_size=%zu\n",
244                         bh->b_state, bh->b_size);
245                 printk("device %pg blocksize: %d\n", bdev,
246                         1 << bd_inode->i_blkbits);
247         }
248 out_unlock:
249         spin_unlock(&bd_mapping->private_lock);
250         put_page(page);
251 out:
252         return ret;
253 }
254 
255 /*
256  * I/O completion handler for block_read_full_page() - pages
257  * which come unlocked at the end of I/O.
258  */
259 static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
260 {
261         unsigned long flags;
262         struct buffer_head *first;
263         struct buffer_head *tmp;
264         struct page *page;
265         int page_uptodate = 1;
266 
267         BUG_ON(!buffer_async_read(bh));
268 
269         page = bh->b_page;
270         if (uptodate) {
271                 set_buffer_uptodate(bh);
272         } else {
273                 clear_buffer_uptodate(bh);
274                 buffer_io_error(bh, ", async page read");
275                 SetPageError(page);
276         }
277 
278         /*
279          * Be _very_ careful from here on. Bad things can happen if
280          * two buffer heads end IO at almost the same time and both
281          * decide that the page is now completely done.
282          */
283         first = page_buffers(page);
284         local_irq_save(flags);
285         bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
286         clear_buffer_async_read(bh);
287         unlock_buffer(bh);
288         tmp = bh;
289         do {
290                 if (!buffer_uptodate(tmp))
291                         page_uptodate = 0;
292                 if (buffer_async_read(tmp)) {
293                         BUG_ON(!buffer_locked(tmp));
294                         goto still_busy;
295                 }
296                 tmp = tmp->b_this_page;
297         } while (tmp != bh);
298         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
299         local_irq_restore(flags);
300 
301         /*
302          * If none of the buffers had errors and they are all
303          * uptodate then we can set the page uptodate.
304          */
305         if (page_uptodate && !PageError(page))
306                 SetPageUptodate(page);
307         unlock_page(page);
308         return;
309 
310 still_busy:
311         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
312         local_irq_restore(flags);
313         return;
314 }
315 
316 /*
317  * Completion handler for block_write_full_page() - pages which are unlocked
318  * during I/O, and which have PageWriteback cleared upon I/O completion.
319  */
320 void end_buffer_async_write(struct buffer_head *bh, int uptodate)
321 {
322         unsigned long flags;
323         struct buffer_head *first;
324         struct buffer_head *tmp;
325         struct page *page;
326 
327         BUG_ON(!buffer_async_write(bh));
328 
329         page = bh->b_page;
330         if (uptodate) {
331                 set_buffer_uptodate(bh);
332         } else {
333                 buffer_io_error(bh, ", lost async page write");
334                 mark_buffer_write_io_error(bh);
335                 clear_buffer_uptodate(bh);
336                 SetPageError(page);
337         }
338 
339         first = page_buffers(page);
340         local_irq_save(flags);
341         bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
342 
343         clear_buffer_async_write(bh);
344         unlock_buffer(bh);
345         tmp = bh->b_this_page;
346         while (tmp != bh) {
347                 if (buffer_async_write(tmp)) {
348                         BUG_ON(!buffer_locked(tmp));
349                         goto still_busy;
350                 }
351                 tmp = tmp->b_this_page;
352         }
353         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
354         local_irq_restore(flags);
355         end_page_writeback(page);
356         return;
357 
358 still_busy:
359         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
360         local_irq_restore(flags);
361         return;
362 }
363 EXPORT_SYMBOL(end_buffer_async_write);
364 
365 /*
366  * If a page's buffers are under async readin (end_buffer_async_read
367  * completion) then there is a possibility that another thread of
368  * control could lock one of the buffers after it has completed
369  * but while some of the other buffers have not completed.  This
370  * locked buffer would confuse end_buffer_async_read() into not unlocking
371  * the page.  So the absence of BH_Async_Read tells end_buffer_async_read()
372  * that this buffer is not under async I/O.
373  *
374  * The page comes unlocked when it has no locked buffer_async buffers
375  * left.
376  *
377  * PageLocked prevents anyone starting new async I/O reads any of
378  * the buffers.
379  *
380  * PageWriteback is used to prevent simultaneous writeout of the same
381  * page.
382  *
383  * PageLocked prevents anyone from starting writeback of a page which is
384  * under read I/O (PageWriteback is only ever set against a locked page).
385  */
386 static void mark_buffer_async_read(struct buffer_head *bh)
387 {
388         bh->b_end_io = end_buffer_async_read;
389         set_buffer_async_read(bh);
390 }
391 
392 static void mark_buffer_async_write_endio(struct buffer_head *bh,
393                                           bh_end_io_t *handler)
394 {
395         bh->b_end_io = handler;
396         set_buffer_async_write(bh);
397 }
398 
399 void mark_buffer_async_write(struct buffer_head *bh)
400 {
401         mark_buffer_async_write_endio(bh, end_buffer_async_write);
402 }
403 EXPORT_SYMBOL(mark_buffer_async_write);
404 
405 
406 /*
407  * fs/buffer.c contains helper functions for buffer-backed address space's
408  * fsync functions.  A common requirement for buffer-based filesystems is
409  * that certain data from the backing blockdev needs to be written out for
410  * a successful fsync().  For example, ext2 indirect blocks need to be
411  * written back and waited upon before fsync() returns.
412  *
413  * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
414  * inode_has_buffers() and invalidate_inode_buffers() are provided for the
415  * management of a list of dependent buffers at ->i_mapping->private_list.
416  *
417  * Locking is a little subtle: try_to_free_buffers() will remove buffers
418  * from their controlling inode's queue when they are being freed.  But
419  * try_to_free_buffers() will be operating against the *blockdev* mapping
420  * at the time, not against the S_ISREG file which depends on those buffers.
421  * So the locking for private_list is via the private_lock in the address_space
422  * which backs the buffers.  Which is different from the address_space 
423  * against which the buffers are listed.  So for a particular address_space,
424  * mapping->private_lock does *not* protect mapping->private_list!  In fact,
425  * mapping->private_list will always be protected by the backing blockdev's
426  * ->private_lock.
427  *
428  * Which introduces a requirement: all buffers on an address_space's
429  * ->private_list must be from the same address_space: the blockdev's.
430  *
431  * address_spaces which do not place buffers at ->private_list via these
432  * utility functions are free to use private_lock and private_list for
433  * whatever they want.  The only requirement is that list_empty(private_list)
434  * be true at clear_inode() time.
435  *
436  * FIXME: clear_inode should not call invalidate_inode_buffers().  The
437  * filesystems should do that.  invalidate_inode_buffers() should just go
438  * BUG_ON(!list_empty).
439  *
440  * FIXME: mark_buffer_dirty_inode() is a data-plane operation.  It should
441  * take an address_space, not an inode.  And it should be called
442  * mark_buffer_dirty_fsync() to clearly define why those buffers are being
443  * queued up.
444  *
445  * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
446  * list if it is already on a list.  Because if the buffer is on a list,
447  * it *must* already be on the right one.  If not, the filesystem is being
448  * silly.  This will save a ton of locking.  But first we have to ensure
449  * that buffers are taken *off* the old inode's list when they are freed
450  * (presumably in truncate).  That requires careful auditing of all
451  * filesystems (do it inside bforget()).  It could also be done by bringing
452  * b_inode back.
453  */
454 
455 /*
456  * The buffer's backing address_space's private_lock must be held
457  */
458 static void __remove_assoc_queue(struct buffer_head *bh)
459 {
460         list_del_init(&bh->b_assoc_buffers);
461         WARN_ON(!bh->b_assoc_map);
462         bh->b_assoc_map = NULL;
463 }
464 
465 int inode_has_buffers(struct inode *inode)
466 {
467         return !list_empty(&inode->i_data.private_list);
468 }
469 
470 /*
471  * osync is designed to support O_SYNC io.  It waits synchronously for
472  * all already-submitted IO to complete, but does not queue any new
473  * writes to the disk.
474  *
475  * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
476  * you dirty the buffers, and then use osync_inode_buffers to wait for
477  * completion.  Any other dirty buffers which are not yet queued for
478  * write will not be flushed to disk by the osync.
479  */
480 static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
481 {
482         struct buffer_head *bh;
483         struct list_head *p;
484         int err = 0;
485 
486         spin_lock(lock);
487 repeat:
488         list_for_each_prev(p, list) {
489                 bh = BH_ENTRY(p);
490                 if (buffer_locked(bh)) {
491                         get_bh(bh);
492                         spin_unlock(lock);
493                         wait_on_buffer(bh);
494                         if (!buffer_uptodate(bh))
495                                 err = -EIO;
496                         brelse(bh);
497                         spin_lock(lock);
498                         goto repeat;
499                 }
500         }
501         spin_unlock(lock);
502         return err;
503 }
504 
505 static void do_thaw_one(struct super_block *sb, void *unused)
506 {
507         while (sb->s_bdev && !thaw_bdev(sb->s_bdev, sb))
508                 printk(KERN_WARNING "Emergency Thaw on %pg\n", sb->s_bdev);
509 }
510 
511 static void do_thaw_all(struct work_struct *work)
512 {
513         iterate_supers(do_thaw_one, NULL);
514         kfree(work);
515         printk(KERN_WARNING "Emergency Thaw complete\n");
516 }
517 
518 /**
519  * emergency_thaw_all -- forcibly thaw every frozen filesystem
520  *
521  * Used for emergency unfreeze of all filesystems via SysRq
522  */
523 void emergency_thaw_all(void)
524 {
525         struct work_struct *work;
526 
527         work = kmalloc(sizeof(*work), GFP_ATOMIC);
528         if (work) {
529                 INIT_WORK(work, do_thaw_all);
530                 schedule_work(work);
531         }
532 }
533 
534 /**
535  * sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers
536  * @mapping: the mapping which wants those buffers written
537  *
538  * Starts I/O against the buffers at mapping->private_list, and waits upon
539  * that I/O.
540  *
541  * Basically, this is a convenience function for fsync().
542  * @mapping is a file or directory which needs those buffers to be written for
543  * a successful fsync().
544  */
545 int sync_mapping_buffers(struct address_space *mapping)
546 {
547         struct address_space *buffer_mapping = mapping->private_data;
548 
549         if (buffer_mapping == NULL || list_empty(&mapping->private_list))
550                 return 0;
551 
552         return fsync_buffers_list(&buffer_mapping->private_lock,
553                                         &mapping->private_list);
554 }
555 EXPORT_SYMBOL(sync_mapping_buffers);
556 
557 /*
558  * Called when we've recently written block `bblock', and it is known that
559  * `bblock' was for a buffer_boundary() buffer.  This means that the block at
560  * `bblock + 1' is probably a dirty indirect block.  Hunt it down and, if it's
561  * dirty, schedule it for IO.  So that indirects merge nicely with their data.
562  */
563 void write_boundary_block(struct block_device *bdev,
564                         sector_t bblock, unsigned blocksize)
565 {
566         struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
567         if (bh) {
568                 if (buffer_dirty(bh))
569                         ll_rw_block(REQ_OP_WRITE, 0, 1, &bh);
570                 put_bh(bh);
571         }
572 }
573 
574 void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
575 {
576         struct address_space *mapping = inode->i_mapping;
577         struct address_space *buffer_mapping = bh->b_page->mapping;
578 
579         mark_buffer_dirty(bh);
580         if (!mapping->private_data) {
581                 mapping->private_data = buffer_mapping;
582         } else {
583                 BUG_ON(mapping->private_data != buffer_mapping);
584         }
585         if (!bh->b_assoc_map) {
586                 spin_lock(&buffer_mapping->private_lock);
587                 list_move_tail(&bh->b_assoc_buffers,
588                                 &mapping->private_list);
589                 bh->b_assoc_map = mapping;
590                 spin_unlock(&buffer_mapping->private_lock);
591         }
592 }
593 EXPORT_SYMBOL(mark_buffer_dirty_inode);
594 
595 /*
596  * Mark the page dirty, and set it dirty in the radix tree, and mark the inode
597  * dirty.
598  *
599  * If warn is true, then emit a warning if the page is not uptodate and has
600  * not been truncated.
601  *
602  * The caller must hold lock_page_memcg().
603  */
604 static void __set_page_dirty(struct page *page, struct address_space *mapping,
605                              int warn)
606 {
607         unsigned long flags;
608 
609         spin_lock_irqsave(&mapping->tree_lock, flags);
610         if (page->mapping) {    /* Race with truncate? */
611                 WARN_ON_ONCE(warn && !PageUptodate(page));
612                 account_page_dirtied(page, mapping);
613                 radix_tree_tag_set(&mapping->page_tree,
614                                 page_index(page), PAGECACHE_TAG_DIRTY);
615         }
616         spin_unlock_irqrestore(&mapping->tree_lock, flags);
617 }
618 
619 /*
620  * Add a page to the dirty page list.
621  *
622  * It is a sad fact of life that this function is called from several places
623  * deeply under spinlocking.  It may not sleep.
624  *
625  * If the page has buffers, the uptodate buffers are set dirty, to preserve
626  * dirty-state coherency between the page and the buffers.  It the page does
627  * not have buffers then when they are later attached they will all be set
628  * dirty.
629  *
630  * The buffers are dirtied before the page is dirtied.  There's a small race
631  * window in which a writepage caller may see the page cleanness but not the
632  * buffer dirtiness.  That's fine.  If this code were to set the page dirty
633  * before the buffers, a concurrent writepage caller could clear the page dirty
634  * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
635  * page on the dirty page list.
636  *
637  * We use private_lock to lock against try_to_free_buffers while using the
638  * page's buffer list.  Also use this to protect against clean buffers being
639  * added to the page after it was set dirty.
640  *
641  * FIXME: may need to call ->reservepage here as well.  That's rather up to the
642  * address_space though.
643  */
644 int __set_page_dirty_buffers(struct page *page)
645 {
646         int newly_dirty;
647         struct address_space *mapping = page_mapping(page);
648 
649         if (unlikely(!mapping))
650                 return !TestSetPageDirty(page);
651 
652         spin_lock(&mapping->private_lock);
653         if (page_has_buffers(page)) {
654                 struct buffer_head *head = page_buffers(page);
655                 struct buffer_head *bh = head;
656 
657                 do {
658                         set_buffer_dirty(bh);
659                         bh = bh->b_this_page;
660                 } while (bh != head);
661         }
662         /*
663          * Lock out page->mem_cgroup migration to keep PageDirty
664          * synchronized with per-memcg dirty page counters.
665          */
666         lock_page_memcg(page);
667         newly_dirty = !TestSetPageDirty(page);
668         spin_unlock(&mapping->private_lock);
669 
670         if (newly_dirty)
671                 __set_page_dirty(page, mapping, 1);
672 
673         unlock_page_memcg(page);
674 
675         if (newly_dirty)
676                 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
677 
678         return newly_dirty;
679 }
680 EXPORT_SYMBOL(__set_page_dirty_buffers);
681 
682 /*
683  * Write out and wait upon a list of buffers.
684  *
685  * We have conflicting pressures: we want to make sure that all
686  * initially dirty buffers get waited on, but that any subsequently
687  * dirtied buffers don't.  After all, we don't want fsync to last
688  * forever if somebody is actively writing to the file.
689  *
690  * Do this in two main stages: first we copy dirty buffers to a
691  * temporary inode list, queueing the writes as we go.  Then we clean
692  * up, waiting for those writes to complete.
693  * 
694  * During this second stage, any subsequent updates to the file may end
695  * up refiling the buffer on the original inode's dirty list again, so
696  * there is a chance we will end up with a buffer queued for write but
697  * not yet completed on that list.  So, as a final cleanup we go through
698  * the osync code to catch these locked, dirty buffers without requeuing
699  * any newly dirty buffers for write.
700  */
701 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
702 {
703         struct buffer_head *bh;
704         struct list_head tmp;
705         struct address_space *mapping;
706         int err = 0, err2;
707         struct blk_plug plug;
708 
709         INIT_LIST_HEAD(&tmp);
710         blk_start_plug(&plug);
711 
712         spin_lock(lock);
713         while (!list_empty(list)) {
714                 bh = BH_ENTRY(list->next);
715                 mapping = bh->b_assoc_map;
716                 __remove_assoc_queue(bh);
717                 /* Avoid race with mark_buffer_dirty_inode() which does
718                  * a lockless check and we rely on seeing the dirty bit */
719                 smp_mb();
720                 if (buffer_dirty(bh) || buffer_locked(bh)) {
721                         list_add(&bh->b_assoc_buffers, &tmp);
722                         bh->b_assoc_map = mapping;
723                         if (buffer_dirty(bh)) {
724                                 get_bh(bh);
725                                 spin_unlock(lock);
726                                 /*
727                                  * Ensure any pending I/O completes so that
728                                  * write_dirty_buffer() actually writes the
729                                  * current contents - it is a noop if I/O is
730                                  * still in flight on potentially older
731                                  * contents.
732                                  */
733                                 write_dirty_buffer(bh, REQ_SYNC);
734 
735                                 /*
736                                  * Kick off IO for the previous mapping. Note
737                                  * that we will not run the very last mapping,
738                                  * wait_on_buffer() will do that for us
739                                  * through sync_buffer().
740                                  */
741                                 brelse(bh);
742                                 spin_lock(lock);
743                         }
744                 }
745         }
746 
747         spin_unlock(lock);
748         blk_finish_plug(&plug);
749         spin_lock(lock);
750 
751         while (!list_empty(&tmp)) {
752                 bh = BH_ENTRY(tmp.prev);
753                 get_bh(bh);
754                 mapping = bh->b_assoc_map;
755                 __remove_assoc_queue(bh);
756                 /* Avoid race with mark_buffer_dirty_inode() which does
757                  * a lockless check and we rely on seeing the dirty bit */
758                 smp_mb();
759                 if (buffer_dirty(bh)) {
760                         list_add(&bh->b_assoc_buffers,
761                                  &mapping->private_list);
762                         bh->b_assoc_map = mapping;
763                 }
764                 spin_unlock(lock);
765                 wait_on_buffer(bh);
766                 if (!buffer_uptodate(bh))
767                         err = -EIO;
768                 brelse(bh);
769                 spin_lock(lock);
770         }
771         
772         spin_unlock(lock);
773         err2 = osync_buffers_list(lock, list);
774         if (err)
775                 return err;
776         else
777                 return err2;
778 }
779 
780 /*
781  * Invalidate any and all dirty buffers on a given inode.  We are
782  * probably unmounting the fs, but that doesn't mean we have already
783  * done a sync().  Just drop the buffers from the inode list.
784  *
785  * NOTE: we take the inode's blockdev's mapping's private_lock.  Which
786  * assumes that all the buffers are against the blockdev.  Not true
787  * for reiserfs.
788  */
789 void invalidate_inode_buffers(struct inode *inode)
790 {
791         if (inode_has_buffers(inode)) {
792                 struct address_space *mapping = &inode->i_data;
793                 struct list_head *list = &mapping->private_list;
794                 struct address_space *buffer_mapping = mapping->private_data;
795 
796                 spin_lock(&buffer_mapping->private_lock);
797                 while (!list_empty(list))
798                         __remove_assoc_queue(BH_ENTRY(list->next));
799                 spin_unlock(&buffer_mapping->private_lock);
800         }
801 }
802 EXPORT_SYMBOL(invalidate_inode_buffers);
803 
804 /*
805  * Remove any clean buffers from the inode's buffer list.  This is called
806  * when we're trying to free the inode itself.  Those buffers can pin it.
807  *
808  * Returns true if all buffers were removed.
809  */
810 int remove_inode_buffers(struct inode *inode)
811 {
812         int ret = 1;
813 
814         if (inode_has_buffers(inode)) {
815                 struct address_space *mapping = &inode->i_data;
816                 struct list_head *list = &mapping->private_list;
817                 struct address_space *buffer_mapping = mapping->private_data;
818 
819                 spin_lock(&buffer_mapping->private_lock);
820                 while (!list_empty(list)) {
821                         struct buffer_head *bh = BH_ENTRY(list->next);
822                         if (buffer_dirty(bh)) {
823                                 ret = 0;
824                                 break;
825                         }
826                         __remove_assoc_queue(bh);
827                 }
828                 spin_unlock(&buffer_mapping->private_lock);
829         }
830         return ret;
831 }
832 
833 /*
834  * Create the appropriate buffers when given a page for data area and
835  * the size of each buffer.. Use the bh->b_this_page linked list to
836  * follow the buffers created.  Return NULL if unable to create more
837  * buffers.
838  *
839  * The retry flag is used to differentiate async IO (paging, swapping)
840  * which may not fail from ordinary buffer allocations.
841  */
842 struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
843                 bool retry)
844 {
845         struct buffer_head *bh, *head;
846         gfp_t gfp = GFP_NOFS;
847         long offset;
848 
849         if (retry)
850                 gfp |= __GFP_NOFAIL;
851 
852         head = NULL;
853         offset = PAGE_SIZE;
854         while ((offset -= size) >= 0) {
855                 bh = alloc_buffer_head(gfp);
856                 if (!bh)
857                         goto no_grow;
858 
859                 bh->b_this_page = head;
860                 bh->b_blocknr = -1;
861                 head = bh;
862 
863                 bh->b_size = size;
864 
865                 /* Link the buffer to its page */
866                 set_bh_page(bh, page, offset);
867         }
868         return head;
869 /*
870  * In case anything failed, we just free everything we got.
871  */
872 no_grow:
873         if (head) {
874                 do {
875                         bh = head;
876                         head = head->b_this_page;
877                         free_buffer_head(bh);
878                 } while (head);
879         }
880 
881         return NULL;
882 }
883 EXPORT_SYMBOL_GPL(alloc_page_buffers);
884 
885 static inline void
886 link_dev_buffers(struct page *page, struct buffer_head *head)
887 {
888         struct buffer_head *bh, *tail;
889 
890         bh = head;
891         do {
892                 tail = bh;
893                 bh = bh->b_this_page;
894         } while (bh);
895         tail->b_this_page = head;
896         attach_page_buffers(page, head);
897 }
898 
899 static sector_t blkdev_max_block(struct block_device *bdev, unsigned int size)
900 {
901         sector_t retval = ~((sector_t)0);
902         loff_t sz = i_size_read(bdev->bd_inode);
903 
904         if (sz) {
905                 unsigned int sizebits = blksize_bits(size);
906                 retval = (sz >> sizebits);
907         }
908         return retval;
909 }
910 
911 /*
912  * Initialise the state of a blockdev page's buffers.
913  */ 
914 static sector_t
915 init_page_buffers(struct page *page, struct block_device *bdev,
916                         sector_t block, int size)
917 {
918         struct buffer_head *head = page_buffers(page);
919         struct buffer_head *bh = head;
920         int uptodate = PageUptodate(page);
921         sector_t end_block = blkdev_max_block(I_BDEV(bdev->bd_inode), size);
922 
923         do {
924                 if (!buffer_mapped(bh)) {
925                         init_buffer(bh, NULL, NULL);
926                         bh->b_bdev = bdev;
927                         bh->b_blocknr = block;
928                         if (uptodate)
929                                 set_buffer_uptodate(bh);
930                         if (block < end_block)
931                                 set_buffer_mapped(bh);
932                 }
933                 block++;
934                 bh = bh->b_this_page;
935         } while (bh != head);
936 
937         /*
938          * Caller needs to validate requested block against end of device.
939          */
940         return end_block;
941 }
942 
943 /*
944  * Create the page-cache page that contains the requested block.
945  *
946  * This is used purely for blockdev mappings.
947  */
948 static int
949 grow_dev_page(struct block_device *bdev, sector_t block,
950               pgoff_t index, int size, int sizebits, gfp_t gfp)
951 {
952         struct inode *inode = bdev->bd_inode;
953         struct page *page;
954         struct buffer_head *bh;
955         sector_t end_block;
956         int ret = 0;            /* Will call free_more_memory() */
957         gfp_t gfp_mask;
958 
959         gfp_mask = mapping_gfp_constraint(inode->i_mapping, ~__GFP_FS) | gfp;
960 
961         /*
962          * XXX: __getblk_slow() can not really deal with failure and
963          * will endlessly loop on improvised global reclaim.  Prefer
964          * looping in the allocator rather than here, at least that
965          * code knows what it's doing.
966          */
967         gfp_mask |= __GFP_NOFAIL;
968 
969         page = find_or_create_page(inode->i_mapping, index, gfp_mask);
970 
971         BUG_ON(!PageLocked(page));
972 
973         if (page_has_buffers(page)) {
974                 bh = page_buffers(page);
975                 if (bh->b_size == size) {
976                         end_block = init_page_buffers(page, bdev,
977                                                 (sector_t)index << sizebits,
978                                                 size);
979                         goto done;
980                 }
981                 if (!try_to_free_buffers(page))
982                         goto failed;
983         }
984 
985         /*
986          * Allocate some buffers for this page
987          */
988         bh = alloc_page_buffers(page, size, true);
989 
990         /*
991          * Link the page to the buffers and initialise them.  Take the
992          * lock to be atomic wrt __find_get_block(), which does not
993          * run under the page lock.
994          */
995         spin_lock(&inode->i_mapping->private_lock);
996         link_dev_buffers(page, bh);
997         end_block = init_page_buffers(page, bdev, (sector_t)index << sizebits,
998                         size);
999         spin_unlock(&inode->i_mapping->private_lock);
1000 done:
1001         ret = (block < end_block) ? 1 : -ENXIO;
1002 failed:
1003         unlock_page(page);
1004         put_page(page);
1005         return ret;
1006 }
1007 
1008 /*
1009  * Create buffers for the specified block device block's page.  If
1010  * that page was dirty, the buffers are set dirty also.
1011  */
1012 static int
1013 grow_buffers(struct block_device *bdev, sector_t block, int size, gfp_t gfp)
1014 {
1015         pgoff_t index;
1016         int sizebits;
1017 
1018         sizebits = -1;
1019         do {
1020                 sizebits++;
1021         } while ((size << sizebits) < PAGE_SIZE);
1022 
1023         index = block >> sizebits;
1024 
1025         /*
1026          * Check for a block which wants to lie outside our maximum possible
1027          * pagecache index.  (this comparison is done using sector_t types).
1028          */
1029         if (unlikely(index != block >> sizebits)) {
1030                 printk(KERN_ERR "%s: requested out-of-range block %llu for "
1031                         "device %pg\n",
1032                         __func__, (unsigned long long)block,
1033                         bdev);
1034                 return -EIO;
1035         }
1036 
1037         /* Create a page with the proper size buffers.. */
1038         return grow_dev_page(bdev, block, index, size, sizebits, gfp);
1039 }
1040 
1041 static struct buffer_head *
1042 __getblk_slow(struct block_device *bdev, sector_t block,
1043              unsigned size, gfp_t gfp)
1044 {
1045         /* Size must be multiple of hard sectorsize */
1046         if (unlikely(size & (bdev_logical_block_size(bdev)-1) ||
1047                         (size < 512 || size > PAGE_SIZE))) {
1048                 printk(KERN_ERR "getblk(): invalid block size %d requested\n",
1049                                         size);
1050                 printk(KERN_ERR "logical block size: %d\n",
1051                                         bdev_logical_block_size(bdev));
1052 
1053                 dump_stack();
1054                 return NULL;
1055         }
1056 
1057         for (;;) {
1058                 struct buffer_head *bh;
1059                 int ret;
1060 
1061                 bh = __find_get_block(bdev, block, size);
1062                 if (bh)
1063                         return bh;
1064 
1065                 ret = grow_buffers(bdev, block, size, gfp);
1066                 if (ret < 0)
1067                         return NULL;
1068         }
1069 }
1070 
1071 /*
1072  * The relationship between dirty buffers and dirty pages:
1073  *
1074  * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1075  * the page is tagged dirty in its radix tree.
1076  *
1077  * At all times, the dirtiness of the buffers represents the dirtiness of
1078  * subsections of the page.  If the page has buffers, the page dirty bit is
1079  * merely a hint about the true dirty state.
1080  *
1081  * When a page is set dirty in its entirety, all its buffers are marked dirty
1082  * (if the page has buffers).
1083  *
1084  * When a buffer is marked dirty, its page is dirtied, but the page's other
1085  * buffers are not.
1086  *
1087  * Also.  When blockdev buffers are explicitly read with bread(), they
1088  * individually become uptodate.  But their backing page remains not
1089  * uptodate - even if all of its buffers are uptodate.  A subsequent
1090  * block_read_full_page() against that page will discover all the uptodate
1091  * buffers, will set the page uptodate and will perform no I/O.
1092  */
1093 
1094 /**
1095  * mark_buffer_dirty - mark a buffer_head as needing writeout
1096  * @bh: the buffer_head to mark dirty
1097  *
1098  * mark_buffer_dirty() will set the dirty bit against the buffer, then set its
1099  * backing page dirty, then tag the page as dirty in its address_space's radix
1100  * tree and then attach the address_space's inode to its superblock's dirty
1101  * inode list.
1102  *
1103  * mark_buffer_dirty() is atomic.  It takes bh->b_page->mapping->private_lock,
1104  * mapping->tree_lock and mapping->host->i_lock.
1105  */
1106 void mark_buffer_dirty(struct buffer_head *bh)
1107 {
1108         WARN_ON_ONCE(!buffer_uptodate(bh));
1109 
1110         trace_block_dirty_buffer(bh);
1111 
1112         /*
1113          * Very *carefully* optimize the it-is-already-dirty case.
1114          *
1115          * Don't let the final "is it dirty" escape to before we
1116          * perhaps modified the buffer.
1117          */
1118         if (buffer_dirty(bh)) {
1119                 smp_mb();
1120                 if (buffer_dirty(bh))
1121                         return;
1122         }
1123 
1124         if (!test_set_buffer_dirty(bh)) {
1125                 struct page *page = bh->b_page;
1126                 struct address_space *mapping = NULL;
1127 
1128                 lock_page_memcg(page);
1129                 if (!TestSetPageDirty(page)) {
1130                         mapping = page_mapping(page);
1131                         if (mapping)
1132                                 __set_page_dirty(page, mapping, 0);
1133                 }
1134                 unlock_page_memcg(page);
1135                 if (mapping)
1136                         __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
1137         }
1138 }
1139 EXPORT_SYMBOL(mark_buffer_dirty);
1140 
1141 void mark_buffer_write_io_error(struct buffer_head *bh)
1142 {
1143         set_buffer_write_io_error(bh);
1144         /* FIXME: do we need to set this in both places? */
1145         if (bh->b_page && bh->b_page->mapping)
1146                 mapping_set_error(bh->b_page->mapping, -EIO);
1147         if (bh->b_assoc_map)
1148                 mapping_set_error(bh->b_assoc_map, -EIO);
1149 }
1150 EXPORT_SYMBOL(mark_buffer_write_io_error);
1151 
1152 /*
1153  * Decrement a buffer_head's reference count.  If all buffers against a page
1154  * have zero reference count, are clean and unlocked, and if the page is clean
1155  * and unlocked then try_to_free_buffers() may strip the buffers from the page
1156  * in preparation for freeing it (sometimes, rarely, buffers are removed from
1157  * a page but it ends up not being freed, and buffers may later be reattached).
1158  */
1159 void __brelse(struct buffer_head * buf)
1160 {
1161         if (atomic_read(&buf->b_count)) {
1162                 put_bh(buf);
1163                 return;
1164         }
1165         WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n");
1166 }
1167 EXPORT_SYMBOL(__brelse);
1168 
1169 /*
1170  * bforget() is like brelse(), except it discards any
1171  * potentially dirty data.
1172  */
1173 void __bforget(struct buffer_head *bh)
1174 {
1175         clear_buffer_dirty(bh);
1176         if (bh->b_assoc_map) {
1177                 struct address_space *buffer_mapping = bh->b_page->mapping;
1178 
1179                 spin_lock(&buffer_mapping->private_lock);
1180                 list_del_init(&bh->b_assoc_buffers);
1181                 bh->b_assoc_map = NULL;
1182                 spin_unlock(&buffer_mapping->private_lock);
1183         }
1184         __brelse(bh);
1185 }
1186 EXPORT_SYMBOL(__bforget);
1187 
1188 static struct buffer_head *__bread_slow(struct buffer_head *bh)
1189 {
1190         lock_buffer(bh);
1191         if (buffer_uptodate(bh)) {
1192                 unlock_buffer(bh);
1193                 return bh;
1194         } else {
1195                 get_bh(bh);
1196                 bh->b_end_io = end_buffer_read_sync;
1197                 submit_bh(REQ_OP_READ, 0, bh);
1198                 wait_on_buffer(bh);
1199                 if (buffer_uptodate(bh))
1200                         return bh;
1201         }
1202         brelse(bh);
1203         return NULL;
1204 }
1205 
1206 /*
1207  * Per-cpu buffer LRU implementation.  To reduce the cost of __find_get_block().
1208  * The bhs[] array is sorted - newest buffer is at bhs[0].  Buffers have their
1209  * refcount elevated by one when they're in an LRU.  A buffer can only appear
1210  * once in a particular CPU's LRU.  A single buffer can be present in multiple
1211  * CPU's LRUs at the same time.
1212  *
1213  * This is a transparent caching front-end to sb_bread(), sb_getblk() and
1214  * sb_find_get_block().
1215  *
1216  * The LRUs themselves only need locking against invalidate_bh_lrus.  We use
1217  * a local interrupt disable for that.
1218  */
1219 
1220 #define BH_LRU_SIZE     16
1221 
1222 struct bh_lru {
1223         struct buffer_head *bhs[BH_LRU_SIZE];
1224 };
1225 
1226 static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
1227 
1228 #ifdef CONFIG_SMP
1229 #define bh_lru_lock()   local_irq_disable()
1230 #define bh_lru_unlock() local_irq_enable()
1231 #else
1232 #define bh_lru_lock()   preempt_disable()
1233 #define bh_lru_unlock() preempt_enable()
1234 #endif
1235 
1236 static inline void check_irqs_on(void)
1237 {
1238 #ifdef irqs_disabled
1239         BUG_ON(irqs_disabled());
1240 #endif
1241 }
1242 
1243 /*
1244  * Install a buffer_head into this cpu's LRU.  If not already in the LRU, it is
1245  * inserted at the front, and the buffer_head at the back if any is evicted.
1246  * Or, if already in the LRU it is moved to the front.
1247  */
1248 static void bh_lru_install(struct buffer_head *bh)
1249 {
1250         struct buffer_head *evictee = bh;
1251         struct bh_lru *b;
1252         int i;
1253 
1254         check_irqs_on();
1255         bh_lru_lock();
1256 
1257         b = this_cpu_ptr(&bh_lrus);
1258         for (i = 0; i < BH_LRU_SIZE; i++) {
1259                 swap(evictee, b->bhs[i]);
1260                 if (evictee == bh) {
1261                         bh_lru_unlock();
1262                         return;
1263                 }
1264         }
1265 
1266         get_bh(bh);
1267         bh_lru_unlock();
1268         brelse(evictee);
1269 }
1270 
1271 /*
1272  * Look up the bh in this cpu's LRU.  If it's there, move it to the head.
1273  */
1274 static struct buffer_head *
1275 lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size)
1276 {
1277         struct buffer_head *ret = NULL;
1278         unsigned int i;
1279 
1280         check_irqs_on();
1281         bh_lru_lock();
1282         for (i = 0; i < BH_LRU_SIZE; i++) {
1283                 struct buffer_head *bh = __this_cpu_read(bh_lrus.bhs[i]);
1284 
1285                 if (bh && bh->b_blocknr == block && bh->b_bdev == bdev &&
1286                     bh->b_size == size) {
1287                         if (i) {
1288                                 while (i) {
1289                                         __this_cpu_write(bh_lrus.bhs[i],
1290                                                 __this_cpu_read(bh_lrus.bhs[i - 1]));
1291                                         i--;
1292                                 }
1293                                 __this_cpu_write(bh_lrus.bhs[0], bh);
1294                         }
1295                         get_bh(bh);
1296                         ret = bh;
1297                         break;
1298                 }
1299         }
1300         bh_lru_unlock();
1301         return ret;
1302 }
1303 
1304 /*
1305  * Perform a pagecache lookup for the matching buffer.  If it's there, refresh
1306  * it in the LRU and mark it as accessed.  If it is not present then return
1307  * NULL
1308  */
1309 struct buffer_head *
1310 __find_get_block(struct block_device *bdev, sector_t block, unsigned size)
1311 {
1312         struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
1313 
1314         if (bh == NULL) {
1315                 /* __find_get_block_slow will mark the page accessed */
1316                 bh = __find_get_block_slow(bdev, block);
1317                 if (bh)
1318                         bh_lru_install(bh);
1319         } else
1320                 touch_buffer(bh);
1321 
1322         return bh;
1323 }
1324 EXPORT_SYMBOL(__find_get_block);
1325 
1326 /*
1327  * __getblk_gfp() will locate (and, if necessary, create) the buffer_head
1328  * which corresponds to the passed block_device, block and size. The
1329  * returned buffer has its reference count incremented.
1330  *
1331  * __getblk_gfp() will lock up the machine if grow_dev_page's
1332  * try_to_free_buffers() attempt is failing.  FIXME, perhaps?
1333  */
1334 struct buffer_head *
1335 __getblk_gfp(struct block_device *bdev, sector_t block,
1336              unsigned size, gfp_t gfp)
1337 {
1338         struct buffer_head *bh = __find_get_block(bdev, block, size);
1339 
1340         might_sleep();
1341         if (bh == NULL)
1342                 bh = __getblk_slow(bdev, block, size, gfp);
1343         return bh;
1344 }
1345 EXPORT_SYMBOL(__getblk_gfp);
1346 
1347 /*
1348  * Do async read-ahead on a buffer..
1349  */
1350 void __breadahead(struct block_device *bdev, sector_t block, unsigned size)
1351 {
1352         struct buffer_head *bh = __getblk(bdev, block, size);
1353         if (likely(bh)) {
1354                 ll_rw_block(REQ_OP_READ, REQ_RAHEAD, 1, &bh);
1355                 brelse(bh);
1356         }
1357 }
1358 EXPORT_SYMBOL(__breadahead);
1359 
1360 /**
1361  *  __bread_gfp() - reads a specified block and returns the bh
1362  *  @bdev: the block_device to read from
1363  *  @block: number of block
1364  *  @size: size (in bytes) to read
1365  *  @gfp: page allocation flag
1366  *
1367  *  Reads a specified block, and returns buffer head that contains it.
1368  *  The page cache can be allocated from non-movable area
1369  *  not to prevent page migration if you set gfp to zero.
1370  *  It returns NULL if the block was unreadable.
1371  */
1372 struct buffer_head *
1373 __bread_gfp(struct block_device *bdev, sector_t block,
1374                    unsigned size, gfp_t gfp)
1375 {
1376         struct buffer_head *bh = __getblk_gfp(bdev, block, size, gfp);
1377 
1378         if (likely(bh) && !buffer_uptodate(bh))
1379                 bh = __bread_slow(bh);
1380         return bh;
1381 }
1382 EXPORT_SYMBOL(__bread_gfp);
1383 
1384 /*
1385  * invalidate_bh_lrus() is called rarely - but not only at unmount.
1386  * This doesn't race because it runs in each cpu either in irq
1387  * or with preempt disabled.
1388  */
1389 static void invalidate_bh_lru(void *arg)
1390 {
1391         struct bh_lru *b = &get_cpu_var(bh_lrus);
1392         int i;
1393 
1394         for (i = 0; i < BH_LRU_SIZE; i++) {
1395                 brelse(b->bhs[i]);
1396                 b->bhs[i] = NULL;
1397         }
1398         put_cpu_var(bh_lrus);
1399 }
1400 
1401 static bool has_bh_in_lru(int cpu, void *dummy)
1402 {
1403         struct bh_lru *b = per_cpu_ptr(&bh_lrus, cpu);
1404         int i;
1405         
1406         for (i = 0; i < BH_LRU_SIZE; i++) {
1407                 if (b->bhs[i])
1408                         return 1;
1409         }
1410 
1411         return 0;
1412 }
1413 
1414 void invalidate_bh_lrus(void)
1415 {
1416         on_each_cpu_cond(has_bh_in_lru, invalidate_bh_lru, NULL, 1, GFP_KERNEL);
1417 }
1418 EXPORT_SYMBOL_GPL(invalidate_bh_lrus);
1419 
1420 void set_bh_page(struct buffer_head *bh,
1421                 struct page *page, unsigned long offset)
1422 {
1423         bh->b_page = page;
1424         BUG_ON(offset >= PAGE_SIZE);
1425         if (PageHighMem(page))
1426                 /*
1427                  * This catches illegal uses and preserves the offset:
1428                  */
1429                 bh->b_data = (char *)(0 + offset);
1430         else
1431                 bh->b_data = page_address(page) + offset;
1432 }
1433 EXPORT_SYMBOL(set_bh_page);
1434 
1435 /*
1436  * Called when truncating a buffer on a page completely.
1437  */
1438 
1439 /* Bits that are cleared during an invalidate */
1440 #define BUFFER_FLAGS_DISCARD \
1441         (1 << BH_Mapped | 1 << BH_New | 1 << BH_Req | \
1442          1 << BH_Delay | 1 << BH_Unwritten)
1443 
1444 static void discard_buffer(struct buffer_head * bh)
1445 {
1446         unsigned long b_state, b_state_old;
1447 
1448         lock_buffer(bh);
1449         clear_buffer_dirty(bh);
1450         bh->b_bdev = NULL;
1451         b_state = bh->b_state;
1452         for (;;) {
1453                 b_state_old = cmpxchg(&bh->b_state, b_state,
1454                                       (b_state & ~BUFFER_FLAGS_DISCARD));
1455                 if (b_state_old == b_state)
1456                         break;
1457                 b_state = b_state_old;
1458         }
1459         unlock_buffer(bh);
1460 }
1461 
1462 /**
1463  * block_invalidatepage - invalidate part or all of a buffer-backed page
1464  *
1465  * @page: the page which is affected
1466  * @offset: start of the range to invalidate
1467  * @length: length of the range to invalidate
1468  *
1469  * block_invalidatepage() is called when all or part of the page has become
1470  * invalidated by a truncate operation.
1471  *
1472  * block_invalidatepage() does not have to release all buffers, but it must
1473  * ensure that no dirty buffer is left outside @offset and that no I/O
1474  * is underway against any of the blocks which are outside the truncation
1475  * point.  Because the caller is about to free (and possibly reuse) those
1476  * blocks on-disk.
1477  */
1478 void block_invalidatepage(struct page *page, unsigned int offset,
1479                           unsigned int length)
1480 {
1481         struct buffer_head *head, *bh, *next;
1482         unsigned int curr_off = 0;
1483         unsigned int stop = length + offset;
1484 
1485         BUG_ON(!PageLocked(page));
1486         if (!page_has_buffers(page))
1487                 goto out;
1488 
1489         /*
1490          * Check for overflow
1491          */
1492         BUG_ON(stop > PAGE_SIZE || stop < length);
1493 
1494         head = page_buffers(page);
1495         bh = head;
1496         do {
1497                 unsigned int next_off = curr_off + bh->b_size;
1498                 next = bh->b_this_page;
1499 
1500                 /*
1501                  * Are we still fully in range ?
1502                  */
1503                 if (next_off > stop)
1504                         goto out;
1505 
1506                 /*
1507                  * is this block fully invalidated?
1508                  */
1509                 if (offset <= curr_off)
1510                         discard_buffer(bh);
1511                 curr_off = next_off;
1512                 bh = next;
1513         } while (bh != head);
1514 
1515         /*
1516          * We release buffers only if the entire page is being invalidated.
1517          * The get_block cached value has been unconditionally invalidated,
1518          * so real IO is not possible anymore.
1519          */
1520         if (offset == 0)
1521                 try_to_release_page(page, 0);
1522 out:
1523         return;
1524 }
1525 EXPORT_SYMBOL(block_invalidatepage);
1526 
1527 
1528 /*
1529  * We attach and possibly dirty the buffers atomically wrt
1530  * __set_page_dirty_buffers() via private_lock.  try_to_free_buffers
1531  * is already excluded via the page lock.
1532  */
1533 void create_empty_buffers(struct page *page,
1534                         unsigned long blocksize, unsigned long b_state)
1535 {
1536         struct buffer_head *bh, *head, *tail;
1537 
1538         head = alloc_page_buffers(page, blocksize, true);
1539         bh = head;
1540         do {
1541                 bh->b_state |= b_state;
1542                 tail = bh;
1543                 bh = bh->b_this_page;
1544         } while (bh);
1545         tail->b_this_page = head;
1546 
1547         spin_lock(&page->mapping->private_lock);
1548         if (PageUptodate(page) || PageDirty(page)) {
1549                 bh = head;
1550                 do {
1551                         if (PageDirty(page))
1552                                 set_buffer_dirty(bh);
1553                         if (PageUptodate(page))
1554                                 set_buffer_uptodate(bh);
1555                         bh = bh->b_this_page;
1556                 } while (bh != head);
1557         }
1558         attach_page_buffers(page, head);
1559         spin_unlock(&page->mapping->private_lock);
1560 }
1561 EXPORT_SYMBOL(create_empty_buffers);
1562 
1563 /**
1564  * clean_bdev_aliases: clean a range of buffers in block device
1565  * @bdev: Block device to clean buffers in
1566  * @block: Start of a range of blocks to clean
1567  * @len: Number of blocks to clean
1568  *
1569  * We are taking a range of blocks for data and we don't want writeback of any
1570  * buffer-cache aliases starting from return from this function and until the
1571  * moment when something will explicitly mark the buffer dirty (hopefully that
1572  * will not happen until we will free that block ;-) We don't even need to mark
1573  * it not-uptodate - nobody can expect anything from a newly allocated buffer
1574  * anyway. We used to use unmap_buffer() for such invalidation, but that was
1575  * wrong. We definitely don't want to mark the alias unmapped, for example - it
1576  * would confuse anyone who might pick it with bread() afterwards...
1577  *
1578  * Also..  Note that bforget() doesn't lock the buffer.  So there can be
1579  * writeout I/O going on against recently-freed buffers.  We don't wait on that
1580  * I/O in bforget() - it's more efficient to wait on the I/O only if we really
1581  * need to.  That happens here.
1582  */
1583 void clean_bdev_aliases(struct block_device *bdev, sector_t block, sector_t len)
1584 {
1585         struct inode *bd_inode = bdev->bd_inode;
1586         struct address_space *bd_mapping = bd_inode->i_mapping;
1587         struct pagevec pvec;
1588         pgoff_t index = block >> (PAGE_SHIFT - bd_inode->i_blkbits);
1589         pgoff_t end;
1590         int i, count;
1591         struct buffer_head *bh;
1592         struct buffer_head *head;
1593 
1594         end = (block + len - 1) >> (PAGE_SHIFT - bd_inode->i_blkbits);
1595         pagevec_init(&pvec);
1596         while (pagevec_lookup_range(&pvec, bd_mapping, &index, end)) {
1597                 count = pagevec_count(&pvec);
1598                 for (i = 0; i < count; i++) {
1599                         struct page *page = pvec.pages[i];
1600 
1601                         if (!page_has_buffers(page))
1602                                 continue;
1603                         /*
1604                          * We use page lock instead of bd_mapping->private_lock
1605                          * to pin buffers here since we can afford to sleep and
1606                          * it scales better than a global spinlock lock.
1607                          */
1608                         lock_page(page);
1609                         /* Recheck when the page is locked which pins bhs */
1610                         if (!page_has_buffers(page))
1611                                 goto unlock_page;
1612                         head = page_buffers(page);
1613                         bh = head;
1614                         do {
1615                                 if (!buffer_mapped(bh) || (bh->b_blocknr < block))
1616                                         goto next;
1617                                 if (bh->b_blocknr >= block + len)
1618                                         break;
1619                                 clear_buffer_dirty(bh);
1620                                 wait_on_buffer(bh);
1621                                 clear_buffer_req(bh);
1622 next:
1623                                 bh = bh->b_this_page;
1624                         } while (bh != head);
1625 unlock_page:
1626                         unlock_page(page);
1627                 }
1628                 pagevec_release(&pvec);
1629                 cond_resched();
1630                 /* End of range already reached? */
1631                 if (index > end || !index)
1632                         break;
1633         }
1634 }
1635 EXPORT_SYMBOL(clean_bdev_aliases);
1636 
1637 /*
1638  * Size is a power-of-two in the range 512..PAGE_SIZE,
1639  * and the case we care about most is PAGE_SIZE.
1640  *
1641  * So this *could* possibly be written with those
1642  * constraints in mind (relevant mostly if some
1643  * architecture has a slow bit-scan instruction)
1644  */
1645 static inline int block_size_bits(unsigned int blocksize)
1646 {
1647         return ilog2(blocksize);
1648 }
1649 
1650 static struct buffer_head *create_page_buffers(struct page *page, struct inode *inode, unsigned int b_state)
1651 {
1652         BUG_ON(!PageLocked(page));
1653 
1654         if (!page_has_buffers(page))
1655                 create_empty_buffers(page, 1 << READ_ONCE(inode->i_blkbits),
1656                                      b_state);
1657         return page_buffers(page);
1658 }
1659 
1660 /*
1661  * NOTE! All mapped/uptodate combinations are valid:
1662  *
1663  *      Mapped  Uptodate        Meaning
1664  *
1665  *      No      No              "unknown" - must do get_block()
1666  *      No      Yes             "hole" - zero-filled
1667  *      Yes     No              "allocated" - allocated on disk, not read in
1668  *      Yes     Yes             "valid" - allocated and up-to-date in memory.
1669  *
1670  * "Dirty" is valid only with the last case (mapped+uptodate).
1671  */
1672 
1673 /*
1674  * While block_write_full_page is writing back the dirty buffers under
1675  * the page lock, whoever dirtied the buffers may decide to clean them
1676  * again at any time.  We handle that by only looking at the buffer
1677  * state inside lock_buffer().
1678  *
1679  * If block_write_full_page() is called for regular writeback
1680  * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1681  * locked buffer.   This only can happen if someone has written the buffer
1682  * directly, with submit_bh().  At the address_space level PageWriteback
1683  * prevents this contention from occurring.
1684  *
1685  * If block_write_full_page() is called with wbc->sync_mode ==
1686  * WB_SYNC_ALL, the writes are posted using REQ_SYNC; this
1687  * causes the writes to be flagged as synchronous writes.
1688  */
1689 int __block_write_full_page(struct inode *inode, struct page *page,
1690                         get_block_t *get_block, struct writeback_control *wbc,
1691                         bh_end_io_t *handler)
1692 {
1693         int err;
1694         sector_t block;
1695         sector_t last_block;
1696         struct buffer_head *bh, *head;
1697         unsigned int blocksize, bbits;
1698         int nr_underway = 0;
1699         int write_flags = wbc_to_write_flags(wbc);
1700 
1701         head = create_page_buffers(page, inode,
1702                                         (1 << BH_Dirty)|(1 << BH_Uptodate));
1703 
1704         /*
1705          * Be very careful.  We have no exclusion from __set_page_dirty_buffers
1706          * here, and the (potentially unmapped) buffers may become dirty at
1707          * any time.  If a buffer becomes dirty here after we've inspected it
1708          * then we just miss that fact, and the page stays dirty.
1709          *
1710          * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
1711          * handle that here by just cleaning them.
1712          */
1713 
1714         bh = head;
1715         blocksize = bh->b_size;
1716         bbits = block_size_bits(blocksize);
1717 
1718         block = (sector_t)page->index << (PAGE_SHIFT - bbits);
1719         last_block = (i_size_read(inode) - 1) >> bbits;
1720 
1721         /*
1722          * Get all the dirty buffers mapped to disk addresses and
1723          * handle any aliases from the underlying blockdev's mapping.
1724          */
1725         do {
1726                 if (block > last_block) {
1727                         /*
1728                          * mapped buffers outside i_size will occur, because
1729                          * this page can be outside i_size when there is a
1730                          * truncate in progress.
1731                          */
1732                         /*
1733                          * The buffer was zeroed by block_write_full_page()
1734                          */
1735                         clear_buffer_dirty(bh);
1736                         set_buffer_uptodate(bh);
1737                 } else if ((!buffer_mapped(bh) || buffer_delay(bh)) &&
1738                            buffer_dirty(bh)) {
1739                         WARN_ON(bh->b_size != blocksize);
1740                         err = get_block(inode, block, bh, 1);
1741                         if (err)
1742                                 goto recover;
1743                         clear_buffer_delay(bh);
1744                         if (buffer_new(bh)) {
1745                                 /* blockdev mappings never come here */
1746                                 clear_buffer_new(bh);
1747                                 clean_bdev_bh_alias(bh);
1748                         }
1749                 }
1750                 bh = bh->b_this_page;
1751                 block++;
1752         } while (bh != head);
1753 
1754         do {
1755                 if (!buffer_mapped(bh))
1756                         continue;
1757                 /*
1758                  * If it's a fully non-blocking write attempt and we cannot
1759                  * lock the buffer then redirty the page.  Note that this can
1760                  * potentially cause a busy-wait loop from writeback threads
1761                  * and kswapd activity, but those code paths have their own
1762                  * higher-level throttling.
1763                  */
1764                 if (wbc->sync_mode != WB_SYNC_NONE) {
1765                         lock_buffer(bh);
1766                 } else if (!trylock_buffer(bh)) {
1767                         redirty_page_for_writepage(wbc, page);
1768                         continue;
1769                 }
1770                 if (test_clear_buffer_dirty(bh)) {
1771                         mark_buffer_async_write_endio(bh, handler);
1772                 } else {
1773                         unlock_buffer(bh);
1774                 }
1775         } while ((bh = bh->b_this_page) != head);
1776 
1777         /*
1778          * The page and its buffers are protected by PageWriteback(), so we can
1779          * drop the bh refcounts early.
1780          */
1781         BUG_ON(PageWriteback(page));
1782         set_page_writeback(page);
1783 
1784         do {
1785                 struct buffer_head *next = bh->b_this_page;
1786                 if (buffer_async_write(bh)) {
1787                         submit_bh_wbc(REQ_OP_WRITE, write_flags, bh,
1788                                         inode->i_write_hint, wbc);
1789                         nr_underway++;
1790                 }
1791                 bh = next;
1792         } while (bh != head);
1793         unlock_page(page);
1794 
1795         err = 0;
1796 done:
1797         if (nr_underway == 0) {
1798                 /*
1799                  * The page was marked dirty, but the buffers were
1800                  * clean.  Someone wrote them back by hand with
1801                  * ll_rw_block/submit_bh.  A rare case.
1802                  */
1803                 end_page_writeback(page);
1804 
1805                 /*
1806                  * The page and buffer_heads can be released at any time from
1807                  * here on.
1808                  */
1809         }
1810         return err;
1811 
1812 recover:
1813         /*
1814          * ENOSPC, or some other error.  We may already have added some
1815          * blocks to the file, so we need to write these out to avoid
1816          * exposing stale data.
1817          * The page is currently locked and not marked for writeback
1818          */
1819         bh = head;
1820         /* Recovery: lock and submit the mapped buffers */
1821         do {
1822                 if (buffer_mapped(bh) && buffer_dirty(bh) &&
1823                     !buffer_delay(bh)) {
1824                         lock_buffer(bh);
1825                         mark_buffer_async_write_endio(bh, handler);
1826                 } else {
1827                         /*
1828                          * The buffer may have been set dirty during
1829                          * attachment to a dirty page.
1830                          */
1831                         clear_buffer_dirty(bh);
1832                 }
1833         } while ((bh = bh->b_this_page) != head);
1834         SetPageError(page);
1835         BUG_ON(PageWriteback(page));
1836         mapping_set_error(page->mapping, err);
1837         set_page_writeback(page);
1838         do {
1839                 struct buffer_head *next = bh->b_this_page;
1840                 if (buffer_async_write(bh)) {
1841                         clear_buffer_dirty(bh);
1842                         submit_bh_wbc(REQ_OP_WRITE, write_flags, bh,
1843                                         inode->i_write_hint, wbc);
1844                         nr_underway++;
1845                 }
1846                 bh = next;
1847         } while (bh != head);
1848         unlock_page(page);
1849         goto done;
1850 }
1851 EXPORT_SYMBOL(__block_write_full_page);
1852 
1853 /*
1854  * If a page has any new buffers, zero them out here, and mark them uptodate
1855  * and dirty so they'll be written out (in order to prevent uninitialised
1856  * block data from leaking). And clear the new bit.
1857  */
1858 void page_zero_new_buffers(struct page *page, unsigned from, unsigned to)
1859 {
1860         unsigned int block_start, block_end;
1861         struct buffer_head *head, *bh;
1862 
1863         BUG_ON(!PageLocked(page));
1864         if (!page_has_buffers(page))
1865                 return;
1866 
1867         bh = head = page_buffers(page);
1868         block_start = 0;
1869         do {
1870                 block_end = block_start + bh->b_size;
1871 
1872                 if (buffer_new(bh)) {
1873                         if (block_end > from && block_start < to) {
1874                                 if (!PageUptodate(page)) {
1875                                         unsigned start, size;
1876 
1877                                         start = max(from, block_start);
1878                                         size = min(to, block_end) - start;
1879 
1880                                         zero_user(page, start, size);
1881                                         set_buffer_uptodate(bh);
1882                                 }
1883 
1884                                 clear_buffer_new(bh);
1885                                 mark_buffer_dirty(bh);
1886                         }
1887                 }
1888 
1889                 block_start = block_end;
1890                 bh = bh->b_this_page;
1891         } while (bh != head);
1892 }
1893 EXPORT_SYMBOL(page_zero_new_buffers);
1894 
1895 static void
1896 iomap_to_bh(struct inode *inode, sector_t block, struct buffer_head *bh,
1897                 struct iomap *iomap)
1898 {
1899         loff_t offset = block << inode->i_blkbits;
1900 
1901         bh->b_bdev = iomap->bdev;
1902 
1903         /*
1904          * Block points to offset in file we need to map, iomap contains
1905          * the offset at which the map starts. If the map ends before the
1906          * current block, then do not map the buffer and let the caller
1907          * handle it.
1908          */
1909         BUG_ON(offset >= iomap->offset + iomap->length);
1910 
1911         switch (iomap->type) {
1912         case IOMAP_HOLE:
1913                 /*
1914                  * If the buffer is not up to date or beyond the current EOF,
1915                  * we need to mark it as new to ensure sub-block zeroing is
1916                  * executed if necessary.
1917                  */
1918                 if (!buffer_uptodate(bh) ||
1919                     (offset >= i_size_read(inode)))
1920                         set_buffer_new(bh);
1921                 break;
1922         case IOMAP_DELALLOC:
1923                 if (!buffer_uptodate(bh) ||
1924                     (offset >= i_size_read(inode)))
1925                         set_buffer_new(bh);
1926                 set_buffer_uptodate(bh);
1927                 set_buffer_mapped(bh);
1928                 set_buffer_delay(bh);
1929                 break;
1930         case IOMAP_UNWRITTEN:
1931                 /*
1932                  * For unwritten regions, we always need to ensure that
1933                  * sub-block writes cause the regions in the block we are not
1934                  * writing to are zeroed. Set the buffer as new to ensure this.
1935                  */
1936                 set_buffer_new(bh);
1937                 set_buffer_unwritten(bh);
1938                 /* FALLTHRU */
1939         case IOMAP_MAPPED:
1940                 if (offset >= i_size_read(inode))
1941                         set_buffer_new(bh);
1942                 bh->b_blocknr = (iomap->addr + offset - iomap->offset) >>
1943                                 inode->i_blkbits;
1944                 set_buffer_mapped(bh);
1945                 break;
1946         }
1947 }
1948 
1949 int __block_write_begin_int(struct page *page, loff_t pos, unsigned len,
1950                 get_block_t *get_block, struct iomap *iomap)
1951 {
1952         unsigned from = pos & (PAGE_SIZE - 1);
1953         unsigned to = from + len;
1954         struct inode *inode = page->mapping->host;
1955         unsigned block_start, block_end;
1956         sector_t block;
1957         int err = 0;
1958         unsigned blocksize, bbits;
1959         struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
1960 
1961         BUG_ON(!PageLocked(page));
1962         BUG_ON(from > PAGE_SIZE);
1963         BUG_ON(to > PAGE_SIZE);
1964         BUG_ON(from > to);
1965 
1966         head = create_page_buffers(page, inode, 0);
1967         blocksize = head->b_size;
1968         bbits = block_size_bits(blocksize);
1969 
1970         block = (sector_t)page->index << (PAGE_SHIFT - bbits);
1971 
1972         for(bh = head, block_start = 0; bh != head || !block_start;
1973             block++, block_start=block_end, bh = bh->b_this_page) {
1974                 block_end = block_start + blocksize;
1975                 if (block_end <= from || block_start >= to) {
1976                         if (PageUptodate(page)) {
1977                                 if (!buffer_uptodate(bh))
1978                                         set_buffer_uptodate(bh);
1979                         }
1980                         continue;
1981                 }
1982                 if (buffer_new(bh))
1983                         clear_buffer_new(bh);
1984                 if (!buffer_mapped(bh)) {
1985                         WARN_ON(bh->b_size != blocksize);
1986                         if (get_block) {
1987                                 err = get_block(inode, block, bh, 1);
1988                                 if (err)
1989                                         break;
1990                         } else {
1991                                 iomap_to_bh(inode, block, bh, iomap);
1992                         }
1993 
1994                         if (buffer_new(bh)) {
1995                                 clean_bdev_bh_alias(bh);
1996                                 if (PageUptodate(page)) {
1997                                         clear_buffer_new(bh);
1998                                         set_buffer_uptodate(bh);
1999                                         mark_buffer_dirty(bh);
2000                                         continue;
2001                                 }
2002                                 if (block_end > to || block_start < from)
2003                                         zero_user_segments(page,
2004                                                 to, block_end,
2005                                                 block_start, from);
2006                                 continue;
2007                         }
2008                 }
2009                 if (PageUptodate(page)) {
2010                         if (!buffer_uptodate(bh))
2011                                 set_buffer_uptodate(bh);
2012                         continue; 
2013                 }
2014                 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
2015                     !buffer_unwritten(bh) &&
2016                      (block_start < from || block_end > to)) {
2017                         ll_rw_block(REQ_OP_READ, 0, 1, &bh);
2018                         *wait_bh++=bh;
2019                 }
2020         }
2021         /*
2022          * If we issued read requests - let them complete.
2023          */
2024         while(wait_bh > wait) {
2025                 wait_on_buffer(*--wait_bh);
2026                 if (!buffer_uptodate(*wait_bh))
2027                         err = -EIO;
2028         }
2029         if (unlikely(err))
2030                 page_zero_new_buffers(page, from, to);
2031         return err;
2032 }
2033 
2034 int __block_write_begin(struct page *page, loff_t pos, unsigned len,
2035                 get_block_t *get_block)
2036 {
2037         return __block_write_begin_int(page, pos, len, get_block, NULL);
2038 }
2039 EXPORT_SYMBOL(__block_write_begin);
2040 
2041 static int __block_commit_write(struct inode *inode, struct page *page,
2042                 unsigned from, unsigned to)
2043 {
2044         unsigned block_start, block_end;
2045         int partial = 0;
2046         unsigned blocksize;
2047         struct buffer_head *bh, *head;
2048 
2049         bh = head = page_buffers(page);
2050         blocksize = bh->b_size;
2051 
2052         block_start = 0;
2053         do {
2054                 block_end = block_start + blocksize;
2055                 if (block_end <= from || block_start >= to) {
2056                         if (!buffer_uptodate(bh))
2057                                 partial = 1;
2058                 } else {
2059                         set_buffer_uptodate(bh);
2060                         mark_buffer_dirty(bh);
2061                 }
2062                 clear_buffer_new(bh);
2063 
2064                 block_start = block_end;
2065                 bh = bh->b_this_page;
2066         } while (bh != head);
2067 
2068         /*
2069          * If this is a partial write which happened to make all buffers
2070          * uptodate then we can optimize away a bogus readpage() for
2071          * the next read(). Here we 'discover' whether the page went
2072          * uptodate as a result of this (potentially partial) write.
2073          */
2074         if (!partial)
2075                 SetPageUptodate(page);
2076         return 0;
2077 }
2078 
2079 /*
2080  * block_write_begin takes care of the basic task of block allocation and
2081  * bringing partial write blocks uptodate first.
2082  *
2083  * The filesystem needs to handle block truncation upon failure.
2084  */
2085 int block_write_begin(struct address_space *mapping, loff_t pos, unsigned len,
2086                 unsigned flags, struct page **pagep, get_block_t *get_block)
2087 {
2088         pgoff_t index = pos >> PAGE_SHIFT;
2089         struct page *page;
2090         int status;
2091 
2092         page = grab_cache_page_write_begin(mapping, index, flags);
2093         if (!page)
2094                 return -ENOMEM;
2095 
2096         status = __block_write_begin(page, pos, len, get_block);
2097         if (unlikely(status)) {
2098                 unlock_page(page);
2099                 put_page(page);
2100                 page = NULL;
2101         }
2102 
2103         *pagep = page;
2104         return status;
2105 }
2106 EXPORT_SYMBOL(block_write_begin);
2107 
2108 int block_write_end(struct file *file, struct address_space *mapping,
2109                         loff_t pos, unsigned len, unsigned copied,
2110                         struct page *page, void *fsdata)
2111 {
2112         struct inode *inode = mapping->host;
2113         unsigned start;
2114 
2115         start = pos & (PAGE_SIZE - 1);
2116 
2117         if (unlikely(copied < len)) {
2118                 /*
2119                  * The buffers that were written will now be uptodate, so we
2120                  * don't have to worry about a readpage reading them and
2121                  * overwriting a partial write. However if we have encountered
2122                  * a short write and only partially written into a buffer, it
2123                  * will not be marked uptodate, so a readpage might come in and
2124                  * destroy our partial write.
2125                  *
2126                  * Do the simplest thing, and just treat any short write to a
2127                  * non uptodate page as a zero-length write, and force the
2128                  * caller to redo the whole thing.
2129                  */
2130                 if (!PageUptodate(page))
2131                         copied = 0;
2132 
2133                 page_zero_new_buffers(page, start+copied, start+len);
2134         }
2135         flush_dcache_page(page);
2136 
2137         /* This could be a short (even 0-length) commit */
2138         __block_commit_write(inode, page, start, start+copied);
2139 
2140         return copied;
2141 }
2142 EXPORT_SYMBOL(block_write_end);
2143 
2144 int generic_write_end(struct file *file, struct address_space *mapping,
2145                         loff_t pos, unsigned len, unsigned copied,
2146                         struct page *page, void *fsdata)
2147 {
2148         struct inode *inode = mapping->host;
2149         loff_t old_size = inode->i_size;
2150         int i_size_changed = 0;
2151 
2152         copied = block_write_end(file, mapping, pos, len, copied, page, fsdata);
2153 
2154         /*
2155          * No need to use i_size_read() here, the i_size
2156          * cannot change under us because we hold i_mutex.
2157          *
2158          * But it's important to update i_size while still holding page lock:
2159          * page writeout could otherwise come in and zero beyond i_size.
2160          */
2161         if (pos+copied > inode->i_size) {
2162                 i_size_write(inode, pos+copied);
2163                 i_size_changed = 1;
2164         }
2165 
2166         unlock_page(page);
2167         put_page(page);
2168 
2169         if (old_size < pos)
2170                 pagecache_isize_extended(inode, old_size, pos);
2171         /*
2172          * Don't mark the inode dirty under page lock. First, it unnecessarily
2173          * makes the holding time of page lock longer. Second, it forces lock
2174          * ordering of page lock and transaction start for journaling
2175          * filesystems.
2176          */
2177         if (i_size_changed)
2178                 mark_inode_dirty(inode);
2179 
2180         return copied;
2181 }
2182 EXPORT_SYMBOL(generic_write_end);
2183 
2184 /*
2185  * block_is_partially_uptodate checks whether buffers within a page are
2186  * uptodate or not.
2187  *
2188  * Returns true if all buffers which correspond to a file portion
2189  * we want to read are uptodate.
2190  */
2191 int block_is_partially_uptodate(struct page *page, unsigned long from,
2192                                         unsigned long count)
2193 {
2194         unsigned block_start, block_end, blocksize;
2195         unsigned to;
2196         struct buffer_head *bh, *head;
2197         int ret = 1;
2198 
2199         if (!page_has_buffers(page))
2200                 return 0;
2201 
2202         head = page_buffers(page);
2203         blocksize = head->b_size;
2204         to = min_t(unsigned, PAGE_SIZE - from, count);
2205         to = from + to;
2206         if (from < blocksize && to > PAGE_SIZE - blocksize)
2207                 return 0;
2208 
2209         bh = head;
2210         block_start = 0;
2211         do {
2212                 block_end = block_start + blocksize;
2213                 if (block_end > from && block_start < to) {
2214                         if (!buffer_uptodate(bh)) {
2215                                 ret = 0;
2216                                 break;
2217                         }
2218                         if (block_end >= to)
2219                                 break;
2220                 }
2221                 block_start = block_end;
2222                 bh = bh->b_this_page;
2223         } while (bh != head);
2224 
2225         return ret;
2226 }
2227 EXPORT_SYMBOL(block_is_partially_uptodate);
2228 
2229 /*
2230  * Generic "read page" function for block devices that have the normal
2231  * get_block functionality. This is most of the block device filesystems.
2232  * Reads the page asynchronously --- the unlock_buffer() and
2233  * set/clear_buffer_uptodate() functions propagate buffer state into the
2234  * page struct once IO has completed.
2235  */
2236 int block_read_full_page(struct page *page, get_block_t *get_block)
2237 {
2238         struct inode *inode = page->mapping->host;
2239         sector_t iblock, lblock;
2240         struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
2241         unsigned int blocksize, bbits;
2242         int nr, i;
2243         int fully_mapped = 1;
2244 
2245         head = create_page_buffers(page, inode, 0);
2246         blocksize = head->b_size;
2247         bbits = block_size_bits(blocksize);
2248 
2249         iblock = (sector_t)page->index << (PAGE_SHIFT - bbits);
2250         lblock = (i_size_read(inode)+blocksize-1) >> bbits;
2251         bh = head;
2252         nr = 0;
2253         i = 0;
2254 
2255         do {
2256                 if (buffer_uptodate(bh))
2257                         continue;
2258 
2259                 if (!buffer_mapped(bh)) {
2260                         int err = 0;
2261 
2262                         fully_mapped = 0;
2263                         if (iblock < lblock) {
2264                                 WARN_ON(bh->b_size != blocksize);
2265                                 err = get_block(inode, iblock, bh, 0);
2266                                 if (err)
2267                                         SetPageError(page);
2268                         }
2269                         if (!buffer_mapped(bh)) {
2270                                 zero_user(page, i * blocksize, blocksize);
2271                                 if (!err)
2272                                         set_buffer_uptodate(bh);
2273                                 continue;
2274                         }
2275                         /*
2276                          * get_block() might have updated the buffer
2277                          * synchronously
2278                          */
2279                         if (buffer_uptodate(bh))
2280                                 continue;
2281                 }
2282                 arr[nr++] = bh;
2283         } while (i++, iblock++, (bh = bh->b_this_page) != head);
2284 
2285         if (fully_mapped)
2286                 SetPageMappedToDisk(page);
2287 
2288         if (!nr) {
2289                 /*
2290                  * All buffers are uptodate - we can set the page uptodate
2291                  * as well. But not if get_block() returned an error.
2292                  */
2293                 if (!PageError(page))
2294                         SetPageUptodate(page);
2295                 unlock_page(page);
2296                 return 0;
2297         }
2298 
2299         /* Stage two: lock the buffers */
2300         for (i = 0; i < nr; i++) {
2301                 bh = arr[i];
2302                 lock_buffer(bh);
2303                 mark_buffer_async_read(bh);
2304         }
2305 
2306         /*
2307          * Stage 3: start the IO.  Check for uptodateness
2308          * inside the buffer lock in case another process reading
2309          * the underlying blockdev brought it uptodate (the sct fix).
2310          */
2311         for (i = 0; i < nr; i++) {
2312                 bh = arr[i];
2313                 if (buffer_uptodate(bh))
2314                         end_buffer_async_read(bh, 1);
2315                 else
2316                         submit_bh(REQ_OP_READ, 0, bh);
2317         }
2318         return 0;
2319 }
2320 EXPORT_SYMBOL(block_read_full_page);
2321 
2322 /* utility function for filesystems that need to do work on expanding
2323  * truncates.  Uses filesystem pagecache writes to allow the filesystem to
2324  * deal with the hole.  
2325  */
2326 int generic_cont_expand_simple(struct inode *inode, loff_t size)
2327 {
2328         struct address_space *mapping = inode->i_mapping;
2329         struct page *page;
2330         void *fsdata;
2331         int err;
2332 
2333         err = inode_newsize_ok(inode, size);
2334         if (err)
2335                 goto out;
2336 
2337         err = pagecache_write_begin(NULL, mapping, size, 0,
2338                                     AOP_FLAG_CONT_EXPAND, &page, &fsdata);
2339         if (err)
2340                 goto out;
2341 
2342         err = pagecache_write_end(NULL, mapping, size, 0, 0, page, fsdata);
2343         BUG_ON(err > 0);
2344 
2345 out:
2346         return err;
2347 }
2348 EXPORT_SYMBOL(generic_cont_expand_simple);
2349 
2350 static int cont_expand_zero(struct file *file, struct address_space *mapping,
2351                             loff_t pos, loff_t *bytes)
2352 {
2353         struct inode *inode = mapping->host;
2354         unsigned int blocksize = i_blocksize(inode);
2355         struct page *page;
2356         void *fsdata;
2357         pgoff_t index, curidx;
2358         loff_t curpos;
2359         unsigned zerofrom, offset, len;
2360         int err = 0;
2361 
2362         index = pos >> PAGE_SHIFT;
2363         offset = pos & ~PAGE_MASK;
2364 
2365         while (index > (curidx = (curpos = *bytes)>>PAGE_SHIFT)) {
2366                 zerofrom = curpos & ~PAGE_MASK;
2367                 if (zerofrom & (blocksize-1)) {
2368                         *bytes |= (blocksize-1);
2369                         (*bytes)++;
2370                 }
2371                 len = PAGE_SIZE - zerofrom;
2372 
2373                 err = pagecache_write_begin(file, mapping, curpos, len, 0,
2374                                             &page, &fsdata);
2375                 if (err)
2376                         goto out;
2377                 zero_user(page, zerofrom, len);
2378                 err = pagecache_write_end(file, mapping, curpos, len, len,
2379                                                 page, fsdata);
2380                 if (err < 0)
2381                         goto out;
2382                 BUG_ON(err != len);
2383                 err = 0;
2384 
2385                 balance_dirty_pages_ratelimited(mapping);
2386 
2387                 if (unlikely(fatal_signal_pending(current))) {
2388                         err = -EINTR;
2389                         goto out;
2390                 }
2391         }
2392 
2393         /* page covers the boundary, find the boundary offset */
2394         if (index == curidx) {
2395                 zerofrom = curpos & ~PAGE_MASK;
2396                 /* if we will expand the thing last block will be filled */
2397                 if (offset <= zerofrom) {
2398                         goto out;
2399                 }
2400                 if (zerofrom & (blocksize-1)) {
2401                         *bytes |= (blocksize-1);
2402                         (*bytes)++;
2403                 }
2404                 len = offset - zerofrom;
2405 
2406                 err = pagecache_write_begin(file, mapping, curpos, len, 0,
2407                                             &page, &fsdata);
2408                 if (err)
2409                         goto out;
2410                 zero_user(page, zerofrom, len);
2411                 err = pagecache_write_end(file, mapping, curpos, len, len,
2412                                                 page, fsdata);
2413                 if (err < 0)
2414                         goto out;
2415                 BUG_ON(err != len);
2416                 err = 0;
2417         }
2418 out:
2419         return err;
2420 }
2421 
2422 /*
2423  * For moronic filesystems that do not allow holes in file.
2424  * We may have to extend the file.
2425  */
2426 int cont_write_begin(struct file *file, struct address_space *mapping,
2427                         loff_t pos, unsigned len, unsigned flags,
2428                         struct page **pagep, void **fsdata,
2429                         get_block_t *get_block, loff_t *bytes)
2430 {
2431         struct inode *inode = mapping->host;
2432         unsigned int blocksize = i_blocksize(inode);
2433         unsigned int zerofrom;
2434         int err;
2435 
2436         err = cont_expand_zero(file, mapping, pos, bytes);
2437         if (err)
2438                 return err;
2439 
2440         zerofrom = *bytes & ~PAGE_MASK;
2441         if (pos+len > *bytes && zerofrom & (blocksize-1)) {
2442                 *bytes |= (blocksize-1);
2443                 (*bytes)++;
2444         }
2445 
2446         return block_write_begin(mapping, pos, len, flags, pagep, get_block);
2447 }
2448 EXPORT_SYMBOL(cont_write_begin);
2449 
2450 int block_commit_write(struct page *page, unsigned from, unsigned to)
2451 {
2452         struct inode *inode = page->mapping->host;
2453         __block_commit_write(inode,page,from,to);
2454         return 0;
2455 }
2456 EXPORT_SYMBOL(block_commit_write);
2457 
2458 /*
2459  * block_page_mkwrite() is not allowed to change the file size as it gets
2460  * called from a page fault handler when a page is first dirtied. Hence we must
2461  * be careful to check for EOF conditions here. We set the page up correctly
2462  * for a written page which means we get ENOSPC checking when writing into
2463  * holes and correct delalloc and unwritten extent mapping on filesystems that
2464  * support these features.
2465  *
2466  * We are not allowed to take the i_mutex here so we have to play games to
2467  * protect against truncate races as the page could now be beyond EOF.  Because
2468  * truncate writes the inode size before removing pages, once we have the
2469  * page lock we can determine safely if the page is beyond EOF. If it is not
2470  * beyond EOF, then the page is guaranteed safe against truncation until we
2471  * unlock the page.
2472  *
2473  * Direct callers of this function should protect against filesystem freezing
2474  * using sb_start_pagefault() - sb_end_pagefault() functions.
2475  */
2476 int block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
2477                          get_block_t get_block)
2478 {
2479         struct page *page = vmf->page;
2480         struct inode *inode = file_inode(vma->vm_file);
2481         unsigned long end;
2482         loff_t size;
2483         int ret;
2484 
2485         lock_page(page);
2486         size = i_size_read(inode);
2487         if ((page->mapping != inode->i_mapping) ||
2488             (page_offset(page) > size)) {
2489                 /* We overload EFAULT to mean page got truncated */
2490                 ret = -EFAULT;
2491                 goto out_unlock;
2492         }
2493 
2494         /* page is wholly or partially inside EOF */
2495         if (((page->index + 1) << PAGE_SHIFT) > size)
2496                 end = size & ~PAGE_MASK;
2497         else
2498                 end = PAGE_SIZE;
2499 
2500         ret = __block_write_begin(page, 0, end, get_block);
2501         if (!ret)
2502                 ret = block_commit_write(page, 0, end);
2503 
2504         if (unlikely(ret < 0))
2505                 goto out_unlock;
2506         set_page_dirty(page);
2507         wait_for_stable_page(page);
2508         return 0;
2509 out_unlock:
2510         unlock_page(page);
2511         return ret;
2512 }
2513 EXPORT_SYMBOL(block_page_mkwrite);
2514 
2515 /*
2516  * nobh_write_begin()'s prereads are special: the buffer_heads are freed
2517  * immediately, while under the page lock.  So it needs a special end_io
2518  * handler which does not touch the bh after unlocking it.
2519  */
2520 static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
2521 {
2522         __end_buffer_read_notouch(bh, uptodate);
2523 }
2524 
2525 /*
2526  * Attach the singly-linked list of buffers created by nobh_write_begin, to
2527  * the page (converting it to circular linked list and taking care of page
2528  * dirty races).
2529  */
2530 static void attach_nobh_buffers(struct page *page, struct buffer_head *head)
2531 {
2532         struct buffer_head *bh;
2533 
2534         BUG_ON(!PageLocked(page));
2535 
2536         spin_lock(&page->mapping->private_lock);
2537         bh = head;
2538         do {
2539                 if (PageDirty(page))
2540                         set_buffer_dirty(bh);
2541                 if (!bh->b_this_page)
2542                         bh->b_this_page = head;
2543                 bh = bh->b_this_page;
2544         } while (bh != head);
2545         attach_page_buffers(page, head);
2546         spin_unlock(&page->mapping->private_lock);
2547 }
2548 
2549 /*
2550  * On entry, the page is fully not uptodate.
2551  * On exit the page is fully uptodate in the areas outside (from,to)
2552  * The filesystem needs to handle block truncation upon failure.
2553  */
2554 int nobh_write_begin(struct address_space *mapping,
2555                         loff_t pos, unsigned len, unsigned flags,
2556                         struct page **pagep, void **fsdata,
2557                         get_block_t *get_block)
2558 {
2559         struct inode *inode = mapping->host;
2560         const unsigned blkbits = inode->i_blkbits;
2561         const unsigned blocksize = 1 << blkbits;
2562         struct buffer_head *head, *bh;
2563         struct page *page;
2564         pgoff_t index;
2565         unsigned from, to;
2566         unsigned block_in_page;
2567         unsigned block_start, block_end;
2568         sector_t block_in_file;
2569         int nr_reads = 0;
2570         int ret = 0;
2571         int is_mapped_to_disk = 1;
2572 
2573         index = pos >> PAGE_SHIFT;
2574         from = pos & (PAGE_SIZE - 1);
2575         to = from + len;
2576 
2577         page = grab_cache_page_write_begin(mapping, index, flags);
2578         if (!page)
2579                 return -ENOMEM;
2580         *pagep = page;
2581         *fsdata = NULL;
2582 
2583         if (page_has_buffers(page)) {
2584                 ret = __block_write_begin(page, pos, len, get_block);
2585                 if (unlikely(ret))
2586                         goto out_release;
2587                 return ret;
2588         }
2589 
2590         if (PageMappedToDisk(page))
2591                 return 0;
2592 
2593         /*
2594          * Allocate buffers so that we can keep track of state, and potentially
2595          * attach them to the page if an error occurs. In the common case of
2596          * no error, they will just be freed again without ever being attached
2597          * to the page (which is all OK, because we're under the page lock).
2598          *
2599          * Be careful: the buffer linked list is a NULL terminated one, rather
2600          * than the circular one we're used to.
2601          */
2602         head = alloc_page_buffers(page, blocksize, false);
2603         if (!head) {
2604                 ret = -ENOMEM;
2605                 goto out_release;
2606         }
2607 
2608         block_in_file = (sector_t)page->index << (PAGE_SHIFT - blkbits);
2609 
2610         /*
2611          * We loop across all blocks in the page, whether or not they are
2612          * part of the affected region.  This is so we can discover if the
2613          * page is fully mapped-to-disk.
2614          */
2615         for (block_start = 0, block_in_page = 0, bh = head;
2616                   block_start < PAGE_SIZE;
2617                   block_in_page++, block_start += blocksize, bh = bh->b_this_page) {
2618                 int create;
2619 
2620                 block_end = block_start + blocksize;
2621                 bh->b_state = 0;
2622                 create = 1;
2623                 if (block_start >= to)
2624                         create = 0;
2625                 ret = get_block(inode, block_in_file + block_in_page,
2626                                         bh, create);
2627                 if (ret)
2628                         goto failed;
2629                 if (!buffer_mapped(bh))
2630                         is_mapped_to_disk = 0;
2631                 if (buffer_new(bh))
2632                         clean_bdev_bh_alias(bh);
2633                 if (PageUptodate(page)) {
2634                         set_buffer_uptodate(bh);
2635                         continue;
2636                 }
2637                 if (buffer_new(bh) || !buffer_mapped(bh)) {
2638                         zero_user_segments(page, block_start, from,
2639                                                         to, block_end);
2640                         continue;
2641                 }
2642                 if (buffer_uptodate(bh))
2643                         continue;       /* reiserfs does this */
2644                 if (block_start < from || block_end > to) {
2645                         lock_buffer(bh);
2646                         bh->b_end_io = end_buffer_read_nobh;
2647                         submit_bh(REQ_OP_READ, 0, bh);
2648                         nr_reads++;
2649                 }
2650         }
2651 
2652         if (nr_reads) {
2653                 /*
2654                  * The page is locked, so these buffers are protected from
2655                  * any VM or truncate activity.  Hence we don't need to care
2656                  * for the buffer_head refcounts.
2657                  */
2658                 for (bh = head; bh; bh = bh->b_this_page) {
2659                         wait_on_buffer(bh);
2660                         if (!buffer_uptodate(bh))
2661                                 ret = -EIO;
2662                 }
2663                 if (ret)
2664                         goto failed;
2665         }
2666 
2667         if (is_mapped_to_disk)
2668                 SetPageMappedToDisk(page);
2669 
2670         *fsdata = head; /* to be released by nobh_write_end */
2671 
2672         return 0;
2673 
2674 failed:
2675         BUG_ON(!ret);
2676         /*
2677          * Error recovery is a bit difficult. We need to zero out blocks that
2678          * were newly allocated, and dirty them to ensure they get written out.
2679          * Buffers need to be attached to the page at this point, otherwise
2680          * the handling of potential IO errors during writeout would be hard
2681          * (could try doing synchronous writeout, but what if that fails too?)
2682          */
2683         attach_nobh_buffers(page, head);
2684         page_zero_new_buffers(page, from, to);
2685 
2686 out_release:
2687         unlock_page(page);
2688         put_page(page);
2689         *pagep = NULL;
2690 
2691         return ret;
2692 }
2693 EXPORT_SYMBOL(nobh_write_begin);
2694 
2695 int nobh_write_end(struct file *file, struct address_space *mapping,
2696                         loff_t pos, unsigned len, unsigned copied,
2697                         struct page *page, void *fsdata)
2698 {
2699         struct inode *inode = page->mapping->host;
2700         struct buffer_head *head = fsdata;
2701         struct buffer_head *bh;
2702         BUG_ON(fsdata != NULL && page_has_buffers(page));
2703 
2704         if (unlikely(copied < len) && head)
2705                 attach_nobh_buffers(page, head);
2706         if (page_has_buffers(page))
2707                 return generic_write_end(file, mapping, pos, len,
2708                                         copied, page, fsdata);
2709 
2710         SetPageUptodate(page);
2711         set_page_dirty(page);
2712         if (pos+copied > inode->i_size) {
2713                 i_size_write(inode, pos+copied);
2714                 mark_inode_dirty(inode);
2715         }
2716 
2717         unlock_page(page);
2718         put_page(page);
2719 
2720         while (head) {
2721                 bh = head;
2722                 head = head->b_this_page;
2723                 free_buffer_head(bh);
2724         }
2725 
2726         return copied;
2727 }
2728 EXPORT_SYMBOL(nobh_write_end);
2729 
2730 /*
2731  * nobh_writepage() - based on block_full_write_page() except
2732  * that it tries to operate without attaching bufferheads to
2733  * the page.
2734  */
2735 int nobh_writepage(struct page *page, get_block_t *get_block,
2736                         struct writeback_control *wbc)
2737 {
2738         struct inode * const inode = page->mapping->host;
2739         loff_t i_size = i_size_read(inode);
2740         const pgoff_t end_index = i_size >> PAGE_SHIFT;
2741         unsigned offset;
2742         int ret;
2743 
2744         /* Is the page fully inside i_size? */
2745         if (page->index < end_index)
2746                 goto out;
2747 
2748         /* Is the page fully outside i_size? (truncate in progress) */
2749         offset = i_size & (PAGE_SIZE-1);
2750         if (page->index >= end_index+1 || !offset) {
2751                 /*
2752                  * The page may have dirty, unmapped buffers.  For example,
2753                  * they may have been added in ext3_writepage().  Make them
2754                  * freeable here, so the page does not leak.
2755                  */
2756 #if 0
2757                 /* Not really sure about this  - do we need this ? */
2758                 if (page->mapping->a_ops->invalidatepage)
2759                         page->mapping->a_ops->invalidatepage(page, offset);
2760 #endif
2761                 unlock_page(page);
2762                 return 0; /* don't care */
2763         }
2764 
2765         /*
2766          * The page straddles i_size.  It must be zeroed out on each and every
2767          * writepage invocation because it may be mmapped.  "A file is mapped
2768          * in multiples of the page size.  For a file that is not a multiple of
2769          * the  page size, the remaining memory is zeroed when mapped, and
2770          * writes to that region are not written out to the file."
2771          */
2772         zero_user_segment(page, offset, PAGE_SIZE);
2773 out:
2774         ret = mpage_writepage(page, get_block, wbc);
2775         if (ret == -EAGAIN)
2776                 ret = __block_write_full_page(inode, page, get_block, wbc,
2777                                               end_buffer_async_write);
2778         return ret;
2779 }
2780 EXPORT_SYMBOL(nobh_writepage);
2781 
2782 int nobh_truncate_page(struct address_space *mapping,
2783                         loff_t from, get_block_t *get_block)
2784 {
2785         pgoff_t index = from >> PAGE_SHIFT;
2786         unsigned offset = from & (PAGE_SIZE-1);
2787         unsigned blocksize;
2788         sector_t iblock;
2789         unsigned length, pos;
2790         struct inode *inode = mapping->host;
2791         struct page *page;
2792         struct buffer_head map_bh;
2793         int err;
2794 
2795         blocksize = i_blocksize(inode);
2796         length = offset & (blocksize - 1);
2797 
2798         /* Block boundary? Nothing to do */
2799         if (!length)
2800                 return 0;
2801 
2802         length = blocksize - length;
2803         iblock = (sector_t)index << (PAGE_SHIFT - inode->i_blkbits);
2804 
2805         page = grab_cache_page(mapping, index);
2806         err = -ENOMEM;
2807         if (!page)
2808                 goto out;
2809 
2810         if (page_has_buffers(page)) {
2811 has_buffers:
2812                 unlock_page(page);
2813                 put_page(page);
2814                 return block_truncate_page(mapping, from, get_block);
2815         }
2816 
2817         /* Find the buffer that contains "offset" */
2818         pos = blocksize;
2819         while (offset >= pos) {
2820                 iblock++;
2821                 pos += blocksize;
2822         }
2823 
2824         map_bh.b_size = blocksize;
2825         map_bh.b_state = 0;
2826         err = get_block(inode, iblock, &map_bh, 0);
2827         if (err)
2828                 goto unlock;
2829         /* unmapped? It's a hole - nothing to do */
2830         if (!buffer_mapped(&map_bh))
2831                 goto unlock;
2832 
2833         /* Ok, it's mapped. Make sure it's up-to-date */
2834         if (!PageUptodate(page)) {
2835                 err = mapping->a_ops->readpage(NULL, page);
2836                 if (err) {
2837                         put_page(page);
2838                         goto out;
2839                 }
2840                 lock_page(page);
2841                 if (!PageUptodate(page)) {
2842                         err = -EIO;
2843                         goto unlock;
2844                 }
2845                 if (page_has_buffers(page))
2846                         goto has_buffers;
2847         }
2848         zero_user(page, offset, length);
2849         set_page_dirty(page);
2850         err = 0;
2851 
2852 unlock:
2853         unlock_page(page);
2854         put_page(page);
2855 out:
2856         return err;
2857 }
2858 EXPORT_SYMBOL(nobh_truncate_page);
2859 
2860 int block_truncate_page(struct address_space *mapping,
2861                         loff_t from, get_block_t *get_block)
2862 {
2863         pgoff_t index = from >> PAGE_SHIFT;
2864         unsigned offset = from & (PAGE_SIZE-1);
2865         unsigned blocksize;
2866         sector_t iblock;
2867         unsigned length, pos;
2868         struct inode *inode = mapping->host;
2869         struct page *page;
2870         struct buffer_head *bh;
2871         int err;
2872 
2873         blocksize = i_blocksize(inode);
2874         length = offset & (blocksize - 1);
2875 
2876         /* Block boundary? Nothing to do */
2877         if (!length)
2878                 return 0;
2879 
2880         length = blocksize - length;
2881         iblock = (sector_t)index << (PAGE_SHIFT - inode->i_blkbits);
2882         
2883         page = grab_cache_page(mapping, index);
2884         err = -ENOMEM;
2885         if (!page)
2886                 goto out;
2887 
2888         if (!page_has_buffers(page))
2889                 create_empty_buffers(page, blocksize, 0);
2890 
2891         /* Find the buffer that contains "offset" */
2892         bh = page_buffers(page);
2893         pos = blocksize;
2894         while (offset >= pos) {
2895                 bh = bh->b_this_page;
2896                 iblock++;
2897                 pos += blocksize;
2898         }
2899 
2900         err = 0;
2901         if (!buffer_mapped(bh)) {
2902                 WARN_ON(bh->b_size != blocksize);
2903                 err = get_block(inode, iblock, bh, 0);
2904                 if (err)
2905                         goto unlock;
2906                 /* unmapped? It's a hole - nothing to do */
2907                 if (!buffer_mapped(bh))
2908                         goto unlock;
2909         }
2910 
2911         /* Ok, it's mapped. Make sure it's up-to-date */
2912         if (PageUptodate(page))
2913                 set_buffer_uptodate(bh);
2914 
2915         if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) {
2916                 err = -EIO;
2917                 ll_rw_block(REQ_OP_READ, 0, 1, &bh);
2918                 wait_on_buffer(bh);
2919                 /* Uhhuh. Read error. Complain and punt. */
2920                 if (!buffer_uptodate(bh))
2921                         goto unlock;
2922         }
2923 
2924         zero_user(page, offset, length);
2925         mark_buffer_dirty(bh);
2926         err = 0;
2927 
2928 unlock:
2929         unlock_page(page);
2930         put_page(page);
2931 out:
2932         return err;
2933 }
2934 EXPORT_SYMBOL(block_truncate_page);
2935 
2936 /*
2937  * The generic ->writepage function for buffer-backed address_spaces
2938  */
2939 int block_write_full_page(struct page *page, get_block_t *get_block,
2940                         struct writeback_control *wbc)
2941 {
2942         struct inode * const inode = page->mapping->host;
2943         loff_t i_size = i_size_read(inode);
2944         const pgoff_t end_index = i_size >> PAGE_SHIFT;
2945         unsigned offset;
2946 
2947         /* Is the page fully inside i_size? */
2948         if (page->index < end_index)
2949                 return __block_write_full_page(inode, page, get_block, wbc,
2950                                                end_buffer_async_write);
2951 
2952         /* Is the page fully outside i_size? (truncate in progress) */
2953         offset = i_size & (PAGE_SIZE-1);
2954         if (page->index >= end_index+1 || !offset) {
2955                 /*
2956                  * The page may have dirty, unmapped buffers.  For example,
2957                  * they may have been added in ext3_writepage().  Make them
2958                  * freeable here, so the page does not leak.
2959                  */
2960                 do_invalidatepage(page, 0, PAGE_SIZE);
2961                 unlock_page(page);
2962                 return 0; /* don't care */
2963         }
2964 
2965         /*
2966          * The page straddles i_size.  It must be zeroed out on each and every
2967          * writepage invocation because it may be mmapped.  "A file is mapped
2968          * in multiples of the page size.  For a file that is not a multiple of
2969          * the  page size, the remaining memory is zeroed when mapped, and
2970          * writes to that region are not written out to the file."
2971          */
2972         zero_user_segment(page, offset, PAGE_SIZE);
2973         return __block_write_full_page(inode, page, get_block, wbc,
2974                                                         end_buffer_async_write);
2975 }
2976 EXPORT_SYMBOL(block_write_full_page);
2977 
2978 sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
2979                             get_block_t *get_block)
2980 {
2981         struct inode *inode = mapping->host;
2982         struct buffer_head tmp = {
2983                 .b_size = i_blocksize(inode),
2984         };
2985 
2986         get_block(inode, block, &tmp, 0);
2987         return tmp.b_blocknr;
2988 }
2989 EXPORT_SYMBOL(generic_block_bmap);
2990 
2991 static void end_bio_bh_io_sync(struct bio *bio)
2992 {
2993         struct buffer_head *bh = bio->bi_private;
2994 
2995         if (unlikely(bio_flagged(bio, BIO_QUIET)))
2996                 set_bit(BH_Quiet, &bh->b_state);
2997 
2998         bh->b_end_io(bh, !bio->bi_status);
2999         bio_put(bio);
3000 }
3001 
3002 /*
3003  * This allows us to do IO even on the odd last sectors
3004  * of a device, even if the block size is some multiple
3005  * of the physical sector size.
3006  *
3007  * We'll just truncate the bio to the size of the device,
3008  * and clear the end of the buffer head manually.
3009  *
3010  * Truly out-of-range accesses will turn into actual IO
3011  * errors, this only handles the "we need to be able to
3012  * do IO at the final sector" case.
3013  */
3014 void guard_bio_eod(int op, struct bio *bio)
3015 {
3016         sector_t maxsector;
3017         struct bio_vec *bvec = &bio->bi_io_vec[bio->bi_vcnt - 1];
3018         unsigned truncated_bytes;
3019         struct hd_struct *part;
3020 
3021         rcu_read_lock();
3022         part = __disk_get_part(bio->bi_disk, bio->bi_partno);
3023         if (part)
3024                 maxsector = part_nr_sects_read(part);
3025         else
3026                 maxsector = get_capacity(bio->bi_disk);
3027         rcu_read_unlock();
3028 
3029         if (!maxsector)
3030                 return;
3031 
3032         /*
3033          * If the *whole* IO is past the end of the device,
3034          * let it through, and the IO layer will turn it into
3035          * an EIO.
3036          */
3037         if (unlikely(bio->bi_iter.bi_sector >= maxsector))
3038                 return;
3039 
3040         maxsector -= bio->bi_iter.bi_sector;
3041         if (likely((bio->bi_iter.bi_size >> 9) <= maxsector))
3042                 return;
3043 
3044         /* Uhhuh. We've got a bio that straddles the device size! */
3045         truncated_bytes = bio->bi_iter.bi_size - (maxsector << 9);
3046 
3047         /* Truncate the bio.. */
3048         bio->bi_iter.bi_size -= truncated_bytes;
3049         bvec->bv_len -= truncated_bytes;
3050 
3051         /* ..and clear the end of the buffer for reads */
3052         if (op == REQ_OP_READ) {
3053                 zero_user(bvec->bv_page, bvec->bv_offset + bvec->bv_len,
3054                                 truncated_bytes);
3055         }
3056 }
3057 
3058 static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh,
3059                          enum rw_hint write_hint, struct writeback_control *wbc)
3060 {
3061         struct bio *bio;
3062 
3063         BUG_ON(!buffer_locked(bh));
3064         BUG_ON(!buffer_mapped(bh));
3065         BUG_ON(!bh->b_end_io);
3066         BUG_ON(buffer_delay(bh));
3067         BUG_ON(buffer_unwritten(bh));
3068 
3069         /*
3070          * Only clear out a write error when rewriting
3071          */
3072         if (test_set_buffer_req(bh) && (op == REQ_OP_WRITE))
3073                 clear_buffer_write_io_error(bh);
3074 
3075         /*
3076          * from here on down, it's all bio -- do the initial mapping,
3077          * submit_bio -> generic_make_request may further map this bio around
3078          */
3079         bio = bio_alloc(GFP_NOIO, 1);
3080 
3081         if (wbc) {
3082                 wbc_init_bio(wbc, bio);
3083                 wbc_account_io(wbc, bh->b_page, bh->b_size);
3084         }
3085 
3086         bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
3087         bio_set_dev(bio, bh->b_bdev);
3088         bio->bi_write_hint = write_hint;
3089 
3090         bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh));
3091         BUG_ON(bio->bi_iter.bi_size != bh->b_size);
3092 
3093         bio->bi_end_io = end_bio_bh_io_sync;
3094         bio->bi_private = bh;
3095 
3096         /* Take care of bh's that straddle the end of the device */
3097         guard_bio_eod(op, bio);
3098 
3099         if (buffer_meta(bh))
3100                 op_flags |= REQ_META;
3101         if (buffer_prio(bh))
3102                 op_flags |= REQ_PRIO;
3103         bio_set_op_attrs(bio, op, op_flags);
3104 
3105         submit_bio(bio);
3106         return 0;
3107 }
3108 
3109 int submit_bh(int op, int op_flags, struct buffer_head *bh)
3110 {
3111         return submit_bh_wbc(op, op_flags, bh, 0, NULL);
3112 }
3113 EXPORT_SYMBOL(submit_bh);
3114 
3115 /**
3116  * ll_rw_block: low-level access to block devices (DEPRECATED)
3117  * @op: whether to %READ or %WRITE
3118  * @op_flags: req_flag_bits
3119  * @nr: number of &struct buffer_heads in the array
3120  * @bhs: array of pointers to &struct buffer_head
3121  *
3122  * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
3123  * requests an I/O operation on them, either a %REQ_OP_READ or a %REQ_OP_WRITE.
3124  * @op_flags contains flags modifying the detailed I/O behavior, most notably
3125  * %REQ_RAHEAD.
3126  *
3127  * This function drops any buffer that it cannot get a lock on (with the
3128  * BH_Lock state bit), any buffer that appears to be clean when doing a write
3129  * request, and any buffer that appears to be up-to-date when doing read
3130  * request.  Further it marks as clean buffers that are processed for
3131  * writing (the buffer cache won't assume that they are actually clean
3132  * until the buffer gets unlocked).
3133  *
3134  * ll_rw_block sets b_end_io to simple completion handler that marks
3135  * the buffer up-to-date (if appropriate), unlocks the buffer and wakes
3136  * any waiters. 
3137  *
3138  * All of the buffers must be for the same device, and must also be a
3139  * multiple of the current approved size for the device.
3140  */
3141 void ll_rw_block(int op, int op_flags,  int nr, struct buffer_head *bhs[])
3142 {
3143         int i;
3144 
3145         for (i = 0; i < nr; i++) {
3146                 struct buffer_head *bh = bhs[i];
3147 
3148                 if (!trylock_buffer(bh))
3149                         continue;
3150                 if (op == WRITE) {
3151                         if (test_clear_buffer_dirty(bh)) {
3152                                 bh->b_end_io = end_buffer_write_sync;
3153                                 get_bh(bh);
3154                                 submit_bh(op, op_flags, bh);
3155                                 continue;
3156                         }
3157                 } else {
3158                         if (!buffer_uptodate(bh)) {
3159                                 bh->b_end_io = end_buffer_read_sync;
3160                                 get_bh(bh);
3161                                 submit_bh(op, op_flags, bh);
3162                                 continue;
3163                         }
3164                 }
3165                 unlock_buffer(bh);
3166         }
3167 }
3168 EXPORT_SYMBOL(ll_rw_block);
3169 
3170 void write_dirty_buffer(struct buffer_head *bh, int op_flags)
3171 {
3172         lock_buffer(bh);
3173         if (!test_clear_buffer_dirty(bh)) {
3174                 unlock_buffer(bh);
3175                 return;
3176         }
3177         bh->b_end_io = end_buffer_write_sync;
3178         get_bh(bh);
3179         submit_bh(REQ_OP_WRITE, op_flags, bh);
3180 }
3181 EXPORT_SYMBOL(write_dirty_buffer);
3182 
3183 /*
3184  * For a data-integrity writeout, we need to wait upon any in-progress I/O
3185  * and then start new I/O and then wait upon it.  The caller must have a ref on
3186  * the buffer_head.
3187  */
3188 int __sync_dirty_buffer(struct buffer_head *bh, int op_flags)
3189 {
3190         int ret = 0;
3191 
3192         WARN_ON(atomic_read(&bh->b_count) < 1);
3193         lock_buffer(bh);
3194         if (test_clear_buffer_dirty(bh)) {
3195                 get_bh(bh);
3196                 bh->b_end_io = end_buffer_write_sync;
3197                 ret = submit_bh(REQ_OP_WRITE, op_flags, bh);
3198                 wait_on_buffer(bh);
3199                 if (!ret && !buffer_uptodate(bh))
3200                         ret = -EIO;
3201         } else {
3202                 unlock_buffer(bh);
3203         }
3204         return ret;
3205 }
3206 EXPORT_SYMBOL(__sync_dirty_buffer);
3207 
3208 int sync_dirty_buffer(struct buffer_head *bh)
3209 {
3210         return __sync_dirty_buffer(bh, REQ_SYNC);
3211 }
3212 EXPORT_SYMBOL(sync_dirty_buffer);
3213 
3214 /*
3215  * try_to_free_buffers() checks if all the buffers on this particular page
3216  * are unused, and releases them if so.
3217  *
3218  * Exclusion against try_to_free_buffers may be obtained by either
3219  * locking the page or by holding its mapping's private_lock.
3220  *
3221  * If the page is dirty but all the buffers are clean then we need to
3222  * be sure to mark the page clean as well.  This is because the page
3223  * may be against a block device, and a later reattachment of buffers
3224  * to a dirty page will set *all* buffers dirty.  Which would corrupt
3225  * filesystem data on the same device.
3226  *
3227  * The same applies to regular filesystem pages: if all the buffers are
3228  * clean then we set the page clean and proceed.  To do that, we require
3229  * total exclusion from __set_page_dirty_buffers().  That is obtained with
3230  * private_lock.
3231  *
3232  * try_to_free_buffers() is non-blocking.
3233  */
3234 static inline int buffer_busy(struct buffer_head *bh)
3235 {
3236         return atomic_read(&bh->b_count) |
3237                 (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
3238 }
3239 
3240 static int
3241 drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
3242 {
3243         struct buffer_head *head = page_buffers(page);
3244         struct buffer_head *bh;
3245 
3246         bh = head;
3247         do {
3248                 if (buffer_busy(bh))
3249                         goto failed;
3250                 bh = bh->b_this_page;
3251         } while (bh != head);
3252 
3253         do {
3254                 struct buffer_head *next = bh->b_this_page;
3255 
3256                 if (bh->b_assoc_map)
3257                         __remove_assoc_queue(bh);
3258                 bh = next;
3259         } while (bh != head);
3260         *buffers_to_free = head;
3261         __clear_page_buffers(page);
3262         return 1;
3263 failed:
3264         return 0;
3265 }
3266 
3267 int try_to_free_buffers(struct page *page)
3268 {
3269         struct address_space * const mapping = page->mapping;
3270         struct buffer_head *buffers_to_free = NULL;
3271         int ret = 0;
3272 
3273         BUG_ON(!PageLocked(page));
3274         if (PageWriteback(page))
3275                 return 0;
3276 
3277         if (mapping == NULL) {          /* can this still happen? */
3278                 ret = drop_buffers(page, &buffers_to_free);
3279                 goto out;
3280         }
3281 
3282         spin_lock(&mapping->private_lock);
3283         ret = drop_buffers(page, &buffers_to_free);
3284 
3285         /*
3286          * If the filesystem writes its buffers by hand (eg ext3)
3287          * then we can have clean buffers against a dirty page.  We
3288          * clean the page here; otherwise the VM will never notice
3289          * that the filesystem did any IO at all.
3290          *
3291          * Also, during truncate, discard_buffer will have marked all
3292          * the page's buffers clean.  We discover that here and clean
3293          * the page also.
3294          *
3295          * private_lock must be held over this entire operation in order
3296          * to synchronise against __set_page_dirty_buffers and prevent the
3297          * dirty bit from being lost.
3298          */
3299         if (ret)
3300                 cancel_dirty_page(page);
3301         spin_unlock(&mapping->private_lock);
3302 out:
3303         if (buffers_to_free) {
3304                 struct buffer_head *bh = buffers_to_free;
3305 
3306                 do {
3307                         struct buffer_head *next = bh->b_this_page;
3308                         free_buffer_head(bh);
3309                         bh = next;
3310                 } while (bh != buffers_to_free);
3311         }
3312         return ret;
3313 }
3314 EXPORT_SYMBOL(try_to_free_buffers);
3315 
3316 /*
3317  * There are no bdflush tunables left.  But distributions are
3318  * still running obsolete flush daemons, so we terminate them here.
3319  *
3320  * Use of bdflush() is deprecated and will be removed in a future kernel.
3321  * The `flush-X' kernel threads fully replace bdflush daemons and this call.
3322  */
3323 SYSCALL_DEFINE2(bdflush, int, func, long, data)
3324 {
3325         static int msg_count;
3326 
3327         if (!capable(CAP_SYS_ADMIN))
3328                 return -EPERM;
3329 
3330         if (msg_count < 5) {
3331                 msg_count++;
3332                 printk(KERN_INFO
3333                         "warning: process `%s' used the obsolete bdflush"
3334                         " system call\n", current->comm);
3335                 printk(KERN_INFO "Fix your initscripts?\n");
3336         }
3337 
3338         if (func == 1)
3339                 do_exit(0);
3340         return 0;
3341 }
3342 
3343 /*
3344  * Buffer-head allocation
3345  */
3346 static struct kmem_cache *bh_cachep __read_mostly;
3347 
3348 /*
3349  * Once the number of bh's in the machine exceeds this level, we start
3350  * stripping them in writeback.
3351  */
3352 static unsigned long max_buffer_heads;
3353 
3354 int buffer_heads_over_limit;
3355 
3356 struct bh_accounting {
3357         int nr;                 /* Number of live bh's */
3358         int ratelimit;          /* Limit cacheline bouncing */
3359 };
3360 
3361 static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
3362 
3363 static void recalc_bh_state(void)
3364 {
3365         int i;
3366         int tot = 0;
3367 
3368         if (__this_cpu_inc_return(bh_accounting.ratelimit) - 1 < 4096)
3369                 return;
3370         __this_cpu_write(bh_accounting.ratelimit, 0);
3371         for_each_online_cpu(i)
3372                 tot += per_cpu(bh_accounting, i).nr;
3373         buffer_heads_over_limit = (tot > max_buffer_heads);
3374 }
3375 
3376 struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
3377 {
3378         struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags);
3379         if (ret) {
3380                 INIT_LIST_HEAD(&ret->b_assoc_buffers);
3381                 preempt_disable();
3382                 __this_cpu_inc(bh_accounting.nr);
3383                 recalc_bh_state();
3384                 preempt_enable();
3385         }
3386         return ret;
3387 }
3388 EXPORT_SYMBOL(alloc_buffer_head);
3389 
3390 void free_buffer_head(struct buffer_head *bh)
3391 {
3392         BUG_ON(!list_empty(&bh->b_assoc_buffers));
3393         kmem_cache_free(bh_cachep, bh);
3394         preempt_disable();
3395         __this_cpu_dec(bh_accounting.nr);
3396         recalc_bh_state();
3397         preempt_enable();
3398 }
3399 EXPORT_SYMBOL(free_buffer_head);
3400 
3401 static int buffer_exit_cpu_dead(unsigned int cpu)
3402 {
3403         int i;
3404         struct bh_lru *b = &per_cpu(bh_lrus, cpu);
3405 
3406         for (i = 0; i < BH_LRU_SIZE; i++) {
3407                 brelse(b->bhs[i]);
3408                 b->bhs[i] = NULL;
3409         }
3410         this_cpu_add(bh_accounting.nr, per_cpu(bh_accounting, cpu).nr);
3411         per_cpu(bh_accounting, cpu).nr = 0;
3412         return 0;
3413 }
3414 
3415 /**
3416  * bh_uptodate_or_lock - Test whether the buffer is uptodate
3417  * @bh: struct buffer_head
3418  *
3419  * Return true if the buffer is up-to-date and false,
3420  * with the buffer locked, if not.
3421  */
3422 int bh_uptodate_or_lock(struct buffer_head *bh)
3423 {
3424         if (!buffer_uptodate(bh)) {
3425                 lock_buffer(bh);
3426                 if (!buffer_uptodate(bh))
3427                         return 0;
3428                 unlock_buffer(bh);
3429         }
3430         return 1;
3431 }
3432 EXPORT_SYMBOL(bh_uptodate_or_lock);
3433 
3434 /**
3435  * bh_submit_read - Submit a locked buffer for reading
3436  * @bh: struct buffer_head
3437  *
3438  * Returns zero on success and -EIO on error.
3439  */
3440 int bh_submit_read(struct buffer_head *bh)
3441 {
3442         BUG_ON(!buffer_locked(bh));
3443 
3444         if (buffer_uptodate(bh)) {
3445                 unlock_buffer(bh);
3446                 return 0;
3447         }
3448 
3449         get_bh(bh);
3450         bh->b_end_io = end_buffer_read_sync;
3451         submit_bh(REQ_OP_READ, 0, bh);
3452         wait_on_buffer(bh);
3453         if (buffer_uptodate(bh))
3454                 return 0;
3455         return -EIO;
3456 }
3457 EXPORT_SYMBOL(bh_submit_read);
3458 
3459 /*
3460  * Seek for SEEK_DATA / SEEK_HOLE within @page, starting at @lastoff.
3461  *
3462  * Returns the offset within the file on success, and -ENOENT otherwise.
3463  */
3464 static loff_t
3465 page_seek_hole_data(struct page *page, loff_t lastoff, int whence)
3466 {
3467         loff_t offset = page_offset(page);
3468         struct buffer_head *bh, *head;
3469         bool seek_data = whence == SEEK_DATA;
3470 
3471         if (lastoff < offset)
3472                 lastoff = offset;
3473 
3474         bh = head = page_buffers(page);
3475         do {
3476                 offset += bh->b_size;
3477                 if (lastoff >= offset)
3478                         continue;
3479 
3480                 /*
3481                  * Unwritten extents that have data in the page cache covering
3482                  * them can be identified by the BH_Unwritten state flag.
3483                  * Pages with multiple buffers might have a mix of holes, data
3484                  * and unwritten extents - any buffer with valid data in it
3485                  * should have BH_Uptodate flag set on it.
3486                  */
3487 
3488                 if ((buffer_unwritten(bh) || buffer_uptodate(bh)) == seek_data)
3489                         return lastoff;
3490 
3491                 lastoff = offset;
3492         } while ((bh = bh->b_this_page) != head);
3493         return -ENOENT;
3494 }
3495 
3496 /*
3497  * Seek for SEEK_DATA / SEEK_HOLE in the page cache.
3498  *
3499  * Within unwritten extents, the page cache determines which parts are holes
3500  * and which are data: unwritten and uptodate buffer heads count as data;
3501  * everything else counts as a hole.
3502  *
3503  * Returns the resulting offset on successs, and -ENOENT otherwise.
3504  */
3505 loff_t
3506 page_cache_seek_hole_data(struct inode *inode, loff_t offset, loff_t length,
3507                           int whence)
3508 {
3509         pgoff_t index = offset >> PAGE_SHIFT;
3510         pgoff_t end = DIV_ROUND_UP(offset + length, PAGE_SIZE);
3511         loff_t lastoff = offset;
3512         struct pagevec pvec;
3513 
3514         if (length <= 0)
3515                 return -ENOENT;
3516 
3517         pagevec_init(&pvec);
3518 
3519         do {
3520                 unsigned nr_pages, i;
3521 
3522                 nr_pages = pagevec_lookup_range(&pvec, inode->i_mapping, &index,
3523                                                 end - 1);
3524                 if (nr_pages == 0)
3525                         break;
3526 
3527                 for (i = 0; i < nr_pages; i++) {
3528                         struct page *page = pvec.pages[i];
3529 
3530                         /*
3531                          * At this point, the page may be truncated or
3532                          * invalidated (changing page->mapping to NULL), or
3533                          * even swizzled back from swapper_space to tmpfs file
3534                          * mapping.  However, page->index will not change
3535                          * because we have a reference on the page.
3536                          *
3537                          * If current page offset is beyond where we've ended,
3538                          * we've found a hole.
3539                          */
3540                         if (whence == SEEK_HOLE &&
3541                             lastoff < page_offset(page))
3542                                 goto check_range;
3543 
3544                         lock_page(page);
3545                         if (likely(page->mapping == inode->i_mapping) &&
3546                             page_has_buffers(page)) {
3547                                 lastoff = page_seek_hole_data(page, lastoff, whence);
3548                                 if (lastoff >= 0) {
3549                                         unlock_page(page);
3550                                         goto check_range;
3551                                 }
3552                         }
3553                         unlock_page(page);
3554                         lastoff = page_offset(page) + PAGE_SIZE;
3555                 }
3556                 pagevec_release(&pvec);
3557         } while (index < end);
3558 
3559         /* When no page at lastoff and we are not done, we found a hole. */
3560         if (whence != SEEK_HOLE)
3561                 goto not_found;
3562 
3563 check_range:
3564         if (lastoff < offset + length)
3565                 goto out;
3566 not_found:
3567         lastoff = -ENOENT;
3568 out:
3569         pagevec_release(&pvec);
3570         return lastoff;
3571 }
3572 
3573 void __init buffer_init(void)
3574 {
3575         unsigned long nrpages;
3576         int ret;
3577 
3578         bh_cachep = kmem_cache_create("buffer_head",
3579                         sizeof(struct buffer_head), 0,
3580                                 (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
3581                                 SLAB_MEM_SPREAD),
3582                                 NULL);
3583 
3584         /*
3585          * Limit the bh occupancy to 10% of ZONE_NORMAL
3586          */
3587         nrpages = (nr_free_buffer_pages() * 10) / 100;
3588         max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
3589         ret = cpuhp_setup_state_nocalls(CPUHP_FS_BUFF_DEAD, "fs/buffer:dead",
3590                                         NULL, buffer_exit_cpu_dead);
3591         WARN_ON(ret < 0);
3592 }
3593 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp