~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/fs/buffer.c

Version: ~ [ linux-5.9 ] ~ [ linux-5.8.14 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.70 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.150 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.200 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.238 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.238 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  *  linux/fs/buffer.c
  3  *
  4  *  Copyright (C) 1991, 1992, 2002  Linus Torvalds
  5  */
  6 
  7 /*
  8  * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
  9  *
 10  * Removed a lot of unnecessary code and simplified things now that
 11  * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
 12  *
 13  * Speed up hash, lru, and free list operations.  Use gfp() for allocating
 14  * hash table, use SLAB cache for buffer heads. SMP threading.  -DaveM
 15  *
 16  * Added 32k buffer block sizes - these are required older ARM systems. - RMK
 17  *
 18  * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
 19  */
 20 
 21 #include <linux/kernel.h>
 22 #include <linux/syscalls.h>
 23 #include <linux/fs.h>
 24 #include <linux/mm.h>
 25 #include <linux/percpu.h>
 26 #include <linux/slab.h>
 27 #include <linux/capability.h>
 28 #include <linux/blkdev.h>
 29 #include <linux/file.h>
 30 #include <linux/quotaops.h>
 31 #include <linux/highmem.h>
 32 #include <linux/export.h>
 33 #include <linux/writeback.h>
 34 #include <linux/hash.h>
 35 #include <linux/suspend.h>
 36 #include <linux/buffer_head.h>
 37 #include <linux/task_io_accounting_ops.h>
 38 #include <linux/bio.h>
 39 #include <linux/notifier.h>
 40 #include <linux/cpu.h>
 41 #include <linux/bitops.h>
 42 #include <linux/mpage.h>
 43 #include <linux/bit_spinlock.h>
 44 #include <trace/events/block.h>
 45 
 46 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
 47 
 48 #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
 49 
 50 void init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private)
 51 {
 52         bh->b_end_io = handler;
 53         bh->b_private = private;
 54 }
 55 EXPORT_SYMBOL(init_buffer);
 56 
 57 inline void touch_buffer(struct buffer_head *bh)
 58 {
 59         trace_block_touch_buffer(bh);
 60         mark_page_accessed(bh->b_page);
 61 }
 62 EXPORT_SYMBOL(touch_buffer);
 63 
 64 static int sleep_on_buffer(void *word)
 65 {
 66         io_schedule();
 67         return 0;
 68 }
 69 
 70 void __lock_buffer(struct buffer_head *bh)
 71 {
 72         wait_on_bit_lock(&bh->b_state, BH_Lock, sleep_on_buffer,
 73                                                         TASK_UNINTERRUPTIBLE);
 74 }
 75 EXPORT_SYMBOL(__lock_buffer);
 76 
 77 void unlock_buffer(struct buffer_head *bh)
 78 {
 79         clear_bit_unlock(BH_Lock, &bh->b_state);
 80         smp_mb__after_clear_bit();
 81         wake_up_bit(&bh->b_state, BH_Lock);
 82 }
 83 EXPORT_SYMBOL(unlock_buffer);
 84 
 85 /*
 86  * Block until a buffer comes unlocked.  This doesn't stop it
 87  * from becoming locked again - you have to lock it yourself
 88  * if you want to preserve its state.
 89  */
 90 void __wait_on_buffer(struct buffer_head * bh)
 91 {
 92         wait_on_bit(&bh->b_state, BH_Lock, sleep_on_buffer, TASK_UNINTERRUPTIBLE);
 93 }
 94 EXPORT_SYMBOL(__wait_on_buffer);
 95 
 96 static void
 97 __clear_page_buffers(struct page *page)
 98 {
 99         ClearPagePrivate(page);
100         set_page_private(page, 0);
101         page_cache_release(page);
102 }
103 
104 
105 static int quiet_error(struct buffer_head *bh)
106 {
107         if (!test_bit(BH_Quiet, &bh->b_state) && printk_ratelimit())
108                 return 0;
109         return 1;
110 }
111 
112 
113 static void buffer_io_error(struct buffer_head *bh)
114 {
115         char b[BDEVNAME_SIZE];
116         printk(KERN_ERR "Buffer I/O error on device %s, logical block %Lu\n",
117                         bdevname(bh->b_bdev, b),
118                         (unsigned long long)bh->b_blocknr);
119 }
120 
121 /*
122  * End-of-IO handler helper function which does not touch the bh after
123  * unlocking it.
124  * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
125  * a race there is benign: unlock_buffer() only use the bh's address for
126  * hashing after unlocking the buffer, so it doesn't actually touch the bh
127  * itself.
128  */
129 static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate)
130 {
131         if (uptodate) {
132                 set_buffer_uptodate(bh);
133         } else {
134                 /* This happens, due to failed READA attempts. */
135                 clear_buffer_uptodate(bh);
136         }
137         unlock_buffer(bh);
138 }
139 
140 /*
141  * Default synchronous end-of-IO handler..  Just mark it up-to-date and
142  * unlock the buffer. This is what ll_rw_block uses too.
143  */
144 void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
145 {
146         __end_buffer_read_notouch(bh, uptodate);
147         put_bh(bh);
148 }
149 EXPORT_SYMBOL(end_buffer_read_sync);
150 
151 void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
152 {
153         char b[BDEVNAME_SIZE];
154 
155         if (uptodate) {
156                 set_buffer_uptodate(bh);
157         } else {
158                 if (!quiet_error(bh)) {
159                         buffer_io_error(bh);
160                         printk(KERN_WARNING "lost page write due to "
161                                         "I/O error on %s\n",
162                                        bdevname(bh->b_bdev, b));
163                 }
164                 set_buffer_write_io_error(bh);
165                 clear_buffer_uptodate(bh);
166         }
167         unlock_buffer(bh);
168         put_bh(bh);
169 }
170 EXPORT_SYMBOL(end_buffer_write_sync);
171 
172 /*
173  * Various filesystems appear to want __find_get_block to be non-blocking.
174  * But it's the page lock which protects the buffers.  To get around this,
175  * we get exclusion from try_to_free_buffers with the blockdev mapping's
176  * private_lock.
177  *
178  * Hack idea: for the blockdev mapping, i_bufferlist_lock contention
179  * may be quite high.  This code could TryLock the page, and if that
180  * succeeds, there is no need to take private_lock. (But if
181  * private_lock is contended then so is mapping->tree_lock).
182  */
183 static struct buffer_head *
184 __find_get_block_slow(struct block_device *bdev, sector_t block)
185 {
186         struct inode *bd_inode = bdev->bd_inode;
187         struct address_space *bd_mapping = bd_inode->i_mapping;
188         struct buffer_head *ret = NULL;
189         pgoff_t index;
190         struct buffer_head *bh;
191         struct buffer_head *head;
192         struct page *page;
193         int all_mapped = 1;
194 
195         index = block >> (PAGE_CACHE_SHIFT - bd_inode->i_blkbits);
196         page = find_get_page(bd_mapping, index);
197         if (!page)
198                 goto out;
199 
200         spin_lock(&bd_mapping->private_lock);
201         if (!page_has_buffers(page))
202                 goto out_unlock;
203         head = page_buffers(page);
204         bh = head;
205         do {
206                 if (!buffer_mapped(bh))
207                         all_mapped = 0;
208                 else if (bh->b_blocknr == block) {
209                         ret = bh;
210                         get_bh(bh);
211                         goto out_unlock;
212                 }
213                 bh = bh->b_this_page;
214         } while (bh != head);
215 
216         /* we might be here because some of the buffers on this page are
217          * not mapped.  This is due to various races between
218          * file io on the block device and getblk.  It gets dealt with
219          * elsewhere, don't buffer_error if we had some unmapped buffers
220          */
221         if (all_mapped) {
222                 char b[BDEVNAME_SIZE];
223 
224                 printk("__find_get_block_slow() failed. "
225                         "block=%llu, b_blocknr=%llu\n",
226                         (unsigned long long)block,
227                         (unsigned long long)bh->b_blocknr);
228                 printk("b_state=0x%08lx, b_size=%zu\n",
229                         bh->b_state, bh->b_size);
230                 printk("device %s blocksize: %d\n", bdevname(bdev, b),
231                         1 << bd_inode->i_blkbits);
232         }
233 out_unlock:
234         spin_unlock(&bd_mapping->private_lock);
235         page_cache_release(page);
236 out:
237         return ret;
238 }
239 
240 /*
241  * Kick the writeback threads then try to free up some ZONE_NORMAL memory.
242  */
243 static void free_more_memory(void)
244 {
245         struct zone *zone;
246         int nid;
247 
248         wakeup_flusher_threads(1024, WB_REASON_FREE_MORE_MEM);
249         yield();
250 
251         for_each_online_node(nid) {
252                 (void)first_zones_zonelist(node_zonelist(nid, GFP_NOFS),
253                                                 gfp_zone(GFP_NOFS), NULL,
254                                                 &zone);
255                 if (zone)
256                         try_to_free_pages(node_zonelist(nid, GFP_NOFS), 0,
257                                                 GFP_NOFS, NULL);
258         }
259 }
260 
261 /*
262  * I/O completion handler for block_read_full_page() - pages
263  * which come unlocked at the end of I/O.
264  */
265 static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
266 {
267         unsigned long flags;
268         struct buffer_head *first;
269         struct buffer_head *tmp;
270         struct page *page;
271         int page_uptodate = 1;
272 
273         BUG_ON(!buffer_async_read(bh));
274 
275         page = bh->b_page;
276         if (uptodate) {
277                 set_buffer_uptodate(bh);
278         } else {
279                 clear_buffer_uptodate(bh);
280                 if (!quiet_error(bh))
281                         buffer_io_error(bh);
282                 SetPageError(page);
283         }
284 
285         /*
286          * Be _very_ careful from here on. Bad things can happen if
287          * two buffer heads end IO at almost the same time and both
288          * decide that the page is now completely done.
289          */
290         first = page_buffers(page);
291         local_irq_save(flags);
292         bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
293         clear_buffer_async_read(bh);
294         unlock_buffer(bh);
295         tmp = bh;
296         do {
297                 if (!buffer_uptodate(tmp))
298                         page_uptodate = 0;
299                 if (buffer_async_read(tmp)) {
300                         BUG_ON(!buffer_locked(tmp));
301                         goto still_busy;
302                 }
303                 tmp = tmp->b_this_page;
304         } while (tmp != bh);
305         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
306         local_irq_restore(flags);
307 
308         /*
309          * If none of the buffers had errors and they are all
310          * uptodate then we can set the page uptodate.
311          */
312         if (page_uptodate && !PageError(page))
313                 SetPageUptodate(page);
314         unlock_page(page);
315         return;
316 
317 still_busy:
318         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
319         local_irq_restore(flags);
320         return;
321 }
322 
323 /*
324  * Completion handler for block_write_full_page() - pages which are unlocked
325  * during I/O, and which have PageWriteback cleared upon I/O completion.
326  */
327 void end_buffer_async_write(struct buffer_head *bh, int uptodate)
328 {
329         char b[BDEVNAME_SIZE];
330         unsigned long flags;
331         struct buffer_head *first;
332         struct buffer_head *tmp;
333         struct page *page;
334 
335         BUG_ON(!buffer_async_write(bh));
336 
337         page = bh->b_page;
338         if (uptodate) {
339                 set_buffer_uptodate(bh);
340         } else {
341                 if (!quiet_error(bh)) {
342                         buffer_io_error(bh);
343                         printk(KERN_WARNING "lost page write due to "
344                                         "I/O error on %s\n",
345                                bdevname(bh->b_bdev, b));
346                 }
347                 set_bit(AS_EIO, &page->mapping->flags);
348                 set_buffer_write_io_error(bh);
349                 clear_buffer_uptodate(bh);
350                 SetPageError(page);
351         }
352 
353         first = page_buffers(page);
354         local_irq_save(flags);
355         bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
356 
357         clear_buffer_async_write(bh);
358         unlock_buffer(bh);
359         tmp = bh->b_this_page;
360         while (tmp != bh) {
361                 if (buffer_async_write(tmp)) {
362                         BUG_ON(!buffer_locked(tmp));
363                         goto still_busy;
364                 }
365                 tmp = tmp->b_this_page;
366         }
367         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
368         local_irq_restore(flags);
369         end_page_writeback(page);
370         return;
371 
372 still_busy:
373         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
374         local_irq_restore(flags);
375         return;
376 }
377 EXPORT_SYMBOL(end_buffer_async_write);
378 
379 /*
380  * If a page's buffers are under async readin (end_buffer_async_read
381  * completion) then there is a possibility that another thread of
382  * control could lock one of the buffers after it has completed
383  * but while some of the other buffers have not completed.  This
384  * locked buffer would confuse end_buffer_async_read() into not unlocking
385  * the page.  So the absence of BH_Async_Read tells end_buffer_async_read()
386  * that this buffer is not under async I/O.
387  *
388  * The page comes unlocked when it has no locked buffer_async buffers
389  * left.
390  *
391  * PageLocked prevents anyone starting new async I/O reads any of
392  * the buffers.
393  *
394  * PageWriteback is used to prevent simultaneous writeout of the same
395  * page.
396  *
397  * PageLocked prevents anyone from starting writeback of a page which is
398  * under read I/O (PageWriteback is only ever set against a locked page).
399  */
400 static void mark_buffer_async_read(struct buffer_head *bh)
401 {
402         bh->b_end_io = end_buffer_async_read;
403         set_buffer_async_read(bh);
404 }
405 
406 static void mark_buffer_async_write_endio(struct buffer_head *bh,
407                                           bh_end_io_t *handler)
408 {
409         bh->b_end_io = handler;
410         set_buffer_async_write(bh);
411 }
412 
413 void mark_buffer_async_write(struct buffer_head *bh)
414 {
415         mark_buffer_async_write_endio(bh, end_buffer_async_write);
416 }
417 EXPORT_SYMBOL(mark_buffer_async_write);
418 
419 
420 /*
421  * fs/buffer.c contains helper functions for buffer-backed address space's
422  * fsync functions.  A common requirement for buffer-based filesystems is
423  * that certain data from the backing blockdev needs to be written out for
424  * a successful fsync().  For example, ext2 indirect blocks need to be
425  * written back and waited upon before fsync() returns.
426  *
427  * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
428  * inode_has_buffers() and invalidate_inode_buffers() are provided for the
429  * management of a list of dependent buffers at ->i_mapping->private_list.
430  *
431  * Locking is a little subtle: try_to_free_buffers() will remove buffers
432  * from their controlling inode's queue when they are being freed.  But
433  * try_to_free_buffers() will be operating against the *blockdev* mapping
434  * at the time, not against the S_ISREG file which depends on those buffers.
435  * So the locking for private_list is via the private_lock in the address_space
436  * which backs the buffers.  Which is different from the address_space 
437  * against which the buffers are listed.  So for a particular address_space,
438  * mapping->private_lock does *not* protect mapping->private_list!  In fact,
439  * mapping->private_list will always be protected by the backing blockdev's
440  * ->private_lock.
441  *
442  * Which introduces a requirement: all buffers on an address_space's
443  * ->private_list must be from the same address_space: the blockdev's.
444  *
445  * address_spaces which do not place buffers at ->private_list via these
446  * utility functions are free to use private_lock and private_list for
447  * whatever they want.  The only requirement is that list_empty(private_list)
448  * be true at clear_inode() time.
449  *
450  * FIXME: clear_inode should not call invalidate_inode_buffers().  The
451  * filesystems should do that.  invalidate_inode_buffers() should just go
452  * BUG_ON(!list_empty).
453  *
454  * FIXME: mark_buffer_dirty_inode() is a data-plane operation.  It should
455  * take an address_space, not an inode.  And it should be called
456  * mark_buffer_dirty_fsync() to clearly define why those buffers are being
457  * queued up.
458  *
459  * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
460  * list if it is already on a list.  Because if the buffer is on a list,
461  * it *must* already be on the right one.  If not, the filesystem is being
462  * silly.  This will save a ton of locking.  But first we have to ensure
463  * that buffers are taken *off* the old inode's list when they are freed
464  * (presumably in truncate).  That requires careful auditing of all
465  * filesystems (do it inside bforget()).  It could also be done by bringing
466  * b_inode back.
467  */
468 
469 /*
470  * The buffer's backing address_space's private_lock must be held
471  */
472 static void __remove_assoc_queue(struct buffer_head *bh)
473 {
474         list_del_init(&bh->b_assoc_buffers);
475         WARN_ON(!bh->b_assoc_map);
476         if (buffer_write_io_error(bh))
477                 set_bit(AS_EIO, &bh->b_assoc_map->flags);
478         bh->b_assoc_map = NULL;
479 }
480 
481 int inode_has_buffers(struct inode *inode)
482 {
483         return !list_empty(&inode->i_data.private_list);
484 }
485 
486 /*
487  * osync is designed to support O_SYNC io.  It waits synchronously for
488  * all already-submitted IO to complete, but does not queue any new
489  * writes to the disk.
490  *
491  * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
492  * you dirty the buffers, and then use osync_inode_buffers to wait for
493  * completion.  Any other dirty buffers which are not yet queued for
494  * write will not be flushed to disk by the osync.
495  */
496 static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
497 {
498         struct buffer_head *bh;
499         struct list_head *p;
500         int err = 0;
501 
502         spin_lock(lock);
503 repeat:
504         list_for_each_prev(p, list) {
505                 bh = BH_ENTRY(p);
506                 if (buffer_locked(bh)) {
507                         get_bh(bh);
508                         spin_unlock(lock);
509                         wait_on_buffer(bh);
510                         if (!buffer_uptodate(bh))
511                                 err = -EIO;
512                         brelse(bh);
513                         spin_lock(lock);
514                         goto repeat;
515                 }
516         }
517         spin_unlock(lock);
518         return err;
519 }
520 
521 static void do_thaw_one(struct super_block *sb, void *unused)
522 {
523         char b[BDEVNAME_SIZE];
524         while (sb->s_bdev && !thaw_bdev(sb->s_bdev, sb))
525                 printk(KERN_WARNING "Emergency Thaw on %s\n",
526                        bdevname(sb->s_bdev, b));
527 }
528 
529 static void do_thaw_all(struct work_struct *work)
530 {
531         iterate_supers(do_thaw_one, NULL);
532         kfree(work);
533         printk(KERN_WARNING "Emergency Thaw complete\n");
534 }
535 
536 /**
537  * emergency_thaw_all -- forcibly thaw every frozen filesystem
538  *
539  * Used for emergency unfreeze of all filesystems via SysRq
540  */
541 void emergency_thaw_all(void)
542 {
543         struct work_struct *work;
544 
545         work = kmalloc(sizeof(*work), GFP_ATOMIC);
546         if (work) {
547                 INIT_WORK(work, do_thaw_all);
548                 schedule_work(work);
549         }
550 }
551 
552 /**
553  * sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers
554  * @mapping: the mapping which wants those buffers written
555  *
556  * Starts I/O against the buffers at mapping->private_list, and waits upon
557  * that I/O.
558  *
559  * Basically, this is a convenience function for fsync().
560  * @mapping is a file or directory which needs those buffers to be written for
561  * a successful fsync().
562  */
563 int sync_mapping_buffers(struct address_space *mapping)
564 {
565         struct address_space *buffer_mapping = mapping->private_data;
566 
567         if (buffer_mapping == NULL || list_empty(&mapping->private_list))
568                 return 0;
569 
570         return fsync_buffers_list(&buffer_mapping->private_lock,
571                                         &mapping->private_list);
572 }
573 EXPORT_SYMBOL(sync_mapping_buffers);
574 
575 /*
576  * Called when we've recently written block `bblock', and it is known that
577  * `bblock' was for a buffer_boundary() buffer.  This means that the block at
578  * `bblock + 1' is probably a dirty indirect block.  Hunt it down and, if it's
579  * dirty, schedule it for IO.  So that indirects merge nicely with their data.
580  */
581 void write_boundary_block(struct block_device *bdev,
582                         sector_t bblock, unsigned blocksize)
583 {
584         struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
585         if (bh) {
586                 if (buffer_dirty(bh))
587                         ll_rw_block(WRITE, 1, &bh);
588                 put_bh(bh);
589         }
590 }
591 
592 void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
593 {
594         struct address_space *mapping = inode->i_mapping;
595         struct address_space *buffer_mapping = bh->b_page->mapping;
596 
597         mark_buffer_dirty(bh);
598         if (!mapping->private_data) {
599                 mapping->private_data = buffer_mapping;
600         } else {
601                 BUG_ON(mapping->private_data != buffer_mapping);
602         }
603         if (!bh->b_assoc_map) {
604                 spin_lock(&buffer_mapping->private_lock);
605                 list_move_tail(&bh->b_assoc_buffers,
606                                 &mapping->private_list);
607                 bh->b_assoc_map = mapping;
608                 spin_unlock(&buffer_mapping->private_lock);
609         }
610 }
611 EXPORT_SYMBOL(mark_buffer_dirty_inode);
612 
613 /*
614  * Mark the page dirty, and set it dirty in the radix tree, and mark the inode
615  * dirty.
616  *
617  * If warn is true, then emit a warning if the page is not uptodate and has
618  * not been truncated.
619  */
620 static void __set_page_dirty(struct page *page,
621                 struct address_space *mapping, int warn)
622 {
623         unsigned long flags;
624 
625         spin_lock_irqsave(&mapping->tree_lock, flags);
626         if (page->mapping) {    /* Race with truncate? */
627                 WARN_ON_ONCE(warn && !PageUptodate(page));
628                 account_page_dirtied(page, mapping);
629                 radix_tree_tag_set(&mapping->page_tree,
630                                 page_index(page), PAGECACHE_TAG_DIRTY);
631         }
632         spin_unlock_irqrestore(&mapping->tree_lock, flags);
633         __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
634 }
635 
636 /*
637  * Add a page to the dirty page list.
638  *
639  * It is a sad fact of life that this function is called from several places
640  * deeply under spinlocking.  It may not sleep.
641  *
642  * If the page has buffers, the uptodate buffers are set dirty, to preserve
643  * dirty-state coherency between the page and the buffers.  It the page does
644  * not have buffers then when they are later attached they will all be set
645  * dirty.
646  *
647  * The buffers are dirtied before the page is dirtied.  There's a small race
648  * window in which a writepage caller may see the page cleanness but not the
649  * buffer dirtiness.  That's fine.  If this code were to set the page dirty
650  * before the buffers, a concurrent writepage caller could clear the page dirty
651  * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
652  * page on the dirty page list.
653  *
654  * We use private_lock to lock against try_to_free_buffers while using the
655  * page's buffer list.  Also use this to protect against clean buffers being
656  * added to the page after it was set dirty.
657  *
658  * FIXME: may need to call ->reservepage here as well.  That's rather up to the
659  * address_space though.
660  */
661 int __set_page_dirty_buffers(struct page *page)
662 {
663         int newly_dirty;
664         struct address_space *mapping = page_mapping(page);
665 
666         if (unlikely(!mapping))
667                 return !TestSetPageDirty(page);
668 
669         spin_lock(&mapping->private_lock);
670         if (page_has_buffers(page)) {
671                 struct buffer_head *head = page_buffers(page);
672                 struct buffer_head *bh = head;
673 
674                 do {
675                         set_buffer_dirty(bh);
676                         bh = bh->b_this_page;
677                 } while (bh != head);
678         }
679         newly_dirty = !TestSetPageDirty(page);
680         spin_unlock(&mapping->private_lock);
681 
682         if (newly_dirty)
683                 __set_page_dirty(page, mapping, 1);
684         return newly_dirty;
685 }
686 EXPORT_SYMBOL(__set_page_dirty_buffers);
687 
688 /*
689  * Write out and wait upon a list of buffers.
690  *
691  * We have conflicting pressures: we want to make sure that all
692  * initially dirty buffers get waited on, but that any subsequently
693  * dirtied buffers don't.  After all, we don't want fsync to last
694  * forever if somebody is actively writing to the file.
695  *
696  * Do this in two main stages: first we copy dirty buffers to a
697  * temporary inode list, queueing the writes as we go.  Then we clean
698  * up, waiting for those writes to complete.
699  * 
700  * During this second stage, any subsequent updates to the file may end
701  * up refiling the buffer on the original inode's dirty list again, so
702  * there is a chance we will end up with a buffer queued for write but
703  * not yet completed on that list.  So, as a final cleanup we go through
704  * the osync code to catch these locked, dirty buffers without requeuing
705  * any newly dirty buffers for write.
706  */
707 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
708 {
709         struct buffer_head *bh;
710         struct list_head tmp;
711         struct address_space *mapping;
712         int err = 0, err2;
713         struct blk_plug plug;
714 
715         INIT_LIST_HEAD(&tmp);
716         blk_start_plug(&plug);
717 
718         spin_lock(lock);
719         while (!list_empty(list)) {
720                 bh = BH_ENTRY(list->next);
721                 mapping = bh->b_assoc_map;
722                 __remove_assoc_queue(bh);
723                 /* Avoid race with mark_buffer_dirty_inode() which does
724                  * a lockless check and we rely on seeing the dirty bit */
725                 smp_mb();
726                 if (buffer_dirty(bh) || buffer_locked(bh)) {
727                         list_add(&bh->b_assoc_buffers, &tmp);
728                         bh->b_assoc_map = mapping;
729                         if (buffer_dirty(bh)) {
730                                 get_bh(bh);
731                                 spin_unlock(lock);
732                                 /*
733                                  * Ensure any pending I/O completes so that
734                                  * write_dirty_buffer() actually writes the
735                                  * current contents - it is a noop if I/O is
736                                  * still in flight on potentially older
737                                  * contents.
738                                  */
739                                 write_dirty_buffer(bh, WRITE_SYNC);
740 
741                                 /*
742                                  * Kick off IO for the previous mapping. Note
743                                  * that we will not run the very last mapping,
744                                  * wait_on_buffer() will do that for us
745                                  * through sync_buffer().
746                                  */
747                                 brelse(bh);
748                                 spin_lock(lock);
749                         }
750                 }
751         }
752 
753         spin_unlock(lock);
754         blk_finish_plug(&plug);
755         spin_lock(lock);
756 
757         while (!list_empty(&tmp)) {
758                 bh = BH_ENTRY(tmp.prev);
759                 get_bh(bh);
760                 mapping = bh->b_assoc_map;
761                 __remove_assoc_queue(bh);
762                 /* Avoid race with mark_buffer_dirty_inode() which does
763                  * a lockless check and we rely on seeing the dirty bit */
764                 smp_mb();
765                 if (buffer_dirty(bh)) {
766                         list_add(&bh->b_assoc_buffers,
767                                  &mapping->private_list);
768                         bh->b_assoc_map = mapping;
769                 }
770                 spin_unlock(lock);
771                 wait_on_buffer(bh);
772                 if (!buffer_uptodate(bh))
773                         err = -EIO;
774                 brelse(bh);
775                 spin_lock(lock);
776         }
777         
778         spin_unlock(lock);
779         err2 = osync_buffers_list(lock, list);
780         if (err)
781                 return err;
782         else
783                 return err2;
784 }
785 
786 /*
787  * Invalidate any and all dirty buffers on a given inode.  We are
788  * probably unmounting the fs, but that doesn't mean we have already
789  * done a sync().  Just drop the buffers from the inode list.
790  *
791  * NOTE: we take the inode's blockdev's mapping's private_lock.  Which
792  * assumes that all the buffers are against the blockdev.  Not true
793  * for reiserfs.
794  */
795 void invalidate_inode_buffers(struct inode *inode)
796 {
797         if (inode_has_buffers(inode)) {
798                 struct address_space *mapping = &inode->i_data;
799                 struct list_head *list = &mapping->private_list;
800                 struct address_space *buffer_mapping = mapping->private_data;
801 
802                 spin_lock(&buffer_mapping->private_lock);
803                 while (!list_empty(list))
804                         __remove_assoc_queue(BH_ENTRY(list->next));
805                 spin_unlock(&buffer_mapping->private_lock);
806         }
807 }
808 EXPORT_SYMBOL(invalidate_inode_buffers);
809 
810 /*
811  * Remove any clean buffers from the inode's buffer list.  This is called
812  * when we're trying to free the inode itself.  Those buffers can pin it.
813  *
814  * Returns true if all buffers were removed.
815  */
816 int remove_inode_buffers(struct inode *inode)
817 {
818         int ret = 1;
819 
820         if (inode_has_buffers(inode)) {
821                 struct address_space *mapping = &inode->i_data;
822                 struct list_head *list = &mapping->private_list;
823                 struct address_space *buffer_mapping = mapping->private_data;
824 
825                 spin_lock(&buffer_mapping->private_lock);
826                 while (!list_empty(list)) {
827                         struct buffer_head *bh = BH_ENTRY(list->next);
828                         if (buffer_dirty(bh)) {
829                                 ret = 0;
830                                 break;
831                         }
832                         __remove_assoc_queue(bh);
833                 }
834                 spin_unlock(&buffer_mapping->private_lock);
835         }
836         return ret;
837 }
838 
839 /*
840  * Create the appropriate buffers when given a page for data area and
841  * the size of each buffer.. Use the bh->b_this_page linked list to
842  * follow the buffers created.  Return NULL if unable to create more
843  * buffers.
844  *
845  * The retry flag is used to differentiate async IO (paging, swapping)
846  * which may not fail from ordinary buffer allocations.
847  */
848 struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
849                 int retry)
850 {
851         struct buffer_head *bh, *head;
852         long offset;
853 
854 try_again:
855         head = NULL;
856         offset = PAGE_SIZE;
857         while ((offset -= size) >= 0) {
858                 bh = alloc_buffer_head(GFP_NOFS);
859                 if (!bh)
860                         goto no_grow;
861 
862                 bh->b_this_page = head;
863                 bh->b_blocknr = -1;
864                 head = bh;
865 
866                 bh->b_size = size;
867 
868                 /* Link the buffer to its page */
869                 set_bh_page(bh, page, offset);
870         }
871         return head;
872 /*
873  * In case anything failed, we just free everything we got.
874  */
875 no_grow:
876         if (head) {
877                 do {
878                         bh = head;
879                         head = head->b_this_page;
880                         free_buffer_head(bh);
881                 } while (head);
882         }
883 
884         /*
885          * Return failure for non-async IO requests.  Async IO requests
886          * are not allowed to fail, so we have to wait until buffer heads
887          * become available.  But we don't want tasks sleeping with 
888          * partially complete buffers, so all were released above.
889          */
890         if (!retry)
891                 return NULL;
892 
893         /* We're _really_ low on memory. Now we just
894          * wait for old buffer heads to become free due to
895          * finishing IO.  Since this is an async request and
896          * the reserve list is empty, we're sure there are 
897          * async buffer heads in use.
898          */
899         free_more_memory();
900         goto try_again;
901 }
902 EXPORT_SYMBOL_GPL(alloc_page_buffers);
903 
904 static inline void
905 link_dev_buffers(struct page *page, struct buffer_head *head)
906 {
907         struct buffer_head *bh, *tail;
908 
909         bh = head;
910         do {
911                 tail = bh;
912                 bh = bh->b_this_page;
913         } while (bh);
914         tail->b_this_page = head;
915         attach_page_buffers(page, head);
916 }
917 
918 static sector_t blkdev_max_block(struct block_device *bdev, unsigned int size)
919 {
920         sector_t retval = ~((sector_t)0);
921         loff_t sz = i_size_read(bdev->bd_inode);
922 
923         if (sz) {
924                 unsigned int sizebits = blksize_bits(size);
925                 retval = (sz >> sizebits);
926         }
927         return retval;
928 }
929 
930 /*
931  * Initialise the state of a blockdev page's buffers.
932  */ 
933 static sector_t
934 init_page_buffers(struct page *page, struct block_device *bdev,
935                         sector_t block, int size)
936 {
937         struct buffer_head *head = page_buffers(page);
938         struct buffer_head *bh = head;
939         int uptodate = PageUptodate(page);
940         sector_t end_block = blkdev_max_block(I_BDEV(bdev->bd_inode), size);
941 
942         do {
943                 if (!buffer_mapped(bh)) {
944                         init_buffer(bh, NULL, NULL);
945                         bh->b_bdev = bdev;
946                         bh->b_blocknr = block;
947                         if (uptodate)
948                                 set_buffer_uptodate(bh);
949                         if (block < end_block)
950                                 set_buffer_mapped(bh);
951                 }
952                 block++;
953                 bh = bh->b_this_page;
954         } while (bh != head);
955 
956         /*
957          * Caller needs to validate requested block against end of device.
958          */
959         return end_block;
960 }
961 
962 /*
963  * Create the page-cache page that contains the requested block.
964  *
965  * This is used purely for blockdev mappings.
966  */
967 static int
968 grow_dev_page(struct block_device *bdev, sector_t block,
969                 pgoff_t index, int size, int sizebits)
970 {
971         struct inode *inode = bdev->bd_inode;
972         struct page *page;
973         struct buffer_head *bh;
974         sector_t end_block;
975         int ret = 0;            /* Will call free_more_memory() */
976 
977         page = find_or_create_page(inode->i_mapping, index,
978                 (mapping_gfp_mask(inode->i_mapping) & ~__GFP_FS)|__GFP_MOVABLE);
979         if (!page)
980                 return ret;
981 
982         BUG_ON(!PageLocked(page));
983 
984         if (page_has_buffers(page)) {
985                 bh = page_buffers(page);
986                 if (bh->b_size == size) {
987                         end_block = init_page_buffers(page, bdev,
988                                                 (sector_t)index << sizebits,
989                                                 size);
990                         goto done;
991                 }
992                 if (!try_to_free_buffers(page))
993                         goto failed;
994         }
995 
996         /*
997          * Allocate some buffers for this page
998          */
999         bh = alloc_page_buffers(page, size, 0);
1000         if (!bh)
1001                 goto failed;
1002 
1003         /*
1004          * Link the page to the buffers and initialise them.  Take the
1005          * lock to be atomic wrt __find_get_block(), which does not
1006          * run under the page lock.
1007          */
1008         spin_lock(&inode->i_mapping->private_lock);
1009         link_dev_buffers(page, bh);
1010         end_block = init_page_buffers(page, bdev, (sector_t)index << sizebits,
1011                         size);
1012         spin_unlock(&inode->i_mapping->private_lock);
1013 done:
1014         ret = (block < end_block) ? 1 : -ENXIO;
1015 failed:
1016         unlock_page(page);
1017         page_cache_release(page);
1018         return ret;
1019 }
1020 
1021 /*
1022  * Create buffers for the specified block device block's page.  If
1023  * that page was dirty, the buffers are set dirty also.
1024  */
1025 static int
1026 grow_buffers(struct block_device *bdev, sector_t block, int size)
1027 {
1028         pgoff_t index;
1029         int sizebits;
1030 
1031         sizebits = -1;
1032         do {
1033                 sizebits++;
1034         } while ((size << sizebits) < PAGE_SIZE);
1035 
1036         index = block >> sizebits;
1037 
1038         /*
1039          * Check for a block which wants to lie outside our maximum possible
1040          * pagecache index.  (this comparison is done using sector_t types).
1041          */
1042         if (unlikely(index != block >> sizebits)) {
1043                 char b[BDEVNAME_SIZE];
1044 
1045                 printk(KERN_ERR "%s: requested out-of-range block %llu for "
1046                         "device %s\n",
1047                         __func__, (unsigned long long)block,
1048                         bdevname(bdev, b));
1049                 return -EIO;
1050         }
1051 
1052         /* Create a page with the proper size buffers.. */
1053         return grow_dev_page(bdev, block, index, size, sizebits);
1054 }
1055 
1056 static struct buffer_head *
1057 __getblk_slow(struct block_device *bdev, sector_t block, int size)
1058 {
1059         /* Size must be multiple of hard sectorsize */
1060         if (unlikely(size & (bdev_logical_block_size(bdev)-1) ||
1061                         (size < 512 || size > PAGE_SIZE))) {
1062                 printk(KERN_ERR "getblk(): invalid block size %d requested\n",
1063                                         size);
1064                 printk(KERN_ERR "logical block size: %d\n",
1065                                         bdev_logical_block_size(bdev));
1066 
1067                 dump_stack();
1068                 return NULL;
1069         }
1070 
1071         for (;;) {
1072                 struct buffer_head *bh;
1073                 int ret;
1074 
1075                 bh = __find_get_block(bdev, block, size);
1076                 if (bh)
1077                         return bh;
1078 
1079                 ret = grow_buffers(bdev, block, size);
1080                 if (ret < 0)
1081                         return NULL;
1082                 if (ret == 0)
1083                         free_more_memory();
1084         }
1085 }
1086 
1087 /*
1088  * The relationship between dirty buffers and dirty pages:
1089  *
1090  * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1091  * the page is tagged dirty in its radix tree.
1092  *
1093  * At all times, the dirtiness of the buffers represents the dirtiness of
1094  * subsections of the page.  If the page has buffers, the page dirty bit is
1095  * merely a hint about the true dirty state.
1096  *
1097  * When a page is set dirty in its entirety, all its buffers are marked dirty
1098  * (if the page has buffers).
1099  *
1100  * When a buffer is marked dirty, its page is dirtied, but the page's other
1101  * buffers are not.
1102  *
1103  * Also.  When blockdev buffers are explicitly read with bread(), they
1104  * individually become uptodate.  But their backing page remains not
1105  * uptodate - even if all of its buffers are uptodate.  A subsequent
1106  * block_read_full_page() against that page will discover all the uptodate
1107  * buffers, will set the page uptodate and will perform no I/O.
1108  */
1109 
1110 /**
1111  * mark_buffer_dirty - mark a buffer_head as needing writeout
1112  * @bh: the buffer_head to mark dirty
1113  *
1114  * mark_buffer_dirty() will set the dirty bit against the buffer, then set its
1115  * backing page dirty, then tag the page as dirty in its address_space's radix
1116  * tree and then attach the address_space's inode to its superblock's dirty
1117  * inode list.
1118  *
1119  * mark_buffer_dirty() is atomic.  It takes bh->b_page->mapping->private_lock,
1120  * mapping->tree_lock and mapping->host->i_lock.
1121  */
1122 void mark_buffer_dirty(struct buffer_head *bh)
1123 {
1124         WARN_ON_ONCE(!buffer_uptodate(bh));
1125 
1126         trace_block_dirty_buffer(bh);
1127 
1128         /*
1129          * Very *carefully* optimize the it-is-already-dirty case.
1130          *
1131          * Don't let the final "is it dirty" escape to before we
1132          * perhaps modified the buffer.
1133          */
1134         if (buffer_dirty(bh)) {
1135                 smp_mb();
1136                 if (buffer_dirty(bh))
1137                         return;
1138         }
1139 
1140         if (!test_set_buffer_dirty(bh)) {
1141                 struct page *page = bh->b_page;
1142                 if (!TestSetPageDirty(page)) {
1143                         struct address_space *mapping = page_mapping(page);
1144                         if (mapping)
1145                                 __set_page_dirty(page, mapping, 0);
1146                 }
1147         }
1148 }
1149 EXPORT_SYMBOL(mark_buffer_dirty);
1150 
1151 /*
1152  * Decrement a buffer_head's reference count.  If all buffers against a page
1153  * have zero reference count, are clean and unlocked, and if the page is clean
1154  * and unlocked then try_to_free_buffers() may strip the buffers from the page
1155  * in preparation for freeing it (sometimes, rarely, buffers are removed from
1156  * a page but it ends up not being freed, and buffers may later be reattached).
1157  */
1158 void __brelse(struct buffer_head * buf)
1159 {
1160         if (atomic_read(&buf->b_count)) {
1161                 put_bh(buf);
1162                 return;
1163         }
1164         WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n");
1165 }
1166 EXPORT_SYMBOL(__brelse);
1167 
1168 /*
1169  * bforget() is like brelse(), except it discards any
1170  * potentially dirty data.
1171  */
1172 void __bforget(struct buffer_head *bh)
1173 {
1174         clear_buffer_dirty(bh);
1175         if (bh->b_assoc_map) {
1176                 struct address_space *buffer_mapping = bh->b_page->mapping;
1177 
1178                 spin_lock(&buffer_mapping->private_lock);
1179                 list_del_init(&bh->b_assoc_buffers);
1180                 bh->b_assoc_map = NULL;
1181                 spin_unlock(&buffer_mapping->private_lock);
1182         }
1183         __brelse(bh);
1184 }
1185 EXPORT_SYMBOL(__bforget);
1186 
1187 static struct buffer_head *__bread_slow(struct buffer_head *bh)
1188 {
1189         lock_buffer(bh);
1190         if (buffer_uptodate(bh)) {
1191                 unlock_buffer(bh);
1192                 return bh;
1193         } else {
1194                 get_bh(bh);
1195                 bh->b_end_io = end_buffer_read_sync;
1196                 submit_bh(READ, bh);
1197                 wait_on_buffer(bh);
1198                 if (buffer_uptodate(bh))
1199                         return bh;
1200         }
1201         brelse(bh);
1202         return NULL;
1203 }
1204 
1205 /*
1206  * Per-cpu buffer LRU implementation.  To reduce the cost of __find_get_block().
1207  * The bhs[] array is sorted - newest buffer is at bhs[0].  Buffers have their
1208  * refcount elevated by one when they're in an LRU.  A buffer can only appear
1209  * once in a particular CPU's LRU.  A single buffer can be present in multiple
1210  * CPU's LRUs at the same time.
1211  *
1212  * This is a transparent caching front-end to sb_bread(), sb_getblk() and
1213  * sb_find_get_block().
1214  *
1215  * The LRUs themselves only need locking against invalidate_bh_lrus.  We use
1216  * a local interrupt disable for that.
1217  */
1218 
1219 #define BH_LRU_SIZE     8
1220 
1221 struct bh_lru {
1222         struct buffer_head *bhs[BH_LRU_SIZE];
1223 };
1224 
1225 static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
1226 
1227 #ifdef CONFIG_SMP
1228 #define bh_lru_lock()   local_irq_disable()
1229 #define bh_lru_unlock() local_irq_enable()
1230 #else
1231 #define bh_lru_lock()   preempt_disable()
1232 #define bh_lru_unlock() preempt_enable()
1233 #endif
1234 
1235 static inline void check_irqs_on(void)
1236 {
1237 #ifdef irqs_disabled
1238         BUG_ON(irqs_disabled());
1239 #endif
1240 }
1241 
1242 /*
1243  * The LRU management algorithm is dopey-but-simple.  Sorry.
1244  */
1245 static void bh_lru_install(struct buffer_head *bh)
1246 {
1247         struct buffer_head *evictee = NULL;
1248 
1249         check_irqs_on();
1250         bh_lru_lock();
1251         if (__this_cpu_read(bh_lrus.bhs[0]) != bh) {
1252                 struct buffer_head *bhs[BH_LRU_SIZE];
1253                 int in;
1254                 int out = 0;
1255 
1256                 get_bh(bh);
1257                 bhs[out++] = bh;
1258                 for (in = 0; in < BH_LRU_SIZE; in++) {
1259                         struct buffer_head *bh2 =
1260                                 __this_cpu_read(bh_lrus.bhs[in]);
1261 
1262                         if (bh2 == bh) {
1263                                 __brelse(bh2);
1264                         } else {
1265                                 if (out >= BH_LRU_SIZE) {
1266                                         BUG_ON(evictee != NULL);
1267                                         evictee = bh2;
1268                                 } else {
1269                                         bhs[out++] = bh2;
1270                                 }
1271                         }
1272                 }
1273                 while (out < BH_LRU_SIZE)
1274                         bhs[out++] = NULL;
1275                 memcpy(__this_cpu_ptr(&bh_lrus.bhs), bhs, sizeof(bhs));
1276         }
1277         bh_lru_unlock();
1278 
1279         if (evictee)
1280                 __brelse(evictee);
1281 }
1282 
1283 /*
1284  * Look up the bh in this cpu's LRU.  If it's there, move it to the head.
1285  */
1286 static struct buffer_head *
1287 lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size)
1288 {
1289         struct buffer_head *ret = NULL;
1290         unsigned int i;
1291 
1292         check_irqs_on();
1293         bh_lru_lock();
1294         for (i = 0; i < BH_LRU_SIZE; i++) {
1295                 struct buffer_head *bh = __this_cpu_read(bh_lrus.bhs[i]);
1296 
1297                 if (bh && bh->b_bdev == bdev &&
1298                                 bh->b_blocknr == block && bh->b_size == size) {
1299                         if (i) {
1300                                 while (i) {
1301                                         __this_cpu_write(bh_lrus.bhs[i],
1302                                                 __this_cpu_read(bh_lrus.bhs[i - 1]));
1303                                         i--;
1304                                 }
1305                                 __this_cpu_write(bh_lrus.bhs[0], bh);
1306                         }
1307                         get_bh(bh);
1308                         ret = bh;
1309                         break;
1310                 }
1311         }
1312         bh_lru_unlock();
1313         return ret;
1314 }
1315 
1316 /*
1317  * Perform a pagecache lookup for the matching buffer.  If it's there, refresh
1318  * it in the LRU and mark it as accessed.  If it is not present then return
1319  * NULL
1320  */
1321 struct buffer_head *
1322 __find_get_block(struct block_device *bdev, sector_t block, unsigned size)
1323 {
1324         struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
1325 
1326         if (bh == NULL) {
1327                 bh = __find_get_block_slow(bdev, block);
1328                 if (bh)
1329                         bh_lru_install(bh);
1330         }
1331         if (bh)
1332                 touch_buffer(bh);
1333         return bh;
1334 }
1335 EXPORT_SYMBOL(__find_get_block);
1336 
1337 /*
1338  * __getblk will locate (and, if necessary, create) the buffer_head
1339  * which corresponds to the passed block_device, block and size. The
1340  * returned buffer has its reference count incremented.
1341  *
1342  * __getblk() will lock up the machine if grow_dev_page's try_to_free_buffers()
1343  * attempt is failing.  FIXME, perhaps?
1344  */
1345 struct buffer_head *
1346 __getblk(struct block_device *bdev, sector_t block, unsigned size)
1347 {
1348         struct buffer_head *bh = __find_get_block(bdev, block, size);
1349 
1350         might_sleep();
1351         if (bh == NULL)
1352                 bh = __getblk_slow(bdev, block, size);
1353         return bh;
1354 }
1355 EXPORT_SYMBOL(__getblk);
1356 
1357 /*
1358  * Do async read-ahead on a buffer..
1359  */
1360 void __breadahead(struct block_device *bdev, sector_t block, unsigned size)
1361 {
1362         struct buffer_head *bh = __getblk(bdev, block, size);
1363         if (likely(bh)) {
1364                 ll_rw_block(READA, 1, &bh);
1365                 brelse(bh);
1366         }
1367 }
1368 EXPORT_SYMBOL(__breadahead);
1369 
1370 /**
1371  *  __bread() - reads a specified block and returns the bh
1372  *  @bdev: the block_device to read from
1373  *  @block: number of block
1374  *  @size: size (in bytes) to read
1375  * 
1376  *  Reads a specified block, and returns buffer head that contains it.
1377  *  It returns NULL if the block was unreadable.
1378  */
1379 struct buffer_head *
1380 __bread(struct block_device *bdev, sector_t block, unsigned size)
1381 {
1382         struct buffer_head *bh = __getblk(bdev, block, size);
1383 
1384         if (likely(bh) && !buffer_uptodate(bh))
1385                 bh = __bread_slow(bh);
1386         return bh;
1387 }
1388 EXPORT_SYMBOL(__bread);
1389 
1390 /*
1391  * invalidate_bh_lrus() is called rarely - but not only at unmount.
1392  * This doesn't race because it runs in each cpu either in irq
1393  * or with preempt disabled.
1394  */
1395 static void invalidate_bh_lru(void *arg)
1396 {
1397         struct bh_lru *b = &get_cpu_var(bh_lrus);
1398         int i;
1399 
1400         for (i = 0; i < BH_LRU_SIZE; i++) {
1401                 brelse(b->bhs[i]);
1402                 b->bhs[i] = NULL;
1403         }
1404         put_cpu_var(bh_lrus);
1405 }
1406 
1407 static bool has_bh_in_lru(int cpu, void *dummy)
1408 {
1409         struct bh_lru *b = per_cpu_ptr(&bh_lrus, cpu);
1410         int i;
1411         
1412         for (i = 0; i < BH_LRU_SIZE; i++) {
1413                 if (b->bhs[i])
1414                         return 1;
1415         }
1416 
1417         return 0;
1418 }
1419 
1420 void invalidate_bh_lrus(void)
1421 {
1422         on_each_cpu_cond(has_bh_in_lru, invalidate_bh_lru, NULL, 1, GFP_KERNEL);
1423 }
1424 EXPORT_SYMBOL_GPL(invalidate_bh_lrus);
1425 
1426 void set_bh_page(struct buffer_head *bh,
1427                 struct page *page, unsigned long offset)
1428 {
1429         bh->b_page = page;
1430         BUG_ON(offset >= PAGE_SIZE);
1431         if (PageHighMem(page))
1432                 /*
1433                  * This catches illegal uses and preserves the offset:
1434                  */
1435                 bh->b_data = (char *)(0 + offset);
1436         else
1437                 bh->b_data = page_address(page) + offset;
1438 }
1439 EXPORT_SYMBOL(set_bh_page);
1440 
1441 /*
1442  * Called when truncating a buffer on a page completely.
1443  */
1444 static void discard_buffer(struct buffer_head * bh)
1445 {
1446         lock_buffer(bh);
1447         clear_buffer_dirty(bh);
1448         bh->b_bdev = NULL;
1449         clear_buffer_mapped(bh);
1450         clear_buffer_req(bh);
1451         clear_buffer_new(bh);
1452         clear_buffer_delay(bh);
1453         clear_buffer_unwritten(bh);
1454         unlock_buffer(bh);
1455 }
1456 
1457 /**
1458  * block_invalidatepage - invalidate part or all of a buffer-backed page
1459  *
1460  * @page: the page which is affected
1461  * @offset: the index of the truncation point
1462  *
1463  * block_invalidatepage() is called when all or part of the page has become
1464  * invalidated by a truncate operation.
1465  *
1466  * block_invalidatepage() does not have to release all buffers, but it must
1467  * ensure that no dirty buffer is left outside @offset and that no I/O
1468  * is underway against any of the blocks which are outside the truncation
1469  * point.  Because the caller is about to free (and possibly reuse) those
1470  * blocks on-disk.
1471  */
1472 void block_invalidatepage(struct page *page, unsigned long offset)
1473 {
1474         struct buffer_head *head, *bh, *next;
1475         unsigned int curr_off = 0;
1476 
1477         BUG_ON(!PageLocked(page));
1478         if (!page_has_buffers(page))
1479                 goto out;
1480 
1481         head = page_buffers(page);
1482         bh = head;
1483         do {
1484                 unsigned int next_off = curr_off + bh->b_size;
1485                 next = bh->b_this_page;
1486 
1487                 /*
1488                  * is this block fully invalidated?
1489                  */
1490                 if (offset <= curr_off)
1491                         discard_buffer(bh);
1492                 curr_off = next_off;
1493                 bh = next;
1494         } while (bh != head);
1495 
1496         /*
1497          * We release buffers only if the entire page is being invalidated.
1498          * The get_block cached value has been unconditionally invalidated,
1499          * so real IO is not possible anymore.
1500          */
1501         if (offset == 0)
1502                 try_to_release_page(page, 0);
1503 out:
1504         return;
1505 }
1506 EXPORT_SYMBOL(block_invalidatepage);
1507 
1508 /*
1509  * We attach and possibly dirty the buffers atomically wrt
1510  * __set_page_dirty_buffers() via private_lock.  try_to_free_buffers
1511  * is already excluded via the page lock.
1512  */
1513 void create_empty_buffers(struct page *page,
1514                         unsigned long blocksize, unsigned long b_state)
1515 {
1516         struct buffer_head *bh, *head, *tail;
1517 
1518         head = alloc_page_buffers(page, blocksize, 1);
1519         bh = head;
1520         do {
1521                 bh->b_state |= b_state;
1522                 tail = bh;
1523                 bh = bh->b_this_page;
1524         } while (bh);
1525         tail->b_this_page = head;
1526 
1527         spin_lock(&page->mapping->private_lock);
1528         if (PageUptodate(page) || PageDirty(page)) {
1529                 bh = head;
1530                 do {
1531                         if (PageDirty(page))
1532                                 set_buffer_dirty(bh);
1533                         if (PageUptodate(page))
1534                                 set_buffer_uptodate(bh);
1535                         bh = bh->b_this_page;
1536                 } while (bh != head);
1537         }
1538         attach_page_buffers(page, head);
1539         spin_unlock(&page->mapping->private_lock);
1540 }
1541 EXPORT_SYMBOL(create_empty_buffers);
1542 
1543 /*
1544  * We are taking a block for data and we don't want any output from any
1545  * buffer-cache aliases starting from return from that function and
1546  * until the moment when something will explicitly mark the buffer
1547  * dirty (hopefully that will not happen until we will free that block ;-)
1548  * We don't even need to mark it not-uptodate - nobody can expect
1549  * anything from a newly allocated buffer anyway. We used to used
1550  * unmap_buffer() for such invalidation, but that was wrong. We definitely
1551  * don't want to mark the alias unmapped, for example - it would confuse
1552  * anyone who might pick it with bread() afterwards...
1553  *
1554  * Also..  Note that bforget() doesn't lock the buffer.  So there can
1555  * be writeout I/O going on against recently-freed buffers.  We don't
1556  * wait on that I/O in bforget() - it's more efficient to wait on the I/O
1557  * only if we really need to.  That happens here.
1558  */
1559 void unmap_underlying_metadata(struct block_device *bdev, sector_t block)
1560 {
1561         struct buffer_head *old_bh;
1562 
1563         might_sleep();
1564 
1565         old_bh = __find_get_block_slow(bdev, block);
1566         if (old_bh) {
1567                 clear_buffer_dirty(old_bh);
1568                 wait_on_buffer(old_bh);
1569                 clear_buffer_req(old_bh);
1570                 __brelse(old_bh);
1571         }
1572 }
1573 EXPORT_SYMBOL(unmap_underlying_metadata);
1574 
1575 /*
1576  * Size is a power-of-two in the range 512..PAGE_SIZE,
1577  * and the case we care about most is PAGE_SIZE.
1578  *
1579  * So this *could* possibly be written with those
1580  * constraints in mind (relevant mostly if some
1581  * architecture has a slow bit-scan instruction)
1582  */
1583 static inline int block_size_bits(unsigned int blocksize)
1584 {
1585         return ilog2(blocksize);
1586 }
1587 
1588 static struct buffer_head *create_page_buffers(struct page *page, struct inode *inode, unsigned int b_state)
1589 {
1590         BUG_ON(!PageLocked(page));
1591 
1592         if (!page_has_buffers(page))
1593                 create_empty_buffers(page, 1 << ACCESS_ONCE(inode->i_blkbits), b_state);
1594         return page_buffers(page);
1595 }
1596 
1597 /*
1598  * NOTE! All mapped/uptodate combinations are valid:
1599  *
1600  *      Mapped  Uptodate        Meaning
1601  *
1602  *      No      No              "unknown" - must do get_block()
1603  *      No      Yes             "hole" - zero-filled
1604  *      Yes     No              "allocated" - allocated on disk, not read in
1605  *      Yes     Yes             "valid" - allocated and up-to-date in memory.
1606  *
1607  * "Dirty" is valid only with the last case (mapped+uptodate).
1608  */
1609 
1610 /*
1611  * While block_write_full_page is writing back the dirty buffers under
1612  * the page lock, whoever dirtied the buffers may decide to clean them
1613  * again at any time.  We handle that by only looking at the buffer
1614  * state inside lock_buffer().
1615  *
1616  * If block_write_full_page() is called for regular writeback
1617  * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1618  * locked buffer.   This only can happen if someone has written the buffer
1619  * directly, with submit_bh().  At the address_space level PageWriteback
1620  * prevents this contention from occurring.
1621  *
1622  * If block_write_full_page() is called with wbc->sync_mode ==
1623  * WB_SYNC_ALL, the writes are posted using WRITE_SYNC; this
1624  * causes the writes to be flagged as synchronous writes.
1625  */
1626 static int __block_write_full_page(struct inode *inode, struct page *page,
1627                         get_block_t *get_block, struct writeback_control *wbc,
1628                         bh_end_io_t *handler)
1629 {
1630         int err;
1631         sector_t block;
1632         sector_t last_block;
1633         struct buffer_head *bh, *head;
1634         unsigned int blocksize, bbits;
1635         int nr_underway = 0;
1636         int write_op = (wbc->sync_mode == WB_SYNC_ALL ?
1637                         WRITE_SYNC : WRITE);
1638 
1639         head = create_page_buffers(page, inode,
1640                                         (1 << BH_Dirty)|(1 << BH_Uptodate));
1641 
1642         /*
1643          * Be very careful.  We have no exclusion from __set_page_dirty_buffers
1644          * here, and the (potentially unmapped) buffers may become dirty at
1645          * any time.  If a buffer becomes dirty here after we've inspected it
1646          * then we just miss that fact, and the page stays dirty.
1647          *
1648          * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
1649          * handle that here by just cleaning them.
1650          */
1651 
1652         bh = head;
1653         blocksize = bh->b_size;
1654         bbits = block_size_bits(blocksize);
1655 
1656         block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);
1657         last_block = (i_size_read(inode) - 1) >> bbits;
1658 
1659         /*
1660          * Get all the dirty buffers mapped to disk addresses and
1661          * handle any aliases from the underlying blockdev's mapping.
1662          */
1663         do {
1664                 if (block > last_block) {
1665                         /*
1666                          * mapped buffers outside i_size will occur, because
1667                          * this page can be outside i_size when there is a
1668                          * truncate in progress.
1669                          */
1670                         /*
1671                          * The buffer was zeroed by block_write_full_page()
1672                          */
1673                         clear_buffer_dirty(bh);
1674                         set_buffer_uptodate(bh);
1675                 } else if ((!buffer_mapped(bh) || buffer_delay(bh)) &&
1676                            buffer_dirty(bh)) {
1677                         WARN_ON(bh->b_size != blocksize);
1678                         err = get_block(inode, block, bh, 1);
1679                         if (err)
1680                                 goto recover;
1681                         clear_buffer_delay(bh);
1682                         if (buffer_new(bh)) {
1683                                 /* blockdev mappings never come here */
1684                                 clear_buffer_new(bh);
1685                                 unmap_underlying_metadata(bh->b_bdev,
1686                                                         bh->b_blocknr);
1687                         }
1688                 }
1689                 bh = bh->b_this_page;
1690                 block++;
1691         } while (bh != head);
1692 
1693         do {
1694                 if (!buffer_mapped(bh))
1695                         continue;
1696                 /*
1697                  * If it's a fully non-blocking write attempt and we cannot
1698                  * lock the buffer then redirty the page.  Note that this can
1699                  * potentially cause a busy-wait loop from writeback threads
1700                  * and kswapd activity, but those code paths have their own
1701                  * higher-level throttling.
1702                  */
1703                 if (wbc->sync_mode != WB_SYNC_NONE) {
1704                         lock_buffer(bh);
1705                 } else if (!trylock_buffer(bh)) {
1706                         redirty_page_for_writepage(wbc, page);
1707                         continue;
1708                 }
1709                 if (test_clear_buffer_dirty(bh)) {
1710                         mark_buffer_async_write_endio(bh, handler);
1711                 } else {
1712                         unlock_buffer(bh);
1713                 }
1714         } while ((bh = bh->b_this_page) != head);
1715 
1716         /*
1717          * The page and its buffers are protected by PageWriteback(), so we can
1718          * drop the bh refcounts early.
1719          */
1720         BUG_ON(PageWriteback(page));
1721         set_page_writeback(page);
1722 
1723         do {
1724                 struct buffer_head *next = bh->b_this_page;
1725                 if (buffer_async_write(bh)) {
1726                         submit_bh(write_op, bh);
1727                         nr_underway++;
1728                 }
1729                 bh = next;
1730         } while (bh != head);
1731         unlock_page(page);
1732 
1733         err = 0;
1734 done:
1735         if (nr_underway == 0) {
1736                 /*
1737                  * The page was marked dirty, but the buffers were
1738                  * clean.  Someone wrote them back by hand with
1739                  * ll_rw_block/submit_bh.  A rare case.
1740                  */
1741                 end_page_writeback(page);
1742 
1743                 /*
1744                  * The page and buffer_heads can be released at any time from
1745                  * here on.
1746                  */
1747         }
1748         return err;
1749 
1750 recover:
1751         /*
1752          * ENOSPC, or some other error.  We may already have added some
1753          * blocks to the file, so we need to write these out to avoid
1754          * exposing stale data.
1755          * The page is currently locked and not marked for writeback
1756          */
1757         bh = head;
1758         /* Recovery: lock and submit the mapped buffers */
1759         do {
1760                 if (buffer_mapped(bh) && buffer_dirty(bh) &&
1761                     !buffer_delay(bh)) {
1762                         lock_buffer(bh);
1763                         mark_buffer_async_write_endio(bh, handler);
1764                 } else {
1765                         /*
1766                          * The buffer may have been set dirty during
1767                          * attachment to a dirty page.
1768                          */
1769                         clear_buffer_dirty(bh);
1770                 }
1771         } while ((bh = bh->b_this_page) != head);
1772         SetPageError(page);
1773         BUG_ON(PageWriteback(page));
1774         mapping_set_error(page->mapping, err);
1775         set_page_writeback(page);
1776         do {
1777                 struct buffer_head *next = bh->b_this_page;
1778                 if (buffer_async_write(bh)) {
1779                         clear_buffer_dirty(bh);
1780                         submit_bh(write_op, bh);
1781                         nr_underway++;
1782                 }
1783                 bh = next;
1784         } while (bh != head);
1785         unlock_page(page);
1786         goto done;
1787 }
1788 
1789 /*
1790  * If a page has any new buffers, zero them out here, and mark them uptodate
1791  * and dirty so they'll be written out (in order to prevent uninitialised
1792  * block data from leaking). And clear the new bit.
1793  */
1794 void page_zero_new_buffers(struct page *page, unsigned from, unsigned to)
1795 {
1796         unsigned int block_start, block_end;
1797         struct buffer_head *head, *bh;
1798 
1799         BUG_ON(!PageLocked(page));
1800         if (!page_has_buffers(page))
1801                 return;
1802 
1803         bh = head = page_buffers(page);
1804         block_start = 0;
1805         do {
1806                 block_end = block_start + bh->b_size;
1807 
1808                 if (buffer_new(bh)) {
1809                         if (block_end > from && block_start < to) {
1810                                 if (!PageUptodate(page)) {
1811                                         unsigned start, size;
1812 
1813                                         start = max(from, block_start);
1814                                         size = min(to, block_end) - start;
1815 
1816                                         zero_user(page, start, size);
1817                                         set_buffer_uptodate(bh);
1818                                 }
1819 
1820                                 clear_buffer_new(bh);
1821                                 mark_buffer_dirty(bh);
1822                         }
1823                 }
1824 
1825                 block_start = block_end;
1826                 bh = bh->b_this_page;
1827         } while (bh != head);
1828 }
1829 EXPORT_SYMBOL(page_zero_new_buffers);
1830 
1831 int __block_write_begin(struct page *page, loff_t pos, unsigned len,
1832                 get_block_t *get_block)
1833 {
1834         unsigned from = pos & (PAGE_CACHE_SIZE - 1);
1835         unsigned to = from + len;
1836         struct inode *inode = page->mapping->host;
1837         unsigned block_start, block_end;
1838         sector_t block;
1839         int err = 0;
1840         unsigned blocksize, bbits;
1841         struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
1842 
1843         BUG_ON(!PageLocked(page));
1844         BUG_ON(from > PAGE_CACHE_SIZE);
1845         BUG_ON(to > PAGE_CACHE_SIZE);
1846         BUG_ON(from > to);
1847 
1848         head = create_page_buffers(page, inode, 0);
1849         blocksize = head->b_size;
1850         bbits = block_size_bits(blocksize);
1851 
1852         block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);
1853 
1854         for(bh = head, block_start = 0; bh != head || !block_start;
1855             block++, block_start=block_end, bh = bh->b_this_page) {
1856                 block_end = block_start + blocksize;
1857                 if (block_end <= from || block_start >= to) {
1858                         if (PageUptodate(page)) {
1859                                 if (!buffer_uptodate(bh))
1860                                         set_buffer_uptodate(bh);
1861                         }
1862                         continue;
1863                 }
1864                 if (buffer_new(bh))
1865                         clear_buffer_new(bh);
1866                 if (!buffer_mapped(bh)) {
1867                         WARN_ON(bh->b_size != blocksize);
1868                         err = get_block(inode, block, bh, 1);
1869                         if (err)
1870                                 break;
1871                         if (buffer_new(bh)) {
1872                                 unmap_underlying_metadata(bh->b_bdev,
1873                                                         bh->b_blocknr);
1874                                 if (PageUptodate(page)) {
1875                                         clear_buffer_new(bh);
1876                                         set_buffer_uptodate(bh);
1877                                         mark_buffer_dirty(bh);
1878                                         continue;
1879                                 }
1880                                 if (block_end > to || block_start < from)
1881                                         zero_user_segments(page,
1882                                                 to, block_end,
1883                                                 block_start, from);
1884                                 continue;
1885                         }
1886                 }
1887                 if (PageUptodate(page)) {
1888                         if (!buffer_uptodate(bh))
1889                                 set_buffer_uptodate(bh);
1890                         continue; 
1891                 }
1892                 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
1893                     !buffer_unwritten(bh) &&
1894                      (block_start < from || block_end > to)) {
1895                         ll_rw_block(READ, 1, &bh);
1896                         *wait_bh++=bh;
1897                 }
1898         }
1899         /*
1900          * If we issued read requests - let them complete.
1901          */
1902         while(wait_bh > wait) {
1903                 wait_on_buffer(*--wait_bh);
1904                 if (!buffer_uptodate(*wait_bh))
1905                         err = -EIO;
1906         }
1907         if (unlikely(err))
1908                 page_zero_new_buffers(page, from, to);
1909         return err;
1910 }
1911 EXPORT_SYMBOL(__block_write_begin);
1912 
1913 static int __block_commit_write(struct inode *inode, struct page *page,
1914                 unsigned from, unsigned to)
1915 {
1916         unsigned block_start, block_end;
1917         int partial = 0;
1918         unsigned blocksize;
1919         struct buffer_head *bh, *head;
1920 
1921         bh = head = page_buffers(page);
1922         blocksize = bh->b_size;
1923 
1924         block_start = 0;
1925         do {
1926                 block_end = block_start + blocksize;
1927                 if (block_end <= from || block_start >= to) {
1928                         if (!buffer_uptodate(bh))
1929                                 partial = 1;
1930                 } else {
1931                         set_buffer_uptodate(bh);
1932                         mark_buffer_dirty(bh);
1933                 }
1934                 clear_buffer_new(bh);
1935 
1936                 block_start = block_end;
1937                 bh = bh->b_this_page;
1938         } while (bh != head);
1939 
1940         /*
1941          * If this is a partial write which happened to make all buffers
1942          * uptodate then we can optimize away a bogus readpage() for
1943          * the next read(). Here we 'discover' whether the page went
1944          * uptodate as a result of this (potentially partial) write.
1945          */
1946         if (!partial)
1947                 SetPageUptodate(page);
1948         return 0;
1949 }
1950 
1951 /*
1952  * block_write_begin takes care of the basic task of block allocation and
1953  * bringing partial write blocks uptodate first.
1954  *
1955  * The filesystem needs to handle block truncation upon failure.
1956  */
1957 int block_write_begin(struct address_space *mapping, loff_t pos, unsigned len,
1958                 unsigned flags, struct page **pagep, get_block_t *get_block)
1959 {
1960         pgoff_t index = pos >> PAGE_CACHE_SHIFT;
1961         struct page *page;
1962         int status;
1963 
1964         page = grab_cache_page_write_begin(mapping, index, flags);
1965         if (!page)
1966                 return -ENOMEM;
1967 
1968         status = __block_write_begin(page, pos, len, get_block);
1969         if (unlikely(status)) {
1970                 unlock_page(page);
1971                 page_cache_release(page);
1972                 page = NULL;
1973         }
1974 
1975         *pagep = page;
1976         return status;
1977 }
1978 EXPORT_SYMBOL(block_write_begin);
1979 
1980 int block_write_end(struct file *file, struct address_space *mapping,
1981                         loff_t pos, unsigned len, unsigned copied,
1982                         struct page *page, void *fsdata)
1983 {
1984         struct inode *inode = mapping->host;
1985         unsigned start;
1986 
1987         start = pos & (PAGE_CACHE_SIZE - 1);
1988 
1989         if (unlikely(copied < len)) {
1990                 /*
1991                  * The buffers that were written will now be uptodate, so we
1992                  * don't have to worry about a readpage reading them and
1993                  * overwriting a partial write. However if we have encountered
1994                  * a short write and only partially written into a buffer, it
1995                  * will not be marked uptodate, so a readpage might come in and
1996                  * destroy our partial write.
1997                  *
1998                  * Do the simplest thing, and just treat any short write to a
1999                  * non uptodate page as a zero-length write, and force the
2000                  * caller to redo the whole thing.
2001                  */
2002                 if (!PageUptodate(page))
2003                         copied = 0;
2004 
2005                 page_zero_new_buffers(page, start+copied, start+len);
2006         }
2007         flush_dcache_page(page);
2008 
2009         /* This could be a short (even 0-length) commit */
2010         __block_commit_write(inode, page, start, start+copied);
2011 
2012         return copied;
2013 }
2014 EXPORT_SYMBOL(block_write_end);
2015 
2016 int generic_write_end(struct file *file, struct address_space *mapping,
2017                         loff_t pos, unsigned len, unsigned copied,
2018                         struct page *page, void *fsdata)
2019 {
2020         struct inode *inode = mapping->host;
2021         loff_t old_size = inode->i_size;
2022         int i_size_changed = 0;
2023 
2024         copied = block_write_end(file, mapping, pos, len, copied, page, fsdata);
2025 
2026         /*
2027          * No need to use i_size_read() here, the i_size
2028          * cannot change under us because we hold i_mutex.
2029          *
2030          * But it's important to update i_size while still holding page lock:
2031          * page writeout could otherwise come in and zero beyond i_size.
2032          */
2033         if (pos+copied > inode->i_size) {
2034                 i_size_write(inode, pos+copied);
2035                 i_size_changed = 1;
2036         }
2037 
2038         unlock_page(page);
2039         page_cache_release(page);
2040 
2041         if (old_size < pos)
2042                 pagecache_isize_extended(inode, old_size, pos);
2043         /*
2044          * Don't mark the inode dirty under page lock. First, it unnecessarily
2045          * makes the holding time of page lock longer. Second, it forces lock
2046          * ordering of page lock and transaction start for journaling
2047          * filesystems.
2048          */
2049         if (i_size_changed)
2050                 mark_inode_dirty(inode);
2051 
2052         return copied;
2053 }
2054 EXPORT_SYMBOL(generic_write_end);
2055 
2056 /*
2057  * block_is_partially_uptodate checks whether buffers within a page are
2058  * uptodate or not.
2059  *
2060  * Returns true if all buffers which correspond to a file portion
2061  * we want to read are uptodate.
2062  */
2063 int block_is_partially_uptodate(struct page *page, read_descriptor_t *desc,
2064                                         unsigned long from)
2065 {
2066         unsigned block_start, block_end, blocksize;
2067         unsigned to;
2068         struct buffer_head *bh, *head;
2069         int ret = 1;
2070 
2071         if (!page_has_buffers(page))
2072                 return 0;
2073 
2074         head = page_buffers(page);
2075         blocksize = head->b_size;
2076         to = min_t(unsigned, PAGE_CACHE_SIZE - from, desc->count);
2077         to = from + to;
2078         if (from < blocksize && to > PAGE_CACHE_SIZE - blocksize)
2079                 return 0;
2080 
2081         bh = head;
2082         block_start = 0;
2083         do {
2084                 block_end = block_start + blocksize;
2085                 if (block_end > from && block_start < to) {
2086                         if (!buffer_uptodate(bh)) {
2087                                 ret = 0;
2088                                 break;
2089                         }
2090                         if (block_end >= to)
2091                                 break;
2092                 }
2093                 block_start = block_end;
2094                 bh = bh->b_this_page;
2095         } while (bh != head);
2096 
2097         return ret;
2098 }
2099 EXPORT_SYMBOL(block_is_partially_uptodate);
2100 
2101 /*
2102  * Generic "read page" function for block devices that have the normal
2103  * get_block functionality. This is most of the block device filesystems.
2104  * Reads the page asynchronously --- the unlock_buffer() and
2105  * set/clear_buffer_uptodate() functions propagate buffer state into the
2106  * page struct once IO has completed.
2107  */
2108 int block_read_full_page(struct page *page, get_block_t *get_block)
2109 {
2110         struct inode *inode = page->mapping->host;
2111         sector_t iblock, lblock;
2112         struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
2113         unsigned int blocksize, bbits;
2114         int nr, i;
2115         int fully_mapped = 1;
2116 
2117         head = create_page_buffers(page, inode, 0);
2118         blocksize = head->b_size;
2119         bbits = block_size_bits(blocksize);
2120 
2121         iblock = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);
2122         lblock = (i_size_read(inode)+blocksize-1) >> bbits;
2123         bh = head;
2124         nr = 0;
2125         i = 0;
2126 
2127         do {
2128                 if (buffer_uptodate(bh))
2129                         continue;
2130 
2131                 if (!buffer_mapped(bh)) {
2132                         int err = 0;
2133 
2134                         fully_mapped = 0;
2135                         if (iblock < lblock) {
2136                                 WARN_ON(bh->b_size != blocksize);
2137                                 err = get_block(inode, iblock, bh, 0);
2138                                 if (err)
2139                                         SetPageError(page);
2140                         }
2141                         if (!buffer_mapped(bh)) {
2142                                 zero_user(page, i * blocksize, blocksize);
2143                                 if (!err)
2144                                         set_buffer_uptodate(bh);
2145                                 continue;
2146                         }
2147                         /*
2148                          * get_block() might have updated the buffer
2149                          * synchronously
2150                          */
2151                         if (buffer_uptodate(bh))
2152                                 continue;
2153                 }
2154                 arr[nr++] = bh;
2155         } while (i++, iblock++, (bh = bh->b_this_page) != head);
2156 
2157         if (fully_mapped)
2158                 SetPageMappedToDisk(page);
2159 
2160         if (!nr) {
2161                 /*
2162                  * All buffers are uptodate - we can set the page uptodate
2163                  * as well. But not if get_block() returned an error.
2164                  */
2165                 if (!PageError(page))
2166                         SetPageUptodate(page);
2167                 unlock_page(page);
2168                 return 0;
2169         }
2170 
2171         /* Stage two: lock the buffers */
2172         for (i = 0; i < nr; i++) {
2173                 bh = arr[i];
2174                 lock_buffer(bh);
2175                 mark_buffer_async_read(bh);
2176         }
2177 
2178         /*
2179          * Stage 3: start the IO.  Check for uptodateness
2180          * inside the buffer lock in case another process reading
2181          * the underlying blockdev brought it uptodate (the sct fix).
2182          */
2183         for (i = 0; i < nr; i++) {
2184                 bh = arr[i];
2185                 if (buffer_uptodate(bh))
2186                         end_buffer_async_read(bh, 1);
2187                 else
2188                         submit_bh(READ, bh);
2189         }
2190         return 0;
2191 }
2192 EXPORT_SYMBOL(block_read_full_page);
2193 
2194 /* utility function for filesystems that need to do work on expanding
2195  * truncates.  Uses filesystem pagecache writes to allow the filesystem to
2196  * deal with the hole.  
2197  */
2198 int generic_cont_expand_simple(struct inode *inode, loff_t size)
2199 {
2200         struct address_space *mapping = inode->i_mapping;
2201         struct page *page;
2202         void *fsdata;
2203         int err;
2204 
2205         err = inode_newsize_ok(inode, size);
2206         if (err)
2207                 goto out;
2208 
2209         err = pagecache_write_begin(NULL, mapping, size, 0,
2210                                 AOP_FLAG_UNINTERRUPTIBLE|AOP_FLAG_CONT_EXPAND,
2211                                 &page, &fsdata);
2212         if (err)
2213                 goto out;
2214 
2215         err = pagecache_write_end(NULL, mapping, size, 0, 0, page, fsdata);
2216         BUG_ON(err > 0);
2217 
2218 out:
2219         return err;
2220 }
2221 EXPORT_SYMBOL(generic_cont_expand_simple);
2222 
2223 static int cont_expand_zero(struct file *file, struct address_space *mapping,
2224                             loff_t pos, loff_t *bytes)
2225 {
2226         struct inode *inode = mapping->host;
2227         unsigned blocksize = 1 << inode->i_blkbits;
2228         struct page *page;
2229         void *fsdata;
2230         pgoff_t index, curidx;
2231         loff_t curpos;
2232         unsigned zerofrom, offset, len;
2233         int err = 0;
2234 
2235         index = pos >> PAGE_CACHE_SHIFT;
2236         offset = pos & ~PAGE_CACHE_MASK;
2237 
2238         while (index > (curidx = (curpos = *bytes)>>PAGE_CACHE_SHIFT)) {
2239                 zerofrom = curpos & ~PAGE_CACHE_MASK;
2240                 if (zerofrom & (blocksize-1)) {
2241                         *bytes |= (blocksize-1);
2242                         (*bytes)++;
2243                 }
2244                 len = PAGE_CACHE_SIZE - zerofrom;
2245 
2246                 err = pagecache_write_begin(file, mapping, curpos, len,
2247                                                 AOP_FLAG_UNINTERRUPTIBLE,
2248                                                 &page, &fsdata);
2249                 if (err)
2250                         goto out;
2251                 zero_user(page, zerofrom, len);
2252                 err = pagecache_write_end(file, mapping, curpos, len, len,
2253                                                 page, fsdata);
2254                 if (err < 0)
2255                         goto out;
2256                 BUG_ON(err != len);
2257                 err = 0;
2258 
2259                 balance_dirty_pages_ratelimited(mapping);
2260 
2261                 if (unlikely(fatal_signal_pending(current))) {
2262                         err = -EINTR;
2263                         goto out;
2264                 }
2265         }
2266 
2267         /* page covers the boundary, find the boundary offset */
2268         if (index == curidx) {
2269                 zerofrom = curpos & ~PAGE_CACHE_MASK;
2270                 /* if we will expand the thing last block will be filled */
2271                 if (offset <= zerofrom) {
2272                         goto out;
2273                 }
2274                 if (zerofrom & (blocksize-1)) {
2275                         *bytes |= (blocksize-1);
2276                         (*bytes)++;
2277                 }
2278                 len = offset - zerofrom;
2279 
2280                 err = pagecache_write_begin(file, mapping, curpos, len,
2281                                                 AOP_FLAG_UNINTERRUPTIBLE,
2282                                                 &page, &fsdata);
2283                 if (err)
2284                         goto out;
2285                 zero_user(page, zerofrom, len);
2286                 err = pagecache_write_end(file, mapping, curpos, len, len,
2287                                                 page, fsdata);
2288                 if (err < 0)
2289                         goto out;
2290                 BUG_ON(err != len);
2291                 err = 0;
2292         }
2293 out:
2294         return err;
2295 }
2296 
2297 /*
2298  * For moronic filesystems that do not allow holes in file.
2299  * We may have to extend the file.
2300  */
2301 int cont_write_begin(struct file *file, struct address_space *mapping,
2302                         loff_t pos, unsigned len, unsigned flags,
2303                         struct page **pagep, void **fsdata,
2304                         get_block_t *get_block, loff_t *bytes)
2305 {
2306         struct inode *inode = mapping->host;
2307         unsigned blocksize = 1 << inode->i_blkbits;
2308         unsigned zerofrom;
2309         int err;
2310 
2311         err = cont_expand_zero(file, mapping, pos, bytes);
2312         if (err)
2313                 return err;
2314 
2315         zerofrom = *bytes & ~PAGE_CACHE_MASK;
2316         if (pos+len > *bytes && zerofrom & (blocksize-1)) {
2317                 *bytes |= (blocksize-1);
2318                 (*bytes)++;
2319         }
2320 
2321         return block_write_begin(mapping, pos, len, flags, pagep, get_block);
2322 }
2323 EXPORT_SYMBOL(cont_write_begin);
2324 
2325 int block_commit_write(struct page *page, unsigned from, unsigned to)
2326 {
2327         struct inode *inode = page->mapping->host;
2328         __block_commit_write(inode,page,from,to);
2329         return 0;
2330 }
2331 EXPORT_SYMBOL(block_commit_write);
2332 
2333 /*
2334  * block_page_mkwrite() is not allowed to change the file size as it gets
2335  * called from a page fault handler when a page is first dirtied. Hence we must
2336  * be careful to check for EOF conditions here. We set the page up correctly
2337  * for a written page which means we get ENOSPC checking when writing into
2338  * holes and correct delalloc and unwritten extent mapping on filesystems that
2339  * support these features.
2340  *
2341  * We are not allowed to take the i_mutex here so we have to play games to
2342  * protect against truncate races as the page could now be beyond EOF.  Because
2343  * truncate writes the inode size before removing pages, once we have the
2344  * page lock we can determine safely if the page is beyond EOF. If it is not
2345  * beyond EOF, then the page is guaranteed safe against truncation until we
2346  * unlock the page.
2347  *
2348  * Direct callers of this function should protect against filesystem freezing
2349  * using sb_start_write() - sb_end_write() functions.
2350  */
2351 int __block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
2352                          get_block_t get_block)
2353 {
2354         struct page *page = vmf->page;
2355         struct inode *inode = file_inode(vma->vm_file);
2356         unsigned long end;
2357         loff_t size;
2358         int ret;
2359 
2360         lock_page(page);
2361         size = i_size_read(inode);
2362         if ((page->mapping != inode->i_mapping) ||
2363             (page_offset(page) > size)) {
2364                 /* We overload EFAULT to mean page got truncated */
2365                 ret = -EFAULT;
2366                 goto out_unlock;
2367         }
2368 
2369         /* page is wholly or partially inside EOF */
2370         if (((page->index + 1) << PAGE_CACHE_SHIFT) > size)
2371                 end = size & ~PAGE_CACHE_MASK;
2372         else
2373                 end = PAGE_CACHE_SIZE;
2374 
2375         ret = __block_write_begin(page, 0, end, get_block);
2376         if (!ret)
2377                 ret = block_commit_write(page, 0, end);
2378 
2379         if (unlikely(ret < 0))
2380                 goto out_unlock;
2381         set_page_dirty(page);
2382         wait_for_stable_page(page);
2383         return 0;
2384 out_unlock:
2385         unlock_page(page);
2386         return ret;
2387 }
2388 EXPORT_SYMBOL(__block_page_mkwrite);
2389 
2390 int block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
2391                    get_block_t get_block)
2392 {
2393         int ret;
2394         struct super_block *sb = file_inode(vma->vm_file)->i_sb;
2395 
2396         sb_start_pagefault(sb);
2397 
2398         /*
2399          * Update file times before taking page lock. We may end up failing the
2400          * fault so this update may be superfluous but who really cares...
2401          */
2402         file_update_time(vma->vm_file);
2403 
2404         ret = __block_page_mkwrite(vma, vmf, get_block);
2405         sb_end_pagefault(sb);
2406         return block_page_mkwrite_return(ret);
2407 }
2408 EXPORT_SYMBOL(block_page_mkwrite);
2409 
2410 /*
2411  * nobh_write_begin()'s prereads are special: the buffer_heads are freed
2412  * immediately, while under the page lock.  So it needs a special end_io
2413  * handler which does not touch the bh after unlocking it.
2414  */
2415 static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
2416 {
2417         __end_buffer_read_notouch(bh, uptodate);
2418 }
2419 
2420 /*
2421  * Attach the singly-linked list of buffers created by nobh_write_begin, to
2422  * the page (converting it to circular linked list and taking care of page
2423  * dirty races).
2424  */
2425 static void attach_nobh_buffers(struct page *page, struct buffer_head *head)
2426 {
2427         struct buffer_head *bh;
2428 
2429         BUG_ON(!PageLocked(page));
2430 
2431         spin_lock(&page->mapping->private_lock);
2432         bh = head;
2433         do {
2434                 if (PageDirty(page))
2435                         set_buffer_dirty(bh);
2436                 if (!bh->b_this_page)
2437                         bh->b_this_page = head;
2438                 bh = bh->b_this_page;
2439         } while (bh != head);
2440         attach_page_buffers(page, head);
2441         spin_unlock(&page->mapping->private_lock);
2442 }
2443 
2444 /*
2445  * On entry, the page is fully not uptodate.
2446  * On exit the page is fully uptodate in the areas outside (from,to)
2447  * The filesystem needs to handle block truncation upon failure.
2448  */
2449 int nobh_write_begin(struct address_space *mapping,
2450                         loff_t pos, unsigned len, unsigned flags,
2451                         struct page **pagep, void **fsdata,
2452                         get_block_t *get_block)
2453 {
2454         struct inode *inode = mapping->host;
2455         const unsigned blkbits = inode->i_blkbits;
2456         const unsigned blocksize = 1 << blkbits;
2457         struct buffer_head *head, *bh;
2458         struct page *page;
2459         pgoff_t index;
2460         unsigned from, to;
2461         unsigned block_in_page;
2462         unsigned block_start, block_end;
2463         sector_t block_in_file;
2464         int nr_reads = 0;
2465         int ret = 0;
2466         int is_mapped_to_disk = 1;
2467 
2468         index = pos >> PAGE_CACHE_SHIFT;
2469         from = pos & (PAGE_CACHE_SIZE - 1);
2470         to = from + len;
2471 
2472         page = grab_cache_page_write_begin(mapping, index, flags);
2473         if (!page)
2474                 return -ENOMEM;
2475         *pagep = page;
2476         *fsdata = NULL;
2477 
2478         if (page_has_buffers(page)) {
2479                 ret = __block_write_begin(page, pos, len, get_block);
2480                 if (unlikely(ret))
2481                         goto out_release;
2482                 return ret;
2483         }
2484 
2485         if (PageMappedToDisk(page))
2486                 return 0;
2487 
2488         /*
2489          * Allocate buffers so that we can keep track of state, and potentially
2490          * attach them to the page if an error occurs. In the common case of
2491          * no error, they will just be freed again without ever being attached
2492          * to the page (which is all OK, because we're under the page lock).
2493          *
2494          * Be careful: the buffer linked list is a NULL terminated one, rather
2495          * than the circular one we're used to.
2496          */
2497         head = alloc_page_buffers(page, blocksize, 0);
2498         if (!head) {
2499                 ret = -ENOMEM;
2500                 goto out_release;
2501         }
2502 
2503         block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits);
2504 
2505         /*
2506          * We loop across all blocks in the page, whether or not they are
2507          * part of the affected region.  This is so we can discover if the
2508          * page is fully mapped-to-disk.
2509          */
2510         for (block_start = 0, block_in_page = 0, bh = head;
2511                   block_start < PAGE_CACHE_SIZE;
2512                   block_in_page++, block_start += blocksize, bh = bh->b_this_page) {
2513                 int create;
2514 
2515                 block_end = block_start + blocksize;
2516                 bh->b_state = 0;
2517                 create = 1;
2518                 if (block_start >= to)
2519                         create = 0;
2520                 ret = get_block(inode, block_in_file + block_in_page,
2521                                         bh, create);
2522                 if (ret)
2523                         goto failed;
2524                 if (!buffer_mapped(bh))
2525                         is_mapped_to_disk = 0;
2526                 if (buffer_new(bh))
2527                         unmap_underlying_metadata(bh->b_bdev, bh->b_blocknr);
2528                 if (PageUptodate(page)) {
2529                         set_buffer_uptodate(bh);
2530                         continue;
2531                 }
2532                 if (buffer_new(bh) || !buffer_mapped(bh)) {
2533                         zero_user_segments(page, block_start, from,
2534                                                         to, block_end);
2535                         continue;
2536                 }
2537                 if (buffer_uptodate(bh))
2538                         continue;       /* reiserfs does this */
2539                 if (block_start < from || block_end > to) {
2540                         lock_buffer(bh);
2541                         bh->b_end_io = end_buffer_read_nobh;
2542                         submit_bh(READ, bh);
2543                         nr_reads++;
2544                 }
2545         }
2546 
2547         if (nr_reads) {
2548                 /*
2549                  * The page is locked, so these buffers are protected from
2550                  * any VM or truncate activity.  Hence we don't need to care
2551                  * for the buffer_head refcounts.
2552                  */
2553                 for (bh = head; bh; bh = bh->b_this_page) {
2554                         wait_on_buffer(bh);
2555                         if (!buffer_uptodate(bh))
2556                                 ret = -EIO;
2557                 }
2558                 if (ret)
2559                         goto failed;
2560         }
2561 
2562         if (is_mapped_to_disk)
2563                 SetPageMappedToDisk(page);
2564 
2565         *fsdata = head; /* to be released by nobh_write_end */
2566 
2567         return 0;
2568 
2569 failed:
2570         BUG_ON(!ret);
2571         /*
2572          * Error recovery is a bit difficult. We need to zero out blocks that
2573          * were newly allocated, and dirty them to ensure they get written out.
2574          * Buffers need to be attached to the page at this point, otherwise
2575          * the handling of potential IO errors during writeout would be hard
2576          * (could try doing synchronous writeout, but what if that fails too?)
2577          */
2578         attach_nobh_buffers(page, head);
2579         page_zero_new_buffers(page, from, to);
2580 
2581 out_release:
2582         unlock_page(page);
2583         page_cache_release(page);
2584         *pagep = NULL;
2585 
2586         return ret;
2587 }
2588 EXPORT_SYMBOL(nobh_write_begin);
2589 
2590 int nobh_write_end(struct file *file, struct address_space *mapping,
2591                         loff_t pos, unsigned len, unsigned copied,
2592                         struct page *page, void *fsdata)
2593 {
2594         struct inode *inode = page->mapping->host;
2595         struct buffer_head *head = fsdata;
2596         struct buffer_head *bh;
2597         BUG_ON(fsdata != NULL && page_has_buffers(page));
2598 
2599         if (unlikely(copied < len) && head)
2600                 attach_nobh_buffers(page, head);
2601         if (page_has_buffers(page))
2602                 return generic_write_end(file, mapping, pos, len,
2603                                         copied, page, fsdata);
2604 
2605         SetPageUptodate(page);
2606         set_page_dirty(page);
2607         if (pos+copied > inode->i_size) {
2608                 i_size_write(inode, pos+copied);
2609                 mark_inode_dirty(inode);
2610         }
2611 
2612         unlock_page(page);
2613         page_cache_release(page);
2614 
2615         while (head) {
2616                 bh = head;
2617                 head = head->b_this_page;
2618                 free_buffer_head(bh);
2619         }
2620 
2621         return copied;
2622 }
2623 EXPORT_SYMBOL(nobh_write_end);
2624 
2625 /*
2626  * nobh_writepage() - based on block_full_write_page() except
2627  * that it tries to operate without attaching bufferheads to
2628  * the page.
2629  */
2630 int nobh_writepage(struct page *page, get_block_t *get_block,
2631                         struct writeback_control *wbc)
2632 {
2633         struct inode * const inode = page->mapping->host;
2634         loff_t i_size = i_size_read(inode);
2635         const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2636         unsigned offset;
2637         int ret;
2638 
2639         /* Is the page fully inside i_size? */
2640         if (page->index < end_index)
2641                 goto out;
2642 
2643         /* Is the page fully outside i_size? (truncate in progress) */
2644         offset = i_size & (PAGE_CACHE_SIZE-1);
2645         if (page->index >= end_index+1 || !offset) {
2646                 /*
2647                  * The page may have dirty, unmapped buffers.  For example,
2648                  * they may have been added in ext3_writepage().  Make them
2649                  * freeable here, so the page does not leak.
2650                  */
2651 #if 0
2652                 /* Not really sure about this  - do we need this ? */
2653                 if (page->mapping->a_ops->invalidatepage)
2654                         page->mapping->a_ops->invalidatepage(page, offset);
2655 #endif
2656                 unlock_page(page);
2657                 return 0; /* don't care */
2658         }
2659 
2660         /*
2661          * The page straddles i_size.  It must be zeroed out on each and every
2662          * writepage invocation because it may be mmapped.  "A file is mapped
2663          * in multiples of the page size.  For a file that is not a multiple of
2664          * the  page size, the remaining memory is zeroed when mapped, and
2665          * writes to that region are not written out to the file."
2666          */
2667         zero_user_segment(page, offset, PAGE_CACHE_SIZE);
2668 out:
2669         ret = mpage_writepage(page, get_block, wbc);
2670         if (ret == -EAGAIN)
2671                 ret = __block_write_full_page(inode, page, get_block, wbc,
2672                                               end_buffer_async_write);
2673         return ret;
2674 }
2675 EXPORT_SYMBOL(nobh_writepage);
2676 
2677 int nobh_truncate_page(struct address_space *mapping,
2678                         loff_t from, get_block_t *get_block)
2679 {
2680         pgoff_t index = from >> PAGE_CACHE_SHIFT;
2681         unsigned offset = from & (PAGE_CACHE_SIZE-1);
2682         unsigned blocksize;
2683         sector_t iblock;
2684         unsigned length, pos;
2685         struct inode *inode = mapping->host;
2686         struct page *page;
2687         struct buffer_head map_bh;
2688         int err;
2689 
2690         blocksize = 1 << inode->i_blkbits;
2691         length = offset & (blocksize - 1);
2692 
2693         /* Block boundary? Nothing to do */
2694         if (!length)
2695                 return 0;
2696 
2697         length = blocksize - length;
2698         iblock = (sector_t)index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2699 
2700         page = grab_cache_page(mapping, index);
2701         err = -ENOMEM;
2702         if (!page)
2703                 goto out;
2704 
2705         if (page_has_buffers(page)) {
2706 has_buffers:
2707                 unlock_page(page);
2708                 page_cache_release(page);
2709                 return block_truncate_page(mapping, from, get_block);
2710         }
2711 
2712         /* Find the buffer that contains "offset" */
2713         pos = blocksize;
2714         while (offset >= pos) {
2715                 iblock++;
2716                 pos += blocksize;
2717         }
2718 
2719         map_bh.b_size = blocksize;
2720         map_bh.b_state = 0;
2721         err = get_block(inode, iblock, &map_bh, 0);
2722         if (err)
2723                 goto unlock;
2724         /* unmapped? It's a hole - nothing to do */
2725         if (!buffer_mapped(&map_bh))
2726                 goto unlock;
2727 
2728         /* Ok, it's mapped. Make sure it's up-to-date */
2729         if (!PageUptodate(page)) {
2730                 err = mapping->a_ops->readpage(NULL, page);
2731                 if (err) {
2732                         page_cache_release(page);
2733                         goto out;
2734                 }
2735                 lock_page(page);
2736                 if (!PageUptodate(page)) {
2737                         err = -EIO;
2738                         goto unlock;
2739                 }
2740                 if (page_has_buffers(page))
2741                         goto has_buffers;
2742         }
2743         zero_user(page, offset, length);
2744         set_page_dirty(page);
2745         err = 0;
2746 
2747 unlock:
2748         unlock_page(page);
2749         page_cache_release(page);
2750 out:
2751         return err;
2752 }
2753 EXPORT_SYMBOL(nobh_truncate_page);
2754 
2755 int block_truncate_page(struct address_space *mapping,
2756                         loff_t from, get_block_t *get_block)
2757 {
2758         pgoff_t index = from >> PAGE_CACHE_SHIFT;
2759         unsigned offset = from & (PAGE_CACHE_SIZE-1);
2760         unsigned blocksize;
2761         sector_t iblock;
2762         unsigned length, pos;
2763         struct inode *inode = mapping->host;
2764         struct page *page;
2765         struct buffer_head *bh;
2766         int err;
2767 
2768         blocksize = 1 << inode->i_blkbits;
2769         length = offset & (blocksize - 1);
2770 
2771         /* Block boundary? Nothing to do */
2772         if (!length)
2773                 return 0;
2774 
2775         length = blocksize - length;
2776         iblock = (sector_t)index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2777         
2778         page = grab_cache_page(mapping, index);
2779         err = -ENOMEM;
2780         if (!page)
2781                 goto out;
2782 
2783         if (!page_has_buffers(page))
2784                 create_empty_buffers(page, blocksize, 0);
2785 
2786         /* Find the buffer that contains "offset" */
2787         bh = page_buffers(page);
2788         pos = blocksize;
2789         while (offset >= pos) {
2790                 bh = bh->b_this_page;
2791                 iblock++;
2792                 pos += blocksize;
2793         }
2794 
2795         err = 0;
2796         if (!buffer_mapped(bh)) {
2797                 WARN_ON(bh->b_size != blocksize);
2798                 err = get_block(inode, iblock, bh, 0);
2799                 if (err)
2800                         goto unlock;
2801                 /* unmapped? It's a hole - nothing to do */
2802                 if (!buffer_mapped(bh))
2803                         goto unlock;
2804         }
2805 
2806         /* Ok, it's mapped. Make sure it's up-to-date */
2807         if (PageUptodate(page))
2808                 set_buffer_uptodate(bh);
2809 
2810         if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) {
2811                 err = -EIO;
2812                 ll_rw_block(READ, 1, &bh);
2813                 wait_on_buffer(bh);
2814                 /* Uhhuh. Read error. Complain and punt. */
2815                 if (!buffer_uptodate(bh))
2816                         goto unlock;
2817         }
2818 
2819         zero_user(page, offset, length);
2820         mark_buffer_dirty(bh);
2821         err = 0;
2822 
2823 unlock:
2824         unlock_page(page);
2825         page_cache_release(page);
2826 out:
2827         return err;
2828 }
2829 EXPORT_SYMBOL(block_truncate_page);
2830 
2831 /*
2832  * The generic ->writepage function for buffer-backed address_spaces
2833  * this form passes in the end_io handler used to finish the IO.
2834  */
2835 int block_write_full_page_endio(struct page *page, get_block_t *get_block,
2836                         struct writeback_control *wbc, bh_end_io_t *handler)
2837 {
2838         struct inode * const inode = page->mapping->host;
2839         loff_t i_size = i_size_read(inode);
2840         const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2841         unsigned offset;
2842 
2843         /* Is the page fully inside i_size? */
2844         if (page->index < end_index)
2845                 return __block_write_full_page(inode, page, get_block, wbc,
2846                                                handler);
2847 
2848         /* Is the page fully outside i_size? (truncate in progress) */
2849         offset = i_size & (PAGE_CACHE_SIZE-1);
2850         if (page->index >= end_index+1 || !offset) {
2851                 /*
2852                  * The page may have dirty, unmapped buffers.  For example,
2853                  * they may have been added in ext3_writepage().  Make them
2854                  * freeable here, so the page does not leak.
2855                  */
2856                 do_invalidatepage(page, 0);
2857                 unlock_page(page);
2858                 return 0; /* don't care */
2859         }
2860 
2861         /*
2862          * The page straddles i_size.  It must be zeroed out on each and every
2863          * writepage invocation because it may be mmapped.  "A file is mapped
2864          * in multiples of the page size.  For a file that is not a multiple of
2865          * the  page size, the remaining memory is zeroed when mapped, and
2866          * writes to that region are not written out to the file."
2867          */
2868         zero_user_segment(page, offset, PAGE_CACHE_SIZE);
2869         return __block_write_full_page(inode, page, get_block, wbc, handler);
2870 }
2871 EXPORT_SYMBOL(block_write_full_page_endio);
2872 
2873 /*
2874  * The generic ->writepage function for buffer-backed address_spaces
2875  */
2876 int block_write_full_page(struct page *page, get_block_t *get_block,
2877                         struct writeback_control *wbc)
2878 {
2879         return block_write_full_page_endio(page, get_block, wbc,
2880                                            end_buffer_async_write);
2881 }
2882 EXPORT_SYMBOL(block_write_full_page);
2883 
2884 sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
2885                             get_block_t *get_block)
2886 {
2887         struct buffer_head tmp;
2888         struct inode *inode = mapping->host;
2889         tmp.b_state = 0;
2890         tmp.b_blocknr = 0;
2891         tmp.b_size = 1 << inode->i_blkbits;
2892         get_block(inode, block, &tmp, 0);
2893         return tmp.b_blocknr;
2894 }
2895 EXPORT_SYMBOL(generic_block_bmap);
2896 
2897 static void end_bio_bh_io_sync(struct bio *bio, int err)
2898 {
2899         struct buffer_head *bh = bio->bi_private;
2900 
2901         if (err == -EOPNOTSUPP) {
2902                 set_bit(BIO_EOPNOTSUPP, &bio->bi_flags);
2903         }
2904 
2905         if (unlikely (test_bit(BIO_QUIET,&bio->bi_flags)))
2906                 set_bit(BH_Quiet, &bh->b_state);
2907 
2908         bh->b_end_io(bh, test_bit(BIO_UPTODATE, &bio->bi_flags));
2909         bio_put(bio);
2910 }
2911 
2912 /*
2913  * This allows us to do IO even on the odd last sectors
2914  * of a device, even if the bh block size is some multiple
2915  * of the physical sector size.
2916  *
2917  * We'll just truncate the bio to the size of the device,
2918  * and clear the end of the buffer head manually.
2919  *
2920  * Truly out-of-range accesses will turn into actual IO
2921  * errors, this only handles the "we need to be able to
2922  * do IO at the final sector" case.
2923  */
2924 static void guard_bh_eod(int rw, struct bio *bio, struct buffer_head *bh)
2925 {
2926         sector_t maxsector;
2927         unsigned bytes;
2928 
2929         maxsector = i_size_read(bio->bi_bdev->bd_inode) >> 9;
2930         if (!maxsector)
2931                 return;
2932 
2933         /*
2934          * If the *whole* IO is past the end of the device,
2935          * let it through, and the IO layer will turn it into
2936          * an EIO.
2937          */
2938         if (unlikely(bio->bi_sector >= maxsector))
2939                 return;
2940 
2941         maxsector -= bio->bi_sector;
2942         bytes = bio->bi_size;
2943         if (likely((bytes >> 9) <= maxsector))
2944                 return;
2945 
2946         /* Uhhuh. We've got a bh that straddles the device size! */
2947         bytes = maxsector << 9;
2948 
2949         /* Truncate the bio.. */
2950         bio->bi_size = bytes;
2951         bio->bi_io_vec[0].bv_len = bytes;
2952 
2953         /* ..and clear the end of the buffer for reads */
2954         if ((rw & RW_MASK) == READ) {
2955                 void *kaddr = kmap_atomic(bh->b_page);
2956                 memset(kaddr + bh_offset(bh) + bytes, 0, bh->b_size - bytes);
2957                 kunmap_atomic(kaddr);
2958                 flush_dcache_page(bh->b_page);
2959         }
2960 }
2961 
2962 int _submit_bh(int rw, struct buffer_head *bh, unsigned long bio_flags)
2963 {
2964         struct bio *bio;
2965         int ret = 0;
2966 
2967         BUG_ON(!buffer_locked(bh));
2968         BUG_ON(!buffer_mapped(bh));
2969         BUG_ON(!bh->b_end_io);
2970         BUG_ON(buffer_delay(bh));
2971         BUG_ON(buffer_unwritten(bh));
2972 
2973         /*
2974          * Only clear out a write error when rewriting
2975          */
2976         if (test_set_buffer_req(bh) && (rw & WRITE))
2977                 clear_buffer_write_io_error(bh);
2978 
2979         /*
2980          * from here on down, it's all bio -- do the initial mapping,
2981          * submit_bio -> generic_make_request may further map this bio around
2982          */
2983         bio = bio_alloc(GFP_NOIO, 1);
2984 
2985         bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9);
2986         bio->bi_bdev = bh->b_bdev;
2987         bio->bi_io_vec[0].bv_page = bh->b_page;
2988         bio->bi_io_vec[0].bv_len = bh->b_size;
2989         bio->bi_io_vec[0].bv_offset = bh_offset(bh);
2990 
2991         bio->bi_vcnt = 1;
2992         bio->bi_size = bh->b_size;
2993 
2994         bio->bi_end_io = end_bio_bh_io_sync;
2995         bio->bi_private = bh;
2996         bio->bi_flags |= bio_flags;
2997 
2998         /* Take care of bh's that straddle the end of the device */
2999         guard_bh_eod(rw, bio, bh);
3000 
3001         if (buffer_meta(bh))
3002                 rw |= REQ_META;
3003         if (buffer_prio(bh))
3004                 rw |= REQ_PRIO;
3005 
3006         bio_get(bio);
3007         submit_bio(rw, bio);
3008 
3009         if (bio_flagged(bio, BIO_EOPNOTSUPP))
3010                 ret = -EOPNOTSUPP;
3011 
3012         bio_put(bio);
3013         return ret;
3014 }
3015 EXPORT_SYMBOL_GPL(_submit_bh);
3016 
3017 int submit_bh(int rw, struct buffer_head *bh)
3018 {
3019         return _submit_bh(rw, bh, 0);
3020 }
3021 EXPORT_SYMBOL(submit_bh);
3022 
3023 /**
3024  * ll_rw_block: low-level access to block devices (DEPRECATED)
3025  * @rw: whether to %READ or %WRITE or maybe %READA (readahead)
3026  * @nr: number of &struct buffer_heads in the array
3027  * @bhs: array of pointers to &struct buffer_head
3028  *
3029  * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
3030  * requests an I/O operation on them, either a %READ or a %WRITE.  The third
3031  * %READA option is described in the documentation for generic_make_request()
3032  * which ll_rw_block() calls.
3033  *
3034  * This function drops any buffer that it cannot get a lock on (with the
3035  * BH_Lock state bit), any buffer that appears to be clean when doing a write
3036  * request, and any buffer that appears to be up-to-date when doing read
3037  * request.  Further it marks as clean buffers that are processed for
3038  * writing (the buffer cache won't assume that they are actually clean
3039  * until the buffer gets unlocked).
3040  *
3041  * ll_rw_block sets b_end_io to simple completion handler that marks
3042  * the buffer up-to-date (if approriate), unlocks the buffer and wakes
3043  * any waiters. 
3044  *
3045  * All of the buffers must be for the same device, and must also be a
3046  * multiple of the current approved size for the device.
3047  */
3048 void ll_rw_block(int rw, int nr, struct buffer_head *bhs[])
3049 {
3050         int i;
3051 
3052         for (i = 0; i < nr; i++) {
3053                 struct buffer_head *bh = bhs[i];
3054 
3055                 if (!trylock_buffer(bh))
3056                         continue;
3057                 if (rw == WRITE) {
3058                         if (test_clear_buffer_dirty(bh)) {
3059                                 bh->b_end_io = end_buffer_write_sync;
3060                                 get_bh(bh);
3061                                 submit_bh(WRITE, bh);
3062                                 continue;
3063                         }
3064                 } else {
3065                         if (!buffer_uptodate(bh)) {
3066                                 bh->b_end_io = end_buffer_read_sync;
3067                                 get_bh(bh);
3068                                 submit_bh(rw, bh);
3069                                 continue;
3070                         }
3071                 }
3072                 unlock_buffer(bh);
3073         }
3074 }
3075 EXPORT_SYMBOL(ll_rw_block);
3076 
3077 void write_dirty_buffer(struct buffer_head *bh, int rw)
3078 {
3079         lock_buffer(bh);
3080         if (!test_clear_buffer_dirty(bh)) {
3081                 unlock_buffer(bh);
3082                 return;
3083         }
3084         bh->b_end_io = end_buffer_write_sync;
3085         get_bh(bh);
3086         submit_bh(rw, bh);
3087 }
3088 EXPORT_SYMBOL(write_dirty_buffer);
3089 
3090 /*
3091  * For a data-integrity writeout, we need to wait upon any in-progress I/O
3092  * and then start new I/O and then wait upon it.  The caller must have a ref on
3093  * the buffer_head.
3094  */
3095 int __sync_dirty_buffer(struct buffer_head *bh, int rw)
3096 {
3097         int ret = 0;
3098 
3099         WARN_ON(atomic_read(&bh->b_count) < 1);
3100         lock_buffer(bh);
3101         if (test_clear_buffer_dirty(bh)) {
3102                 get_bh(bh);
3103                 bh->b_end_io = end_buffer_write_sync;
3104                 ret = submit_bh(rw, bh);
3105                 wait_on_buffer(bh);
3106                 if (!ret && !buffer_uptodate(bh))
3107                         ret = -EIO;
3108         } else {
3109                 unlock_buffer(bh);
3110         }
3111         return ret;
3112 }
3113 EXPORT_SYMBOL(__sync_dirty_buffer);
3114 
3115 int sync_dirty_buffer(struct buffer_head *bh)
3116 {
3117         return __sync_dirty_buffer(bh, WRITE_SYNC);
3118 }
3119 EXPORT_SYMBOL(sync_dirty_buffer);
3120 
3121 /*
3122  * try_to_free_buffers() checks if all the buffers on this particular page
3123  * are unused, and releases them if so.
3124  *
3125  * Exclusion against try_to_free_buffers may be obtained by either
3126  * locking the page or by holding its mapping's private_lock.
3127  *
3128  * If the page is dirty but all the buffers are clean then we need to
3129  * be sure to mark the page clean as well.  This is because the page
3130  * may be against a block device, and a later reattachment of buffers
3131  * to a dirty page will set *all* buffers dirty.  Which would corrupt
3132  * filesystem data on the same device.
3133  *
3134  * The same applies to regular filesystem pages: if all the buffers are
3135  * clean then we set the page clean and proceed.  To do that, we require
3136  * total exclusion from __set_page_dirty_buffers().  That is obtained with
3137  * private_lock.
3138  *
3139  * try_to_free_buffers() is non-blocking.
3140  */
3141 static inline int buffer_busy(struct buffer_head *bh)
3142 {
3143         return atomic_read(&bh->b_count) |
3144                 (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
3145 }
3146 
3147 static int
3148 drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
3149 {
3150         struct buffer_head *head = page_buffers(page);
3151         struct buffer_head *bh;
3152 
3153         bh = head;
3154         do {
3155                 if (buffer_write_io_error(bh) && page->mapping)
3156                         set_bit(AS_EIO, &page->mapping->flags);
3157                 if (buffer_busy(bh))
3158                         goto failed;
3159                 bh = bh->b_this_page;
3160         } while (bh != head);
3161 
3162         do {
3163                 struct buffer_head *next = bh->b_this_page;
3164 
3165                 if (bh->b_assoc_map)
3166                         __remove_assoc_queue(bh);
3167                 bh = next;
3168         } while (bh != head);
3169         *buffers_to_free = head;
3170         __clear_page_buffers(page);
3171         return 1;
3172 failed:
3173         return 0;
3174 }
3175 
3176 int try_to_free_buffers(struct page *page)
3177 {
3178         struct address_space * const mapping = page->mapping;
3179         struct buffer_head *buffers_to_free = NULL;
3180         int ret = 0;
3181 
3182         BUG_ON(!PageLocked(page));
3183         if (PageWriteback(page))
3184                 return 0;
3185 
3186         if (mapping == NULL) {          /* can this still happen? */
3187                 ret = drop_buffers(page, &buffers_to_free);
3188                 goto out;
3189         }
3190 
3191         spin_lock(&mapping->private_lock);
3192         ret = drop_buffers(page, &buffers_to_free);
3193 
3194         /*
3195          * If the filesystem writes its buffers by hand (eg ext3)
3196          * then we can have clean buffers against a dirty page.  We
3197          * clean the page here; otherwise the VM will never notice
3198          * that the filesystem did any IO at all.
3199          *
3200          * Also, during truncate, discard_buffer will have marked all
3201          * the page's buffers clean.  We discover that here and clean
3202          * the page also.
3203          *
3204          * private_lock must be held over this entire operation in order
3205          * to synchronise against __set_page_dirty_buffers and prevent the
3206          * dirty bit from being lost.
3207          */
3208         if (ret)
3209                 cancel_dirty_page(page, PAGE_CACHE_SIZE);
3210         spin_unlock(&mapping->private_lock);
3211 out:
3212         if (buffers_to_free) {
3213                 struct buffer_head *bh = buffers_to_free;
3214 
3215                 do {
3216                         struct buffer_head *next = bh->b_this_page;
3217                         free_buffer_head(bh);
3218                         bh = next;
3219                 } while (bh != buffers_to_free);
3220         }
3221         return ret;
3222 }
3223 EXPORT_SYMBOL(try_to_free_buffers);
3224 
3225 /*
3226  * There are no bdflush tunables left.  But distributions are
3227  * still running obsolete flush daemons, so we terminate them here.
3228  *
3229  * Use of bdflush() is deprecated and will be removed in a future kernel.
3230  * The `flush-X' kernel threads fully replace bdflush daemons and this call.
3231  */
3232 SYSCALL_DEFINE2(bdflush, int, func, long, data)
3233 {
3234         static int msg_count;
3235 
3236         if (!capable(CAP_SYS_ADMIN))
3237                 return -EPERM;
3238 
3239         if (msg_count < 5) {
3240                 msg_count++;
3241                 printk(KERN_INFO
3242                         "warning: process `%s' used the obsolete bdflush"
3243                         " system call\n", current->comm);
3244                 printk(KERN_INFO "Fix your initscripts?\n");
3245         }
3246 
3247         if (func == 1)
3248                 do_exit(0);
3249         return 0;
3250 }
3251 
3252 /*
3253  * Buffer-head allocation
3254  */
3255 static struct kmem_cache *bh_cachep __read_mostly;
3256 
3257 /*
3258  * Once the number of bh's in the machine exceeds this level, we start
3259  * stripping them in writeback.
3260  */
3261 static unsigned long max_buffer_heads;
3262 
3263 int buffer_heads_over_limit;
3264 
3265 struct bh_accounting {
3266         int nr;                 /* Number of live bh's */
3267         int ratelimit;          /* Limit cacheline bouncing */
3268 };
3269 
3270 static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
3271 
3272 static void recalc_bh_state(void)
3273 {
3274         int i;
3275         int tot = 0;
3276 
3277         if (__this_cpu_inc_return(bh_accounting.ratelimit) - 1 < 4096)
3278                 return;
3279         __this_cpu_write(bh_accounting.ratelimit, 0);
3280         for_each_online_cpu(i)
3281                 tot += per_cpu(bh_accounting, i).nr;
3282         buffer_heads_over_limit = (tot > max_buffer_heads);
3283 }
3284 
3285 struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
3286 {
3287         struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags);
3288         if (ret) {
3289                 INIT_LIST_HEAD(&ret->b_assoc_buffers);
3290                 preempt_disable();
3291                 __this_cpu_inc(bh_accounting.nr);
3292                 recalc_bh_state();
3293                 preempt_enable();
3294         }
3295         return ret;
3296 }
3297 EXPORT_SYMBOL(alloc_buffer_head);
3298 
3299 void free_buffer_head(struct buffer_head *bh)
3300 {
3301         BUG_ON(!list_empty(&bh->b_assoc_buffers));
3302         kmem_cache_free(bh_cachep, bh);
3303         preempt_disable();
3304         __this_cpu_dec(bh_accounting.nr);
3305         recalc_bh_state();
3306         preempt_enable();
3307 }
3308 EXPORT_SYMBOL(free_buffer_head);
3309 
3310 static void buffer_exit_cpu(int cpu)
3311 {
3312         int i;
3313         struct bh_lru *b = &per_cpu(bh_lrus, cpu);
3314 
3315         for (i = 0; i < BH_LRU_SIZE; i++) {
3316                 brelse(b->bhs[i]);
3317                 b->bhs[i] = NULL;
3318         }
3319         this_cpu_add(bh_accounting.nr, per_cpu(bh_accounting, cpu).nr);
3320         per_cpu(bh_accounting, cpu).nr = 0;
3321 }
3322 
3323 static int buffer_cpu_notify(struct notifier_block *self,
3324                               unsigned long action, void *hcpu)
3325 {
3326         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN)
3327                 buffer_exit_cpu((unsigned long)hcpu);
3328         return NOTIFY_OK;
3329 }
3330 
3331 /**
3332  * bh_uptodate_or_lock - Test whether the buffer is uptodate
3333  * @bh: struct buffer_head
3334  *
3335  * Return true if the buffer is up-to-date and false,
3336  * with the buffer locked, if not.
3337  */
3338 int bh_uptodate_or_lock(struct buffer_head *bh)
3339 {
3340         if (!buffer_uptodate(bh)) {
3341                 lock_buffer(bh);
3342                 if (!buffer_uptodate(bh))
3343                         return 0;
3344                 unlock_buffer(bh);
3345         }
3346         return 1;
3347 }
3348 EXPORT_SYMBOL(bh_uptodate_or_lock);
3349 
3350 /**
3351  * bh_submit_read - Submit a locked buffer for reading
3352  * @bh: struct buffer_head
3353  *
3354  * Returns zero on success and -EIO on error.
3355  */
3356 int bh_submit_read(struct buffer_head *bh)
3357 {
3358         BUG_ON(!buffer_locked(bh));
3359 
3360         if (buffer_uptodate(bh)) {
3361                 unlock_buffer(bh);
3362                 return 0;
3363         }
3364 
3365         get_bh(bh);
3366         bh->b_end_io = end_buffer_read_sync;
3367         submit_bh(READ, bh);
3368         wait_on_buffer(bh);
3369         if (buffer_uptodate(bh))
3370                 return 0;
3371         return -EIO;
3372 }
3373 EXPORT_SYMBOL(bh_submit_read);
3374 
3375 void __init buffer_init(void)
3376 {
3377         unsigned long nrpages;
3378 
3379         bh_cachep = kmem_cache_create("buffer_head",
3380                         sizeof(struct buffer_head), 0,
3381                                 (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
3382                                 SLAB_MEM_SPREAD),
3383                                 NULL);
3384 
3385         /*
3386          * Limit the bh occupancy to 10% of ZONE_NORMAL
3387          */
3388         nrpages = (nr_free_buffer_pages() * 10) / 100;
3389         max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
3390         hotcpu_notifier(buffer_cpu_notify, 0);
3391 }
3392 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp