~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/fs/xfs/xfs_buf_item.c

Version: ~ [ linux-5.10-rc5 ] ~ [ linux-5.9.10 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.79 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.159 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.208 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.245 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.245 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Copyright (c) 2000-2005 Silicon Graphics, Inc.
  3  * All Rights Reserved.
  4  *
  5  * This program is free software; you can redistribute it and/or
  6  * modify it under the terms of the GNU General Public License as
  7  * published by the Free Software Foundation.
  8  *
  9  * This program is distributed in the hope that it would be useful,
 10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 12  * GNU General Public License for more details.
 13  *
 14  * You should have received a copy of the GNU General Public License
 15  * along with this program; if not, write the Free Software Foundation,
 16  * Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 17  */
 18 #include "xfs.h"
 19 #include "xfs_fs.h"
 20 #include "xfs_format.h"
 21 #include "xfs_log_format.h"
 22 #include "xfs_trans_resv.h"
 23 #include "xfs_bit.h"
 24 #include "xfs_sb.h"
 25 #include "xfs_mount.h"
 26 #include "xfs_trans.h"
 27 #include "xfs_buf_item.h"
 28 #include "xfs_trans_priv.h"
 29 #include "xfs_error.h"
 30 #include "xfs_trace.h"
 31 #include "xfs_log.h"
 32 
 33 
 34 kmem_zone_t     *xfs_buf_item_zone;
 35 
 36 static inline struct xfs_buf_log_item *BUF_ITEM(struct xfs_log_item *lip)
 37 {
 38         return container_of(lip, struct xfs_buf_log_item, bli_item);
 39 }
 40 
 41 STATIC void     xfs_buf_do_callbacks(struct xfs_buf *bp);
 42 
 43 static inline int
 44 xfs_buf_log_format_size(
 45         struct xfs_buf_log_format *blfp)
 46 {
 47         return offsetof(struct xfs_buf_log_format, blf_data_map) +
 48                         (blfp->blf_map_size * sizeof(blfp->blf_data_map[0]));
 49 }
 50 
 51 /*
 52  * This returns the number of log iovecs needed to log the
 53  * given buf log item.
 54  *
 55  * It calculates this as 1 iovec for the buf log format structure
 56  * and 1 for each stretch of non-contiguous chunks to be logged.
 57  * Contiguous chunks are logged in a single iovec.
 58  *
 59  * If the XFS_BLI_STALE flag has been set, then log nothing.
 60  */
 61 STATIC void
 62 xfs_buf_item_size_segment(
 63         struct xfs_buf_log_item *bip,
 64         struct xfs_buf_log_format *blfp,
 65         int                     *nvecs,
 66         int                     *nbytes)
 67 {
 68         struct xfs_buf          *bp = bip->bli_buf;
 69         int                     next_bit;
 70         int                     last_bit;
 71 
 72         last_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size, 0);
 73         if (last_bit == -1)
 74                 return;
 75 
 76         /*
 77          * initial count for a dirty buffer is 2 vectors - the format structure
 78          * and the first dirty region.
 79          */
 80         *nvecs += 2;
 81         *nbytes += xfs_buf_log_format_size(blfp) + XFS_BLF_CHUNK;
 82 
 83         while (last_bit != -1) {
 84                 /*
 85                  * This takes the bit number to start looking from and
 86                  * returns the next set bit from there.  It returns -1
 87                  * if there are no more bits set or the start bit is
 88                  * beyond the end of the bitmap.
 89                  */
 90                 next_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size,
 91                                         last_bit + 1);
 92                 /*
 93                  * If we run out of bits, leave the loop,
 94                  * else if we find a new set of bits bump the number of vecs,
 95                  * else keep scanning the current set of bits.
 96                  */
 97                 if (next_bit == -1) {
 98                         break;
 99                 } else if (next_bit != last_bit + 1) {
100                         last_bit = next_bit;
101                         (*nvecs)++;
102                 } else if (xfs_buf_offset(bp, next_bit * XFS_BLF_CHUNK) !=
103                            (xfs_buf_offset(bp, last_bit * XFS_BLF_CHUNK) +
104                             XFS_BLF_CHUNK)) {
105                         last_bit = next_bit;
106                         (*nvecs)++;
107                 } else {
108                         last_bit++;
109                 }
110                 *nbytes += XFS_BLF_CHUNK;
111         }
112 }
113 
114 /*
115  * This returns the number of log iovecs needed to log the given buf log item.
116  *
117  * It calculates this as 1 iovec for the buf log format structure and 1 for each
118  * stretch of non-contiguous chunks to be logged.  Contiguous chunks are logged
119  * in a single iovec.
120  *
121  * Discontiguous buffers need a format structure per region that that is being
122  * logged. This makes the changes in the buffer appear to log recovery as though
123  * they came from separate buffers, just like would occur if multiple buffers
124  * were used instead of a single discontiguous buffer. This enables
125  * discontiguous buffers to be in-memory constructs, completely transparent to
126  * what ends up on disk.
127  *
128  * If the XFS_BLI_STALE flag has been set, then log nothing but the buf log
129  * format structures.
130  */
131 STATIC void
132 xfs_buf_item_size(
133         struct xfs_log_item     *lip,
134         int                     *nvecs,
135         int                     *nbytes)
136 {
137         struct xfs_buf_log_item *bip = BUF_ITEM(lip);
138         int                     i;
139 
140         ASSERT(atomic_read(&bip->bli_refcount) > 0);
141         if (bip->bli_flags & XFS_BLI_STALE) {
142                 /*
143                  * The buffer is stale, so all we need to log
144                  * is the buf log format structure with the
145                  * cancel flag in it.
146                  */
147                 trace_xfs_buf_item_size_stale(bip);
148                 ASSERT(bip->__bli_format.blf_flags & XFS_BLF_CANCEL);
149                 *nvecs += bip->bli_format_count;
150                 for (i = 0; i < bip->bli_format_count; i++) {
151                         *nbytes += xfs_buf_log_format_size(&bip->bli_formats[i]);
152                 }
153                 return;
154         }
155 
156         ASSERT(bip->bli_flags & XFS_BLI_LOGGED);
157 
158         if (bip->bli_flags & XFS_BLI_ORDERED) {
159                 /*
160                  * The buffer has been logged just to order it.
161                  * It is not being included in the transaction
162                  * commit, so no vectors are used at all.
163                  */
164                 trace_xfs_buf_item_size_ordered(bip);
165                 *nvecs = XFS_LOG_VEC_ORDERED;
166                 return;
167         }
168 
169         /*
170          * the vector count is based on the number of buffer vectors we have
171          * dirty bits in. This will only be greater than one when we have a
172          * compound buffer with more than one segment dirty. Hence for compound
173          * buffers we need to track which segment the dirty bits correspond to,
174          * and when we move from one segment to the next increment the vector
175          * count for the extra buf log format structure that will need to be
176          * written.
177          */
178         for (i = 0; i < bip->bli_format_count; i++) {
179                 xfs_buf_item_size_segment(bip, &bip->bli_formats[i],
180                                           nvecs, nbytes);
181         }
182         trace_xfs_buf_item_size(bip);
183 }
184 
185 static inline void
186 xfs_buf_item_copy_iovec(
187         struct xfs_log_vec      *lv,
188         struct xfs_log_iovec    **vecp,
189         struct xfs_buf          *bp,
190         uint                    offset,
191         int                     first_bit,
192         uint                    nbits)
193 {
194         offset += first_bit * XFS_BLF_CHUNK;
195         xlog_copy_iovec(lv, vecp, XLOG_REG_TYPE_BCHUNK,
196                         xfs_buf_offset(bp, offset),
197                         nbits * XFS_BLF_CHUNK);
198 }
199 
200 static inline bool
201 xfs_buf_item_straddle(
202         struct xfs_buf          *bp,
203         uint                    offset,
204         int                     next_bit,
205         int                     last_bit)
206 {
207         return xfs_buf_offset(bp, offset + (next_bit << XFS_BLF_SHIFT)) !=
208                 (xfs_buf_offset(bp, offset + (last_bit << XFS_BLF_SHIFT)) +
209                  XFS_BLF_CHUNK);
210 }
211 
212 static void
213 xfs_buf_item_format_segment(
214         struct xfs_buf_log_item *bip,
215         struct xfs_log_vec      *lv,
216         struct xfs_log_iovec    **vecp,
217         uint                    offset,
218         struct xfs_buf_log_format *blfp)
219 {
220         struct xfs_buf  *bp = bip->bli_buf;
221         uint            base_size;
222         int             first_bit;
223         int             last_bit;
224         int             next_bit;
225         uint            nbits;
226 
227         /* copy the flags across from the base format item */
228         blfp->blf_flags = bip->__bli_format.blf_flags;
229 
230         /*
231          * Base size is the actual size of the ondisk structure - it reflects
232          * the actual size of the dirty bitmap rather than the size of the in
233          * memory structure.
234          */
235         base_size = xfs_buf_log_format_size(blfp);
236 
237         first_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size, 0);
238         if (!(bip->bli_flags & XFS_BLI_STALE) && first_bit == -1) {
239                 /*
240                  * If the map is not be dirty in the transaction, mark
241                  * the size as zero and do not advance the vector pointer.
242                  */
243                 return;
244         }
245 
246         blfp = xlog_copy_iovec(lv, vecp, XLOG_REG_TYPE_BFORMAT, blfp, base_size);
247         blfp->blf_size = 1;
248 
249         if (bip->bli_flags & XFS_BLI_STALE) {
250                 /*
251                  * The buffer is stale, so all we need to log
252                  * is the buf log format structure with the
253                  * cancel flag in it.
254                  */
255                 trace_xfs_buf_item_format_stale(bip);
256                 ASSERT(blfp->blf_flags & XFS_BLF_CANCEL);
257                 return;
258         }
259 
260 
261         /*
262          * Fill in an iovec for each set of contiguous chunks.
263          */
264         last_bit = first_bit;
265         nbits = 1;
266         for (;;) {
267                 /*
268                  * This takes the bit number to start looking from and
269                  * returns the next set bit from there.  It returns -1
270                  * if there are no more bits set or the start bit is
271                  * beyond the end of the bitmap.
272                  */
273                 next_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size,
274                                         (uint)last_bit + 1);
275                 /*
276                  * If we run out of bits fill in the last iovec and get out of
277                  * the loop.  Else if we start a new set of bits then fill in
278                  * the iovec for the series we were looking at and start
279                  * counting the bits in the new one.  Else we're still in the
280                  * same set of bits so just keep counting and scanning.
281                  */
282                 if (next_bit == -1) {
283                         xfs_buf_item_copy_iovec(lv, vecp, bp, offset,
284                                                 first_bit, nbits);
285                         blfp->blf_size++;
286                         break;
287                 } else if (next_bit != last_bit + 1 ||
288                            xfs_buf_item_straddle(bp, offset, next_bit, last_bit)) {
289                         xfs_buf_item_copy_iovec(lv, vecp, bp, offset,
290                                                 first_bit, nbits);
291                         blfp->blf_size++;
292                         first_bit = next_bit;
293                         last_bit = next_bit;
294                         nbits = 1;
295                 } else {
296                         last_bit++;
297                         nbits++;
298                 }
299         }
300 }
301 
302 /*
303  * This is called to fill in the vector of log iovecs for the
304  * given log buf item.  It fills the first entry with a buf log
305  * format structure, and the rest point to contiguous chunks
306  * within the buffer.
307  */
308 STATIC void
309 xfs_buf_item_format(
310         struct xfs_log_item     *lip,
311         struct xfs_log_vec      *lv)
312 {
313         struct xfs_buf_log_item *bip = BUF_ITEM(lip);
314         struct xfs_buf          *bp = bip->bli_buf;
315         struct xfs_log_iovec    *vecp = NULL;
316         uint                    offset = 0;
317         int                     i;
318 
319         ASSERT(atomic_read(&bip->bli_refcount) > 0);
320         ASSERT((bip->bli_flags & XFS_BLI_LOGGED) ||
321                (bip->bli_flags & XFS_BLI_STALE));
322         ASSERT((bip->bli_flags & XFS_BLI_STALE) ||
323                (xfs_blft_from_flags(&bip->__bli_format) > XFS_BLFT_UNKNOWN_BUF
324                 && xfs_blft_from_flags(&bip->__bli_format) < XFS_BLFT_MAX_BUF));
325 
326 
327         /*
328          * If it is an inode buffer, transfer the in-memory state to the
329          * format flags and clear the in-memory state.
330          *
331          * For buffer based inode allocation, we do not transfer
332          * this state if the inode buffer allocation has not yet been committed
333          * to the log as setting the XFS_BLI_INODE_BUF flag will prevent
334          * correct replay of the inode allocation.
335          *
336          * For icreate item based inode allocation, the buffers aren't written
337          * to the journal during allocation, and hence we should always tag the
338          * buffer as an inode buffer so that the correct unlinked list replay
339          * occurs during recovery.
340          */
341         if (bip->bli_flags & XFS_BLI_INODE_BUF) {
342                 if (xfs_sb_version_hascrc(&lip->li_mountp->m_sb) ||
343                     !((bip->bli_flags & XFS_BLI_INODE_ALLOC_BUF) &&
344                       xfs_log_item_in_current_chkpt(lip)))
345                         bip->__bli_format.blf_flags |= XFS_BLF_INODE_BUF;
346                 bip->bli_flags &= ~XFS_BLI_INODE_BUF;
347         }
348 
349         if ((bip->bli_flags & (XFS_BLI_ORDERED|XFS_BLI_STALE)) ==
350                                                         XFS_BLI_ORDERED) {
351                 /*
352                  * The buffer has been logged just to order it.  It is not being
353                  * included in the transaction commit, so don't format it.
354                  */
355                 trace_xfs_buf_item_format_ordered(bip);
356                 return;
357         }
358 
359         for (i = 0; i < bip->bli_format_count; i++) {
360                 xfs_buf_item_format_segment(bip, lv, &vecp, offset,
361                                             &bip->bli_formats[i]);
362                 offset += bp->b_maps[i].bm_len;
363         }
364 
365         /*
366          * Check to make sure everything is consistent.
367          */
368         trace_xfs_buf_item_format(bip);
369 }
370 
371 /*
372  * This is called to pin the buffer associated with the buf log item in memory
373  * so it cannot be written out.
374  *
375  * We also always take a reference to the buffer log item here so that the bli
376  * is held while the item is pinned in memory. This means that we can
377  * unconditionally drop the reference count a transaction holds when the
378  * transaction is completed.
379  */
380 STATIC void
381 xfs_buf_item_pin(
382         struct xfs_log_item     *lip)
383 {
384         struct xfs_buf_log_item *bip = BUF_ITEM(lip);
385 
386         ASSERT(atomic_read(&bip->bli_refcount) > 0);
387         ASSERT((bip->bli_flags & XFS_BLI_LOGGED) ||
388                (bip->bli_flags & XFS_BLI_ORDERED) ||
389                (bip->bli_flags & XFS_BLI_STALE));
390 
391         trace_xfs_buf_item_pin(bip);
392 
393         atomic_inc(&bip->bli_refcount);
394         atomic_inc(&bip->bli_buf->b_pin_count);
395 }
396 
397 /*
398  * This is called to unpin the buffer associated with the buf log
399  * item which was previously pinned with a call to xfs_buf_item_pin().
400  *
401  * Also drop the reference to the buf item for the current transaction.
402  * If the XFS_BLI_STALE flag is set and we are the last reference,
403  * then free up the buf log item and unlock the buffer.
404  *
405  * If the remove flag is set we are called from uncommit in the
406  * forced-shutdown path.  If that is true and the reference count on
407  * the log item is going to drop to zero we need to free the item's
408  * descriptor in the transaction.
409  */
410 STATIC void
411 xfs_buf_item_unpin(
412         struct xfs_log_item     *lip,
413         int                     remove)
414 {
415         struct xfs_buf_log_item *bip = BUF_ITEM(lip);
416         xfs_buf_t       *bp = bip->bli_buf;
417         struct xfs_ail  *ailp = lip->li_ailp;
418         int             stale = bip->bli_flags & XFS_BLI_STALE;
419         int             freed;
420 
421         ASSERT(bp->b_fspriv == bip);
422         ASSERT(atomic_read(&bip->bli_refcount) > 0);
423 
424         trace_xfs_buf_item_unpin(bip);
425 
426         freed = atomic_dec_and_test(&bip->bli_refcount);
427 
428         if (atomic_dec_and_test(&bp->b_pin_count))
429                 wake_up_all(&bp->b_waiters);
430 
431         if (freed && stale) {
432                 ASSERT(bip->bli_flags & XFS_BLI_STALE);
433                 ASSERT(xfs_buf_islocked(bp));
434                 ASSERT(XFS_BUF_ISSTALE(bp));
435                 ASSERT(bip->__bli_format.blf_flags & XFS_BLF_CANCEL);
436 
437                 trace_xfs_buf_item_unpin_stale(bip);
438 
439                 if (remove) {
440                         /*
441                          * If we are in a transaction context, we have to
442                          * remove the log item from the transaction as we are
443                          * about to release our reference to the buffer.  If we
444                          * don't, the unlock that occurs later in
445                          * xfs_trans_uncommit() will try to reference the
446                          * buffer which we no longer have a hold on.
447                          */
448                         if (lip->li_desc)
449                                 xfs_trans_del_item(lip);
450 
451                         /*
452                          * Since the transaction no longer refers to the buffer,
453                          * the buffer should no longer refer to the transaction.
454                          */
455                         bp->b_transp = NULL;
456                 }
457 
458                 /*
459                  * If we get called here because of an IO error, we may
460                  * or may not have the item on the AIL. xfs_trans_ail_delete()
461                  * will take care of that situation.
462                  * xfs_trans_ail_delete() drops the AIL lock.
463                  */
464                 if (bip->bli_flags & XFS_BLI_STALE_INODE) {
465                         xfs_buf_do_callbacks(bp);
466                         bp->b_fspriv = NULL;
467                         bp->b_iodone = NULL;
468                 } else {
469                         spin_lock(&ailp->xa_lock);
470                         xfs_trans_ail_delete(ailp, lip, SHUTDOWN_LOG_IO_ERROR);
471                         xfs_buf_item_relse(bp);
472                         ASSERT(bp->b_fspriv == NULL);
473                 }
474                 xfs_buf_relse(bp);
475         } else if (freed && remove) {
476                 /*
477                  * There are currently two references to the buffer - the active
478                  * LRU reference and the buf log item. What we are about to do
479                  * here - simulate a failed IO completion - requires 3
480                  * references.
481                  *
482                  * The LRU reference is removed by the xfs_buf_stale() call. The
483                  * buf item reference is removed by the xfs_buf_iodone()
484                  * callback that is run by xfs_buf_do_callbacks() during ioend
485                  * processing (via the bp->b_iodone callback), and then finally
486                  * the ioend processing will drop the IO reference if the buffer
487                  * is marked XBF_ASYNC.
488                  *
489                  * Hence we need to take an additional reference here so that IO
490                  * completion processing doesn't free the buffer prematurely.
491                  */
492                 xfs_buf_lock(bp);
493                 xfs_buf_hold(bp);
494                 bp->b_flags |= XBF_ASYNC;
495                 xfs_buf_ioerror(bp, -EIO);
496                 XFS_BUF_UNDONE(bp);
497                 xfs_buf_stale(bp);
498                 xfs_buf_ioend(bp);
499         }
500 }
501 
502 /*
503  * Buffer IO error rate limiting. Limit it to no more than 10 messages per 30
504  * seconds so as to not spam logs too much on repeated detection of the same
505  * buffer being bad..
506  */
507 
508 static DEFINE_RATELIMIT_STATE(xfs_buf_write_fail_rl_state, 30 * HZ, 10);
509 
510 STATIC uint
511 xfs_buf_item_push(
512         struct xfs_log_item     *lip,
513         struct list_head        *buffer_list)
514 {
515         struct xfs_buf_log_item *bip = BUF_ITEM(lip);
516         struct xfs_buf          *bp = bip->bli_buf;
517         uint                    rval = XFS_ITEM_SUCCESS;
518 
519         if (xfs_buf_ispinned(bp))
520                 return XFS_ITEM_PINNED;
521         if (!xfs_buf_trylock(bp)) {
522                 /*
523                  * If we have just raced with a buffer being pinned and it has
524                  * been marked stale, we could end up stalling until someone else
525                  * issues a log force to unpin the stale buffer. Check for the
526                  * race condition here so xfsaild recognizes the buffer is pinned
527                  * and queues a log force to move it along.
528                  */
529                 if (xfs_buf_ispinned(bp))
530                         return XFS_ITEM_PINNED;
531                 return XFS_ITEM_LOCKED;
532         }
533 
534         ASSERT(!(bip->bli_flags & XFS_BLI_STALE));
535 
536         trace_xfs_buf_item_push(bip);
537 
538         /* has a previous flush failed due to IO errors? */
539         if ((bp->b_flags & XBF_WRITE_FAIL) &&
540             ___ratelimit(&xfs_buf_write_fail_rl_state, "XFS:")) {
541                 xfs_warn(bp->b_target->bt_mount,
542 "Detected failing async write on buffer block 0x%llx. Retrying async write.",
543                          (long long)bp->b_bn);
544         }
545 
546         if (!xfs_buf_delwri_queue(bp, buffer_list))
547                 rval = XFS_ITEM_FLUSHING;
548         xfs_buf_unlock(bp);
549         return rval;
550 }
551 
552 /*
553  * Release the buffer associated with the buf log item.  If there is no dirty
554  * logged data associated with the buffer recorded in the buf log item, then
555  * free the buf log item and remove the reference to it in the buffer.
556  *
557  * This call ignores the recursion count.  It is only called when the buffer
558  * should REALLY be unlocked, regardless of the recursion count.
559  *
560  * We unconditionally drop the transaction's reference to the log item. If the
561  * item was logged, then another reference was taken when it was pinned, so we
562  * can safely drop the transaction reference now.  This also allows us to avoid
563  * potential races with the unpin code freeing the bli by not referencing the
564  * bli after we've dropped the reference count.
565  *
566  * If the XFS_BLI_HOLD flag is set in the buf log item, then free the log item
567  * if necessary but do not unlock the buffer.  This is for support of
568  * xfs_trans_bhold(). Make sure the XFS_BLI_HOLD field is cleared if we don't
569  * free the item.
570  */
571 STATIC void
572 xfs_buf_item_unlock(
573         struct xfs_log_item     *lip)
574 {
575         struct xfs_buf_log_item *bip = BUF_ITEM(lip);
576         struct xfs_buf          *bp = bip->bli_buf;
577         bool                    clean;
578         bool                    aborted;
579         int                     flags;
580 
581         /* Clear the buffer's association with this transaction. */
582         bp->b_transp = NULL;
583 
584         /*
585          * If this is a transaction abort, don't return early.  Instead, allow
586          * the brelse to happen.  Normally it would be done for stale
587          * (cancelled) buffers at unpin time, but we'll never go through the
588          * pin/unpin cycle if we abort inside commit.
589          */
590         aborted = (lip->li_flags & XFS_LI_ABORTED) ? true : false;
591         /*
592          * Before possibly freeing the buf item, copy the per-transaction state
593          * so we can reference it safely later after clearing it from the
594          * buffer log item.
595          */
596         flags = bip->bli_flags;
597         bip->bli_flags &= ~(XFS_BLI_LOGGED | XFS_BLI_HOLD | XFS_BLI_ORDERED);
598 
599         /*
600          * If the buf item is marked stale, then don't do anything.  We'll
601          * unlock the buffer and free the buf item when the buffer is unpinned
602          * for the last time.
603          */
604         if (flags & XFS_BLI_STALE) {
605                 trace_xfs_buf_item_unlock_stale(bip);
606                 ASSERT(bip->__bli_format.blf_flags & XFS_BLF_CANCEL);
607                 if (!aborted) {
608                         atomic_dec(&bip->bli_refcount);
609                         return;
610                 }
611         }
612 
613         trace_xfs_buf_item_unlock(bip);
614 
615         /*
616          * If the buf item isn't tracking any data, free it, otherwise drop the
617          * reference we hold to it. If we are aborting the transaction, this may
618          * be the only reference to the buf item, so we free it anyway
619          * regardless of whether it is dirty or not. A dirty abort implies a
620          * shutdown, anyway.
621          *
622          * Ordered buffers are dirty but may have no recorded changes, so ensure
623          * we only release clean items here.
624          */
625         clean = (flags & XFS_BLI_DIRTY) ? false : true;
626         if (clean) {
627                 int i;
628                 for (i = 0; i < bip->bli_format_count; i++) {
629                         if (!xfs_bitmap_empty(bip->bli_formats[i].blf_data_map,
630                                      bip->bli_formats[i].blf_map_size)) {
631                                 clean = false;
632                                 break;
633                         }
634                 }
635         }
636 
637         /*
638          * Clean buffers, by definition, cannot be in the AIL. However, aborted
639          * buffers may be dirty and hence in the AIL. Therefore if we are
640          * aborting a buffer and we've just taken the last refernce away, we
641          * have to check if it is in the AIL before freeing it. We need to free
642          * it in this case, because an aborted transaction has already shut the
643          * filesystem down and this is the last chance we will have to do so.
644          */
645         if (atomic_dec_and_test(&bip->bli_refcount)) {
646                 if (clean)
647                         xfs_buf_item_relse(bp);
648                 else if (aborted) {
649                         ASSERT(XFS_FORCED_SHUTDOWN(lip->li_mountp));
650                         if (lip->li_flags & XFS_LI_IN_AIL) {
651                                 spin_lock(&lip->li_ailp->xa_lock);
652                                 xfs_trans_ail_delete(lip->li_ailp, lip,
653                                                      SHUTDOWN_LOG_IO_ERROR);
654                         }
655                         xfs_buf_item_relse(bp);
656                 }
657         }
658 
659         if (!(flags & XFS_BLI_HOLD))
660                 xfs_buf_relse(bp);
661 }
662 
663 /*
664  * This is called to find out where the oldest active copy of the
665  * buf log item in the on disk log resides now that the last log
666  * write of it completed at the given lsn.
667  * We always re-log all the dirty data in a buffer, so usually the
668  * latest copy in the on disk log is the only one that matters.  For
669  * those cases we simply return the given lsn.
670  *
671  * The one exception to this is for buffers full of newly allocated
672  * inodes.  These buffers are only relogged with the XFS_BLI_INODE_BUF
673  * flag set, indicating that only the di_next_unlinked fields from the
674  * inodes in the buffers will be replayed during recovery.  If the
675  * original newly allocated inode images have not yet been flushed
676  * when the buffer is so relogged, then we need to make sure that we
677  * keep the old images in the 'active' portion of the log.  We do this
678  * by returning the original lsn of that transaction here rather than
679  * the current one.
680  */
681 STATIC xfs_lsn_t
682 xfs_buf_item_committed(
683         struct xfs_log_item     *lip,
684         xfs_lsn_t               lsn)
685 {
686         struct xfs_buf_log_item *bip = BUF_ITEM(lip);
687 
688         trace_xfs_buf_item_committed(bip);
689 
690         if ((bip->bli_flags & XFS_BLI_INODE_ALLOC_BUF) && lip->li_lsn != 0)
691                 return lip->li_lsn;
692         return lsn;
693 }
694 
695 STATIC void
696 xfs_buf_item_committing(
697         struct xfs_log_item     *lip,
698         xfs_lsn_t               commit_lsn)
699 {
700 }
701 
702 /*
703  * This is the ops vector shared by all buf log items.
704  */
705 static const struct xfs_item_ops xfs_buf_item_ops = {
706         .iop_size       = xfs_buf_item_size,
707         .iop_format     = xfs_buf_item_format,
708         .iop_pin        = xfs_buf_item_pin,
709         .iop_unpin      = xfs_buf_item_unpin,
710         .iop_unlock     = xfs_buf_item_unlock,
711         .iop_committed  = xfs_buf_item_committed,
712         .iop_push       = xfs_buf_item_push,
713         .iop_committing = xfs_buf_item_committing
714 };
715 
716 STATIC int
717 xfs_buf_item_get_format(
718         struct xfs_buf_log_item *bip,
719         int                     count)
720 {
721         ASSERT(bip->bli_formats == NULL);
722         bip->bli_format_count = count;
723 
724         if (count == 1) {
725                 bip->bli_formats = &bip->__bli_format;
726                 return 0;
727         }
728 
729         bip->bli_formats = kmem_zalloc(count * sizeof(struct xfs_buf_log_format),
730                                 KM_SLEEP);
731         if (!bip->bli_formats)
732                 return -ENOMEM;
733         return 0;
734 }
735 
736 STATIC void
737 xfs_buf_item_free_format(
738         struct xfs_buf_log_item *bip)
739 {
740         if (bip->bli_formats != &bip->__bli_format) {
741                 kmem_free(bip->bli_formats);
742                 bip->bli_formats = NULL;
743         }
744 }
745 
746 /*
747  * Allocate a new buf log item to go with the given buffer.
748  * Set the buffer's b_fsprivate field to point to the new
749  * buf log item.  If there are other item's attached to the
750  * buffer (see xfs_buf_attach_iodone() below), then put the
751  * buf log item at the front.
752  */
753 void
754 xfs_buf_item_init(
755         xfs_buf_t       *bp,
756         xfs_mount_t     *mp)
757 {
758         xfs_log_item_t          *lip = bp->b_fspriv;
759         xfs_buf_log_item_t      *bip;
760         int                     chunks;
761         int                     map_size;
762         int                     error;
763         int                     i;
764 
765         /*
766          * Check to see if there is already a buf log item for
767          * this buffer.  If there is, it is guaranteed to be
768          * the first.  If we do already have one, there is
769          * nothing to do here so return.
770          */
771         ASSERT(bp->b_target->bt_mount == mp);
772         if (lip != NULL && lip->li_type == XFS_LI_BUF)
773                 return;
774 
775         bip = kmem_zone_zalloc(xfs_buf_item_zone, KM_SLEEP);
776         xfs_log_item_init(mp, &bip->bli_item, XFS_LI_BUF, &xfs_buf_item_ops);
777         bip->bli_buf = bp;
778         xfs_buf_hold(bp);
779 
780         /*
781          * chunks is the number of XFS_BLF_CHUNK size pieces the buffer
782          * can be divided into. Make sure not to truncate any pieces.
783          * map_size is the size of the bitmap needed to describe the
784          * chunks of the buffer.
785          *
786          * Discontiguous buffer support follows the layout of the underlying
787          * buffer. This makes the implementation as simple as possible.
788          */
789         error = xfs_buf_item_get_format(bip, bp->b_map_count);
790         ASSERT(error == 0);
791 
792         for (i = 0; i < bip->bli_format_count; i++) {
793                 chunks = DIV_ROUND_UP(BBTOB(bp->b_maps[i].bm_len),
794                                       XFS_BLF_CHUNK);
795                 map_size = DIV_ROUND_UP(chunks, NBWORD);
796 
797                 bip->bli_formats[i].blf_type = XFS_LI_BUF;
798                 bip->bli_formats[i].blf_blkno = bp->b_maps[i].bm_bn;
799                 bip->bli_formats[i].blf_len = bp->b_maps[i].bm_len;
800                 bip->bli_formats[i].blf_map_size = map_size;
801         }
802 
803         /*
804          * Put the buf item into the list of items attached to the
805          * buffer at the front.
806          */
807         if (bp->b_fspriv)
808                 bip->bli_item.li_bio_list = bp->b_fspriv;
809         bp->b_fspriv = bip;
810 }
811 
812 
813 /*
814  * Mark bytes first through last inclusive as dirty in the buf
815  * item's bitmap.
816  */
817 static void
818 xfs_buf_item_log_segment(
819         uint                    first,
820         uint                    last,
821         uint                    *map)
822 {
823         uint            first_bit;
824         uint            last_bit;
825         uint            bits_to_set;
826         uint            bits_set;
827         uint            word_num;
828         uint            *wordp;
829         uint            bit;
830         uint            end_bit;
831         uint            mask;
832 
833         /*
834          * Convert byte offsets to bit numbers.
835          */
836         first_bit = first >> XFS_BLF_SHIFT;
837         last_bit = last >> XFS_BLF_SHIFT;
838 
839         /*
840          * Calculate the total number of bits to be set.
841          */
842         bits_to_set = last_bit - first_bit + 1;
843 
844         /*
845          * Get a pointer to the first word in the bitmap
846          * to set a bit in.
847          */
848         word_num = first_bit >> BIT_TO_WORD_SHIFT;
849         wordp = &map[word_num];
850 
851         /*
852          * Calculate the starting bit in the first word.
853          */
854         bit = first_bit & (uint)(NBWORD - 1);
855 
856         /*
857          * First set any bits in the first word of our range.
858          * If it starts at bit 0 of the word, it will be
859          * set below rather than here.  That is what the variable
860          * bit tells us. The variable bits_set tracks the number
861          * of bits that have been set so far.  End_bit is the number
862          * of the last bit to be set in this word plus one.
863          */
864         if (bit) {
865                 end_bit = MIN(bit + bits_to_set, (uint)NBWORD);
866                 mask = ((1 << (end_bit - bit)) - 1) << bit;
867                 *wordp |= mask;
868                 wordp++;
869                 bits_set = end_bit - bit;
870         } else {
871                 bits_set = 0;
872         }
873 
874         /*
875          * Now set bits a whole word at a time that are between
876          * first_bit and last_bit.
877          */
878         while ((bits_to_set - bits_set) >= NBWORD) {
879                 *wordp |= 0xffffffff;
880                 bits_set += NBWORD;
881                 wordp++;
882         }
883 
884         /*
885          * Finally, set any bits left to be set in one last partial word.
886          */
887         end_bit = bits_to_set - bits_set;
888         if (end_bit) {
889                 mask = (1 << end_bit) - 1;
890                 *wordp |= mask;
891         }
892 }
893 
894 /*
895  * Mark bytes first through last inclusive as dirty in the buf
896  * item's bitmap.
897  */
898 void
899 xfs_buf_item_log(
900         xfs_buf_log_item_t      *bip,
901         uint                    first,
902         uint                    last)
903 {
904         int                     i;
905         uint                    start;
906         uint                    end;
907         struct xfs_buf          *bp = bip->bli_buf;
908 
909         /*
910          * walk each buffer segment and mark them dirty appropriately.
911          */
912         start = 0;
913         for (i = 0; i < bip->bli_format_count; i++) {
914                 if (start > last)
915                         break;
916                 end = start + BBTOB(bp->b_maps[i].bm_len);
917                 if (first > end) {
918                         start += BBTOB(bp->b_maps[i].bm_len);
919                         continue;
920                 }
921                 if (first < start)
922                         first = start;
923                 if (end > last)
924                         end = last;
925 
926                 xfs_buf_item_log_segment(first, end,
927                                          &bip->bli_formats[i].blf_data_map[0]);
928 
929                 start += bp->b_maps[i].bm_len;
930         }
931 }
932 
933 
934 /*
935  * Return 1 if the buffer has been logged or ordered in a transaction (at any
936  * point, not just the current transaction) and 0 if not.
937  */
938 uint
939 xfs_buf_item_dirty(
940         xfs_buf_log_item_t      *bip)
941 {
942         return (bip->bli_flags & XFS_BLI_DIRTY);
943 }
944 
945 STATIC void
946 xfs_buf_item_free(
947         xfs_buf_log_item_t      *bip)
948 {
949         xfs_buf_item_free_format(bip);
950         kmem_zone_free(xfs_buf_item_zone, bip);
951 }
952 
953 /*
954  * This is called when the buf log item is no longer needed.  It should
955  * free the buf log item associated with the given buffer and clear
956  * the buffer's pointer to the buf log item.  If there are no more
957  * items in the list, clear the b_iodone field of the buffer (see
958  * xfs_buf_attach_iodone() below).
959  */
960 void
961 xfs_buf_item_relse(
962         xfs_buf_t       *bp)
963 {
964         xfs_buf_log_item_t      *bip = bp->b_fspriv;
965 
966         trace_xfs_buf_item_relse(bp, _RET_IP_);
967         ASSERT(!(bip->bli_item.li_flags & XFS_LI_IN_AIL));
968 
969         bp->b_fspriv = bip->bli_item.li_bio_list;
970         if (bp->b_fspriv == NULL)
971                 bp->b_iodone = NULL;
972 
973         xfs_buf_rele(bp);
974         xfs_buf_item_free(bip);
975 }
976 
977 
978 /*
979  * Add the given log item with its callback to the list of callbacks
980  * to be called when the buffer's I/O completes.  If it is not set
981  * already, set the buffer's b_iodone() routine to be
982  * xfs_buf_iodone_callbacks() and link the log item into the list of
983  * items rooted at b_fsprivate.  Items are always added as the second
984  * entry in the list if there is a first, because the buf item code
985  * assumes that the buf log item is first.
986  */
987 void
988 xfs_buf_attach_iodone(
989         xfs_buf_t       *bp,
990         void            (*cb)(xfs_buf_t *, xfs_log_item_t *),
991         xfs_log_item_t  *lip)
992 {
993         xfs_log_item_t  *head_lip;
994 
995         ASSERT(xfs_buf_islocked(bp));
996 
997         lip->li_cb = cb;
998         head_lip = bp->b_fspriv;
999         if (head_lip) {
1000                 lip->li_bio_list = head_lip->li_bio_list;
1001                 head_lip->li_bio_list = lip;
1002         } else {
1003                 bp->b_fspriv = lip;
1004         }
1005 
1006         ASSERT(bp->b_iodone == NULL ||
1007                bp->b_iodone == xfs_buf_iodone_callbacks);
1008         bp->b_iodone = xfs_buf_iodone_callbacks;
1009 }
1010 
1011 /*
1012  * We can have many callbacks on a buffer. Running the callbacks individually
1013  * can cause a lot of contention on the AIL lock, so we allow for a single
1014  * callback to be able to scan the remaining lip->li_bio_list for other items
1015  * of the same type and callback to be processed in the first call.
1016  *
1017  * As a result, the loop walking the callback list below will also modify the
1018  * list. it removes the first item from the list and then runs the callback.
1019  * The loop then restarts from the new head of the list. This allows the
1020  * callback to scan and modify the list attached to the buffer and we don't
1021  * have to care about maintaining a next item pointer.
1022  */
1023 STATIC void
1024 xfs_buf_do_callbacks(
1025         struct xfs_buf          *bp)
1026 {
1027         struct xfs_log_item     *lip;
1028 
1029         while ((lip = bp->b_fspriv) != NULL) {
1030                 bp->b_fspriv = lip->li_bio_list;
1031                 ASSERT(lip->li_cb != NULL);
1032                 /*
1033                  * Clear the next pointer so we don't have any
1034                  * confusion if the item is added to another buf.
1035                  * Don't touch the log item after calling its
1036                  * callback, because it could have freed itself.
1037                  */
1038                 lip->li_bio_list = NULL;
1039                 lip->li_cb(bp, lip);
1040         }
1041 }
1042 
1043 /*
1044  * This is the iodone() function for buffers which have had callbacks
1045  * attached to them by xfs_buf_attach_iodone().  It should remove each
1046  * log item from the buffer's list and call the callback of each in turn.
1047  * When done, the buffer's fsprivate field is set to NULL and the buffer
1048  * is unlocked with a call to iodone().
1049  */
1050 void
1051 xfs_buf_iodone_callbacks(
1052         struct xfs_buf          *bp)
1053 {
1054         struct xfs_log_item     *lip = bp->b_fspriv;
1055         struct xfs_mount        *mp = lip->li_mountp;
1056         static ulong            lasttime;
1057         static xfs_buftarg_t    *lasttarg;
1058 
1059         if (likely(!bp->b_error))
1060                 goto do_callbacks;
1061 
1062         /*
1063          * If we've already decided to shutdown the filesystem because of
1064          * I/O errors, there's no point in giving this a retry.
1065          */
1066         if (XFS_FORCED_SHUTDOWN(mp)) {
1067                 xfs_buf_stale(bp);
1068                 XFS_BUF_DONE(bp);
1069                 trace_xfs_buf_item_iodone(bp, _RET_IP_);
1070                 goto do_callbacks;
1071         }
1072 
1073         if (bp->b_target != lasttarg ||
1074             time_after(jiffies, (lasttime + 5*HZ))) {
1075                 lasttime = jiffies;
1076                 xfs_buf_ioerror_alert(bp, __func__);
1077         }
1078         lasttarg = bp->b_target;
1079 
1080         /*
1081          * If the write was asynchronous then no one will be looking for the
1082          * error.  Clear the error state and write the buffer out again.
1083          *
1084          * XXX: This helps against transient write errors, but we need to find
1085          * a way to shut the filesystem down if the writes keep failing.
1086          *
1087          * In practice we'll shut the filesystem down soon as non-transient
1088          * errors tend to affect the whole device and a failing log write
1089          * will make us give up.  But we really ought to do better here.
1090          */
1091         if (XFS_BUF_ISASYNC(bp)) {
1092                 ASSERT(bp->b_iodone != NULL);
1093 
1094                 trace_xfs_buf_item_iodone_async(bp, _RET_IP_);
1095 
1096                 xfs_buf_ioerror(bp, 0); /* errno of 0 unsets the flag */
1097 
1098                 if (!(bp->b_flags & (XBF_STALE|XBF_WRITE_FAIL))) {
1099                         bp->b_flags |= XBF_WRITE | XBF_ASYNC |
1100                                        XBF_DONE | XBF_WRITE_FAIL;
1101                         xfs_buf_submit(bp);
1102                 } else {
1103                         xfs_buf_relse(bp);
1104                 }
1105 
1106                 return;
1107         }
1108 
1109         /*
1110          * If the write of the buffer was synchronous, we want to make
1111          * sure to return the error to the caller of xfs_bwrite().
1112          */
1113         xfs_buf_stale(bp);
1114         XFS_BUF_DONE(bp);
1115 
1116         trace_xfs_buf_error_relse(bp, _RET_IP_);
1117 
1118 do_callbacks:
1119         xfs_buf_do_callbacks(bp);
1120         bp->b_fspriv = NULL;
1121         bp->b_iodone = NULL;
1122         xfs_buf_ioend(bp);
1123 }
1124 
1125 /*
1126  * This is the iodone() function for buffers which have been
1127  * logged.  It is called when they are eventually flushed out.
1128  * It should remove the buf item from the AIL, and free the buf item.
1129  * It is called by xfs_buf_iodone_callbacks() above which will take
1130  * care of cleaning up the buffer itself.
1131  */
1132 void
1133 xfs_buf_iodone(
1134         struct xfs_buf          *bp,
1135         struct xfs_log_item     *lip)
1136 {
1137         struct xfs_ail          *ailp = lip->li_ailp;
1138 
1139         ASSERT(BUF_ITEM(lip)->bli_buf == bp);
1140 
1141         xfs_buf_rele(bp);
1142 
1143         /*
1144          * If we are forcibly shutting down, this may well be
1145          * off the AIL already. That's because we simulate the
1146          * log-committed callbacks to unpin these buffers. Or we may never
1147          * have put this item on AIL because of the transaction was
1148          * aborted forcibly. xfs_trans_ail_delete() takes care of these.
1149          *
1150          * Either way, AIL is useless if we're forcing a shutdown.
1151          */
1152         spin_lock(&ailp->xa_lock);
1153         xfs_trans_ail_delete(ailp, lip, SHUTDOWN_CORRUPT_INCORE);
1154         xfs_buf_item_free(BUF_ITEM(lip));
1155 }
1156 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp