~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/fs/xfs/xfs_icache.c

Version: ~ [ linux-5.12-rc7 ] ~ [ linux-5.11.13 ] ~ [ linux-5.10.29 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.111 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.186 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.230 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.266 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.266 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Copyright (c) 2000-2005 Silicon Graphics, Inc.
  3  * All Rights Reserved.
  4  *
  5  * This program is free software; you can redistribute it and/or
  6  * modify it under the terms of the GNU General Public License as
  7  * published by the Free Software Foundation.
  8  *
  9  * This program is distributed in the hope that it would be useful,
 10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 12  * GNU General Public License for more details.
 13  *
 14  * You should have received a copy of the GNU General Public License
 15  * along with this program; if not, write the Free Software Foundation,
 16  * Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 17  */
 18 #include "xfs.h"
 19 #include "xfs_fs.h"
 20 #include "xfs_format.h"
 21 #include "xfs_log_format.h"
 22 #include "xfs_trans_resv.h"
 23 #include "xfs_sb.h"
 24 #include "xfs_mount.h"
 25 #include "xfs_inode.h"
 26 #include "xfs_error.h"
 27 #include "xfs_trans.h"
 28 #include "xfs_trans_priv.h"
 29 #include "xfs_inode_item.h"
 30 #include "xfs_quota.h"
 31 #include "xfs_trace.h"
 32 #include "xfs_icache.h"
 33 #include "xfs_bmap_util.h"
 34 #include "xfs_dquot_item.h"
 35 #include "xfs_dquot.h"
 36 #include "xfs_reflink.h"
 37 
 38 #include <linux/kthread.h>
 39 #include <linux/freezer.h>
 40 #include <linux/iversion.h>
 41 
 42 /*
 43  * Allocate and initialise an xfs_inode.
 44  */
 45 struct xfs_inode *
 46 xfs_inode_alloc(
 47         struct xfs_mount        *mp,
 48         xfs_ino_t               ino)
 49 {
 50         struct xfs_inode        *ip;
 51 
 52         /*
 53          * if this didn't occur in transactions, we could use
 54          * KM_MAYFAIL and return NULL here on ENOMEM. Set the
 55          * code up to do this anyway.
 56          */
 57         ip = kmem_zone_alloc(xfs_inode_zone, KM_SLEEP);
 58         if (!ip)
 59                 return NULL;
 60         if (inode_init_always(mp->m_super, VFS_I(ip))) {
 61                 kmem_zone_free(xfs_inode_zone, ip);
 62                 return NULL;
 63         }
 64 
 65         /* VFS doesn't initialise i_mode! */
 66         VFS_I(ip)->i_mode = 0;
 67 
 68         XFS_STATS_INC(mp, vn_active);
 69         ASSERT(atomic_read(&ip->i_pincount) == 0);
 70         ASSERT(!xfs_isiflocked(ip));
 71         ASSERT(ip->i_ino == 0);
 72 
 73         /* initialise the xfs inode */
 74         ip->i_ino = ino;
 75         ip->i_mount = mp;
 76         memset(&ip->i_imap, 0, sizeof(struct xfs_imap));
 77         ip->i_afp = NULL;
 78         ip->i_cowfp = NULL;
 79         ip->i_cnextents = 0;
 80         ip->i_cformat = XFS_DINODE_FMT_EXTENTS;
 81         memset(&ip->i_df, 0, sizeof(xfs_ifork_t));
 82         ip->i_flags = 0;
 83         ip->i_delayed_blks = 0;
 84         memset(&ip->i_d, 0, sizeof(ip->i_d));
 85 
 86         return ip;
 87 }
 88 
 89 STATIC void
 90 xfs_inode_free_callback(
 91         struct rcu_head         *head)
 92 {
 93         struct inode            *inode = container_of(head, struct inode, i_rcu);
 94         struct xfs_inode        *ip = XFS_I(inode);
 95 
 96         switch (VFS_I(ip)->i_mode & S_IFMT) {
 97         case S_IFREG:
 98         case S_IFDIR:
 99         case S_IFLNK:
100                 xfs_idestroy_fork(ip, XFS_DATA_FORK);
101                 break;
102         }
103 
104         if (ip->i_afp)
105                 xfs_idestroy_fork(ip, XFS_ATTR_FORK);
106         if (ip->i_cowfp)
107                 xfs_idestroy_fork(ip, XFS_COW_FORK);
108 
109         if (ip->i_itemp) {
110                 ASSERT(!(ip->i_itemp->ili_item.li_flags & XFS_LI_IN_AIL));
111                 xfs_inode_item_destroy(ip);
112                 ip->i_itemp = NULL;
113         }
114 
115         kmem_zone_free(xfs_inode_zone, ip);
116 }
117 
118 static void
119 __xfs_inode_free(
120         struct xfs_inode        *ip)
121 {
122         /* asserts to verify all state is correct here */
123         ASSERT(atomic_read(&ip->i_pincount) == 0);
124         XFS_STATS_DEC(ip->i_mount, vn_active);
125 
126         call_rcu(&VFS_I(ip)->i_rcu, xfs_inode_free_callback);
127 }
128 
129 void
130 xfs_inode_free(
131         struct xfs_inode        *ip)
132 {
133         ASSERT(!xfs_isiflocked(ip));
134 
135         /*
136          * Because we use RCU freeing we need to ensure the inode always
137          * appears to be reclaimed with an invalid inode number when in the
138          * free state. The ip->i_flags_lock provides the barrier against lookup
139          * races.
140          */
141         spin_lock(&ip->i_flags_lock);
142         ip->i_flags = XFS_IRECLAIM;
143         ip->i_ino = 0;
144         spin_unlock(&ip->i_flags_lock);
145 
146         __xfs_inode_free(ip);
147 }
148 
149 /*
150  * Queue a new inode reclaim pass if there are reclaimable inodes and there
151  * isn't a reclaim pass already in progress. By default it runs every 5s based
152  * on the xfs periodic sync default of 30s. Perhaps this should have it's own
153  * tunable, but that can be done if this method proves to be ineffective or too
154  * aggressive.
155  */
156 static void
157 xfs_reclaim_work_queue(
158         struct xfs_mount        *mp)
159 {
160 
161         rcu_read_lock();
162         if (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_RECLAIM_TAG)) {
163                 queue_delayed_work(mp->m_reclaim_workqueue, &mp->m_reclaim_work,
164                         msecs_to_jiffies(xfs_syncd_centisecs / 6 * 10));
165         }
166         rcu_read_unlock();
167 }
168 
169 /*
170  * This is a fast pass over the inode cache to try to get reclaim moving on as
171  * many inodes as possible in a short period of time. It kicks itself every few
172  * seconds, as well as being kicked by the inode cache shrinker when memory
173  * goes low. It scans as quickly as possible avoiding locked inodes or those
174  * already being flushed, and once done schedules a future pass.
175  */
176 void
177 xfs_reclaim_worker(
178         struct work_struct *work)
179 {
180         struct xfs_mount *mp = container_of(to_delayed_work(work),
181                                         struct xfs_mount, m_reclaim_work);
182 
183         xfs_reclaim_inodes(mp, SYNC_TRYLOCK);
184         xfs_reclaim_work_queue(mp);
185 }
186 
187 static void
188 xfs_perag_set_reclaim_tag(
189         struct xfs_perag        *pag)
190 {
191         struct xfs_mount        *mp = pag->pag_mount;
192 
193         lockdep_assert_held(&pag->pag_ici_lock);
194         if (pag->pag_ici_reclaimable++)
195                 return;
196 
197         /* propagate the reclaim tag up into the perag radix tree */
198         spin_lock(&mp->m_perag_lock);
199         radix_tree_tag_set(&mp->m_perag_tree, pag->pag_agno,
200                            XFS_ICI_RECLAIM_TAG);
201         spin_unlock(&mp->m_perag_lock);
202 
203         /* schedule periodic background inode reclaim */
204         xfs_reclaim_work_queue(mp);
205 
206         trace_xfs_perag_set_reclaim(mp, pag->pag_agno, -1, _RET_IP_);
207 }
208 
209 static void
210 xfs_perag_clear_reclaim_tag(
211         struct xfs_perag        *pag)
212 {
213         struct xfs_mount        *mp = pag->pag_mount;
214 
215         lockdep_assert_held(&pag->pag_ici_lock);
216         if (--pag->pag_ici_reclaimable)
217                 return;
218 
219         /* clear the reclaim tag from the perag radix tree */
220         spin_lock(&mp->m_perag_lock);
221         radix_tree_tag_clear(&mp->m_perag_tree, pag->pag_agno,
222                              XFS_ICI_RECLAIM_TAG);
223         spin_unlock(&mp->m_perag_lock);
224         trace_xfs_perag_clear_reclaim(mp, pag->pag_agno, -1, _RET_IP_);
225 }
226 
227 
228 /*
229  * We set the inode flag atomically with the radix tree tag.
230  * Once we get tag lookups on the radix tree, this inode flag
231  * can go away.
232  */
233 void
234 xfs_inode_set_reclaim_tag(
235         struct xfs_inode        *ip)
236 {
237         struct xfs_mount        *mp = ip->i_mount;
238         struct xfs_perag        *pag;
239 
240         pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino));
241         spin_lock(&pag->pag_ici_lock);
242         spin_lock(&ip->i_flags_lock);
243 
244         radix_tree_tag_set(&pag->pag_ici_root, XFS_INO_TO_AGINO(mp, ip->i_ino),
245                            XFS_ICI_RECLAIM_TAG);
246         xfs_perag_set_reclaim_tag(pag);
247         __xfs_iflags_set(ip, XFS_IRECLAIMABLE);
248 
249         spin_unlock(&ip->i_flags_lock);
250         spin_unlock(&pag->pag_ici_lock);
251         xfs_perag_put(pag);
252 }
253 
254 STATIC void
255 xfs_inode_clear_reclaim_tag(
256         struct xfs_perag        *pag,
257         xfs_ino_t               ino)
258 {
259         radix_tree_tag_clear(&pag->pag_ici_root,
260                              XFS_INO_TO_AGINO(pag->pag_mount, ino),
261                              XFS_ICI_RECLAIM_TAG);
262         xfs_perag_clear_reclaim_tag(pag);
263 }
264 
265 static void
266 xfs_inew_wait(
267         struct xfs_inode        *ip)
268 {
269         wait_queue_head_t *wq = bit_waitqueue(&ip->i_flags, __XFS_INEW_BIT);
270         DEFINE_WAIT_BIT(wait, &ip->i_flags, __XFS_INEW_BIT);
271 
272         do {
273                 prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE);
274                 if (!xfs_iflags_test(ip, XFS_INEW))
275                         break;
276                 schedule();
277         } while (true);
278         finish_wait(wq, &wait.wq_entry);
279 }
280 
281 /*
282  * When we recycle a reclaimable inode, we need to re-initialise the VFS inode
283  * part of the structure. This is made more complex by the fact we store
284  * information about the on-disk values in the VFS inode and so we can't just
285  * overwrite the values unconditionally. Hence we save the parameters we
286  * need to retain across reinitialisation, and rewrite them into the VFS inode
287  * after reinitialisation even if it fails.
288  */
289 static int
290 xfs_reinit_inode(
291         struct xfs_mount        *mp,
292         struct inode            *inode)
293 {
294         int             error;
295         uint32_t        nlink = inode->i_nlink;
296         uint32_t        generation = inode->i_generation;
297         uint64_t        version = inode_peek_iversion(inode);
298         umode_t         mode = inode->i_mode;
299         dev_t           dev = inode->i_rdev;
300 
301         error = inode_init_always(mp->m_super, inode);
302 
303         set_nlink(inode, nlink);
304         inode->i_generation = generation;
305         inode_set_iversion_queried(inode, version);
306         inode->i_mode = mode;
307         inode->i_rdev = dev;
308         return error;
309 }
310 
311 /*
312  * If we are allocating a new inode, then check what was returned is
313  * actually a free, empty inode. If we are not allocating an inode,
314  * then check we didn't find a free inode.
315  *
316  * Returns:
317  *      0               if the inode free state matches the lookup context
318  *      -ENOENT         if the inode is free and we are not allocating
319  *      -EFSCORRUPTED   if there is any state mismatch at all
320  */
321 static int
322 xfs_iget_check_free_state(
323         struct xfs_inode        *ip,
324         int                     flags)
325 {
326         if (flags & XFS_IGET_CREATE) {
327                 /* should be a free inode */
328                 if (VFS_I(ip)->i_mode != 0) {
329                         xfs_warn(ip->i_mount,
330 "Corruption detected! Free inode 0x%llx not marked free! (mode 0x%x)",
331                                 ip->i_ino, VFS_I(ip)->i_mode);
332                         return -EFSCORRUPTED;
333                 }
334 
335                 if (ip->i_d.di_nblocks != 0) {
336                         xfs_warn(ip->i_mount,
337 "Corruption detected! Free inode 0x%llx has blocks allocated!",
338                                 ip->i_ino);
339                         return -EFSCORRUPTED;
340                 }
341                 return 0;
342         }
343 
344         /* should be an allocated inode */
345         if (VFS_I(ip)->i_mode == 0)
346                 return -ENOENT;
347 
348         return 0;
349 }
350 
351 /*
352  * Check the validity of the inode we just found it the cache
353  */
354 static int
355 xfs_iget_cache_hit(
356         struct xfs_perag        *pag,
357         struct xfs_inode        *ip,
358         xfs_ino_t               ino,
359         int                     flags,
360         int                     lock_flags) __releases(RCU)
361 {
362         struct inode            *inode = VFS_I(ip);
363         struct xfs_mount        *mp = ip->i_mount;
364         int                     error;
365 
366         /*
367          * check for re-use of an inode within an RCU grace period due to the
368          * radix tree nodes not being updated yet. We monitor for this by
369          * setting the inode number to zero before freeing the inode structure.
370          * If the inode has been reallocated and set up, then the inode number
371          * will not match, so check for that, too.
372          */
373         spin_lock(&ip->i_flags_lock);
374         if (ip->i_ino != ino) {
375                 trace_xfs_iget_skip(ip);
376                 XFS_STATS_INC(mp, xs_ig_frecycle);
377                 error = -EAGAIN;
378                 goto out_error;
379         }
380 
381 
382         /*
383          * If we are racing with another cache hit that is currently
384          * instantiating this inode or currently recycling it out of
385          * reclaimabe state, wait for the initialisation to complete
386          * before continuing.
387          *
388          * XXX(hch): eventually we should do something equivalent to
389          *           wait_on_inode to wait for these flags to be cleared
390          *           instead of polling for it.
391          */
392         if (ip->i_flags & (XFS_INEW|XFS_IRECLAIM)) {
393                 trace_xfs_iget_skip(ip);
394                 XFS_STATS_INC(mp, xs_ig_frecycle);
395                 error = -EAGAIN;
396                 goto out_error;
397         }
398 
399         /*
400          * Check the inode free state is valid. This also detects lookup
401          * racing with unlinks.
402          */
403         error = xfs_iget_check_free_state(ip, flags);
404         if (error)
405                 goto out_error;
406 
407         /*
408          * If IRECLAIMABLE is set, we've torn down the VFS inode already.
409          * Need to carefully get it back into useable state.
410          */
411         if (ip->i_flags & XFS_IRECLAIMABLE) {
412                 trace_xfs_iget_reclaim(ip);
413 
414                 if (flags & XFS_IGET_INCORE) {
415                         error = -EAGAIN;
416                         goto out_error;
417                 }
418 
419                 /*
420                  * We need to set XFS_IRECLAIM to prevent xfs_reclaim_inode
421                  * from stomping over us while we recycle the inode.  We can't
422                  * clear the radix tree reclaimable tag yet as it requires
423                  * pag_ici_lock to be held exclusive.
424                  */
425                 ip->i_flags |= XFS_IRECLAIM;
426 
427                 spin_unlock(&ip->i_flags_lock);
428                 rcu_read_unlock();
429 
430                 error = xfs_reinit_inode(mp, inode);
431                 if (error) {
432                         bool wake;
433                         /*
434                          * Re-initializing the inode failed, and we are in deep
435                          * trouble.  Try to re-add it to the reclaim list.
436                          */
437                         rcu_read_lock();
438                         spin_lock(&ip->i_flags_lock);
439                         wake = !!__xfs_iflags_test(ip, XFS_INEW);
440                         ip->i_flags &= ~(XFS_INEW | XFS_IRECLAIM);
441                         if (wake)
442                                 wake_up_bit(&ip->i_flags, __XFS_INEW_BIT);
443                         ASSERT(ip->i_flags & XFS_IRECLAIMABLE);
444                         trace_xfs_iget_reclaim_fail(ip);
445                         goto out_error;
446                 }
447 
448                 spin_lock(&pag->pag_ici_lock);
449                 spin_lock(&ip->i_flags_lock);
450 
451                 /*
452                  * Clear the per-lifetime state in the inode as we are now
453                  * effectively a new inode and need to return to the initial
454                  * state before reuse occurs.
455                  */
456                 ip->i_flags &= ~XFS_IRECLAIM_RESET_FLAGS;
457                 ip->i_flags |= XFS_INEW;
458                 xfs_inode_clear_reclaim_tag(pag, ip->i_ino);
459                 inode->i_state = I_NEW;
460 
461                 ASSERT(!rwsem_is_locked(&inode->i_rwsem));
462                 init_rwsem(&inode->i_rwsem);
463 
464                 spin_unlock(&ip->i_flags_lock);
465                 spin_unlock(&pag->pag_ici_lock);
466         } else {
467                 /* If the VFS inode is being torn down, pause and try again. */
468                 if (!igrab(inode)) {
469                         trace_xfs_iget_skip(ip);
470                         error = -EAGAIN;
471                         goto out_error;
472                 }
473 
474                 /* We've got a live one. */
475                 spin_unlock(&ip->i_flags_lock);
476                 rcu_read_unlock();
477                 trace_xfs_iget_hit(ip);
478         }
479 
480         if (lock_flags != 0)
481                 xfs_ilock(ip, lock_flags);
482 
483         if (!(flags & XFS_IGET_INCORE))
484                 xfs_iflags_clear(ip, XFS_ISTALE | XFS_IDONTCACHE);
485         XFS_STATS_INC(mp, xs_ig_found);
486 
487         return 0;
488 
489 out_error:
490         spin_unlock(&ip->i_flags_lock);
491         rcu_read_unlock();
492         return error;
493 }
494 
495 
496 static int
497 xfs_iget_cache_miss(
498         struct xfs_mount        *mp,
499         struct xfs_perag        *pag,
500         xfs_trans_t             *tp,
501         xfs_ino_t               ino,
502         struct xfs_inode        **ipp,
503         int                     flags,
504         int                     lock_flags)
505 {
506         struct xfs_inode        *ip;
507         int                     error;
508         xfs_agino_t             agino = XFS_INO_TO_AGINO(mp, ino);
509         int                     iflags;
510 
511         ip = xfs_inode_alloc(mp, ino);
512         if (!ip)
513                 return -ENOMEM;
514 
515         error = xfs_iread(mp, tp, ip, flags);
516         if (error)
517                 goto out_destroy;
518 
519         if (!xfs_inode_verify_forks(ip)) {
520                 error = -EFSCORRUPTED;
521                 goto out_destroy;
522         }
523 
524         trace_xfs_iget_miss(ip);
525 
526 
527         /*
528          * Check the inode free state is valid. This also detects lookup
529          * racing with unlinks.
530          */
531         error = xfs_iget_check_free_state(ip, flags);
532         if (error)
533                 goto out_destroy;
534 
535         /*
536          * Preload the radix tree so we can insert safely under the
537          * write spinlock. Note that we cannot sleep inside the preload
538          * region. Since we can be called from transaction context, don't
539          * recurse into the file system.
540          */
541         if (radix_tree_preload(GFP_NOFS)) {
542                 error = -EAGAIN;
543                 goto out_destroy;
544         }
545 
546         /*
547          * Because the inode hasn't been added to the radix-tree yet it can't
548          * be found by another thread, so we can do the non-sleeping lock here.
549          */
550         if (lock_flags) {
551                 if (!xfs_ilock_nowait(ip, lock_flags))
552                         BUG();
553         }
554 
555         /*
556          * These values must be set before inserting the inode into the radix
557          * tree as the moment it is inserted a concurrent lookup (allowed by the
558          * RCU locking mechanism) can find it and that lookup must see that this
559          * is an inode currently under construction (i.e. that XFS_INEW is set).
560          * The ip->i_flags_lock that protects the XFS_INEW flag forms the
561          * memory barrier that ensures this detection works correctly at lookup
562          * time.
563          */
564         iflags = XFS_INEW;
565         if (flags & XFS_IGET_DONTCACHE)
566                 iflags |= XFS_IDONTCACHE;
567         ip->i_udquot = NULL;
568         ip->i_gdquot = NULL;
569         ip->i_pdquot = NULL;
570         xfs_iflags_set(ip, iflags);
571 
572         /* insert the new inode */
573         spin_lock(&pag->pag_ici_lock);
574         error = radix_tree_insert(&pag->pag_ici_root, agino, ip);
575         if (unlikely(error)) {
576                 WARN_ON(error != -EEXIST);
577                 XFS_STATS_INC(mp, xs_ig_dup);
578                 error = -EAGAIN;
579                 goto out_preload_end;
580         }
581         spin_unlock(&pag->pag_ici_lock);
582         radix_tree_preload_end();
583 
584         *ipp = ip;
585         return 0;
586 
587 out_preload_end:
588         spin_unlock(&pag->pag_ici_lock);
589         radix_tree_preload_end();
590         if (lock_flags)
591                 xfs_iunlock(ip, lock_flags);
592 out_destroy:
593         __destroy_inode(VFS_I(ip));
594         xfs_inode_free(ip);
595         return error;
596 }
597 
598 /*
599  * Look up an inode by number in the given file system.
600  * The inode is looked up in the cache held in each AG.
601  * If the inode is found in the cache, initialise the vfs inode
602  * if necessary.
603  *
604  * If it is not in core, read it in from the file system's device,
605  * add it to the cache and initialise the vfs inode.
606  *
607  * The inode is locked according to the value of the lock_flags parameter.
608  * This flag parameter indicates how and if the inode's IO lock and inode lock
609  * should be taken.
610  *
611  * mp -- the mount point structure for the current file system.  It points
612  *       to the inode hash table.
613  * tp -- a pointer to the current transaction if there is one.  This is
614  *       simply passed through to the xfs_iread() call.
615  * ino -- the number of the inode desired.  This is the unique identifier
616  *        within the file system for the inode being requested.
617  * lock_flags -- flags indicating how to lock the inode.  See the comment
618  *               for xfs_ilock() for a list of valid values.
619  */
620 int
621 xfs_iget(
622         xfs_mount_t     *mp,
623         xfs_trans_t     *tp,
624         xfs_ino_t       ino,
625         uint            flags,
626         uint            lock_flags,
627         xfs_inode_t     **ipp)
628 {
629         xfs_inode_t     *ip;
630         int             error;
631         xfs_perag_t     *pag;
632         xfs_agino_t     agino;
633 
634         /*
635          * xfs_reclaim_inode() uses the ILOCK to ensure an inode
636          * doesn't get freed while it's being referenced during a
637          * radix tree traversal here.  It assumes this function
638          * aqcuires only the ILOCK (and therefore it has no need to
639          * involve the IOLOCK in this synchronization).
640          */
641         ASSERT((lock_flags & (XFS_IOLOCK_EXCL | XFS_IOLOCK_SHARED)) == 0);
642 
643         /* reject inode numbers outside existing AGs */
644         if (!ino || XFS_INO_TO_AGNO(mp, ino) >= mp->m_sb.sb_agcount)
645                 return -EINVAL;
646 
647         XFS_STATS_INC(mp, xs_ig_attempts);
648 
649         /* get the perag structure and ensure that it's inode capable */
650         pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ino));
651         agino = XFS_INO_TO_AGINO(mp, ino);
652 
653 again:
654         error = 0;
655         rcu_read_lock();
656         ip = radix_tree_lookup(&pag->pag_ici_root, agino);
657 
658         if (ip) {
659                 error = xfs_iget_cache_hit(pag, ip, ino, flags, lock_flags);
660                 if (error)
661                         goto out_error_or_again;
662         } else {
663                 rcu_read_unlock();
664                 if (flags & XFS_IGET_INCORE) {
665                         error = -ENODATA;
666                         goto out_error_or_again;
667                 }
668                 XFS_STATS_INC(mp, xs_ig_missed);
669 
670                 error = xfs_iget_cache_miss(mp, pag, tp, ino, &ip,
671                                                         flags, lock_flags);
672                 if (error)
673                         goto out_error_or_again;
674         }
675         xfs_perag_put(pag);
676 
677         *ipp = ip;
678 
679         /*
680          * If we have a real type for an on-disk inode, we can setup the inode
681          * now.  If it's a new inode being created, xfs_ialloc will handle it.
682          */
683         if (xfs_iflags_test(ip, XFS_INEW) && VFS_I(ip)->i_mode != 0)
684                 xfs_setup_existing_inode(ip);
685         return 0;
686 
687 out_error_or_again:
688         if (!(flags & XFS_IGET_INCORE) && error == -EAGAIN) {
689                 delay(1);
690                 goto again;
691         }
692         xfs_perag_put(pag);
693         return error;
694 }
695 
696 /*
697  * "Is this a cached inode that's also allocated?"
698  *
699  * Look up an inode by number in the given file system.  If the inode is
700  * in cache and isn't in purgatory, return 1 if the inode is allocated
701  * and 0 if it is not.  For all other cases (not in cache, being torn
702  * down, etc.), return a negative error code.
703  *
704  * The caller has to prevent inode allocation and freeing activity,
705  * presumably by locking the AGI buffer.   This is to ensure that an
706  * inode cannot transition from allocated to freed until the caller is
707  * ready to allow that.  If the inode is in an intermediate state (new,
708  * reclaimable, or being reclaimed), -EAGAIN will be returned; if the
709  * inode is not in the cache, -ENOENT will be returned.  The caller must
710  * deal with these scenarios appropriately.
711  *
712  * This is a specialized use case for the online scrubber; if you're
713  * reading this, you probably want xfs_iget.
714  */
715 int
716 xfs_icache_inode_is_allocated(
717         struct xfs_mount        *mp,
718         struct xfs_trans        *tp,
719         xfs_ino_t               ino,
720         bool                    *inuse)
721 {
722         struct xfs_inode        *ip;
723         int                     error;
724 
725         error = xfs_iget(mp, tp, ino, XFS_IGET_INCORE, 0, &ip);
726         if (error)
727                 return error;
728 
729         *inuse = !!(VFS_I(ip)->i_mode);
730         IRELE(ip);
731         return 0;
732 }
733 
734 /*
735  * The inode lookup is done in batches to keep the amount of lock traffic and
736  * radix tree lookups to a minimum. The batch size is a trade off between
737  * lookup reduction and stack usage. This is in the reclaim path, so we can't
738  * be too greedy.
739  */
740 #define XFS_LOOKUP_BATCH        32
741 
742 STATIC int
743 xfs_inode_ag_walk_grab(
744         struct xfs_inode        *ip,
745         int                     flags)
746 {
747         struct inode            *inode = VFS_I(ip);
748         bool                    newinos = !!(flags & XFS_AGITER_INEW_WAIT);
749 
750         ASSERT(rcu_read_lock_held());
751 
752         /*
753          * check for stale RCU freed inode
754          *
755          * If the inode has been reallocated, it doesn't matter if it's not in
756          * the AG we are walking - we are walking for writeback, so if it
757          * passes all the "valid inode" checks and is dirty, then we'll write
758          * it back anyway.  If it has been reallocated and still being
759          * initialised, the XFS_INEW check below will catch it.
760          */
761         spin_lock(&ip->i_flags_lock);
762         if (!ip->i_ino)
763                 goto out_unlock_noent;
764 
765         /* avoid new or reclaimable inodes. Leave for reclaim code to flush */
766         if ((!newinos && __xfs_iflags_test(ip, XFS_INEW)) ||
767             __xfs_iflags_test(ip, XFS_IRECLAIMABLE | XFS_IRECLAIM))
768                 goto out_unlock_noent;
769         spin_unlock(&ip->i_flags_lock);
770 
771         /* nothing to sync during shutdown */
772         if (XFS_FORCED_SHUTDOWN(ip->i_mount))
773                 return -EFSCORRUPTED;
774 
775         /* If we can't grab the inode, it must on it's way to reclaim. */
776         if (!igrab(inode))
777                 return -ENOENT;
778 
779         /* inode is valid */
780         return 0;
781 
782 out_unlock_noent:
783         spin_unlock(&ip->i_flags_lock);
784         return -ENOENT;
785 }
786 
787 STATIC int
788 xfs_inode_ag_walk(
789         struct xfs_mount        *mp,
790         struct xfs_perag        *pag,
791         int                     (*execute)(struct xfs_inode *ip, int flags,
792                                            void *args),
793         int                     flags,
794         void                    *args,
795         int                     tag,
796         int                     iter_flags)
797 {
798         uint32_t                first_index;
799         int                     last_error = 0;
800         int                     skipped;
801         int                     done;
802         int                     nr_found;
803 
804 restart:
805         done = 0;
806         skipped = 0;
807         first_index = 0;
808         nr_found = 0;
809         do {
810                 struct xfs_inode *batch[XFS_LOOKUP_BATCH];
811                 int             error = 0;
812                 int             i;
813 
814                 rcu_read_lock();
815 
816                 if (tag == -1)
817                         nr_found = radix_tree_gang_lookup(&pag->pag_ici_root,
818                                         (void **)batch, first_index,
819                                         XFS_LOOKUP_BATCH);
820                 else
821                         nr_found = radix_tree_gang_lookup_tag(
822                                         &pag->pag_ici_root,
823                                         (void **) batch, first_index,
824                                         XFS_LOOKUP_BATCH, tag);
825 
826                 if (!nr_found) {
827                         rcu_read_unlock();
828                         break;
829                 }
830 
831                 /*
832                  * Grab the inodes before we drop the lock. if we found
833                  * nothing, nr == 0 and the loop will be skipped.
834                  */
835                 for (i = 0; i < nr_found; i++) {
836                         struct xfs_inode *ip = batch[i];
837 
838                         if (done || xfs_inode_ag_walk_grab(ip, iter_flags))
839                                 batch[i] = NULL;
840 
841                         /*
842                          * Update the index for the next lookup. Catch
843                          * overflows into the next AG range which can occur if
844                          * we have inodes in the last block of the AG and we
845                          * are currently pointing to the last inode.
846                          *
847                          * Because we may see inodes that are from the wrong AG
848                          * due to RCU freeing and reallocation, only update the
849                          * index if it lies in this AG. It was a race that lead
850                          * us to see this inode, so another lookup from the
851                          * same index will not find it again.
852                          */
853                         if (XFS_INO_TO_AGNO(mp, ip->i_ino) != pag->pag_agno)
854                                 continue;
855                         first_index = XFS_INO_TO_AGINO(mp, ip->i_ino + 1);
856                         if (first_index < XFS_INO_TO_AGINO(mp, ip->i_ino))
857                                 done = 1;
858                 }
859 
860                 /* unlock now we've grabbed the inodes. */
861                 rcu_read_unlock();
862 
863                 for (i = 0; i < nr_found; i++) {
864                         if (!batch[i])
865                                 continue;
866                         if ((iter_flags & XFS_AGITER_INEW_WAIT) &&
867                             xfs_iflags_test(batch[i], XFS_INEW))
868                                 xfs_inew_wait(batch[i]);
869                         error = execute(batch[i], flags, args);
870                         IRELE(batch[i]);
871                         if (error == -EAGAIN) {
872                                 skipped++;
873                                 continue;
874                         }
875                         if (error && last_error != -EFSCORRUPTED)
876                                 last_error = error;
877                 }
878 
879                 /* bail out if the filesystem is corrupted.  */
880                 if (error == -EFSCORRUPTED)
881                         break;
882 
883                 cond_resched();
884 
885         } while (nr_found && !done);
886 
887         if (skipped) {
888                 delay(1);
889                 goto restart;
890         }
891         return last_error;
892 }
893 
894 /*
895  * Background scanning to trim post-EOF preallocated space. This is queued
896  * based on the 'speculative_prealloc_lifetime' tunable (5m by default).
897  */
898 void
899 xfs_queue_eofblocks(
900         struct xfs_mount *mp)
901 {
902         rcu_read_lock();
903         if (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_EOFBLOCKS_TAG))
904                 queue_delayed_work(mp->m_eofblocks_workqueue,
905                                    &mp->m_eofblocks_work,
906                                    msecs_to_jiffies(xfs_eofb_secs * 1000));
907         rcu_read_unlock();
908 }
909 
910 void
911 xfs_eofblocks_worker(
912         struct work_struct *work)
913 {
914         struct xfs_mount *mp = container_of(to_delayed_work(work),
915                                 struct xfs_mount, m_eofblocks_work);
916         xfs_icache_free_eofblocks(mp, NULL);
917         xfs_queue_eofblocks(mp);
918 }
919 
920 /*
921  * Background scanning to trim preallocated CoW space. This is queued
922  * based on the 'speculative_cow_prealloc_lifetime' tunable (5m by default).
923  * (We'll just piggyback on the post-EOF prealloc space workqueue.)
924  */
925 void
926 xfs_queue_cowblocks(
927         struct xfs_mount *mp)
928 {
929         rcu_read_lock();
930         if (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_COWBLOCKS_TAG))
931                 queue_delayed_work(mp->m_eofblocks_workqueue,
932                                    &mp->m_cowblocks_work,
933                                    msecs_to_jiffies(xfs_cowb_secs * 1000));
934         rcu_read_unlock();
935 }
936 
937 void
938 xfs_cowblocks_worker(
939         struct work_struct *work)
940 {
941         struct xfs_mount *mp = container_of(to_delayed_work(work),
942                                 struct xfs_mount, m_cowblocks_work);
943         xfs_icache_free_cowblocks(mp, NULL);
944         xfs_queue_cowblocks(mp);
945 }
946 
947 int
948 xfs_inode_ag_iterator_flags(
949         struct xfs_mount        *mp,
950         int                     (*execute)(struct xfs_inode *ip, int flags,
951                                            void *args),
952         int                     flags,
953         void                    *args,
954         int                     iter_flags)
955 {
956         struct xfs_perag        *pag;
957         int                     error = 0;
958         int                     last_error = 0;
959         xfs_agnumber_t          ag;
960 
961         ag = 0;
962         while ((pag = xfs_perag_get(mp, ag))) {
963                 ag = pag->pag_agno + 1;
964                 error = xfs_inode_ag_walk(mp, pag, execute, flags, args, -1,
965                                           iter_flags);
966                 xfs_perag_put(pag);
967                 if (error) {
968                         last_error = error;
969                         if (error == -EFSCORRUPTED)
970                                 break;
971                 }
972         }
973         return last_error;
974 }
975 
976 int
977 xfs_inode_ag_iterator(
978         struct xfs_mount        *mp,
979         int                     (*execute)(struct xfs_inode *ip, int flags,
980                                            void *args),
981         int                     flags,
982         void                    *args)
983 {
984         return xfs_inode_ag_iterator_flags(mp, execute, flags, args, 0);
985 }
986 
987 int
988 xfs_inode_ag_iterator_tag(
989         struct xfs_mount        *mp,
990         int                     (*execute)(struct xfs_inode *ip, int flags,
991                                            void *args),
992         int                     flags,
993         void                    *args,
994         int                     tag)
995 {
996         struct xfs_perag        *pag;
997         int                     error = 0;
998         int                     last_error = 0;
999         xfs_agnumber_t          ag;
1000 
1001         ag = 0;
1002         while ((pag = xfs_perag_get_tag(mp, ag, tag))) {
1003                 ag = pag->pag_agno + 1;
1004                 error = xfs_inode_ag_walk(mp, pag, execute, flags, args, tag,
1005                                           0);
1006                 xfs_perag_put(pag);
1007                 if (error) {
1008                         last_error = error;
1009                         if (error == -EFSCORRUPTED)
1010                                 break;
1011                 }
1012         }
1013         return last_error;
1014 }
1015 
1016 /*
1017  * Grab the inode for reclaim exclusively.
1018  * Return 0 if we grabbed it, non-zero otherwise.
1019  */
1020 STATIC int
1021 xfs_reclaim_inode_grab(
1022         struct xfs_inode        *ip,
1023         int                     flags)
1024 {
1025         ASSERT(rcu_read_lock_held());
1026 
1027         /* quick check for stale RCU freed inode */
1028         if (!ip->i_ino)
1029                 return 1;
1030 
1031         /*
1032          * If we are asked for non-blocking operation, do unlocked checks to
1033          * see if the inode already is being flushed or in reclaim to avoid
1034          * lock traffic.
1035          */
1036         if ((flags & SYNC_TRYLOCK) &&
1037             __xfs_iflags_test(ip, XFS_IFLOCK | XFS_IRECLAIM))
1038                 return 1;
1039 
1040         /*
1041          * The radix tree lock here protects a thread in xfs_iget from racing
1042          * with us starting reclaim on the inode.  Once we have the
1043          * XFS_IRECLAIM flag set it will not touch us.
1044          *
1045          * Due to RCU lookup, we may find inodes that have been freed and only
1046          * have XFS_IRECLAIM set.  Indeed, we may see reallocated inodes that
1047          * aren't candidates for reclaim at all, so we must check the
1048          * XFS_IRECLAIMABLE is set first before proceeding to reclaim.
1049          */
1050         spin_lock(&ip->i_flags_lock);
1051         if (!__xfs_iflags_test(ip, XFS_IRECLAIMABLE) ||
1052             __xfs_iflags_test(ip, XFS_IRECLAIM)) {
1053                 /* not a reclaim candidate. */
1054                 spin_unlock(&ip->i_flags_lock);
1055                 return 1;
1056         }
1057         __xfs_iflags_set(ip, XFS_IRECLAIM);
1058         spin_unlock(&ip->i_flags_lock);
1059         return 0;
1060 }
1061 
1062 /*
1063  * Inodes in different states need to be treated differently. The following
1064  * table lists the inode states and the reclaim actions necessary:
1065  *
1066  *      inode state          iflush ret         required action
1067  *      ---------------      ----------         ---------------
1068  *      bad                     -               reclaim
1069  *      shutdown                EIO             unpin and reclaim
1070  *      clean, unpinned         0               reclaim
1071  *      stale, unpinned         0               reclaim
1072  *      clean, pinned(*)        0               requeue
1073  *      stale, pinned           EAGAIN          requeue
1074  *      dirty, async            -               requeue
1075  *      dirty, sync             0               reclaim
1076  *
1077  * (*) dgc: I don't think the clean, pinned state is possible but it gets
1078  * handled anyway given the order of checks implemented.
1079  *
1080  * Also, because we get the flush lock first, we know that any inode that has
1081  * been flushed delwri has had the flush completed by the time we check that
1082  * the inode is clean.
1083  *
1084  * Note that because the inode is flushed delayed write by AIL pushing, the
1085  * flush lock may already be held here and waiting on it can result in very
1086  * long latencies.  Hence for sync reclaims, where we wait on the flush lock,
1087  * the caller should push the AIL first before trying to reclaim inodes to
1088  * minimise the amount of time spent waiting.  For background relaim, we only
1089  * bother to reclaim clean inodes anyway.
1090  *
1091  * Hence the order of actions after gaining the locks should be:
1092  *      bad             => reclaim
1093  *      shutdown        => unpin and reclaim
1094  *      pinned, async   => requeue
1095  *      pinned, sync    => unpin
1096  *      stale           => reclaim
1097  *      clean           => reclaim
1098  *      dirty, async    => requeue
1099  *      dirty, sync     => flush, wait and reclaim
1100  */
1101 STATIC int
1102 xfs_reclaim_inode(
1103         struct xfs_inode        *ip,
1104         struct xfs_perag        *pag,
1105         int                     sync_mode)
1106 {
1107         struct xfs_buf          *bp = NULL;
1108         xfs_ino_t               ino = ip->i_ino; /* for radix_tree_delete */
1109         int                     error;
1110 
1111 restart:
1112         error = 0;
1113         xfs_ilock(ip, XFS_ILOCK_EXCL);
1114         if (!xfs_iflock_nowait(ip)) {
1115                 if (!(sync_mode & SYNC_WAIT))
1116                         goto out;
1117                 xfs_iflock(ip);
1118         }
1119 
1120         if (XFS_FORCED_SHUTDOWN(ip->i_mount)) {
1121                 xfs_iunpin_wait(ip);
1122                 /* xfs_iflush_abort() drops the flush lock */
1123                 xfs_iflush_abort(ip, false);
1124                 goto reclaim;
1125         }
1126         if (xfs_ipincount(ip)) {
1127                 if (!(sync_mode & SYNC_WAIT))
1128                         goto out_ifunlock;
1129                 xfs_iunpin_wait(ip);
1130         }
1131         if (xfs_iflags_test(ip, XFS_ISTALE) || xfs_inode_clean(ip)) {
1132                 xfs_ifunlock(ip);
1133                 goto reclaim;
1134         }
1135 
1136         /*
1137          * Never flush out dirty data during non-blocking reclaim, as it would
1138          * just contend with AIL pushing trying to do the same job.
1139          */
1140         if (!(sync_mode & SYNC_WAIT))
1141                 goto out_ifunlock;
1142 
1143         /*
1144          * Now we have an inode that needs flushing.
1145          *
1146          * Note that xfs_iflush will never block on the inode buffer lock, as
1147          * xfs_ifree_cluster() can lock the inode buffer before it locks the
1148          * ip->i_lock, and we are doing the exact opposite here.  As a result,
1149          * doing a blocking xfs_imap_to_bp() to get the cluster buffer would
1150          * result in an ABBA deadlock with xfs_ifree_cluster().
1151          *
1152          * As xfs_ifree_cluser() must gather all inodes that are active in the
1153          * cache to mark them stale, if we hit this case we don't actually want
1154          * to do IO here - we want the inode marked stale so we can simply
1155          * reclaim it.  Hence if we get an EAGAIN error here,  just unlock the
1156          * inode, back off and try again.  Hopefully the next pass through will
1157          * see the stale flag set on the inode.
1158          */
1159         error = xfs_iflush(ip, &bp);
1160         if (error == -EAGAIN) {
1161                 xfs_iunlock(ip, XFS_ILOCK_EXCL);
1162                 /* backoff longer than in xfs_ifree_cluster */
1163                 delay(2);
1164                 goto restart;
1165         }
1166 
1167         if (!error) {
1168                 error = xfs_bwrite(bp);
1169                 xfs_buf_relse(bp);
1170         }
1171 
1172 reclaim:
1173         ASSERT(!xfs_isiflocked(ip));
1174 
1175         /*
1176          * Because we use RCU freeing we need to ensure the inode always appears
1177          * to be reclaimed with an invalid inode number when in the free state.
1178          * We do this as early as possible under the ILOCK so that
1179          * xfs_iflush_cluster() and xfs_ifree_cluster() can be guaranteed to
1180          * detect races with us here. By doing this, we guarantee that once
1181          * xfs_iflush_cluster() or xfs_ifree_cluster() has locked XFS_ILOCK that
1182          * it will see either a valid inode that will serialise correctly, or it
1183          * will see an invalid inode that it can skip.
1184          */
1185         spin_lock(&ip->i_flags_lock);
1186         ip->i_flags = XFS_IRECLAIM;
1187         ip->i_ino = 0;
1188         spin_unlock(&ip->i_flags_lock);
1189 
1190         xfs_iunlock(ip, XFS_ILOCK_EXCL);
1191 
1192         XFS_STATS_INC(ip->i_mount, xs_ig_reclaims);
1193         /*
1194          * Remove the inode from the per-AG radix tree.
1195          *
1196          * Because radix_tree_delete won't complain even if the item was never
1197          * added to the tree assert that it's been there before to catch
1198          * problems with the inode life time early on.
1199          */
1200         spin_lock(&pag->pag_ici_lock);
1201         if (!radix_tree_delete(&pag->pag_ici_root,
1202                                 XFS_INO_TO_AGINO(ip->i_mount, ino)))
1203                 ASSERT(0);
1204         xfs_perag_clear_reclaim_tag(pag);
1205         spin_unlock(&pag->pag_ici_lock);
1206 
1207         /*
1208          * Here we do an (almost) spurious inode lock in order to coordinate
1209          * with inode cache radix tree lookups.  This is because the lookup
1210          * can reference the inodes in the cache without taking references.
1211          *
1212          * We make that OK here by ensuring that we wait until the inode is
1213          * unlocked after the lookup before we go ahead and free it.
1214          */
1215         xfs_ilock(ip, XFS_ILOCK_EXCL);
1216         xfs_qm_dqdetach(ip);
1217         xfs_iunlock(ip, XFS_ILOCK_EXCL);
1218 
1219         __xfs_inode_free(ip);
1220         return error;
1221 
1222 out_ifunlock:
1223         xfs_ifunlock(ip);
1224 out:
1225         xfs_iflags_clear(ip, XFS_IRECLAIM);
1226         xfs_iunlock(ip, XFS_ILOCK_EXCL);
1227         /*
1228          * We could return -EAGAIN here to make reclaim rescan the inode tree in
1229          * a short while. However, this just burns CPU time scanning the tree
1230          * waiting for IO to complete and the reclaim work never goes back to
1231          * the idle state. Instead, return 0 to let the next scheduled
1232          * background reclaim attempt to reclaim the inode again.
1233          */
1234         return 0;
1235 }
1236 
1237 /*
1238  * Walk the AGs and reclaim the inodes in them. Even if the filesystem is
1239  * corrupted, we still want to try to reclaim all the inodes. If we don't,
1240  * then a shut down during filesystem unmount reclaim walk leak all the
1241  * unreclaimed inodes.
1242  */
1243 STATIC int
1244 xfs_reclaim_inodes_ag(
1245         struct xfs_mount        *mp,
1246         int                     flags,
1247         int                     *nr_to_scan)
1248 {
1249         struct xfs_perag        *pag;
1250         int                     error = 0;
1251         int                     last_error = 0;
1252         xfs_agnumber_t          ag;
1253         int                     trylock = flags & SYNC_TRYLOCK;
1254         int                     skipped;
1255 
1256 restart:
1257         ag = 0;
1258         skipped = 0;
1259         while ((pag = xfs_perag_get_tag(mp, ag, XFS_ICI_RECLAIM_TAG))) {
1260                 unsigned long   first_index = 0;
1261                 int             done = 0;
1262                 int             nr_found = 0;
1263 
1264                 ag = pag->pag_agno + 1;
1265 
1266                 if (trylock) {
1267                         if (!mutex_trylock(&pag->pag_ici_reclaim_lock)) {
1268                                 skipped++;
1269                                 xfs_perag_put(pag);
1270                                 continue;
1271                         }
1272                         first_index = pag->pag_ici_reclaim_cursor;
1273                 } else
1274                         mutex_lock(&pag->pag_ici_reclaim_lock);
1275 
1276                 do {
1277                         struct xfs_inode *batch[XFS_LOOKUP_BATCH];
1278                         int     i;
1279 
1280                         rcu_read_lock();
1281                         nr_found = radix_tree_gang_lookup_tag(
1282                                         &pag->pag_ici_root,
1283                                         (void **)batch, first_index,
1284                                         XFS_LOOKUP_BATCH,
1285                                         XFS_ICI_RECLAIM_TAG);
1286                         if (!nr_found) {
1287                                 done = 1;
1288                                 rcu_read_unlock();
1289                                 break;
1290                         }
1291 
1292                         /*
1293                          * Grab the inodes before we drop the lock. if we found
1294                          * nothing, nr == 0 and the loop will be skipped.
1295                          */
1296                         for (i = 0; i < nr_found; i++) {
1297                                 struct xfs_inode *ip = batch[i];
1298 
1299                                 if (done || xfs_reclaim_inode_grab(ip, flags))
1300                                         batch[i] = NULL;
1301 
1302                                 /*
1303                                  * Update the index for the next lookup. Catch
1304                                  * overflows into the next AG range which can
1305                                  * occur if we have inodes in the last block of
1306                                  * the AG and we are currently pointing to the
1307                                  * last inode.
1308                                  *
1309                                  * Because we may see inodes that are from the
1310                                  * wrong AG due to RCU freeing and
1311                                  * reallocation, only update the index if it
1312                                  * lies in this AG. It was a race that lead us
1313                                  * to see this inode, so another lookup from
1314                                  * the same index will not find it again.
1315                                  */
1316                                 if (XFS_INO_TO_AGNO(mp, ip->i_ino) !=
1317                                                                 pag->pag_agno)
1318                                         continue;
1319                                 first_index = XFS_INO_TO_AGINO(mp, ip->i_ino + 1);
1320                                 if (first_index < XFS_INO_TO_AGINO(mp, ip->i_ino))
1321                                         done = 1;
1322                         }
1323 
1324                         /* unlock now we've grabbed the inodes. */
1325                         rcu_read_unlock();
1326 
1327                         for (i = 0; i < nr_found; i++) {
1328                                 if (!batch[i])
1329                                         continue;
1330                                 error = xfs_reclaim_inode(batch[i], pag, flags);
1331                                 if (error && last_error != -EFSCORRUPTED)
1332                                         last_error = error;
1333                         }
1334 
1335                         *nr_to_scan -= XFS_LOOKUP_BATCH;
1336 
1337                         cond_resched();
1338 
1339                 } while (nr_found && !done && *nr_to_scan > 0);
1340 
1341                 if (trylock && !done)
1342                         pag->pag_ici_reclaim_cursor = first_index;
1343                 else
1344                         pag->pag_ici_reclaim_cursor = 0;
1345                 mutex_unlock(&pag->pag_ici_reclaim_lock);
1346                 xfs_perag_put(pag);
1347         }
1348 
1349         /*
1350          * if we skipped any AG, and we still have scan count remaining, do
1351          * another pass this time using blocking reclaim semantics (i.e
1352          * waiting on the reclaim locks and ignoring the reclaim cursors). This
1353          * ensure that when we get more reclaimers than AGs we block rather
1354          * than spin trying to execute reclaim.
1355          */
1356         if (skipped && (flags & SYNC_WAIT) && *nr_to_scan > 0) {
1357                 trylock = 0;
1358                 goto restart;
1359         }
1360         return last_error;
1361 }
1362 
1363 int
1364 xfs_reclaim_inodes(
1365         xfs_mount_t     *mp,
1366         int             mode)
1367 {
1368         int             nr_to_scan = INT_MAX;
1369 
1370         return xfs_reclaim_inodes_ag(mp, mode, &nr_to_scan);
1371 }
1372 
1373 /*
1374  * Scan a certain number of inodes for reclaim.
1375  *
1376  * When called we make sure that there is a background (fast) inode reclaim in
1377  * progress, while we will throttle the speed of reclaim via doing synchronous
1378  * reclaim of inodes. That means if we come across dirty inodes, we wait for
1379  * them to be cleaned, which we hope will not be very long due to the
1380  * background walker having already kicked the IO off on those dirty inodes.
1381  */
1382 long
1383 xfs_reclaim_inodes_nr(
1384         struct xfs_mount        *mp,
1385         int                     nr_to_scan)
1386 {
1387         /* kick background reclaimer and push the AIL */
1388         xfs_reclaim_work_queue(mp);
1389         xfs_ail_push_all(mp->m_ail);
1390 
1391         return xfs_reclaim_inodes_ag(mp, SYNC_TRYLOCK | SYNC_WAIT, &nr_to_scan);
1392 }
1393 
1394 /*
1395  * Return the number of reclaimable inodes in the filesystem for
1396  * the shrinker to determine how much to reclaim.
1397  */
1398 int
1399 xfs_reclaim_inodes_count(
1400         struct xfs_mount        *mp)
1401 {
1402         struct xfs_perag        *pag;
1403         xfs_agnumber_t          ag = 0;
1404         int                     reclaimable = 0;
1405 
1406         while ((pag = xfs_perag_get_tag(mp, ag, XFS_ICI_RECLAIM_TAG))) {
1407                 ag = pag->pag_agno + 1;
1408                 reclaimable += pag->pag_ici_reclaimable;
1409                 xfs_perag_put(pag);
1410         }
1411         return reclaimable;
1412 }
1413 
1414 STATIC int
1415 xfs_inode_match_id(
1416         struct xfs_inode        *ip,
1417         struct xfs_eofblocks    *eofb)
1418 {
1419         if ((eofb->eof_flags & XFS_EOF_FLAGS_UID) &&
1420             !uid_eq(VFS_I(ip)->i_uid, eofb->eof_uid))
1421                 return 0;
1422 
1423         if ((eofb->eof_flags & XFS_EOF_FLAGS_GID) &&
1424             !gid_eq(VFS_I(ip)->i_gid, eofb->eof_gid))
1425                 return 0;
1426 
1427         if ((eofb->eof_flags & XFS_EOF_FLAGS_PRID) &&
1428             xfs_get_projid(ip) != eofb->eof_prid)
1429                 return 0;
1430 
1431         return 1;
1432 }
1433 
1434 /*
1435  * A union-based inode filtering algorithm. Process the inode if any of the
1436  * criteria match. This is for global/internal scans only.
1437  */
1438 STATIC int
1439 xfs_inode_match_id_union(
1440         struct xfs_inode        *ip,
1441         struct xfs_eofblocks    *eofb)
1442 {
1443         if ((eofb->eof_flags & XFS_EOF_FLAGS_UID) &&
1444             uid_eq(VFS_I(ip)->i_uid, eofb->eof_uid))
1445                 return 1;
1446 
1447         if ((eofb->eof_flags & XFS_EOF_FLAGS_GID) &&
1448             gid_eq(VFS_I(ip)->i_gid, eofb->eof_gid))
1449                 return 1;
1450 
1451         if ((eofb->eof_flags & XFS_EOF_FLAGS_PRID) &&
1452             xfs_get_projid(ip) == eofb->eof_prid)
1453                 return 1;
1454 
1455         return 0;
1456 }
1457 
1458 STATIC int
1459 xfs_inode_free_eofblocks(
1460         struct xfs_inode        *ip,
1461         int                     flags,
1462         void                    *args)
1463 {
1464         int ret = 0;
1465         struct xfs_eofblocks *eofb = args;
1466         int match;
1467 
1468         if (!xfs_can_free_eofblocks(ip, false)) {
1469                 /* inode could be preallocated or append-only */
1470                 trace_xfs_inode_free_eofblocks_invalid(ip);
1471                 xfs_inode_clear_eofblocks_tag(ip);
1472                 return 0;
1473         }
1474 
1475         /*
1476          * If the mapping is dirty the operation can block and wait for some
1477          * time. Unless we are waiting, skip it.
1478          */
1479         if (!(flags & SYNC_WAIT) &&
1480             mapping_tagged(VFS_I(ip)->i_mapping, PAGECACHE_TAG_DIRTY))
1481                 return 0;
1482 
1483         if (eofb) {
1484                 if (eofb->eof_flags & XFS_EOF_FLAGS_UNION)
1485                         match = xfs_inode_match_id_union(ip, eofb);
1486                 else
1487                         match = xfs_inode_match_id(ip, eofb);
1488                 if (!match)
1489                         return 0;
1490 
1491                 /* skip the inode if the file size is too small */
1492                 if (eofb->eof_flags & XFS_EOF_FLAGS_MINFILESIZE &&
1493                     XFS_ISIZE(ip) < eofb->eof_min_file_size)
1494                         return 0;
1495         }
1496 
1497         /*
1498          * If the caller is waiting, return -EAGAIN to keep the background
1499          * scanner moving and revisit the inode in a subsequent pass.
1500          */
1501         if (!xfs_ilock_nowait(ip, XFS_IOLOCK_EXCL)) {
1502                 if (flags & SYNC_WAIT)
1503                         ret = -EAGAIN;
1504                 return ret;
1505         }
1506         ret = xfs_free_eofblocks(ip);
1507         xfs_iunlock(ip, XFS_IOLOCK_EXCL);
1508 
1509         return ret;
1510 }
1511 
1512 static int
1513 __xfs_icache_free_eofblocks(
1514         struct xfs_mount        *mp,
1515         struct xfs_eofblocks    *eofb,
1516         int                     (*execute)(struct xfs_inode *ip, int flags,
1517                                            void *args),
1518         int                     tag)
1519 {
1520         int flags = SYNC_TRYLOCK;
1521 
1522         if (eofb && (eofb->eof_flags & XFS_EOF_FLAGS_SYNC))
1523                 flags = SYNC_WAIT;
1524 
1525         return xfs_inode_ag_iterator_tag(mp, execute, flags,
1526                                          eofb, tag);
1527 }
1528 
1529 int
1530 xfs_icache_free_eofblocks(
1531         struct xfs_mount        *mp,
1532         struct xfs_eofblocks    *eofb)
1533 {
1534         return __xfs_icache_free_eofblocks(mp, eofb, xfs_inode_free_eofblocks,
1535                         XFS_ICI_EOFBLOCKS_TAG);
1536 }
1537 
1538 /*
1539  * Run eofblocks scans on the quotas applicable to the inode. For inodes with
1540  * multiple quotas, we don't know exactly which quota caused an allocation
1541  * failure. We make a best effort by including each quota under low free space
1542  * conditions (less than 1% free space) in the scan.
1543  */
1544 static int
1545 __xfs_inode_free_quota_eofblocks(
1546         struct xfs_inode        *ip,
1547         int                     (*execute)(struct xfs_mount *mp,
1548                                            struct xfs_eofblocks *eofb))
1549 {
1550         int scan = 0;
1551         struct xfs_eofblocks eofb = {0};
1552         struct xfs_dquot *dq;
1553 
1554         /*
1555          * Run a sync scan to increase effectiveness and use the union filter to
1556          * cover all applicable quotas in a single scan.
1557          */
1558         eofb.eof_flags = XFS_EOF_FLAGS_UNION|XFS_EOF_FLAGS_SYNC;
1559 
1560         if (XFS_IS_UQUOTA_ENFORCED(ip->i_mount)) {
1561                 dq = xfs_inode_dquot(ip, XFS_DQ_USER);
1562                 if (dq && xfs_dquot_lowsp(dq)) {
1563                         eofb.eof_uid = VFS_I(ip)->i_uid;
1564                         eofb.eof_flags |= XFS_EOF_FLAGS_UID;
1565                         scan = 1;
1566                 }
1567         }
1568 
1569         if (XFS_IS_GQUOTA_ENFORCED(ip->i_mount)) {
1570                 dq = xfs_inode_dquot(ip, XFS_DQ_GROUP);
1571                 if (dq && xfs_dquot_lowsp(dq)) {
1572                         eofb.eof_gid = VFS_I(ip)->i_gid;
1573                         eofb.eof_flags |= XFS_EOF_FLAGS_GID;
1574                         scan = 1;
1575                 }
1576         }
1577 
1578         if (scan)
1579                 execute(ip->i_mount, &eofb);
1580 
1581         return scan;
1582 }
1583 
1584 int
1585 xfs_inode_free_quota_eofblocks(
1586         struct xfs_inode *ip)
1587 {
1588         return __xfs_inode_free_quota_eofblocks(ip, xfs_icache_free_eofblocks);
1589 }
1590 
1591 static inline unsigned long
1592 xfs_iflag_for_tag(
1593         int             tag)
1594 {
1595         switch (tag) {
1596         case XFS_ICI_EOFBLOCKS_TAG:
1597                 return XFS_IEOFBLOCKS;
1598         case XFS_ICI_COWBLOCKS_TAG:
1599                 return XFS_ICOWBLOCKS;
1600         default:
1601                 ASSERT(0);
1602                 return 0;
1603         }
1604 }
1605 
1606 static void
1607 __xfs_inode_set_blocks_tag(
1608         xfs_inode_t     *ip,
1609         void            (*execute)(struct xfs_mount *mp),
1610         void            (*set_tp)(struct xfs_mount *mp, xfs_agnumber_t agno,
1611                                   int error, unsigned long caller_ip),
1612         int             tag)
1613 {
1614         struct xfs_mount *mp = ip->i_mount;
1615         struct xfs_perag *pag;
1616         int tagged;
1617 
1618         /*
1619          * Don't bother locking the AG and looking up in the radix trees
1620          * if we already know that we have the tag set.
1621          */
1622         if (ip->i_flags & xfs_iflag_for_tag(tag))
1623                 return;
1624         spin_lock(&ip->i_flags_lock);
1625         ip->i_flags |= xfs_iflag_for_tag(tag);
1626         spin_unlock(&ip->i_flags_lock);
1627 
1628         pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino));
1629         spin_lock(&pag->pag_ici_lock);
1630 
1631         tagged = radix_tree_tagged(&pag->pag_ici_root, tag);
1632         radix_tree_tag_set(&pag->pag_ici_root,
1633                            XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino), tag);
1634         if (!tagged) {
1635                 /* propagate the eofblocks tag up into the perag radix tree */
1636                 spin_lock(&ip->i_mount->m_perag_lock);
1637                 radix_tree_tag_set(&ip->i_mount->m_perag_tree,
1638                                    XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino),
1639                                    tag);
1640                 spin_unlock(&ip->i_mount->m_perag_lock);
1641 
1642                 /* kick off background trimming */
1643                 execute(ip->i_mount);
1644 
1645                 set_tp(ip->i_mount, pag->pag_agno, -1, _RET_IP_);
1646         }
1647 
1648         spin_unlock(&pag->pag_ici_lock);
1649         xfs_perag_put(pag);
1650 }
1651 
1652 void
1653 xfs_inode_set_eofblocks_tag(
1654         xfs_inode_t     *ip)
1655 {
1656         trace_xfs_inode_set_eofblocks_tag(ip);
1657         return __xfs_inode_set_blocks_tag(ip, xfs_queue_eofblocks,
1658                         trace_xfs_perag_set_eofblocks,
1659                         XFS_ICI_EOFBLOCKS_TAG);
1660 }
1661 
1662 static void
1663 __xfs_inode_clear_blocks_tag(
1664         xfs_inode_t     *ip,
1665         void            (*clear_tp)(struct xfs_mount *mp, xfs_agnumber_t agno,
1666                                     int error, unsigned long caller_ip),
1667         int             tag)
1668 {
1669         struct xfs_mount *mp = ip->i_mount;
1670         struct xfs_perag *pag;
1671 
1672         spin_lock(&ip->i_flags_lock);
1673         ip->i_flags &= ~xfs_iflag_for_tag(tag);
1674         spin_unlock(&ip->i_flags_lock);
1675 
1676         pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino));
1677         spin_lock(&pag->pag_ici_lock);
1678 
1679         radix_tree_tag_clear(&pag->pag_ici_root,
1680                              XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino), tag);
1681         if (!radix_tree_tagged(&pag->pag_ici_root, tag)) {
1682                 /* clear the eofblocks tag from the perag radix tree */
1683                 spin_lock(&ip->i_mount->m_perag_lock);
1684                 radix_tree_tag_clear(&ip->i_mount->m_perag_tree,
1685                                      XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino),
1686                                      tag);
1687                 spin_unlock(&ip->i_mount->m_perag_lock);
1688                 clear_tp(ip->i_mount, pag->pag_agno, -1, _RET_IP_);
1689         }
1690 
1691         spin_unlock(&pag->pag_ici_lock);
1692         xfs_perag_put(pag);
1693 }
1694 
1695 void
1696 xfs_inode_clear_eofblocks_tag(
1697         xfs_inode_t     *ip)
1698 {
1699         trace_xfs_inode_clear_eofblocks_tag(ip);
1700         return __xfs_inode_clear_blocks_tag(ip,
1701                         trace_xfs_perag_clear_eofblocks, XFS_ICI_EOFBLOCKS_TAG);
1702 }
1703 
1704 /*
1705  * Set ourselves up to free CoW blocks from this file.  If it's already clean
1706  * then we can bail out quickly, but otherwise we must back off if the file
1707  * is undergoing some kind of write.
1708  */
1709 static bool
1710 xfs_prep_free_cowblocks(
1711         struct xfs_inode        *ip,
1712         struct xfs_ifork        *ifp)
1713 {
1714         /*
1715          * Just clear the tag if we have an empty cow fork or none at all. It's
1716          * possible the inode was fully unshared since it was originally tagged.
1717          */
1718         if (!xfs_is_reflink_inode(ip) || !ifp->if_bytes) {
1719                 trace_xfs_inode_free_cowblocks_invalid(ip);
1720                 xfs_inode_clear_cowblocks_tag(ip);
1721                 return false;
1722         }
1723 
1724         /*
1725          * If the mapping is dirty or under writeback we cannot touch the
1726          * CoW fork.  Leave it alone if we're in the midst of a directio.
1727          */
1728         if ((VFS_I(ip)->i_state & I_DIRTY_PAGES) ||
1729             mapping_tagged(VFS_I(ip)->i_mapping, PAGECACHE_TAG_DIRTY) ||
1730             mapping_tagged(VFS_I(ip)->i_mapping, PAGECACHE_TAG_WRITEBACK) ||
1731             atomic_read(&VFS_I(ip)->i_dio_count))
1732                 return false;
1733 
1734         return true;
1735 }
1736 
1737 /*
1738  * Automatic CoW Reservation Freeing
1739  *
1740  * These functions automatically garbage collect leftover CoW reservations
1741  * that were made on behalf of a cowextsize hint when we start to run out
1742  * of quota or when the reservations sit around for too long.  If the file
1743  * has dirty pages or is undergoing writeback, its CoW reservations will
1744  * be retained.
1745  *
1746  * The actual garbage collection piggybacks off the same code that runs
1747  * the speculative EOF preallocation garbage collector.
1748  */
1749 STATIC int
1750 xfs_inode_free_cowblocks(
1751         struct xfs_inode        *ip,
1752         int                     flags,
1753         void                    *args)
1754 {
1755         struct xfs_eofblocks    *eofb = args;
1756         struct xfs_ifork        *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK);
1757         int                     match;
1758         int                     ret = 0;
1759 
1760         if (!xfs_prep_free_cowblocks(ip, ifp))
1761                 return 0;
1762 
1763         if (eofb) {
1764                 if (eofb->eof_flags & XFS_EOF_FLAGS_UNION)
1765                         match = xfs_inode_match_id_union(ip, eofb);
1766                 else
1767                         match = xfs_inode_match_id(ip, eofb);
1768                 if (!match)
1769                         return 0;
1770 
1771                 /* skip the inode if the file size is too small */
1772                 if (eofb->eof_flags & XFS_EOF_FLAGS_MINFILESIZE &&
1773                     XFS_ISIZE(ip) < eofb->eof_min_file_size)
1774                         return 0;
1775         }
1776 
1777         /* Free the CoW blocks */
1778         xfs_ilock(ip, XFS_IOLOCK_EXCL);
1779         xfs_ilock(ip, XFS_MMAPLOCK_EXCL);
1780 
1781         /*
1782          * Check again, nobody else should be able to dirty blocks or change
1783          * the reflink iflag now that we have the first two locks held.
1784          */
1785         if (xfs_prep_free_cowblocks(ip, ifp))
1786                 ret = xfs_reflink_cancel_cow_range(ip, 0, NULLFILEOFF, false);
1787 
1788         xfs_iunlock(ip, XFS_MMAPLOCK_EXCL);
1789         xfs_iunlock(ip, XFS_IOLOCK_EXCL);
1790 
1791         return ret;
1792 }
1793 
1794 int
1795 xfs_icache_free_cowblocks(
1796         struct xfs_mount        *mp,
1797         struct xfs_eofblocks    *eofb)
1798 {
1799         return __xfs_icache_free_eofblocks(mp, eofb, xfs_inode_free_cowblocks,
1800                         XFS_ICI_COWBLOCKS_TAG);
1801 }
1802 
1803 int
1804 xfs_inode_free_quota_cowblocks(
1805         struct xfs_inode *ip)
1806 {
1807         return __xfs_inode_free_quota_eofblocks(ip, xfs_icache_free_cowblocks);
1808 }
1809 
1810 void
1811 xfs_inode_set_cowblocks_tag(
1812         xfs_inode_t     *ip)
1813 {
1814         trace_xfs_inode_set_cowblocks_tag(ip);
1815         return __xfs_inode_set_blocks_tag(ip, xfs_queue_cowblocks,
1816                         trace_xfs_perag_set_cowblocks,
1817                         XFS_ICI_COWBLOCKS_TAG);
1818 }
1819 
1820 void
1821 xfs_inode_clear_cowblocks_tag(
1822         xfs_inode_t     *ip)
1823 {
1824         trace_xfs_inode_clear_cowblocks_tag(ip);
1825         return __xfs_inode_clear_blocks_tag(ip,
1826                         trace_xfs_perag_clear_cowblocks, XFS_ICI_COWBLOCKS_TAG);
1827 }
1828 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp