~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/fs/fscache/page.c

Version: ~ [ linux-5.2-rc1 ] ~ [ linux-5.1.2 ] ~ [ linux-5.0.16 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.43 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.119 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.176 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.179 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.139 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.67 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /* Cache page management and data I/O routines
  2  *
  3  * Copyright (C) 2004-2008 Red Hat, Inc. All Rights Reserved.
  4  * Written by David Howells (dhowells@redhat.com)
  5  *
  6  * This program is free software; you can redistribute it and/or
  7  * modify it under the terms of the GNU General Public License
  8  * as published by the Free Software Foundation; either version
  9  * 2 of the License, or (at your option) any later version.
 10  */
 11 
 12 #define FSCACHE_DEBUG_LEVEL PAGE
 13 #include <linux/module.h>
 14 #include <linux/fscache-cache.h>
 15 #include <linux/buffer_head.h>
 16 #include <linux/pagevec.h>
 17 #include <linux/slab.h>
 18 #include "internal.h"
 19 
 20 /*
 21  * check to see if a page is being written to the cache
 22  */
 23 bool __fscache_check_page_write(struct fscache_cookie *cookie, struct page *page)
 24 {
 25         void *val;
 26 
 27         rcu_read_lock();
 28         val = radix_tree_lookup(&cookie->stores, page->index);
 29         rcu_read_unlock();
 30 
 31         return val != NULL;
 32 }
 33 EXPORT_SYMBOL(__fscache_check_page_write);
 34 
 35 /*
 36  * wait for a page to finish being written to the cache
 37  */
 38 void __fscache_wait_on_page_write(struct fscache_cookie *cookie, struct page *page)
 39 {
 40         wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0);
 41 
 42         wait_event(*wq, !__fscache_check_page_write(cookie, page));
 43 }
 44 EXPORT_SYMBOL(__fscache_wait_on_page_write);
 45 
 46 /*
 47  * decide whether a page can be released, possibly by cancelling a store to it
 48  * - we're allowed to sleep if __GFP_WAIT is flagged
 49  */
 50 bool __fscache_maybe_release_page(struct fscache_cookie *cookie,
 51                                   struct page *page,
 52                                   gfp_t gfp)
 53 {
 54         struct page *xpage;
 55         void *val;
 56 
 57         _enter("%p,%p,%x", cookie, page, gfp);
 58 
 59         rcu_read_lock();
 60         val = radix_tree_lookup(&cookie->stores, page->index);
 61         if (!val) {
 62                 rcu_read_unlock();
 63                 fscache_stat(&fscache_n_store_vmscan_not_storing);
 64                 __fscache_uncache_page(cookie, page);
 65                 return true;
 66         }
 67 
 68         /* see if the page is actually undergoing storage - if so we can't get
 69          * rid of it till the cache has finished with it */
 70         if (radix_tree_tag_get(&cookie->stores, page->index,
 71                                FSCACHE_COOKIE_STORING_TAG)) {
 72                 rcu_read_unlock();
 73                 goto page_busy;
 74         }
 75 
 76         /* the page is pending storage, so we attempt to cancel the store and
 77          * discard the store request so that the page can be reclaimed */
 78         spin_lock(&cookie->stores_lock);
 79         rcu_read_unlock();
 80 
 81         if (radix_tree_tag_get(&cookie->stores, page->index,
 82                                FSCACHE_COOKIE_STORING_TAG)) {
 83                 /* the page started to undergo storage whilst we were looking,
 84                  * so now we can only wait or return */
 85                 spin_unlock(&cookie->stores_lock);
 86                 goto page_busy;
 87         }
 88 
 89         xpage = radix_tree_delete(&cookie->stores, page->index);
 90         spin_unlock(&cookie->stores_lock);
 91 
 92         if (xpage) {
 93                 fscache_stat(&fscache_n_store_vmscan_cancelled);
 94                 fscache_stat(&fscache_n_store_radix_deletes);
 95                 ASSERTCMP(xpage, ==, page);
 96         } else {
 97                 fscache_stat(&fscache_n_store_vmscan_gone);
 98         }
 99 
100         wake_up_bit(&cookie->flags, 0);
101         if (xpage)
102                 page_cache_release(xpage);
103         __fscache_uncache_page(cookie, page);
104         return true;
105 
106 page_busy:
107         /* we might want to wait here, but that could deadlock the allocator as
108          * the work threads writing to the cache may all end up sleeping
109          * on memory allocation */
110         fscache_stat(&fscache_n_store_vmscan_busy);
111         return false;
112 }
113 EXPORT_SYMBOL(__fscache_maybe_release_page);
114 
115 /*
116  * note that a page has finished being written to the cache
117  */
118 static void fscache_end_page_write(struct fscache_object *object,
119                                    struct page *page)
120 {
121         struct fscache_cookie *cookie;
122         struct page *xpage = NULL;
123 
124         spin_lock(&object->lock);
125         cookie = object->cookie;
126         if (cookie) {
127                 /* delete the page from the tree if it is now no longer
128                  * pending */
129                 spin_lock(&cookie->stores_lock);
130                 radix_tree_tag_clear(&cookie->stores, page->index,
131                                      FSCACHE_COOKIE_STORING_TAG);
132                 if (!radix_tree_tag_get(&cookie->stores, page->index,
133                                         FSCACHE_COOKIE_PENDING_TAG)) {
134                         fscache_stat(&fscache_n_store_radix_deletes);
135                         xpage = radix_tree_delete(&cookie->stores, page->index);
136                 }
137                 spin_unlock(&cookie->stores_lock);
138                 wake_up_bit(&cookie->flags, 0);
139         }
140         spin_unlock(&object->lock);
141         if (xpage)
142                 page_cache_release(xpage);
143 }
144 
145 /*
146  * actually apply the changed attributes to a cache object
147  */
148 static void fscache_attr_changed_op(struct fscache_operation *op)
149 {
150         struct fscache_object *object = op->object;
151         int ret;
152 
153         _enter("{OBJ%x OP%x}", object->debug_id, op->debug_id);
154 
155         fscache_stat(&fscache_n_attr_changed_calls);
156 
157         if (fscache_object_is_active(object)) {
158                 fscache_set_op_state(op, "CallFS");
159                 fscache_stat(&fscache_n_cop_attr_changed);
160                 ret = object->cache->ops->attr_changed(object);
161                 fscache_stat_d(&fscache_n_cop_attr_changed);
162                 fscache_set_op_state(op, "Done");
163                 if (ret < 0)
164                         fscache_abort_object(object);
165         }
166 
167         _leave("");
168 }
169 
170 /*
171  * notification that the attributes on an object have changed
172  */
173 int __fscache_attr_changed(struct fscache_cookie *cookie)
174 {
175         struct fscache_operation *op;
176         struct fscache_object *object;
177 
178         _enter("%p", cookie);
179 
180         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
181 
182         fscache_stat(&fscache_n_attr_changed);
183 
184         op = kzalloc(sizeof(*op), GFP_KERNEL);
185         if (!op) {
186                 fscache_stat(&fscache_n_attr_changed_nomem);
187                 _leave(" = -ENOMEM");
188                 return -ENOMEM;
189         }
190 
191         fscache_operation_init(op, fscache_attr_changed_op, NULL);
192         op->flags = FSCACHE_OP_ASYNC | (1 << FSCACHE_OP_EXCLUSIVE);
193         fscache_set_op_name(op, "Attr");
194 
195         spin_lock(&cookie->lock);
196 
197         if (hlist_empty(&cookie->backing_objects))
198                 goto nobufs;
199         object = hlist_entry(cookie->backing_objects.first,
200                              struct fscache_object, cookie_link);
201 
202         if (fscache_submit_exclusive_op(object, op) < 0)
203                 goto nobufs;
204         spin_unlock(&cookie->lock);
205         fscache_stat(&fscache_n_attr_changed_ok);
206         fscache_put_operation(op);
207         _leave(" = 0");
208         return 0;
209 
210 nobufs:
211         spin_unlock(&cookie->lock);
212         kfree(op);
213         fscache_stat(&fscache_n_attr_changed_nobufs);
214         _leave(" = %d", -ENOBUFS);
215         return -ENOBUFS;
216 }
217 EXPORT_SYMBOL(__fscache_attr_changed);
218 
219 /*
220  * release a retrieval op reference
221  */
222 static void fscache_release_retrieval_op(struct fscache_operation *_op)
223 {
224         struct fscache_retrieval *op =
225                 container_of(_op, struct fscache_retrieval, op);
226 
227         _enter("{OP%x}", op->op.debug_id);
228 
229         fscache_hist(fscache_retrieval_histogram, op->start_time);
230         if (op->context)
231                 fscache_put_context(op->op.object->cookie, op->context);
232 
233         _leave("");
234 }
235 
236 /*
237  * allocate a retrieval op
238  */
239 static struct fscache_retrieval *fscache_alloc_retrieval(
240         struct address_space *mapping,
241         fscache_rw_complete_t end_io_func,
242         void *context)
243 {
244         struct fscache_retrieval *op;
245 
246         /* allocate a retrieval operation and attempt to submit it */
247         op = kzalloc(sizeof(*op), GFP_NOIO);
248         if (!op) {
249                 fscache_stat(&fscache_n_retrievals_nomem);
250                 return NULL;
251         }
252 
253         fscache_operation_init(&op->op, NULL, fscache_release_retrieval_op);
254         op->op.flags    = FSCACHE_OP_MYTHREAD | (1 << FSCACHE_OP_WAITING);
255         op->mapping     = mapping;
256         op->end_io_func = end_io_func;
257         op->context     = context;
258         op->start_time  = jiffies;
259         INIT_LIST_HEAD(&op->to_do);
260         fscache_set_op_name(&op->op, "Retr");
261         return op;
262 }
263 
264 /*
265  * wait for a deferred lookup to complete
266  */
267 static int fscache_wait_for_deferred_lookup(struct fscache_cookie *cookie)
268 {
269         unsigned long jif;
270 
271         _enter("");
272 
273         if (!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags)) {
274                 _leave(" = 0 [imm]");
275                 return 0;
276         }
277 
278         fscache_stat(&fscache_n_retrievals_wait);
279 
280         jif = jiffies;
281         if (wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,
282                         fscache_wait_bit_interruptible,
283                         TASK_INTERRUPTIBLE) != 0) {
284                 fscache_stat(&fscache_n_retrievals_intr);
285                 _leave(" = -ERESTARTSYS");
286                 return -ERESTARTSYS;
287         }
288 
289         ASSERT(!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags));
290 
291         smp_rmb();
292         fscache_hist(fscache_retrieval_delay_histogram, jif);
293         _leave(" = 0 [dly]");
294         return 0;
295 }
296 
297 /*
298  * wait for an object to become active (or dead)
299  */
300 static int fscache_wait_for_retrieval_activation(struct fscache_object *object,
301                                                  struct fscache_retrieval *op,
302                                                  atomic_t *stat_op_waits,
303                                                  atomic_t *stat_object_dead)
304 {
305         int ret;
306 
307         if (!test_bit(FSCACHE_OP_WAITING, &op->op.flags))
308                 goto check_if_dead;
309 
310         _debug(">>> WT");
311         fscache_stat(stat_op_waits);
312         if (wait_on_bit(&op->op.flags, FSCACHE_OP_WAITING,
313                         fscache_wait_bit_interruptible,
314                         TASK_INTERRUPTIBLE) < 0) {
315                 ret = fscache_cancel_op(&op->op);
316                 if (ret == 0)
317                         return -ERESTARTSYS;
318 
319                 /* it's been removed from the pending queue by another party,
320                  * so we should get to run shortly */
321                 wait_on_bit(&op->op.flags, FSCACHE_OP_WAITING,
322                             fscache_wait_bit, TASK_UNINTERRUPTIBLE);
323         }
324         _debug("<<< GO");
325 
326 check_if_dead:
327         if (unlikely(fscache_object_is_dead(object))) {
328                 fscache_stat(stat_object_dead);
329                 return -ENOBUFS;
330         }
331         return 0;
332 }
333 
334 /*
335  * read a page from the cache or allocate a block in which to store it
336  * - we return:
337  *   -ENOMEM    - out of memory, nothing done
338  *   -ERESTARTSYS - interrupted
339  *   -ENOBUFS   - no backing object available in which to cache the block
340  *   -ENODATA   - no data available in the backing object for this block
341  *   0          - dispatched a read - it'll call end_io_func() when finished
342  */
343 int __fscache_read_or_alloc_page(struct fscache_cookie *cookie,
344                                  struct page *page,
345                                  fscache_rw_complete_t end_io_func,
346                                  void *context,
347                                  gfp_t gfp)
348 {
349         struct fscache_retrieval *op;
350         struct fscache_object *object;
351         int ret;
352 
353         _enter("%p,%p,,,", cookie, page);
354 
355         fscache_stat(&fscache_n_retrievals);
356 
357         if (hlist_empty(&cookie->backing_objects))
358                 goto nobufs;
359 
360         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
361         ASSERTCMP(page, !=, NULL);
362 
363         if (fscache_wait_for_deferred_lookup(cookie) < 0)
364                 return -ERESTARTSYS;
365 
366         op = fscache_alloc_retrieval(page->mapping, end_io_func, context);
367         if (!op) {
368                 _leave(" = -ENOMEM");
369                 return -ENOMEM;
370         }
371         fscache_set_op_name(&op->op, "RetrRA1");
372 
373         spin_lock(&cookie->lock);
374 
375         if (hlist_empty(&cookie->backing_objects))
376                 goto nobufs_unlock;
377         object = hlist_entry(cookie->backing_objects.first,
378                              struct fscache_object, cookie_link);
379 
380         ASSERTCMP(object->state, >, FSCACHE_OBJECT_LOOKING_UP);
381 
382         atomic_inc(&object->n_reads);
383         set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags);
384 
385         if (fscache_submit_op(object, &op->op) < 0)
386                 goto nobufs_unlock;
387         spin_unlock(&cookie->lock);
388 
389         fscache_stat(&fscache_n_retrieval_ops);
390 
391         /* pin the netfs read context in case we need to do the actual netfs
392          * read because we've encountered a cache read failure */
393         fscache_get_context(object->cookie, op->context);
394 
395         /* we wait for the operation to become active, and then process it
396          * *here*, in this thread, and not in the thread pool */
397         ret = fscache_wait_for_retrieval_activation(
398                 object, op,
399                 __fscache_stat(&fscache_n_retrieval_op_waits),
400                 __fscache_stat(&fscache_n_retrievals_object_dead));
401         if (ret < 0)
402                 goto error;
403 
404         /* ask the cache to honour the operation */
405         if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {
406                 fscache_stat(&fscache_n_cop_allocate_page);
407                 ret = object->cache->ops->allocate_page(op, page, gfp);
408                 fscache_stat_d(&fscache_n_cop_allocate_page);
409                 if (ret == 0)
410                         ret = -ENODATA;
411         } else {
412                 fscache_stat(&fscache_n_cop_read_or_alloc_page);
413                 ret = object->cache->ops->read_or_alloc_page(op, page, gfp);
414                 fscache_stat_d(&fscache_n_cop_read_or_alloc_page);
415         }
416 
417 error:
418         if (ret == -ENOMEM)
419                 fscache_stat(&fscache_n_retrievals_nomem);
420         else if (ret == -ERESTARTSYS)
421                 fscache_stat(&fscache_n_retrievals_intr);
422         else if (ret == -ENODATA)
423                 fscache_stat(&fscache_n_retrievals_nodata);
424         else if (ret < 0)
425                 fscache_stat(&fscache_n_retrievals_nobufs);
426         else
427                 fscache_stat(&fscache_n_retrievals_ok);
428 
429         fscache_put_retrieval(op);
430         _leave(" = %d", ret);
431         return ret;
432 
433 nobufs_unlock:
434         spin_unlock(&cookie->lock);
435         kfree(op);
436 nobufs:
437         fscache_stat(&fscache_n_retrievals_nobufs);
438         _leave(" = -ENOBUFS");
439         return -ENOBUFS;
440 }
441 EXPORT_SYMBOL(__fscache_read_or_alloc_page);
442 
443 /*
444  * read a list of page from the cache or allocate a block in which to store
445  * them
446  * - we return:
447  *   -ENOMEM    - out of memory, some pages may be being read
448  *   -ERESTARTSYS - interrupted, some pages may be being read
449  *   -ENOBUFS   - no backing object or space available in which to cache any
450  *                pages not being read
451  *   -ENODATA   - no data available in the backing object for some or all of
452  *                the pages
453  *   0          - dispatched a read on all pages
454  *
455  * end_io_func() will be called for each page read from the cache as it is
456  * finishes being read
457  *
458  * any pages for which a read is dispatched will be removed from pages and
459  * nr_pages
460  */
461 int __fscache_read_or_alloc_pages(struct fscache_cookie *cookie,
462                                   struct address_space *mapping,
463                                   struct list_head *pages,
464                                   unsigned *nr_pages,
465                                   fscache_rw_complete_t end_io_func,
466                                   void *context,
467                                   gfp_t gfp)
468 {
469         struct fscache_retrieval *op;
470         struct fscache_object *object;
471         int ret;
472 
473         _enter("%p,,%d,,,", cookie, *nr_pages);
474 
475         fscache_stat(&fscache_n_retrievals);
476 
477         if (hlist_empty(&cookie->backing_objects))
478                 goto nobufs;
479 
480         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
481         ASSERTCMP(*nr_pages, >, 0);
482         ASSERT(!list_empty(pages));
483 
484         if (fscache_wait_for_deferred_lookup(cookie) < 0)
485                 return -ERESTARTSYS;
486 
487         op = fscache_alloc_retrieval(mapping, end_io_func, context);
488         if (!op)
489                 return -ENOMEM;
490         fscache_set_op_name(&op->op, "RetrRAN");
491 
492         spin_lock(&cookie->lock);
493 
494         if (hlist_empty(&cookie->backing_objects))
495                 goto nobufs_unlock;
496         object = hlist_entry(cookie->backing_objects.first,
497                              struct fscache_object, cookie_link);
498 
499         atomic_inc(&object->n_reads);
500         set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags);
501 
502         if (fscache_submit_op(object, &op->op) < 0)
503                 goto nobufs_unlock;
504         spin_unlock(&cookie->lock);
505 
506         fscache_stat(&fscache_n_retrieval_ops);
507 
508         /* pin the netfs read context in case we need to do the actual netfs
509          * read because we've encountered a cache read failure */
510         fscache_get_context(object->cookie, op->context);
511 
512         /* we wait for the operation to become active, and then process it
513          * *here*, in this thread, and not in the thread pool */
514         ret = fscache_wait_for_retrieval_activation(
515                 object, op,
516                 __fscache_stat(&fscache_n_retrieval_op_waits),
517                 __fscache_stat(&fscache_n_retrievals_object_dead));
518         if (ret < 0)
519                 goto error;
520 
521         /* ask the cache to honour the operation */
522         if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {
523                 fscache_stat(&fscache_n_cop_allocate_pages);
524                 ret = object->cache->ops->allocate_pages(
525                         op, pages, nr_pages, gfp);
526                 fscache_stat_d(&fscache_n_cop_allocate_pages);
527         } else {
528                 fscache_stat(&fscache_n_cop_read_or_alloc_pages);
529                 ret = object->cache->ops->read_or_alloc_pages(
530                         op, pages, nr_pages, gfp);
531                 fscache_stat_d(&fscache_n_cop_read_or_alloc_pages);
532         }
533 
534 error:
535         if (ret == -ENOMEM)
536                 fscache_stat(&fscache_n_retrievals_nomem);
537         else if (ret == -ERESTARTSYS)
538                 fscache_stat(&fscache_n_retrievals_intr);
539         else if (ret == -ENODATA)
540                 fscache_stat(&fscache_n_retrievals_nodata);
541         else if (ret < 0)
542                 fscache_stat(&fscache_n_retrievals_nobufs);
543         else
544                 fscache_stat(&fscache_n_retrievals_ok);
545 
546         fscache_put_retrieval(op);
547         _leave(" = %d", ret);
548         return ret;
549 
550 nobufs_unlock:
551         spin_unlock(&cookie->lock);
552         kfree(op);
553 nobufs:
554         fscache_stat(&fscache_n_retrievals_nobufs);
555         _leave(" = -ENOBUFS");
556         return -ENOBUFS;
557 }
558 EXPORT_SYMBOL(__fscache_read_or_alloc_pages);
559 
560 /*
561  * allocate a block in the cache on which to store a page
562  * - we return:
563  *   -ENOMEM    - out of memory, nothing done
564  *   -ERESTARTSYS - interrupted
565  *   -ENOBUFS   - no backing object available in which to cache the block
566  *   0          - block allocated
567  */
568 int __fscache_alloc_page(struct fscache_cookie *cookie,
569                          struct page *page,
570                          gfp_t gfp)
571 {
572         struct fscache_retrieval *op;
573         struct fscache_object *object;
574         int ret;
575 
576         _enter("%p,%p,,,", cookie, page);
577 
578         fscache_stat(&fscache_n_allocs);
579 
580         if (hlist_empty(&cookie->backing_objects))
581                 goto nobufs;
582 
583         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
584         ASSERTCMP(page, !=, NULL);
585 
586         if (fscache_wait_for_deferred_lookup(cookie) < 0)
587                 return -ERESTARTSYS;
588 
589         op = fscache_alloc_retrieval(page->mapping, NULL, NULL);
590         if (!op)
591                 return -ENOMEM;
592         fscache_set_op_name(&op->op, "RetrAL1");
593 
594         spin_lock(&cookie->lock);
595 
596         if (hlist_empty(&cookie->backing_objects))
597                 goto nobufs_unlock;
598         object = hlist_entry(cookie->backing_objects.first,
599                              struct fscache_object, cookie_link);
600 
601         if (fscache_submit_op(object, &op->op) < 0)
602                 goto nobufs_unlock;
603         spin_unlock(&cookie->lock);
604 
605         fscache_stat(&fscache_n_alloc_ops);
606 
607         ret = fscache_wait_for_retrieval_activation(
608                 object, op,
609                 __fscache_stat(&fscache_n_alloc_op_waits),
610                 __fscache_stat(&fscache_n_allocs_object_dead));
611         if (ret < 0)
612                 goto error;
613 
614         /* ask the cache to honour the operation */
615         fscache_stat(&fscache_n_cop_allocate_page);
616         ret = object->cache->ops->allocate_page(op, page, gfp);
617         fscache_stat_d(&fscache_n_cop_allocate_page);
618 
619 error:
620         if (ret == -ERESTARTSYS)
621                 fscache_stat(&fscache_n_allocs_intr);
622         else if (ret < 0)
623                 fscache_stat(&fscache_n_allocs_nobufs);
624         else
625                 fscache_stat(&fscache_n_allocs_ok);
626 
627         fscache_put_retrieval(op);
628         _leave(" = %d", ret);
629         return ret;
630 
631 nobufs_unlock:
632         spin_unlock(&cookie->lock);
633         kfree(op);
634 nobufs:
635         fscache_stat(&fscache_n_allocs_nobufs);
636         _leave(" = -ENOBUFS");
637         return -ENOBUFS;
638 }
639 EXPORT_SYMBOL(__fscache_alloc_page);
640 
641 /*
642  * release a write op reference
643  */
644 static void fscache_release_write_op(struct fscache_operation *_op)
645 {
646         _enter("{OP%x}", _op->debug_id);
647 }
648 
649 /*
650  * perform the background storage of a page into the cache
651  */
652 static void fscache_write_op(struct fscache_operation *_op)
653 {
654         struct fscache_storage *op =
655                 container_of(_op, struct fscache_storage, op);
656         struct fscache_object *object = op->op.object;
657         struct fscache_cookie *cookie;
658         struct page *page;
659         unsigned n;
660         void *results[1];
661         int ret;
662 
663         _enter("{OP%x,%d}", op->op.debug_id, atomic_read(&op->op.usage));
664 
665         fscache_set_op_state(&op->op, "GetPage");
666 
667         spin_lock(&object->lock);
668         cookie = object->cookie;
669 
670         if (!fscache_object_is_active(object) || !cookie) {
671                 spin_unlock(&object->lock);
672                 _leave("");
673                 return;
674         }
675 
676         spin_lock(&cookie->stores_lock);
677 
678         fscache_stat(&fscache_n_store_calls);
679 
680         /* find a page to store */
681         page = NULL;
682         n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0, 1,
683                                        FSCACHE_COOKIE_PENDING_TAG);
684         if (n != 1)
685                 goto superseded;
686         page = results[0];
687         _debug("gang %d [%lx]", n, page->index);
688         if (page->index > op->store_limit) {
689                 fscache_stat(&fscache_n_store_pages_over_limit);
690                 goto superseded;
691         }
692 
693         radix_tree_tag_set(&cookie->stores, page->index,
694                            FSCACHE_COOKIE_STORING_TAG);
695         radix_tree_tag_clear(&cookie->stores, page->index,
696                              FSCACHE_COOKIE_PENDING_TAG);
697 
698         spin_unlock(&cookie->stores_lock);
699         spin_unlock(&object->lock);
700 
701         fscache_set_op_state(&op->op, "Store");
702         fscache_stat(&fscache_n_store_pages);
703         fscache_stat(&fscache_n_cop_write_page);
704         ret = object->cache->ops->write_page(op, page);
705         fscache_stat_d(&fscache_n_cop_write_page);
706         fscache_set_op_state(&op->op, "EndWrite");
707         fscache_end_page_write(object, page);
708         if (ret < 0) {
709                 fscache_set_op_state(&op->op, "Abort");
710                 fscache_abort_object(object);
711         } else {
712                 fscache_enqueue_operation(&op->op);
713         }
714 
715         _leave("");
716         return;
717 
718 superseded:
719         /* this writer is going away and there aren't any more things to
720          * write */
721         _debug("cease");
722         spin_unlock(&cookie->stores_lock);
723         clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);
724         spin_unlock(&object->lock);
725         _leave("");
726 }
727 
728 /*
729  * request a page be stored in the cache
730  * - returns:
731  *   -ENOMEM    - out of memory, nothing done
732  *   -ENOBUFS   - no backing object available in which to cache the page
733  *   0          - dispatched a write - it'll call end_io_func() when finished
734  *
735  * if the cookie still has a backing object at this point, that object can be
736  * in one of a few states with respect to storage processing:
737  *
738  *  (1) negative lookup, object not yet created (FSCACHE_COOKIE_CREATING is
739  *      set)
740  *
741  *      (a) no writes yet (set FSCACHE_COOKIE_PENDING_FILL and queue deferred
742  *          fill op)
743  *
744  *      (b) writes deferred till post-creation (mark page for writing and
745  *          return immediately)
746  *
747  *  (2) negative lookup, object created, initial fill being made from netfs
748  *      (FSCACHE_COOKIE_INITIAL_FILL is set)
749  *
750  *      (a) fill point not yet reached this page (mark page for writing and
751  *          return)
752  *
753  *      (b) fill point passed this page (queue op to store this page)
754  *
755  *  (3) object extant (queue op to store this page)
756  *
757  * any other state is invalid
758  */
759 int __fscache_write_page(struct fscache_cookie *cookie,
760                          struct page *page,
761                          gfp_t gfp)
762 {
763         struct fscache_storage *op;
764         struct fscache_object *object;
765         int ret;
766 
767         _enter("%p,%x,", cookie, (u32) page->flags);
768 
769         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
770         ASSERT(PageFsCache(page));
771 
772         fscache_stat(&fscache_n_stores);
773 
774         op = kzalloc(sizeof(*op), GFP_NOIO);
775         if (!op)
776                 goto nomem;
777 
778         fscache_operation_init(&op->op, fscache_write_op,
779                                fscache_release_write_op);
780         op->op.flags = FSCACHE_OP_ASYNC | (1 << FSCACHE_OP_WAITING);
781         fscache_set_op_name(&op->op, "Write1");
782 
783         ret = radix_tree_preload(gfp & ~__GFP_HIGHMEM);
784         if (ret < 0)
785                 goto nomem_free;
786 
787         ret = -ENOBUFS;
788         spin_lock(&cookie->lock);
789 
790         if (hlist_empty(&cookie->backing_objects))
791                 goto nobufs;
792         object = hlist_entry(cookie->backing_objects.first,
793                              struct fscache_object, cookie_link);
794         if (test_bit(FSCACHE_IOERROR, &object->cache->flags))
795                 goto nobufs;
796 
797         /* add the page to the pending-storage radix tree on the backing
798          * object */
799         spin_lock(&object->lock);
800         spin_lock(&cookie->stores_lock);
801 
802         _debug("store limit %llx", (unsigned long long) object->store_limit);
803 
804         ret = radix_tree_insert(&cookie->stores, page->index, page);
805         if (ret < 0) {
806                 if (ret == -EEXIST)
807                         goto already_queued;
808                 _debug("insert failed %d", ret);
809                 goto nobufs_unlock_obj;
810         }
811 
812         radix_tree_tag_set(&cookie->stores, page->index,
813                            FSCACHE_COOKIE_PENDING_TAG);
814         page_cache_get(page);
815 
816         /* we only want one writer at a time, but we do need to queue new
817          * writers after exclusive ops */
818         if (test_and_set_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags))
819                 goto already_pending;
820 
821         spin_unlock(&cookie->stores_lock);
822         spin_unlock(&object->lock);
823 
824         op->op.debug_id = atomic_inc_return(&fscache_op_debug_id);
825         op->store_limit = object->store_limit;
826 
827         if (fscache_submit_op(object, &op->op) < 0)
828                 goto submit_failed;
829 
830         spin_unlock(&cookie->lock);
831         radix_tree_preload_end();
832         fscache_stat(&fscache_n_store_ops);
833         fscache_stat(&fscache_n_stores_ok);
834 
835         /* the work queue now carries its own ref on the object */
836         fscache_put_operation(&op->op);
837         _leave(" = 0");
838         return 0;
839 
840 already_queued:
841         fscache_stat(&fscache_n_stores_again);
842 already_pending:
843         spin_unlock(&cookie->stores_lock);
844         spin_unlock(&object->lock);
845         spin_unlock(&cookie->lock);
846         radix_tree_preload_end();
847         kfree(op);
848         fscache_stat(&fscache_n_stores_ok);
849         _leave(" = 0");
850         return 0;
851 
852 submit_failed:
853         spin_lock(&cookie->stores_lock);
854         radix_tree_delete(&cookie->stores, page->index);
855         spin_unlock(&cookie->stores_lock);
856         page_cache_release(page);
857         ret = -ENOBUFS;
858         goto nobufs;
859 
860 nobufs_unlock_obj:
861         spin_unlock(&cookie->stores_lock);
862         spin_unlock(&object->lock);
863 nobufs:
864         spin_unlock(&cookie->lock);
865         radix_tree_preload_end();
866         kfree(op);
867         fscache_stat(&fscache_n_stores_nobufs);
868         _leave(" = -ENOBUFS");
869         return -ENOBUFS;
870 
871 nomem_free:
872         kfree(op);
873 nomem:
874         fscache_stat(&fscache_n_stores_oom);
875         _leave(" = -ENOMEM");
876         return -ENOMEM;
877 }
878 EXPORT_SYMBOL(__fscache_write_page);
879 
880 /*
881  * remove a page from the cache
882  */
883 void __fscache_uncache_page(struct fscache_cookie *cookie, struct page *page)
884 {
885         struct fscache_object *object;
886 
887         _enter(",%p", page);
888 
889         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
890         ASSERTCMP(page, !=, NULL);
891 
892         fscache_stat(&fscache_n_uncaches);
893 
894         /* cache withdrawal may beat us to it */
895         if (!PageFsCache(page))
896                 goto done;
897 
898         /* get the object */
899         spin_lock(&cookie->lock);
900 
901         if (hlist_empty(&cookie->backing_objects)) {
902                 ClearPageFsCache(page);
903                 goto done_unlock;
904         }
905 
906         object = hlist_entry(cookie->backing_objects.first,
907                              struct fscache_object, cookie_link);
908 
909         /* there might now be stuff on disk we could read */
910         clear_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
911 
912         /* only invoke the cache backend if we managed to mark the page
913          * uncached here; this deals with synchronisation vs withdrawal */
914         if (TestClearPageFsCache(page) &&
915             object->cache->ops->uncache_page) {
916                 /* the cache backend releases the cookie lock */
917                 fscache_stat(&fscache_n_cop_uncache_page);
918                 object->cache->ops->uncache_page(object, page);
919                 fscache_stat_d(&fscache_n_cop_uncache_page);
920                 goto done;
921         }
922 
923 done_unlock:
924         spin_unlock(&cookie->lock);
925 done:
926         _leave("");
927 }
928 EXPORT_SYMBOL(__fscache_uncache_page);
929 
930 /**
931  * fscache_mark_pages_cached - Mark pages as being cached
932  * @op: The retrieval op pages are being marked for
933  * @pagevec: The pages to be marked
934  *
935  * Mark a bunch of netfs pages as being cached.  After this is called,
936  * the netfs must call fscache_uncache_page() to remove the mark.
937  */
938 void fscache_mark_pages_cached(struct fscache_retrieval *op,
939                                struct pagevec *pagevec)
940 {
941         struct fscache_cookie *cookie = op->op.object->cookie;
942         unsigned long loop;
943 
944 #ifdef CONFIG_FSCACHE_STATS
945         atomic_add(pagevec->nr, &fscache_n_marks);
946 #endif
947 
948         for (loop = 0; loop < pagevec->nr; loop++) {
949                 struct page *page = pagevec->pages[loop];
950 
951                 _debug("- mark %p{%lx}", page, page->index);
952                 if (TestSetPageFsCache(page)) {
953                         static bool once_only;
954                         if (!once_only) {
955                                 once_only = true;
956                                 printk(KERN_WARNING "FS-Cache:"
957                                        " Cookie type %s marked page %lx"
958                                        " multiple times\n",
959                                        cookie->def->name, page->index);
960                         }
961                 }
962         }
963 
964         if (cookie->def->mark_pages_cached)
965                 cookie->def->mark_pages_cached(cookie->netfs_data,
966                                                op->mapping, pagevec);
967         pagevec_reinit(pagevec);
968 }
969 EXPORT_SYMBOL(fscache_mark_pages_cached);
970 
971 /*
972  * Uncache all the pages in an inode that are marked PG_fscache, assuming them
973  * to be associated with the given cookie.
974  */
975 void __fscache_uncache_all_inode_pages(struct fscache_cookie *cookie,
976                                        struct inode *inode)
977 {
978         struct address_space *mapping = inode->i_mapping;
979         struct pagevec pvec;
980         pgoff_t next;
981         int i;
982 
983         _enter("%p,%p", cookie, inode);
984 
985         if (!mapping || mapping->nrpages == 0) {
986                 _leave(" [no pages]");
987                 return;
988         }
989 
990         pagevec_init(&pvec, 0);
991         next = 0;
992         do {
993                 if (!pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE))
994                         break;
995                 for (i = 0; i < pagevec_count(&pvec); i++) {
996                         struct page *page = pvec.pages[i];
997                         next = page->index;
998                         if (PageFsCache(page)) {
999                                 __fscache_wait_on_page_write(cookie, page);
1000                                 __fscache_uncache_page(cookie, page);
1001                         }
1002                 }
1003                 pagevec_release(&pvec);
1004                 cond_resched();
1005         } while (++next);
1006 
1007         _leave("");
1008 }
1009 EXPORT_SYMBOL(__fscache_uncache_all_inode_pages);
1010 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp