~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/fs/btrfs/inode.c

Version: ~ [ linux-4.17-rc1 ] ~ [ linux-4.16.2 ] ~ [ linux-4.15.17 ] ~ [ linux-4.14.34 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.94 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.128 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.51 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.105 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.56 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.101 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.27.62 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 /*
  3  * Copyright (C) 2007 Oracle.  All rights reserved.
  4  */
  5 
  6 #include <linux/kernel.h>
  7 #include <linux/bio.h>
  8 #include <linux/buffer_head.h>
  9 #include <linux/file.h>
 10 #include <linux/fs.h>
 11 #include <linux/pagemap.h>
 12 #include <linux/highmem.h>
 13 #include <linux/time.h>
 14 #include <linux/init.h>
 15 #include <linux/string.h>
 16 #include <linux/backing-dev.h>
 17 #include <linux/mpage.h>
 18 #include <linux/swap.h>
 19 #include <linux/writeback.h>
 20 #include <linux/compat.h>
 21 #include <linux/bit_spinlock.h>
 22 #include <linux/xattr.h>
 23 #include <linux/posix_acl.h>
 24 #include <linux/falloc.h>
 25 #include <linux/slab.h>
 26 #include <linux/ratelimit.h>
 27 #include <linux/mount.h>
 28 #include <linux/btrfs.h>
 29 #include <linux/blkdev.h>
 30 #include <linux/posix_acl_xattr.h>
 31 #include <linux/uio.h>
 32 #include <linux/magic.h>
 33 #include <linux/iversion.h>
 34 #include "ctree.h"
 35 #include "disk-io.h"
 36 #include "transaction.h"
 37 #include "btrfs_inode.h"
 38 #include "print-tree.h"
 39 #include "ordered-data.h"
 40 #include "xattr.h"
 41 #include "tree-log.h"
 42 #include "volumes.h"
 43 #include "compression.h"
 44 #include "locking.h"
 45 #include "free-space-cache.h"
 46 #include "inode-map.h"
 47 #include "backref.h"
 48 #include "props.h"
 49 #include "qgroup.h"
 50 #include "dedupe.h"
 51 
 52 struct btrfs_iget_args {
 53         struct btrfs_key *location;
 54         struct btrfs_root *root;
 55 };
 56 
 57 struct btrfs_dio_data {
 58         u64 reserve;
 59         u64 unsubmitted_oe_range_start;
 60         u64 unsubmitted_oe_range_end;
 61         int overwrite;
 62 };
 63 
 64 static const struct inode_operations btrfs_dir_inode_operations;
 65 static const struct inode_operations btrfs_symlink_inode_operations;
 66 static const struct inode_operations btrfs_dir_ro_inode_operations;
 67 static const struct inode_operations btrfs_special_inode_operations;
 68 static const struct inode_operations btrfs_file_inode_operations;
 69 static const struct address_space_operations btrfs_aops;
 70 static const struct address_space_operations btrfs_symlink_aops;
 71 static const struct file_operations btrfs_dir_file_operations;
 72 static const struct extent_io_ops btrfs_extent_io_ops;
 73 
 74 static struct kmem_cache *btrfs_inode_cachep;
 75 struct kmem_cache *btrfs_trans_handle_cachep;
 76 struct kmem_cache *btrfs_path_cachep;
 77 struct kmem_cache *btrfs_free_space_cachep;
 78 
 79 #define S_SHIFT 12
 80 static const unsigned char btrfs_type_by_mode[S_IFMT >> S_SHIFT] = {
 81         [S_IFREG >> S_SHIFT]    = BTRFS_FT_REG_FILE,
 82         [S_IFDIR >> S_SHIFT]    = BTRFS_FT_DIR,
 83         [S_IFCHR >> S_SHIFT]    = BTRFS_FT_CHRDEV,
 84         [S_IFBLK >> S_SHIFT]    = BTRFS_FT_BLKDEV,
 85         [S_IFIFO >> S_SHIFT]    = BTRFS_FT_FIFO,
 86         [S_IFSOCK >> S_SHIFT]   = BTRFS_FT_SOCK,
 87         [S_IFLNK >> S_SHIFT]    = BTRFS_FT_SYMLINK,
 88 };
 89 
 90 static int btrfs_setsize(struct inode *inode, struct iattr *attr);
 91 static int btrfs_truncate(struct inode *inode, bool skip_writeback);
 92 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent);
 93 static noinline int cow_file_range(struct inode *inode,
 94                                    struct page *locked_page,
 95                                    u64 start, u64 end, u64 delalloc_end,
 96                                    int *page_started, unsigned long *nr_written,
 97                                    int unlock, struct btrfs_dedupe_hash *hash);
 98 static struct extent_map *create_io_em(struct inode *inode, u64 start, u64 len,
 99                                        u64 orig_start, u64 block_start,
100                                        u64 block_len, u64 orig_block_len,
101                                        u64 ram_bytes, int compress_type,
102                                        int type);
103 
104 static void __endio_write_update_ordered(struct inode *inode,
105                                          const u64 offset, const u64 bytes,
106                                          const bool uptodate);
107 
108 /*
109  * Cleanup all submitted ordered extents in specified range to handle errors
110  * from the fill_dellaloc() callback.
111  *
112  * NOTE: caller must ensure that when an error happens, it can not call
113  * extent_clear_unlock_delalloc() to clear both the bits EXTENT_DO_ACCOUNTING
114  * and EXTENT_DELALLOC simultaneously, because that causes the reserved metadata
115  * to be released, which we want to happen only when finishing the ordered
116  * extent (btrfs_finish_ordered_io()). Also note that the caller of the
117  * fill_delalloc() callback already does proper cleanup for the first page of
118  * the range, that is, it invokes the callback writepage_end_io_hook() for the
119  * range of the first page.
120  */
121 static inline void btrfs_cleanup_ordered_extents(struct inode *inode,
122                                                  const u64 offset,
123                                                  const u64 bytes)
124 {
125         unsigned long index = offset >> PAGE_SHIFT;
126         unsigned long end_index = (offset + bytes - 1) >> PAGE_SHIFT;
127         struct page *page;
128 
129         while (index <= end_index) {
130                 page = find_get_page(inode->i_mapping, index);
131                 index++;
132                 if (!page)
133                         continue;
134                 ClearPagePrivate2(page);
135                 put_page(page);
136         }
137         return __endio_write_update_ordered(inode, offset + PAGE_SIZE,
138                                             bytes - PAGE_SIZE, false);
139 }
140 
141 static int btrfs_dirty_inode(struct inode *inode);
142 
143 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
144 void btrfs_test_inode_set_ops(struct inode *inode)
145 {
146         BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
147 }
148 #endif
149 
150 static int btrfs_init_inode_security(struct btrfs_trans_handle *trans,
151                                      struct inode *inode,  struct inode *dir,
152                                      const struct qstr *qstr)
153 {
154         int err;
155 
156         err = btrfs_init_acl(trans, inode, dir);
157         if (!err)
158                 err = btrfs_xattr_security_init(trans, inode, dir, qstr);
159         return err;
160 }
161 
162 /*
163  * this does all the hard work for inserting an inline extent into
164  * the btree.  The caller should have done a btrfs_drop_extents so that
165  * no overlapping inline items exist in the btree
166  */
167 static int insert_inline_extent(struct btrfs_trans_handle *trans,
168                                 struct btrfs_path *path, int extent_inserted,
169                                 struct btrfs_root *root, struct inode *inode,
170                                 u64 start, size_t size, size_t compressed_size,
171                                 int compress_type,
172                                 struct page **compressed_pages)
173 {
174         struct extent_buffer *leaf;
175         struct page *page = NULL;
176         char *kaddr;
177         unsigned long ptr;
178         struct btrfs_file_extent_item *ei;
179         int ret;
180         size_t cur_size = size;
181         unsigned long offset;
182 
183         if (compressed_size && compressed_pages)
184                 cur_size = compressed_size;
185 
186         inode_add_bytes(inode, size);
187 
188         if (!extent_inserted) {
189                 struct btrfs_key key;
190                 size_t datasize;
191 
192                 key.objectid = btrfs_ino(BTRFS_I(inode));
193                 key.offset = start;
194                 key.type = BTRFS_EXTENT_DATA_KEY;
195 
196                 datasize = btrfs_file_extent_calc_inline_size(cur_size);
197                 path->leave_spinning = 1;
198                 ret = btrfs_insert_empty_item(trans, root, path, &key,
199                                               datasize);
200                 if (ret)
201                         goto fail;
202         }
203         leaf = path->nodes[0];
204         ei = btrfs_item_ptr(leaf, path->slots[0],
205                             struct btrfs_file_extent_item);
206         btrfs_set_file_extent_generation(leaf, ei, trans->transid);
207         btrfs_set_file_extent_type(leaf, ei, BTRFS_FILE_EXTENT_INLINE);
208         btrfs_set_file_extent_encryption(leaf, ei, 0);
209         btrfs_set_file_extent_other_encoding(leaf, ei, 0);
210         btrfs_set_file_extent_ram_bytes(leaf, ei, size);
211         ptr = btrfs_file_extent_inline_start(ei);
212 
213         if (compress_type != BTRFS_COMPRESS_NONE) {
214                 struct page *cpage;
215                 int i = 0;
216                 while (compressed_size > 0) {
217                         cpage = compressed_pages[i];
218                         cur_size = min_t(unsigned long, compressed_size,
219                                        PAGE_SIZE);
220 
221                         kaddr = kmap_atomic(cpage);
222                         write_extent_buffer(leaf, kaddr, ptr, cur_size);
223                         kunmap_atomic(kaddr);
224 
225                         i++;
226                         ptr += cur_size;
227                         compressed_size -= cur_size;
228                 }
229                 btrfs_set_file_extent_compression(leaf, ei,
230                                                   compress_type);
231         } else {
232                 page = find_get_page(inode->i_mapping,
233                                      start >> PAGE_SHIFT);
234                 btrfs_set_file_extent_compression(leaf, ei, 0);
235                 kaddr = kmap_atomic(page);
236                 offset = start & (PAGE_SIZE - 1);
237                 write_extent_buffer(leaf, kaddr + offset, ptr, size);
238                 kunmap_atomic(kaddr);
239                 put_page(page);
240         }
241         btrfs_mark_buffer_dirty(leaf);
242         btrfs_release_path(path);
243 
244         /*
245          * we're an inline extent, so nobody can
246          * extend the file past i_size without locking
247          * a page we already have locked.
248          *
249          * We must do any isize and inode updates
250          * before we unlock the pages.  Otherwise we
251          * could end up racing with unlink.
252          */
253         BTRFS_I(inode)->disk_i_size = inode->i_size;
254         ret = btrfs_update_inode(trans, root, inode);
255 
256 fail:
257         return ret;
258 }
259 
260 
261 /*
262  * conditionally insert an inline extent into the file.  This
263  * does the checks required to make sure the data is small enough
264  * to fit as an inline extent.
265  */
266 static noinline int cow_file_range_inline(struct inode *inode, u64 start,
267                                           u64 end, size_t compressed_size,
268                                           int compress_type,
269                                           struct page **compressed_pages)
270 {
271         struct btrfs_root *root = BTRFS_I(inode)->root;
272         struct btrfs_fs_info *fs_info = root->fs_info;
273         struct btrfs_trans_handle *trans;
274         u64 isize = i_size_read(inode);
275         u64 actual_end = min(end + 1, isize);
276         u64 inline_len = actual_end - start;
277         u64 aligned_end = ALIGN(end, fs_info->sectorsize);
278         u64 data_len = inline_len;
279         int ret;
280         struct btrfs_path *path;
281         int extent_inserted = 0;
282         u32 extent_item_size;
283 
284         if (compressed_size)
285                 data_len = compressed_size;
286 
287         if (start > 0 ||
288             actual_end > fs_info->sectorsize ||
289             data_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info) ||
290             (!compressed_size &&
291             (actual_end & (fs_info->sectorsize - 1)) == 0) ||
292             end + 1 < isize ||
293             data_len > fs_info->max_inline) {
294                 return 1;
295         }
296 
297         path = btrfs_alloc_path();
298         if (!path)
299                 return -ENOMEM;
300 
301         trans = btrfs_join_transaction(root);
302         if (IS_ERR(trans)) {
303                 btrfs_free_path(path);
304                 return PTR_ERR(trans);
305         }
306         trans->block_rsv = &BTRFS_I(inode)->block_rsv;
307 
308         if (compressed_size && compressed_pages)
309                 extent_item_size = btrfs_file_extent_calc_inline_size(
310                    compressed_size);
311         else
312                 extent_item_size = btrfs_file_extent_calc_inline_size(
313                     inline_len);
314 
315         ret = __btrfs_drop_extents(trans, root, inode, path,
316                                    start, aligned_end, NULL,
317                                    1, 1, extent_item_size, &extent_inserted);
318         if (ret) {
319                 btrfs_abort_transaction(trans, ret);
320                 goto out;
321         }
322 
323         if (isize > actual_end)
324                 inline_len = min_t(u64, isize, actual_end);
325         ret = insert_inline_extent(trans, path, extent_inserted,
326                                    root, inode, start,
327                                    inline_len, compressed_size,
328                                    compress_type, compressed_pages);
329         if (ret && ret != -ENOSPC) {
330                 btrfs_abort_transaction(trans, ret);
331                 goto out;
332         } else if (ret == -ENOSPC) {
333                 ret = 1;
334                 goto out;
335         }
336 
337         set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags);
338         btrfs_drop_extent_cache(BTRFS_I(inode), start, aligned_end - 1, 0);
339 out:
340         /*
341          * Don't forget to free the reserved space, as for inlined extent
342          * it won't count as data extent, free them directly here.
343          * And at reserve time, it's always aligned to page size, so
344          * just free one page here.
345          */
346         btrfs_qgroup_free_data(inode, NULL, 0, PAGE_SIZE);
347         btrfs_free_path(path);
348         btrfs_end_transaction(trans);
349         return ret;
350 }
351 
352 struct async_extent {
353         u64 start;
354         u64 ram_size;
355         u64 compressed_size;
356         struct page **pages;
357         unsigned long nr_pages;
358         int compress_type;
359         struct list_head list;
360 };
361 
362 struct async_cow {
363         struct inode *inode;
364         struct btrfs_root *root;
365         struct page *locked_page;
366         u64 start;
367         u64 end;
368         unsigned int write_flags;
369         struct list_head extents;
370         struct btrfs_work work;
371 };
372 
373 static noinline int add_async_extent(struct async_cow *cow,
374                                      u64 start, u64 ram_size,
375                                      u64 compressed_size,
376                                      struct page **pages,
377                                      unsigned long nr_pages,
378                                      int compress_type)
379 {
380         struct async_extent *async_extent;
381 
382         async_extent = kmalloc(sizeof(*async_extent), GFP_NOFS);
383         BUG_ON(!async_extent); /* -ENOMEM */
384         async_extent->start = start;
385         async_extent->ram_size = ram_size;
386         async_extent->compressed_size = compressed_size;
387         async_extent->pages = pages;
388         async_extent->nr_pages = nr_pages;
389         async_extent->compress_type = compress_type;
390         list_add_tail(&async_extent->list, &cow->extents);
391         return 0;
392 }
393 
394 static inline int inode_need_compress(struct inode *inode, u64 start, u64 end)
395 {
396         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
397 
398         /* force compress */
399         if (btrfs_test_opt(fs_info, FORCE_COMPRESS))
400                 return 1;
401         /* defrag ioctl */
402         if (BTRFS_I(inode)->defrag_compress)
403                 return 1;
404         /* bad compression ratios */
405         if (BTRFS_I(inode)->flags & BTRFS_INODE_NOCOMPRESS)
406                 return 0;
407         if (btrfs_test_opt(fs_info, COMPRESS) ||
408             BTRFS_I(inode)->flags & BTRFS_INODE_COMPRESS ||
409             BTRFS_I(inode)->prop_compress)
410                 return btrfs_compress_heuristic(inode, start, end);
411         return 0;
412 }
413 
414 static inline void inode_should_defrag(struct btrfs_inode *inode,
415                 u64 start, u64 end, u64 num_bytes, u64 small_write)
416 {
417         /* If this is a small write inside eof, kick off a defrag */
418         if (num_bytes < small_write &&
419             (start > 0 || end + 1 < inode->disk_i_size))
420                 btrfs_add_inode_defrag(NULL, inode);
421 }
422 
423 /*
424  * we create compressed extents in two phases.  The first
425  * phase compresses a range of pages that have already been
426  * locked (both pages and state bits are locked).
427  *
428  * This is done inside an ordered work queue, and the compression
429  * is spread across many cpus.  The actual IO submission is step
430  * two, and the ordered work queue takes care of making sure that
431  * happens in the same order things were put onto the queue by
432  * writepages and friends.
433  *
434  * If this code finds it can't get good compression, it puts an
435  * entry onto the work queue to write the uncompressed bytes.  This
436  * makes sure that both compressed inodes and uncompressed inodes
437  * are written in the same order that the flusher thread sent them
438  * down.
439  */
440 static noinline void compress_file_range(struct inode *inode,
441                                         struct page *locked_page,
442                                         u64 start, u64 end,
443                                         struct async_cow *async_cow,
444                                         int *num_added)
445 {
446         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
447         u64 blocksize = fs_info->sectorsize;
448         u64 actual_end;
449         u64 isize = i_size_read(inode);
450         int ret = 0;
451         struct page **pages = NULL;
452         unsigned long nr_pages;
453         unsigned long total_compressed = 0;
454         unsigned long total_in = 0;
455         int i;
456         int will_compress;
457         int compress_type = fs_info->compress_type;
458         int redirty = 0;
459 
460         inode_should_defrag(BTRFS_I(inode), start, end, end - start + 1,
461                         SZ_16K);
462 
463         actual_end = min_t(u64, isize, end + 1);
464 again:
465         will_compress = 0;
466         nr_pages = (end >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1;
467         BUILD_BUG_ON((BTRFS_MAX_COMPRESSED % PAGE_SIZE) != 0);
468         nr_pages = min_t(unsigned long, nr_pages,
469                         BTRFS_MAX_COMPRESSED / PAGE_SIZE);
470 
471         /*
472          * we don't want to send crud past the end of i_size through
473          * compression, that's just a waste of CPU time.  So, if the
474          * end of the file is before the start of our current
475          * requested range of bytes, we bail out to the uncompressed
476          * cleanup code that can deal with all of this.
477          *
478          * It isn't really the fastest way to fix things, but this is a
479          * very uncommon corner.
480          */
481         if (actual_end <= start)
482                 goto cleanup_and_bail_uncompressed;
483 
484         total_compressed = actual_end - start;
485 
486         /*
487          * skip compression for a small file range(<=blocksize) that
488          * isn't an inline extent, since it doesn't save disk space at all.
489          */
490         if (total_compressed <= blocksize &&
491            (start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size))
492                 goto cleanup_and_bail_uncompressed;
493 
494         total_compressed = min_t(unsigned long, total_compressed,
495                         BTRFS_MAX_UNCOMPRESSED);
496         total_in = 0;
497         ret = 0;
498 
499         /*
500          * we do compression for mount -o compress and when the
501          * inode has not been flagged as nocompress.  This flag can
502          * change at any time if we discover bad compression ratios.
503          */
504         if (inode_need_compress(inode, start, end)) {
505                 WARN_ON(pages);
506                 pages = kcalloc(nr_pages, sizeof(struct page *), GFP_NOFS);
507                 if (!pages) {
508                         /* just bail out to the uncompressed code */
509                         goto cont;
510                 }
511 
512                 if (BTRFS_I(inode)->defrag_compress)
513                         compress_type = BTRFS_I(inode)->defrag_compress;
514                 else if (BTRFS_I(inode)->prop_compress)
515                         compress_type = BTRFS_I(inode)->prop_compress;
516 
517                 /*
518                  * we need to call clear_page_dirty_for_io on each
519                  * page in the range.  Otherwise applications with the file
520                  * mmap'd can wander in and change the page contents while
521                  * we are compressing them.
522                  *
523                  * If the compression fails for any reason, we set the pages
524                  * dirty again later on.
525                  *
526                  * Note that the remaining part is redirtied, the start pointer
527                  * has moved, the end is the original one.
528                  */
529                 if (!redirty) {
530                         extent_range_clear_dirty_for_io(inode, start, end);
531                         redirty = 1;
532                 }
533 
534                 /* Compression level is applied here and only here */
535                 ret = btrfs_compress_pages(
536                         compress_type | (fs_info->compress_level << 4),
537                                            inode->i_mapping, start,
538                                            pages,
539                                            &nr_pages,
540                                            &total_in,
541                                            &total_compressed);
542 
543                 if (!ret) {
544                         unsigned long offset = total_compressed &
545                                 (PAGE_SIZE - 1);
546                         struct page *page = pages[nr_pages - 1];
547                         char *kaddr;
548 
549                         /* zero the tail end of the last page, we might be
550                          * sending it down to disk
551                          */
552                         if (offset) {
553                                 kaddr = kmap_atomic(page);
554                                 memset(kaddr + offset, 0,
555                                        PAGE_SIZE - offset);
556                                 kunmap_atomic(kaddr);
557                         }
558                         will_compress = 1;
559                 }
560         }
561 cont:
562         if (start == 0) {
563                 /* lets try to make an inline extent */
564                 if (ret || total_in < actual_end) {
565                         /* we didn't compress the entire range, try
566                          * to make an uncompressed inline extent.
567                          */
568                         ret = cow_file_range_inline(inode, start, end, 0,
569                                                     BTRFS_COMPRESS_NONE, NULL);
570                 } else {
571                         /* try making a compressed inline extent */
572                         ret = cow_file_range_inline(inode, start, end,
573                                                     total_compressed,
574                                                     compress_type, pages);
575                 }
576                 if (ret <= 0) {
577                         unsigned long clear_flags = EXTENT_DELALLOC |
578                                 EXTENT_DELALLOC_NEW | EXTENT_DEFRAG |
579                                 EXTENT_DO_ACCOUNTING;
580                         unsigned long page_error_op;
581 
582                         page_error_op = ret < 0 ? PAGE_SET_ERROR : 0;
583 
584                         /*
585                          * inline extent creation worked or returned error,
586                          * we don't need to create any more async work items.
587                          * Unlock and free up our temp pages.
588                          *
589                          * We use DO_ACCOUNTING here because we need the
590                          * delalloc_release_metadata to be done _after_ we drop
591                          * our outstanding extent for clearing delalloc for this
592                          * range.
593                          */
594                         extent_clear_unlock_delalloc(inode, start, end, end,
595                                                      NULL, clear_flags,
596                                                      PAGE_UNLOCK |
597                                                      PAGE_CLEAR_DIRTY |
598                                                      PAGE_SET_WRITEBACK |
599                                                      page_error_op |
600                                                      PAGE_END_WRITEBACK);
601                         goto free_pages_out;
602                 }
603         }
604 
605         if (will_compress) {
606                 /*
607                  * we aren't doing an inline extent round the compressed size
608                  * up to a block size boundary so the allocator does sane
609                  * things
610                  */
611                 total_compressed = ALIGN(total_compressed, blocksize);
612 
613                 /*
614                  * one last check to make sure the compression is really a
615                  * win, compare the page count read with the blocks on disk,
616                  * compression must free at least one sector size
617                  */
618                 total_in = ALIGN(total_in, PAGE_SIZE);
619                 if (total_compressed + blocksize <= total_in) {
620                         *num_added += 1;
621 
622                         /*
623                          * The async work queues will take care of doing actual
624                          * allocation on disk for these compressed pages, and
625                          * will submit them to the elevator.
626                          */
627                         add_async_extent(async_cow, start, total_in,
628                                         total_compressed, pages, nr_pages,
629                                         compress_type);
630 
631                         if (start + total_in < end) {
632                                 start += total_in;
633                                 pages = NULL;
634                                 cond_resched();
635                                 goto again;
636                         }
637                         return;
638                 }
639         }
640         if (pages) {
641                 /*
642                  * the compression code ran but failed to make things smaller,
643                  * free any pages it allocated and our page pointer array
644                  */
645                 for (i = 0; i < nr_pages; i++) {
646                         WARN_ON(pages[i]->mapping);
647                         put_page(pages[i]);
648                 }
649                 kfree(pages);
650                 pages = NULL;
651                 total_compressed = 0;
652                 nr_pages = 0;
653 
654                 /* flag the file so we don't compress in the future */
655                 if (!btrfs_test_opt(fs_info, FORCE_COMPRESS) &&
656                     !(BTRFS_I(inode)->prop_compress)) {
657                         BTRFS_I(inode)->flags |= BTRFS_INODE_NOCOMPRESS;
658                 }
659         }
660 cleanup_and_bail_uncompressed:
661         /*
662          * No compression, but we still need to write the pages in the file
663          * we've been given so far.  redirty the locked page if it corresponds
664          * to our extent and set things up for the async work queue to run
665          * cow_file_range to do the normal delalloc dance.
666          */
667         if (page_offset(locked_page) >= start &&
668             page_offset(locked_page) <= end)
669                 __set_page_dirty_nobuffers(locked_page);
670                 /* unlocked later on in the async handlers */
671 
672         if (redirty)
673                 extent_range_redirty_for_io(inode, start, end);
674         add_async_extent(async_cow, start, end - start + 1, 0, NULL, 0,
675                          BTRFS_COMPRESS_NONE);
676         *num_added += 1;
677 
678         return;
679 
680 free_pages_out:
681         for (i = 0; i < nr_pages; i++) {
682                 WARN_ON(pages[i]->mapping);
683                 put_page(pages[i]);
684         }
685         kfree(pages);
686 }
687 
688 static void free_async_extent_pages(struct async_extent *async_extent)
689 {
690         int i;
691 
692         if (!async_extent->pages)
693                 return;
694 
695         for (i = 0; i < async_extent->nr_pages; i++) {
696                 WARN_ON(async_extent->pages[i]->mapping);
697                 put_page(async_extent->pages[i]);
698         }
699         kfree(async_extent->pages);
700         async_extent->nr_pages = 0;
701         async_extent->pages = NULL;
702 }
703 
704 /*
705  * phase two of compressed writeback.  This is the ordered portion
706  * of the code, which only gets called in the order the work was
707  * queued.  We walk all the async extents created by compress_file_range
708  * and send them down to the disk.
709  */
710 static noinline void submit_compressed_extents(struct inode *inode,
711                                               struct async_cow *async_cow)
712 {
713         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
714         struct async_extent *async_extent;
715         u64 alloc_hint = 0;
716         struct btrfs_key ins;
717         struct extent_map *em;
718         struct btrfs_root *root = BTRFS_I(inode)->root;
719         struct extent_io_tree *io_tree;
720         int ret = 0;
721 
722 again:
723         while (!list_empty(&async_cow->extents)) {
724                 async_extent = list_entry(async_cow->extents.next,
725                                           struct async_extent, list);
726                 list_del(&async_extent->list);
727 
728                 io_tree = &BTRFS_I(inode)->io_tree;
729 
730 retry:
731                 /* did the compression code fall back to uncompressed IO? */
732                 if (!async_extent->pages) {
733                         int page_started = 0;
734                         unsigned long nr_written = 0;
735 
736                         lock_extent(io_tree, async_extent->start,
737                                          async_extent->start +
738                                          async_extent->ram_size - 1);
739 
740                         /* allocate blocks */
741                         ret = cow_file_range(inode, async_cow->locked_page,
742                                              async_extent->start,
743                                              async_extent->start +
744                                              async_extent->ram_size - 1,
745                                              async_extent->start +
746                                              async_extent->ram_size - 1,
747                                              &page_started, &nr_written, 0,
748                                              NULL);
749 
750                         /* JDM XXX */
751 
752                         /*
753                          * if page_started, cow_file_range inserted an
754                          * inline extent and took care of all the unlocking
755                          * and IO for us.  Otherwise, we need to submit
756                          * all those pages down to the drive.
757                          */
758                         if (!page_started && !ret)
759                                 extent_write_locked_range(inode,
760                                                   async_extent->start,
761                                                   async_extent->start +
762                                                   async_extent->ram_size - 1,
763                                                   WB_SYNC_ALL);
764                         else if (ret)
765                                 unlock_page(async_cow->locked_page);
766                         kfree(async_extent);
767                         cond_resched();
768                         continue;
769                 }
770 
771                 lock_extent(io_tree, async_extent->start,
772                             async_extent->start + async_extent->ram_size - 1);
773 
774                 ret = btrfs_reserve_extent(root, async_extent->ram_size,
775                                            async_extent->compressed_size,
776                                            async_extent->compressed_size,
777                                            0, alloc_hint, &ins, 1, 1);
778                 if (ret) {
779                         free_async_extent_pages(async_extent);
780 
781                         if (ret == -ENOSPC) {
782                                 unlock_extent(io_tree, async_extent->start,
783                                               async_extent->start +
784                                               async_extent->ram_size - 1);
785 
786                                 /*
787                                  * we need to redirty the pages if we decide to
788                                  * fallback to uncompressed IO, otherwise we
789                                  * will not submit these pages down to lower
790                                  * layers.
791                                  */
792                                 extent_range_redirty_for_io(inode,
793                                                 async_extent->start,
794                                                 async_extent->start +
795                                                 async_extent->ram_size - 1);
796 
797                                 goto retry;
798                         }
799                         goto out_free;
800                 }
801                 /*
802                  * here we're doing allocation and writeback of the
803                  * compressed pages
804                  */
805                 em = create_io_em(inode, async_extent->start,
806                                   async_extent->ram_size, /* len */
807                                   async_extent->start, /* orig_start */
808                                   ins.objectid, /* block_start */
809                                   ins.offset, /* block_len */
810                                   ins.offset, /* orig_block_len */
811                                   async_extent->ram_size, /* ram_bytes */
812                                   async_extent->compress_type,
813                                   BTRFS_ORDERED_COMPRESSED);
814                 if (IS_ERR(em))
815                         /* ret value is not necessary due to void function */
816                         goto out_free_reserve;
817                 free_extent_map(em);
818 
819                 ret = btrfs_add_ordered_extent_compress(inode,
820                                                 async_extent->start,
821                                                 ins.objectid,
822                                                 async_extent->ram_size,
823                                                 ins.offset,
824                                                 BTRFS_ORDERED_COMPRESSED,
825                                                 async_extent->compress_type);
826                 if (ret) {
827                         btrfs_drop_extent_cache(BTRFS_I(inode),
828                                                 async_extent->start,
829                                                 async_extent->start +
830                                                 async_extent->ram_size - 1, 0);
831                         goto out_free_reserve;
832                 }
833                 btrfs_dec_block_group_reservations(fs_info, ins.objectid);
834 
835                 /*
836                  * clear dirty, set writeback and unlock the pages.
837                  */
838                 extent_clear_unlock_delalloc(inode, async_extent->start,
839                                 async_extent->start +
840                                 async_extent->ram_size - 1,
841                                 async_extent->start +
842                                 async_extent->ram_size - 1,
843                                 NULL, EXTENT_LOCKED | EXTENT_DELALLOC,
844                                 PAGE_UNLOCK | PAGE_CLEAR_DIRTY |
845                                 PAGE_SET_WRITEBACK);
846                 if (btrfs_submit_compressed_write(inode,
847                                     async_extent->start,
848                                     async_extent->ram_size,
849                                     ins.objectid,
850                                     ins.offset, async_extent->pages,
851                                     async_extent->nr_pages,
852                                     async_cow->write_flags)) {
853                         struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
854                         struct page *p = async_extent->pages[0];
855                         const u64 start = async_extent->start;
856                         const u64 end = start + async_extent->ram_size - 1;
857 
858                         p->mapping = inode->i_mapping;
859                         tree->ops->writepage_end_io_hook(p, start, end,
860                                                          NULL, 0);
861                         p->mapping = NULL;
862                         extent_clear_unlock_delalloc(inode, start, end, end,
863                                                      NULL, 0,
864                                                      PAGE_END_WRITEBACK |
865                                                      PAGE_SET_ERROR);
866                         free_async_extent_pages(async_extent);
867                 }
868                 alloc_hint = ins.objectid + ins.offset;
869                 kfree(async_extent);
870                 cond_resched();
871         }
872         return;
873 out_free_reserve:
874         btrfs_dec_block_group_reservations(fs_info, ins.objectid);
875         btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1);
876 out_free:
877         extent_clear_unlock_delalloc(inode, async_extent->start,
878                                      async_extent->start +
879                                      async_extent->ram_size - 1,
880                                      async_extent->start +
881                                      async_extent->ram_size - 1,
882                                      NULL, EXTENT_LOCKED | EXTENT_DELALLOC |
883                                      EXTENT_DELALLOC_NEW |
884                                      EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING,
885                                      PAGE_UNLOCK | PAGE_CLEAR_DIRTY |
886                                      PAGE_SET_WRITEBACK | PAGE_END_WRITEBACK |
887                                      PAGE_SET_ERROR);
888         free_async_extent_pages(async_extent);
889         kfree(async_extent);
890         goto again;
891 }
892 
893 static u64 get_extent_allocation_hint(struct inode *inode, u64 start,
894                                       u64 num_bytes)
895 {
896         struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
897         struct extent_map *em;
898         u64 alloc_hint = 0;
899 
900         read_lock(&em_tree->lock);
901         em = search_extent_mapping(em_tree, start, num_bytes);
902         if (em) {
903                 /*
904                  * if block start isn't an actual block number then find the
905                  * first block in this inode and use that as a hint.  If that
906                  * block is also bogus then just don't worry about it.
907                  */
908                 if (em->block_start >= EXTENT_MAP_LAST_BYTE) {
909                         free_extent_map(em);
910                         em = search_extent_mapping(em_tree, 0, 0);
911                         if (em && em->block_start < EXTENT_MAP_LAST_BYTE)
912                                 alloc_hint = em->block_start;
913                         if (em)
914                                 free_extent_map(em);
915                 } else {
916                         alloc_hint = em->block_start;
917                         free_extent_map(em);
918                 }
919         }
920         read_unlock(&em_tree->lock);
921 
922         return alloc_hint;
923 }
924 
925 /*
926  * when extent_io.c finds a delayed allocation range in the file,
927  * the call backs end up in this code.  The basic idea is to
928  * allocate extents on disk for the range, and create ordered data structs
929  * in ram to track those extents.
930  *
931  * locked_page is the page that writepage had locked already.  We use
932  * it to make sure we don't do extra locks or unlocks.
933  *
934  * *page_started is set to one if we unlock locked_page and do everything
935  * required to start IO on it.  It may be clean and already done with
936  * IO when we return.
937  */
938 static noinline int cow_file_range(struct inode *inode,
939                                    struct page *locked_page,
940                                    u64 start, u64 end, u64 delalloc_end,
941                                    int *page_started, unsigned long *nr_written,
942                                    int unlock, struct btrfs_dedupe_hash *hash)
943 {
944         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
945         struct btrfs_root *root = BTRFS_I(inode)->root;
946         u64 alloc_hint = 0;
947         u64 num_bytes;
948         unsigned long ram_size;
949         u64 cur_alloc_size = 0;
950         u64 blocksize = fs_info->sectorsize;
951         struct btrfs_key ins;
952         struct extent_map *em;
953         unsigned clear_bits;
954         unsigned long page_ops;
955         bool extent_reserved = false;
956         int ret = 0;
957 
958         if (btrfs_is_free_space_inode(BTRFS_I(inode))) {
959                 WARN_ON_ONCE(1);
960                 ret = -EINVAL;
961                 goto out_unlock;
962         }
963 
964         num_bytes = ALIGN(end - start + 1, blocksize);
965         num_bytes = max(blocksize,  num_bytes);
966         ASSERT(num_bytes <= btrfs_super_total_bytes(fs_info->super_copy));
967 
968         inode_should_defrag(BTRFS_I(inode), start, end, num_bytes, SZ_64K);
969 
970         if (start == 0) {
971                 /* lets try to make an inline extent */
972                 ret = cow_file_range_inline(inode, start, end, 0,
973                                             BTRFS_COMPRESS_NONE, NULL);
974                 if (ret == 0) {
975                         /*
976                          * We use DO_ACCOUNTING here because we need the
977                          * delalloc_release_metadata to be run _after_ we drop
978                          * our outstanding extent for clearing delalloc for this
979                          * range.
980                          */
981                         extent_clear_unlock_delalloc(inode, start, end,
982                                      delalloc_end, NULL,
983                                      EXTENT_LOCKED | EXTENT_DELALLOC |
984                                      EXTENT_DELALLOC_NEW | EXTENT_DEFRAG |
985                                      EXTENT_DO_ACCOUNTING, PAGE_UNLOCK |
986                                      PAGE_CLEAR_DIRTY | PAGE_SET_WRITEBACK |
987                                      PAGE_END_WRITEBACK);
988                         *nr_written = *nr_written +
989                              (end - start + PAGE_SIZE) / PAGE_SIZE;
990                         *page_started = 1;
991                         goto out;
992                 } else if (ret < 0) {
993                         goto out_unlock;
994                 }
995         }
996 
997         alloc_hint = get_extent_allocation_hint(inode, start, num_bytes);
998         btrfs_drop_extent_cache(BTRFS_I(inode), start,
999                         start + num_bytes - 1, 0);
1000 
1001         while (num_bytes > 0) {
1002                 cur_alloc_size = num_bytes;
1003                 ret = btrfs_reserve_extent(root, cur_alloc_size, cur_alloc_size,
1004                                            fs_info->sectorsize, 0, alloc_hint,
1005                                            &ins, 1, 1);
1006                 if (ret < 0)
1007                         goto out_unlock;
1008                 cur_alloc_size = ins.offset;
1009                 extent_reserved = true;
1010 
1011                 ram_size = ins.offset;
1012                 em = create_io_em(inode, start, ins.offset, /* len */
1013                                   start, /* orig_start */
1014                                   ins.objectid, /* block_start */
1015                                   ins.offset, /* block_len */
1016                                   ins.offset, /* orig_block_len */
1017                                   ram_size, /* ram_bytes */
1018                                   BTRFS_COMPRESS_NONE, /* compress_type */
1019                                   BTRFS_ORDERED_REGULAR /* type */);
1020                 if (IS_ERR(em))
1021                         goto out_reserve;
1022                 free_extent_map(em);
1023 
1024                 ret = btrfs_add_ordered_extent(inode, start, ins.objectid,
1025                                                ram_size, cur_alloc_size, 0);
1026                 if (ret)
1027                         goto out_drop_extent_cache;
1028 
1029                 if (root->root_key.objectid ==
1030                     BTRFS_DATA_RELOC_TREE_OBJECTID) {
1031                         ret = btrfs_reloc_clone_csums(inode, start,
1032                                                       cur_alloc_size);
1033                         /*
1034                          * Only drop cache here, and process as normal.
1035                          *
1036                          * We must not allow extent_clear_unlock_delalloc()
1037                          * at out_unlock label to free meta of this ordered
1038                          * extent, as its meta should be freed by
1039                          * btrfs_finish_ordered_io().
1040                          *
1041                          * So we must continue until @start is increased to
1042                          * skip current ordered extent.
1043                          */
1044                         if (ret)
1045                                 btrfs_drop_extent_cache(BTRFS_I(inode), start,
1046                                                 start + ram_size - 1, 0);
1047                 }
1048 
1049                 btrfs_dec_block_group_reservations(fs_info, ins.objectid);
1050 
1051                 /* we're not doing compressed IO, don't unlock the first
1052                  * page (which the caller expects to stay locked), don't
1053                  * clear any dirty bits and don't set any writeback bits
1054                  *
1055                  * Do set the Private2 bit so we know this page was properly
1056                  * setup for writepage
1057                  */
1058                 page_ops = unlock ? PAGE_UNLOCK : 0;
1059                 page_ops |= PAGE_SET_PRIVATE2;
1060 
1061                 extent_clear_unlock_delalloc(inode, start,
1062                                              start + ram_size - 1,
1063                                              delalloc_end, locked_page,
1064                                              EXTENT_LOCKED | EXTENT_DELALLOC,
1065                                              page_ops);
1066                 if (num_bytes < cur_alloc_size)
1067                         num_bytes = 0;
1068                 else
1069                         num_bytes -= cur_alloc_size;
1070                 alloc_hint = ins.objectid + ins.offset;
1071                 start += cur_alloc_size;
1072                 extent_reserved = false;
1073 
1074                 /*
1075                  * btrfs_reloc_clone_csums() error, since start is increased
1076                  * extent_clear_unlock_delalloc() at out_unlock label won't
1077                  * free metadata of current ordered extent, we're OK to exit.
1078                  */
1079                 if (ret)
1080                         goto out_unlock;
1081         }
1082 out:
1083         return ret;
1084 
1085 out_drop_extent_cache:
1086         btrfs_drop_extent_cache(BTRFS_I(inode), start, start + ram_size - 1, 0);
1087 out_reserve:
1088         btrfs_dec_block_group_reservations(fs_info, ins.objectid);
1089         btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1);
1090 out_unlock:
1091         clear_bits = EXTENT_LOCKED | EXTENT_DELALLOC | EXTENT_DELALLOC_NEW |
1092                 EXTENT_DEFRAG | EXTENT_CLEAR_META_RESV;
1093         page_ops = PAGE_UNLOCK | PAGE_CLEAR_DIRTY | PAGE_SET_WRITEBACK |
1094                 PAGE_END_WRITEBACK;
1095         /*
1096          * If we reserved an extent for our delalloc range (or a subrange) and
1097          * failed to create the respective ordered extent, then it means that
1098          * when we reserved the extent we decremented the extent's size from
1099          * the data space_info's bytes_may_use counter and incremented the
1100          * space_info's bytes_reserved counter by the same amount. We must make
1101          * sure extent_clear_unlock_delalloc() does not try to decrement again
1102          * the data space_info's bytes_may_use counter, therefore we do not pass
1103          * it the flag EXTENT_CLEAR_DATA_RESV.
1104          */
1105         if (extent_reserved) {
1106                 extent_clear_unlock_delalloc(inode, start,
1107                                              start + cur_alloc_size,
1108                                              start + cur_alloc_size,
1109                                              locked_page,
1110                                              clear_bits,
1111                                              page_ops);
1112                 start += cur_alloc_size;
1113                 if (start >= end)
1114                         goto out;
1115         }
1116         extent_clear_unlock_delalloc(inode, start, end, delalloc_end,
1117                                      locked_page,
1118                                      clear_bits | EXTENT_CLEAR_DATA_RESV,
1119                                      page_ops);
1120         goto out;
1121 }
1122 
1123 /*
1124  * work queue call back to started compression on a file and pages
1125  */
1126 static noinline void async_cow_start(struct btrfs_work *work)
1127 {
1128         struct async_cow *async_cow;
1129         int num_added = 0;
1130         async_cow = container_of(work, struct async_cow, work);
1131 
1132         compress_file_range(async_cow->inode, async_cow->locked_page,
1133                             async_cow->start, async_cow->end, async_cow,
1134                             &num_added);
1135         if (num_added == 0) {
1136                 btrfs_add_delayed_iput(async_cow->inode);
1137                 async_cow->inode = NULL;
1138         }
1139 }
1140 
1141 /*
1142  * work queue call back to submit previously compressed pages
1143  */
1144 static noinline void async_cow_submit(struct btrfs_work *work)
1145 {
1146         struct btrfs_fs_info *fs_info;
1147         struct async_cow *async_cow;
1148         struct btrfs_root *root;
1149         unsigned long nr_pages;
1150 
1151         async_cow = container_of(work, struct async_cow, work);
1152 
1153         root = async_cow->root;
1154         fs_info = root->fs_info;
1155         nr_pages = (async_cow->end - async_cow->start + PAGE_SIZE) >>
1156                 PAGE_SHIFT;
1157 
1158         /*
1159          * atomic_sub_return implies a barrier for waitqueue_active
1160          */
1161         if (atomic_sub_return(nr_pages, &fs_info->async_delalloc_pages) <
1162             5 * SZ_1M &&
1163             waitqueue_active(&fs_info->async_submit_wait))
1164                 wake_up(&fs_info->async_submit_wait);
1165 
1166         if (async_cow->inode)
1167                 submit_compressed_extents(async_cow->inode, async_cow);
1168 }
1169 
1170 static noinline void async_cow_free(struct btrfs_work *work)
1171 {
1172         struct async_cow *async_cow;
1173         async_cow = container_of(work, struct async_cow, work);
1174         if (async_cow->inode)
1175                 btrfs_add_delayed_iput(async_cow->inode);
1176         kfree(async_cow);
1177 }
1178 
1179 static int cow_file_range_async(struct inode *inode, struct page *locked_page,
1180                                 u64 start, u64 end, int *page_started,
1181                                 unsigned long *nr_written,
1182                                 unsigned int write_flags)
1183 {
1184         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1185         struct async_cow *async_cow;
1186         struct btrfs_root *root = BTRFS_I(inode)->root;
1187         unsigned long nr_pages;
1188         u64 cur_end;
1189 
1190         clear_extent_bit(&BTRFS_I(inode)->io_tree, start, end, EXTENT_LOCKED,
1191                          1, 0, NULL);
1192         while (start < end) {
1193                 async_cow = kmalloc(sizeof(*async_cow), GFP_NOFS);
1194                 BUG_ON(!async_cow); /* -ENOMEM */
1195                 async_cow->inode = igrab(inode);
1196                 async_cow->root = root;
1197                 async_cow->locked_page = locked_page;
1198                 async_cow->start = start;
1199                 async_cow->write_flags = write_flags;
1200 
1201                 if (BTRFS_I(inode)->flags & BTRFS_INODE_NOCOMPRESS &&
1202                     !btrfs_test_opt(fs_info, FORCE_COMPRESS))
1203                         cur_end = end;
1204                 else
1205                         cur_end = min(end, start + SZ_512K - 1);
1206 
1207                 async_cow->end = cur_end;
1208                 INIT_LIST_HEAD(&async_cow->extents);
1209 
1210                 btrfs_init_work(&async_cow->work,
1211                                 btrfs_delalloc_helper,
1212                                 async_cow_start, async_cow_submit,
1213                                 async_cow_free);
1214 
1215                 nr_pages = (cur_end - start + PAGE_SIZE) >>
1216                         PAGE_SHIFT;
1217                 atomic_add(nr_pages, &fs_info->async_delalloc_pages);
1218 
1219                 btrfs_queue_work(fs_info->delalloc_workers, &async_cow->work);
1220 
1221                 *nr_written += nr_pages;
1222                 start = cur_end + 1;
1223         }
1224         *page_started = 1;
1225         return 0;
1226 }
1227 
1228 static noinline int csum_exist_in_range(struct btrfs_fs_info *fs_info,
1229                                         u64 bytenr, u64 num_bytes)
1230 {
1231         int ret;
1232         struct btrfs_ordered_sum *sums;
1233         LIST_HEAD(list);
1234 
1235         ret = btrfs_lookup_csums_range(fs_info->csum_root, bytenr,
1236                                        bytenr + num_bytes - 1, &list, 0);
1237         if (ret == 0 && list_empty(&list))
1238                 return 0;
1239 
1240         while (!list_empty(&list)) {
1241                 sums = list_entry(list.next, struct btrfs_ordered_sum, list);
1242                 list_del(&sums->list);
1243                 kfree(sums);
1244         }
1245         if (ret < 0)
1246                 return ret;
1247         return 1;
1248 }
1249 
1250 /*
1251  * when nowcow writeback call back.  This checks for snapshots or COW copies
1252  * of the extents that exist in the file, and COWs the file as required.
1253  *
1254  * If no cow copies or snapshots exist, we write directly to the existing
1255  * blocks on disk
1256  */
1257 static noinline int run_delalloc_nocow(struct inode *inode,
1258                                        struct page *locked_page,
1259                               u64 start, u64 end, int *page_started, int force,
1260                               unsigned long *nr_written)
1261 {
1262         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1263         struct btrfs_root *root = BTRFS_I(inode)->root;
1264         struct extent_buffer *leaf;
1265         struct btrfs_path *path;
1266         struct btrfs_file_extent_item *fi;
1267         struct btrfs_key found_key;
1268         struct extent_map *em;
1269         u64 cow_start;
1270         u64 cur_offset;
1271         u64 extent_end;
1272         u64 extent_offset;
1273         u64 disk_bytenr;
1274         u64 num_bytes;
1275         u64 disk_num_bytes;
1276         u64 ram_bytes;
1277         int extent_type;
1278         int ret, err;
1279         int type;
1280         int nocow;
1281         int check_prev = 1;
1282         bool nolock;
1283         u64 ino = btrfs_ino(BTRFS_I(inode));
1284 
1285         path = btrfs_alloc_path();
1286         if (!path) {
1287                 extent_clear_unlock_delalloc(inode, start, end, end,
1288                                              locked_page,
1289                                              EXTENT_LOCKED | EXTENT_DELALLOC |
1290                                              EXTENT_DO_ACCOUNTING |
1291                                              EXTENT_DEFRAG, PAGE_UNLOCK |
1292                                              PAGE_CLEAR_DIRTY |
1293                                              PAGE_SET_WRITEBACK |
1294                                              PAGE_END_WRITEBACK);
1295                 return -ENOMEM;
1296         }
1297 
1298         nolock = btrfs_is_free_space_inode(BTRFS_I(inode));
1299 
1300         cow_start = (u64)-1;
1301         cur_offset = start;
1302         while (1) {
1303                 ret = btrfs_lookup_file_extent(NULL, root, path, ino,
1304                                                cur_offset, 0);
1305                 if (ret < 0)
1306                         goto error;
1307                 if (ret > 0 && path->slots[0] > 0 && check_prev) {
1308                         leaf = path->nodes[0];
1309                         btrfs_item_key_to_cpu(leaf, &found_key,
1310                                               path->slots[0] - 1);
1311                         if (found_key.objectid == ino &&
1312                             found_key.type == BTRFS_EXTENT_DATA_KEY)
1313                                 path->slots[0]--;
1314                 }
1315                 check_prev = 0;
1316 next_slot:
1317                 leaf = path->nodes[0];
1318                 if (path->slots[0] >= btrfs_header_nritems(leaf)) {
1319                         ret = btrfs_next_leaf(root, path);
1320                         if (ret < 0) {
1321                                 if (cow_start != (u64)-1)
1322                                         cur_offset = cow_start;
1323                                 goto error;
1324                         }
1325                         if (ret > 0)
1326                                 break;
1327                         leaf = path->nodes[0];
1328                 }
1329 
1330                 nocow = 0;
1331                 disk_bytenr = 0;
1332                 num_bytes = 0;
1333                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
1334 
1335                 if (found_key.objectid > ino)
1336                         break;
1337                 if (WARN_ON_ONCE(found_key.objectid < ino) ||
1338                     found_key.type < BTRFS_EXTENT_DATA_KEY) {
1339                         path->slots[0]++;
1340                         goto next_slot;
1341                 }
1342                 if (found_key.type > BTRFS_EXTENT_DATA_KEY ||
1343                     found_key.offset > end)
1344                         break;
1345 
1346                 if (found_key.offset > cur_offset) {
1347                         extent_end = found_key.offset;
1348                         extent_type = 0;
1349                         goto out_check;
1350                 }
1351 
1352                 fi = btrfs_item_ptr(leaf, path->slots[0],
1353                                     struct btrfs_file_extent_item);
1354                 extent_type = btrfs_file_extent_type(leaf, fi);
1355 
1356                 ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi);
1357                 if (extent_type == BTRFS_FILE_EXTENT_REG ||
1358                     extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
1359                         disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
1360                         extent_offset = btrfs_file_extent_offset(leaf, fi);
1361                         extent_end = found_key.offset +
1362                                 btrfs_file_extent_num_bytes(leaf, fi);
1363                         disk_num_bytes =
1364                                 btrfs_file_extent_disk_num_bytes(leaf, fi);
1365                         if (extent_end <= start) {
1366                                 path->slots[0]++;
1367                                 goto next_slot;
1368                         }
1369                         if (disk_bytenr == 0)
1370                                 goto out_check;
1371                         if (btrfs_file_extent_compression(leaf, fi) ||
1372                             btrfs_file_extent_encryption(leaf, fi) ||
1373                             btrfs_file_extent_other_encoding(leaf, fi))
1374                                 goto out_check;
1375                         if (extent_type == BTRFS_FILE_EXTENT_REG && !force)
1376                                 goto out_check;
1377                         if (btrfs_extent_readonly(fs_info, disk_bytenr))
1378                                 goto out_check;
1379                         ret = btrfs_cross_ref_exist(root, ino,
1380                                                     found_key.offset -
1381                                                     extent_offset, disk_bytenr);
1382                         if (ret) {
1383                                 /*
1384                                  * ret could be -EIO if the above fails to read
1385                                  * metadata.
1386                                  */
1387                                 if (ret < 0) {
1388                                         if (cow_start != (u64)-1)
1389                                                 cur_offset = cow_start;
1390                                         goto error;
1391                                 }
1392 
1393                                 WARN_ON_ONCE(nolock);
1394                                 goto out_check;
1395                         }
1396                         disk_bytenr += extent_offset;
1397                         disk_bytenr += cur_offset - found_key.offset;
1398                         num_bytes = min(end + 1, extent_end) - cur_offset;
1399                         /*
1400                          * if there are pending snapshots for this root,
1401                          * we fall into common COW way.
1402                          */
1403                         if (!nolock) {
1404                                 err = btrfs_start_write_no_snapshotting(root);
1405                                 if (!err)
1406                                         goto out_check;
1407                         }
1408                         /*
1409                          * force cow if csum exists in the range.
1410                          * this ensure that csum for a given extent are
1411                          * either valid or do not exist.
1412                          */
1413                         ret = csum_exist_in_range(fs_info, disk_bytenr,
1414                                                   num_bytes);
1415                         if (ret) {
1416                                 if (!nolock)
1417                                         btrfs_end_write_no_snapshotting(root);
1418 
1419                                 /*
1420                                  * ret could be -EIO if the above fails to read
1421                                  * metadata.
1422                                  */
1423                                 if (ret < 0) {
1424                                         if (cow_start != (u64)-1)
1425                                                 cur_offset = cow_start;
1426                                         goto error;
1427                                 }
1428                                 WARN_ON_ONCE(nolock);
1429                                 goto out_check;
1430                         }
1431                         if (!btrfs_inc_nocow_writers(fs_info, disk_bytenr)) {
1432                                 if (!nolock)
1433                                         btrfs_end_write_no_snapshotting(root);
1434                                 goto out_check;
1435                         }
1436                         nocow = 1;
1437                 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
1438                         extent_end = found_key.offset +
1439                                 btrfs_file_extent_inline_len(leaf,
1440                                                      path->slots[0], fi);
1441                         extent_end = ALIGN(extent_end,
1442                                            fs_info->sectorsize);
1443                 } else {
1444                         BUG_ON(1);
1445                 }
1446 out_check:
1447                 if (extent_end <= start) {
1448                         path->slots[0]++;
1449                         if (!nolock && nocow)
1450                                 btrfs_end_write_no_snapshotting(root);
1451                         if (nocow)
1452                                 btrfs_dec_nocow_writers(fs_info, disk_bytenr);
1453                         goto next_slot;
1454                 }
1455                 if (!nocow) {
1456                         if (cow_start == (u64)-1)
1457                                 cow_start = cur_offset;
1458                         cur_offset = extent_end;
1459                         if (cur_offset > end)
1460                                 break;
1461                         path->slots[0]++;
1462                         goto next_slot;
1463                 }
1464 
1465                 btrfs_release_path(path);
1466                 if (cow_start != (u64)-1) {
1467                         ret = cow_file_range(inode, locked_page,
1468                                              cow_start, found_key.offset - 1,
1469                                              end, page_started, nr_written, 1,
1470                                              NULL);
1471                         if (ret) {
1472                                 if (!nolock && nocow)
1473                                         btrfs_end_write_no_snapshotting(root);
1474                                 if (nocow)
1475                                         btrfs_dec_nocow_writers(fs_info,
1476                                                                 disk_bytenr);
1477                                 goto error;
1478                         }
1479                         cow_start = (u64)-1;
1480                 }
1481 
1482                 if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
1483                         u64 orig_start = found_key.offset - extent_offset;
1484 
1485                         em = create_io_em(inode, cur_offset, num_bytes,
1486                                           orig_start,
1487                                           disk_bytenr, /* block_start */
1488                                           num_bytes, /* block_len */
1489                                           disk_num_bytes, /* orig_block_len */
1490                                           ram_bytes, BTRFS_COMPRESS_NONE,
1491                                           BTRFS_ORDERED_PREALLOC);
1492                         if (IS_ERR(em)) {
1493                                 if (!nolock && nocow)
1494                                         btrfs_end_write_no_snapshotting(root);
1495                                 if (nocow)
1496                                         btrfs_dec_nocow_writers(fs_info,
1497                                                                 disk_bytenr);
1498                                 ret = PTR_ERR(em);
1499                                 goto error;
1500                         }
1501                         free_extent_map(em);
1502                 }
1503 
1504                 if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
1505                         type = BTRFS_ORDERED_PREALLOC;
1506                 } else {
1507                         type = BTRFS_ORDERED_NOCOW;
1508                 }
1509 
1510                 ret = btrfs_add_ordered_extent(inode, cur_offset, disk_bytenr,
1511                                                num_bytes, num_bytes, type);
1512                 if (nocow)
1513                         btrfs_dec_nocow_writers(fs_info, disk_bytenr);
1514                 BUG_ON(ret); /* -ENOMEM */
1515 
1516                 if (root->root_key.objectid ==
1517                     BTRFS_DATA_RELOC_TREE_OBJECTID)
1518                         /*
1519                          * Error handled later, as we must prevent
1520                          * extent_clear_unlock_delalloc() in error handler
1521                          * from freeing metadata of created ordered extent.
1522                          */
1523                         ret = btrfs_reloc_clone_csums(inode, cur_offset,
1524                                                       num_bytes);
1525 
1526                 extent_clear_unlock_delalloc(inode, cur_offset,
1527                                              cur_offset + num_bytes - 1, end,
1528                                              locked_page, EXTENT_LOCKED |
1529                                              EXTENT_DELALLOC |
1530                                              EXTENT_CLEAR_DATA_RESV,
1531                                              PAGE_UNLOCK | PAGE_SET_PRIVATE2);
1532 
1533                 if (!nolock && nocow)
1534                         btrfs_end_write_no_snapshotting(root);
1535                 cur_offset = extent_end;
1536 
1537                 /*
1538                  * btrfs_reloc_clone_csums() error, now we're OK to call error
1539                  * handler, as metadata for created ordered extent will only
1540                  * be freed by btrfs_finish_ordered_io().
1541                  */
1542                 if (ret)
1543                         goto error;
1544                 if (cur_offset > end)
1545                         break;
1546         }
1547         btrfs_release_path(path);
1548 
1549         if (cur_offset <= end && cow_start == (u64)-1) {
1550                 cow_start = cur_offset;
1551                 cur_offset = end;
1552         }
1553 
1554         if (cow_start != (u64)-1) {
1555                 ret = cow_file_range(inode, locked_page, cow_start, end, end,
1556                                      page_started, nr_written, 1, NULL);
1557                 if (ret)
1558                         goto error;
1559         }
1560 
1561 error:
1562         if (ret && cur_offset < end)
1563                 extent_clear_unlock_delalloc(inode, cur_offset, end, end,
1564                                              locked_page, EXTENT_LOCKED |
1565                                              EXTENT_DELALLOC | EXTENT_DEFRAG |
1566                                              EXTENT_DO_ACCOUNTING, PAGE_UNLOCK |
1567                                              PAGE_CLEAR_DIRTY |
1568                                              PAGE_SET_WRITEBACK |
1569                                              PAGE_END_WRITEBACK);
1570         btrfs_free_path(path);
1571         return ret;
1572 }
1573 
1574 static inline int need_force_cow(struct inode *inode, u64 start, u64 end)
1575 {
1576 
1577         if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) &&
1578             !(BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC))
1579                 return 0;
1580 
1581         /*
1582          * @defrag_bytes is a hint value, no spinlock held here,
1583          * if is not zero, it means the file is defragging.
1584          * Force cow if given extent needs to be defragged.
1585          */
1586         if (BTRFS_I(inode)->defrag_bytes &&
1587             test_range_bit(&BTRFS_I(inode)->io_tree, start, end,
1588                            EXTENT_DEFRAG, 0, NULL))
1589                 return 1;
1590 
1591         return 0;
1592 }
1593 
1594 /*
1595  * extent_io.c call back to do delayed allocation processing
1596  */
1597 static int run_delalloc_range(void *private_data, struct page *locked_page,
1598                               u64 start, u64 end, int *page_started,
1599                               unsigned long *nr_written,
1600                               struct writeback_control *wbc)
1601 {
1602         struct inode *inode = private_data;
1603         int ret;
1604         int force_cow = need_force_cow(inode, start, end);
1605         unsigned int write_flags = wbc_to_write_flags(wbc);
1606 
1607         if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW && !force_cow) {
1608                 ret = run_delalloc_nocow(inode, locked_page, start, end,
1609                                          page_started, 1, nr_written);
1610         } else if (BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC && !force_cow) {
1611                 ret = run_delalloc_nocow(inode, locked_page, start, end,
1612                                          page_started, 0, nr_written);
1613         } else if (!inode_need_compress(inode, start, end)) {
1614                 ret = cow_file_range(inode, locked_page, start, end, end,
1615                                       page_started, nr_written, 1, NULL);
1616         } else {
1617                 set_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
1618                         &BTRFS_I(inode)->runtime_flags);
1619                 ret = cow_file_range_async(inode, locked_page, start, end,
1620                                            page_started, nr_written,
1621                                            write_flags);
1622         }
1623         if (ret)
1624                 btrfs_cleanup_ordered_extents(inode, start, end - start + 1);
1625         return ret;
1626 }
1627 
1628 static void btrfs_split_extent_hook(void *private_data,
1629                                     struct extent_state *orig, u64 split)
1630 {
1631         struct inode *inode = private_data;
1632         u64 size;
1633 
1634         /* not delalloc, ignore it */
1635         if (!(orig->state & EXTENT_DELALLOC))
1636                 return;
1637 
1638         size = orig->end - orig->start + 1;
1639         if (size > BTRFS_MAX_EXTENT_SIZE) {
1640                 u32 num_extents;
1641                 u64 new_size;
1642 
1643                 /*
1644                  * See the explanation in btrfs_merge_extent_hook, the same
1645                  * applies here, just in reverse.
1646                  */
1647                 new_size = orig->end - split + 1;
1648                 num_extents = count_max_extents(new_size);
1649                 new_size = split - orig->start;
1650                 num_extents += count_max_extents(new_size);
1651                 if (count_max_extents(size) >= num_extents)
1652                         return;
1653         }
1654 
1655         spin_lock(&BTRFS_I(inode)->lock);
1656         btrfs_mod_outstanding_extents(BTRFS_I(inode), 1);
1657         spin_unlock(&BTRFS_I(inode)->lock);
1658 }
1659 
1660 /*
1661  * extent_io.c merge_extent_hook, used to track merged delayed allocation
1662  * extents so we can keep track of new extents that are just merged onto old
1663  * extents, such as when we are doing sequential writes, so we can properly
1664  * account for the metadata space we'll need.
1665  */
1666 static void btrfs_merge_extent_hook(void *private_data,
1667                                     struct extent_state *new,
1668                                     struct extent_state *other)
1669 {
1670         struct inode *inode = private_data;
1671         u64 new_size, old_size;
1672         u32 num_extents;
1673 
1674         /* not delalloc, ignore it */
1675         if (!(other->state & EXTENT_DELALLOC))
1676                 return;
1677 
1678         if (new->start > other->start)
1679                 new_size = new->end - other->start + 1;
1680         else
1681                 new_size = other->end - new->start + 1;
1682 
1683         /* we're not bigger than the max, unreserve the space and go */
1684         if (new_size <= BTRFS_MAX_EXTENT_SIZE) {
1685                 spin_lock(&BTRFS_I(inode)->lock);
1686                 btrfs_mod_outstanding_extents(BTRFS_I(inode), -1);
1687                 spin_unlock(&BTRFS_I(inode)->lock);
1688                 return;
1689         }
1690 
1691         /*
1692          * We have to add up either side to figure out how many extents were
1693          * accounted for before we merged into one big extent.  If the number of
1694          * extents we accounted for is <= the amount we need for the new range
1695          * then we can return, otherwise drop.  Think of it like this
1696          *
1697          * [ 4k][MAX_SIZE]
1698          *
1699          * So we've grown the extent by a MAX_SIZE extent, this would mean we
1700          * need 2 outstanding extents, on one side we have 1 and the other side
1701          * we have 1 so they are == and we can return.  But in this case
1702          *
1703          * [MAX_SIZE+4k][MAX_SIZE+4k]
1704          *
1705          * Each range on their own accounts for 2 extents, but merged together
1706          * they are only 3 extents worth of accounting, so we need to drop in
1707          * this case.
1708          */
1709         old_size = other->end - other->start + 1;
1710         num_extents = count_max_extents(old_size);
1711         old_size = new->end - new->start + 1;
1712         num_extents += count_max_extents(old_size);
1713         if (count_max_extents(new_size) >= num_extents)
1714                 return;
1715 
1716         spin_lock(&BTRFS_I(inode)->lock);
1717         btrfs_mod_outstanding_extents(BTRFS_I(inode), -1);
1718         spin_unlock(&BTRFS_I(inode)->lock);
1719 }
1720 
1721 static void btrfs_add_delalloc_inodes(struct btrfs_root *root,
1722                                       struct inode *inode)
1723 {
1724         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1725 
1726         spin_lock(&root->delalloc_lock);
1727         if (list_empty(&BTRFS_I(inode)->delalloc_inodes)) {
1728                 list_add_tail(&BTRFS_I(inode)->delalloc_inodes,
1729                               &root->delalloc_inodes);
1730                 set_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1731                         &BTRFS_I(inode)->runtime_flags);
1732                 root->nr_delalloc_inodes++;
1733                 if (root->nr_delalloc_inodes == 1) {
1734                         spin_lock(&fs_info->delalloc_root_lock);
1735                         BUG_ON(!list_empty(&root->delalloc_root));
1736                         list_add_tail(&root->delalloc_root,
1737                                       &fs_info->delalloc_roots);
1738                         spin_unlock(&fs_info->delalloc_root_lock);
1739                 }
1740         }
1741         spin_unlock(&root->delalloc_lock);
1742 }
1743 
1744 static void btrfs_del_delalloc_inode(struct btrfs_root *root,
1745                                      struct btrfs_inode *inode)
1746 {
1747         struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
1748 
1749         spin_lock(&root->delalloc_lock);
1750         if (!list_empty(&inode->delalloc_inodes)) {
1751                 list_del_init(&inode->delalloc_inodes);
1752                 clear_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1753                           &inode->runtime_flags);
1754                 root->nr_delalloc_inodes--;
1755                 if (!root->nr_delalloc_inodes) {
1756                         spin_lock(&fs_info->delalloc_root_lock);
1757                         BUG_ON(list_empty(&root->delalloc_root));
1758                         list_del_init(&root->delalloc_root);
1759                         spin_unlock(&fs_info->delalloc_root_lock);
1760                 }
1761         }
1762         spin_unlock(&root->delalloc_lock);
1763 }
1764 
1765 /*
1766  * extent_io.c set_bit_hook, used to track delayed allocation
1767  * bytes in this file, and to maintain the list of inodes that
1768  * have pending delalloc work to be done.
1769  */
1770 static void btrfs_set_bit_hook(void *private_data,
1771                                struct extent_state *state, unsigned *bits)
1772 {
1773         struct inode *inode = private_data;
1774 
1775         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1776 
1777         if ((*bits & EXTENT_DEFRAG) && !(*bits & EXTENT_DELALLOC))
1778                 WARN_ON(1);
1779         /*
1780          * set_bit and clear bit hooks normally require _irqsave/restore
1781          * but in this case, we are only testing for the DELALLOC
1782          * bit, which is only set or cleared with irqs on
1783          */
1784         if (!(state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) {
1785                 struct btrfs_root *root = BTRFS_I(inode)->root;
1786                 u64 len = state->end + 1 - state->start;
1787                 u32 num_extents = count_max_extents(len);
1788                 bool do_list = !btrfs_is_free_space_inode(BTRFS_I(inode));
1789 
1790                 spin_lock(&BTRFS_I(inode)->lock);
1791                 btrfs_mod_outstanding_extents(BTRFS_I(inode), num_extents);
1792                 spin_unlock(&BTRFS_I(inode)->lock);
1793 
1794                 /* For sanity tests */
1795                 if (btrfs_is_testing(fs_info))
1796                         return;
1797 
1798                 percpu_counter_add_batch(&fs_info->delalloc_bytes, len,
1799                                          fs_info->delalloc_batch);
1800                 spin_lock(&BTRFS_I(inode)->lock);
1801                 BTRFS_I(inode)->delalloc_bytes += len;
1802                 if (*bits & EXTENT_DEFRAG)
1803                         BTRFS_I(inode)->defrag_bytes += len;
1804                 if (do_list && !test_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1805                                          &BTRFS_I(inode)->runtime_flags))
1806                         btrfs_add_delalloc_inodes(root, inode);
1807                 spin_unlock(&BTRFS_I(inode)->lock);
1808         }
1809 
1810         if (!(state->state & EXTENT_DELALLOC_NEW) &&
1811             (*bits & EXTENT_DELALLOC_NEW)) {
1812                 spin_lock(&BTRFS_I(inode)->lock);
1813                 BTRFS_I(inode)->new_delalloc_bytes += state->end + 1 -
1814                         state->start;
1815                 spin_unlock(&BTRFS_I(inode)->lock);
1816         }
1817 }
1818 
1819 /*
1820  * extent_io.c clear_bit_hook, see set_bit_hook for why
1821  */
1822 static void btrfs_clear_bit_hook(void *private_data,
1823                                  struct extent_state *state,
1824                                  unsigned *bits)
1825 {
1826         struct btrfs_inode *inode = BTRFS_I((struct inode *)private_data);
1827         struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
1828         u64 len = state->end + 1 - state->start;
1829         u32 num_extents = count_max_extents(len);
1830 
1831         if ((state->state & EXTENT_DEFRAG) && (*bits & EXTENT_DEFRAG)) {
1832                 spin_lock(&inode->lock);
1833                 inode->defrag_bytes -= len;
1834                 spin_unlock(&inode->lock);
1835         }
1836 
1837         /*
1838          * set_bit and clear bit hooks normally require _irqsave/restore
1839          * but in this case, we are only testing for the DELALLOC
1840          * bit, which is only set or cleared with irqs on
1841          */
1842         if ((state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) {
1843                 struct btrfs_root *root = inode->root;
1844                 bool do_list = !btrfs_is_free_space_inode(inode);
1845 
1846                 spin_lock(&inode->lock);
1847                 btrfs_mod_outstanding_extents(inode, -num_extents);
1848                 spin_unlock(&inode->lock);
1849 
1850                 /*
1851                  * We don't reserve metadata space for space cache inodes so we
1852                  * don't need to call dellalloc_release_metadata if there is an
1853                  * error.
1854                  */
1855                 if (*bits & EXTENT_CLEAR_META_RESV &&
1856                     root != fs_info->tree_root)
1857                         btrfs_delalloc_release_metadata(inode, len, false);
1858 
1859                 /* For sanity tests. */
1860                 if (btrfs_is_testing(fs_info))
1861                         return;
1862 
1863                 if (root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID &&
1864                     do_list && !(state->state & EXTENT_NORESERVE) &&
1865                     (*bits & EXTENT_CLEAR_DATA_RESV))
1866                         btrfs_free_reserved_data_space_noquota(
1867                                         &inode->vfs_inode,
1868                                         state->start, len);
1869 
1870                 percpu_counter_add_batch(&fs_info->delalloc_bytes, -len,
1871                                          fs_info->delalloc_batch);
1872                 spin_lock(&inode->lock);
1873                 inode->delalloc_bytes -= len;
1874                 if (do_list && inode->delalloc_bytes == 0 &&
1875                     test_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1876                                         &inode->runtime_flags))
1877                         btrfs_del_delalloc_inode(root, inode);
1878                 spin_unlock(&inode->lock);
1879         }
1880 
1881         if ((state->state & EXTENT_DELALLOC_NEW) &&
1882             (*bits & EXTENT_DELALLOC_NEW)) {
1883                 spin_lock(&inode->lock);
1884                 ASSERT(inode->new_delalloc_bytes >= len);
1885                 inode->new_delalloc_bytes -= len;
1886                 spin_unlock(&inode->lock);
1887         }
1888 }
1889 
1890 /*
1891  * extent_io.c merge_bio_hook, this must check the chunk tree to make sure
1892  * we don't create bios that span stripes or chunks
1893  *
1894  * return 1 if page cannot be merged to bio
1895  * return 0 if page can be merged to bio
1896  * return error otherwise
1897  */
1898 int btrfs_merge_bio_hook(struct page *page, unsigned long offset,
1899                          size_t size, struct bio *bio,
1900                          unsigned long bio_flags)
1901 {
1902         struct inode *inode = page->mapping->host;
1903         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1904         u64 logical = (u64)bio->bi_iter.bi_sector << 9;
1905         u64 length = 0;
1906         u64 map_length;
1907         int ret;
1908 
1909         if (bio_flags & EXTENT_BIO_COMPRESSED)
1910                 return 0;
1911 
1912         length = bio->bi_iter.bi_size;
1913         map_length = length;
1914         ret = btrfs_map_block(fs_info, btrfs_op(bio), logical, &map_length,
1915                               NULL, 0);
1916         if (ret < 0)
1917                 return ret;
1918         if (map_length < length + size)
1919                 return 1;
1920         return 0;
1921 }
1922 
1923 /*
1924  * in order to insert checksums into the metadata in large chunks,
1925  * we wait until bio submission time.   All the pages in the bio are
1926  * checksummed and sums are attached onto the ordered extent record.
1927  *
1928  * At IO completion time the cums attached on the ordered extent record
1929  * are inserted into the btree
1930  */
1931 static blk_status_t btrfs_submit_bio_start(void *private_data, struct bio *bio,
1932                                     u64 bio_offset)
1933 {
1934         struct inode *inode = private_data;
1935         blk_status_t ret = 0;
1936 
1937         ret = btrfs_csum_one_bio(inode, bio, 0, 0);
1938         BUG_ON(ret); /* -ENOMEM */
1939         return 0;
1940 }
1941 
1942 /*
1943  * in order to insert checksums into the metadata in large chunks,
1944  * we wait until bio submission time.   All the pages in the bio are
1945  * checksummed and sums are attached onto the ordered extent record.
1946  *
1947  * At IO completion time the cums attached on the ordered extent record
1948  * are inserted into the btree
1949  */
1950 static blk_status_t btrfs_submit_bio_done(void *private_data, struct bio *bio,
1951                           int mirror_num)
1952 {
1953         struct inode *inode = private_data;
1954         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1955         blk_status_t ret;
1956 
1957         ret = btrfs_map_bio(fs_info, bio, mirror_num, 1);
1958         if (ret) {
1959                 bio->bi_status = ret;
1960                 bio_endio(bio);
1961         }
1962         return ret;
1963 }
1964 
1965 /*
1966  * extent_io.c submission hook. This does the right thing for csum calculation
1967  * on write, or reading the csums from the tree before a read.
1968  *
1969  * Rules about async/sync submit,
1970  * a) read:                             sync submit
1971  *
1972  * b) write without checksum:           sync submit
1973  *
1974  * c) write with checksum:
1975  *    c-1) if bio is issued by fsync:   sync submit
1976  *         (sync_writers != 0)
1977  *
1978  *    c-2) if root is reloc root:       sync submit
1979  *         (only in case of buffered IO)
1980  *
1981  *    c-3) otherwise:                   async submit
1982  */
1983 static blk_status_t btrfs_submit_bio_hook(void *private_data, struct bio *bio,
1984                                  int mirror_num, unsigned long bio_flags,
1985                                  u64 bio_offset)
1986 {
1987         struct inode *inode = private_data;
1988         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1989         struct btrfs_root *root = BTRFS_I(inode)->root;
1990         enum btrfs_wq_endio_type metadata = BTRFS_WQ_ENDIO_DATA;
1991         blk_status_t ret = 0;
1992         int skip_sum;
1993         int async = !atomic_read(&BTRFS_I(inode)->sync_writers);
1994 
1995         skip_sum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM;
1996 
1997         if (btrfs_is_free_space_inode(BTRFS_I(inode)))
1998                 metadata = BTRFS_WQ_ENDIO_FREE_SPACE;
1999 
2000         if (bio_op(bio) != REQ_OP_WRITE) {
2001                 ret = btrfs_bio_wq_end_io(fs_info, bio, metadata);
2002                 if (ret)
2003                         goto out;
2004 
2005                 if (bio_flags & EXTENT_BIO_COMPRESSED) {
2006                         ret = btrfs_submit_compressed_read(inode, bio,
2007                                                            mirror_num,
2008                                                            bio_flags);
2009                         goto out;
2010                 } else if (!skip_sum) {
2011                         ret = btrfs_lookup_bio_sums(inode, bio, NULL);
2012                         if (ret)
2013                                 goto out;
2014                 }
2015                 goto mapit;
2016         } else if (async && !skip_sum) {
2017                 /* csum items have already been cloned */
2018                 if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
2019                         goto mapit;
2020                 /* we're doing a write, do the async checksumming */
2021                 ret = btrfs_wq_submit_bio(fs_info, bio, mirror_num, bio_flags,
2022                                           bio_offset, inode,
2023                                           btrfs_submit_bio_start,
2024                                           btrfs_submit_bio_done);
2025                 goto out;
2026         } else if (!skip_sum) {
2027                 ret = btrfs_csum_one_bio(inode, bio, 0, 0);
2028                 if (ret)
2029                         goto out;
2030         }
2031 
2032 mapit:
2033         ret = btrfs_map_bio(fs_info, bio, mirror_num, 0);
2034 
2035 out:
2036         if (ret) {
2037                 bio->bi_status = ret;
2038                 bio_endio(bio);
2039         }
2040         return ret;
2041 }
2042 
2043 /*
2044  * given a list of ordered sums record them in the inode.  This happens
2045  * at IO completion time based on sums calculated at bio submission time.
2046  */
2047 static noinline int add_pending_csums(struct btrfs_trans_handle *trans,
2048                              struct inode *inode, struct list_head *list)
2049 {
2050         struct btrfs_ordered_sum *sum;
2051         int ret;
2052 
2053         list_for_each_entry(sum, list, list) {
2054                 trans->adding_csums = true;
2055                 ret = btrfs_csum_file_blocks(trans,
2056                        BTRFS_I(inode)->root->fs_info->csum_root, sum);
2057                 trans->adding_csums = false;
2058                 if (ret)
2059                         return ret;
2060         }
2061         return 0;
2062 }
2063 
2064 int btrfs_set_extent_delalloc(struct inode *inode, u64 start, u64 end,
2065                               unsigned int extra_bits,
2066                               struct extent_state **cached_state, int dedupe)
2067 {
2068         WARN_ON((end & (PAGE_SIZE - 1)) == 0);
2069         return set_extent_delalloc(&BTRFS_I(inode)->io_tree, start, end,
2070                                    extra_bits, cached_state);
2071 }
2072 
2073 /* see btrfs_writepage_start_hook for details on why this is required */
2074 struct btrfs_writepage_fixup {
2075         struct page *page;
2076         struct btrfs_work work;
2077 };
2078 
2079 static void btrfs_writepage_fixup_worker(struct btrfs_work *work)
2080 {
2081         struct btrfs_writepage_fixup *fixup;
2082         struct btrfs_ordered_extent *ordered;
2083         struct extent_state *cached_state = NULL;
2084         struct extent_changeset *data_reserved = NULL;
2085         struct page *page;
2086         struct inode *inode;
2087         u64 page_start;
2088         u64 page_end;
2089         int ret;
2090 
2091         fixup = container_of(work, struct btrfs_writepage_fixup, work);
2092         page = fixup->page;
2093 again:
2094         lock_page(page);
2095         if (!page->mapping || !PageDirty(page) || !PageChecked(page)) {
2096                 ClearPageChecked(page);
2097                 goto out_page;
2098         }
2099 
2100         inode = page->mapping->host;
2101         page_start = page_offset(page);
2102         page_end = page_offset(page) + PAGE_SIZE - 1;
2103 
2104         lock_extent_bits(&BTRFS_I(inode)->io_tree, page_start, page_end,
2105                          &cached_state);
2106 
2107         /* already ordered? We're done */
2108         if (PagePrivate2(page))
2109                 goto out;
2110 
2111         ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), page_start,
2112                                         PAGE_SIZE);
2113         if (ordered) {
2114                 unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start,
2115                                      page_end, &cached_state);
2116                 unlock_page(page);
2117                 btrfs_start_ordered_extent(inode, ordered, 1);
2118                 btrfs_put_ordered_extent(ordered);
2119                 goto again;
2120         }
2121 
2122         ret = btrfs_delalloc_reserve_space(inode, &data_reserved, page_start,
2123                                            PAGE_SIZE);
2124         if (ret) {
2125                 mapping_set_error(page->mapping, ret);
2126                 end_extent_writepage(page, ret, page_start, page_end);
2127                 ClearPageChecked(page);
2128                 goto out;
2129          }
2130 
2131         ret = btrfs_set_extent_delalloc(inode, page_start, page_end, 0,
2132                                         &cached_state, 0);
2133         if (ret) {
2134                 mapping_set_error(page->mapping, ret);
2135                 end_extent_writepage(page, ret, page_start, page_end);
2136                 ClearPageChecked(page);
2137                 goto out;
2138         }
2139 
2140         ClearPageChecked(page);
2141         set_page_dirty(page);
2142         btrfs_delalloc_release_extents(BTRFS_I(inode), PAGE_SIZE, false);
2143 out:
2144         unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start, page_end,
2145                              &cached_state);
2146 out_page:
2147         unlock_page(page);
2148         put_page(page);
2149         kfree(fixup);
2150         extent_changeset_free(data_reserved);
2151 }
2152 
2153 /*
2154  * There are a few paths in the higher layers of the kernel that directly
2155  * set the page dirty bit without asking the filesystem if it is a
2156  * good idea.  This causes problems because we want to make sure COW
2157  * properly happens and the data=ordered rules are followed.
2158  *
2159  * In our case any range that doesn't have the ORDERED bit set
2160  * hasn't been properly setup for IO.  We kick off an async process
2161  * to fix it up.  The async helper will wait for ordered extents, set
2162  * the delalloc bit and make it safe to write the page.
2163  */
2164 static int btrfs_writepage_start_hook(struct page *page, u64 start, u64 end)
2165 {
2166         struct inode *inode = page->mapping->host;
2167         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2168         struct btrfs_writepage_fixup *fixup;
2169 
2170         /* this page is properly in the ordered list */
2171         if (TestClearPagePrivate2(page))
2172                 return 0;
2173 
2174         if (PageChecked(page))
2175                 return -EAGAIN;
2176 
2177         fixup = kzalloc(sizeof(*fixup), GFP_NOFS);
2178         if (!fixup)
2179                 return -EAGAIN;
2180 
2181         SetPageChecked(page);
2182         get_page(page);
2183         btrfs_init_work(&fixup->work, btrfs_fixup_helper,
2184                         btrfs_writepage_fixup_worker, NULL, NULL);
2185         fixup->page = page;
2186         btrfs_queue_work(fs_info->fixup_workers, &fixup->work);
2187         return -EBUSY;
2188 }
2189 
2190 static int insert_reserved_file_extent(struct btrfs_trans_handle *trans,
2191                                        struct inode *inode, u64 file_pos,
2192                                        u64 disk_bytenr, u64 disk_num_bytes,
2193                                        u64 num_bytes, u64 ram_bytes,
2194                                        u8 compression, u8 encryption,
2195                                        u16 other_encoding, int extent_type)
2196 {
2197         struct btrfs_root *root = BTRFS_I(inode)->root;
2198         struct btrfs_file_extent_item *fi;
2199         struct btrfs_path *path;
2200         struct extent_buffer *leaf;
2201         struct btrfs_key ins;
2202         u64 qg_released;
2203         int extent_inserted = 0;
2204         int ret;
2205 
2206         path = btrfs_alloc_path();
2207         if (!path)
2208                 return -ENOMEM;
2209 
2210         /*
2211          * we may be replacing one extent in the tree with another.
2212          * The new extent is pinned in the extent map, and we don't want
2213          * to drop it from the cache until it is completely in the btree.
2214          *
2215          * So, tell btrfs_drop_extents to leave this extent in the cache.
2216          * the caller is expected to unpin it and allow it to be merged
2217          * with the others.
2218          */
2219         ret = __btrfs_drop_extents(trans, root, inode, path, file_pos,
2220                                    file_pos + num_bytes, NULL, 0,
2221                                    1, sizeof(*fi), &extent_inserted);
2222         if (ret)
2223                 goto out;
2224 
2225         if (!extent_inserted) {
2226                 ins.objectid = btrfs_ino(BTRFS_I(inode));
2227                 ins.offset = file_pos;
2228                 ins.type = BTRFS_EXTENT_DATA_KEY;
2229 
2230                 path->leave_spinning = 1;
2231                 ret = btrfs_insert_empty_item(trans, root, path, &ins,
2232                                               sizeof(*fi));
2233                 if (ret)
2234                         goto out;
2235         }
2236         leaf = path->nodes[0];
2237         fi = btrfs_item_ptr(leaf, path->slots[0],
2238                             struct btrfs_file_extent_item);
2239         btrfs_set_file_extent_generation(leaf, fi, trans->transid);
2240         btrfs_set_file_extent_type(leaf, fi, extent_type);
2241         btrfs_set_file_extent_disk_bytenr(leaf, fi, disk_bytenr);
2242         btrfs_set_file_extent_disk_num_bytes(leaf, fi, disk_num_bytes);
2243         btrfs_set_file_extent_offset(leaf, fi, 0);
2244         btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes);
2245         btrfs_set_file_extent_ram_bytes(leaf, fi, ram_bytes);
2246         btrfs_set_file_extent_compression(leaf, fi, compression);
2247         btrfs_set_file_extent_encryption(leaf, fi, encryption);
2248         btrfs_set_file_extent_other_encoding(leaf, fi, other_encoding);
2249 
2250         btrfs_mark_buffer_dirty(leaf);
2251         btrfs_release_path(path);
2252 
2253         inode_add_bytes(inode, num_bytes);
2254 
2255         ins.objectid = disk_bytenr;
2256         ins.offset = disk_num_bytes;
2257         ins.type = BTRFS_EXTENT_ITEM_KEY;
2258 
2259         /*
2260          * Release the reserved range from inode dirty range map, as it is
2261          * already moved into delayed_ref_head
2262          */
2263         ret = btrfs_qgroup_release_data(inode, file_pos, ram_bytes);
2264         if (ret < 0)
2265                 goto out;
2266         qg_released = ret;
2267         ret = btrfs_alloc_reserved_file_extent(trans, root,
2268                                                btrfs_ino(BTRFS_I(inode)),
2269                                                file_pos, qg_released, &ins);
2270 out:
2271         btrfs_free_path(path);
2272 
2273         return ret;
2274 }
2275 
2276 /* snapshot-aware defrag */
2277 struct sa_defrag_extent_backref {
2278         struct rb_node node;
2279         struct old_sa_defrag_extent *old;
2280         u64 root_id;
2281         u64 inum;
2282         u64 file_pos;
2283         u64 extent_offset;
2284         u64 num_bytes;
2285         u64 generation;
2286 };
2287 
2288 struct old_sa_defrag_extent {
2289         struct list_head list;
2290         struct new_sa_defrag_extent *new;
2291 
2292         u64 extent_offset;
2293         u64 bytenr;
2294         u64 offset;
2295         u64 len;
2296         int count;
2297 };
2298 
2299 struct new_sa_defrag_extent {
2300         struct rb_root root;
2301         struct list_head head;
2302         struct btrfs_path *path;
2303         struct inode *inode;
2304         u64 file_pos;
2305         u64 len;
2306         u64 bytenr;
2307         u64 disk_len;
2308         u8 compress_type;
2309 };
2310 
2311 static int backref_comp(struct sa_defrag_extent_backref *b1,
2312                         struct sa_defrag_extent_backref *b2)
2313 {
2314         if (b1->root_id < b2->root_id)
2315                 return -1;
2316         else if (b1->root_id > b2->root_id)
2317                 return 1;
2318 
2319         if (b1->inum < b2->inum)
2320                 return -1;
2321         else if (b1->inum > b2->inum)
2322                 return 1;
2323 
2324         if (b1->file_pos < b2->file_pos)
2325                 return -1;
2326         else if (b1->file_pos > b2->file_pos)
2327                 return 1;
2328 
2329         /*
2330          * [------------------------------] ===> (a range of space)
2331          *     |<--->|   |<---->| =============> (fs/file tree A)
2332          * |<---------------------------->| ===> (fs/file tree B)
2333          *
2334          * A range of space can refer to two file extents in one tree while
2335          * refer to only one file extent in another tree.
2336          *
2337          * So we may process a disk offset more than one time(two extents in A)
2338          * and locate at the same extent(one extent in B), then insert two same
2339          * backrefs(both refer to the extent in B).
2340          */
2341         return 0;
2342 }
2343 
2344 static void backref_insert(struct rb_root *root,
2345                            struct sa_defrag_extent_backref *backref)
2346 {
2347         struct rb_node **p = &root->rb_node;
2348         struct rb_node *parent = NULL;
2349         struct sa_defrag_extent_backref *entry;
2350         int ret;
2351 
2352         while (*p) {
2353                 parent = *p;
2354                 entry = rb_entry(parent, struct sa_defrag_extent_backref, node);
2355 
2356                 ret = backref_comp(backref, entry);
2357                 if (ret < 0)
2358                         p = &(*p)->rb_left;
2359                 else
2360                         p = &(*p)->rb_right;
2361         }
2362 
2363         rb_link_node(&backref->node, parent, p);
2364         rb_insert_color(&backref->node, root);
2365 }
2366 
2367 /*
2368  * Note the backref might has changed, and in this case we just return 0.
2369  */
2370 static noinline int record_one_backref(u64 inum, u64 offset, u64 root_id,
2371                                        void *ctx)
2372 {
2373         struct btrfs_file_extent_item *extent;
2374         struct old_sa_defrag_extent *old = ctx;
2375         struct new_sa_defrag_extent *new = old->new;
2376         struct btrfs_path *path = new->path;
2377         struct btrfs_key key;
2378         struct btrfs_root *root;
2379         struct sa_defrag_extent_backref *backref;
2380         struct extent_buffer *leaf;
2381         struct inode *inode = new->inode;
2382         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2383         int slot;
2384         int ret;
2385         u64 extent_offset;
2386         u64 num_bytes;
2387 
2388         if (BTRFS_I(inode)->root->root_key.objectid == root_id &&
2389             inum == btrfs_ino(BTRFS_I(inode)))
2390                 return 0;
2391 
2392         key.objectid = root_id;
2393         key.type = BTRFS_ROOT_ITEM_KEY;
2394         key.offset = (u64)-1;
2395 
2396         root = btrfs_read_fs_root_no_name(fs_info, &key);
2397         if (IS_ERR(root)) {
2398                 if (PTR_ERR(root) == -ENOENT)
2399                         return 0;
2400                 WARN_ON(1);
2401                 btrfs_debug(fs_info, "inum=%llu, offset=%llu, root_id=%llu",
2402                          inum, offset, root_id);
2403                 return PTR_ERR(root);
2404         }
2405 
2406         key.objectid = inum;
2407         key.type = BTRFS_EXTENT_DATA_KEY;
2408         if (offset > (u64)-1 << 32)
2409                 key.offset = 0;
2410         else
2411                 key.offset = offset;
2412 
2413         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2414         if (WARN_ON(ret < 0))
2415                 return ret;
2416         ret = 0;
2417 
2418         while (1) {
2419                 cond_resched();
2420 
2421                 leaf = path->nodes[0];
2422                 slot = path->slots[0];
2423 
2424                 if (slot >= btrfs_header_nritems(leaf)) {
2425                         ret = btrfs_next_leaf(root, path);
2426                         if (ret < 0) {
2427                                 goto out;
2428                         } else if (ret > 0) {
2429                                 ret = 0;
2430                                 goto out;
2431                         }
2432                         continue;
2433                 }
2434 
2435                 path->slots[0]++;
2436 
2437                 btrfs_item_key_to_cpu(leaf, &key, slot);
2438 
2439                 if (key.objectid > inum)
2440                         goto out;
2441 
2442                 if (key.objectid < inum || key.type != BTRFS_EXTENT_DATA_KEY)
2443                         continue;
2444 
2445                 extent = btrfs_item_ptr(leaf, slot,
2446                                         struct btrfs_file_extent_item);
2447 
2448                 if (btrfs_file_extent_disk_bytenr(leaf, extent) != old->bytenr)
2449                         continue;
2450 
2451                 /*
2452                  * 'offset' refers to the exact key.offset,
2453                  * NOT the 'offset' field in btrfs_extent_data_ref, ie.
2454                  * (key.offset - extent_offset).
2455                  */
2456                 if (key.offset != offset)
2457                         continue;
2458 
2459                 extent_offset = btrfs_file_extent_offset(leaf, extent);
2460                 num_bytes = btrfs_file_extent_num_bytes(leaf, extent);
2461 
2462                 if (extent_offset >= old->extent_offset + old->offset +
2463                     old->len || extent_offset + num_bytes <=
2464                     old->extent_offset + old->offset)
2465                         continue;
2466                 break;
2467         }
2468 
2469         backref = kmalloc(sizeof(*backref), GFP_NOFS);
2470         if (!backref) {
2471                 ret = -ENOENT;
2472                 goto out;
2473         }
2474 
2475         backref->root_id = root_id;
2476         backref->inum = inum;
2477         backref->file_pos = offset;
2478         backref->num_bytes = num_bytes;
2479         backref->extent_offset = extent_offset;
2480         backref->generation = btrfs_file_extent_generation(leaf, extent);
2481         backref->old = old;
2482         backref_insert(&new->root, backref);
2483         old->count++;
2484 out:
2485         btrfs_release_path(path);
2486         WARN_ON(ret);
2487         return ret;
2488 }
2489 
2490 static noinline bool record_extent_backrefs(struct btrfs_path *path,
2491                                    struct new_sa_defrag_extent *new)
2492 {
2493         struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb);
2494         struct old_sa_defrag_extent *old, *tmp;
2495         int ret;
2496 
2497         new->path = path;
2498 
2499         list_for_each_entry_safe(old, tmp, &new->head, list) {
2500                 ret = iterate_inodes_from_logical(old->bytenr +
2501                                                   old->extent_offset, fs_info,
2502                                                   path, record_one_backref,
2503                                                   old, false);
2504                 if (ret < 0 && ret != -ENOENT)
2505                         return false;
2506 
2507                 /* no backref to be processed for this extent */
2508                 if (!old->count) {
2509                         list_del(&old->list);
2510                         kfree(old);
2511                 }
2512         }
2513 
2514         if (list_empty(&new->head))
2515                 return false;
2516 
2517         return true;
2518 }
2519 
2520 static int relink_is_mergable(struct extent_buffer *leaf,
2521                               struct btrfs_file_extent_item *fi,
2522                               struct new_sa_defrag_extent *new)
2523 {
2524         if (btrfs_file_extent_disk_bytenr(leaf, fi) != new->bytenr)
2525                 return 0;
2526 
2527         if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG)
2528                 return 0;
2529 
2530         if (btrfs_file_extent_compression(leaf, fi) != new->compress_type)
2531                 return 0;
2532 
2533         if (btrfs_file_extent_encryption(leaf, fi) ||
2534             btrfs_file_extent_other_encoding(leaf, fi))
2535                 return 0;
2536 
2537         return 1;
2538 }
2539 
2540 /*
2541  * Note the backref might has changed, and in this case we just return 0.
2542  */
2543 static noinline int relink_extent_backref(struct btrfs_path *path,
2544                                  struct sa_defrag_extent_backref *prev,
2545                                  struct sa_defrag_extent_backref *backref)
2546 {
2547         struct btrfs_file_extent_item *extent;
2548         struct btrfs_file_extent_item *item;
2549         struct btrfs_ordered_extent *ordered;
2550         struct btrfs_trans_handle *trans;
2551         struct btrfs_root *root;
2552         struct btrfs_key key;
2553         struct extent_buffer *leaf;
2554         struct old_sa_defrag_extent *old = backref->old;
2555         struct new_sa_defrag_extent *new = old->new;
2556         struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb);
2557         struct inode *inode;
2558         struct extent_state *cached = NULL;
2559         int ret = 0;
2560         u64 start;
2561         u64 len;
2562         u64 lock_start;
2563         u64 lock_end;
2564         bool merge = false;
2565         int index;
2566 
2567         if (prev && prev->root_id == backref->root_id &&
2568             prev->inum == backref->inum &&
2569             prev->file_pos + prev->num_bytes == backref->file_pos)
2570                 merge = true;
2571 
2572         /* step 1: get root */
2573         key.objectid = backref->root_id;
2574         key.type = BTRFS_ROOT_ITEM_KEY;
2575         key.offset = (u64)-1;
2576 
2577         index = srcu_read_lock(&fs_info->subvol_srcu);
2578 
2579         root = btrfs_read_fs_root_no_name(fs_info, &key);
2580         if (IS_ERR(root)) {
2581                 srcu_read_unlock(&fs_info->subvol_srcu, index);
2582                 if (PTR_ERR(root) == -ENOENT)
2583                         return 0;
2584                 return PTR_ERR(root);
2585         }
2586 
2587         if (btrfs_root_readonly(root)) {
2588                 srcu_read_unlock(&fs_info->subvol_srcu, index);
2589                 return 0;
2590         }
2591 
2592         /* step 2: get inode */
2593         key.objectid = backref->inum;
2594         key.type = BTRFS_INODE_ITEM_KEY;
2595         key.offset = 0;
2596 
2597         inode = btrfs_iget(fs_info->sb, &key, root, NULL);
2598         if (IS_ERR(inode)) {
2599                 srcu_read_unlock(&fs_info->subvol_srcu, index);
2600                 return 0;
2601         }
2602 
2603         srcu_read_unlock(&fs_info->subvol_srcu, index);
2604 
2605         /* step 3: relink backref */
2606         lock_start = backref->file_pos;
2607         lock_end = backref->file_pos + backref->num_bytes - 1;
2608         lock_extent_bits(&BTRFS_I(inode)->io_tree, lock_start, lock_end,
2609                          &cached);
2610 
2611         ordered = btrfs_lookup_first_ordered_extent(inode, lock_end);
2612         if (ordered) {
2613                 btrfs_put_ordered_extent(ordered);
2614                 goto out_unlock;
2615         }
2616 
2617         trans = btrfs_join_transaction(root);
2618         if (IS_ERR(trans)) {
2619                 ret = PTR_ERR(trans);
2620                 goto out_unlock;
2621         }
2622 
2623         key.objectid = backref->inum;
2624         key.type = BTRFS_EXTENT_DATA_KEY;
2625         key.offset = backref->file_pos;
2626 
2627         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2628         if (ret < 0) {
2629                 goto out_free_path;
2630         } else if (ret > 0) {
2631                 ret = 0;
2632                 goto out_free_path;
2633         }
2634 
2635         extent = btrfs_item_ptr(path->nodes[0], path->slots[0],
2636                                 struct btrfs_file_extent_item);
2637 
2638         if (btrfs_file_extent_generation(path->nodes[0], extent) !=
2639             backref->generation)
2640                 goto out_free_path;
2641 
2642         btrfs_release_path(path);
2643 
2644         start = backref->file_pos;
2645         if (backref->extent_offset < old->extent_offset + old->offset)
2646                 start += old->extent_offset + old->offset -
2647                          backref->extent_offset;
2648 
2649         len = min(backref->extent_offset + backref->num_bytes,
2650                   old->extent_offset + old->offset + old->len);
2651         len -= max(backref->extent_offset, old->extent_offset + old->offset);
2652 
2653         ret = btrfs_drop_extents(trans, root, inode, start,
2654                                  start + len, 1);
2655         if (ret)
2656                 goto out_free_path;
2657 again:
2658         key.objectid = btrfs_ino(BTRFS_I(inode));
2659         key.type = BTRFS_EXTENT_DATA_KEY;
2660         key.offset = start;
2661 
2662         path->leave_spinning = 1;
2663         if (merge) {
2664                 struct btrfs_file_extent_item *fi;
2665                 u64 extent_len;
2666                 struct btrfs_key found_key;
2667 
2668                 ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
2669                 if (ret < 0)
2670                         goto out_free_path;
2671 
2672                 path->slots[0]--;
2673                 leaf = path->nodes[0];
2674                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
2675 
2676                 fi = btrfs_item_ptr(leaf, path->slots[0],
2677                                     struct btrfs_file_extent_item);
2678                 extent_len = btrfs_file_extent_num_bytes(leaf, fi);
2679 
2680                 if (extent_len + found_key.offset == start &&
2681                     relink_is_mergable(leaf, fi, new)) {
2682                         btrfs_set_file_extent_num_bytes(leaf, fi,
2683                                                         extent_len + len);
2684                         btrfs_mark_buffer_dirty(leaf);
2685                         inode_add_bytes(inode, len);
2686 
2687                         ret = 1;
2688                         goto out_free_path;
2689                 } else {
2690                         merge = false;
2691                         btrfs_release_path(path);
2692                         goto again;
2693                 }
2694         }
2695 
2696         ret = btrfs_insert_empty_item(trans, root, path, &key,
2697                                         sizeof(*extent));
2698         if (ret) {
2699                 btrfs_abort_transaction(trans, ret);
2700                 goto out_free_path;
2701         }
2702 
2703         leaf = path->nodes[0];
2704         item = btrfs_item_ptr(leaf, path->slots[0],
2705                                 struct btrfs_file_extent_item);
2706         btrfs_set_file_extent_disk_bytenr(leaf, item, new->bytenr);
2707         btrfs_set_file_extent_disk_num_bytes(leaf, item, new->disk_len);
2708         btrfs_set_file_extent_offset(leaf, item, start - new->file_pos);
2709         btrfs_set_file_extent_num_bytes(leaf, item, len);
2710         btrfs_set_file_extent_ram_bytes(leaf, item, new->len);
2711         btrfs_set_file_extent_generation(leaf, item, trans->transid);
2712         btrfs_set_file_extent_type(leaf, item, BTRFS_FILE_EXTENT_REG);
2713         btrfs_set_file_extent_compression(leaf, item, new->compress_type);
2714         btrfs_set_file_extent_encryption(leaf, item, 0);
2715         btrfs_set_file_extent_other_encoding(leaf, item, 0);
2716 
2717         btrfs_mark_buffer_dirty(leaf);
2718         inode_add_bytes(inode, len);
2719         btrfs_release_path(path);
2720 
2721         ret = btrfs_inc_extent_ref(trans, root, new->bytenr,
2722                         new->disk_len, 0,
2723                         backref->root_id, backref->inum,
2724                         new->file_pos); /* start - extent_offset */
2725         if (ret) {
2726                 btrfs_abort_transaction(trans, ret);
2727                 goto out_free_path;
2728         }
2729 
2730         ret = 1;
2731 out_free_path:
2732         btrfs_release_path(path);
2733         path->leave_spinning = 0;
2734         btrfs_end_transaction(trans);
2735 out_unlock:
2736         unlock_extent_cached(&BTRFS_I(inode)->io_tree, lock_start, lock_end,
2737                              &cached);
2738         iput(inode);
2739         return ret;
2740 }
2741 
2742 static void free_sa_defrag_extent(struct new_sa_defrag_extent *new)
2743 {
2744         struct old_sa_defrag_extent *old, *tmp;
2745 
2746         if (!new)
2747                 return;
2748 
2749         list_for_each_entry_safe(old, tmp, &new->head, list) {
2750                 kfree(old);
2751         }
2752         kfree(new);
2753 }
2754 
2755 static void relink_file_extents(struct new_sa_defrag_extent *new)
2756 {
2757         struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb);
2758         struct btrfs_path *path;
2759         struct sa_defrag_extent_backref *backref;
2760         struct sa_defrag_extent_backref *prev = NULL;
2761         struct inode *inode;
2762         struct rb_node *node;
2763         int ret;
2764 
2765         inode = new->inode;
2766 
2767         path = btrfs_alloc_path();
2768         if (!path)
2769                 return;
2770 
2771         if (!record_extent_backrefs(path, new)) {
2772                 btrfs_free_path(path);
2773                 goto out;
2774         }
2775         btrfs_release_path(path);
2776 
2777         while (1) {
2778                 node = rb_first(&new->root);
2779                 if (!node)
2780                         break;
2781                 rb_erase(node, &new->root);
2782 
2783                 backref = rb_entry(node, struct sa_defrag_extent_backref, node);
2784 
2785                 ret = relink_extent_backref(path, prev, backref);
2786                 WARN_ON(ret < 0);
2787 
2788                 kfree(prev);
2789 
2790                 if (ret == 1)
2791                         prev = backref;
2792                 else
2793                         prev = NULL;
2794                 cond_resched();
2795         }
2796         kfree(prev);
2797 
2798         btrfs_free_path(path);
2799 out:
2800         free_sa_defrag_extent(new);
2801 
2802         atomic_dec(&fs_info->defrag_running);
2803         wake_up(&fs_info->transaction_wait);
2804 }
2805 
2806 static struct new_sa_defrag_extent *
2807 record_old_file_extents(struct inode *inode,
2808                         struct btrfs_ordered_extent *ordered)
2809 {
2810         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2811         struct btrfs_root *root = BTRFS_I(inode)->root;
2812         struct btrfs_path *path;
2813         struct btrfs_key key;
2814         struct old_sa_defrag_extent *old;
2815         struct new_sa_defrag_extent *new;
2816         int ret;
2817 
2818         new = kmalloc(sizeof(*new), GFP_NOFS);
2819         if (!new)
2820                 return NULL;
2821 
2822         new->inode = inode;
2823         new->file_pos = ordered->file_offset;
2824         new->len = ordered->len;
2825         new->bytenr = ordered->start;
2826         new->disk_len = ordered->disk_len;
2827         new->compress_type = ordered->compress_type;
2828         new->root = RB_ROOT;
2829         INIT_LIST_HEAD(&new->head);
2830 
2831         path = btrfs_alloc_path();
2832         if (!path)
2833                 goto out_kfree;
2834 
2835         key.objectid = btrfs_ino(BTRFS_I(inode));
2836         key.type = BTRFS_EXTENT_DATA_KEY;
2837         key.offset = new->file_pos;
2838 
2839         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2840         if (ret < 0)
2841                 goto out_free_path;
2842         if (ret > 0 && path->slots[0] > 0)
2843                 path->slots[0]--;
2844 
2845         /* find out all the old extents for the file range */
2846         while (1) {
2847                 struct btrfs_file_extent_item *extent;
2848                 struct extent_buffer *l;
2849                 int slot;
2850                 u64 num_bytes;
2851                 u64 offset;
2852                 u64 end;
2853                 u64 disk_bytenr;
2854                 u64 extent_offset;
2855 
2856                 l = path->nodes[0];
2857                 slot = path->slots[0];
2858 
2859                 if (slot >= btrfs_header_nritems(l)) {
2860                         ret = btrfs_next_leaf(root, path);
2861                         if (ret < 0)
2862                                 goto out_free_path;
2863                         else if (ret > 0)
2864                                 break;
2865                         continue;
2866                 }
2867 
2868                 btrfs_item_key_to_cpu(l, &key, slot);
2869 
2870                 if (key.objectid != btrfs_ino(BTRFS_I(inode)))
2871                         break;
2872                 if (key.type != BTRFS_EXTENT_DATA_KEY)
2873                         break;
2874                 if (key.offset >= new->file_pos + new->len)
2875                         break;
2876 
2877                 extent = btrfs_item_ptr(l, slot, struct btrfs_file_extent_item);
2878 
2879                 num_bytes = btrfs_file_extent_num_bytes(l, extent);
2880                 if (key.offset + num_bytes < new->file_pos)
2881                         goto next;
2882 
2883                 disk_bytenr = btrfs_file_extent_disk_bytenr(l, extent);
2884                 if (!disk_bytenr)
2885                         goto next;
2886 
2887                 extent_offset = btrfs_file_extent_offset(l, extent);
2888 
2889                 old = kmalloc(sizeof(*old), GFP_NOFS);
2890                 if (!old)
2891                         goto out_free_path;
2892 
2893                 offset = max(new->file_pos, key.offset);
2894                 end = min(new->file_pos + new->len, key.offset + num_bytes);
2895 
2896                 old->bytenr = disk_bytenr;
2897                 old->extent_offset = extent_offset;
2898                 old->offset = offset - key.offset;
2899                 old->len = end - offset;
2900                 old->new = new;
2901                 old->count = 0;
2902                 list_add_tail(&old->list, &new->head);
2903 next:
2904                 path->slots[0]++;
2905                 cond_resched();
2906         }
2907 
2908         btrfs_free_path(path);
2909         atomic_inc(&fs_info->defrag_running);
2910 
2911         return new;
2912 
2913 out_free_path:
2914         btrfs_free_path(path);
2915 out_kfree:
2916         free_sa_defrag_extent(new);
2917         return NULL;
2918 }
2919 
2920 static void btrfs_release_delalloc_bytes(struct btrfs_fs_info *fs_info,
2921                                          u64 start, u64 len)
2922 {
2923         struct btrfs_block_group_cache *cache;
2924 
2925         cache = btrfs_lookup_block_group(fs_info, start);
2926         ASSERT(cache);
2927 
2928         spin_lock(&cache->lock);
2929         cache->delalloc_bytes -= len;
2930         spin_unlock(&cache->lock);
2931 
2932         btrfs_put_block_group(cache);
2933 }
2934 
2935 /* as ordered data IO finishes, this gets called so we can finish
2936  * an ordered extent if the range of bytes in the file it covers are
2937  * fully written.
2938  */
2939 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent)
2940 {
2941         struct inode *inode = ordered_extent->inode;
2942         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2943         struct btrfs_root *root = BTRFS_I(inode)->root;
2944         struct btrfs_trans_handle *trans = NULL;
2945         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
2946         struct extent_state *cached_state = NULL;
2947         struct new_sa_defrag_extent *new = NULL;
2948         int compress_type = 0;
2949         int ret = 0;
2950         u64 logical_len = ordered_extent->len;
2951         bool nolock;
2952         bool truncated = false;
2953         bool range_locked = false;
2954         bool clear_new_delalloc_bytes = false;
2955 
2956         if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) &&
2957             !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags) &&
2958             !test_bit(BTRFS_ORDERED_DIRECT, &ordered_extent->flags))
2959                 clear_new_delalloc_bytes = true;
2960 
2961         nolock = btrfs_is_free_space_inode(BTRFS_I(inode));
2962 
2963         if (test_bit(BTRFS_ORDERED_IOERR, &ordered_extent->flags)) {
2964                 ret = -EIO;
2965                 goto out;
2966         }
2967 
2968         btrfs_free_io_failure_record(BTRFS_I(inode),
2969                         ordered_extent->file_offset,
2970                         ordered_extent->file_offset +
2971                         ordered_extent->len - 1);
2972 
2973         if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) {
2974                 truncated = true;
2975                 logical_len = ordered_extent->truncated_len;
2976                 /* Truncated the entire extent, don't bother adding */
2977                 if (!logical_len)
2978                         goto out;
2979         }
2980 
2981         if (test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) {
2982                 BUG_ON(!list_empty(&ordered_extent->list)); /* Logic error */
2983 
2984                 /*
2985                  * For mwrite(mmap + memset to write) case, we still reserve
2986                  * space for NOCOW range.
2987                  * As NOCOW won't cause a new delayed ref, just free the space
2988                  */
2989                 btrfs_qgroup_free_data(inode, NULL, ordered_extent->file_offset,
2990                                        ordered_extent->len);
2991                 btrfs_ordered_update_i_size(inode, 0, ordered_extent);
2992                 if (nolock)
2993                         trans = btrfs_join_transaction_nolock(root);
2994                 else
2995                         trans = btrfs_join_transaction(root);
2996                 if (IS_ERR(trans)) {
2997                         ret = PTR_ERR(trans);
2998                         trans = NULL;
2999                         goto out;
3000                 }
3001                 trans->block_rsv = &BTRFS_I(inode)->block_rsv;
3002                 ret = btrfs_update_inode_fallback(trans, root, inode);
3003                 if (ret) /* -ENOMEM or corruption */
3004                         btrfs_abort_transaction(trans, ret);
3005                 goto out;
3006         }
3007 
3008         range_locked = true;
3009         lock_extent_bits(io_tree, ordered_extent->file_offset,
3010                          ordered_extent->file_offset + ordered_extent->len - 1,
3011                          &cached_state);
3012 
3013         ret = test_range_bit(io_tree, ordered_extent->file_offset,
3014                         ordered_extent->file_offset + ordered_extent->len - 1,
3015                         EXTENT_DEFRAG, 0, cached_state);
3016         if (ret) {
3017                 u64 last_snapshot = btrfs_root_last_snapshot(&root->root_item);
3018                 if (0 && last_snapshot >= BTRFS_I(inode)->generation)
3019                         /* the inode is shared */
3020                         new = record_old_file_extents(inode, ordered_extent);
3021 
3022                 clear_extent_bit(io_tree, ordered_extent->file_offset,
3023                         ordered_extent->file_offset + ordered_extent->len - 1,
3024                         EXTENT_DEFRAG, 0, 0, &cached_state);
3025         }
3026 
3027         if (nolock)
3028                 trans = btrfs_join_transaction_nolock(root);
3029         else
3030                 trans = btrfs_join_transaction(root);
3031         if (IS_ERR(trans)) {
3032                 ret = PTR_ERR(trans);
3033                 trans = NULL;
3034                 goto out;
3035         }
3036 
3037         trans->block_rsv = &BTRFS_I(inode)->block_rsv;
3038 
3039         if (test_bit(BTRFS_ORDERED_COMPRESSED, &ordered_extent->flags))
3040                 compress_type = ordered_extent->compress_type;
3041         if (test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) {
3042                 BUG_ON(compress_type);
3043                 btrfs_qgroup_free_data(inode, NULL, ordered_extent->file_offset,
3044                                        ordered_extent->len);
3045                 ret = btrfs_mark_extent_written(trans, BTRFS_I(inode),
3046                                                 ordered_extent->file_offset,
3047                                                 ordered_extent->file_offset +
3048                                                 logical_len);
3049         } else {
3050                 BUG_ON(root == fs_info->tree_root);
3051                 ret = insert_reserved_file_extent(trans, inode,
3052                                                 ordered_extent->file_offset,
3053                                                 ordered_extent->start,
3054                                                 ordered_extent->disk_len,
3055                                                 logical_len, logical_len,
3056                                                 compress_type, 0, 0,
3057                                                 BTRFS_FILE_EXTENT_REG);
3058                 if (!ret)
3059                         btrfs_release_delalloc_bytes(fs_info,
3060                                                      ordered_extent->start,
3061                                                      ordered_extent->disk_len);
3062         }
3063         unpin_extent_cache(&BTRFS_I(inode)->extent_tree,
3064                            ordered_extent->file_offset, ordered_extent->len,
3065                            trans->transid);
3066         if (ret < 0) {
3067                 btrfs_abort_transaction(trans, ret);
3068                 goto out;
3069         }
3070 
3071         ret = add_pending_csums(trans, inode, &ordered_extent->list);
3072         if (ret) {
3073                 btrfs_abort_transaction(trans, ret);
3074                 goto out;
3075         }
3076 
3077         btrfs_ordered_update_i_size(inode, 0, ordered_extent);
3078         ret = btrfs_update_inode_fallback(trans, root, inode);
3079         if (ret) { /* -ENOMEM or corruption */
3080                 btrfs_abort_transaction(trans, ret);
3081                 goto out;
3082         }
3083         ret = 0;
3084 out:
3085         if (range_locked || clear_new_delalloc_bytes) {
3086                 unsigned int clear_bits = 0;
3087 
3088                 if (range_locked)
3089                         clear_bits |= EXTENT_LOCKED;
3090                 if (clear_new_delalloc_bytes)
3091                         clear_bits |= EXTENT_DELALLOC_NEW;
3092                 clear_extent_bit(&BTRFS_I(inode)->io_tree,
3093                                  ordered_extent->file_offset,
3094                                  ordered_extent->file_offset +
3095                                  ordered_extent->len - 1,
3096                                  clear_bits,
3097                                  (clear_bits & EXTENT_LOCKED) ? 1 : 0,
3098                                  0, &cached_state);
3099         }
3100 
3101         if (trans)
3102                 btrfs_end_transaction(trans);
3103 
3104         if (ret || truncated) {
3105                 u64 start, end;
3106 
3107                 if (truncated)
3108                         start = ordered_extent->file_offset + logical_len;
3109                 else
3110                         start = ordered_extent->file_offset;
3111                 end = ordered_extent->file_offset + ordered_extent->len - 1;
3112                 clear_extent_uptodate(io_tree, start, end, NULL);
3113 
3114                 /* Drop the cache for the part of the extent we didn't write. */
3115                 btrfs_drop_extent_cache(BTRFS_I(inode), start, end, 0);
3116 
3117                 /*
3118                  * If the ordered extent had an IOERR or something else went
3119                  * wrong we need to return the space for this ordered extent
3120                  * back to the allocator.  We only free the extent in the
3121                  * truncated case if we didn't write out the extent at all.
3122                  */
3123                 if ((ret || !logical_len) &&
3124                     !test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) &&
3125                     !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags))
3126                         btrfs_free_reserved_extent(fs_info,
3127                                                    ordered_extent->start,
3128                                                    ordered_extent->disk_len, 1);
3129         }
3130 
3131 
3132         /*
3133          * This needs to be done to make sure anybody waiting knows we are done
3134          * updating everything for this ordered extent.
3135          */
3136         btrfs_remove_ordered_extent(inode, ordered_extent);
3137 
3138         /* for snapshot-aware defrag */
3139         if (new) {
3140                 if (ret) {
3141                         free_sa_defrag_extent(new);
3142                         atomic_dec(&fs_info->defrag_running);
3143                 } else {
3144                         relink_file_extents(new);
3145                 }
3146         }
3147 
3148         /* once for us */
3149         btrfs_put_ordered_extent(ordered_extent);
3150         /* once for the tree */
3151         btrfs_put_ordered_extent(ordered_extent);
3152 
3153         return ret;
3154 }
3155 
3156 static void finish_ordered_fn(struct btrfs_work *work)
3157 {
3158         struct btrfs_ordered_extent *ordered_extent;
3159         ordered_extent = container_of(work, struct btrfs_ordered_extent, work);
3160         btrfs_finish_ordered_io(ordered_extent);
3161 }
3162 
3163 static void btrfs_writepage_end_io_hook(struct page *page, u64 start, u64 end,
3164                                 struct extent_state *state, int uptodate)
3165 {
3166         struct inode *inode = page->mapping->host;
3167         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
3168         struct btrfs_ordered_extent *ordered_extent = NULL;
3169         struct btrfs_workqueue *wq;
3170         btrfs_work_func_t func;
3171 
3172         trace_btrfs_writepage_end_io_hook(page, start, end, uptodate);
3173 
3174         ClearPagePrivate2(page);
3175         if (!btrfs_dec_test_ordered_pending(inode, &ordered_extent, start,
3176                                             end - start + 1, uptodate))
3177                 return;
3178 
3179         if (btrfs_is_free_space_inode(BTRFS_I(inode))) {
3180                 wq = fs_info->endio_freespace_worker;
3181                 func = btrfs_freespace_write_helper;
3182         } else {
3183                 wq = fs_info->endio_write_workers;
3184                 func = btrfs_endio_write_helper;
3185         }
3186 
3187         btrfs_init_work(&ordered_extent->work, func, finish_ordered_fn, NULL,
3188                         NULL);
3189         btrfs_queue_work(wq, &ordered_extent->work);
3190 }
3191 
3192 static int __readpage_endio_check(struct inode *inode,
3193                                   struct btrfs_io_bio *io_bio,
3194                                   int icsum, struct page *page,
3195                                   int pgoff, u64 start, size_t len)
3196 {
3197         char *kaddr;
3198         u32 csum_expected;
3199         u32 csum = ~(u32)0;
3200 
3201         csum_expected = *(((u32 *)io_bio->csum) + icsum);
3202 
3203         kaddr = kmap_atomic(page);
3204         csum = btrfs_csum_data(kaddr + pgoff, csum,  len);
3205         btrfs_csum_final(csum, (u8 *)&csum);
3206         if (csum != csum_expected)
3207                 goto zeroit;
3208 
3209         kunmap_atomic(kaddr);
3210         return 0;
3211 zeroit:
3212         btrfs_print_data_csum_error(BTRFS_I(inode), start, csum, csum_expected,
3213                                     io_bio->mirror_num);
3214         memset(kaddr + pgoff, 1, len);
3215         flush_dcache_page(page);
3216         kunmap_atomic(kaddr);
3217         return -EIO;
3218 }
3219 
3220 /*
3221  * when reads are done, we need to check csums to verify the data is correct
3222  * if there's a match, we allow the bio to finish.  If not, the code in
3223  * extent_io.c will try to find good copies for us.
3224  */
3225 static int btrfs_readpage_end_io_hook(struct btrfs_io_bio *io_bio,
3226                                       u64 phy_offset, struct page *page,
3227                                       u64 start, u64 end, int mirror)
3228 {
3229         size_t offset = start - page_offset(page);
3230         struct inode *inode = page->mapping->host;
3231         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
3232         struct btrfs_root *root = BTRFS_I(inode)->root;
3233 
3234         if (PageChecked(page)) {
3235                 ClearPageChecked(page);
3236                 return 0;
3237         }
3238 
3239         if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)
3240                 return 0;
3241 
3242         if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID &&
3243             test_range_bit(io_tree, start, end, EXTENT_NODATASUM, 1, NULL)) {
3244                 clear_extent_bits(io_tree, start, end, EXTENT_NODATASUM);
3245                 return 0;
3246         }
3247 
3248         phy_offset >>= inode->i_sb->s_blocksize_bits;
3249         return __readpage_endio_check(inode, io_bio, phy_offset, page, offset,
3250                                       start, (size_t)(end - start + 1));
3251 }
3252 
3253 /*
3254  * btrfs_add_delayed_iput - perform a delayed iput on @inode
3255  *
3256  * @inode: The inode we want to perform iput on
3257  *
3258  * This function uses the generic vfs_inode::i_count to track whether we should
3259  * just decrement it (in case it's > 1) or if this is the last iput then link
3260  * the inode to the delayed iput machinery. Delayed iputs are processed at
3261  * transaction commit time/superblock commit/cleaner kthread.
3262  */
3263 void btrfs_add_delayed_iput(struct inode *inode)
3264 {
3265         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
3266         struct btrfs_inode *binode = BTRFS_I(inode);
3267 
3268         if (atomic_add_unless(&inode->i_count, -1, 1))
3269                 return;
3270 
3271         spin_lock(&fs_info->delayed_iput_lock);
3272         ASSERT(list_empty(&binode->delayed_iput));
3273         list_add_tail(&binode->delayed_iput, &fs_info->delayed_iputs);
3274         spin_unlock(&fs_info->delayed_iput_lock);
3275 }
3276 
3277 void btrfs_run_delayed_iputs(struct btrfs_fs_info *fs_info)
3278 {
3279 
3280         spin_lock(&fs_info->delayed_iput_lock);
3281         while (!list_empty(&fs_info->delayed_iputs)) {
3282                 struct btrfs_inode *inode;
3283 
3284                 inode = list_first_entry(&fs_info->delayed_iputs,
3285                                 struct btrfs_inode, delayed_iput);
3286                 list_del_init(&inode->delayed_iput);
3287                 spin_unlock(&fs_info->delayed_iput_lock);
3288                 iput(&inode->vfs_inode);
3289                 spin_lock(&fs_info->delayed_iput_lock);
3290         }
3291         spin_unlock(&fs_info->delayed_iput_lock);
3292 }
3293 
3294 /*
3295  * This is called in transaction commit time. If there are no orphan
3296  * files in the subvolume, it removes orphan item and frees block_rsv
3297  * structure.
3298  */
3299 void btrfs_orphan_commit_root(struct btrfs_trans_handle *trans,
3300                               struct btrfs_root *root)
3301 {
3302         struct btrfs_fs_info *fs_info = root->fs_info;
3303         struct btrfs_block_rsv *block_rsv;
3304         int ret;
3305 
3306         if (atomic_read(&root->orphan_inodes) ||
3307             root->orphan_cleanup_state != ORPHAN_CLEANUP_DONE)
3308                 return;
3309 
3310         spin_lock(&root->orphan_lock);
3311         if (atomic_read(&root->orphan_inodes)) {
3312                 spin_unlock(&root->orphan_lock);
3313                 return;
3314         }
3315 
3316         if (root->orphan_cleanup_state != ORPHAN_CLEANUP_DONE) {
3317                 spin_unlock(&root->orphan_lock);
3318                 return;
3319         }
3320 
3321         block_rsv = root->orphan_block_rsv;
3322         root->orphan_block_rsv = NULL;
3323         spin_unlock(&root->orphan_lock);
3324 
3325         if (test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state) &&
3326             btrfs_root_refs(&root->root_item) > 0) {
3327                 ret = btrfs_del_orphan_item(trans, fs_info->tree_root,
3328                                             root->root_key.objectid);
3329                 if (ret)
3330                         btrfs_abort_transaction(trans, ret);
3331                 else
3332                         clear_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED,
3333                                   &root->state);
3334         }
3335 
3336         if (block_rsv) {
3337                 WARN_ON(block_rsv->size > 0);
3338                 btrfs_free_block_rsv(fs_info, block_rsv);
3339         }
3340 }
3341 
3342 /*
3343  * This creates an orphan entry for the given inode in case something goes
3344  * wrong in the middle of an unlink/truncate.
3345  *
3346  * NOTE: caller of this function should reserve 5 units of metadata for
3347  *       this function.
3348  */
3349 int btrfs_orphan_add(struct btrfs_trans_handle *trans,
3350                 struct btrfs_inode *inode)
3351 {
3352         struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
3353         struct btrfs_root *root = inode->root;
3354         struct btrfs_block_rsv *block_rsv = NULL;
3355         int reserve = 0;
3356         bool insert = false;
3357         int ret;
3358 
3359         if (!root->orphan_block_rsv) {
3360                 block_rsv = btrfs_alloc_block_rsv(fs_info,
3361                                                   BTRFS_BLOCK_RSV_TEMP);
3362                 if (!block_rsv)
3363                         return -ENOMEM;
3364         }
3365 
3366         if (!test_and_set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3367                               &inode->runtime_flags))
3368                 insert = true;
3369 
3370         if (!test_and_set_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
3371                               &inode->runtime_flags))
3372                 reserve = 1;
3373 
3374         spin_lock(&root->orphan_lock);
3375         /* If someone has created ->orphan_block_rsv, be happy to use it. */
3376         if (!root->orphan_block_rsv) {
3377                 root->orphan_block_rsv = block_rsv;
3378         } else if (block_rsv) {
3379                 btrfs_free_block_rsv(fs_info, block_rsv);
3380                 block_rsv = NULL;
3381         }
3382 
3383         if (insert)
3384                 atomic_inc(&root->orphan_inodes);
3385         spin_unlock(&root->orphan_lock);
3386 
3387         /* grab metadata reservation from transaction handle */
3388         if (reserve) {
3389                 ret = btrfs_orphan_reserve_metadata(trans, inode);
3390                 ASSERT(!ret);
3391                 if (ret) {
3392                         /*
3393                          * dec doesn't need spin_lock as ->orphan_block_rsv
3394                          * would be released only if ->orphan_inodes is
3395                          * zero.
3396                          */
3397                         atomic_dec(&root->orphan_inodes);
3398                         clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
3399                                   &inode->runtime_flags);
3400                         if (insert)
3401                                 clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3402                                           &inode->runtime_flags);
3403                         return ret;
3404                 }
3405         }
3406 
3407         /* insert an orphan item to track this unlinked/truncated file */
3408         if (insert) {
3409                 ret = btrfs_insert_orphan_item(trans, root, btrfs_ino(inode));
3410                 if (ret) {
3411                         if (reserve) {
3412                                 clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
3413                                           &inode->runtime_flags);
3414                                 btrfs_orphan_release_metadata(inode);
3415                         }
3416                         /*
3417                          * btrfs_orphan_commit_root may race with us and set
3418                          * ->orphan_block_rsv to zero, in order to avoid that,
3419                          * decrease ->orphan_inodes after everything is done.
3420                          */
3421                         atomic_dec(&root->orphan_inodes);
3422                         if (ret != -EEXIST) {
3423                                 clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3424                                           &inode->runtime_flags);
3425                                 btrfs_abort_transaction(trans, ret);
3426                                 return ret;
3427                         }
3428                 }
3429                 ret = 0;
3430         }
3431 
3432         return 0;
3433 }
3434 
3435 /*
3436  * We have done the truncate/delete so we can go ahead and remove the orphan
3437  * item for this particular inode.
3438  */
3439 static int btrfs_orphan_del(struct btrfs_trans_handle *trans,
3440                             struct btrfs_inode *inode)
3441 {
3442         struct btrfs_root *root = inode->root;
3443         int delete_item = 0;
3444         int ret = 0;
3445 
3446         if (test_and_clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3447                                &inode->runtime_flags))
3448                 delete_item = 1;
3449 
3450         if (delete_item && trans)
3451                 ret = btrfs_del_orphan_item(trans, root, btrfs_ino(inode));
3452 
3453         if (test_and_clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
3454                                &inode->runtime_flags))
3455                 btrfs_orphan_release_metadata(inode);
3456 
3457         /*
3458          * btrfs_orphan_commit_root may race with us and set ->orphan_block_rsv
3459          * to zero, in order to avoid that, decrease ->orphan_inodes after
3460          * everything is done.
3461          */
3462         if (delete_item)
3463                 atomic_dec(&root->orphan_inodes);
3464 
3465         return ret;
3466 }
3467 
3468 /*
3469  * this cleans up any orphans that may be left on the list from the last use
3470  * of this root.
3471  */
3472 int btrfs_orphan_cleanup(struct btrfs_root *root)
3473 {
3474         struct btrfs_fs_info *fs_info = root->fs_info;
3475         struct btrfs_path *path;
3476         struct extent_buffer *leaf;
3477         struct btrfs_key key, found_key;
3478         struct btrfs_trans_handle *trans;
3479         struct inode *inode;
3480         u64 last_objectid = 0;
3481         int ret = 0, nr_unlink = 0, nr_truncate = 0;
3482 
3483         if (cmpxchg(&root->orphan_cleanup_state, 0, ORPHAN_CLEANUP_STARTED))
3484                 return 0;
3485 
3486         path = btrfs_alloc_path();
3487         if (!path) {
3488                 ret = -ENOMEM;
3489                 goto out;
3490         }
3491         path->reada = READA_BACK;
3492 
3493         key.objectid = BTRFS_ORPHAN_OBJECTID;
3494         key.type = BTRFS_ORPHAN_ITEM_KEY;
3495         key.offset = (u64)-1;
3496 
3497         while (1) {
3498                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3499                 if (ret < 0)
3500                         goto out;
3501 
3502                 /*
3503                  * if ret == 0 means we found what we were searching for, which
3504                  * is weird, but possible, so only screw with path if we didn't
3505                  * find the key and see if we have stuff that matches
3506                  */
3507                 if (ret > 0) {
3508                         ret = 0;
3509                         if (path->slots[0] == 0)
3510                                 break;
3511                         path->slots[0]--;
3512                 }
3513 
3514                 /* pull out the item */
3515                 leaf = path->nodes[0];
3516                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
3517 
3518                 /* make sure the item matches what we want */
3519                 if (found_key.objectid != BTRFS_ORPHAN_OBJECTID)
3520                         break;
3521                 if (found_key.type != BTRFS_ORPHAN_ITEM_KEY)
3522                         break;
3523 
3524                 /* release the path since we're done with it */
3525                 btrfs_release_path(path);
3526 
3527                 /*
3528                  * this is where we are basically btrfs_lookup, without the
3529                  * crossing root thing.  we store the inode number in the
3530                  * offset of the orphan item.
3531                  */
3532 
3533                 if (found_key.offset == last_objectid) {
3534                         btrfs_err(fs_info,
3535                                   "Error removing orphan entry, stopping orphan cleanup");
3536                         ret = -EINVAL;
3537                         goto out;
3538                 }
3539 
3540                 last_objectid = found_key.offset;
3541 
3542                 found_key.objectid = found_key.offset;
3543                 found_key.type = BTRFS_INODE_ITEM_KEY;
3544                 found_key.offset = 0;
3545                 inode = btrfs_iget(fs_info->sb, &found_key, root, NULL);
3546                 ret = PTR_ERR_OR_ZERO(inode);
3547                 if (ret && ret != -ENOENT)
3548                         goto out;
3549 
3550                 if (ret == -ENOENT && root == fs_info->tree_root) {
3551                         struct btrfs_root *dead_root;
3552                         struct btrfs_fs_info *fs_info = root->fs_info;
3553                         int is_dead_root = 0;
3554 
3555                         /*
3556                          * this is an orphan in the tree root. Currently these
3557                          * could come from 2 sources:
3558                          *  a) a snapshot deletion in progress
3559                          *  b) a free space cache inode
3560                          * We need to distinguish those two, as the snapshot
3561                          * orphan must not get deleted.
3562                          * find_dead_roots already ran before us, so if this
3563                          * is a snapshot deletion, we should find the root
3564                          * in the dead_roots list
3565                          */
3566                         spin_lock(&fs_info->trans_lock);
3567                         list_for_each_entry(dead_root, &fs_info->dead_roots,
3568                                             root_list) {
3569                                 if (dead_root->root_key.objectid ==
3570                                     found_key.objectid) {
3571                                         is_dead_root = 1;
3572                                         break;
3573                                 }
3574                         }
3575                         spin_unlock(&fs_info->trans_lock);
3576                         if (is_dead_root) {
3577                                 /* prevent this orphan from being found again */
3578                                 key.offset = found_key.objectid - 1;
3579                                 continue;
3580                         }
3581                 }
3582                 /*
3583                  * Inode is already gone but the orphan item is still there,
3584                  * kill the orphan item.
3585                  */
3586                 if (ret == -ENOENT) {
3587                         trans = btrfs_start_transaction(root, 1);
3588                         if (IS_ERR(trans)) {
3589                                 ret = PTR_ERR(trans);
3590                                 goto out;
3591                         }
3592                         btrfs_debug(fs_info, "auto deleting %Lu",
3593                                     found_key.objectid);
3594                         ret = btrfs_del_orphan_item(trans, root,
3595                                                     found_key.objectid);
3596                         btrfs_end_transaction(trans);
3597                         if (ret)
3598                                 goto out;
3599                         continue;
3600                 }
3601 
3602                 /*
3603                  * add this inode to the orphan list so btrfs_orphan_del does
3604                  * the proper thing when we hit it
3605                  */
3606                 set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3607                         &BTRFS_I(inode)->runtime_flags);
3608                 atomic_inc(&root->orphan_inodes);
3609 
3610                 /* if we have links, this was a truncate, lets do that */
3611                 if (inode->i_nlink) {
3612                         if (WARN_ON(!S_ISREG(inode->i_mode))) {
3613                                 iput(inode);
3614                                 continue;
3615                         }
3616                         nr_truncate++;
3617 
3618                         /* 1 for the orphan item deletion. */
3619                         trans = btrfs_start_transaction(root, 1);
3620                         if (IS_ERR(trans)) {
3621                                 iput(inode);
3622                                 ret = PTR_ERR(trans);
3623                                 goto out;
3624                         }
3625                         ret = btrfs_orphan_add(trans, BTRFS_I(inode));
3626                         btrfs_end_transaction(trans);
3627                         if (ret) {
3628                                 iput(inode);
3629                                 goto out;
3630                         }
3631 
3632                         ret = btrfs_truncate(inode, false);
3633                         if (ret)
3634                                 btrfs_orphan_del(NULL, BTRFS_I(inode));
3635                 } else {
3636                         nr_unlink++;
3637                 }
3638 
3639                 /* this will do delete_inode and everything for us */
3640                 iput(inode);
3641                 if (ret)
3642                         goto out;
3643         }
3644         /* release the path since we're done with it */
3645         btrfs_release_path(path);
3646 
3647         root->orphan_cleanup_state = ORPHAN_CLEANUP_DONE;
3648 
3649         if (root->orphan_block_rsv)
3650                 btrfs_block_rsv_release(fs_info, root->orphan_block_rsv,
3651                                         (u64)-1);
3652 
3653         if (root->orphan_block_rsv ||
3654             test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state)) {
3655                 trans = btrfs_join_transaction(root);
3656                 if (!IS_ERR(trans))
3657                         btrfs_end_transaction(trans);
3658         }
3659 
3660         if (nr_unlink)
3661                 btrfs_debug(fs_info, "unlinked %d orphans", nr_unlink);
3662         if (nr_truncate)
3663                 btrfs_debug(fs_info, "truncated %d orphans", nr_truncate);
3664 
3665 out:
3666         if (ret)
3667                 btrfs_err(fs_info, "could not do orphan cleanup %d", ret);
3668         btrfs_free_path(path);
3669         return ret;
3670 }
3671 
3672 /*
3673  * very simple check to peek ahead in the leaf looking for xattrs.  If we
3674  * don't find any xattrs, we know there can't be any acls.
3675  *
3676  * slot is the slot the inode is in, objectid is the objectid of the inode
3677  */
3678 static noinline int acls_after_inode_item(struct extent_buffer *leaf,
3679                                           int slot, u64 objectid,
3680                                           int *first_xattr_slot)
3681 {
3682         u32 nritems = btrfs_header_nritems(leaf);
3683         struct btrfs_key found_key;
3684         static u64 xattr_access = 0;
3685         static u64 xattr_default = 0;
3686         int scanned = 0;
3687 
3688         if (!xattr_access) {
3689                 xattr_access = btrfs_name_hash(XATTR_NAME_POSIX_ACL_ACCESS,
3690                                         strlen(XATTR_NAME_POSIX_ACL_ACCESS));
3691                 xattr_default = btrfs_name_hash(XATTR_NAME_POSIX_ACL_DEFAULT,
3692                                         strlen(XATTR_NAME_POSIX_ACL_DEFAULT));
3693         }
3694 
3695         slot++;
3696         *first_xattr_slot = -1;
3697         while (slot < nritems) {
3698                 btrfs_item_key_to_cpu(leaf, &found_key, slot);
3699 
3700                 /* we found a different objectid, there must not be acls */
3701                 if (found_key.objectid != objectid)
3702                         return 0;
3703 
3704                 /* we found an xattr, assume we've got an acl */
3705                 if (found_key.type == BTRFS_XATTR_ITEM_KEY) {
3706                         if (*first_xattr_slot == -1)
3707                                 *first_xattr_slot = slot;
3708                         if (found_key.offset == xattr_access ||
3709                             found_key.offset == xattr_default)
3710                                 return 1;
3711                 }
3712 
3713                 /*
3714                  * we found a key greater than an xattr key, there can't
3715                  * be any acls later on
3716                  */
3717                 if (found_key.type > BTRFS_XATTR_ITEM_KEY)
3718                         return 0;
3719 
3720                 slot++;
3721                 scanned++;
3722 
3723                 /*
3724                  * it goes inode, inode backrefs, xattrs, extents,
3725                  * so if there are a ton of hard links to an inode there can
3726                  * be a lot of backrefs.  Don't waste time searching too hard,
3727                  * this is just an optimization
3728                  */
3729                 if (scanned >= 8)
3730                         break;
3731         }
3732         /* we hit the end of the leaf before we found an xattr or
3733          * something larger than an xattr.  We have to assume the inode
3734          * has acls
3735          */
3736         if (*first_xattr_slot == -1)
3737                 *first_xattr_slot = slot;
3738         return 1;
3739 }
3740 
3741 /*
3742  * read an inode from the btree into the in-memory inode
3743  */
3744 static int btrfs_read_locked_inode(struct inode *inode)
3745 {
3746         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
3747         struct btrfs_path *path;
3748         struct extent_buffer *leaf;
3749         struct btrfs_inode_item *inode_item;
3750         struct btrfs_root *root = BTRFS_I(inode)->root;
3751         struct btrfs_key location;
3752         unsigned long ptr;
3753         int maybe_acls;
3754         u32 rdev;
3755         int ret;
3756         bool filled = false;
3757         int first_xattr_slot;
3758 
3759         ret = btrfs_fill_inode(inode, &rdev);
3760         if (!ret)
3761                 filled = true;
3762 
3763         path = btrfs_alloc_path();
3764         if (!path) {
3765                 ret = -ENOMEM;
3766                 goto make_bad;
3767         }
3768 
3769         memcpy(&location, &BTRFS_I(inode)->location, sizeof(location));
3770 
3771         ret = btrfs_lookup_inode(NULL, root, path, &location, 0);
3772         if (ret) {
3773                 if (ret > 0)
3774                         ret = -ENOENT;
3775                 goto make_bad;
3776         }
3777 
3778         leaf = path->nodes[0];
3779 
3780         if (filled)
3781                 goto cache_index;
3782 
3783         inode_item = btrfs_item_ptr(leaf, path->slots[0],
3784                                     struct btrfs_inode_item);
3785         inode->i_mode = btrfs_inode_mode(leaf, inode_item);
3786         set_nlink(inode, btrfs_inode_nlink(leaf, inode_item));
3787         i_uid_write(inode, btrfs_inode_uid(leaf, inode_item));
3788         i_gid_write(inode, btrfs_inode_gid(leaf, inode_item));
3789         btrfs_i_size_write(BTRFS_I(inode), btrfs_inode_size(leaf, inode_item));
3790 
3791         inode->i_atime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->atime);
3792         inode->i_atime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->atime);
3793 
3794         inode->i_mtime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->mtime);
3795         inode->i_mtime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->mtime);
3796 
3797         inode->i_ctime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->ctime);
3798         inode->i_ctime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->ctime);
3799 
3800         BTRFS_I(inode)->i_otime.tv_sec =
3801                 btrfs_timespec_sec(leaf, &inode_item->otime);
3802         BTRFS_I(inode)->i_otime.tv_nsec =
3803                 btrfs_timespec_nsec(leaf, &inode_item->otime);
3804 
3805         inode_set_bytes(inode, btrfs_inode_nbytes(leaf, inode_item));
3806         BTRFS_I(inode)->generation = btrfs_inode_generation(leaf, inode_item);
3807         BTRFS_I(inode)->last_trans = btrfs_inode_transid(leaf, inode_item);
3808 
3809         inode_set_iversion_queried(inode,
3810                                    btrfs_inode_sequence(leaf, inode_item));
3811         inode->i_generation = BTRFS_I(inode)->generation;
3812         inode->i_rdev = 0;
3813         rdev = btrfs_inode_rdev(leaf, inode_item);
3814 
3815         BTRFS_I(inode)->index_cnt = (u64)-1;
3816         BTRFS_I(inode)->flags = btrfs_inode_flags(leaf, inode_item);
3817 
3818 cache_index:
3819         /*
3820          * If we were modified in the current generation and evicted from memory
3821          * and then re-read we need to do a full sync since we don't have any
3822          * idea about which extents were modified before we were evicted from
3823          * cache.
3824          *
3825          * This is required for both inode re-read from disk and delayed inode
3826          * in delayed_nodes_tree.
3827          */
3828         if (BTRFS_I(inode)->last_trans == fs_info->generation)
3829                 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
3830                         &BTRFS_I(inode)->runtime_flags);
3831 
3832         /*
3833          * We don't persist the id of the transaction where an unlink operation
3834          * against the inode was last made. So here we assume the inode might
3835          * have been evicted, and therefore the exact value of last_unlink_trans
3836          * lost, and set it to last_trans to avoid metadata inconsistencies
3837          * between the inode and its parent if the inode is fsync'ed and the log
3838          * replayed. For example, in the scenario:
3839          *
3840          * touch mydir/foo
3841          * ln mydir/foo mydir/bar
3842          * sync
3843          * unlink mydir/bar
3844          * echo 2 > /proc/sys/vm/drop_caches   # evicts inode
3845          * xfs_io -c fsync mydir/foo
3846          * <power failure>
3847          * mount fs, triggers fsync log replay
3848          *
3849          * We must make sure that when we fsync our inode foo we also log its
3850          * parent inode, otherwise after log replay the parent still has the
3851          * dentry with the "bar" name but our inode foo has a link count of 1
3852          * and doesn't have an inode ref with the name "bar" anymore.
3853          *
3854          * Setting last_unlink_trans to last_trans is a pessimistic approach,
3855          * but it guarantees correctness at the expense of occasional full
3856          * transaction commits on fsync if our inode is a directory, or if our
3857          * inode is not a directory, logging its parent unnecessarily.
3858          */
3859         BTRFS_I(inode)->last_unlink_trans = BTRFS_I(inode)->last_trans;
3860 
3861         path->slots[0]++;
3862         if (inode->i_nlink != 1 ||
3863             path->slots[0] >= btrfs_header_nritems(leaf))
3864                 goto cache_acl;
3865 
3866         btrfs_item_key_to_cpu(leaf, &location, path->slots[0]);
3867         if (location.objectid != btrfs_ino(BTRFS_I(inode)))
3868                 goto cache_acl;
3869 
3870         ptr = btrfs_item_ptr_offset(leaf, path->slots[0]);
3871         if (location.type == BTRFS_INODE_REF_KEY) {
3872                 struct btrfs_inode_ref *ref;
3873 
3874                 ref = (struct btrfs_inode_ref *)ptr;
3875                 BTRFS_I(inode)->dir_index = btrfs_inode_ref_index(leaf, ref);
3876         } else if (location.type == BTRFS_INODE_EXTREF_KEY) {
3877                 struct btrfs_inode_extref *extref;
3878 
3879                 extref = (struct btrfs_inode_extref *)ptr;
3880                 BTRFS_I(inode)->dir_index = btrfs_inode_extref_index(leaf,
3881                                                                      extref);
3882         }
3883 cache_acl:
3884         /*
3885          * try to precache a NULL acl entry for files that don't have
3886          * any xattrs or acls
3887          */
3888         maybe_acls = acls_after_inode_item(leaf, path->slots[0],
3889                         btrfs_ino(BTRFS_I(inode)), &first_xattr_slot);
3890         if (first_xattr_slot != -1) {
3891                 path->slots[0] = first_xattr_slot;
3892                 ret = btrfs_load_inode_props(inode, path);
3893                 if (ret)
3894                         btrfs_err(fs_info,
3895                                   "error loading props for ino %llu (root %llu): %d",
3896                                   btrfs_ino(BTRFS_I(inode)),
3897                                   root->root_key.objectid, ret);
3898         }
3899         btrfs_free_path(path);
3900 
3901         if (!maybe_acls)
3902                 cache_no_acl(inode);
3903 
3904         switch (inode->i_mode & S_IFMT) {
3905         case S_IFREG:
3906                 inode->i_mapping->a_ops = &btrfs_aops;
3907                 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
3908                 inode->i_fop = &btrfs_file_operations;
3909                 inode->i_op = &btrfs_file_inode_operations;
3910                 break;
3911         case S_IFDIR:
3912                 inode->i_fop = &btrfs_dir_file_operations;
3913                 inode->i_op = &btrfs_dir_inode_operations;
3914                 break;
3915         case S_IFLNK:
3916                 inode->i_op = &btrfs_symlink_inode_operations;
3917                 inode_nohighmem(inode);
3918                 inode->i_mapping->a_ops = &btrfs_symlink_aops;
3919                 break;
3920         default:
3921                 inode->i_op = &btrfs_special_inode_operations;
3922                 init_special_inode(inode, inode->i_mode, rdev);
3923                 break;
3924         }
3925 
3926         btrfs_update_iflags(inode);
3927         return 0;
3928 
3929 make_bad:
3930         btrfs_free_path(path);
3931         make_bad_inode(inode);
3932         return ret;
3933 }
3934 
3935 /*
3936  * given a leaf and an inode, copy the inode fields into the leaf
3937  */
3938 static void fill_inode_item(struct btrfs_trans_handle *trans,
3939                             struct extent_buffer *leaf,
3940                             struct btrfs_inode_item *item,
3941                             struct inode *inode)
3942 {
3943         struct btrfs_map_token token;
3944 
3945         btrfs_init_map_token(&token);
3946 
3947         btrfs_set_token_inode_uid(leaf, item, i_uid_read(inode), &token);
3948         btrfs_set_token_inode_gid(leaf, item, i_gid_read(inode), &token);
3949         btrfs_set_token_inode_size(leaf, item, BTRFS_I(inode)->disk_i_size,
3950                                    &token);
3951         btrfs_set_token_inode_mode(leaf, item, inode->i_mode, &token);
3952         btrfs_set_token_inode_nlink(leaf, item, inode->i_nlink, &token);
3953 
3954         btrfs_set_token_timespec_sec(leaf, &item->atime,
3955                                      inode->i_atime.tv_sec, &token);
3956         btrfs_set_token_timespec_nsec(leaf, &item->atime,
3957                                       inode->i_atime.tv_nsec, &token);
3958 
3959         btrfs_set_token_timespec_sec(leaf, &item->mtime,
3960                                      inode->i_mtime.tv_sec, &token);
3961         btrfs_set_token_timespec_nsec(leaf, &item->mtime,
3962                                       inode->i_mtime.tv_nsec, &token);
3963 
3964         btrfs_set_token_timespec_sec(leaf, &item->ctime,
3965                                      inode->i_ctime.tv_sec, &token);
3966         btrfs_set_token_timespec_nsec(leaf, &item->ctime,
3967                                       inode->i_ctime.tv_nsec, &token);
3968 
3969         btrfs_set_token_timespec_sec(leaf, &item->otime,
3970                                      BTRFS_I(inode)->i_otime.tv_sec, &token);
3971         btrfs_set_token_timespec_nsec(leaf, &item->otime,
3972                                       BTRFS_I(inode)->i_otime.tv_nsec, &token);
3973 
3974         btrfs_set_token_inode_nbytes(leaf, item, inode_get_bytes(inode),
3975                                      &token);
3976         btrfs_set_token_inode_generation(leaf, item, BTRFS_I(inode)->generation,
3977                                          &token);
3978         btrfs_set_token_inode_sequence(leaf, item, inode_peek_iversion(inode),
3979                                        &token);
3980         btrfs_set_token_inode_transid(leaf, item, trans->transid, &token);
3981         btrfs_set_token_inode_rdev(leaf, item, inode->i_rdev, &token);
3982         btrfs_set_token_inode_flags(leaf, item, BTRFS_I(inode)->flags, &token);
3983         btrfs_set_token_inode_block_group(leaf, item, 0, &token);
3984 }
3985 
3986 /*
3987  * copy everything in the in-memory inode into the btree.
3988  */
3989 static noinline int btrfs_update_inode_item(struct btrfs_trans_handle *trans,
3990                                 struct btrfs_root *root, struct inode *inode)
3991 {
3992         struct btrfs_inode_item *inode_item;
3993         struct btrfs_path *path;
3994         struct extent_buffer *leaf;
3995         int ret;
3996 
3997         path = btrfs_alloc_path();
3998         if (!path)
3999                 return -ENOMEM;
4000 
4001         path->leave_spinning = 1;
4002         ret = btrfs_lookup_inode(trans, root, path, &BTRFS_I(inode)->location,
4003                                  1);
4004         if (ret) {
4005                 if (ret > 0)
4006                         ret = -ENOENT;
4007                 goto failed;
4008         }
4009 
4010         leaf = path->nodes[0];
4011         inode_item = btrfs_item_ptr(leaf, path->slots[0],
4012                                     struct btrfs_inode_item);
4013 
4014         fill_inode_item(trans, leaf, inode_item, inode);
4015         btrfs_mark_buffer_dirty(leaf);
4016         btrfs_set_inode_last_trans(trans, inode);
4017         ret = 0;
4018 failed:
4019         btrfs_free_path(path);
4020         return ret;
4021 }
4022 
4023 /*
4024  * copy everything in the in-memory inode into the btree.
4025  */
4026 noinline int btrfs_update_inode(struct btrfs_trans_handle *trans,
4027                                 struct btrfs_root *root, struct inode *inode)
4028 {
4029         struct btrfs_fs_info *fs_info = root->fs_info;
4030         int ret;
4031 
4032         /*
4033          * If the inode is a free space inode, we can deadlock during commit
4034          * if we put it into the delayed code.
4035          *
4036          * The data relocation inode should also be directly updated
4037          * without delay
4038          */
4039         if (!btrfs_is_free_space_inode(BTRFS_I(inode))
4040             && root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID
4041             && !test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) {
4042                 btrfs_update_root_times(trans, root);
4043 
4044                 ret = btrfs_delayed_update_inode(trans, root, inode);
4045                 if (!ret)
4046                         btrfs_set_inode_last_trans(trans, inode);
4047                 return ret;
4048         }
4049 
4050         return btrfs_update_inode_item(trans, root, inode);
4051 }
4052 
4053 noinline int btrfs_update_inode_fallback(struct btrfs_trans_handle *trans,
4054                                          struct btrfs_root *root,
4055                                          struct inode *inode)
4056 {
4057         int ret;
4058 
4059         ret = btrfs_update_inode(trans, root, inode);
4060         if (ret == -ENOSPC)
4061                 return btrfs_update_inode_item(trans, root, inode);
4062         return ret;
4063 }
4064 
4065 /*
4066  * unlink helper that gets used here in inode.c and in the tree logging
4067  * recovery code.  It remove a link in a directory with a given name, and
4068  * also drops the back refs in the inode to the directory
4069  */
4070 static int __btrfs_unlink_inode(struct btrfs_trans_handle *trans,
4071                                 struct btrfs_root *root,
4072                                 struct btrfs_inode *dir,
4073                                 struct btrfs_inode *inode,
4074                                 const char *name, int name_len)
4075 {
4076         struct btrfs_fs_info *fs_info = root->fs_info;
4077         struct btrfs_path *path;
4078         int ret = 0;
4079         struct extent_buffer *leaf;
4080         struct btrfs_dir_item *di;
4081         struct btrfs_key key;
4082         u64 index;
4083         u64 ino = btrfs_ino(inode);
4084         u64 dir_ino = btrfs_ino(dir);
4085 
4086         path = btrfs_alloc_path();
4087         if (!path) {
4088                 ret = -ENOMEM;
4089                 goto out;
4090         }
4091 
4092         path->leave_spinning = 1;
4093         di = btrfs_lookup_dir_item(trans, root, path, dir_ino,
4094                                     name, name_len, -1);
4095         if (IS_ERR(di)) {
4096                 ret = PTR_ERR(di);
4097                 goto err;
4098         }
4099         if (!di) {
4100                 ret = -ENOENT;
4101                 goto err;
4102         }
4103         leaf = path->nodes[0];
4104         btrfs_dir_item_key_to_cpu(leaf, di, &key);
4105         ret = btrfs_delete_one_dir_name(trans, root, path, di);
4106         if (ret)
4107                 goto err;
4108         btrfs_release_path(path);
4109 
4110         /*
4111          * If we don't have dir index, we have to get it by looking up
4112          * the inode ref, since we get the inode ref, remove it directly,
4113          * it is unnecessary to do delayed deletion.
4114          *
4115          * But if we have dir index, needn't search inode ref to get it.
4116          * Since the inode ref is close to the inode item, it is better
4117          * that we delay to delete it, and just do this deletion when
4118          * we update the inode item.
4119          */
4120         if (inode->dir_index) {
4121                 ret = btrfs_delayed_delete_inode_ref(inode);
4122                 if (!ret) {
4123                         index = inode->dir_index;
4124                         goto skip_backref;
4125                 }
4126         }
4127 
4128         ret = btrfs_del_inode_ref(trans, root, name, name_len, ino,
4129                                   dir_ino, &index);
4130         if (ret) {
4131                 btrfs_info(fs_info,
4132                         "failed to delete reference to %.*s, inode %llu parent %llu",
4133                         name_len, name, ino, dir_ino);
4134                 btrfs_abort_transaction(trans, ret);
4135                 goto err;
4136         }
4137 skip_backref:
4138         ret = btrfs_delete_delayed_dir_index(trans, fs_info, dir, index);
4139         if (ret) {
4140                 btrfs_abort_transaction(trans, ret);
4141                 goto err;
4142         }
4143 
4144         ret = btrfs_del_inode_ref_in_log(trans, root, name, name_len, inode,
4145                         dir_ino);
4146         if (ret != 0 && ret != -ENOENT) {
4147                 btrfs_abort_transaction(trans, ret);
4148                 goto err;
4149         }
4150 
4151         ret = btrfs_del_dir_entries_in_log(trans, root, name, name_len, dir,
4152                         index);
4153         if (ret == -ENOENT)
4154                 ret = 0;
4155         else if (ret)
4156                 btrfs_abort_transaction(trans, ret);
4157 err:
4158         btrfs_free_path(path);
4159         if (ret)
4160                 goto out;
4161 
4162         btrfs_i_size_write(dir, dir->vfs_inode.i_size - name_len * 2);
4163         inode_inc_iversion(&inode->vfs_inode);
4164         inode_inc_iversion(&dir->vfs_inode);
4165         inode->vfs_inode.i_ctime = dir->vfs_inode.i_mtime =
4166                 dir->vfs_inode.i_ctime = current_time(&inode->vfs_inode);
4167         ret = btrfs_update_inode(trans, root, &dir->vfs_inode);
4168 out:
4169         return ret;
4170 }
4171 
4172 int btrfs_unlink_inode(struct btrfs_trans_handle *trans,
4173                        struct btrfs_root *root,
4174                        struct btrfs_inode *dir, struct btrfs_inode *inode,
4175                        const char *name, int name_len)
4176 {
4177         int ret;
4178         ret = __btrfs_unlink_inode(trans, root, dir, inode, name, name_len);
4179         if (!ret) {
4180                 drop_nlink(&inode->vfs_inode);
4181                 ret = btrfs_update_inode(trans, root, &inode->vfs_inode);
4182         }
4183         return ret;
4184 }
4185 
4186 /*
4187  * helper to start transaction for unlink and rmdir.
4188  *
4189  * unlink and rmdir are special in btrfs, they do not always free space, so
4190  * if we cannot make our reservations the normal way try and see if there is
4191  * plenty of slack room in the global reserve to migrate, otherwise we cannot
4192  * allow the unlink to occur.
4193  */
4194 static struct btrfs_trans_handle *__unlink_start_trans(struct inode *dir)
4195 {
4196         struct btrfs_root *root = BTRFS_I(dir)->root;
4197 
4198         /*
4199          * 1 for the possible orphan item
4200          * 1 for the dir item
4201          * 1 for the dir index
4202          * 1 for the inode ref
4203          * 1 for the inode
4204          */
4205         return btrfs_start_transaction_fallback_global_rsv(root, 5, 5);
4206 }
4207 
4208 static int btrfs_unlink(struct inode *dir, struct dentry *dentry)
4209 {
4210         struct btrfs_root *root = BTRFS_I(dir)->root;
4211         struct btrfs_trans_handle *trans;
4212         struct inode *inode = d_inode(dentry);
4213         int ret;
4214 
4215         trans = __unlink_start_trans(dir);
4216         if (IS_ERR(trans))
4217                 return PTR_ERR(trans);
4218 
4219         btrfs_record_unlink_dir(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)),
4220                         0);
4221 
4222         ret = btrfs_unlink_inode(trans, root, BTRFS_I(dir),
4223                         BTRFS_I(d_inode(dentry)), dentry->d_name.name,
4224                         dentry->d_name.len);
4225         if (ret)
4226                 goto out;
4227 
4228         if (inode->i_nlink == 0) {
4229                 ret = btrfs_orphan_add(trans, BTRFS_I(inode));
4230                 if (ret)
4231                         goto out;
4232         }
4233 
4234 out:
4235         btrfs_end_transaction(trans);
4236         btrfs_btree_balance_dirty(root->fs_info);
4237         return ret;
4238 }
4239 
4240 int btrfs_unlink_subvol(struct btrfs_trans_handle *trans,
4241                         struct btrfs_root *root,
4242                         struct inode *dir, u64 objectid,
4243                         const char *name, int name_len)
4244 {
4245         struct btrfs_fs_info *fs_info = root->fs_info;
4246         struct btrfs_path *path;
4247         struct extent_buffer *leaf;
4248         struct btrfs_dir_item *di;
4249         struct btrfs_key key;
4250         u64 index;
4251         int ret;
4252         u64 dir_ino = btrfs_ino(BTRFS_I(dir));
4253 
4254         path = btrfs_alloc_path();
4255         if (!path)
4256                 return -ENOMEM;
4257 
4258         di = btrfs_lookup_dir_item(trans, root, path, dir_ino,
4259                                    name, name_len, -1);
4260         if (IS_ERR_OR_NULL(di)) {
4261                 if (!di)
4262                         ret = -ENOENT;
4263                 else
4264                         ret = PTR_ERR(di);
4265                 goto out;
4266         }
4267 
4268         leaf = path->nodes[0];
4269         btrfs_dir_item_key_to_cpu(leaf, di, &key);
4270         WARN_ON(key.type != BTRFS_ROOT_ITEM_KEY || key.objectid != objectid);
4271         ret = btrfs_delete_one_dir_name(trans, root, path, di);
4272         if (ret) {
4273                 btrfs_abort_transaction(trans, ret);
4274                 goto out;
4275         }
4276         btrfs_release_path(path);
4277 
4278         ret = btrfs_del_root_ref(trans, fs_info, objectid,
4279                                  root->root_key.objectid, dir_ino,
4280                                  &index, name, name_len);
4281         if (ret < 0) {
4282                 if (ret != -ENOENT) {
4283                         btrfs_abort_transaction(trans, ret);
4284                         goto out;
4285                 }
4286                 di = btrfs_search_dir_index_item(root, path, dir_ino,
4287                                                  name, name_len);
4288                 if (IS_ERR_OR_NULL(di)) {
4289                         if (!di)
4290                                 ret = -ENOENT;
4291                         else
4292                                 ret = PTR_ERR(di);
4293                         btrfs_abort_transaction(trans, ret);
4294                         goto out;
4295                 }
4296 
4297                 leaf = path->nodes[0];
4298                 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
4299                 btrfs_release_path(path);
4300                 index = key.offset;
4301         }
4302         btrfs_release_path(path);
4303 
4304         ret = btrfs_delete_delayed_dir_index(trans, fs_info, BTRFS_I(dir), index);
4305         if (ret) {
4306                 btrfs_abort_transaction(trans, ret);
4307                 goto out;
4308         }
4309 
4310         btrfs_i_size_write(BTRFS_I(dir), dir->i_size - name_len * 2);
4311         inode_inc_iversion(dir);
4312         dir->i_mtime = dir->i_ctime = current_time(dir);
4313         ret = btrfs_update_inode_fallback(trans, root, dir);
4314         if (ret)
4315                 btrfs_abort_transaction(trans, ret);
4316 out:
4317         btrfs_free_path(path);
4318         return ret;
4319 }
4320 
4321 static int btrfs_rmdir(struct inode *dir, struct dentry *dentry)
4322 {
4323         struct inode *inode = d_inode(dentry);
4324         int err = 0;
4325         struct btrfs_root *root = BTRFS_I(dir)->root;
4326         struct btrfs_trans_handle *trans;
4327         u64 last_unlink_trans;
4328 
4329         if (inode->i_size > BTRFS_EMPTY_DIR_SIZE)
4330                 return -ENOTEMPTY;
4331         if (btrfs_ino(BTRFS_I(inode)) == BTRFS_FIRST_FREE_OBJECTID)
4332                 return -EPERM;
4333 
4334         trans = __unlink_start_trans(dir);
4335         if (IS_ERR(trans))
4336                 return PTR_ERR(trans);
4337 
4338         if (unlikely(btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) {
4339                 err = btrfs_unlink_subvol(trans, root, dir,
4340                                           BTRFS_I(inode)->location.objectid,
4341                                           dentry->d_name.name,
4342                                           dentry->d_name.len);
4343                 goto out;
4344         }
4345 
4346         err = btrfs_orphan_add(trans, BTRFS_I(inode));
4347         if (err)
4348                 goto out;
4349 
4350         last_unlink_trans = BTRFS_I(inode)->last_unlink_trans;
4351 
4352         /* now the directory is empty */
4353         err = btrfs_unlink_inode(trans, root, BTRFS_I(dir),
4354                         BTRFS_I(d_inode(dentry)), dentry->d_name.name,
4355                         dentry->d_name.len);
4356         if (!err) {
4357                 btrfs_i_size_write(BTRFS_I(inode), 0);
4358                 /*
4359                  * Propagate the last_unlink_trans value of the deleted dir to
4360                  * its parent directory. This is to prevent an unrecoverable
4361                  * log tree in the case we do something like this:
4362                  * 1) create dir foo
4363                  * 2) create snapshot under dir foo
4364                  * 3) delete the snapshot
4365                  * 4) rmdir foo
4366                  * 5) mkdir foo
4367                  * 6) fsync foo or some file inside foo
4368                  */
4369                 if (last_unlink_trans >= trans->transid)
4370                         BTRFS_I(dir)->last_unlink_trans = last_unlink_trans;
4371         }
4372 out:
4373         btrfs_end_transaction(trans);
4374         btrfs_btree_balance_dirty(root->fs_info);
4375 
4376         return err;
4377 }
4378 
4379 static int truncate_space_check(struct btrfs_trans_handle *trans,
4380                                 struct btrfs_root *root,
4381                                 u64 bytes_deleted)
4382 {
4383         struct btrfs_fs_info *fs_info = root->fs_info;
4384         int ret;
4385 
4386         /*
4387          * This is only used to apply pressure to the enospc system, we don't
4388          * intend to use this reservation at all.
4389          */
4390         bytes_deleted = btrfs_csum_bytes_to_leaves(fs_info, bytes_deleted);
4391         bytes_deleted *= fs_info->nodesize;
4392         ret = btrfs_block_rsv_add(root, &fs_info->trans_block_rsv,
4393                                   bytes_deleted, BTRFS_RESERVE_NO_FLUSH);
4394         if (!ret) {
4395                 trace_btrfs_space_reservation(fs_info, "transaction",
4396                                               trans->transid,
4397                                               bytes_deleted, 1);
4398                 trans->bytes_reserved += bytes_deleted;
4399         }
4400         return ret;
4401 
4402 }
4403 
4404 /*
4405  * Return this if we need to call truncate_block for the last bit of the
4406  * truncate.
4407  */
4408 #define NEED_TRUNCATE_BLOCK 1
4409 
4410 /*
4411  * this can truncate away extent items, csum items and directory items.
4412  * It starts at a high offset and removes keys until it can't find
4413  * any higher than new_size
4414  *
4415  * csum items that cross the new i_size are truncated to the new size
4416  * as well.
4417  *
4418  * min_type is the minimum key type to truncate down to.  If set to 0, this
4419  * will kill all the items on this inode, including the INODE_ITEM_KEY.
4420  */
4421 int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans,
4422                                struct btrfs_root *root,
4423                                struct inode *inode,
4424                                u64 new_size, u32 min_type)
4425 {
4426         struct btrfs_fs_info *fs_info = root->fs_info;
4427         struct btrfs_path *path;
4428         struct extent_buffer *leaf;
4429         struct btrfs_file_extent_item *fi;
4430         struct btrfs_key key;
4431         struct btrfs_key found_key;
4432         u64 extent_start = 0;
4433         u64 extent_num_bytes = 0;
4434         u64 extent_offset = 0;
4435         u64 item_end = 0;
4436         u64 last_size = new_size;
4437         u32 found_type = (u8)-1;
4438         int found_extent;
4439         int del_item;
4440         int pending_del_nr = 0;
4441         int pending_del_slot = 0;
4442         int extent_type = -1;
4443         int ret;
4444         int err = 0;
4445         u64 ino = btrfs_ino(BTRFS_I(inode));
4446         u64 bytes_deleted = 0;
4447         bool be_nice = false;
4448         bool should_throttle = false;
4449         bool should_end = false;
4450 
4451         BUG_ON(new_size > 0 && min_type != BTRFS_EXTENT_DATA_KEY);
4452 
4453         /*
4454          * for non-free space inodes and ref cows, we want to back off from
4455          * time to time
4456          */
4457         if (!btrfs_is_free_space_inode(BTRFS_I(inode)) &&
4458             test_bit(BTRFS_ROOT_REF_COWS, &root->state))
4459                 be_nice = true;
4460 
4461         path = btrfs_alloc_path();
4462         if (!path)
4463                 return -ENOMEM;
4464         path->reada = READA_BACK;
4465 
4466         /*
4467          * We want to drop from the next block forward in case this new size is
4468          * not block aligned since we will be keeping the last block of the
4469          * extent just the way it is.
4470          */
4471         if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) ||
4472             root == fs_info->tree_root)
4473                 btrfs_drop_extent_cache(BTRFS_I(inode), ALIGN(new_size,
4474                                         fs_info->sectorsize),
4475                                         (u64)-1, 0);
4476 
4477         /*
4478          * This function is also used to drop the items in the log tree before
4479          * we relog the inode, so if root != BTRFS_I(inode)->root, it means
4480          * it is used to drop the loged items. So we shouldn't kill the delayed
4481          * items.
4482          */
4483         if (min_type == 0 && root == BTRFS_I(inode)->root)
4484                 btrfs_kill_delayed_inode_items(BTRFS_I(inode));
4485 
4486         key.objectid = ino;
4487         key.offset = (u64)-1;
4488         key.type = (u8)-1;
4489 
4490 search_again:
4491         /*
4492          * with a 16K leaf size and 128MB extents, you can actually queue
4493          * up a huge file in a single leaf.  Most of the time that
4494          * bytes_deleted is > 0, it will be huge by the time we get here
4495          */
4496         if (be_nice && bytes_deleted > SZ_32M) {
4497                 if (btrfs_should_end_transaction(trans)) {
4498                         err = -EAGAIN;
4499                         goto error;
4500                 }
4501         }
4502 
4503 
4504         path->leave_spinning = 1;
4505         ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
4506         if (ret < 0) {
4507                 err = ret;
4508                 goto out;
4509         }
4510 
4511         if (ret > 0) {
4512                 /* there are no items in the tree for us to truncate, we're
4513                  * done
4514                  */
4515                 if (path->slots[0] == 0)
4516                         goto out;
4517                 path->slots[0]--;
4518         }
4519 
4520         while (1) {
4521                 fi = NULL;
4522                 leaf = path->nodes[0];
4523                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
4524                 found_type = found_key.type;
4525 
4526                 if (found_key.objectid != ino)
4527                         break;
4528 
4529                 if (found_type < min_type)
4530                         break;
4531 
4532                 item_end = found_key.offset;
4533                 if (found_type == BTRFS_EXTENT_DATA_KEY) {
4534                         fi = btrfs_item_ptr(leaf, path->slots[0],
4535                                             struct btrfs_file_extent_item);
4536                         extent_type = btrfs_file_extent_type(leaf, fi);
4537                         if (extent_type != BTRFS_FILE_EXTENT_INLINE) {
4538                                 item_end +=
4539                                     btrfs_file_extent_num_bytes(leaf, fi);
4540 
4541                                 trace_btrfs_truncate_show_fi_regular(
4542                                         BTRFS_I(inode), leaf, fi,
4543                                         found_key.offset);
4544                         } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
4545                                 item_end += btrfs_file_extent_inline_len(leaf,
4546                                                          path->slots[0], fi);
4547 
4548                                 trace_btrfs_truncate_show_fi_inline(
4549                                         BTRFS_I(inode), leaf, fi, path->slots[0],
4550                                         found_key.offset);
4551                         }
4552                         item_end--;
4553                 }
4554                 if (found_type > min_type) {
4555                         del_item = 1;
4556                 } else {
4557                         if (item_end < new_size)
4558                                 break;
4559                         if (found_key.offset >= new_size)
4560                                 del_item = 1;
4561                         else
4562                                 del_item = 0;
4563                 }
4564                 found_extent = 0;
4565                 /* FIXME, shrink the extent if the ref count is only 1 */
4566                 if (found_type != BTRFS_EXTENT_DATA_KEY)
4567                         goto delete;
4568 
4569                 if (extent_type != BTRFS_FILE_EXTENT_INLINE) {
4570                         u64 num_dec;
4571                         extent_start = btrfs_file_extent_disk_bytenr(leaf, fi);
4572                         if (!del_item) {
4573                                 u64 orig_num_bytes =
4574                                         btrfs_file_extent_num_bytes(leaf, fi);
4575                                 extent_num_bytes = ALIGN(new_size -
4576                                                 found_key.offset,
4577                                                 fs_info->sectorsize);
4578                                 btrfs_set_file_extent_num_bytes(leaf, fi,
4579                                                          extent_num_bytes);
4580                                 num_dec = (orig_num_bytes -
4581                                            extent_num_bytes);
4582                                 if (test_bit(BTRFS_ROOT_REF_COWS,
4583                                              &root->state) &&
4584                                     extent_start != 0)
4585                                         inode_sub_bytes(inode, num_dec);
4586                                 btrfs_mark_buffer_dirty(leaf);
4587                         } else {
4588                                 extent_num_bytes =
4589                                         btrfs_file_extent_disk_num_bytes(leaf,
4590                                                                          fi);
4591                                 extent_offset = found_key.offset -
4592                                         btrfs_file_extent_offset(leaf, fi);
4593 
4594                                 /* FIXME blocksize != 4096 */
4595                                 num_dec = btrfs_file_extent_num_bytes(leaf, fi);
4596                                 if (extent_start != 0) {
4597                                         found_extent = 1;
4598                                         if (test_bit(BTRFS_ROOT_REF_COWS,
4599                                                      &root->state))
4600                                                 inode_sub_bytes(inode, num_dec);
4601                                 }
4602                         }
4603                 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
4604                         /*
4605                          * we can't truncate inline items that have had
4606                          * special encodings
4607                          */
4608                         if (!del_item &&
4609                             btrfs_file_extent_encryption(leaf, fi) == 0 &&
4610                             btrfs_file_extent_other_encoding(leaf, fi) == 0 &&
4611                             btrfs_file_extent_compression(leaf, fi) == 0) {
4612                                 u32 size = (u32)(new_size - found_key.offset);
4613 
4614                                 btrfs_set_file_extent_ram_bytes(leaf, fi, size);
4615                                 size = btrfs_file_extent_calc_inline_size(size);
4616                                 btrfs_truncate_item(root->fs_info, path, size, 1);
4617                         } else if (!del_item) {
4618                                 /*
4619                                  * We have to bail so the last_size is set to
4620                                  * just before this extent.
4621                                  */
4622                                 err = NEED_TRUNCATE_BLOCK;
4623                                 break;
4624                         }
4625 
4626                         if (test_bit(BTRFS_ROOT_REF_COWS, &root->state))
4627                                 inode_sub_bytes(inode, item_end + 1 - new_size);
4628                 }
4629 delete:
4630                 if (del_item)
4631                         last_size = found_key.offset;
4632                 else
4633                         last_size = new_size;
4634                 if (del_item) {
4635                         if (!pending_del_nr) {
4636                                 /* no pending yet, add ourselves */
4637                                 pending_del_slot = path->slots[0];
4638                                 pending_del_nr = 1;
4639                         } else if (pending_del_nr &&
4640                                    path->slots[0] + 1 == pending_del_slot) {
4641                                 /* hop on the pending chunk */
4642                                 pending_del_nr++;
4643                                 pending_del_slot = path->slots[0];
4644                         } else {
4645                                 BUG();
4646                         }
4647                 } else {
4648                         break;
4649                 }
4650                 should_throttle = false;
4651 
4652                 if (found_extent &&
4653                     (test_bit(BTRFS_ROOT_REF_COWS, &root->state) ||
4654                      root == fs_info->tree_root)) {
4655                         btrfs_set_path_blocking(path);
4656                         bytes_deleted += extent_num_bytes;
4657                         ret = btrfs_free_extent(trans, root, extent_start,
4658                                                 extent_num_bytes, 0,
4659                                                 btrfs_header_owner(leaf),
4660                                                 ino, extent_offset);
4661                         BUG_ON(ret);
4662                         if (btrfs_should_throttle_delayed_refs(trans, fs_info))
4663                                 btrfs_async_run_delayed_refs(fs_info,
4664                                         trans->delayed_ref_updates * 2,
4665                                         trans->transid, 0);
4666                         if (be_nice) {
4667                                 if (truncate_space_check(trans, root,
4668                                                          extent_num_bytes)) {
4669                                         should_end = true;
4670                                 }
4671                                 if (btrfs_should_throttle_delayed_refs(trans,
4672                                                                        fs_info))
4673                                         should_throttle = true;
4674                         }
4675                 }
4676 
4677                 if (found_type == BTRFS_INODE_ITEM_KEY)
4678                         break;
4679 
4680                 if (path->slots[0] == 0 ||
4681                     path->slots[0] != pending_del_slot ||
4682                     should_throttle || should_end) {
4683                         if (pending_del_nr) {
4684                                 ret = btrfs_del_items(trans, root, path,
4685                                                 pending_del_slot,
4686                                                 pending_del_nr);
4687                                 if (ret) {
4688                                         btrfs_abort_transaction(trans, ret);
4689                                         goto error;
4690                                 }
4691                                 pending_del_nr = 0;
4692                         }
4693                         btrfs_release_path(path);
4694                         if (should_throttle) {
4695                                 unsigned long updates = trans->delayed_ref_updates;
4696                                 if (updates) {
4697                                         trans->delayed_ref_updates = 0;
4698                                         ret = btrfs_run_delayed_refs(trans,
4699                                                                    updates * 2);
4700                                         if (ret && !err)
4701                                                 err = ret;
4702                                 }
4703                         }
4704                         /*
4705                          * if we failed to refill our space rsv, bail out
4706                          * and let the transaction restart
4707                          */
4708                         if (should_end) {
4709                                 err = -EAGAIN;
4710                                 goto error;
4711                         }
4712                         goto search_again;
4713                 } else {
4714                         path->slots[0]--;
4715                 }
4716         }
4717 out:
4718         if (pending_del_nr) {
4719                 ret = btrfs_del_items(trans, root, path, pending_del_slot,
4720                                       pending_del_nr);
4721                 if (ret)
4722                         btrfs_abort_transaction(trans, ret);
4723         }
4724 error:
4725         if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
4726                 ASSERT(last_size >= new_size);
4727                 if (!err && last_size > new_size)
4728                         last_size = new_size;
4729                 btrfs_ordered_update_i_size(inode, last_size, NULL);
4730         }
4731 
4732         btrfs_free_path(path);
4733 
4734         if (be_nice && bytes_deleted > SZ_32M) {
4735                 unsigned long updates = trans->delayed_ref_updates;
4736                 if (updates) {
4737                         trans->delayed_ref_updates = 0;
4738                         ret = btrfs_run_delayed_refs(trans, updates * 2);
4739                         if (ret && !err)
4740                                 err = ret;
4741                 }
4742         }
4743         return err;
4744 }
4745 
4746 /*
4747  * btrfs_truncate_block - read, zero a chunk and write a block
4748  * @inode - inode that we're zeroing
4749  * @from - the offset to start zeroing
4750  * @len - the length to zero, 0 to zero the entire range respective to the
4751  *      offset
4752  * @front - zero up to the offset instead of from the offset on
4753  *
4754  * This will find the block for the "from" offset and cow the block and zero the
4755  * part we want to zero.  This is used with truncate and hole punching.
4756  */
4757 int btrfs_truncate_block(struct inode *inode, loff_t from, loff_t len,
4758                         int front)
4759 {
4760         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
4761         struct address_space *mapping = inode->i_mapping;
4762         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
4763         struct btrfs_ordered_extent *ordered;
4764         struct extent_state *cached_state = NULL;
4765         struct extent_changeset *data_reserved = NULL;
4766         char *kaddr;
4767         u32 blocksize = fs_info->sectorsize;
4768         pgoff_t index = from >> PAGE_SHIFT;
4769         unsigned offset = from & (blocksize - 1);
4770         struct page *page;
4771         gfp_t mask = btrfs_alloc_write_mask(mapping);
4772         int ret = 0;
4773         u64 block_start;
4774         u64 block_end;
4775 
4776         if (IS_ALIGNED(offset, blocksize) &&
4777             (!len || IS_ALIGNED(len, blocksize)))
4778                 goto out;
4779 
4780         block_start = round_down(from, blocksize);
4781         block_end = block_start + blocksize - 1;
4782 
4783         ret = btrfs_delalloc_reserve_space(inode, &data_reserved,
4784                                            block_start, blocksize);
4785         if (ret)
4786                 goto out;
4787 
4788 again:
4789         page = find_or_create_page(mapping, index, mask);
4790         if (!page) {
4791                 btrfs_delalloc_release_space(inode, data_reserved,
4792                                              block_start, blocksize, true);
4793                 btrfs_delalloc_release_extents(BTRFS_I(inode), blocksize, true);
4794                 ret = -ENOMEM;
4795                 goto out;
4796         }
4797 
4798         if (!PageUptodate(page)) {
4799                 ret = btrfs_readpage(NULL, page);
4800                 lock_page(page);
4801                 if (page->mapping != mapping) {
4802                         unlock_page(page);
4803                         put_page(page);
4804                         goto again;
4805                 }
4806                 if (!PageUptodate(page)) {
4807                         ret = -EIO;
4808                         goto out_unlock;
4809                 }
4810         }
4811         wait_on_page_writeback(page);
4812 
4813         lock_extent_bits(io_tree, block_start, block_end, &cached_state);
4814         set_page_extent_mapped(page);
4815 
4816         ordered = btrfs_lookup_ordered_extent(inode, block_start);
4817         if (ordered) {
4818                 unlock_extent_cached(io_tree, block_start, block_end,
4819                                      &cached_state);
4820                 unlock_page(page);
4821                 put_page(page);
4822                 btrfs_start_ordered_extent(inode, ordered, 1);
4823                 btrfs_put_ordered_extent(ordered);
4824                 goto again;
4825         }
4826 
4827         clear_extent_bit(&BTRFS_I(inode)->io_tree, block_start, block_end,
4828                           EXTENT_DIRTY | EXTENT_DELALLOC |
4829                           EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG,
4830                           0, 0, &cached_state);
4831 
4832         ret = btrfs_set_extent_delalloc(inode, block_start, block_end, 0,
4833                                         &cached_state, 0);
4834         if (ret) {
4835                 unlock_extent_cached(io_tree, block_start, block_end,
4836                                      &cached_state);
4837                 goto out_unlock;
4838         }
4839 
4840         if (offset != blocksize) {
4841                 if (!len)
4842                         len = blocksize - offset;
4843                 kaddr = kmap(page);
4844                 if (front)
4845                         memset(kaddr + (block_start - page_offset(page)),
4846                                 0, offset);
4847                 else
4848                         memset(kaddr + (block_start - page_offset(page)) +  offset,
4849                                 0, len);
4850                 flush_dcache_page(page);
4851                 kunmap(page);
4852         }
4853         ClearPageChecked(page);
4854         set_page_dirty(page);
4855         unlock_extent_cached(io_tree, block_start, block_end, &cached_state);
4856 
4857 out_unlock:
4858         if (ret)
4859                 btrfs_delalloc_release_space(inode, data_reserved, block_start,
4860                                              blocksize, true);
4861         btrfs_delalloc_release_extents(BTRFS_I(inode), blocksize, (ret != 0));
4862         unlock_page(page);
4863         put_page(page);
4864 out:
4865         extent_changeset_free(data_reserved);
4866         return ret;
4867 }
4868 
4869 static int maybe_insert_hole(struct btrfs_root *root, struct inode *inode,
4870                              u64 offset, u64 len)
4871 {
4872         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
4873         struct btrfs_trans_handle *trans;
4874         int ret;
4875 
4876         /*
4877          * Still need to make sure the inode looks like it's been updated so
4878          * that any holes get logged if we fsync.
4879          */
4880         if (btrfs_fs_incompat(fs_info, NO_HOLES)) {
4881                 BTRFS_I(inode)->last_trans = fs_info->generation;
4882                 BTRFS_I(inode)->last_sub_trans = root->log_transid;
4883                 BTRFS_I(inode)->last_log_commit = root->last_log_commit;
4884                 return 0;
4885         }
4886 
4887         /*
4888          * 1 - for the one we're dropping
4889          * 1 - for the one we're adding
4890          * 1 - for updating the inode.
4891          */
4892         trans = btrfs_start_transaction(root, 3);
4893         if (IS_ERR(trans))
4894                 return PTR_ERR(trans);
4895 
4896         ret = btrfs_drop_extents(trans, root, inode, offset, offset + len, 1);
4897         if (ret) {
4898                 btrfs_abort_transaction(trans, ret);
4899                 btrfs_end_transaction(trans);
4900                 return ret;
4901         }
4902 
4903         ret = btrfs_insert_file_extent(trans, root, btrfs_ino(BTRFS_I(inode)),
4904                         offset, 0, 0, len, 0, len, 0, 0, 0);
4905         if (ret)
4906                 btrfs_abort_transaction(trans, ret);
4907         else
4908                 btrfs_update_inode(trans, root, inode);
4909         btrfs_end_transaction(trans);
4910         return ret;
4911 }
4912 
4913 /*
4914  * This function puts in dummy file extents for the area we're creating a hole
4915  * for.  So if we are truncating this file to a larger size we need to insert
4916  * these file extents so that btrfs_get_extent will return a EXTENT_MAP_HOLE for
4917  * the range between oldsize and size
4918  */
4919 int btrfs_cont_expand(struct inode *inode, loff_t oldsize, loff_t size)
4920 {
4921         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
4922         struct btrfs_root *root = BTRFS_I(inode)->root;
4923         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
4924         struct extent_map *em = NULL;
4925         struct extent_state *cached_state = NULL;
4926         struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
4927         u64 hole_start = ALIGN(oldsize, fs_info->sectorsize);
4928         u64 block_end = ALIGN(size, fs_info->sectorsize);
4929         u64 last_byte;
4930         u64 cur_offset;
4931         u64 hole_size;
4932         int err = 0;
4933 
4934         /*
4935          * If our size started in the middle of a block we need to zero out the
4936          * rest of the block before we expand the i_size, otherwise we could
4937          * expose stale data.
4938          */
4939         err = btrfs_truncate_block(inode, oldsize, 0, 0);
4940         if (err)
4941                 return err;
4942 
4943         if (size <= hole_start)
4944                 return 0;
4945 
4946         while (1) {
4947                 struct btrfs_ordered_extent *ordered;
4948 
4949                 lock_extent_bits(io_tree, hole_start, block_end - 1,
4950                                  &cached_state);
4951                 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), hole_start,
4952                                                      block_end - hole_start);
4953                 if (!ordered)
4954                         break;
4955                 unlock_extent_cached(io_tree, hole_start, block_end - 1,
4956                                      &cached_state);
4957                 btrfs_start_ordered_extent(inode, ordered, 1);
4958                 btrfs_put_ordered_extent(ordered);
4959         }
4960 
4961         cur_offset = hole_start;
4962         while (1) {
4963                 em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, cur_offset,
4964                                 block_end - cur_offset, 0);
4965                 if (IS_ERR(em)) {
4966                         err = PTR_ERR(em);
4967                         em = NULL;
4968                         break;
4969                 }
4970                 last_byte = min(extent_map_end(em), block_end);
4971                 last_byte = ALIGN(last_byte, fs_info->sectorsize);
4972                 if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) {
4973                         struct extent_map *hole_em;
4974                         hole_size = last_byte - cur_offset;
4975 
4976                         err = maybe_insert_hole(root, inode, cur_offset,
4977                                                 hole_size);
4978                         if (err)
4979                                 break;
4980                         btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset,
4981                                                 cur_offset + hole_size - 1, 0);
4982                         hole_em = alloc_extent_map();
4983                         if (!hole_em) {
4984                                 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
4985                                         &BTRFS_I(inode)->runtime_flags);
4986                                 goto next;
4987                         }
4988                         hole_em->start = cur_offset;
4989                         hole_em->len = hole_size;
4990                         hole_em->orig_start = cur_offset;
4991 
4992                         hole_em->block_start = EXTENT_MAP_HOLE;
4993                         hole_em->block_len = 0;
4994                         hole_em->orig_block_len = 0;
4995                         hole_em->ram_bytes = hole_size;
4996                         hole_em->bdev = fs_info->fs_devices->latest_bdev;
4997                         hole_em->compress_type = BTRFS_COMPRESS_NONE;
4998                         hole_em->generation = fs_info->generation;
4999 
5000                         while (1) {
5001                                 write_lock(&em_tree->lock);
5002                                 err = add_extent_mapping(em_tree, hole_em, 1);
5003                                 write_unlock(&em_tree->lock);
5004                                 if (err != -EEXIST)
5005                                         break;
5006                                 btrfs_drop_extent_cache(BTRFS_I(inode),
5007                                                         cur_offset,
5008                                                         cur_offset +
5009                                                         hole_size - 1, 0);
5010                         }
5011                         free_extent_map(hole_em);
5012                 }
5013 next:
5014                 free_extent_map(em);
5015                 em = NULL;
5016                 cur_offset = last_byte;
5017                 if (cur_offset >= block_end)
5018                         break;
5019         }
5020         free_extent_map(em);
5021         unlock_extent_cached(io_tree, hole_start, block_end - 1, &cached_state);
5022         return err;
5023 }
5024 
5025 static int btrfs_setsize(struct inode *inode, struct iattr *attr)
5026 {
5027         struct btrfs_root *root = BTRFS_I(inode)->root;
5028         struct btrfs_trans_handle *trans;
5029         loff_t oldsize = i_size_read(inode);
5030         loff_t newsize = attr->ia_size;
5031         int mask = attr->ia_valid;
5032         int ret;
5033 
5034         /*
5035          * The regular truncate() case without ATTR_CTIME and ATTR_MTIME is a
5036          * special case where we need to update the times despite not having
5037          * these flags set.  For all other operations the VFS set these flags
5038          * explicitly if it wants a timestamp update.
5039          */
5040         if (newsize != oldsize) {
5041                 inode_inc_iversion(inode);
5042                 if (!(mask & (ATTR_CTIME | ATTR_MTIME)))
5043                         inode->i_ctime = inode->i_mtime =
5044                                 current_time(inode);
5045         }
5046 
5047         if (newsize > oldsize) {
5048                 /*
5049                  * Don't do an expanding truncate while snapshotting is ongoing.
5050                  * This is to ensure the snapshot captures a fully consistent
5051                  * state of this file - if the snapshot captures this expanding
5052                  * truncation, it must capture all writes that happened before
5053                  * this truncation.
5054                  */
5055                 btrfs_wait_for_snapshot_creation(root);
5056                 ret = btrfs_cont_expand(inode, oldsize, newsize);
5057                 if (ret) {
5058                         btrfs_end_write_no_snapshotting(root);
5059                         return ret;
5060                 }
5061 
5062                 trans = btrfs_start_transaction(root, 1);
5063                 if (IS_ERR(trans)) {
5064                         btrfs_end_write_no_snapshotting(root);
5065                         return PTR_ERR(trans);
5066                 }
5067 
5068                 i_size_write(inode, newsize);
5069                 btrfs_ordered_update_i_size(inode, i_size_read(inode), NULL);
5070                 pagecache_isize_extended(inode, oldsize, newsize);
5071                 ret = btrfs_update_inode(trans, root, inode);
5072                 btrfs_end_write_no_snapshotting(root);
5073                 btrfs_end_transaction(trans);
5074         } else {
5075 
5076                 /*
5077                  * We're truncating a file that used to have good data down to
5078                  * zero. Make sure it gets into the ordered flush list so that
5079                  * any new writes get down to disk quickly.
5080                  */
5081                 if (newsize == 0)
5082                         set_bit(BTRFS_INODE_ORDERED_DATA_CLOSE,
5083                                 &BTRFS_I(inode)->runtime_flags);
5084 
5085                 /*
5086                  * 1 for the orphan item we're going to add
5087                  * 1 for the orphan item deletion.
5088                  */
5089                 trans = btrfs_start_transaction(root, 2);
5090                 if (IS_ERR(trans))
5091                         return PTR_ERR(trans);
5092 
5093                 /*
5094                  * We need to do this in case we fail at _any_ point during the
5095                  * actual truncate.  Once we do the truncate_setsize we could
5096                  * invalidate pages which forces any outstanding ordered io to
5097                  * be instantly completed which will give us extents that need
5098                  * to be truncated.  If we fail to get an orphan inode down we
5099                  * could have left over extents that were never meant to live,
5100                  * so we need to guarantee from this point on that everything
5101                  * will be consistent.
5102                  */
5103                 ret = btrfs_orphan_add(trans, BTRFS_I(inode));
5104                 btrfs_end_transaction(trans);
5105                 if (ret)
5106                         return ret;
5107 
5108                 /* we don't support swapfiles, so vmtruncate shouldn't fail */
5109                 truncate_setsize(inode, newsize);
5110 
5111                 /* Disable nonlocked read DIO to avoid the end less truncate */
5112                 btrfs_inode_block_unlocked_dio(BTRFS_I(inode));
5113                 inode_dio_wait(inode);
5114                 btrfs_inode_resume_unlocked_dio(BTRFS_I(inode));
5115 
5116                 ret = btrfs_truncate(inode, newsize == oldsize);
5117                 if (ret && inode->i_nlink) {
5118                         int err;
5119 
5120                         /* To get a stable disk_i_size */
5121                         err = btrfs_wait_ordered_range(inode, 0, (u64)-1);
5122                         if (err) {
5123                                 btrfs_orphan_del(NULL, BTRFS_I(inode));
5124                                 return err;
5125                         }
5126 
5127                         /*
5128                          * failed to truncate, disk_i_size is only adjusted down
5129                          * as we remove extents, so it should represent the true
5130                          * size of the inode, so reset the in memory size and
5131                          * delete our orphan entry.
5132                          */
5133                         trans = btrfs_join_transaction(root);
5134                         if (IS_ERR(trans)) {
5135                                 btrfs_orphan_del(NULL, BTRFS_I(inode));
5136                                 return ret;
5137                         }
5138                         i_size_write(inode, BTRFS_I(inode)->disk_i_size);
5139                         err = btrfs_orphan_del(trans, BTRFS_I(inode));
5140                         if (err)
5141                                 btrfs_abort_transaction(trans, err);
5142                         btrfs_end_transaction(trans);
5143                 }
5144         }
5145 
5146         return ret;
5147 }
5148 
5149 static int btrfs_setattr(struct dentry *dentry, struct iattr *attr)
5150 {
5151         struct inode *inode = d_inode(dentry);
5152         struct btrfs_root *root = BTRFS_I(inode)->root;
5153         int err;
5154 
5155         if (btrfs_root_readonly(root))
5156                 return -EROFS;
5157 
5158         err = setattr_prepare(dentry, attr);
5159         if (err)
5160                 return err;
5161 
5162         if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
5163                 err = btrfs_setsize(inode, attr);
5164                 if (err)
5165                         return err;
5166         }
5167 
5168         if (attr->ia_valid) {
5169                 setattr_copy(inode, attr);
5170                 inode_inc_iversion(inode);
5171                 err = btrfs_dirty_inode(inode);
5172 
5173                 if (!err && attr->ia_valid & ATTR_MODE)
5174                         err = posix_acl_chmod(inode, inode->i_mode);
5175         }
5176 
5177         return err;
5178 }
5179 
5180 /*
5181  * While truncating the inode pages during eviction, we get the VFS calling
5182  * btrfs_invalidatepage() against each page of the inode. This is slow because
5183  * the calls to btrfs_invalidatepage() result in a huge amount of calls to
5184  * lock_extent_bits() and clear_extent_bit(), which keep merging and splitting
5185  * extent_state structures over and over, wasting lots of time.
5186  *
5187  * Therefore if the inode is being evicted, let btrfs_invalidatepage() skip all
5188  * those expensive operations on a per page basis and do only the ordered io
5189  * finishing, while we release here the extent_map and extent_state structures,
5190  * without the excessive merging and splitting.
5191  */
5192 static void evict_inode_truncate_pages(struct inode *inode)
5193 {
5194         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
5195         struct extent_map_tree *map_tree = &BTRFS_I(inode)->extent_tree;
5196         struct rb_node *node;
5197 
5198         ASSERT(inode->i_state & I_FREEING);
5199         truncate_inode_pages_final(&inode->i_data);
5200 
5201         write_lock(&map_tree->lock);
5202         while (!RB_EMPTY_ROOT(&map_tree->map)) {
5203                 struct extent_map *em;
5204 
5205                 node = rb_first(&map_tree->map);
5206                 em = rb_entry(node, struct extent_map, rb_node);
5207                 clear_bit(EXTENT_FLAG_PINNED, &em->flags);
5208                 clear_bit(EXTENT_FLAG_LOGGING, &em->flags);
5209                 remove_extent_mapping(map_tree, em);
5210                 free_extent_map(em);
5211                 if (need_resched()) {
5212                         write_unlock(&map_tree->lock);
5213                         cond_resched();
5214                         write_lock(&map_tree->lock);
5215                 }
5216         }
5217         write_unlock(&map_tree->lock);
5218 
5219         /*
5220          * Keep looping until we have no more ranges in the io tree.
5221          * We can have ongoing bios started by readpages (called from readahead)
5222          * that have their endio callback (extent_io.c:end_bio_extent_readpage)
5223          * still in progress (unlocked the pages in the bio but did not yet
5224          * unlocked the ranges in the io tree). Therefore this means some
5225          * ranges can still be locked and eviction started because before
5226          * submitting those bios, which are executed by a separate task (work
5227          * queue kthread), inode references (inode->i_count) were not taken
5228          * (which would be dropped in the end io callback of each bio).
5229          * Therefore here we effectively end up waiting for those bios and
5230          * anyone else holding locked ranges without having bumped the inode's
5231          * reference count - if we don't do it, when they access the inode's
5232          * io_tree to unlock a range it may be too late, leading to an
5233          * use-after-free issue.
5234          */
5235         spin_lock(&io_tree->lock);
5236         while (!RB_EMPTY_ROOT(&io_tree->state)) {
5237                 struct extent_state *state;
5238                 struct extent_state *cached_state = NULL;
5239                 u64 start;
5240                 u64 end;
5241 
5242                 node = rb_first(&io_tree->state);
5243                 state = rb_entry(node, struct extent_state, rb_node);
5244                 start = state->start;
5245                 end = state->end;
5246                 spin_unlock(&io_tree->lock);
5247 
5248                 lock_extent_bits(io_tree, start, end, &cached_state);
5249 
5250                 /*
5251                  * If still has DELALLOC flag, the extent didn't reach disk,
5252                  * and its reserved space won't be freed by delayed_ref.
5253                  * So we need to free its reserved space here.
5254                  * (Refer to comment in btrfs_invalidatepage, case 2)
5255                  *
5256                  * Note, end is the bytenr of last byte, so we need + 1 here.
5257                  */
5258                 if (state->state & EXTENT_DELALLOC)
5259                         btrfs_qgroup_free_data(inode, NULL, start, end - start + 1);
5260 
5261                 clear_extent_bit(io_tree, start, end,
5262                                  EXTENT_LOCKED | EXTENT_DIRTY |
5263                                  EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING |
5264                                  EXTENT_DEFRAG, 1, 1, &cached_state);
5265 
5266                 cond_resched();
5267                 spin_lock(&io_tree->lock);
5268         }
5269         spin_unlock(&io_tree->lock);
5270 }
5271 
5272 void btrfs_evict_inode(struct inode *inode)
5273 {
5274         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
5275         struct btrfs_trans_handle *trans;
5276         struct btrfs_root *root = BTRFS_I(inode)->root;
5277         struct btrfs_block_rsv *rsv, *global_rsv;
5278         int steal_from_global = 0;
5279         u64 min_size;
5280         int ret;
5281 
5282         trace_btrfs_inode_evict(inode);
5283 
5284         if (!root) {
5285                 clear_inode(inode);
5286                 return;
5287         }
5288 
5289         min_size = btrfs_calc_trunc_metadata_size(fs_info, 1);
5290 
5291         evict_inode_truncate_pages(inode);
5292 
5293         if (inode->i_nlink &&
5294             ((btrfs_root_refs(&root->root_item) != 0 &&
5295               root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID) ||
5296              btrfs_is_free_space_inode(BTRFS_I(inode))))
5297                 goto no_delete;
5298 
5299         if (is_bad_inode(inode)) {
5300                 btrfs_orphan_del(NULL, BTRFS_I(inode));
5301                 goto no_delete;
5302         }
5303         /* do we really want it for ->i_nlink > 0 and zero btrfs_root_refs? */
5304         if (!special_file(inode->i_mode))
5305                 btrfs_wait_ordered_range(inode, 0, (u64)-1);
5306 
5307         btrfs_free_io_failure_record(BTRFS_I(inode), 0, (u64)-1);
5308 
5309         if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) {
5310                 BUG_ON(test_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
5311                                  &BTRFS_I(inode)->runtime_flags));
5312                 goto no_delete;
5313         }
5314 
5315         if (inode->i_nlink > 0) {
5316                 BUG_ON(btrfs_root_refs(&root->root_item) != 0 &&
5317                        root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID);
5318                 goto no_delete;
5319         }
5320 
5321         ret = btrfs_commit_inode_delayed_inode(BTRFS_I(inode));
5322         if (ret) {
5323                 btrfs_orphan_del(NULL, BTRFS_I(inode));
5324                 goto no_delete;
5325         }
5326 
5327         rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP);
5328         if (!rsv) {
5329                 btrfs_orphan_del(NULL, BTRFS_I(inode));
5330                 goto no_delete;
5331         }
5332         rsv->size = min_size;
5333         rsv->failfast = 1;
5334         global_rsv = &fs_info->global_block_rsv;
5335 
5336         btrfs_i_size_write(BTRFS_I(inode), 0);
5337 
5338         /*
5339          * This is a bit simpler than btrfs_truncate since we've already
5340          * reserved our space for our orphan item in the unlink, so we just
5341          * need to reserve some slack space in case we add bytes and update
5342          * inode item when doing the truncate.
5343          */
5344         while (1) {
5345                 ret = btrfs_block_rsv_refill(root, rsv, min_size,
5346                                              BTRFS_RESERVE_FLUSH_LIMIT);
5347 
5348                 /*
5349                  * Try and steal from the global reserve since we will
5350                  * likely not use this space anyway, we want to try as
5351                  * hard as possible to get this to work.
5352                  */
5353                 if (ret)
5354                         steal_from_global++;
5355                 else
5356                         steal_from_global = 0;
5357                 ret = 0;
5358 
5359                 /*
5360                  * steal_from_global == 0: we reserved stuff, hooray!
5361                  * steal_from_global == 1: we didn't reserve stuff, boo!
5362                  * steal_from_global == 2: we've committed, still not a lot of
5363                  * room but maybe we'll have room in the global reserve this
5364                  * time.
5365                  * steal_from_global == 3: abandon all hope!
5366                  */
5367                 if (steal_from_global > 2) {
5368                         btrfs_warn(fs_info,
5369                                    "Could not get space for a delete, will truncate on mount %d",
5370                                    ret);
5371                         btrfs_orphan_del(NULL, BTRFS_I(inode));
5372                         btrfs_free_block_rsv(fs_info, rsv);
5373                         goto no_delete;
5374                 }
5375 
5376                 trans = btrfs_join_transaction(root);
5377                 if (IS_ERR(trans)) {
5378                         btrfs_orphan_del(NULL, BTRFS_I(inode));
5379                         btrfs_free_block_rsv(fs_info, rsv);
5380                         goto no_delete;
5381                 }
5382 
5383                 /*
5384                  * We can't just steal from the global reserve, we need to make
5385                  * sure there is room to do it, if not we need to commit and try
5386                  * again.
5387                  */
5388                 if (steal_from_global) {
5389                         if (!btrfs_check_space_for_delayed_refs(trans, fs_info))
5390                                 ret = btrfs_block_rsv_migrate(global_rsv, rsv,
5391                                                               min_size, 0);
5392                         else
5393                                 ret = -ENOSPC;
5394                 }
5395 
5396                 /*
5397                  * Couldn't steal from the global reserve, we have too much
5398                  * pending stuff built up, commit the transaction and try it
5399                  * again.
5400                  */
5401                 if (ret) {
5402                         ret = btrfs_commit_transaction(trans);
5403                         if (ret) {
5404                                 btrfs_orphan_del(NULL, BTRFS_I(inode));
5405                                 btrfs_free_block_rsv(fs_info, rsv);
5406                                 goto no_delete;
5407                         }
5408                         continue;
5409                 } else {
5410                         steal_from_global = 0;
5411                 }
5412 
5413                 trans->block_rsv = rsv;
5414 
5415                 ret = btrfs_truncate_inode_items(trans, root, inode, 0, 0);
5416                 if (ret != -ENOSPC && ret != -EAGAIN)
5417                         break;
5418 
5419                 trans->block_rsv = &fs_info->trans_block_rsv;
5420                 btrfs_end_transaction(trans);
5421                 trans = NULL;
5422                 btrfs_btree_balance_dirty(fs_info);
5423         }
5424 
5425         btrfs_free_block_rsv(fs_info, rsv);
5426 
5427         /*
5428          * Errors here aren't a big deal, it just means we leave orphan items
5429          * in the tree.  They will be cleaned up on the next mount.
5430          */
5431         if (ret == 0) {
5432                 trans->block_rsv = root->orphan_block_rsv;
5433                 btrfs_orphan_del(trans, BTRFS_I(inode));
5434         } else {
5435                 btrfs_orphan_del(NULL, BTRFS_I(inode));
5436         }
5437 
5438         trans->block_rsv = &fs_info->trans_block_rsv;
5439         if (!(root == fs_info->tree_root ||
5440               root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID))
5441                 btrfs_return_ino(root, btrfs_ino(BTRFS_I(inode)));
5442 
5443         btrfs_end_transaction(trans);
5444         btrfs_btree_balance_dirty(fs_info);
5445 no_delete:
5446         btrfs_remove_delayed_node(BTRFS_I(inode));
5447         clear_inode(inode);
5448 }
5449 
5450 /*
5451  * this returns the key found in the dir entry in the location pointer.
5452  * If no dir entries were found, returns -ENOENT.
5453  * If found a corrupted location in dir entry, returns -EUCLEAN.
5454  */
5455 static int btrfs_inode_by_name(struct inode *dir, struct dentry *dentry,
5456                                struct btrfs_key *location)
5457 {
5458         const char *name = dentry->d_name.name;
5459         int namelen = dentry->d_name.len;
5460         struct btrfs_dir_item *di;
5461         struct btrfs_path *path;
5462         struct btrfs_root *root = BTRFS_I(dir)->root;
5463         int ret = 0;
5464 
5465         path = btrfs_alloc_path();
5466         if (!path)
5467                 return -ENOMEM;
5468 
5469         di = btrfs_lookup_dir_item(NULL, root, path, btrfs_ino(BTRFS_I(dir)),
5470                         name, namelen, 0);
5471         if (!di) {
5472                 ret = -ENOENT;
5473                 goto out;
5474         }
5475         if (IS_ERR(di)) {
5476                 ret = PTR_ERR(di);
5477                 goto out;
5478         }
5479 
5480         btrfs_dir_item_key_to_cpu(path->nodes[0], di, location);
5481         if (location->type != BTRFS_INODE_ITEM_KEY &&
5482             location->type != BTRFS_ROOT_ITEM_KEY) {
5483                 ret = -EUCLEAN;
5484                 btrfs_warn(root->fs_info,
5485 "%s gets something invalid in DIR_ITEM (name %s, directory ino %llu, location(%llu %u %llu))",
5486                            __func__, name, btrfs_ino(BTRFS_I(dir)),
5487                            location->objectid, location->type, location->offset);
5488         }
5489 out:
5490         btrfs_free_path(path);
5491         return ret;
5492 }
5493 
5494 /*
5495  * when we hit a tree root in a directory, the btrfs part of the inode
5496  * needs to be changed to reflect the root directory of the tree root.  This
5497  * is kind of like crossing a mount point.
5498  */
5499 static int fixup_tree_root_location(struct btrfs_fs_info *fs_info,
5500                                     struct inode *dir,
5501                                     struct dentry *dentry,
5502                                     struct btrfs_key *location,
5503                                     struct btrfs_root **sub_root)
5504 {
5505         struct btrfs_path *path;
5506         struct btrfs_root *new_root;
5507         struct btrfs_root_ref *ref;
5508         struct extent_buffer *leaf;
5509         struct btrfs_key key;
5510         int ret;
5511         int err = 0;
5512 
5513         path = btrfs_alloc_path();
5514         if (!path) {
5515                 err = -ENOMEM;
5516                 goto out;
5517         }
5518 
5519         err = -ENOENT;
5520         key.objectid = BTRFS_I(dir)->root->root_key.objectid;
5521         key.type = BTRFS_ROOT_REF_KEY;
5522         key.offset = location->objectid;
5523 
5524         ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0);
5525         if (ret) {
5526                 if (ret < 0)
5527                         err = ret;
5528                 goto out;
5529         }
5530 
5531         leaf = path->nodes[0];
5532         ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref);
5533         if (btrfs_root_ref_dirid(leaf, ref) != btrfs_ino(BTRFS_I(dir)) ||
5534             btrfs_root_ref_name_len(leaf, ref) != dentry->d_name.len)
5535                 goto out;
5536 
5537         ret = memcmp_extent_buffer(leaf, dentry->d_name.name,
5538                                    (unsigned long)(ref + 1),
5539                                    dentry->d_name.len);
5540         if (ret)
5541                 goto out;
5542 
5543         btrfs_release_path(path);
5544 
5545         new_root = btrfs_read_fs_root_no_name(fs_info, location);
5546         if (IS_ERR(new_root)) {
5547                 err = PTR_ERR(new_root);
5548                 goto out;
5549         }
5550 
5551         *sub_root = new_root;
5552         location->objectid = btrfs_root_dirid(&new_root->root_item);
5553         location->type = BTRFS_INODE_ITEM_KEY;
5554         location->offset = 0;
5555         err = 0;
5556 out:
5557         btrfs_free_path(path);
5558         return err;
5559 }
5560 
5561 static void inode_tree_add(struct inode *inode)
5562 {
5563         struct btrfs_root *root = BTRFS_I(inode)->root;
5564         struct btrfs_inode *entry;
5565         struct rb_node **p;
5566         struct rb_node *parent;
5567         struct rb_node *new = &BTRFS_I(inode)->rb_node;
5568         u64 ino = btrfs_ino(BTRFS_I(inode));
5569 
5570         if (inode_unhashed(inode))
5571                 return;
5572         parent = NULL;
5573         spin_lock(&root->inode_lock);
5574         p = &root->inode_tree.rb_node;
5575         while (*p) {
5576                 parent = *p;
5577                 entry = rb_entry(parent, struct btrfs_inode, rb_node);
5578 
5579                 if (ino < btrfs_ino(BTRFS_I(&entry->vfs_inode)))
5580                         p = &parent->rb_left;
5581                 else if (ino > btrfs_ino(BTRFS_I(&entry->vfs_inode)))
5582                         p = &parent->rb_right;
5583                 else {
5584                         WARN_ON(!(entry->vfs_inode.i_state &
5585                                   (I_WILL_FREE | I_FREEING)));
5586                         rb_replace_node(parent, new, &root->inode_tree);
5587                         RB_CLEAR_NODE(parent);
5588                         spin_unlock(&root->inode_lock);
5589                         return;
5590                 }
5591         }
5592         rb_link_node(new, parent, p);
5593         rb_insert_color(new, &root->inode_tree);
5594         spin_unlock(&root->inode_lock);
5595 }
5596 
5597 static void inode_tree_del(struct inode *inode)
5598 {
5599         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
5600         struct btrfs_root *root = BTRFS_I(inode)->root;
5601         int empty = 0;
5602 
5603         spin_lock(&root->inode_lock);
5604         if (!RB_EMPTY_NODE(&BTRFS_I(inode)->rb_node)) {
5605                 rb_erase(&BTRFS_I(inode)->rb_node, &root->inode_tree);
5606                 RB_CLEAR_NODE(&BTRFS_I(inode)->rb_node);
5607                 empty = RB_EMPTY_ROOT(&root->inode_tree);
5608         }
5609         spin_unlock(&root->inode_lock);
5610 
5611         if (empty && btrfs_root_refs(&root->root_item) == 0) {
5612                 synchronize_srcu(&fs_info->subvol_srcu);
5613                 spin_lock(&root->inode_lock);
5614                 empty = RB_EMPTY_ROOT(&root->inode_tree);
5615                 spin_unlock(&root->inode_lock);
5616                 if (empty)
5617                         btrfs_add_dead_root(root);
5618         }
5619 }
5620 
5621 void btrfs_invalidate_inodes(struct btrfs_root *root)
5622 {
5623         struct btrfs_fs_info *fs_info = root->fs_info;
5624         struct rb_node *node;
5625         struct rb_node *prev;
5626         struct btrfs_inode *entry;
5627         struct inode *inode;
5628         u64 objectid = 0;
5629 
5630         if (!test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state))
5631                 WARN_ON(btrfs_root_refs(&root->root_item) != 0);
5632 
5633         spin_lock(&root->inode_lock);
5634 again:
5635         node = root->inode_tree.rb_node;
5636         prev = NULL;
5637         while (node) {
5638                 prev = node;
5639                 entry = rb_entry(node, struct btrfs_inode, rb_node);
5640 
5641                 if (objectid < btrfs_ino(BTRFS_I(&entry->vfs_inode)))
5642                         node = node->rb_left;
5643                 else if (objectid > btrfs_ino(BTRFS_I(&entry->vfs_inode)))
5644                         node = node->rb_right;
5645                 else
5646                         break;
5647         }
5648         if (!node) {
5649                 while (prev) {
5650                         entry = rb_entry(prev, struct btrfs_inode, rb_node);
5651                         if (objectid <= btrfs_ino(BTRFS_I(&entry->vfs_inode))) {
5652                                 node = prev;
5653                                 break;
5654                         }
5655                         prev = rb_next(prev);
5656                 }
5657         }
5658         while (node) {
5659                 entry = rb_entry(node, struct btrfs_inode, rb_node);
5660                 objectid = btrfs_ino(BTRFS_I(&entry->vfs_inode)) + 1;
5661                 inode = igrab(&entry->vfs_inode);
5662                 if (inode) {
5663                         spin_unlock(&root->inode_lock);
5664                         if (atomic_read(&inode->i_count) > 1)
5665                                 d_prune_aliases(inode);
5666                         /*
5667                          * btrfs_drop_inode will have it removed from
5668                          * the inode cache when its usage count
5669                          * hits zero.
5670                          */
5671                         iput(inode);
5672                         cond_resched();
5673                         spin_lock(&root->inode_lock);
5674                         goto again;
5675                 }
5676 
5677                 if (cond_resched_lock(&root->inode_lock))
5678                         goto again;
5679 
5680                 node = rb_next(node);
5681         }
5682         spin_unlock(&root->inode_lock);
5683 }
5684 
5685 static int btrfs_init_locked_inode(struct inode *inode, void *p)
5686 {
5687         struct btrfs_iget_args *args = p;
5688         inode->i_ino = args->location->objectid;
5689         memcpy(&BTRFS_I(inode)->location, args->location,
5690                sizeof(*args->location));
5691         BTRFS_I(inode)->root = args->root;
5692         return 0;
5693 }
5694 
5695 static int btrfs_find_actor(struct inode *inode, void *opaque)
5696 {
5697         struct btrfs_iget_args *args = opaque;
5698         return args->location->objectid == BTRFS_I(inode)->location.objectid &&
5699                 args->root == BTRFS_I(inode)->root;
5700 }
5701 
5702 static struct inode *btrfs_iget_locked(struct super_block *s,
5703                                        struct btrfs_key *location,
5704                                        struct btrfs_root *root)
5705 {
5706         struct inode *inode;
5707         struct btrfs_iget_args args;
5708         unsigned long hashval = btrfs_inode_hash(location->objectid, root);
5709 
5710         args.location = location;
5711         args.root = root;
5712 
5713         inode = iget5_locked(s, hashval, btrfs_find_actor,
5714                              btrfs_init_locked_inode,
5715                              (void *)&args);
5716         return inode;
5717 }
5718 
5719 /* Get an inode object given its location and corresponding root.
5720  * Returns in *is_new if the inode was read from disk
5721  */
5722 struct inode *btrfs_iget(struct super_block *s, struct btrfs_key *location,
5723                          struct btrfs_root *root, int *new)
5724 {
5725         struct inode *inode;
5726 
5727         inode = btrfs_iget_locked(s, location, root);
5728         if (!inode)
5729                 return ERR_PTR(-ENOMEM);
5730 
5731         if (inode->i_state & I_NEW) {
5732                 int ret;
5733 
5734                 ret = btrfs_read_locked_inode(inode);
5735                 if (!is_bad_inode(inode)) {
5736                         inode_tree_add(inode);
5737                         unlock_new_inode(inode);
5738                         if (new)
5739                                 *new = 1;
5740                 } else {
5741                         unlock_new_inode(inode);
5742                         iput(inode);
5743                         ASSERT(ret < 0);
5744                         inode = ERR_PTR(ret < 0 ? ret : -ESTALE);
5745                 }
5746         }
5747 
5748         return inode;
5749 }
5750 
5751 static struct inode *new_simple_dir(struct super_block *s,
5752                                     struct btrfs_key *key,
5753                                     struct btrfs_root *root)
5754 {
5755         struct inode *inode = new_inode(s);
5756 
5757         if (!inode)
5758                 return ERR_PTR(-ENOMEM);
5759 
5760         BTRFS_I(inode)->root = root;
5761         memcpy(&BTRFS_I(inode)->location, key, sizeof(*key));
5762         set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags);
5763 
5764         inode->i_ino = BTRFS_EMPTY_SUBVOL_DIR_OBJECTID;
5765         inode->i_op = &btrfs_dir_ro_inode_operations;
5766         inode->i_opflags &= ~IOP_XATTR;
5767         inode->i_fop = &simple_dir_operations;
5768         inode->i_mode = S_IFDIR | S_IRUGO | S_IWUSR | S_IXUGO;
5769         inode->i_mtime = current_time(inode);
5770         inode->i_atime = inode->i_mtime;
5771         inode->i_ctime = inode->i_mtime;
5772         BTRFS_I(inode)->i_otime = inode->i_mtime;
5773 
5774         return inode;
5775 }
5776 
5777 struct inode *btrfs_lookup_dentry(struct inode *dir, struct dentry *dentry)
5778 {
5779         struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
5780         struct inode *inode;
5781         struct btrfs_root *root = BTRFS_I(dir)->root;
5782         struct btrfs_root *sub_root = root;
5783         struct btrfs_key location;
5784         int index;
5785         int ret = 0;
5786 
5787         if (dentry->d_name.len > BTRFS_NAME_LEN)
5788                 return ERR_PTR(-ENAMETOOLONG);
5789 
5790         ret = btrfs_inode_by_name(dir, dentry, &location);
5791         if (ret < 0)
5792                 return ERR_PTR(ret);
5793 
5794         if (location.type == BTRFS_INODE_ITEM_KEY) {
5795                 inode = btrfs_iget(dir->i_sb, &location, root, NULL);
5796                 return inode;
5797         }
5798 
5799         index = srcu_read_lock(&fs_info->subvol_srcu);
5800         ret = fixup_tree_root_location(fs_info, dir, dentry,
5801                                        &location, &sub_root);
5802         if (ret < 0) {
5803                 if (ret != -ENOENT)
5804                         inode = ERR_PTR(ret);
5805                 else
5806                         inode = new_simple_dir(dir->i_sb, &location, sub_root);
5807         } else {
5808                 inode = btrfs_iget(dir->i_sb, &location, sub_root, NULL);
5809         }
5810         srcu_read_unlock(&fs_info->subvol_srcu, index);
5811 
5812         if (!IS_ERR(inode) && root != sub_root) {
5813                 down_read(&fs_info->cleanup_work_sem);
5814                 if (!sb_rdonly(inode->i_sb))
5815                         ret = btrfs_orphan_cleanup(sub_root);
5816                 up_read(&fs_info->cleanup_work_sem);
5817                 if (ret) {
5818                         iput(inode);
5819                         inode = ERR_PTR(ret);
5820                 }
5821         }
5822 
5823         return inode;
5824 }
5825 
5826 static int btrfs_dentry_delete(const struct dentry *dentry)
5827 {
5828         struct btrfs_root *root;
5829         struct inode *inode = d_inode(dentry);
5830 
5831         if (!inode && !IS_ROOT(dentry))
5832                 inode = d_inode(dentry->d_parent);
5833 
5834         if (inode) {
5835                 root = BTRFS_I(inode)->root;
5836                 if (btrfs_root_refs(&root->root_item) == 0)
5837                         return 1;
5838 
5839                 if (btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)
5840                         return 1;
5841         }
5842         return 0;
5843 }
5844 
5845 static void btrfs_dentry_release(struct dentry *dentry)
5846 {
5847         kfree(dentry->d_fsdata);
5848 }
5849 
5850 static struct dentry *btrfs_lookup(struct inode *dir, struct dentry *dentry,
5851                                    unsigned int flags)
5852 {
5853         struct inode *inode;
5854 
5855         inode = btrfs_lookup_dentry(dir, dentry);
5856         if (IS_ERR(inode)) {
5857                 if (PTR_ERR(inode) == -ENOENT)
5858                         inode = NULL;
5859                 else
5860                         return ERR_CAST(inode);
5861         }
5862 
5863         return d_splice_alias(inode, dentry);
5864 }
5865 
5866 unsigned char btrfs_filetype_table[] = {
5867         DT_UNKNOWN, DT_REG, DT_DIR, DT_CHR, DT_BLK, DT_FIFO, DT_SOCK, DT_LNK
5868 };
5869 
5870 /*
5871  * All this infrastructure exists because dir_emit can fault, and we are holding
5872  * the tree lock when doing readdir.  For now just allocate a buffer and copy
5873  * our information into that, and then dir_emit from the buffer.  This is
5874  * similar to what NFS does, only we don't keep the buffer around in pagecache
5875  * because I'm afraid I'll mess that up.  Long term we need to make filldir do
5876  * copy_to_user_inatomic so we don't have to worry about page faulting under the
5877  * tree lock.
5878  */
5879 static int btrfs_opendir(struct inode *inode, struct file *file)
5880 {
5881         struct btrfs_file_private *private;
5882 
5883         private = kzalloc(sizeof(struct btrfs_file_private), GFP_KERNEL);
5884         if (!private)
5885                 return -ENOMEM;
5886         private->filldir_buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
5887         if (!private->filldir_buf) {
5888                 kfree(private);
5889                 return -ENOMEM;
5890         }
5891         file->private_data = private;
5892         return 0;
5893 }
5894 
5895 struct dir_entry {
5896         u64 ino;
5897         u64 offset;
5898         unsigned type;
5899         int name_len;
5900 };
5901 
5902 static int btrfs_filldir(void *addr, int entries, struct dir_context *ctx)
5903 {
5904         while (entries--) {
5905                 struct dir_entry *entry = addr;
5906                 char *name = (char *)(entry + 1);
5907 
5908                 ctx->pos = entry->offset;
5909                 if (!dir_emit(ctx, name, entry->name_len, entry->ino,
5910                               entry->type))
5911                         return 1;
5912                 addr += sizeof(struct dir_entry) + entry->name_len;
5913                 ctx->pos++;
5914         }
5915         return 0;
5916 }
5917 
5918 static int btrfs_real_readdir(struct file *file, struct dir_context *ctx)
5919 {
5920         struct inode *inode = file_inode(file);
5921         struct btrfs_root *root = BTRFS_I(inode)->root;
5922         struct btrfs_file_private *private = file->private_data;
5923         struct btrfs_dir_item *di;
5924         struct btrfs_key key;
5925         struct btrfs_key found_key;
5926         struct btrfs_path *path;
5927         void *addr;
5928         struct list_head ins_list;
5929         struct list_head del_list;
5930         int ret;
5931         struct extent_buffer *leaf;
5932         int slot;
5933         char *name_ptr;
5934         int name_len;
5935         int entries = 0;
5936         int total_len = 0;
5937         bool put = false;
5938         struct btrfs_key location;
5939 
5940         if (!dir_emit_dots(file, ctx))
5941                 return 0;
5942 
5943         path = btrfs_alloc_path();
5944         if (!path)
5945                 return -ENOMEM;
5946 
5947         addr = private->filldir_buf;
5948         path->reada = READA_FORWARD;
5949 
5950         INIT_LIST_HEAD(&ins_list);
5951         INIT_LIST_HEAD(&del_list);
5952         put = btrfs_readdir_get_delayed_items(inode, &ins_list, &del_list);
5953 
5954 again:
5955         key.type = BTRFS_DIR_INDEX_KEY;
5956         key.offset = ctx->pos;
5957         key.objectid = btrfs_ino(BTRFS_I(inode));
5958 
5959         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
5960         if (ret < 0)
5961                 goto err;
5962 
5963         while (1) {
5964                 struct dir_entry *entry;
5965 
5966                 leaf = path->nodes[0];
5967                 slot = path->slots[0];
5968                 if (slot >= btrfs_header_nritems(leaf)) {
5969                         ret = btrfs_next_leaf(root, path);
5970                         if (ret < 0)
5971                                 goto err;
5972                         else if (ret > 0)
5973                                 break;
5974                         continue;
5975                 }
5976 
5977                 btrfs_item_key_to_cpu(leaf, &found_key, slot);
5978 
5979                 if (found_key.objectid != key.objectid)
5980                         break;
5981                 if (found_key.type != BTRFS_DIR_INDEX_KEY)
5982                         break;
5983                 if (found_key.offset < ctx->pos)
5984                         goto next;
5985                 if (btrfs_should_delete_dir_index(&del_list, found_key.offset))
5986                         goto next;
5987                 di = btrfs_item_ptr(leaf, slot, struct btrfs_dir_item);
5988                 name_len = btrfs_dir_name_len(leaf, di);
5989                 if ((total_len + sizeof(struct dir_entry) + name_len) >=
5990                     PAGE_SIZE) {
5991                         btrfs_release_path(path);
5992                         ret = btrfs_filldir(private->filldir_buf, entries, ctx);
5993                         if (ret)
5994                                 goto nopos;
5995                         addr = private->filldir_buf;
5996                         entries = 0;
5997                         total_len = 0;
5998                         goto again;
5999                 }
6000 
6001                 entry = addr;
6002                 entry->name_len = name_len;
6003                 name_ptr = (char *)(entry + 1);
6004                 read_extent_buffer(leaf, name_ptr, (unsigned long)(di + 1),
6005                                    name_len);
6006                 entry->type = btrfs_filetype_table[btrfs_dir_type(leaf, di)];
6007                 btrfs_dir_item_key_to_cpu(leaf, di, &location);
6008                 entry->ino = location.objectid;
6009                 entry->offset = found_key.offset;
6010                 entries++;
6011                 addr += sizeof(struct dir_entry) + name_len;
6012                 total_len += sizeof(struct dir_entry) + name_len;
6013 next:
6014                 path->slots[0]++;
6015         }
6016         btrfs_release_path(path);
6017 
6018         ret = btrfs_filldir(private->filldir_buf, entries, ctx);
6019         if (ret)
6020                 goto nopos;
6021 
6022         ret = btrfs_readdir_delayed_dir_index(ctx, &ins_list);
6023         if (ret)
6024                 goto nopos;
6025 
6026         /*
6027          * Stop new entries from being returned after we return the last
6028          * entry.
6029          *
6030          * New directory entries are assigned a strictly increasing
6031          * offset.  This means that new entries created during readdir
6032          * are *guaranteed* to be seen in the future by that readdir.
6033          * This has broken buggy programs which operate on names as
6034          * they're returned by readdir.  Until we re-use freed offsets
6035          * we have this hack to stop new entries from being returned
6036          * under the assumption that they'll never reach this huge
6037          * offset.
6038          *
6039          * This is being careful not to overflow 32bit loff_t unless the
6040          * last entry requires it because doing so has broken 32bit apps
6041          * in the past.
6042          */
6043         if (ctx->pos >= INT_MAX)
6044                 ctx->pos = LLONG_MAX;
6045         else
6046                 ctx->pos = INT_MAX;
6047 nopos:
6048         ret = 0;
6049 err:
6050         if (put)
6051                 btrfs_readdir_put_delayed_items(inode, &ins_list, &del_list);
6052         btrfs_free_path(path);
6053         return ret;
6054 }
6055 
6056 int btrfs_write_inode(struct inode *inode, struct writeback_control *wbc)
6057 {
6058         struct btrfs_root *root = BTRFS_I(inode)->root;
6059         struct btrfs_trans_handle *trans;
6060         int ret = 0;
6061         bool nolock = false;
6062 
6063         if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags))
6064                 return 0;
6065 
6066         if (btrfs_fs_closing(root->fs_info) &&
6067                         btrfs_is_free_space_inode(BTRFS_I(inode)))
6068                 nolock = true;
6069 
6070         if (wbc->sync_mode == WB_SYNC_ALL) {
6071                 if (nolock)
6072                         trans = btrfs_join_transaction_nolock(root);
6073                 else
6074                         trans = btrfs_join_transaction(root);
6075                 if (IS_ERR(trans))
6076                         return PTR_ERR(trans);
6077                 ret = btrfs_commit_transaction(trans);
6078         }
6079         return ret;
6080 }
6081 
6082 /*
6083  * This is somewhat expensive, updating the tree every time the
6084  * inode changes.  But, it is most likely to find the inode in cache.
6085  * FIXME, needs more benchmarking...there are no reasons other than performance
6086  * to keep or drop this code.
6087  */
6088 static int btrfs_dirty_inode(struct inode *inode)
6089 {
6090         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
6091         struct btrfs_root *root = BTRFS_I(inode)->root;
6092         struct btrfs_trans_handle *trans;
6093         int ret;
6094 
6095         if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags))
6096                 return 0;
6097 
6098         trans = btrfs_join_transaction(root);
6099         if (IS_ERR(trans))
6100                 return PTR_ERR(trans);
6101 
6102         ret = btrfs_update_inode(trans, root, inode);
6103         if (ret && ret == -ENOSPC) {
6104                 /* whoops, lets try again with the full transaction */
6105                 btrfs_end_transaction(trans);
6106                 trans = btrfs_start_transaction(root, 1);
6107                 if (IS_ERR(trans))
6108                         return PTR_ERR(trans);
6109 
6110                 ret = btrfs_update_inode(trans, root, inode);
6111         }
6112         btrfs_end_transaction(trans);
6113         if (BTRFS_I(inode)->delayed_node)
6114                 btrfs_balance_delayed_items(fs_info);
6115 
6116         return ret;
6117 }
6118 
6119 /*
6120  * This is a copy of file_update_time.  We need this so we can return error on
6121  * ENOSPC for updating the inode in the case of file write and mmap writes.
6122  */
6123 static int btrfs_update_time(struct inode *inode, struct timespec *now,
6124                              int flags)
6125 {
6126         struct btrfs_root *root = BTRFS_I(inode)->root;
6127         bool dirty = flags & ~S_VERSION;
6128 
6129         if (btrfs_root_readonly(root))
6130                 return -EROFS;
6131 
6132         if (flags & S_VERSION)
6133                 dirty |= inode_maybe_inc_iversion(inode, dirty);
6134         if (flags & S_CTIME)
6135                 inode->i_ctime = *now;
6136         if (flags & S_MTIME)
6137                 inode->i_mtime = *now;
6138         if (flags & S_ATIME)
6139                 inode->i_atime = *now;
6140         return dirty ? btrfs_dirty_inode(inode) : 0;
6141 }
6142 
6143 /*
6144  * find the highest existing sequence number in a directory
6145  * and then set the in-memory index_cnt variable to reflect
6146  * free sequence numbers
6147  */
6148 static int btrfs_set_inode_index_count(struct btrfs_inode *inode)
6149 {
6150         struct btrfs_root *root = inode->root;
6151         struct btrfs_key key, found_key;
6152         struct btrfs_path *path;
6153         struct extent_buffer *leaf;
6154         int ret;
6155 
6156         key.objectid = btrfs_ino(inode);
6157         key.type = BTRFS_DIR_INDEX_KEY;
6158         key.offset = (u64)-1;
6159 
6160         path = btrfs_alloc_path();
6161         if (!path)
6162                 return -ENOMEM;
6163 
6164         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
6165         if (ret < 0)
6166                 goto out;
6167         /* FIXME: we should be able to handle this */
6168         if (ret == 0)
6169                 goto out;
6170         ret = 0;
6171 
6172         /*
6173          * MAGIC NUMBER EXPLANATION:
6174          * since we search a directory based on f_pos we have to start at 2
6175          * since '.' and '..' have f_pos of 0 and 1 respectively, so everybody
6176          * else has to start at 2
6177          */
6178         if (path->slots[0] == 0) {
6179                 inode->index_cnt = 2;
6180                 goto out;
6181         }
6182 
6183         path->slots[0]--;
6184 
6185         leaf = path->nodes[0];
6186         btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
6187 
6188         if (found_key.objectid != btrfs_ino(inode) ||
6189             found_key.type != BTRFS_DIR_INDEX_KEY) {
6190                 inode->index_cnt = 2;
6191                 goto out;
6192         }
6193 
6194         inode->index_cnt = found_key.offset + 1;
6195 out:
6196         btrfs_free_path(path);
6197         return ret;
6198 }
6199 
6200 /*
6201  * helper to find a free sequence number in a given directory.  This current
6202  * code is very simple, later versions will do smarter things in the btree
6203  */
6204 int btrfs_set_inode_index(struct btrfs_inode *dir, u64 *index)
6205 {
6206         int ret = 0;
6207 
6208         if (dir->index_cnt == (u64)-1) {
6209                 ret = btrfs_inode_delayed_dir_index_count(dir);
6210                 if (ret) {
6211                         ret = btrfs_set_inode_index_count(dir);
6212                         if (ret)
6213                                 return ret;
6214                 }
6215         }
6216 
6217         *index = dir->index_cnt;
6218         dir->index_cnt++;
6219 
6220         return ret;
6221 }
6222 
6223 static int btrfs_insert_inode_locked(struct inode *inode)
6224 {
6225         struct btrfs_iget_args args;
6226         args.location = &BTRFS_I(inode)->location;
6227         args.root = BTRFS_I(inode)->root;
6228 
6229         return insert_inode_locked4(inode,
6230                    btrfs_inode_hash(inode->i_ino, BTRFS_I(inode)->root),
6231                    btrfs_find_actor, &args);
6232 }
6233 
6234 /*
6235  * Inherit flags from the parent inode.
6236  *
6237  * Currently only the compression flags and the cow flags are inherited.
6238  */
6239 static void btrfs_inherit_iflags(struct inode *inode, struct inode *dir)
6240 {
6241         unsigned int flags;
6242 
6243         if (!dir)
6244                 return;
6245 
6246         flags = BTRFS_I(dir)->flags;
6247 
6248         if (flags & BTRFS_INODE_NOCOMPRESS) {
6249                 BTRFS_I(inode)->flags &= ~BTRFS_INODE_COMPRESS;
6250                 BTRFS_I(inode)->flags |= BTRFS_INODE_NOCOMPRESS;
6251         } else if (flags & BTRFS_INODE_COMPRESS) {
6252                 BTRFS_I(inode)->flags &= ~BTRFS_INODE_NOCOMPRESS;
6253                 BTRFS_I(inode)->flags |= BTRFS_INODE_COMPRESS;
6254         }
6255 
6256         if (flags & BTRFS_INODE_NODATACOW) {
6257                 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW;
6258                 if (S_ISREG(inode->i_mode))
6259                         BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM;
6260         }
6261 
6262         btrfs_update_iflags(inode);
6263 }
6264 
6265 static struct inode *btrfs_new_inode(struct btrfs_trans_handle *trans,
6266                                      struct btrfs_root *root,
6267                                      struct inode *dir,
6268                                      const char *name, int name_len,
6269                                      u64 ref_objectid, u64 objectid,
6270                                      umode_t mode, u64 *index)
6271 {
6272         struct btrfs_fs_info *fs_info = root->fs_info;
6273         struct inode *inode;
6274         struct btrfs_inode_item *inode_item;
6275         struct btrfs_key *location;
6276         struct btrfs_path *path;
6277         struct btrfs_inode_ref *ref;
6278         struct btrfs_key key[2];
6279         u32 sizes[2];
6280         int nitems = name ? 2 : 1;
6281         unsigned long ptr;
6282         int ret;
6283 
6284         path = btrfs_alloc_path();
6285         if (!path)
6286                 return ERR_PTR(-ENOMEM);
6287 
6288         inode = new_inode(fs_info->sb);
6289         if (!inode) {
6290                 btrfs_free_path(path);
6291                 return ERR_PTR(-ENOMEM);
6292         }
6293 
6294         /*
6295          * O_TMPFILE, set link count to 0, so that after this point,
6296          * we fill in an inode item with the correct link count.
6297          */
6298         if (!name)
6299                 set_nlink(inode, 0);
6300 
6301         /*
6302          * we have to initialize this early, so we can reclaim the inode
6303          * number if we fail afterwards in this function.
6304          */
6305         inode->i_ino = objectid;
6306 
6307         if (dir && name) {
6308                 trace_btrfs_inode_request(dir);
6309 
6310                 ret = btrfs_set_inode_index(BTRFS_I(dir), index);
6311                 if (ret) {
6312                         btrfs_free_path(path);
6313                         iput(inode);
6314                         return ERR_PTR(ret);
6315                 }
6316         } else if (dir) {
6317                 *index = 0;
6318         }
6319         /*
6320          * index_cnt is ignored for everything but a dir,
6321          * btrfs_set_inode_index_count has an explanation for the magic
6322          * number
6323          */
6324         BTRFS_I(inode)->index_cnt = 2;
6325         BTRFS_I(inode)->dir_index = *index;
6326         BTRFS_I(inode)->root = root;
6327         BTRFS_I(inode)->generation = trans->transid;
6328         inode->i_generation = BTRFS_I(inode)->generation;
6329 
6330         /*
6331          * We could have gotten an inode number from somebody who was fsynced
6332          * and then removed in this same transaction, so let's just set full
6333          * sync since it will be a full sync anyway and this will blow away the
6334          * old info in the log.
6335          */
6336         set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags);
6337 
6338         key[0].objectid = objectid;
6339         key[0].type = BTRFS_INODE_ITEM_KEY;
6340         key[0].offset = 0;
6341 
6342         sizes[0] = sizeof(struct btrfs_inode_item);
6343 
6344         if (name) {
6345                 /*
6346                  * Start new inodes with an inode_ref. This is slightly more
6347                  * efficient for small numbers of hard links since they will
6348                  * be packed into one item. Extended refs will kick in if we
6349                  * add more hard links than can fit in the ref item.
6350                  */
6351                 key[1].objectid = objectid;
6352                 key[1].type = BTRFS_INODE_REF_KEY;
6353                 key[1].offset = ref_objectid;
6354 
6355                 sizes[1] = name_len + sizeof(*ref);
6356         }
6357 
6358         location = &BTRFS_I(inode)->location;
6359         location->objectid = objectid;
6360         location->offset = 0;
6361         location->type = BTRFS_INODE_ITEM_KEY;
6362 
6363         ret = btrfs_insert_inode_locked(inode);
6364         if (ret < 0)
6365                 goto fail;
6366 
6367         path->leave_spinning = 1;
6368         ret = btrfs_insert_empty_items(trans, root, path, key, sizes, nitems);
6369         if (ret != 0)
6370                 goto fail_unlock;
6371 
6372         inode_init_owner(inode, dir, mode);
6373         inode_set_bytes(inode, 0);
6374 
6375         inode->i_mtime = current_time(inode);
6376         inode->i_atime = inode->i_mtime;
6377         inode->i_ctime = inode->i_mtime;
6378         BTRFS_I(inode)->i_otime = inode->i_mtime;
6379 
6380         inode_item = btrfs_item_ptr(path->nodes[0], path->slots[0],
6381                                   struct btrfs_inode_item);
6382         memzero_extent_buffer(path->nodes[0], (unsigned long)inode_item,
6383                              sizeof(*inode_item));
6384         fill_inode_item(trans, path->nodes[0], inode_item, inode);
6385 
6386         if (name) {
6387                 ref = btrfs_item_ptr(path->nodes[0], path->slots[0] + 1,
6388                                      struct btrfs_inode_ref);
6389                 btrfs_set_inode_ref_name_len(path->nodes[0], ref, name_len);
6390                 btrfs_set_inode_ref_index(path->nodes[0], ref, *index);
6391                 ptr = (unsigned long)(ref + 1);
6392                 write_extent_buffer(path->nodes[0], name, ptr, name_len);
6393         }
6394 
6395         btrfs_mark_buffer_dirty(path->nodes[0]);
6396         btrfs_free_path(path);
6397 
6398         btrfs_inherit_iflags(inode, dir);
6399 
6400         if (S_ISREG(mode)) {
6401                 if (btrfs_test_opt(fs_info, NODATASUM))
6402                         BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM;
6403                 if (btrfs_test_opt(fs_info, NODATACOW))
6404                         BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW |
6405                                 BTRFS_INODE_NODATASUM;
6406         }
6407 
6408         inode_tree_add(inode);
6409 
6410         trace_btrfs_inode_new(inode);
6411         btrfs_set_inode_last_trans(trans, inode);
6412 
6413         btrfs_update_root_times(trans, root);
6414 
6415         ret = btrfs_inode_inherit_props(trans, inode, dir);
6416         if (ret)
6417                 btrfs_err(fs_info,
6418                           "error inheriting props for ino %llu (root %llu): %d",
6419                         btrfs_ino(BTRFS_I(inode)), root->root_key.objectid, ret);
6420 
6421         return inode;
6422 
6423 fail_unlock:
6424         unlock_new_inode(inode);
6425 fail:
6426         if (dir && name)
6427                 BTRFS_I(dir)->index_cnt--;
6428         btrfs_free_path(path);
6429         iput(inode);
6430         return ERR_PTR(ret);
6431 }
6432 
6433 static inline u8 btrfs_inode_type(struct inode *inode)
6434 {
6435         return btrfs_type_by_mode[(inode->i_mode & S_IFMT) >> S_SHIFT];
6436 }
6437 
6438 /*
6439  * utility function to add 'inode' into 'parent_inode' with
6440  * a give name and a given sequence number.
6441  * if 'add_backref' is true, also insert a backref from the
6442  * inode to the parent directory.
6443  */
6444 int btrfs_add_link(struct btrfs_trans_handle *trans,
6445                    struct btrfs_inode *parent_inode, struct btrfs_inode *inode,
6446                    const char *name, int name_len, int add_backref, u64 index)
6447 {
6448         struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
6449         int ret = 0;
6450         struct btrfs_key key;
6451         struct btrfs_root *root = parent_inode->root;
6452         u64 ino = btrfs_ino(inode);
6453         u64 parent_ino = btrfs_ino(parent_inode);
6454 
6455         if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6456                 memcpy(&key, &inode->root->root_key, sizeof(key));
6457         } else {
6458                 key.objectid = ino;
6459                 key.type = BTRFS_INODE_ITEM_KEY;
6460                 key.offset = 0;
6461         }
6462 
6463         if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6464                 ret = btrfs_add_root_ref(trans, fs_info, key.objectid,
6465                                          root->root_key.objectid, parent_ino,
6466                                          index, name, name_len);
6467         } else if (add_backref) {
6468                 ret = btrfs_insert_inode_ref(trans, root, name, name_len, ino,
6469                                              parent_ino, index);
6470         }
6471 
6472         /* Nothing to clean up yet */
6473         if (ret)
6474                 return ret;
6475 
6476         ret = btrfs_insert_dir_item(trans, root, name, name_len,
6477                                     parent_inode, &key,
6478                                     btrfs_inode_type(&inode->vfs_inode), index);
6479         if (ret == -EEXIST || ret == -EOVERFLOW)
6480                 goto fail_dir_item;
6481         else if (ret) {
6482                 btrfs_abort_transaction(trans, ret);
6483                 return ret;
6484         }
6485 
6486         btrfs_i_size_write(parent_inode, parent_inode->vfs_inode.i_size +
6487                            name_len * 2);
6488         inode_inc_iversion(&parent_inode->vfs_inode);
6489         parent_inode->vfs_inode.i_mtime = parent_inode->vfs_inode.i_ctime =
6490                 current_time(&parent_inode->vfs_inode);
6491         ret = btrfs_update_inode(trans, root, &parent_inode->vfs_inode);
6492         if (ret)
6493                 btrfs_abort_transaction(trans, ret);
6494         return ret;
6495 
6496 fail_dir_item:
6497         if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6498                 u64 local_index;
6499                 int err;
6500                 err = btrfs_del_root_ref(trans, fs_info, key.objectid,
6501                                          root->root_key.objectid, parent_ino,
6502                                          &local_index, name, name_len);
6503 
6504         } else if (add_backref) {
6505                 u64 local_index;
6506                 int err;
6507 
6508                 err = btrfs_del_inode_ref(trans, root, name, name_len,
6509                                           ino, parent_ino, &local_index);
6510         }
6511         return ret;
6512 }
6513 
6514 static int btrfs_add_nondir(struct btrfs_trans_handle *trans,
6515                             struct btrfs_inode *dir, struct dentry *dentry,
6516                             struct btrfs_inode *inode, int backref, u64 index)
6517 {
6518         int err = btrfs_add_link(trans, dir, inode,
6519                                  dentry->d_name.name, dentry->d_name.len,
6520                                  backref, index);
6521         if (err > 0)
6522                 err = -EEXIST;
6523         return err;
6524 }
6525 
6526 static int btrfs_mknod(struct inode *dir, struct dentry *dentry,
6527                         umode_t mode, dev_t rdev)
6528 {
6529         struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
6530         struct btrfs_trans_handle *trans;
6531         struct btrfs_root *root = BTRFS_I(dir)->root;
6532         struct inode *inode = NULL;
6533         int err;
6534         int drop_inode = 0;
6535         u64 objectid;
6536         u64 index = 0;
6537 
6538         /*
6539          * 2 for inode item and ref
6540          * 2 for dir items
6541          * 1 for xattr if selinux is on
6542          */
6543         trans = btrfs_start_transaction(root, 5);
6544         if (IS_ERR(trans))
6545                 return PTR_ERR(trans);
6546 
6547         err = btrfs_find_free_ino(root, &objectid);
6548         if (err)
6549                 goto out_unlock;
6550 
6551         inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
6552                         dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid,
6553                         mode, &index);
6554         if (IS_ERR(inode)) {
6555                 err = PTR_ERR(inode);
6556                 goto out_unlock;
6557         }
6558 
6559         /*
6560         * If the active LSM wants to access the inode during
6561         * d_instantiate it needs these. Smack checks to see
6562         * if the filesystem supports xattrs by looking at the
6563         * ops vector.
6564         */
6565         inode->i_op = &btrfs_special_inode_operations;
6566         init_special_inode(inode, inode->i_mode, rdev);
6567 
6568         err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
6569         if (err)
6570                 goto out_unlock_inode;
6571 
6572         err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode),
6573                         0, index);
6574         if (err) {
6575                 goto out_unlock_inode;
6576         } else {
6577                 btrfs_update_inode(trans, root, inode);
6578                 unlock_new_inode(inode);
6579                 d_instantiate(dentry, inode);
6580         }
6581 
6582 out_unlock:
6583         btrfs_end_transaction(trans);
6584         btrfs_btree_balance_dirty(fs_info);
6585         if (drop_inode) {
6586                 inode_dec_link_count(inode);
6587                 iput(inode);
6588         }
6589         return err;
6590 
6591 out_unlock_inode:
6592         drop_inode = 1;
6593         unlock_new_inode(inode);
6594         goto out_unlock;
6595 
6596 }
6597 
6598 static int btrfs_create(struct inode *dir, struct dentry *dentry,
6599                         umode_t mode, bool excl)
6600 {
6601         struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
6602         struct btrfs_trans_handle *trans;
6603         struct btrfs_root *root = BTRFS_I(dir)->root;
6604         struct inode *inode = NULL;
6605         int drop_inode_on_err = 0;
6606         int err;
6607         u64 objectid;
6608         u64 index = 0;
6609 
6610         /*
6611          * 2 for inode item and ref
6612          * 2 for dir items
6613          * 1 for xattr if selinux is on
6614          */
6615         trans = btrfs_start_transaction(root, 5);
6616         if (IS_ERR(trans))
6617                 return PTR_ERR(trans);
6618 
6619         err = btrfs_find_free_ino(root, &objectid);
6620         if (err)
6621                 goto out_unlock;
6622 
6623         inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
6624                         dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid,
6625                         mode, &index);
6626         if (IS_ERR(inode)) {
6627                 err = PTR_ERR(inode);
6628                 goto out_unlock;
6629         }
6630         drop_inode_on_err = 1;
6631         /*
6632         * If the active LSM wants to access the inode during
6633         * d_instantiate it needs these. Smack checks to see
6634         * if the filesystem supports xattrs by looking at the
6635         * ops vector.
6636         */
6637         inode->i_fop = &btrfs_file_operations;
6638         inode->i_op = &btrfs_file_inode_operations;
6639         inode->i_mapping->a_ops = &btrfs_aops;
6640 
6641         err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
6642         if (err)
6643                 goto out_unlock_inode;
6644 
6645         err = btrfs_update_inode(trans, root, inode);
6646         if (err)
6647                 goto out_unlock_inode;
6648 
6649         err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode),
6650                         0, index);
6651         if (err)
6652                 goto out_unlock_inode;
6653 
6654         BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
6655         unlock_new_inode(inode);
6656         d_instantiate(dentry, inode);
6657 
6658 out_unlock:
6659         btrfs_end_transaction(trans);
6660         if (err && drop_inode_on_err) {
6661                 inode_dec_link_count(inode);
6662                 iput(inode);
6663         }
6664         btrfs_btree_balance_dirty(fs_info);
6665         return err;
6666 
6667 out_unlock_inode:
6668         unlock_new_inode(inode);
6669         goto out_unlock;
6670 
6671 }
6672 
6673 static int btrfs_link(struct dentry *old_dentry, struct inode *dir,
6674                       struct dentry *dentry)
6675 {
6676         struct btrfs_trans_handle *trans = NULL;
6677         struct btrfs_root *root = BTRFS_I(dir)->root;
6678         struct inode *inode = d_inode(old_dentry);
6679         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
6680         u64 index;
6681         int err;
6682         int drop_inode = 0;
6683 
6684         /* do not allow sys_link's with other subvols of the same device */
6685         if (root->objectid != BTRFS_I(inode)->root->objectid)
6686                 return -EXDEV;
6687 
6688         if (inode->i_nlink >= BTRFS_LINK_MAX)
6689                 return -EMLINK;
6690 
6691         err = btrfs_set_inode_index(BTRFS_I(dir), &index);
6692         if (err)
6693                 goto fail;
6694 
6695         /*
6696          * 2 items for inode and inode ref
6697          * 2 items for dir items
6698          * 1 item for parent inode
6699          */
6700         trans = btrfs_start_transaction(root, 5);
6701         if (IS_ERR(trans)) {
6702                 err = PTR_ERR(trans);
6703                 trans = NULL;
6704                 goto fail;
6705         }
6706 
6707         /* There are several dir indexes for this inode, clear the cache. */
6708         BTRFS_I(inode)->dir_index = 0ULL;
6709         inc_nlink(inode);
6710         inode_inc_iversion(inode);
6711         inode->i_ctime = current_time(inode);
6712         ihold(inode);
6713         set_bit(BTRFS_INODE_COPY_EVERYTHING, &BTRFS_I(inode)->runtime_flags);
6714 
6715         err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode),
6716                         1, index);
6717 
6718         if (err) {
6719                 drop_inode = 1;
6720         } else {
6721                 struct dentry *parent = dentry->d_parent;
6722                 err = btrfs_update_inode(trans, root, inode);
6723                 if (err)
6724                         goto fail;
6725                 if (inode->i_nlink == 1) {
6726                         /*
6727                          * If new hard link count is 1, it's a file created
6728                          * with open(2) O_TMPFILE flag.
6729                          */
6730                         err = btrfs_orphan_del(trans, BTRFS_I(inode));
6731                         if (err)
6732                                 goto fail;
6733                 }
6734                 d_instantiate(dentry, inode);
6735                 btrfs_log_new_name(trans, BTRFS_I(inode), NULL, parent);
6736         }
6737 
6738 fail:
6739         if (trans)
6740                 btrfs_end_transaction(trans);
6741         if (drop_inode) {
6742                 inode_dec_link_count(inode);
6743                 iput(inode);
6744         }
6745         btrfs_btree_balance_dirty(fs_info);
6746         return err;
6747 }
6748 
6749 static int btrfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
6750 {
6751         struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
6752         struct inode *inode = NULL;
6753         struct btrfs_trans_handle *trans;
6754         struct btrfs_root *root = BTRFS_I(dir)->root;
6755         int err = 0;
6756         int drop_on_err = 0;
6757         u64 objectid = 0;
6758         u64 index = 0;
6759 
6760         /*
6761          * 2 items for inode and ref
6762          * 2 items for dir items
6763          * 1 for xattr if selinux is on
6764          */
6765         trans = btrfs_start_transaction(root, 5);
6766         if (IS_ERR(trans))
6767                 return PTR_ERR(trans);
6768 
6769         err = btrfs_find_free_ino(root, &objectid);
6770         if (err)
6771                 goto out_fail;
6772 
6773         inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
6774                         dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid,
6775                         S_IFDIR | mode, &index);
6776         if (IS_ERR(inode)) {
6777                 err = PTR_ERR(inode);
6778                 goto out_fail;
6779         }
6780 
6781         drop_on_err = 1;
6782         /* these must be set before we unlock the inode */
6783         inode->i_op = &btrfs_dir_inode_operations;
6784         inode->i_fop = &btrfs_dir_file_operations;
6785 
6786         err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
6787         if (err)
6788                 goto out_fail_inode;
6789 
6790         btrfs_i_size_write(BTRFS_I(inode), 0);
6791         err = btrfs_update_inode(trans, root, inode);
6792         if (err)
6793                 goto out_fail_inode;
6794 
6795         err = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode),
6796                         dentry->d_name.name,
6797                         dentry->d_name.len, 0, index);
6798         if (err)
6799                 goto out_fail_inode;
6800 
6801         d_instantiate(dentry, inode);
6802         /*
6803          * mkdir is special.  We're unlocking after we call d_instantiate
6804          * to avoid a race with nfsd calling d_instantiate.
6805          */
6806         unlock_new_inode(inode);
6807         drop_on_err = 0;
6808 
6809 out_fail:
6810         btrfs_end_transaction(trans);
6811         if (drop_on_err) {
6812                 inode_dec_link_count(inode);
6813                 iput(inode);
6814         }
6815         btrfs_btree_balance_dirty(fs_info);
6816         return err;
6817 
6818 out_fail_inode:
6819         unlock_new_inode(inode);
6820         goto out_fail;
6821 }
6822 
6823 static noinline int uncompress_inline(struct btrfs_path *path,
6824                                       struct page *page,
6825                                       size_t pg_offset, u64 extent_offset,
6826                                       struct btrfs_file_extent_item *item)
6827 {
6828         int ret;
6829         struct extent_buffer *leaf = path->nodes[0];
6830         char *tmp;
6831         size_t max_size;
6832         unsigned long inline_size;
6833         unsigned long ptr;
6834         int compress_type;
6835 
6836         WARN_ON(pg_offset != 0);
6837         compress_type = btrfs_file_extent_compression(leaf, item);
6838         max_size = btrfs_file_extent_ram_bytes(leaf, item);
6839         inline_size = btrfs_file_extent_inline_item_len(leaf,
6840                                         btrfs_item_nr(path->slots[0]));
6841         tmp = kmalloc(inline_size, GFP_NOFS);
6842         if (!tmp)
6843                 return -ENOMEM;
6844         ptr = btrfs_file_extent_inline_start(item);
6845 
6846         read_extent_buffer(leaf, tmp, ptr, inline_size);
6847 
6848         max_size = min_t(unsigned long, PAGE_SIZE, max_size);
6849         ret = btrfs_decompress(compress_type, tmp, page,
6850                                extent_offset, inline_size, max_size);
6851 
6852         /*
6853          * decompression code contains a memset to fill in any space between the end
6854          * of the uncompressed data and the end of max_size in case the decompressed
6855          * data ends up shorter than ram_bytes.  That doesn't cover the hole between
6856          * the end of an inline extent and the beginning of the next block, so we
6857          * cover that region here.
6858          */
6859 
6860         if (max_size + pg_offset < PAGE_SIZE) {
6861                 char *map = kmap(page);
6862                 memset(map + pg_offset + max_size, 0, PAGE_SIZE - max_size - pg_offset);
6863                 kunmap(page);
6864         }
6865         kfree(tmp);
6866         return ret;
6867 }
6868 
6869 /*
6870  * a bit scary, this does extent mapping from logical file offset to the disk.
6871  * the ugly parts come from merging extents from the disk with the in-ram
6872  * representation.  This gets more complex because of the data=ordered code,
6873  * where the in-ram extents might be locked pending data=ordered completion.
6874  *
6875  * This also copies inline extents directly into the page.
6876  */
6877 struct extent_map *btrfs_get_extent(struct btrfs_inode *inode,
6878                 struct page *page,
6879             size_t pg_offset, u64 start, u64 len,
6880                 int create)
6881 {
6882         struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
6883         int ret;
6884         int err = 0;
6885         u64 extent_start = 0;
6886         u64 extent_end = 0;
6887         u64 objectid = btrfs_ino(inode);
6888         u32 found_type;
6889         struct btrfs_path *path = NULL;
6890         struct btrfs_root *root = inode->root;
6891         struct btrfs_file_extent_item *item;
6892         struct extent_buffer *leaf;
6893         struct btrfs_key found_key;
6894         struct extent_map *em = NULL;
6895         struct extent_map_tree *em_tree = &inode->extent_tree;
6896         struct extent_io_tree *io_tree = &inode->io_tree;
6897         const bool new_inline = !page || create;
6898 
6899         read_lock(&em_tree->lock);
6900         em = lookup_extent_mapping(em_tree, start, len);
6901         if (em)
6902                 em->bdev = fs_info->fs_devices->latest_bdev;
6903         read_unlock(&em_tree->lock);
6904 
6905         if (em) {
6906                 if (em->start > start || em->start + em->len <= start)
6907                         free_extent_map(em);
6908                 else if (em->block_start == EXTENT_MAP_INLINE && page)
6909                         free_extent_map(em);
6910                 else
6911                         goto out;
6912         }
6913         em = alloc_extent_map();
6914         if (!em) {
6915                 err = -ENOMEM;
6916                 goto out;
6917         }
6918         em->bdev = fs_info->fs_devices->latest_bdev;
6919         em->start = EXTENT_MAP_HOLE;
6920         em->orig_start = EXTENT_MAP_HOLE;
6921         em->len = (u64)-1;
6922         em->block_len = (u64)-1;
6923 
6924         if (!path) {
6925                 path = btrfs_alloc_path();
6926                 if (!path) {
6927                         err = -ENOMEM;
6928                         goto out;
6929                 }
6930                 /*
6931                  * Chances are we'll be called again, so go ahead and do
6932                  * readahead
6933                  */
6934                 path->reada = READA_FORWARD;
6935         }
6936 
6937         ret = btrfs_lookup_file_extent(NULL, root, path, objectid, start, 0);
6938         if (ret < 0) {
6939                 err = ret;
6940                 goto out;
6941         }
6942 
6943         if (ret != 0) {
6944                 if (path->slots[0] == 0)
6945                         goto not_found;
6946                 path->slots[0]--;
6947         }
6948 
6949         leaf = path->nodes[0];
6950         item = btrfs_item_ptr(leaf, path->slots[0],
6951                               struct btrfs_file_extent_item);
6952         /* are we inside the extent that was found? */
6953         btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
6954         found_type = found_key.type;
6955         if (found_key.objectid != objectid ||
6956             found_type != BTRFS_EXTENT_DATA_KEY) {
6957                 /*
6958                  * If we backup past the first extent we want to move forward
6959                  * and see if there is an extent in front of us, otherwise we'll
6960                  * say there is a hole for our whole search range which can
6961                  * cause problems.
6962                  */
6963                 extent_end = start;
6964                 goto next;
6965         }
6966 
6967         found_type = btrfs_file_extent_type(leaf, item);
6968         extent_start = found_key.offset;
6969         if (found_type == BTRFS_FILE_EXTENT_REG ||
6970             found_type == BTRFS_FILE_EXTENT_PREALLOC) {
6971                 extent_end = extent_start +
6972                        btrfs_file_extent_num_bytes(leaf, item);
6973 
6974                 trace_btrfs_get_extent_show_fi_regular(inode, leaf, item,
6975                                                        extent_start);
6976         } else if (found_type == BTRFS_FILE_EXTENT_INLINE) {
6977                 size_t size;
6978                 size = btrfs_file_extent_inline_len(leaf, path->slots[0], item);
6979                 extent_end = ALIGN(extent_start + size,
6980                                    fs_info->sectorsize);
6981 
6982                 trace_btrfs_get_extent_show_fi_inline(inode, leaf, item,
6983                                                       path->slots[0],
6984                                                       extent_start);
6985         }
6986 next:
6987         if (start >= extent_end) {
6988                 path->slots[0]++;
6989                 if (path->slots[0] >= btrfs_header_nritems(leaf)) {
6990                         ret = btrfs_next_leaf(root, path);
6991                         if (ret < 0) {
6992                                 err = ret;
6993                                 goto out;
6994                         }
6995                         if (ret > 0)
6996                                 goto not_found;
6997                         leaf = path->nodes[0];
6998                 }
6999                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
7000                 if (found_key.objectid != objectid ||
7001                     found_key.type != BTRFS_EXTENT_DATA_KEY)
7002                         goto not_found;
7003                 if (start + len <= found_key.offset)
7004                         goto not_found;
7005                 if (start > found_key.offset)
7006                         goto next;
7007                 em->start = start;
7008                 em->orig_start = start;
7009                 em->len = found_key.offset - start;
7010                 goto not_found_em;
7011         }
7012 
7013         btrfs_extent_item_to_extent_map(inode, path, item,
7014                         new_inline, em);
7015 
7016         if (found_type == BTRFS_FILE_EXTENT_REG ||
7017             found_type == BTRFS_FILE_EXTENT_PREALLOC) {
7018                 goto insert;
7019         } else if (found_type == BTRFS_FILE_EXTENT_INLINE) {
7020                 unsigned long ptr;
7021                 char *map;
7022                 size_t size;
7023                 size_t extent_offset;
7024                 size_t copy_size;
7025 
7026                 if (new_inline)
7027                         goto out;
7028 
7029                 size = btrfs_file_extent_inline_len(leaf, path->slots[0], item);
7030                 extent_offset = page_offset(page) + pg_offset - extent_start;
7031                 copy_size = min_t(u64, PAGE_SIZE - pg_offset,
7032                                   size - extent_offset);
7033                 em->start = extent_start + extent_offset;
7034                 em->len = ALIGN(copy_size, fs_info->sectorsize);
7035                 em->orig_block_len = em->len;
7036                 em->orig_start = em->start;
7037                 ptr = btrfs_file_extent_inline_start(item) + extent_offset;
7038                 if (!PageUptodate(page)) {
7039                         if (btrfs_file_extent_compression(leaf, item) !=
7040                             BTRFS_COMPRESS_NONE) {
7041                                 ret = uncompress_inline(path, page, pg_offset,
7042                                                         extent_offset, item);
7043                                 if (ret) {
7044                                         err = ret;
7045                                         goto out;
7046                                 }
7047                         } else {
7048                                 map = kmap(page);
7049                                 read_extent_buffer(leaf, map + pg_offset, ptr,
7050                                                    copy_size);
7051                                 if (pg_offset + copy_size < PAGE_SIZE) {
7052                                         memset(map + pg_offset + copy_size, 0,
7053                                                PAGE_SIZE - pg_offset -
7054                                                copy_size);
7055                                 }
7056                                 kunmap(page);
7057                         }
7058                         flush_dcache_page(page);
7059                 }
7060                 set_extent_uptodate(io_tree, em->start,
7061                                     extent_map_end(em) - 1, NULL, GFP_NOFS);
7062                 goto insert;
7063         }
7064 not_found:
7065         em->start = start;
7066         em->orig_start = start;
7067         em->len = len;
7068 not_found_em:
7069         em->block_start = EXTENT_MAP_HOLE;
7070 insert:
7071         btrfs_release_path(path);
7072         if (em->start > start || extent_map_end(em) <= start) {
7073                 btrfs_err(fs_info,
7074                           "bad extent! em: [%llu %llu] passed [%llu %llu]",
7075                           em->start, em->len, start, len);
7076                 err = -EIO;
7077                 goto out;
7078         }
7079 
7080         err = 0;
7081         write_lock(&em_tree->lock);
7082         err = btrfs_add_extent_mapping(em_tree, &em, start, len);
7083         write_unlock(&em_tree->lock);
7084 out:
7085 
7086         trace_btrfs_get_extent(root, inode, em);
7087 
7088         btrfs_free_path(path);
7089         if (err) {
7090                 free_extent_map(em);
7091                 return ERR_PTR(err);
7092         }
7093         BUG_ON(!em); /* Error is always set */
7094         return em;
7095 }
7096 
7097 struct extent_map *btrfs_get_extent_fiemap(struct btrfs_inode *inode,
7098                 struct page *page,
7099                 size_t pg_offset, u64 start, u64 len,
7100                 int create)
7101 {
7102         struct extent_map *em;
7103         struct extent_map *hole_em = NULL;
7104         u64 range_start = start;
7105         u64 end;
7106         u64 found;
7107         u64 found_end;
7108         int err = 0;
7109 
7110         em = btrfs_get_extent(inode, page, pg_offset, start, len, create);
7111         if (IS_ERR(em))
7112                 return em;
7113         /*
7114          * If our em maps to:
7115          * - a hole or
7116          * - a pre-alloc extent,
7117          * there might actually be delalloc bytes behind it.
7118          */
7119         if (em->block_start != EXTENT_MAP_HOLE &&
7120             !test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
7121                 return em;
7122         else
7123                 hole_em = em;
7124 
7125         /* check to see if we've wrapped (len == -1 or similar) */
7126         end = start + len;
7127         if (end < start)
7128                 end = (u64)-1;
7129         else
7130                 end -= 1;
7131 
7132         em = NULL;
7133 
7134         /* ok, we didn't find anything, lets look for delalloc */
7135         found = count_range_bits(&inode->io_tree, &range_start,
7136                                  end, len, EXTENT_DELALLOC, 1);
7137         found_end = range_start + found;
7138         if (found_end < range_start)
7139                 found_end = (u64)-1;
7140 
7141         /*
7142          * we didn't find anything useful, return
7143          * the original results from get_extent()
7144          */
7145         if (range_start > end || found_end <= start) {
7146                 em = hole_em;
7147                 hole_em = NULL;
7148                 goto out;
7149         }
7150 
7151         /* adjust the range_start to make sure it doesn't
7152          * go backwards from the start they passed in
7153          */
7154         range_start = max(start, range_start);
7155         found = found_end - range_start;
7156 
7157         if (found > 0) {
7158                 u64 hole_start = start;
7159                 u64 hole_len = len;
7160 
7161                 em = alloc_extent_map();
7162                 if (!em) {
7163                         err = -ENOMEM;
7164                         goto out;
7165                 }
7166                 /*
7167                  * when btrfs_get_extent can't find anything it
7168                  * returns one huge hole
7169                  *
7170                  * make sure what it found really fits our range, and
7171                  * adjust to make sure it is based on the start from
7172                  * the caller
7173                  */
7174                 if (hole_em) {
7175                         u64 calc_end = extent_map_end(hole_em);
7176 
7177                         if (calc_end <= start || (hole_em->start > end)) {
7178                                 free_extent_map(hole_em);
7179                                 hole_em = NULL;
7180                         } else {
7181                                 hole_start = max(hole_em->start, start);
7182                                 hole_len = calc_end - hole_start;
7183                         }
7184                 }
7185                 em->bdev = NULL;
7186                 if (hole_em && range_start > hole_start) {
7187                         /* our hole starts before our delalloc, so we
7188                          * have to return just the parts of the hole
7189                          * that go until  the delalloc starts
7190                          */
7191                         em->len = min(hole_len,
7192                                       range_start - hole_start);
7193                         em->start = hole_start;
7194                         em->orig_start = hole_start;
7195                         /*
7196                          * don't adjust block start at all,
7197                          * it is fixed at EXTENT_MAP_HOLE
7198                          */
7199                         em->block_start = hole_em->block_start;
7200                         em->block_len = hole_len;
7201                         if (test_bit(EXTENT_FLAG_PREALLOC, &hole_em->flags))
7202                                 set_bit(EXTENT_FLAG_PREALLOC, &em->flags);
7203                 } else {
7204                         em->start = range_start;
7205                         em->len = found;
7206                         em->orig_start = range_start;
7207                         em->block_start = EXTENT_MAP_DELALLOC;
7208                         em->block_len = found;
7209                 }
7210         } else {
7211                 return hole_em;
7212         }
7213 out:
7214 
7215         free_extent_map(hole_em);
7216         if (err) {
7217                 free_extent_map(em);
7218                 return ERR_PTR(err);
7219         }
7220         return em;
7221 }
7222 
7223 static struct extent_map *btrfs_create_dio_extent(struct inode *inode,
7224                                                   const u64 start,
7225                                                   const u64 len,
7226                                                   const u64 orig_start,
7227                                                   const u64 block_start,
7228                                                   const u64 block_len,
7229                                                   const u64 orig_block_len,
7230                                                   const u64 ram_bytes,
7231                                                   const int type)
7232 {
7233         struct extent_map *em = NULL;
7234         int ret;
7235 
7236         if (type != BTRFS_ORDERED_NOCOW) {
7237                 em = create_io_em(inode, start, len, orig_start,
7238                                   block_start, block_len, orig_block_len,
7239                                   ram_bytes,
7240                                   BTRFS_COMPRESS_NONE, /* compress_type */
7241                                   type);
7242                 if (IS_ERR(em))
7243                         goto out;
7244         }
7245         ret = btrfs_add_ordered_extent_dio(inode, start, block_start,
7246                                            len, block_len, type);
7247         if (ret) {
7248                 if (em) {
7249                         free_extent_map(em);
7250                         btrfs_drop_extent_cache(BTRFS_I(inode), start,
7251                                                 start + len - 1, 0);
7252                 }
7253                 em = ERR_PTR(ret);
7254         }
7255  out:
7256 
7257         return em;
7258 }
7259 
7260 static struct extent_map *btrfs_new_extent_direct(struct inode *inode,
7261                                                   u64 start, u64 len)
7262 {
7263         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7264         struct btrfs_root *root = BTRFS_I(inode)->root;
7265         struct extent_map *em;
7266         struct btrfs_key ins;
7267         u64 alloc_hint;
7268         int ret;
7269 
7270         alloc_hint = get_extent_allocation_hint(inode, start, len);
7271         ret = btrfs_reserve_extent(root, len, len, fs_info->sectorsize,
7272                                    0, alloc_hint, &ins, 1, 1);
7273         if (ret)
7274                 return ERR_PTR(ret);
7275 
7276         em = btrfs_create_dio_extent(inode, start, ins.offset, start,
7277                                      ins.objectid, ins.offset, ins.offset,
7278                                      ins.offset, BTRFS_ORDERED_REGULAR);
7279         btrfs_dec_block_group_reservations(fs_info, ins.objectid);
7280         if (IS_ERR(em))
7281                 btrfs_free_reserved_extent(fs_info, ins.objectid,
7282                                            ins.offset, 1);
7283 
7284         return em;
7285 }
7286 
7287 /*
7288  * returns 1 when the nocow is safe, < 1 on error, 0 if the
7289  * block must be cow'd
7290  */
7291 noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len,
7292                               u64 *orig_start, u64 *orig_block_len,
7293                               u64 *ram_bytes)
7294 {
7295         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7296         struct btrfs_path *path;
7297         int ret;
7298         struct extent_buffer *leaf;
7299         struct btrfs_root *root = BTRFS_I(inode)->root;
7300         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
7301         struct btrfs_file_extent_item *fi;
7302         struct btrfs_key key;
7303         u64 disk_bytenr;
7304         u64 backref_offset;
7305         u64 extent_end;
7306         u64 num_bytes;
7307         int slot;
7308         int found_type;
7309         bool nocow = (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW);
7310 
7311         path = btrfs_alloc_path();
7312         if (!path)
7313                 return -ENOMEM;
7314 
7315         ret = btrfs_lookup_file_extent(NULL, root, path,
7316                         btrfs_ino(BTRFS_I(inode)), offset, 0);
7317         if (ret < 0)
7318                 goto out;
7319 
7320         slot = path->slots[0];
7321         if (ret == 1) {
7322                 if (slot == 0) {
7323                         /* can't find the item, must cow */
7324                         ret = 0;
7325                         goto out;
7326                 }
7327                 slot--;
7328         }
7329         ret = 0;
7330         leaf = path->nodes[0];
7331         btrfs_item_key_to_cpu(leaf, &key, slot);
7332         if (key.objectid != btrfs_ino(BTRFS_I(inode)) ||
7333             key.type != BTRFS_EXTENT_DATA_KEY) {
7334                 /* not our file or wrong item type, must cow */
7335                 goto out;
7336         }
7337 
7338         if (key.offset > offset) {
7339                 /* Wrong offset, must cow */
7340                 goto out;
7341         }
7342 
7343         fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
7344         found_type = btrfs_file_extent_type(leaf, fi);
7345         if (found_type != BTRFS_FILE_EXTENT_REG &&
7346             found_type != BTRFS_FILE_EXTENT_PREALLOC) {
7347                 /* not a regular extent, must cow */
7348                 goto out;
7349         }
7350 
7351         if (!nocow && found_type == BTRFS_FILE_EXTENT_REG)
7352                 goto out;
7353 
7354         extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
7355         if (extent_end <= offset)
7356                 goto out;
7357 
7358         disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
7359         if (disk_bytenr == 0)
7360                 goto out;
7361 
7362         if (btrfs_file_extent_compression(leaf, fi) ||
7363             btrfs_file_extent_encryption(leaf, fi) ||
7364             btrfs_file_extent_other_encoding(leaf, fi))
7365                 goto out;
7366 
7367         backref_offset = btrfs_file_extent_offset(leaf, fi);
7368 
7369         if (orig_start) {
7370                 *orig_start = key.offset - backref_offset;
7371                 *orig_block_len = btrfs_file_extent_disk_num_bytes(leaf, fi);
7372                 *ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi);
7373         }
7374 
7375         if (btrfs_extent_readonly(fs_info, disk_bytenr))
7376                 goto out;
7377 
7378         num_bytes = min(offset + *len, extent_end) - offset;
7379         if (!nocow && found_type == BTRFS_FILE_EXTENT_PREALLOC) {
7380                 u64 range_end;
7381 
7382                 range_end = round_up(offset + num_bytes,
7383                                      root->fs_info->sectorsize) - 1;
7384                 ret = test_range_bit(io_tree, offset, range_end,
7385                                      EXTENT_DELALLOC, 0, NULL);
7386                 if (ret) {
7387                         ret = -EAGAIN;
7388                         goto out;
7389                 }
7390         }
7391 
7392         btrfs_release_path(path);
7393 
7394         /*
7395          * look for other files referencing this extent, if we
7396          * find any we must cow
7397          */
7398 
7399         ret = btrfs_cross_ref_exist(root, btrfs_ino(BTRFS_I(inode)),
7400                                     key.offset - backref_offset, disk_bytenr);
7401         if (ret) {
7402                 ret = 0;
7403                 goto out;
7404         }
7405 
7406         /*
7407          * adjust disk_bytenr and num_bytes to cover just the bytes
7408          * in this extent we are about to write.  If there
7409          * are any csums in that range we have to cow in order
7410          * to keep the csums correct
7411          */
7412         disk_bytenr += backref_offset;
7413         disk_bytenr += offset - key.offset;
7414         if (csum_exist_in_range(fs_info, disk_bytenr, num_bytes))
7415                 goto out;
7416         /*
7417          * all of the above have passed, it is safe to overwrite this extent
7418          * without cow
7419          */
7420         *len = num_bytes;
7421         ret = 1;
7422 out:
7423         btrfs_free_path(path);
7424         return ret;
7425 }
7426 
7427 static int lock_extent_direct(struct inode *inode, u64 lockstart, u64 lockend,
7428                               struct extent_state **cached_state, int writing)
7429 {
7430         struct btrfs_ordered_extent *ordered;
7431         int ret = 0;
7432 
7433         while (1) {
7434                 lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend,
7435                                  cached_state);
7436                 /*
7437                  * We're concerned with the entire range that we're going to be
7438                  * doing DIO to, so we need to make sure there's no ordered
7439                  * extents in this range.
7440                  */
7441                 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), lockstart,
7442                                                      lockend - lockstart + 1);
7443 
7444                 /*
7445                  * We need to make sure there are no buffered pages in this
7446                  * range either, we could have raced between the invalidate in
7447                  * generic_file_direct_write and locking the extent.  The
7448                  * invalidate needs to happen so that reads after a write do not
7449                  * get stale data.
7450                  */
7451                 if (!ordered &&
7452                     (!writing || !filemap_range_has_page(inode->i_mapping,
7453                                                          lockstart, lockend)))
7454                         break;
7455 
7456                 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend,
7457                                      cached_state);
7458 
7459                 if (ordered) {
7460                         /*
7461                          * If we are doing a DIO read and the ordered extent we
7462                          * found is for a buffered write, we can not wait for it
7463                          * to complete and retry, because if we do so we can
7464                          * deadlock with concurrent buffered writes on page
7465                          * locks. This happens only if our DIO read covers more
7466                          * than one extent map, if at this point has already
7467                          * created an ordered extent for a previous extent map
7468                          * and locked its range in the inode's io tree, and a