~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/fs/splice.c

Version: ~ [ linux-5.16-rc3 ] ~ [ linux-5.15.5 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.82 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.162 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.218 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.256 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.291 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.293 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * "splice": joining two ropes together by interweaving their strands.
  3  *
  4  * This is the "extended pipe" functionality, where a pipe is used as
  5  * an arbitrary in-memory buffer. Think of a pipe as a small kernel
  6  * buffer that you can use to transfer data from one end to the other.
  7  *
  8  * The traditional unix read/write is extended with a "splice()" operation
  9  * that transfers data buffers to or from a pipe buffer.
 10  *
 11  * Named by Larry McVoy, original implementation from Linus, extended by
 12  * Jens to support splicing to files, network, direct splicing, etc and
 13  * fixing lots of bugs.
 14  *
 15  * Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk>
 16  * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org>
 17  * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu>
 18  *
 19  */
 20 #include <linux/fs.h>
 21 #include <linux/file.h>
 22 #include <linux/pagemap.h>
 23 #include <linux/splice.h>
 24 #include <linux/memcontrol.h>
 25 #include <linux/mm_inline.h>
 26 #include <linux/swap.h>
 27 #include <linux/writeback.h>
 28 #include <linux/export.h>
 29 #include <linux/syscalls.h>
 30 #include <linux/uio.h>
 31 #include <linux/security.h>
 32 #include <linux/gfp.h>
 33 #include <linux/socket.h>
 34 #include <linux/compat.h>
 35 #include "internal.h"
 36 
 37 /*
 38  * Attempt to steal a page from a pipe buffer. This should perhaps go into
 39  * a vm helper function, it's already simplified quite a bit by the
 40  * addition of remove_mapping(). If success is returned, the caller may
 41  * attempt to reuse this page for another destination.
 42  */
 43 static int page_cache_pipe_buf_steal(struct pipe_inode_info *pipe,
 44                                      struct pipe_buffer *buf)
 45 {
 46         struct page *page = buf->page;
 47         struct address_space *mapping;
 48 
 49         lock_page(page);
 50 
 51         mapping = page_mapping(page);
 52         if (mapping) {
 53                 WARN_ON(!PageUptodate(page));
 54 
 55                 /*
 56                  * At least for ext2 with nobh option, we need to wait on
 57                  * writeback completing on this page, since we'll remove it
 58                  * from the pagecache.  Otherwise truncate wont wait on the
 59                  * page, allowing the disk blocks to be reused by someone else
 60                  * before we actually wrote our data to them. fs corruption
 61                  * ensues.
 62                  */
 63                 wait_on_page_writeback(page);
 64 
 65                 if (page_has_private(page) &&
 66                     !try_to_release_page(page, GFP_KERNEL))
 67                         goto out_unlock;
 68 
 69                 /*
 70                  * If we succeeded in removing the mapping, set LRU flag
 71                  * and return good.
 72                  */
 73                 if (remove_mapping(mapping, page)) {
 74                         buf->flags |= PIPE_BUF_FLAG_LRU;
 75                         return 0;
 76                 }
 77         }
 78 
 79         /*
 80          * Raced with truncate or failed to remove page from current
 81          * address space, unlock and return failure.
 82          */
 83 out_unlock:
 84         unlock_page(page);
 85         return 1;
 86 }
 87 
 88 static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe,
 89                                         struct pipe_buffer *buf)
 90 {
 91         page_cache_release(buf->page);
 92         buf->flags &= ~PIPE_BUF_FLAG_LRU;
 93 }
 94 
 95 /*
 96  * Check whether the contents of buf is OK to access. Since the content
 97  * is a page cache page, IO may be in flight.
 98  */
 99 static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe,
100                                        struct pipe_buffer *buf)
101 {
102         struct page *page = buf->page;
103         int err;
104 
105         if (!PageUptodate(page)) {
106                 lock_page(page);
107 
108                 /*
109                  * Page got truncated/unhashed. This will cause a 0-byte
110                  * splice, if this is the first page.
111                  */
112                 if (!page->mapping) {
113                         err = -ENODATA;
114                         goto error;
115                 }
116 
117                 /*
118                  * Uh oh, read-error from disk.
119                  */
120                 if (!PageUptodate(page)) {
121                         err = -EIO;
122                         goto error;
123                 }
124 
125                 /*
126                  * Page is ok afterall, we are done.
127                  */
128                 unlock_page(page);
129         }
130 
131         return 0;
132 error:
133         unlock_page(page);
134         return err;
135 }
136 
137 const struct pipe_buf_operations page_cache_pipe_buf_ops = {
138         .can_merge = 0,
139         .map = generic_pipe_buf_map,
140         .unmap = generic_pipe_buf_unmap,
141         .confirm = page_cache_pipe_buf_confirm,
142         .release = page_cache_pipe_buf_release,
143         .steal = page_cache_pipe_buf_steal,
144         .get = generic_pipe_buf_get,
145 };
146 
147 static int user_page_pipe_buf_steal(struct pipe_inode_info *pipe,
148                                     struct pipe_buffer *buf)
149 {
150         if (!(buf->flags & PIPE_BUF_FLAG_GIFT))
151                 return 1;
152 
153         buf->flags |= PIPE_BUF_FLAG_LRU;
154         return generic_pipe_buf_steal(pipe, buf);
155 }
156 
157 static const struct pipe_buf_operations user_page_pipe_buf_ops = {
158         .can_merge = 0,
159         .map = generic_pipe_buf_map,
160         .unmap = generic_pipe_buf_unmap,
161         .confirm = generic_pipe_buf_confirm,
162         .release = page_cache_pipe_buf_release,
163         .steal = user_page_pipe_buf_steal,
164         .get = generic_pipe_buf_get,
165 };
166 
167 static void wakeup_pipe_readers(struct pipe_inode_info *pipe)
168 {
169         smp_mb();
170         if (waitqueue_active(&pipe->wait))
171                 wake_up_interruptible(&pipe->wait);
172         kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
173 }
174 
175 /**
176  * splice_to_pipe - fill passed data into a pipe
177  * @pipe:       pipe to fill
178  * @spd:        data to fill
179  *
180  * Description:
181  *    @spd contains a map of pages and len/offset tuples, along with
182  *    the struct pipe_buf_operations associated with these pages. This
183  *    function will link that data to the pipe.
184  *
185  */
186 ssize_t splice_to_pipe(struct pipe_inode_info *pipe,
187                        struct splice_pipe_desc *spd)
188 {
189         unsigned int spd_pages = spd->nr_pages;
190         int ret, do_wakeup, page_nr;
191 
192         if (!spd_pages)
193                 return 0;
194 
195         ret = 0;
196         do_wakeup = 0;
197         page_nr = 0;
198 
199         pipe_lock(pipe);
200 
201         for (;;) {
202                 if (!pipe->readers) {
203                         send_sig(SIGPIPE, current, 0);
204                         if (!ret)
205                                 ret = -EPIPE;
206                         break;
207                 }
208 
209                 if (pipe->nrbufs < pipe->buffers) {
210                         int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);
211                         struct pipe_buffer *buf = pipe->bufs + newbuf;
212 
213                         buf->page = spd->pages[page_nr];
214                         buf->offset = spd->partial[page_nr].offset;
215                         buf->len = spd->partial[page_nr].len;
216                         buf->private = spd->partial[page_nr].private;
217                         buf->ops = spd->ops;
218                         if (spd->flags & SPLICE_F_GIFT)
219                                 buf->flags |= PIPE_BUF_FLAG_GIFT;
220 
221                         pipe->nrbufs++;
222                         page_nr++;
223                         ret += buf->len;
224 
225                         if (pipe->files)
226                                 do_wakeup = 1;
227 
228                         if (!--spd->nr_pages)
229                                 break;
230                         if (pipe->nrbufs < pipe->buffers)
231                                 continue;
232 
233                         break;
234                 }
235 
236                 if (spd->flags & SPLICE_F_NONBLOCK) {
237                         if (!ret)
238                                 ret = -EAGAIN;
239                         break;
240                 }
241 
242                 if (signal_pending(current)) {
243                         if (!ret)
244                                 ret = -ERESTARTSYS;
245                         break;
246                 }
247 
248                 if (do_wakeup) {
249                         smp_mb();
250                         if (waitqueue_active(&pipe->wait))
251                                 wake_up_interruptible_sync(&pipe->wait);
252                         kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
253                         do_wakeup = 0;
254                 }
255 
256                 pipe->waiting_writers++;
257                 pipe_wait(pipe);
258                 pipe->waiting_writers--;
259         }
260 
261         pipe_unlock(pipe);
262 
263         if (do_wakeup)
264                 wakeup_pipe_readers(pipe);
265 
266         while (page_nr < spd_pages)
267                 spd->spd_release(spd, page_nr++);
268 
269         return ret;
270 }
271 
272 void spd_release_page(struct splice_pipe_desc *spd, unsigned int i)
273 {
274         page_cache_release(spd->pages[i]);
275 }
276 
277 /*
278  * Check if we need to grow the arrays holding pages and partial page
279  * descriptions.
280  */
281 int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd)
282 {
283         unsigned int buffers = ACCESS_ONCE(pipe->buffers);
284 
285         spd->nr_pages_max = buffers;
286         if (buffers <= PIPE_DEF_BUFFERS)
287                 return 0;
288 
289         spd->pages = kmalloc(buffers * sizeof(struct page *), GFP_KERNEL);
290         spd->partial = kmalloc(buffers * sizeof(struct partial_page), GFP_KERNEL);
291 
292         if (spd->pages && spd->partial)
293                 return 0;
294 
295         kfree(spd->pages);
296         kfree(spd->partial);
297         return -ENOMEM;
298 }
299 
300 void splice_shrink_spd(struct splice_pipe_desc *spd)
301 {
302         if (spd->nr_pages_max <= PIPE_DEF_BUFFERS)
303                 return;
304 
305         kfree(spd->pages);
306         kfree(spd->partial);
307 }
308 
309 static int
310 __generic_file_splice_read(struct file *in, loff_t *ppos,
311                            struct pipe_inode_info *pipe, size_t len,
312                            unsigned int flags)
313 {
314         struct address_space *mapping = in->f_mapping;
315         unsigned int loff, nr_pages, req_pages;
316         struct page *pages[PIPE_DEF_BUFFERS];
317         struct partial_page partial[PIPE_DEF_BUFFERS];
318         struct page *page;
319         pgoff_t index, end_index;
320         loff_t isize;
321         int error, page_nr;
322         struct splice_pipe_desc spd = {
323                 .pages = pages,
324                 .partial = partial,
325                 .nr_pages_max = PIPE_DEF_BUFFERS,
326                 .flags = flags,
327                 .ops = &page_cache_pipe_buf_ops,
328                 .spd_release = spd_release_page,
329         };
330 
331         if (splice_grow_spd(pipe, &spd))
332                 return -ENOMEM;
333 
334         index = *ppos >> PAGE_CACHE_SHIFT;
335         loff = *ppos & ~PAGE_CACHE_MASK;
336         req_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
337         nr_pages = min(req_pages, spd.nr_pages_max);
338 
339         /*
340          * Lookup the (hopefully) full range of pages we need.
341          */
342         spd.nr_pages = find_get_pages_contig(mapping, index, nr_pages, spd.pages);
343         index += spd.nr_pages;
344 
345         /*
346          * If find_get_pages_contig() returned fewer pages than we needed,
347          * readahead/allocate the rest and fill in the holes.
348          */
349         if (spd.nr_pages < nr_pages)
350                 page_cache_sync_readahead(mapping, &in->f_ra, in,
351                                 index, req_pages - spd.nr_pages);
352 
353         error = 0;
354         while (spd.nr_pages < nr_pages) {
355                 /*
356                  * Page could be there, find_get_pages_contig() breaks on
357                  * the first hole.
358                  */
359                 page = find_get_page(mapping, index);
360                 if (!page) {
361                         /*
362                          * page didn't exist, allocate one.
363                          */
364                         page = page_cache_alloc_cold(mapping);
365                         if (!page)
366                                 break;
367 
368                         error = add_to_page_cache_lru(page, mapping, index,
369                                                 GFP_KERNEL);
370                         if (unlikely(error)) {
371                                 page_cache_release(page);
372                                 if (error == -EEXIST)
373                                         continue;
374                                 break;
375                         }
376                         /*
377                          * add_to_page_cache() locks the page, unlock it
378                          * to avoid convoluting the logic below even more.
379                          */
380                         unlock_page(page);
381                 }
382 
383                 spd.pages[spd.nr_pages++] = page;
384                 index++;
385         }
386 
387         /*
388          * Now loop over the map and see if we need to start IO on any
389          * pages, fill in the partial map, etc.
390          */
391         index = *ppos >> PAGE_CACHE_SHIFT;
392         nr_pages = spd.nr_pages;
393         spd.nr_pages = 0;
394         for (page_nr = 0; page_nr < nr_pages; page_nr++) {
395                 unsigned int this_len;
396 
397                 if (!len)
398                         break;
399 
400                 /*
401                  * this_len is the max we'll use from this page
402                  */
403                 this_len = min_t(unsigned long, len, PAGE_CACHE_SIZE - loff);
404                 page = spd.pages[page_nr];
405 
406                 if (PageReadahead(page))
407                         page_cache_async_readahead(mapping, &in->f_ra, in,
408                                         page, index, req_pages - page_nr);
409 
410                 /*
411                  * If the page isn't uptodate, we may need to start io on it
412                  */
413                 if (!PageUptodate(page)) {
414                         lock_page(page);
415 
416                         /*
417                          * Page was truncated, or invalidated by the
418                          * filesystem.  Redo the find/create, but this time the
419                          * page is kept locked, so there's no chance of another
420                          * race with truncate/invalidate.
421                          */
422                         if (!page->mapping) {
423                                 unlock_page(page);
424                                 page = find_or_create_page(mapping, index,
425                                                 mapping_gfp_mask(mapping));
426 
427                                 if (!page) {
428                                         error = -ENOMEM;
429                                         break;
430                                 }
431                                 page_cache_release(spd.pages[page_nr]);
432                                 spd.pages[page_nr] = page;
433                         }
434                         /*
435                          * page was already under io and is now done, great
436                          */
437                         if (PageUptodate(page)) {
438                                 unlock_page(page);
439                                 goto fill_it;
440                         }
441 
442                         /*
443                          * need to read in the page
444                          */
445                         error = mapping->a_ops->readpage(in, page);
446                         if (unlikely(error)) {
447                                 /*
448                                  * We really should re-lookup the page here,
449                                  * but it complicates things a lot. Instead
450                                  * lets just do what we already stored, and
451                                  * we'll get it the next time we are called.
452                                  */
453                                 if (error == AOP_TRUNCATED_PAGE)
454                                         error = 0;
455 
456                                 break;
457                         }
458                 }
459 fill_it:
460                 /*
461                  * i_size must be checked after PageUptodate.
462                  */
463                 isize = i_size_read(mapping->host);
464                 end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
465                 if (unlikely(!isize || index > end_index))
466                         break;
467 
468                 /*
469                  * if this is the last page, see if we need to shrink
470                  * the length and stop
471                  */
472                 if (end_index == index) {
473                         unsigned int plen;
474 
475                         /*
476                          * max good bytes in this page
477                          */
478                         plen = ((isize - 1) & ~PAGE_CACHE_MASK) + 1;
479                         if (plen <= loff)
480                                 break;
481 
482                         /*
483                          * force quit after adding this page
484                          */
485                         this_len = min(this_len, plen - loff);
486                         len = this_len;
487                 }
488 
489                 spd.partial[page_nr].offset = loff;
490                 spd.partial[page_nr].len = this_len;
491                 len -= this_len;
492                 loff = 0;
493                 spd.nr_pages++;
494                 index++;
495         }
496 
497         /*
498          * Release any pages at the end, if we quit early. 'page_nr' is how far
499          * we got, 'nr_pages' is how many pages are in the map.
500          */
501         while (page_nr < nr_pages)
502                 page_cache_release(spd.pages[page_nr++]);
503         in->f_ra.prev_pos = (loff_t)index << PAGE_CACHE_SHIFT;
504 
505         if (spd.nr_pages)
506                 error = splice_to_pipe(pipe, &spd);
507 
508         splice_shrink_spd(&spd);
509         return error;
510 }
511 
512 /**
513  * generic_file_splice_read - splice data from file to a pipe
514  * @in:         file to splice from
515  * @ppos:       position in @in
516  * @pipe:       pipe to splice to
517  * @len:        number of bytes to splice
518  * @flags:      splice modifier flags
519  *
520  * Description:
521  *    Will read pages from given file and fill them into a pipe. Can be
522  *    used as long as the address_space operations for the source implements
523  *    a readpage() hook.
524  *
525  */
526 ssize_t generic_file_splice_read(struct file *in, loff_t *ppos,
527                                  struct pipe_inode_info *pipe, size_t len,
528                                  unsigned int flags)
529 {
530         loff_t isize, left;
531         int ret;
532 
533         isize = i_size_read(in->f_mapping->host);
534         if (unlikely(*ppos >= isize))
535                 return 0;
536 
537         left = isize - *ppos;
538         if (unlikely(left < len))
539                 len = left;
540 
541         ret = __generic_file_splice_read(in, ppos, pipe, len, flags);
542         if (ret > 0) {
543                 *ppos += ret;
544                 file_accessed(in);
545         }
546 
547         return ret;
548 }
549 EXPORT_SYMBOL(generic_file_splice_read);
550 
551 static const struct pipe_buf_operations default_pipe_buf_ops = {
552         .can_merge = 0,
553         .map = generic_pipe_buf_map,
554         .unmap = generic_pipe_buf_unmap,
555         .confirm = generic_pipe_buf_confirm,
556         .release = generic_pipe_buf_release,
557         .steal = generic_pipe_buf_steal,
558         .get = generic_pipe_buf_get,
559 };
560 
561 static int generic_pipe_buf_nosteal(struct pipe_inode_info *pipe,
562                                     struct pipe_buffer *buf)
563 {
564         return 1;
565 }
566 
567 /* Pipe buffer operations for a socket and similar. */
568 const struct pipe_buf_operations nosteal_pipe_buf_ops = {
569         .can_merge = 0,
570         .map = generic_pipe_buf_map,
571         .unmap = generic_pipe_buf_unmap,
572         .confirm = generic_pipe_buf_confirm,
573         .release = generic_pipe_buf_release,
574         .steal = generic_pipe_buf_nosteal,
575         .get = generic_pipe_buf_get,
576 };
577 EXPORT_SYMBOL(nosteal_pipe_buf_ops);
578 
579 static ssize_t kernel_readv(struct file *file, const struct iovec *vec,
580                             unsigned long vlen, loff_t offset)
581 {
582         mm_segment_t old_fs;
583         loff_t pos = offset;
584         ssize_t res;
585 
586         old_fs = get_fs();
587         set_fs(get_ds());
588         /* The cast to a user pointer is valid due to the set_fs() */
589         res = vfs_readv(file, (const struct iovec __user *)vec, vlen, &pos);
590         set_fs(old_fs);
591 
592         return res;
593 }
594 
595 ssize_t kernel_write(struct file *file, const char *buf, size_t count,
596                             loff_t pos)
597 {
598         mm_segment_t old_fs;
599         ssize_t res;
600 
601         old_fs = get_fs();
602         set_fs(get_ds());
603         /* The cast to a user pointer is valid due to the set_fs() */
604         res = vfs_write(file, (__force const char __user *)buf, count, &pos);
605         set_fs(old_fs);
606 
607         return res;
608 }
609 EXPORT_SYMBOL(kernel_write);
610 
611 ssize_t default_file_splice_read(struct file *in, loff_t *ppos,
612                                  struct pipe_inode_info *pipe, size_t len,
613                                  unsigned int flags)
614 {
615         unsigned int nr_pages;
616         unsigned int nr_freed;
617         size_t offset;
618         struct page *pages[PIPE_DEF_BUFFERS];
619         struct partial_page partial[PIPE_DEF_BUFFERS];
620         struct iovec *vec, __vec[PIPE_DEF_BUFFERS];
621         ssize_t res;
622         size_t this_len;
623         int error;
624         int i;
625         struct splice_pipe_desc spd = {
626                 .pages = pages,
627                 .partial = partial,
628                 .nr_pages_max = PIPE_DEF_BUFFERS,
629                 .flags = flags,
630                 .ops = &default_pipe_buf_ops,
631                 .spd_release = spd_release_page,
632         };
633 
634         if (splice_grow_spd(pipe, &spd))
635                 return -ENOMEM;
636 
637         res = -ENOMEM;
638         vec = __vec;
639         if (spd.nr_pages_max > PIPE_DEF_BUFFERS) {
640                 vec = kmalloc(spd.nr_pages_max * sizeof(struct iovec), GFP_KERNEL);
641                 if (!vec)
642                         goto shrink_ret;
643         }
644 
645         offset = *ppos & ~PAGE_CACHE_MASK;
646         nr_pages = (len + offset + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
647 
648         for (i = 0; i < nr_pages && i < spd.nr_pages_max && len; i++) {
649                 struct page *page;
650 
651                 page = alloc_page(GFP_USER);
652                 error = -ENOMEM;
653                 if (!page)
654                         goto err;
655 
656                 this_len = min_t(size_t, len, PAGE_CACHE_SIZE - offset);
657                 vec[i].iov_base = (void __user *) page_address(page);
658                 vec[i].iov_len = this_len;
659                 spd.pages[i] = page;
660                 spd.nr_pages++;
661                 len -= this_len;
662                 offset = 0;
663         }
664 
665         res = kernel_readv(in, vec, spd.nr_pages, *ppos);
666         if (res < 0) {
667                 error = res;
668                 goto err;
669         }
670 
671         error = 0;
672         if (!res)
673                 goto err;
674 
675         nr_freed = 0;
676         for (i = 0; i < spd.nr_pages; i++) {
677                 this_len = min_t(size_t, vec[i].iov_len, res);
678                 spd.partial[i].offset = 0;
679                 spd.partial[i].len = this_len;
680                 if (!this_len) {
681                         __free_page(spd.pages[i]);
682                         spd.pages[i] = NULL;
683                         nr_freed++;
684                 }
685                 res -= this_len;
686         }
687         spd.nr_pages -= nr_freed;
688 
689         res = splice_to_pipe(pipe, &spd);
690         if (res > 0)
691                 *ppos += res;
692 
693 shrink_ret:
694         if (vec != __vec)
695                 kfree(vec);
696         splice_shrink_spd(&spd);
697         return res;
698 
699 err:
700         for (i = 0; i < spd.nr_pages; i++)
701                 __free_page(spd.pages[i]);
702 
703         res = error;
704         goto shrink_ret;
705 }
706 EXPORT_SYMBOL(default_file_splice_read);
707 
708 /*
709  * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos'
710  * using sendpage(). Return the number of bytes sent.
711  */
712 static int pipe_to_sendpage(struct pipe_inode_info *pipe,
713                             struct pipe_buffer *buf, struct splice_desc *sd)
714 {
715         struct file *file = sd->u.file;
716         loff_t pos = sd->pos;
717         int more;
718 
719         if (!likely(file->f_op->sendpage))
720                 return -EINVAL;
721 
722         more = (sd->flags & SPLICE_F_MORE) ? MSG_MORE : 0;
723 
724         if (sd->len < sd->total_len && pipe->nrbufs > 1)
725                 more |= MSG_SENDPAGE_NOTLAST;
726 
727         return file->f_op->sendpage(file, buf->page, buf->offset,
728                                     sd->len, &pos, more);
729 }
730 
731 /*
732  * This is a little more tricky than the file -> pipe splicing. There are
733  * basically three cases:
734  *
735  *      - Destination page already exists in the address space and there
736  *        are users of it. For that case we have no other option that
737  *        copying the data. Tough luck.
738  *      - Destination page already exists in the address space, but there
739  *        are no users of it. Make sure it's uptodate, then drop it. Fall
740  *        through to last case.
741  *      - Destination page does not exist, we can add the pipe page to
742  *        the page cache and avoid the copy.
743  *
744  * If asked to move pages to the output file (SPLICE_F_MOVE is set in
745  * sd->flags), we attempt to migrate pages from the pipe to the output
746  * file address space page cache. This is possible if no one else has
747  * the pipe page referenced outside of the pipe and page cache. If
748  * SPLICE_F_MOVE isn't set, or we cannot move the page, we simply create
749  * a new page in the output file page cache and fill/dirty that.
750  */
751 int pipe_to_file(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
752                  struct splice_desc *sd)
753 {
754         struct file *file = sd->u.file;
755         struct address_space *mapping = file->f_mapping;
756         unsigned int offset, this_len;
757         struct page *page;
758         void *fsdata;
759         int ret;
760 
761         offset = sd->pos & ~PAGE_CACHE_MASK;
762 
763         this_len = sd->len;
764         if (this_len + offset > PAGE_CACHE_SIZE)
765                 this_len = PAGE_CACHE_SIZE - offset;
766 
767         ret = pagecache_write_begin(file, mapping, sd->pos, this_len,
768                                 AOP_FLAG_UNINTERRUPTIBLE, &page, &fsdata);
769         if (unlikely(ret))
770                 goto out;
771 
772         if (buf->page != page) {
773                 char *src = buf->ops->map(pipe, buf, 1);
774                 char *dst = kmap_atomic(page);
775 
776                 memcpy(dst + offset, src + buf->offset, this_len);
777                 flush_dcache_page(page);
778                 kunmap_atomic(dst);
779                 buf->ops->unmap(pipe, buf, src);
780         }
781         ret = pagecache_write_end(file, mapping, sd->pos, this_len, this_len,
782                                 page, fsdata);
783 out:
784         return ret;
785 }
786 EXPORT_SYMBOL(pipe_to_file);
787 
788 static void wakeup_pipe_writers(struct pipe_inode_info *pipe)
789 {
790         smp_mb();
791         if (waitqueue_active(&pipe->wait))
792                 wake_up_interruptible(&pipe->wait);
793         kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
794 }
795 
796 /**
797  * splice_from_pipe_feed - feed available data from a pipe to a file
798  * @pipe:       pipe to splice from
799  * @sd:         information to @actor
800  * @actor:      handler that splices the data
801  *
802  * Description:
803  *    This function loops over the pipe and calls @actor to do the
804  *    actual moving of a single struct pipe_buffer to the desired
805  *    destination.  It returns when there's no more buffers left in
806  *    the pipe or if the requested number of bytes (@sd->total_len)
807  *    have been copied.  It returns a positive number (one) if the
808  *    pipe needs to be filled with more data, zero if the required
809  *    number of bytes have been copied and -errno on error.
810  *
811  *    This, together with splice_from_pipe_{begin,end,next}, may be
812  *    used to implement the functionality of __splice_from_pipe() when
813  *    locking is required around copying the pipe buffers to the
814  *    destination.
815  */
816 int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd,
817                           splice_actor *actor)
818 {
819         int ret;
820 
821         while (pipe->nrbufs) {
822                 struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;
823                 const struct pipe_buf_operations *ops = buf->ops;
824 
825                 sd->len = buf->len;
826                 if (sd->len > sd->total_len)
827                         sd->len = sd->total_len;
828 
829                 ret = buf->ops->confirm(pipe, buf);
830                 if (unlikely(ret)) {
831                         if (ret == -ENODATA)
832                                 ret = 0;
833                         return ret;
834                 }
835 
836                 ret = actor(pipe, buf, sd);
837                 if (ret <= 0)
838                         return ret;
839 
840                 buf->offset += ret;
841                 buf->len -= ret;
842 
843                 sd->num_spliced += ret;
844                 sd->len -= ret;
845                 sd->pos += ret;
846                 sd->total_len -= ret;
847 
848                 if (!buf->len) {
849                         buf->ops = NULL;
850                         ops->release(pipe, buf);
851                         pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);
852                         pipe->nrbufs--;
853                         if (pipe->files)
854                                 sd->need_wakeup = true;
855                 }
856 
857                 if (!sd->total_len)
858                         return 0;
859         }
860 
861         return 1;
862 }
863 EXPORT_SYMBOL(splice_from_pipe_feed);
864 
865 /**
866  * splice_from_pipe_next - wait for some data to splice from
867  * @pipe:       pipe to splice from
868  * @sd:         information about the splice operation
869  *
870  * Description:
871  *    This function will wait for some data and return a positive
872  *    value (one) if pipe buffers are available.  It will return zero
873  *    or -errno if no more data needs to be spliced.
874  */
875 int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd)
876 {
877         while (!pipe->nrbufs) {
878                 if (!pipe->writers)
879                         return 0;
880 
881                 if (!pipe->waiting_writers && sd->num_spliced)
882                         return 0;
883 
884                 if (sd->flags & SPLICE_F_NONBLOCK)
885                         return -EAGAIN;
886 
887                 if (signal_pending(current))
888                         return -ERESTARTSYS;
889 
890                 if (sd->need_wakeup) {
891                         wakeup_pipe_writers(pipe);
892                         sd->need_wakeup = false;
893                 }
894 
895                 pipe_wait(pipe);
896         }
897 
898         return 1;
899 }
900 EXPORT_SYMBOL(splice_from_pipe_next);
901 
902 /**
903  * splice_from_pipe_begin - start splicing from pipe
904  * @sd:         information about the splice operation
905  *
906  * Description:
907  *    This function should be called before a loop containing
908  *    splice_from_pipe_next() and splice_from_pipe_feed() to
909  *    initialize the necessary fields of @sd.
910  */
911 void splice_from_pipe_begin(struct splice_desc *sd)
912 {
913         sd->num_spliced = 0;
914         sd->need_wakeup = false;
915 }
916 EXPORT_SYMBOL(splice_from_pipe_begin);
917 
918 /**
919  * splice_from_pipe_end - finish splicing from pipe
920  * @pipe:       pipe to splice from
921  * @sd:         information about the splice operation
922  *
923  * Description:
924  *    This function will wake up pipe writers if necessary.  It should
925  *    be called after a loop containing splice_from_pipe_next() and
926  *    splice_from_pipe_feed().
927  */
928 void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd)
929 {
930         if (sd->need_wakeup)
931                 wakeup_pipe_writers(pipe);
932 }
933 EXPORT_SYMBOL(splice_from_pipe_end);
934 
935 /**
936  * __splice_from_pipe - splice data from a pipe to given actor
937  * @pipe:       pipe to splice from
938  * @sd:         information to @actor
939  * @actor:      handler that splices the data
940  *
941  * Description:
942  *    This function does little more than loop over the pipe and call
943  *    @actor to do the actual moving of a single struct pipe_buffer to
944  *    the desired destination. See pipe_to_file, pipe_to_sendpage, or
945  *    pipe_to_user.
946  *
947  */
948 ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd,
949                            splice_actor *actor)
950 {
951         int ret;
952 
953         splice_from_pipe_begin(sd);
954         do {
955                 cond_resched();
956                 ret = splice_from_pipe_next(pipe, sd);
957                 if (ret > 0)
958                         ret = splice_from_pipe_feed(pipe, sd, actor);
959         } while (ret > 0);
960         splice_from_pipe_end(pipe, sd);
961 
962         return sd->num_spliced ? sd->num_spliced : ret;
963 }
964 EXPORT_SYMBOL(__splice_from_pipe);
965 
966 /**
967  * splice_from_pipe - splice data from a pipe to a file
968  * @pipe:       pipe to splice from
969  * @out:        file to splice to
970  * @ppos:       position in @out
971  * @len:        how many bytes to splice
972  * @flags:      splice modifier flags
973  * @actor:      handler that splices the data
974  *
975  * Description:
976  *    See __splice_from_pipe. This function locks the pipe inode,
977  *    otherwise it's identical to __splice_from_pipe().
978  *
979  */
980 ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,
981                          loff_t *ppos, size_t len, unsigned int flags,
982                          splice_actor *actor)
983 {
984         ssize_t ret;
985         struct splice_desc sd = {
986                 .total_len = len,
987                 .flags = flags,
988                 .pos = *ppos,
989                 .u.file = out,
990         };
991 
992         pipe_lock(pipe);
993         ret = __splice_from_pipe(pipe, &sd, actor);
994         pipe_unlock(pipe);
995 
996         return ret;
997 }
998 
999 /**
1000  * generic_file_splice_write - splice data from a pipe to a file
1001  * @pipe:       pipe info
1002  * @out:        file to write to
1003  * @ppos:       position in @out
1004  * @len:        number of bytes to splice
1005  * @flags:      splice modifier flags
1006  *
1007  * Description:
1008  *    Will either move or copy pages (determined by @flags options) from
1009  *    the given pipe inode to the given file.
1010  *
1011  */
1012 ssize_t
1013 generic_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
1014                           loff_t *ppos, size_t len, unsigned int flags)
1015 {
1016         struct address_space *mapping = out->f_mapping;
1017         struct inode *inode = mapping->host;
1018         struct splice_desc sd = {
1019                 .flags = flags,
1020                 .u.file = out,
1021         };
1022         ssize_t ret;
1023 
1024         ret = generic_write_checks(out, ppos, &len, S_ISBLK(inode->i_mode));
1025         if (ret)
1026                 return ret;
1027         sd.total_len = len;
1028         sd.pos = *ppos;
1029 
1030         pipe_lock(pipe);
1031 
1032         splice_from_pipe_begin(&sd);
1033         do {
1034                 ret = splice_from_pipe_next(pipe, &sd);
1035                 if (ret <= 0)
1036                         break;
1037 
1038                 mutex_lock_nested(&inode->i_mutex, I_MUTEX_CHILD);
1039                 ret = file_remove_suid(out);
1040                 if (!ret) {
1041                         ret = file_update_time(out);
1042                         if (!ret)
1043                                 ret = splice_from_pipe_feed(pipe, &sd,
1044                                                             pipe_to_file);
1045                 }
1046                 mutex_unlock(&inode->i_mutex);
1047         } while (ret > 0);
1048         splice_from_pipe_end(pipe, &sd);
1049 
1050         pipe_unlock(pipe);
1051 
1052         if (sd.num_spliced)
1053                 ret = sd.num_spliced;
1054 
1055         if (ret > 0) {
1056                 int err;
1057 
1058                 err = generic_write_sync(out, *ppos, ret);
1059                 if (err)
1060                         ret = err;
1061                 else
1062                         *ppos += ret;
1063                 balance_dirty_pages_ratelimited(mapping);
1064         }
1065 
1066         return ret;
1067 }
1068 
1069 EXPORT_SYMBOL(generic_file_splice_write);
1070 
1071 static int write_pipe_buf(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
1072                           struct splice_desc *sd)
1073 {
1074         int ret;
1075         void *data;
1076         loff_t tmp = sd->pos;
1077 
1078         data = buf->ops->map(pipe, buf, 0);
1079         ret = __kernel_write(sd->u.file, data + buf->offset, sd->len, &tmp);
1080         buf->ops->unmap(pipe, buf, data);
1081 
1082         return ret;
1083 }
1084 
1085 static ssize_t default_file_splice_write(struct pipe_inode_info *pipe,
1086                                          struct file *out, loff_t *ppos,
1087                                          size_t len, unsigned int flags)
1088 {
1089         ssize_t ret;
1090 
1091         ret = splice_from_pipe(pipe, out, ppos, len, flags, write_pipe_buf);
1092         if (ret > 0)
1093                 *ppos += ret;
1094 
1095         return ret;
1096 }
1097 
1098 /**
1099  * generic_splice_sendpage - splice data from a pipe to a socket
1100  * @pipe:       pipe to splice from
1101  * @out:        socket to write to
1102  * @ppos:       position in @out
1103  * @len:        number of bytes to splice
1104  * @flags:      splice modifier flags
1105  *
1106  * Description:
1107  *    Will send @len bytes from the pipe to a network socket. No data copying
1108  *    is involved.
1109  *
1110  */
1111 ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,
1112                                 loff_t *ppos, size_t len, unsigned int flags)
1113 {
1114         return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage);
1115 }
1116 
1117 EXPORT_SYMBOL(generic_splice_sendpage);
1118 
1119 /*
1120  * Attempt to initiate a splice from pipe to file.
1121  */
1122 static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
1123                            loff_t *ppos, size_t len, unsigned int flags)
1124 {
1125         ssize_t (*splice_write)(struct pipe_inode_info *, struct file *,
1126                                 loff_t *, size_t, unsigned int);
1127 
1128         if (out->f_op->splice_write)
1129                 splice_write = out->f_op->splice_write;
1130         else
1131                 splice_write = default_file_splice_write;
1132 
1133         return splice_write(pipe, out, ppos, len, flags);
1134 }
1135 
1136 /*
1137  * Attempt to initiate a splice from a file to a pipe.
1138  */
1139 static long do_splice_to(struct file *in, loff_t *ppos,
1140                          struct pipe_inode_info *pipe, size_t len,
1141                          unsigned int flags)
1142 {
1143         ssize_t (*splice_read)(struct file *, loff_t *,
1144                                struct pipe_inode_info *, size_t, unsigned int);
1145         int ret;
1146 
1147         if (unlikely(!(in->f_mode & FMODE_READ)))
1148                 return -EBADF;
1149 
1150         ret = rw_verify_area(READ, in, ppos, len);
1151         if (unlikely(ret < 0))
1152                 return ret;
1153 
1154         if (in->f_op->splice_read)
1155                 splice_read = in->f_op->splice_read;
1156         else
1157                 splice_read = default_file_splice_read;
1158 
1159         return splice_read(in, ppos, pipe, len, flags);
1160 }
1161 
1162 /**
1163  * splice_direct_to_actor - splices data directly between two non-pipes
1164  * @in:         file to splice from
1165  * @sd:         actor information on where to splice to
1166  * @actor:      handles the data splicing
1167  *
1168  * Description:
1169  *    This is a special case helper to splice directly between two
1170  *    points, without requiring an explicit pipe. Internally an allocated
1171  *    pipe is cached in the process, and reused during the lifetime of
1172  *    that process.
1173  *
1174  */
1175 ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd,
1176                                splice_direct_actor *actor)
1177 {
1178         struct pipe_inode_info *pipe;
1179         long ret, bytes;
1180         umode_t i_mode;
1181         size_t len;
1182         int i, flags, more;
1183 
1184         /*
1185          * We require the input being a regular file, as we don't want to
1186          * randomly drop data for eg socket -> socket splicing. Use the
1187          * piped splicing for that!
1188          */
1189         i_mode = file_inode(in)->i_mode;
1190         if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode)))
1191                 return -EINVAL;
1192 
1193         /*
1194          * neither in nor out is a pipe, setup an internal pipe attached to
1195          * 'out' and transfer the wanted data from 'in' to 'out' through that
1196          */
1197         pipe = current->splice_pipe;
1198         if (unlikely(!pipe)) {
1199                 pipe = alloc_pipe_info();
1200                 if (!pipe)
1201                         return -ENOMEM;
1202 
1203                 /*
1204                  * We don't have an immediate reader, but we'll read the stuff
1205                  * out of the pipe right after the splice_to_pipe(). So set
1206                  * PIPE_READERS appropriately.
1207                  */
1208                 pipe->readers = 1;
1209 
1210                 current->splice_pipe = pipe;
1211         }
1212 
1213         /*
1214          * Do the splice.
1215          */
1216         ret = 0;
1217         bytes = 0;
1218         len = sd->total_len;
1219         flags = sd->flags;
1220 
1221         /*
1222          * Don't block on output, we have to drain the direct pipe.
1223          */
1224         sd->flags &= ~SPLICE_F_NONBLOCK;
1225         more = sd->flags & SPLICE_F_MORE;
1226 
1227         while (len) {
1228                 size_t read_len;
1229                 loff_t pos = sd->pos, prev_pos = pos;
1230 
1231                 ret = do_splice_to(in, &pos, pipe, len, flags);
1232                 if (unlikely(ret <= 0))
1233                         goto out_release;
1234 
1235                 read_len = ret;
1236                 sd->total_len = read_len;
1237 
1238                 /*
1239                  * If more data is pending, set SPLICE_F_MORE
1240                  * If this is the last data and SPLICE_F_MORE was not set
1241                  * initially, clears it.
1242                  */
1243                 if (read_len < len)
1244                         sd->flags |= SPLICE_F_MORE;
1245                 else if (!more)
1246                         sd->flags &= ~SPLICE_F_MORE;
1247                 /*
1248                  * NOTE: nonblocking mode only applies to the input. We
1249                  * must not do the output in nonblocking mode as then we
1250                  * could get stuck data in the internal pipe:
1251                  */
1252                 ret = actor(pipe, sd);
1253                 if (unlikely(ret <= 0)) {
1254                         sd->pos = prev_pos;
1255                         goto out_release;
1256                 }
1257 
1258                 bytes += ret;
1259                 len -= ret;
1260                 sd->pos = pos;
1261 
1262                 if (ret < read_len) {
1263                         sd->pos = prev_pos + ret;
1264                         goto out_release;
1265                 }
1266         }
1267 
1268 done:
1269         pipe->nrbufs = pipe->curbuf = 0;
1270         file_accessed(in);
1271         return bytes;
1272 
1273 out_release:
1274         /*
1275          * If we did an incomplete transfer we must release
1276          * the pipe buffers in question:
1277          */
1278         for (i = 0; i < pipe->buffers; i++) {
1279                 struct pipe_buffer *buf = pipe->bufs + i;
1280 
1281                 if (buf->ops) {
1282                         buf->ops->release(pipe, buf);
1283                         buf->ops = NULL;
1284                 }
1285         }
1286 
1287         if (!bytes)
1288                 bytes = ret;
1289 
1290         goto done;
1291 }
1292 EXPORT_SYMBOL(splice_direct_to_actor);
1293 
1294 static int direct_splice_actor(struct pipe_inode_info *pipe,
1295                                struct splice_desc *sd)
1296 {
1297         struct file *file = sd->u.file;
1298 
1299         return do_splice_from(pipe, file, sd->opos, sd->total_len,
1300                               sd->flags);
1301 }
1302 
1303 /**
1304  * do_splice_direct - splices data directly between two files
1305  * @in:         file to splice from
1306  * @ppos:       input file offset
1307  * @out:        file to splice to
1308  * @opos:       output file offset
1309  * @len:        number of bytes to splice
1310  * @flags:      splice modifier flags
1311  *
1312  * Description:
1313  *    For use by do_sendfile(). splice can easily emulate sendfile, but
1314  *    doing it in the application would incur an extra system call
1315  *    (splice in + splice out, as compared to just sendfile()). So this helper
1316  *    can splice directly through a process-private pipe.
1317  *
1318  */
1319 long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
1320                       loff_t *opos, size_t len, unsigned int flags)
1321 {
1322         struct splice_desc sd = {
1323                 .len            = len,
1324                 .total_len      = len,
1325                 .flags          = flags,
1326                 .pos            = *ppos,
1327                 .u.file         = out,
1328                 .opos           = opos,
1329         };
1330         long ret;
1331 
1332         if (unlikely(!(out->f_mode & FMODE_WRITE)))
1333                 return -EBADF;
1334 
1335         if (unlikely(out->f_flags & O_APPEND))
1336                 return -EINVAL;
1337 
1338         ret = rw_verify_area(WRITE, out, opos, len);
1339         if (unlikely(ret < 0))
1340                 return ret;
1341 
1342         ret = splice_direct_to_actor(in, &sd, direct_splice_actor);
1343         if (ret > 0)
1344                 *ppos = sd.pos;
1345 
1346         return ret;
1347 }
1348 
1349 static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
1350                                struct pipe_inode_info *opipe,
1351                                size_t len, unsigned int flags);
1352 
1353 /*
1354  * Determine where to splice to/from.
1355  */
1356 static long do_splice(struct file *in, loff_t __user *off_in,
1357                       struct file *out, loff_t __user *off_out,
1358                       size_t len, unsigned int flags)
1359 {
1360         struct pipe_inode_info *ipipe;
1361         struct pipe_inode_info *opipe;
1362         loff_t offset;
1363         long ret;
1364 
1365         ipipe = get_pipe_info(in);
1366         opipe = get_pipe_info(out);
1367 
1368         if (ipipe && opipe) {
1369                 if (off_in || off_out)
1370                         return -ESPIPE;
1371 
1372                 if (!(in->f_mode & FMODE_READ))
1373                         return -EBADF;
1374 
1375                 if (!(out->f_mode & FMODE_WRITE))
1376                         return -EBADF;
1377 
1378                 /* Splicing to self would be fun, but... */
1379                 if (ipipe == opipe)
1380                         return -EINVAL;
1381 
1382                 return splice_pipe_to_pipe(ipipe, opipe, len, flags);
1383         }
1384 
1385         if (ipipe) {
1386                 if (off_in)
1387                         return -ESPIPE;
1388                 if (off_out) {
1389                         if (!(out->f_mode & FMODE_PWRITE))
1390                                 return -EINVAL;
1391                         if (copy_from_user(&offset, off_out, sizeof(loff_t)))
1392                                 return -EFAULT;
1393                 } else {
1394                         offset = out->f_pos;
1395                 }
1396 
1397                 if (unlikely(!(out->f_mode & FMODE_WRITE)))
1398                         return -EBADF;
1399 
1400                 if (unlikely(out->f_flags & O_APPEND))
1401                         return -EINVAL;
1402 
1403                 ret = rw_verify_area(WRITE, out, &offset, len);
1404                 if (unlikely(ret < 0))
1405                         return ret;
1406 
1407                 file_start_write(out);
1408                 ret = do_splice_from(ipipe, out, &offset, len, flags);
1409                 file_end_write(out);
1410 
1411                 if (!off_out)
1412                         out->f_pos = offset;
1413                 else if (copy_to_user(off_out, &offset, sizeof(loff_t)))
1414                         ret = -EFAULT;
1415 
1416                 return ret;
1417         }
1418 
1419         if (opipe) {
1420                 if (off_out)
1421                         return -ESPIPE;
1422                 if (off_in) {
1423                         if (!(in->f_mode & FMODE_PREAD))
1424                                 return -EINVAL;
1425                         if (copy_from_user(&offset, off_in, sizeof(loff_t)))
1426                                 return -EFAULT;
1427                 } else {
1428                         offset = in->f_pos;
1429                 }
1430 
1431                 ret = do_splice_to(in, &offset, opipe, len, flags);
1432 
1433                 if (!off_in)
1434                         in->f_pos = offset;
1435                 else if (copy_to_user(off_in, &offset, sizeof(loff_t)))
1436                         ret = -EFAULT;
1437 
1438                 return ret;
1439         }
1440 
1441         return -EINVAL;
1442 }
1443 
1444 /*
1445  * Map an iov into an array of pages and offset/length tupples. With the
1446  * partial_page structure, we can map several non-contiguous ranges into
1447  * our ones pages[] map instead of splitting that operation into pieces.
1448  * Could easily be exported as a generic helper for other users, in which
1449  * case one would probably want to add a 'max_nr_pages' parameter as well.
1450  */
1451 static int get_iovec_page_array(const struct iovec __user *iov,
1452                                 unsigned int nr_vecs, struct page **pages,
1453                                 struct partial_page *partial, bool aligned,
1454                                 unsigned int pipe_buffers)
1455 {
1456         int buffers = 0, error = 0;
1457 
1458         while (nr_vecs) {
1459                 unsigned long off, npages;
1460                 struct iovec entry;
1461                 void __user *base;
1462                 size_t len;
1463                 int i;
1464 
1465                 error = -EFAULT;
1466                 if (copy_from_user(&entry, iov, sizeof(entry)))
1467                         break;
1468 
1469                 base = entry.iov_base;
1470                 len = entry.iov_len;
1471 
1472                 /*
1473                  * Sanity check this iovec. 0 read succeeds.
1474                  */
1475                 error = 0;
1476                 if (unlikely(!len))
1477                         break;
1478                 error = -EFAULT;
1479                 if (!access_ok(VERIFY_READ, base, len))
1480                         break;
1481 
1482                 /*
1483                  * Get this base offset and number of pages, then map
1484                  * in the user pages.
1485                  */
1486                 off = (unsigned long) base & ~PAGE_MASK;
1487 
1488                 /*
1489                  * If asked for alignment, the offset must be zero and the
1490                  * length a multiple of the PAGE_SIZE.
1491                  */
1492                 error = -EINVAL;
1493                 if (aligned && (off || len & ~PAGE_MASK))
1494                         break;
1495 
1496                 npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1497                 if (npages > pipe_buffers - buffers)
1498                         npages = pipe_buffers - buffers;
1499 
1500                 error = get_user_pages_fast((unsigned long)base, npages,
1501                                         0, &pages[buffers]);
1502 
1503                 if (unlikely(error <= 0))
1504                         break;
1505 
1506                 /*
1507                  * Fill this contiguous range into the partial page map.
1508                  */
1509                 for (i = 0; i < error; i++) {
1510                         const int plen = min_t(size_t, len, PAGE_SIZE - off);
1511 
1512                         partial[buffers].offset = off;
1513                         partial[buffers].len = plen;
1514 
1515                         off = 0;
1516                         len -= plen;
1517                         buffers++;
1518                 }
1519 
1520                 /*
1521                  * We didn't complete this iov, stop here since it probably
1522                  * means we have to move some of this into a pipe to
1523                  * be able to continue.
1524                  */
1525                 if (len)
1526                         break;
1527 
1528                 /*
1529                  * Don't continue if we mapped fewer pages than we asked for,
1530                  * or if we mapped the max number of pages that we have
1531                  * room for.
1532                  */
1533                 if (error < npages || buffers == pipe_buffers)
1534                         break;
1535 
1536                 nr_vecs--;
1537                 iov++;
1538         }
1539 
1540         if (buffers)
1541                 return buffers;
1542 
1543         return error;
1544 }
1545 
1546 static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
1547                         struct splice_desc *sd)
1548 {
1549         char *src;
1550         int ret;
1551 
1552         /*
1553          * See if we can use the atomic maps, by prefaulting in the
1554          * pages and doing an atomic copy
1555          */
1556         if (!fault_in_pages_writeable(sd->u.userptr, sd->len)) {
1557                 src = buf->ops->map(pipe, buf, 1);
1558                 ret = __copy_to_user_inatomic(sd->u.userptr, src + buf->offset,
1559                                                         sd->len);
1560                 buf->ops->unmap(pipe, buf, src);
1561                 if (!ret) {
1562                         ret = sd->len;
1563                         goto out;
1564                 }
1565         }
1566 
1567         /*
1568          * No dice, use slow non-atomic map and copy
1569          */
1570         src = buf->ops->map(pipe, buf, 0);
1571 
1572         ret = sd->len;
1573         if (copy_to_user(sd->u.userptr, src + buf->offset, sd->len))
1574                 ret = -EFAULT;
1575 
1576         buf->ops->unmap(pipe, buf, src);
1577 out:
1578         if (ret > 0)
1579                 sd->u.userptr += ret;
1580         return ret;
1581 }
1582 
1583 /*
1584  * For lack of a better implementation, implement vmsplice() to userspace
1585  * as a simple copy of the pipes pages to the user iov.
1586  */
1587 static long vmsplice_to_user(struct file *file, const struct iovec __user *iov,
1588                              unsigned long nr_segs, unsigned int flags)
1589 {
1590         struct pipe_inode_info *pipe;
1591         struct splice_desc sd;
1592         ssize_t size;
1593         int error;
1594         long ret;
1595 
1596         pipe = get_pipe_info(file);
1597         if (!pipe)
1598                 return -EBADF;
1599 
1600         pipe_lock(pipe);
1601 
1602         error = ret = 0;
1603         while (nr_segs) {
1604                 void __user *base;
1605                 size_t len;
1606 
1607                 /*
1608                  * Get user address base and length for this iovec.
1609                  */
1610                 error = get_user(base, &iov->iov_base);
1611                 if (unlikely(error))
1612                         break;
1613                 error = get_user(len, &iov->iov_len);
1614                 if (unlikely(error))
1615                         break;
1616 
1617                 /*
1618                  * Sanity check this iovec. 0 read succeeds.
1619                  */
1620                 if (unlikely(!len))
1621                         break;
1622                 if (unlikely(!base)) {
1623                         error = -EFAULT;
1624                         break;
1625                 }
1626 
1627                 if (unlikely(!access_ok(VERIFY_WRITE, base, len))) {
1628                         error = -EFAULT;
1629                         break;
1630                 }
1631 
1632                 sd.len = 0;
1633                 sd.total_len = len;
1634                 sd.flags = flags;
1635                 sd.u.userptr = base;
1636                 sd.pos = 0;
1637 
1638                 size = __splice_from_pipe(pipe, &sd, pipe_to_user);
1639                 if (size < 0) {
1640                         if (!ret)
1641                                 ret = size;
1642 
1643                         break;
1644                 }
1645 
1646                 ret += size;
1647 
1648                 if (size < len)
1649                         break;
1650 
1651                 nr_segs--;
1652                 iov++;
1653         }
1654 
1655         pipe_unlock(pipe);
1656 
1657         if (!ret)
1658                 ret = error;
1659 
1660         return ret;
1661 }
1662 
1663 /*
1664  * vmsplice splices a user address range into a pipe. It can be thought of
1665  * as splice-from-memory, where the regular splice is splice-from-file (or
1666  * to file). In both cases the output is a pipe, naturally.
1667  */
1668 static long vmsplice_to_pipe(struct file *file, const struct iovec __user *iov,
1669                              unsigned long nr_segs, unsigned int flags)
1670 {
1671         struct pipe_inode_info *pipe;
1672         struct page *pages[PIPE_DEF_BUFFERS];
1673         struct partial_page partial[PIPE_DEF_BUFFERS];
1674         struct splice_pipe_desc spd = {
1675                 .pages = pages,
1676                 .partial = partial,
1677                 .nr_pages_max = PIPE_DEF_BUFFERS,
1678                 .flags = flags,
1679                 .ops = &user_page_pipe_buf_ops,
1680                 .spd_release = spd_release_page,
1681         };
1682         long ret;
1683 
1684         pipe = get_pipe_info(file);
1685         if (!pipe)
1686                 return -EBADF;
1687 
1688         if (splice_grow_spd(pipe, &spd))
1689                 return -ENOMEM;
1690 
1691         spd.nr_pages = get_iovec_page_array(iov, nr_segs, spd.pages,
1692                                             spd.partial, false,
1693                                             spd.nr_pages_max);
1694         if (spd.nr_pages <= 0)
1695                 ret = spd.nr_pages;
1696         else
1697                 ret = splice_to_pipe(pipe, &spd);
1698 
1699         splice_shrink_spd(&spd);
1700         return ret;
1701 }
1702 
1703 /*
1704  * Note that vmsplice only really supports true splicing _from_ user memory
1705  * to a pipe, not the other way around. Splicing from user memory is a simple
1706  * operation that can be supported without any funky alignment restrictions
1707  * or nasty vm tricks. We simply map in the user memory and fill them into
1708  * a pipe. The reverse isn't quite as easy, though. There are two possible
1709  * solutions for that:
1710  *
1711  *      - memcpy() the data internally, at which point we might as well just
1712  *        do a regular read() on the buffer anyway.
1713  *      - Lots of nasty vm tricks, that are neither fast nor flexible (it
1714  *        has restriction limitations on both ends of the pipe).
1715  *
1716  * Currently we punt and implement it as a normal copy, see pipe_to_user().
1717  *
1718  */
1719 SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, iov,
1720                 unsigned long, nr_segs, unsigned int, flags)
1721 {
1722         struct fd f;
1723         long error;
1724 
1725         if (unlikely(nr_segs > UIO_MAXIOV))
1726                 return -EINVAL;
1727         else if (unlikely(!nr_segs))
1728                 return 0;
1729 
1730         error = -EBADF;
1731         f = fdget(fd);
1732         if (f.file) {
1733                 if (f.file->f_mode & FMODE_WRITE)
1734                         error = vmsplice_to_pipe(f.file, iov, nr_segs, flags);
1735                 else if (f.file->f_mode & FMODE_READ)
1736                         error = vmsplice_to_user(f.file, iov, nr_segs, flags);
1737 
1738                 fdput(f);
1739         }
1740 
1741         return error;
1742 }
1743 
1744 #ifdef CONFIG_COMPAT
1745 COMPAT_SYSCALL_DEFINE4(vmsplice, int, fd, const struct compat_iovec __user *, iov32,
1746                     unsigned int, nr_segs, unsigned int, flags)
1747 {
1748         unsigned i;
1749         struct iovec __user *iov;
1750         if (nr_segs > UIO_MAXIOV)
1751                 return -EINVAL;
1752         iov = compat_alloc_user_space(nr_segs * sizeof(struct iovec));
1753         for (i = 0; i < nr_segs; i++) {
1754                 struct compat_iovec v;
1755                 if (get_user(v.iov_base, &iov32[i].iov_base) ||
1756                     get_user(v.iov_len, &iov32[i].iov_len) ||
1757                     put_user(compat_ptr(v.iov_base), &iov[i].iov_base) ||
1758                     put_user(v.iov_len, &iov[i].iov_len))
1759                         return -EFAULT;
1760         }
1761         return sys_vmsplice(fd, iov, nr_segs, flags);
1762 }
1763 #endif
1764 
1765 SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in,
1766                 int, fd_out, loff_t __user *, off_out,
1767                 size_t, len, unsigned int, flags)
1768 {
1769         struct fd in, out;
1770         long error;
1771 
1772         if (unlikely(!len))
1773                 return 0;
1774 
1775         error = -EBADF;
1776         in = fdget(fd_in);
1777         if (in.file) {
1778                 if (in.file->f_mode & FMODE_READ) {
1779                         out = fdget(fd_out);
1780                         if (out.file) {
1781                                 if (out.file->f_mode & FMODE_WRITE)
1782                                         error = do_splice(in.file, off_in,
1783                                                           out.file, off_out,
1784                                                           len, flags);
1785                                 fdput(out);
1786                         }
1787                 }
1788                 fdput(in);
1789         }
1790         return error;
1791 }
1792 
1793 /*
1794  * Make sure there's data to read. Wait for input if we can, otherwise
1795  * return an appropriate error.
1796  */
1797 static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1798 {
1799         int ret;
1800 
1801         /*
1802          * Check ->nrbufs without the inode lock first. This function
1803          * is speculative anyways, so missing one is ok.
1804          */
1805         if (pipe->nrbufs)
1806                 return 0;
1807 
1808         ret = 0;
1809         pipe_lock(pipe);
1810 
1811         while (!pipe->nrbufs) {
1812                 if (signal_pending(current)) {
1813                         ret = -ERESTARTSYS;
1814                         break;
1815                 }
1816                 if (!pipe->writers)
1817                         break;
1818                 if (!pipe->waiting_writers) {
1819                         if (flags & SPLICE_F_NONBLOCK) {
1820                                 ret = -EAGAIN;
1821                                 break;
1822                         }
1823                 }
1824                 pipe_wait(pipe);
1825         }
1826 
1827         pipe_unlock(pipe);
1828         return ret;
1829 }
1830 
1831 /*
1832  * Make sure there's writeable room. Wait for room if we can, otherwise
1833  * return an appropriate error.
1834  */
1835 static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1836 {
1837         int ret;
1838 
1839         /*
1840          * Check ->nrbufs without the inode lock first. This function
1841          * is speculative anyways, so missing one is ok.
1842          */
1843         if (pipe->nrbufs < pipe->buffers)
1844                 return 0;
1845 
1846         ret = 0;
1847         pipe_lock(pipe);
1848 
1849         while (pipe->nrbufs >= pipe->buffers) {
1850                 if (!pipe->readers) {
1851                         send_sig(SIGPIPE, current, 0);
1852                         ret = -EPIPE;
1853                         break;
1854                 }
1855                 if (flags & SPLICE_F_NONBLOCK) {
1856                         ret = -EAGAIN;
1857                         break;
1858                 }
1859                 if (signal_pending(current)) {
1860                         ret = -ERESTARTSYS;
1861                         break;
1862                 }
1863                 pipe->waiting_writers++;
1864                 pipe_wait(pipe);
1865                 pipe->waiting_writers--;
1866         }
1867 
1868         pipe_unlock(pipe);
1869         return ret;
1870 }
1871 
1872 /*
1873  * Splice contents of ipipe to opipe.
1874  */
1875 static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
1876                                struct pipe_inode_info *opipe,
1877                                size_t len, unsigned int flags)
1878 {
1879         struct pipe_buffer *ibuf, *obuf;
1880         int ret = 0, nbuf;
1881         bool input_wakeup = false;
1882 
1883 
1884 retry:
1885         ret = ipipe_prep(ipipe, flags);
1886         if (ret)
1887                 return ret;
1888 
1889         ret = opipe_prep(opipe, flags);
1890         if (ret)
1891                 return ret;
1892 
1893         /*
1894          * Potential ABBA deadlock, work around it by ordering lock
1895          * grabbing by pipe info address. Otherwise two different processes
1896          * could deadlock (one doing tee from A -> B, the other from B -> A).
1897          */
1898         pipe_double_lock(ipipe, opipe);
1899 
1900         do {
1901                 if (!opipe->readers) {
1902                         send_sig(SIGPIPE, current, 0);
1903                         if (!ret)
1904                                 ret = -EPIPE;
1905                         break;
1906                 }
1907 
1908                 if (!ipipe->nrbufs && !ipipe->writers)
1909                         break;
1910 
1911                 /*
1912                  * Cannot make any progress, because either the input
1913                  * pipe is empty or the output pipe is full.
1914                  */
1915                 if (!ipipe->nrbufs || opipe->nrbufs >= opipe->buffers) {
1916                         /* Already processed some buffers, break */
1917                         if (ret)
1918                                 break;
1919 
1920                         if (flags & SPLICE_F_NONBLOCK) {
1921                                 ret = -EAGAIN;
1922                                 break;
1923                         }
1924 
1925                         /*
1926                          * We raced with another reader/writer and haven't
1927                          * managed to process any buffers.  A zero return
1928                          * value means EOF, so retry instead.
1929                          */
1930                         pipe_unlock(ipipe);
1931                         pipe_unlock(opipe);
1932                         goto retry;
1933                 }
1934 
1935                 ibuf = ipipe->bufs + ipipe->curbuf;
1936                 nbuf = (opipe->curbuf + opipe->nrbufs) & (opipe->buffers - 1);
1937                 obuf = opipe->bufs + nbuf;
1938 
1939                 if (len >= ibuf->len) {
1940                         /*
1941                          * Simply move the whole buffer from ipipe to opipe
1942                          */
1943                         *obuf = *ibuf;
1944                         ibuf->ops = NULL;
1945                         opipe->nrbufs++;
1946                         ipipe->curbuf = (ipipe->curbuf + 1) & (ipipe->buffers - 1);
1947                         ipipe->nrbufs--;
1948                         input_wakeup = true;
1949                 } else {
1950                         /*
1951                          * Get a reference to this pipe buffer,
1952                          * so we can copy the contents over.
1953                          */
1954                         ibuf->ops->get(ipipe, ibuf);
1955                         *obuf = *ibuf;
1956 
1957                         /*
1958                          * Don't inherit the gift flag, we need to
1959                          * prevent multiple steals of this page.
1960                          */
1961                         obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1962 
1963                         obuf->len = len;
1964                         opipe->nrbufs++;
1965                         ibuf->offset += obuf->len;
1966                         ibuf->len -= obuf->len;
1967                 }
1968                 ret += obuf->len;
1969                 len -= obuf->len;
1970         } while (len);
1971 
1972         pipe_unlock(ipipe);
1973         pipe_unlock(opipe);
1974 
1975         /*
1976          * If we put data in the output pipe, wakeup any potential readers.
1977          */
1978         if (ret > 0)
1979                 wakeup_pipe_readers(opipe);
1980 
1981         if (input_wakeup)
1982                 wakeup_pipe_writers(ipipe);
1983 
1984         return ret;
1985 }
1986 
1987 /*
1988  * Link contents of ipipe to opipe.
1989  */
1990 static int link_pipe(struct pipe_inode_info *ipipe,
1991                      struct pipe_inode_info *opipe,
1992                      size_t len, unsigned int flags)
1993 {
1994         struct pipe_buffer *ibuf, *obuf;
1995         int ret = 0, i = 0, nbuf;
1996 
1997         /*
1998          * Potential ABBA deadlock, work around it by ordering lock
1999          * grabbing by pipe info address. Otherwise two different processes
2000          * could deadlock (one doing tee from A -> B, the other from B -> A).
2001          */
2002         pipe_double_lock(ipipe, opipe);
2003 
2004         do {
2005                 if (!opipe->readers) {
2006                         send_sig(SIGPIPE, current, 0);
2007                         if (!ret)
2008                                 ret = -EPIPE;
2009                         break;
2010                 }
2011 
2012                 /*
2013                  * If we have iterated all input buffers or ran out of
2014                  * output room, break.
2015                  */
2016                 if (i >= ipipe->nrbufs || opipe->nrbufs >= opipe->buffers)
2017                         break;
2018 
2019                 ibuf = ipipe->bufs + ((ipipe->curbuf + i) & (ipipe->buffers-1));
2020                 nbuf = (opipe->curbuf + opipe->nrbufs) & (opipe->buffers - 1);
2021 
2022                 /*
2023                  * Get a reference to this pipe buffer,
2024                  * so we can copy the contents over.
2025                  */
2026                 ibuf->ops->get(ipipe, ibuf);
2027 
2028                 obuf = opipe->bufs + nbuf;
2029                 *obuf = *ibuf;
2030 
2031                 /*
2032                  * Don't inherit the gift flag, we need to
2033                  * prevent multiple steals of this page.
2034                  */
2035                 obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
2036 
2037                 if (obuf->len > len)
2038                         obuf->len = len;
2039 
2040                 opipe->nrbufs++;
2041                 ret += obuf->len;
2042                 len -= obuf->len;
2043                 i++;
2044         } while (len);
2045 
2046         /*
2047          * return EAGAIN if we have the potential of some data in the
2048          * future, otherwise just return 0
2049          */
2050         if (!ret && ipipe->waiting_writers && (flags & SPLICE_F_NONBLOCK))
2051                 ret = -EAGAIN;
2052 
2053         pipe_unlock(ipipe);
2054         pipe_unlock(opipe);
2055 
2056         /*
2057          * If we put data in the output pipe, wakeup any potential readers.
2058          */
2059         if (ret > 0)
2060                 wakeup_pipe_readers(opipe);
2061 
2062         return ret;
2063 }
2064 
2065 /*
2066  * This is a tee(1) implementation that works on pipes. It doesn't copy
2067  * any data, it simply references the 'in' pages on the 'out' pipe.
2068  * The 'flags' used are the SPLICE_F_* variants, currently the only
2069  * applicable one is SPLICE_F_NONBLOCK.
2070  */
2071 static long do_tee(struct file *in, struct file *out, size_t len,
2072                    unsigned int flags)
2073 {
2074         struct pipe_inode_info *ipipe = get_pipe_info(in);
2075         struct pipe_inode_info *opipe = get_pipe_info(out);
2076         int ret = -EINVAL;
2077 
2078         /*
2079          * Duplicate the contents of ipipe to opipe without actually
2080          * copying the data.
2081          */
2082         if (ipipe && opipe && ipipe != opipe) {
2083                 /*
2084                  * Keep going, unless we encounter an error. The ipipe/opipe
2085                  * ordering doesn't really matter.
2086                  */
2087                 ret = ipipe_prep(ipipe, flags);
2088                 if (!ret) {
2089                         ret = opipe_prep(opipe, flags);
2090                         if (!ret)
2091                                 ret = link_pipe(ipipe, opipe, len, flags);
2092                 }
2093         }
2094 
2095         return ret;
2096 }
2097 
2098 SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags)
2099 {
2100         struct fd in;
2101         int error;
2102 
2103         if (unlikely(!len))
2104                 return 0;
2105 
2106         error = -EBADF;
2107         in = fdget(fdin);
2108         if (in.file) {
2109                 if (in.file->f_mode & FMODE_READ) {
2110                         struct fd out = fdget(fdout);
2111                         if (out.file) {
2112                                 if (out.file->f_mode & FMODE_WRITE)
2113                                         error = do_tee(in.file, out.file,
2114                                                         len, flags);
2115                                 fdput(out);
2116                         }
2117                 }
2118                 fdput(in);
2119         }
2120 
2121         return error;
2122 }
2123 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp