~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/lib/iov_iter.c

Version: ~ [ linux-5.11-rc3 ] ~ [ linux-5.10.7 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.89 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.167 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.215 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.251 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.251 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 #include <linux/export.h>
  2 #include <linux/uio.h>
  3 #include <linux/pagemap.h>
  4 #include <linux/slab.h>
  5 #include <linux/vmalloc.h>
  6 #include <net/checksum.h>
  7 
  8 #define iterate_iovec(i, n, __v, __p, skip, STEP) {     \
  9         size_t left;                                    \
 10         size_t wanted = n;                              \
 11         __p = i->iov;                                   \
 12         __v.iov_len = min(n, __p->iov_len - skip);      \
 13         if (likely(__v.iov_len)) {                      \
 14                 __v.iov_base = __p->iov_base + skip;    \
 15                 left = (STEP);                          \
 16                 __v.iov_len -= left;                    \
 17                 skip += __v.iov_len;                    \
 18                 n -= __v.iov_len;                       \
 19         } else {                                        \
 20                 left = 0;                               \
 21         }                                               \
 22         while (unlikely(!left && n)) {                  \
 23                 __p++;                                  \
 24                 __v.iov_len = min(n, __p->iov_len);     \
 25                 if (unlikely(!__v.iov_len))             \
 26                         continue;                       \
 27                 __v.iov_base = __p->iov_base;           \
 28                 left = (STEP);                          \
 29                 __v.iov_len -= left;                    \
 30                 skip = __v.iov_len;                     \
 31                 n -= __v.iov_len;                       \
 32         }                                               \
 33         n = wanted - n;                                 \
 34 }
 35 
 36 #define iterate_kvec(i, n, __v, __p, skip, STEP) {      \
 37         size_t wanted = n;                              \
 38         __p = i->kvec;                                  \
 39         __v.iov_len = min(n, __p->iov_len - skip);      \
 40         if (likely(__v.iov_len)) {                      \
 41                 __v.iov_base = __p->iov_base + skip;    \
 42                 (void)(STEP);                           \
 43                 skip += __v.iov_len;                    \
 44                 n -= __v.iov_len;                       \
 45         }                                               \
 46         while (unlikely(n)) {                           \
 47                 __p++;                                  \
 48                 __v.iov_len = min(n, __p->iov_len);     \
 49                 if (unlikely(!__v.iov_len))             \
 50                         continue;                       \
 51                 __v.iov_base = __p->iov_base;           \
 52                 (void)(STEP);                           \
 53                 skip = __v.iov_len;                     \
 54                 n -= __v.iov_len;                       \
 55         }                                               \
 56         n = wanted;                                     \
 57 }
 58 
 59 #define iterate_bvec(i, n, __v, __bi, skip, STEP) {     \
 60         struct bvec_iter __start;                       \
 61         __start.bi_size = n;                            \
 62         __start.bi_bvec_done = skip;                    \
 63         __start.bi_idx = 0;                             \
 64         for_each_bvec(__v, i->bvec, __bi, __start) {    \
 65                 if (!__v.bv_len)                        \
 66                         continue;                       \
 67                 (void)(STEP);                           \
 68         }                                               \
 69 }
 70 
 71 #define iterate_all_kinds(i, n, v, I, B, K) {                   \
 72         size_t skip = i->iov_offset;                            \
 73         if (unlikely(i->type & ITER_BVEC)) {                    \
 74                 struct bio_vec v;                               \
 75                 struct bvec_iter __bi;                          \
 76                 iterate_bvec(i, n, v, __bi, skip, (B))          \
 77         } else if (unlikely(i->type & ITER_KVEC)) {             \
 78                 const struct kvec *kvec;                        \
 79                 struct kvec v;                                  \
 80                 iterate_kvec(i, n, v, kvec, skip, (K))          \
 81         } else {                                                \
 82                 const struct iovec *iov;                        \
 83                 struct iovec v;                                 \
 84                 iterate_iovec(i, n, v, iov, skip, (I))          \
 85         }                                                       \
 86 }
 87 
 88 #define iterate_and_advance(i, n, v, I, B, K) {                 \
 89         if (unlikely(i->count < n))                             \
 90                 n = i->count;                                   \
 91         if (i->count) {                                         \
 92                 size_t skip = i->iov_offset;                    \
 93                 if (unlikely(i->type & ITER_BVEC)) {            \
 94                         const struct bio_vec *bvec = i->bvec;   \
 95                         struct bio_vec v;                       \
 96                         struct bvec_iter __bi;                  \
 97                         iterate_bvec(i, n, v, __bi, skip, (B))  \
 98                         i->bvec = __bvec_iter_bvec(i->bvec, __bi);      \
 99                         i->nr_segs -= i->bvec - bvec;           \
100                         skip = __bi.bi_bvec_done;               \
101                 } else if (unlikely(i->type & ITER_KVEC)) {     \
102                         const struct kvec *kvec;                \
103                         struct kvec v;                          \
104                         iterate_kvec(i, n, v, kvec, skip, (K))  \
105                         if (skip == kvec->iov_len) {            \
106                                 kvec++;                         \
107                                 skip = 0;                       \
108                         }                                       \
109                         i->nr_segs -= kvec - i->kvec;           \
110                         i->kvec = kvec;                         \
111                 } else {                                        \
112                         const struct iovec *iov;                \
113                         struct iovec v;                         \
114                         iterate_iovec(i, n, v, iov, skip, (I))  \
115                         if (skip == iov->iov_len) {             \
116                                 iov++;                          \
117                                 skip = 0;                       \
118                         }                                       \
119                         i->nr_segs -= iov - i->iov;             \
120                         i->iov = iov;                           \
121                 }                                               \
122                 i->count -= n;                                  \
123                 i->iov_offset = skip;                           \
124         }                                                       \
125 }
126 
127 static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,
128                          struct iov_iter *i)
129 {
130         size_t skip, copy, left, wanted;
131         const struct iovec *iov;
132         char __user *buf;
133         void *kaddr, *from;
134 
135         if (unlikely(bytes > i->count))
136                 bytes = i->count;
137 
138         if (unlikely(!bytes))
139                 return 0;
140 
141         wanted = bytes;
142         iov = i->iov;
143         skip = i->iov_offset;
144         buf = iov->iov_base + skip;
145         copy = min(bytes, iov->iov_len - skip);
146 
147         if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_writeable(buf, copy)) {
148                 kaddr = kmap_atomic(page);
149                 from = kaddr + offset;
150 
151                 /* first chunk, usually the only one */
152                 left = __copy_to_user_inatomic(buf, from, copy);
153                 copy -= left;
154                 skip += copy;
155                 from += copy;
156                 bytes -= copy;
157 
158                 while (unlikely(!left && bytes)) {
159                         iov++;
160                         buf = iov->iov_base;
161                         copy = min(bytes, iov->iov_len);
162                         left = __copy_to_user_inatomic(buf, from, copy);
163                         copy -= left;
164                         skip = copy;
165                         from += copy;
166                         bytes -= copy;
167                 }
168                 if (likely(!bytes)) {
169                         kunmap_atomic(kaddr);
170                         goto done;
171                 }
172                 offset = from - kaddr;
173                 buf += copy;
174                 kunmap_atomic(kaddr);
175                 copy = min(bytes, iov->iov_len - skip);
176         }
177         /* Too bad - revert to non-atomic kmap */
178 
179         kaddr = kmap(page);
180         from = kaddr + offset;
181         left = __copy_to_user(buf, from, copy);
182         copy -= left;
183         skip += copy;
184         from += copy;
185         bytes -= copy;
186         while (unlikely(!left && bytes)) {
187                 iov++;
188                 buf = iov->iov_base;
189                 copy = min(bytes, iov->iov_len);
190                 left = __copy_to_user(buf, from, copy);
191                 copy -= left;
192                 skip = copy;
193                 from += copy;
194                 bytes -= copy;
195         }
196         kunmap(page);
197 
198 done:
199         if (skip == iov->iov_len) {
200                 iov++;
201                 skip = 0;
202         }
203         i->count -= wanted - bytes;
204         i->nr_segs -= iov - i->iov;
205         i->iov = iov;
206         i->iov_offset = skip;
207         return wanted - bytes;
208 }
209 
210 static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,
211                          struct iov_iter *i)
212 {
213         size_t skip, copy, left, wanted;
214         const struct iovec *iov;
215         char __user *buf;
216         void *kaddr, *to;
217 
218         if (unlikely(bytes > i->count))
219                 bytes = i->count;
220 
221         if (unlikely(!bytes))
222                 return 0;
223 
224         wanted = bytes;
225         iov = i->iov;
226         skip = i->iov_offset;
227         buf = iov->iov_base + skip;
228         copy = min(bytes, iov->iov_len - skip);
229 
230         if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_readable(buf, copy)) {
231                 kaddr = kmap_atomic(page);
232                 to = kaddr + offset;
233 
234                 /* first chunk, usually the only one */
235                 left = __copy_from_user_inatomic(to, buf, copy);
236                 copy -= left;
237                 skip += copy;
238                 to += copy;
239                 bytes -= copy;
240 
241                 while (unlikely(!left && bytes)) {
242                         iov++;
243                         buf = iov->iov_base;
244                         copy = min(bytes, iov->iov_len);
245                         left = __copy_from_user_inatomic(to, buf, copy);
246                         copy -= left;
247                         skip = copy;
248                         to += copy;
249                         bytes -= copy;
250                 }
251                 if (likely(!bytes)) {
252                         kunmap_atomic(kaddr);
253                         goto done;
254                 }
255                 offset = to - kaddr;
256                 buf += copy;
257                 kunmap_atomic(kaddr);
258                 copy = min(bytes, iov->iov_len - skip);
259         }
260         /* Too bad - revert to non-atomic kmap */
261 
262         kaddr = kmap(page);
263         to = kaddr + offset;
264         left = __copy_from_user(to, buf, copy);
265         copy -= left;
266         skip += copy;
267         to += copy;
268         bytes -= copy;
269         while (unlikely(!left && bytes)) {
270                 iov++;
271                 buf = iov->iov_base;
272                 copy = min(bytes, iov->iov_len);
273                 left = __copy_from_user(to, buf, copy);
274                 copy -= left;
275                 skip = copy;
276                 to += copy;
277                 bytes -= copy;
278         }
279         kunmap(page);
280 
281 done:
282         if (skip == iov->iov_len) {
283                 iov++;
284                 skip = 0;
285         }
286         i->count -= wanted - bytes;
287         i->nr_segs -= iov - i->iov;
288         i->iov = iov;
289         i->iov_offset = skip;
290         return wanted - bytes;
291 }
292 
293 /*
294  * Fault in one or more iovecs of the given iov_iter, to a maximum length of
295  * bytes.  For each iovec, fault in each page that constitutes the iovec.
296  *
297  * Return 0 on success, or non-zero if the memory could not be accessed (i.e.
298  * because it is an invalid address).
299  */
300 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
301 {
302         size_t skip = i->iov_offset;
303         const struct iovec *iov;
304         int err;
305         struct iovec v;
306 
307         if (!(i->type & (ITER_BVEC|ITER_KVEC))) {
308                 iterate_iovec(i, bytes, v, iov, skip, ({
309                         err = fault_in_multipages_readable(v.iov_base,
310                                         v.iov_len);
311                         if (unlikely(err))
312                         return err;
313                 0;}))
314         }
315         return 0;
316 }
317 EXPORT_SYMBOL(iov_iter_fault_in_readable);
318 
319 void iov_iter_init(struct iov_iter *i, int direction,
320                         const struct iovec *iov, unsigned long nr_segs,
321                         size_t count)
322 {
323         /* It will get better.  Eventually... */
324         if (segment_eq(get_fs(), KERNEL_DS)) {
325                 direction |= ITER_KVEC;
326                 i->type = direction;
327                 i->kvec = (struct kvec *)iov;
328         } else {
329                 i->type = direction;
330                 i->iov = iov;
331         }
332         i->nr_segs = nr_segs;
333         i->iov_offset = 0;
334         i->count = count;
335 }
336 EXPORT_SYMBOL(iov_iter_init);
337 
338 static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len)
339 {
340         char *from = kmap_atomic(page);
341         memcpy(to, from + offset, len);
342         kunmap_atomic(from);
343 }
344 
345 static void memcpy_to_page(struct page *page, size_t offset, const char *from, size_t len)
346 {
347         char *to = kmap_atomic(page);
348         memcpy(to + offset, from, len);
349         kunmap_atomic(to);
350 }
351 
352 static void memzero_page(struct page *page, size_t offset, size_t len)
353 {
354         char *addr = kmap_atomic(page);
355         memset(addr + offset, 0, len);
356         kunmap_atomic(addr);
357 }
358 
359 size_t copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
360 {
361         const char *from = addr;
362         iterate_and_advance(i, bytes, v,
363                 __copy_to_user(v.iov_base, (from += v.iov_len) - v.iov_len,
364                                v.iov_len),
365                 memcpy_to_page(v.bv_page, v.bv_offset,
366                                (from += v.bv_len) - v.bv_len, v.bv_len),
367                 memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len)
368         )
369 
370         return bytes;
371 }
372 EXPORT_SYMBOL(copy_to_iter);
373 
374 size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
375 {
376         char *to = addr;
377         iterate_and_advance(i, bytes, v,
378                 __copy_from_user((to += v.iov_len) - v.iov_len, v.iov_base,
379                                  v.iov_len),
380                 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
381                                  v.bv_offset, v.bv_len),
382                 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
383         )
384 
385         return bytes;
386 }
387 EXPORT_SYMBOL(copy_from_iter);
388 
389 size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
390 {
391         char *to = addr;
392         iterate_and_advance(i, bytes, v,
393                 __copy_from_user_nocache((to += v.iov_len) - v.iov_len,
394                                          v.iov_base, v.iov_len),
395                 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
396                                  v.bv_offset, v.bv_len),
397                 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
398         )
399 
400         return bytes;
401 }
402 EXPORT_SYMBOL(copy_from_iter_nocache);
403 
404 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
405                          struct iov_iter *i)
406 {
407         if (i->type & (ITER_BVEC|ITER_KVEC)) {
408                 void *kaddr = kmap_atomic(page);
409                 size_t wanted = copy_to_iter(kaddr + offset, bytes, i);
410                 kunmap_atomic(kaddr);
411                 return wanted;
412         } else
413                 return copy_page_to_iter_iovec(page, offset, bytes, i);
414 }
415 EXPORT_SYMBOL(copy_page_to_iter);
416 
417 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
418                          struct iov_iter *i)
419 {
420         if (i->type & (ITER_BVEC|ITER_KVEC)) {
421                 void *kaddr = kmap_atomic(page);
422                 size_t wanted = copy_from_iter(kaddr + offset, bytes, i);
423                 kunmap_atomic(kaddr);
424                 return wanted;
425         } else
426                 return copy_page_from_iter_iovec(page, offset, bytes, i);
427 }
428 EXPORT_SYMBOL(copy_page_from_iter);
429 
430 size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
431 {
432         iterate_and_advance(i, bytes, v,
433                 __clear_user(v.iov_base, v.iov_len),
434                 memzero_page(v.bv_page, v.bv_offset, v.bv_len),
435                 memset(v.iov_base, 0, v.iov_len)
436         )
437 
438         return bytes;
439 }
440 EXPORT_SYMBOL(iov_iter_zero);
441 
442 size_t iov_iter_copy_from_user_atomic(struct page *page,
443                 struct iov_iter *i, unsigned long offset, size_t bytes)
444 {
445         char *kaddr = kmap_atomic(page), *p = kaddr + offset;
446         iterate_all_kinds(i, bytes, v,
447                 __copy_from_user_inatomic((p += v.iov_len) - v.iov_len,
448                                           v.iov_base, v.iov_len),
449                 memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page,
450                                  v.bv_offset, v.bv_len),
451                 memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
452         )
453         kunmap_atomic(kaddr);
454         return bytes;
455 }
456 EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);
457 
458 void iov_iter_advance(struct iov_iter *i, size_t size)
459 {
460         iterate_and_advance(i, size, v, 0, 0, 0)
461 }
462 EXPORT_SYMBOL(iov_iter_advance);
463 
464 /*
465  * Return the count of just the current iov_iter segment.
466  */
467 size_t iov_iter_single_seg_count(const struct iov_iter *i)
468 {
469         if (i->nr_segs == 1)
470                 return i->count;
471         else if (i->type & ITER_BVEC)
472                 return min(i->count, i->bvec->bv_len - i->iov_offset);
473         else
474                 return min(i->count, i->iov->iov_len - i->iov_offset);
475 }
476 EXPORT_SYMBOL(iov_iter_single_seg_count);
477 
478 void iov_iter_kvec(struct iov_iter *i, int direction,
479                         const struct kvec *kvec, unsigned long nr_segs,
480                         size_t count)
481 {
482         BUG_ON(!(direction & ITER_KVEC));
483         i->type = direction;
484         i->kvec = kvec;
485         i->nr_segs = nr_segs;
486         i->iov_offset = 0;
487         i->count = count;
488 }
489 EXPORT_SYMBOL(iov_iter_kvec);
490 
491 void iov_iter_bvec(struct iov_iter *i, int direction,
492                         const struct bio_vec *bvec, unsigned long nr_segs,
493                         size_t count)
494 {
495         BUG_ON(!(direction & ITER_BVEC));
496         i->type = direction;
497         i->bvec = bvec;
498         i->nr_segs = nr_segs;
499         i->iov_offset = 0;
500         i->count = count;
501 }
502 EXPORT_SYMBOL(iov_iter_bvec);
503 
504 unsigned long iov_iter_alignment(const struct iov_iter *i)
505 {
506         unsigned long res = 0;
507         size_t size = i->count;
508 
509         if (!size)
510                 return 0;
511 
512         iterate_all_kinds(i, size, v,
513                 (res |= (unsigned long)v.iov_base | v.iov_len, 0),
514                 res |= v.bv_offset | v.bv_len,
515                 res |= (unsigned long)v.iov_base | v.iov_len
516         )
517         return res;
518 }
519 EXPORT_SYMBOL(iov_iter_alignment);
520 
521 unsigned long iov_iter_gap_alignment(const struct iov_iter *i)
522 {
523         unsigned long res = 0;
524         size_t size = i->count;
525         if (!size)
526                 return 0;
527 
528         iterate_all_kinds(i, size, v,
529                 (res |= (!res ? 0 : (unsigned long)v.iov_base) |
530                         (size != v.iov_len ? size : 0), 0),
531                 (res |= (!res ? 0 : (unsigned long)v.bv_offset) |
532                         (size != v.bv_len ? size : 0)),
533                 (res |= (!res ? 0 : (unsigned long)v.iov_base) |
534                         (size != v.iov_len ? size : 0))
535                 );
536                 return res;
537 }
538 EXPORT_SYMBOL(iov_iter_gap_alignment);
539 
540 ssize_t iov_iter_get_pages(struct iov_iter *i,
541                    struct page **pages, size_t maxsize, unsigned maxpages,
542                    size_t *start)
543 {
544         if (maxsize > i->count)
545                 maxsize = i->count;
546 
547         if (!maxsize)
548                 return 0;
549 
550         iterate_all_kinds(i, maxsize, v, ({
551                 unsigned long addr = (unsigned long)v.iov_base;
552                 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
553                 int n;
554                 int res;
555 
556                 if (len > maxpages * PAGE_SIZE)
557                         len = maxpages * PAGE_SIZE;
558                 addr &= ~(PAGE_SIZE - 1);
559                 n = DIV_ROUND_UP(len, PAGE_SIZE);
560                 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);
561                 if (unlikely(res < 0))
562                         return res;
563                 return (res == n ? len : res * PAGE_SIZE) - *start;
564         0;}),({
565                 /* can't be more than PAGE_SIZE */
566                 *start = v.bv_offset;
567                 get_page(*pages = v.bv_page);
568                 return v.bv_len;
569         }),({
570                 return -EFAULT;
571         })
572         )
573         return 0;
574 }
575 EXPORT_SYMBOL(iov_iter_get_pages);
576 
577 static struct page **get_pages_array(size_t n)
578 {
579         struct page **p = kmalloc(n * sizeof(struct page *), GFP_KERNEL);
580         if (!p)
581                 p = vmalloc(n * sizeof(struct page *));
582         return p;
583 }
584 
585 ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
586                    struct page ***pages, size_t maxsize,
587                    size_t *start)
588 {
589         struct page **p;
590 
591         if (maxsize > i->count)
592                 maxsize = i->count;
593 
594         if (!maxsize)
595                 return 0;
596 
597         iterate_all_kinds(i, maxsize, v, ({
598                 unsigned long addr = (unsigned long)v.iov_base;
599                 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
600                 int n;
601                 int res;
602 
603                 addr &= ~(PAGE_SIZE - 1);
604                 n = DIV_ROUND_UP(len, PAGE_SIZE);
605                 p = get_pages_array(n);
606                 if (!p)
607                         return -ENOMEM;
608                 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p);
609                 if (unlikely(res < 0)) {
610                         kvfree(p);
611                         return res;
612                 }
613                 *pages = p;
614                 return (res == n ? len : res * PAGE_SIZE) - *start;
615         0;}),({
616                 /* can't be more than PAGE_SIZE */
617                 *start = v.bv_offset;
618                 *pages = p = get_pages_array(1);
619                 if (!p)
620                         return -ENOMEM;
621                 get_page(*p = v.bv_page);
622                 return v.bv_len;
623         }),({
624                 return -EFAULT;
625         })
626         )
627         return 0;
628 }
629 EXPORT_SYMBOL(iov_iter_get_pages_alloc);
630 
631 size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,
632                                struct iov_iter *i)
633 {
634         char *to = addr;
635         __wsum sum, next;
636         size_t off = 0;
637         sum = *csum;
638         iterate_and_advance(i, bytes, v, ({
639                 int err = 0;
640                 next = csum_and_copy_from_user(v.iov_base, 
641                                                (to += v.iov_len) - v.iov_len,
642                                                v.iov_len, 0, &err);
643                 if (!err) {
644                         sum = csum_block_add(sum, next, off);
645                         off += v.iov_len;
646                 }
647                 err ? v.iov_len : 0;
648         }), ({
649                 char *p = kmap_atomic(v.bv_page);
650                 next = csum_partial_copy_nocheck(p + v.bv_offset,
651                                                  (to += v.bv_len) - v.bv_len,
652                                                  v.bv_len, 0);
653                 kunmap_atomic(p);
654                 sum = csum_block_add(sum, next, off);
655                 off += v.bv_len;
656         }),({
657                 next = csum_partial_copy_nocheck(v.iov_base,
658                                                  (to += v.iov_len) - v.iov_len,
659                                                  v.iov_len, 0);
660                 sum = csum_block_add(sum, next, off);
661                 off += v.iov_len;
662         })
663         )
664         *csum = sum;
665         return bytes;
666 }
667 EXPORT_SYMBOL(csum_and_copy_from_iter);
668 
669 size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum,
670                              struct iov_iter *i)
671 {
672         const char *from = addr;
673         __wsum sum, next;
674         size_t off = 0;
675         sum = *csum;
676         iterate_and_advance(i, bytes, v, ({
677                 int err = 0;
678                 next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len,
679                                              v.iov_base, 
680                                              v.iov_len, 0, &err);
681                 if (!err) {
682                         sum = csum_block_add(sum, next, off);
683                         off += v.iov_len;
684                 }
685                 err ? v.iov_len : 0;
686         }), ({
687                 char *p = kmap_atomic(v.bv_page);
688                 next = csum_partial_copy_nocheck((from += v.bv_len) - v.bv_len,
689                                                  p + v.bv_offset,
690                                                  v.bv_len, 0);
691                 kunmap_atomic(p);
692                 sum = csum_block_add(sum, next, off);
693                 off += v.bv_len;
694         }),({
695                 next = csum_partial_copy_nocheck((from += v.iov_len) - v.iov_len,
696                                                  v.iov_base,
697                                                  v.iov_len, 0);
698                 sum = csum_block_add(sum, next, off);
699                 off += v.iov_len;
700         })
701         )
702         *csum = sum;
703         return bytes;
704 }
705 EXPORT_SYMBOL(csum_and_copy_to_iter);
706 
707 int iov_iter_npages(const struct iov_iter *i, int maxpages)
708 {
709         size_t size = i->count;
710         int npages = 0;
711 
712         if (!size)
713                 return 0;
714 
715         iterate_all_kinds(i, size, v, ({
716                 unsigned long p = (unsigned long)v.iov_base;
717                 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
718                         - p / PAGE_SIZE;
719                 if (npages >= maxpages)
720                         return maxpages;
721         0;}),({
722                 npages++;
723                 if (npages >= maxpages)
724                         return maxpages;
725         }),({
726                 unsigned long p = (unsigned long)v.iov_base;
727                 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
728                         - p / PAGE_SIZE;
729                 if (npages >= maxpages)
730                         return maxpages;
731         })
732         )
733         return npages;
734 }
735 EXPORT_SYMBOL(iov_iter_npages);
736 
737 const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags)
738 {
739         *new = *old;
740         if (new->type & ITER_BVEC)
741                 return new->bvec = kmemdup(new->bvec,
742                                     new->nr_segs * sizeof(struct bio_vec),
743                                     flags);
744         else
745                 /* iovec and kvec have identical layout */
746                 return new->iov = kmemdup(new->iov,
747                                    new->nr_segs * sizeof(struct iovec),
748                                    flags);
749 }
750 EXPORT_SYMBOL(dup_iter);
751 
752 int import_iovec(int type, const struct iovec __user * uvector,
753                  unsigned nr_segs, unsigned fast_segs,
754                  struct iovec **iov, struct iov_iter *i)
755 {
756         ssize_t n;
757         struct iovec *p;
758         n = rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
759                                   *iov, &p);
760         if (n < 0) {
761                 if (p != *iov)
762                         kfree(p);
763                 *iov = NULL;
764                 return n;
765         }
766         iov_iter_init(i, type, p, nr_segs, n);
767         *iov = p == *iov ? NULL : p;
768         return 0;
769 }
770 EXPORT_SYMBOL(import_iovec);
771 
772 #ifdef CONFIG_COMPAT
773 #include <linux/compat.h>
774 
775 int compat_import_iovec(int type, const struct compat_iovec __user * uvector,
776                  unsigned nr_segs, unsigned fast_segs,
777                  struct iovec **iov, struct iov_iter *i)
778 {
779         ssize_t n;
780         struct iovec *p;
781         n = compat_rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
782                                   *iov, &p);
783         if (n < 0) {
784                 if (p != *iov)
785                         kfree(p);
786                 *iov = NULL;
787                 return n;
788         }
789         iov_iter_init(i, type, p, nr_segs, n);
790         *iov = p == *iov ? NULL : p;
791         return 0;
792 }
793 #endif
794 
795 int import_single_range(int rw, void __user *buf, size_t len,
796                  struct iovec *iov, struct iov_iter *i)
797 {
798         if (len > MAX_RW_COUNT)
799                 len = MAX_RW_COUNT;
800         if (unlikely(!access_ok(!rw, buf, len)))
801                 return -EFAULT;
802 
803         iov->iov_base = buf;
804         iov->iov_len = len;
805         iov_iter_init(i, rw, iov, 1, len);
806         return 0;
807 }
808 EXPORT_SYMBOL(import_single_range);
809 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp