~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/lib/scatterlist.c

Version: ~ [ linux-5.8 ] ~ [ linux-5.7.14 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.57 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.138 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.193 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.232 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.232 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Copyright (C) 2007 Jens Axboe <jens.axboe@oracle.com>
  3  *
  4  * Scatterlist handling helpers.
  5  *
  6  * This source code is licensed under the GNU General Public License,
  7  * Version 2. See the file COPYING for more details.
  8  */
  9 #include <linux/export.h>
 10 #include <linux/slab.h>
 11 #include <linux/scatterlist.h>
 12 #include <linux/highmem.h>
 13 #include <linux/kmemleak.h>
 14 
 15 /**
 16  * sg_next - return the next scatterlist entry in a list
 17  * @sg:         The current sg entry
 18  *
 19  * Description:
 20  *   Usually the next entry will be @sg@ + 1, but if this sg element is part
 21  *   of a chained scatterlist, it could jump to the start of a new
 22  *   scatterlist array.
 23  *
 24  **/
 25 struct scatterlist *sg_next(struct scatterlist *sg)
 26 {
 27         if (sg_is_last(sg))
 28                 return NULL;
 29 
 30         sg++;
 31         if (unlikely(sg_is_chain(sg)))
 32                 sg = sg_chain_ptr(sg);
 33 
 34         return sg;
 35 }
 36 EXPORT_SYMBOL(sg_next);
 37 
 38 /**
 39  * sg_nents - return total count of entries in scatterlist
 40  * @sg:         The scatterlist
 41  *
 42  * Description:
 43  * Allows to know how many entries are in sg, taking into acount
 44  * chaining as well
 45  *
 46  **/
 47 int sg_nents(struct scatterlist *sg)
 48 {
 49         int nents;
 50         for (nents = 0; sg; sg = sg_next(sg))
 51                 nents++;
 52         return nents;
 53 }
 54 EXPORT_SYMBOL(sg_nents);
 55 
 56 /**
 57  * sg_nents_for_len - return total count of entries in scatterlist
 58  *                    needed to satisfy the supplied length
 59  * @sg:         The scatterlist
 60  * @len:        The total required length
 61  *
 62  * Description:
 63  * Determines the number of entries in sg that are required to meet
 64  * the supplied length, taking into acount chaining as well
 65  *
 66  * Returns:
 67  *   the number of sg entries needed, negative error on failure
 68  *
 69  **/
 70 int sg_nents_for_len(struct scatterlist *sg, u64 len)
 71 {
 72         int nents;
 73         u64 total;
 74 
 75         if (!len)
 76                 return 0;
 77 
 78         for (nents = 0, total = 0; sg; sg = sg_next(sg)) {
 79                 nents++;
 80                 total += sg->length;
 81                 if (total >= len)
 82                         return nents;
 83         }
 84 
 85         return -EINVAL;
 86 }
 87 EXPORT_SYMBOL(sg_nents_for_len);
 88 
 89 /**
 90  * sg_last - return the last scatterlist entry in a list
 91  * @sgl:        First entry in the scatterlist
 92  * @nents:      Number of entries in the scatterlist
 93  *
 94  * Description:
 95  *   Should only be used casually, it (currently) scans the entire list
 96  *   to get the last entry.
 97  *
 98  *   Note that the @sgl@ pointer passed in need not be the first one,
 99  *   the important bit is that @nents@ denotes the number of entries that
100  *   exist from @sgl@.
101  *
102  **/
103 struct scatterlist *sg_last(struct scatterlist *sgl, unsigned int nents)
104 {
105         struct scatterlist *sg, *ret = NULL;
106         unsigned int i;
107 
108         for_each_sg(sgl, sg, nents, i)
109                 ret = sg;
110 
111         BUG_ON(!sg_is_last(ret));
112         return ret;
113 }
114 EXPORT_SYMBOL(sg_last);
115 
116 /**
117  * sg_init_table - Initialize SG table
118  * @sgl:           The SG table
119  * @nents:         Number of entries in table
120  *
121  * Notes:
122  *   If this is part of a chained sg table, sg_mark_end() should be
123  *   used only on the last table part.
124  *
125  **/
126 void sg_init_table(struct scatterlist *sgl, unsigned int nents)
127 {
128         memset(sgl, 0, sizeof(*sgl) * nents);
129         sg_init_marker(sgl, nents);
130 }
131 EXPORT_SYMBOL(sg_init_table);
132 
133 /**
134  * sg_init_one - Initialize a single entry sg list
135  * @sg:          SG entry
136  * @buf:         Virtual address for IO
137  * @buflen:      IO length
138  *
139  **/
140 void sg_init_one(struct scatterlist *sg, const void *buf, unsigned int buflen)
141 {
142         sg_init_table(sg, 1);
143         sg_set_buf(sg, buf, buflen);
144 }
145 EXPORT_SYMBOL(sg_init_one);
146 
147 /*
148  * The default behaviour of sg_alloc_table() is to use these kmalloc/kfree
149  * helpers.
150  */
151 static struct scatterlist *sg_kmalloc(unsigned int nents, gfp_t gfp_mask)
152 {
153         if (nents == SG_MAX_SINGLE_ALLOC) {
154                 /*
155                  * Kmemleak doesn't track page allocations as they are not
156                  * commonly used (in a raw form) for kernel data structures.
157                  * As we chain together a list of pages and then a normal
158                  * kmalloc (tracked by kmemleak), in order to for that last
159                  * allocation not to become decoupled (and thus a
160                  * false-positive) we need to inform kmemleak of all the
161                  * intermediate allocations.
162                  */
163                 void *ptr = (void *) __get_free_page(gfp_mask);
164                 kmemleak_alloc(ptr, PAGE_SIZE, 1, gfp_mask);
165                 return ptr;
166         } else
167                 return kmalloc_array(nents, sizeof(struct scatterlist),
168                                      gfp_mask);
169 }
170 
171 static void sg_kfree(struct scatterlist *sg, unsigned int nents)
172 {
173         if (nents == SG_MAX_SINGLE_ALLOC) {
174                 kmemleak_free(sg);
175                 free_page((unsigned long) sg);
176         } else
177                 kfree(sg);
178 }
179 
180 /**
181  * __sg_free_table - Free a previously mapped sg table
182  * @table:      The sg table header to use
183  * @max_ents:   The maximum number of entries per single scatterlist
184  * @skip_first_chunk: don't free the (preallocated) first scatterlist chunk
185  * @free_fn:    Free function
186  *
187  *  Description:
188  *    Free an sg table previously allocated and setup with
189  *    __sg_alloc_table().  The @max_ents value must be identical to
190  *    that previously used with __sg_alloc_table().
191  *
192  **/
193 void __sg_free_table(struct sg_table *table, unsigned int max_ents,
194                      bool skip_first_chunk, sg_free_fn *free_fn)
195 {
196         struct scatterlist *sgl, *next;
197 
198         if (unlikely(!table->sgl))
199                 return;
200 
201         sgl = table->sgl;
202         while (table->orig_nents) {
203                 unsigned int alloc_size = table->orig_nents;
204                 unsigned int sg_size;
205 
206                 /*
207                  * If we have more than max_ents segments left,
208                  * then assign 'next' to the sg table after the current one.
209                  * sg_size is then one less than alloc size, since the last
210                  * element is the chain pointer.
211                  */
212                 if (alloc_size > max_ents) {
213                         next = sg_chain_ptr(&sgl[max_ents - 1]);
214                         alloc_size = max_ents;
215                         sg_size = alloc_size - 1;
216                 } else {
217                         sg_size = alloc_size;
218                         next = NULL;
219                 }
220 
221                 table->orig_nents -= sg_size;
222                 if (skip_first_chunk)
223                         skip_first_chunk = false;
224                 else
225                         free_fn(sgl, alloc_size);
226                 sgl = next;
227         }
228 
229         table->sgl = NULL;
230 }
231 EXPORT_SYMBOL(__sg_free_table);
232 
233 /**
234  * sg_free_table - Free a previously allocated sg table
235  * @table:      The mapped sg table header
236  *
237  **/
238 void sg_free_table(struct sg_table *table)
239 {
240         __sg_free_table(table, SG_MAX_SINGLE_ALLOC, false, sg_kfree);
241 }
242 EXPORT_SYMBOL(sg_free_table);
243 
244 /**
245  * __sg_alloc_table - Allocate and initialize an sg table with given allocator
246  * @table:      The sg table header to use
247  * @nents:      Number of entries in sg list
248  * @max_ents:   The maximum number of entries the allocator returns per call
249  * @gfp_mask:   GFP allocation mask
250  * @alloc_fn:   Allocator to use
251  *
252  * Description:
253  *   This function returns a @table @nents long. The allocator is
254  *   defined to return scatterlist chunks of maximum size @max_ents.
255  *   Thus if @nents is bigger than @max_ents, the scatterlists will be
256  *   chained in units of @max_ents.
257  *
258  * Notes:
259  *   If this function returns non-0 (eg failure), the caller must call
260  *   __sg_free_table() to cleanup any leftover allocations.
261  *
262  **/
263 int __sg_alloc_table(struct sg_table *table, unsigned int nents,
264                      unsigned int max_ents, struct scatterlist *first_chunk,
265                      gfp_t gfp_mask, sg_alloc_fn *alloc_fn)
266 {
267         struct scatterlist *sg, *prv;
268         unsigned int left;
269 
270         memset(table, 0, sizeof(*table));
271 
272         if (nents == 0)
273                 return -EINVAL;
274 #ifdef CONFIG_ARCH_NO_SG_CHAIN
275         if (WARN_ON_ONCE(nents > max_ents))
276                 return -EINVAL;
277 #endif
278 
279         left = nents;
280         prv = NULL;
281         do {
282                 unsigned int sg_size, alloc_size = left;
283 
284                 if (alloc_size > max_ents) {
285                         alloc_size = max_ents;
286                         sg_size = alloc_size - 1;
287                 } else
288                         sg_size = alloc_size;
289 
290                 left -= sg_size;
291 
292                 if (first_chunk) {
293                         sg = first_chunk;
294                         first_chunk = NULL;
295                 } else {
296                         sg = alloc_fn(alloc_size, gfp_mask);
297                 }
298                 if (unlikely(!sg)) {
299                         /*
300                          * Adjust entry count to reflect that the last
301                          * entry of the previous table won't be used for
302                          * linkage.  Without this, sg_kfree() may get
303                          * confused.
304                          */
305                         if (prv)
306                                 table->nents = ++table->orig_nents;
307 
308                         return -ENOMEM;
309                 }
310 
311                 sg_init_table(sg, alloc_size);
312                 table->nents = table->orig_nents += sg_size;
313 
314                 /*
315                  * If this is the first mapping, assign the sg table header.
316                  * If this is not the first mapping, chain previous part.
317                  */
318                 if (prv)
319                         sg_chain(prv, max_ents, sg);
320                 else
321                         table->sgl = sg;
322 
323                 /*
324                  * If no more entries after this one, mark the end
325                  */
326                 if (!left)
327                         sg_mark_end(&sg[sg_size - 1]);
328 
329                 prv = sg;
330         } while (left);
331 
332         return 0;
333 }
334 EXPORT_SYMBOL(__sg_alloc_table);
335 
336 /**
337  * sg_alloc_table - Allocate and initialize an sg table
338  * @table:      The sg table header to use
339  * @nents:      Number of entries in sg list
340  * @gfp_mask:   GFP allocation mask
341  *
342  *  Description:
343  *    Allocate and initialize an sg table. If @nents@ is larger than
344  *    SG_MAX_SINGLE_ALLOC a chained sg table will be setup.
345  *
346  **/
347 int sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask)
348 {
349         int ret;
350 
351         ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC,
352                                NULL, gfp_mask, sg_kmalloc);
353         if (unlikely(ret))
354                 __sg_free_table(table, SG_MAX_SINGLE_ALLOC, false, sg_kfree);
355 
356         return ret;
357 }
358 EXPORT_SYMBOL(sg_alloc_table);
359 
360 /**
361  * __sg_alloc_table_from_pages - Allocate and initialize an sg table from
362  *                               an array of pages
363  * @sgt:         The sg table header to use
364  * @pages:       Pointer to an array of page pointers
365  * @n_pages:     Number of pages in the pages array
366  * @offset:      Offset from start of the first page to the start of a buffer
367  * @size:        Number of valid bytes in the buffer (after offset)
368  * @max_segment: Maximum size of a scatterlist node in bytes (page aligned)
369  * @gfp_mask:    GFP allocation mask
370  *
371  *  Description:
372  *    Allocate and initialize an sg table from a list of pages. Contiguous
373  *    ranges of the pages are squashed into a single scatterlist node up to the
374  *    maximum size specified in @max_segment. An user may provide an offset at a
375  *    start and a size of valid data in a buffer specified by the page array.
376  *    The returned sg table is released by sg_free_table.
377  *
378  * Returns:
379  *   0 on success, negative error on failure
380  */
381 int __sg_alloc_table_from_pages(struct sg_table *sgt, struct page **pages,
382                                 unsigned int n_pages, unsigned int offset,
383                                 unsigned long size, unsigned int max_segment,
384                                 gfp_t gfp_mask)
385 {
386         unsigned int chunks, cur_page, seg_len, i;
387         int ret;
388         struct scatterlist *s;
389 
390         if (WARN_ON(!max_segment || offset_in_page(max_segment)))
391                 return -EINVAL;
392 
393         /* compute number of contiguous chunks */
394         chunks = 1;
395         seg_len = 0;
396         for (i = 1; i < n_pages; i++) {
397                 seg_len += PAGE_SIZE;
398                 if (seg_len >= max_segment ||
399                     page_to_pfn(pages[i]) != page_to_pfn(pages[i - 1]) + 1) {
400                         chunks++;
401                         seg_len = 0;
402                 }
403         }
404 
405         ret = sg_alloc_table(sgt, chunks, gfp_mask);
406         if (unlikely(ret))
407                 return ret;
408 
409         /* merging chunks and putting them into the scatterlist */
410         cur_page = 0;
411         for_each_sg(sgt->sgl, s, sgt->orig_nents, i) {
412                 unsigned int j, chunk_size;
413 
414                 /* look for the end of the current chunk */
415                 seg_len = 0;
416                 for (j = cur_page + 1; j < n_pages; j++) {
417                         seg_len += PAGE_SIZE;
418                         if (seg_len >= max_segment ||
419                             page_to_pfn(pages[j]) !=
420                             page_to_pfn(pages[j - 1]) + 1)
421                                 break;
422                 }
423 
424                 chunk_size = ((j - cur_page) << PAGE_SHIFT) - offset;
425                 sg_set_page(s, pages[cur_page],
426                             min_t(unsigned long, size, chunk_size), offset);
427                 size -= chunk_size;
428                 offset = 0;
429                 cur_page = j;
430         }
431 
432         return 0;
433 }
434 EXPORT_SYMBOL(__sg_alloc_table_from_pages);
435 
436 /**
437  * sg_alloc_table_from_pages - Allocate and initialize an sg table from
438  *                             an array of pages
439  * @sgt:         The sg table header to use
440  * @pages:       Pointer to an array of page pointers
441  * @n_pages:     Number of pages in the pages array
442  * @offset:      Offset from start of the first page to the start of a buffer
443  * @size:        Number of valid bytes in the buffer (after offset)
444  * @gfp_mask:    GFP allocation mask
445  *
446  *  Description:
447  *    Allocate and initialize an sg table from a list of pages. Contiguous
448  *    ranges of the pages are squashed into a single scatterlist node. A user
449  *    may provide an offset at a start and a size of valid data in a buffer
450  *    specified by the page array. The returned sg table is released by
451  *    sg_free_table.
452  *
453  * Returns:
454  *   0 on success, negative error on failure
455  */
456 int sg_alloc_table_from_pages(struct sg_table *sgt, struct page **pages,
457                               unsigned int n_pages, unsigned int offset,
458                               unsigned long size, gfp_t gfp_mask)
459 {
460         return __sg_alloc_table_from_pages(sgt, pages, n_pages, offset, size,
461                                            SCATTERLIST_MAX_SEGMENT, gfp_mask);
462 }
463 EXPORT_SYMBOL(sg_alloc_table_from_pages);
464 
465 #ifdef CONFIG_SGL_ALLOC
466 
467 /**
468  * sgl_alloc_order - allocate a scatterlist and its pages
469  * @length: Length in bytes of the scatterlist. Must be at least one
470  * @order: Second argument for alloc_pages()
471  * @chainable: Whether or not to allocate an extra element in the scatterlist
472  *      for scatterlist chaining purposes
473  * @gfp: Memory allocation flags
474  * @nent_p: [out] Number of entries in the scatterlist that have pages
475  *
476  * Returns: A pointer to an initialized scatterlist or %NULL upon failure.
477  */
478 struct scatterlist *sgl_alloc_order(unsigned long long length,
479                                     unsigned int order, bool chainable,
480                                     gfp_t gfp, unsigned int *nent_p)
481 {
482         struct scatterlist *sgl, *sg;
483         struct page *page;
484         unsigned int nent, nalloc;
485         u32 elem_len;
486 
487         nent = round_up(length, PAGE_SIZE << order) >> (PAGE_SHIFT + order);
488         /* Check for integer overflow */
489         if (length > (nent << (PAGE_SHIFT + order)))
490                 return NULL;
491         nalloc = nent;
492         if (chainable) {
493                 /* Check for integer overflow */
494                 if (nalloc + 1 < nalloc)
495                         return NULL;
496                 nalloc++;
497         }
498         sgl = kmalloc_array(nalloc, sizeof(struct scatterlist),
499                             (gfp & ~GFP_DMA) | __GFP_ZERO);
500         if (!sgl)
501                 return NULL;
502 
503         sg_init_table(sgl, nalloc);
504         sg = sgl;
505         while (length) {
506                 elem_len = min_t(u64, length, PAGE_SIZE << order);
507                 page = alloc_pages(gfp, order);
508                 if (!page) {
509                         sgl_free(sgl);
510                         return NULL;
511                 }
512 
513                 sg_set_page(sg, page, elem_len, 0);
514                 length -= elem_len;
515                 sg = sg_next(sg);
516         }
517         WARN_ONCE(length, "length = %lld\n", length);
518         if (nent_p)
519                 *nent_p = nent;
520         return sgl;
521 }
522 EXPORT_SYMBOL(sgl_alloc_order);
523 
524 /**
525  * sgl_alloc - allocate a scatterlist and its pages
526  * @length: Length in bytes of the scatterlist
527  * @gfp: Memory allocation flags
528  * @nent_p: [out] Number of entries in the scatterlist
529  *
530  * Returns: A pointer to an initialized scatterlist or %NULL upon failure.
531  */
532 struct scatterlist *sgl_alloc(unsigned long long length, gfp_t gfp,
533                               unsigned int *nent_p)
534 {
535         return sgl_alloc_order(length, 0, false, gfp, nent_p);
536 }
537 EXPORT_SYMBOL(sgl_alloc);
538 
539 /**
540  * sgl_free_n_order - free a scatterlist and its pages
541  * @sgl: Scatterlist with one or more elements
542  * @nents: Maximum number of elements to free
543  * @order: Second argument for __free_pages()
544  *
545  * Notes:
546  * - If several scatterlists have been chained and each chain element is
547  *   freed separately then it's essential to set nents correctly to avoid that a
548  *   page would get freed twice.
549  * - All pages in a chained scatterlist can be freed at once by setting @nents
550  *   to a high number.
551  */
552 void sgl_free_n_order(struct scatterlist *sgl, int nents, int order)
553 {
554         struct scatterlist *sg;
555         struct page *page;
556         int i;
557 
558         for_each_sg(sgl, sg, nents, i) {
559                 if (!sg)
560                         break;
561                 page = sg_page(sg);
562                 if (page)
563                         __free_pages(page, order);
564         }
565         kfree(sgl);
566 }
567 EXPORT_SYMBOL(sgl_free_n_order);
568 
569 /**
570  * sgl_free_order - free a scatterlist and its pages
571  * @sgl: Scatterlist with one or more elements
572  * @order: Second argument for __free_pages()
573  */
574 void sgl_free_order(struct scatterlist *sgl, int order)
575 {
576         sgl_free_n_order(sgl, INT_MAX, order);
577 }
578 EXPORT_SYMBOL(sgl_free_order);
579 
580 /**
581  * sgl_free - free a scatterlist and its pages
582  * @sgl: Scatterlist with one or more elements
583  */
584 void sgl_free(struct scatterlist *sgl)
585 {
586         sgl_free_order(sgl, 0);
587 }
588 EXPORT_SYMBOL(sgl_free);
589 
590 #endif /* CONFIG_SGL_ALLOC */
591 
592 void __sg_page_iter_start(struct sg_page_iter *piter,
593                           struct scatterlist *sglist, unsigned int nents,
594                           unsigned long pgoffset)
595 {
596         piter->__pg_advance = 0;
597         piter->__nents = nents;
598 
599         piter->sg = sglist;
600         piter->sg_pgoffset = pgoffset;
601 }
602 EXPORT_SYMBOL(__sg_page_iter_start);
603 
604 static int sg_page_count(struct scatterlist *sg)
605 {
606         return PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT;
607 }
608 
609 bool __sg_page_iter_next(struct sg_page_iter *piter)
610 {
611         if (!piter->__nents || !piter->sg)
612                 return false;
613 
614         piter->sg_pgoffset += piter->__pg_advance;
615         piter->__pg_advance = 1;
616 
617         while (piter->sg_pgoffset >= sg_page_count(piter->sg)) {
618                 piter->sg_pgoffset -= sg_page_count(piter->sg);
619                 piter->sg = sg_next(piter->sg);
620                 if (!--piter->__nents || !piter->sg)
621                         return false;
622         }
623 
624         return true;
625 }
626 EXPORT_SYMBOL(__sg_page_iter_next);
627 
628 /**
629  * sg_miter_start - start mapping iteration over a sg list
630  * @miter: sg mapping iter to be started
631  * @sgl: sg list to iterate over
632  * @nents: number of sg entries
633  *
634  * Description:
635  *   Starts mapping iterator @miter.
636  *
637  * Context:
638  *   Don't care.
639  */
640 void sg_miter_start(struct sg_mapping_iter *miter, struct scatterlist *sgl,
641                     unsigned int nents, unsigned int flags)
642 {
643         memset(miter, 0, sizeof(struct sg_mapping_iter));
644 
645         __sg_page_iter_start(&miter->piter, sgl, nents, 0);
646         WARN_ON(!(flags & (SG_MITER_TO_SG | SG_MITER_FROM_SG)));
647         miter->__flags = flags;
648 }
649 EXPORT_SYMBOL(sg_miter_start);
650 
651 static bool sg_miter_get_next_page(struct sg_mapping_iter *miter)
652 {
653         if (!miter->__remaining) {
654                 struct scatterlist *sg;
655                 unsigned long pgoffset;
656 
657                 if (!__sg_page_iter_next(&miter->piter))
658                         return false;
659 
660                 sg = miter->piter.sg;
661                 pgoffset = miter->piter.sg_pgoffset;
662 
663                 miter->__offset = pgoffset ? 0 : sg->offset;
664                 miter->__remaining = sg->offset + sg->length -
665                                 (pgoffset << PAGE_SHIFT) - miter->__offset;
666                 miter->__remaining = min_t(unsigned long, miter->__remaining,
667                                            PAGE_SIZE - miter->__offset);
668         }
669 
670         return true;
671 }
672 
673 /**
674  * sg_miter_skip - reposition mapping iterator
675  * @miter: sg mapping iter to be skipped
676  * @offset: number of bytes to plus the current location
677  *
678  * Description:
679  *   Sets the offset of @miter to its current location plus @offset bytes.
680  *   If mapping iterator @miter has been proceeded by sg_miter_next(), this
681  *   stops @miter.
682  *
683  * Context:
684  *   Don't care if @miter is stopped, or not proceeded yet.
685  *   Otherwise, preemption disabled if the SG_MITER_ATOMIC is set.
686  *
687  * Returns:
688  *   true if @miter contains the valid mapping.  false if end of sg
689  *   list is reached.
690  */
691 bool sg_miter_skip(struct sg_mapping_iter *miter, off_t offset)
692 {
693         sg_miter_stop(miter);
694 
695         while (offset) {
696                 off_t consumed;
697 
698                 if (!sg_miter_get_next_page(miter))
699                         return false;
700 
701                 consumed = min_t(off_t, offset, miter->__remaining);
702                 miter->__offset += consumed;
703                 miter->__remaining -= consumed;
704                 offset -= consumed;
705         }
706 
707         return true;
708 }
709 EXPORT_SYMBOL(sg_miter_skip);
710 
711 /**
712  * sg_miter_next - proceed mapping iterator to the next mapping
713  * @miter: sg mapping iter to proceed
714  *
715  * Description:
716  *   Proceeds @miter to the next mapping.  @miter should have been started
717  *   using sg_miter_start().  On successful return, @miter->page,
718  *   @miter->addr and @miter->length point to the current mapping.
719  *
720  * Context:
721  *   Preemption disabled if SG_MITER_ATOMIC.  Preemption must stay disabled
722  *   till @miter is stopped.  May sleep if !SG_MITER_ATOMIC.
723  *
724  * Returns:
725  *   true if @miter contains the next mapping.  false if end of sg
726  *   list is reached.
727  */
728 bool sg_miter_next(struct sg_mapping_iter *miter)
729 {
730         sg_miter_stop(miter);
731 
732         /*
733          * Get to the next page if necessary.
734          * __remaining, __offset is adjusted by sg_miter_stop
735          */
736         if (!sg_miter_get_next_page(miter))
737                 return false;
738 
739         miter->page = sg_page_iter_page(&miter->piter);
740         miter->consumed = miter->length = miter->__remaining;
741 
742         if (miter->__flags & SG_MITER_ATOMIC)
743                 miter->addr = kmap_atomic(miter->page) + miter->__offset;
744         else
745                 miter->addr = kmap(miter->page) + miter->__offset;
746 
747         return true;
748 }
749 EXPORT_SYMBOL(sg_miter_next);
750 
751 /**
752  * sg_miter_stop - stop mapping iteration
753  * @miter: sg mapping iter to be stopped
754  *
755  * Description:
756  *   Stops mapping iterator @miter.  @miter should have been started
757  *   using sg_miter_start().  A stopped iteration can be resumed by
758  *   calling sg_miter_next() on it.  This is useful when resources (kmap)
759  *   need to be released during iteration.
760  *
761  * Context:
762  *   Preemption disabled if the SG_MITER_ATOMIC is set.  Don't care
763  *   otherwise.
764  */
765 void sg_miter_stop(struct sg_mapping_iter *miter)
766 {
767         WARN_ON(miter->consumed > miter->length);
768 
769         /* drop resources from the last iteration */
770         if (miter->addr) {
771                 miter->__offset += miter->consumed;
772                 miter->__remaining -= miter->consumed;
773 
774                 if ((miter->__flags & SG_MITER_TO_SG) &&
775                     !PageSlab(miter->page))
776                         flush_kernel_dcache_page(miter->page);
777 
778                 if (miter->__flags & SG_MITER_ATOMIC) {
779                         WARN_ON_ONCE(preemptible());
780                         kunmap_atomic(miter->addr);
781                 } else
782                         kunmap(miter->page);
783 
784                 miter->page = NULL;
785                 miter->addr = NULL;
786                 miter->length = 0;
787                 miter->consumed = 0;
788         }
789 }
790 EXPORT_SYMBOL(sg_miter_stop);
791 
792 /**
793  * sg_copy_buffer - Copy data between a linear buffer and an SG list
794  * @sgl:                 The SG list
795  * @nents:               Number of SG entries
796  * @buf:                 Where to copy from
797  * @buflen:              The number of bytes to copy
798  * @skip:                Number of bytes to skip before copying
799  * @to_buffer:           transfer direction (true == from an sg list to a
800  *                       buffer, false == from a buffer to an sg list
801  *
802  * Returns the number of copied bytes.
803  *
804  **/
805 size_t sg_copy_buffer(struct scatterlist *sgl, unsigned int nents, void *buf,
806                       size_t buflen, off_t skip, bool to_buffer)
807 {
808         unsigned int offset = 0;
809         struct sg_mapping_iter miter;
810         unsigned int sg_flags = SG_MITER_ATOMIC;
811 
812         if (to_buffer)
813                 sg_flags |= SG_MITER_FROM_SG;
814         else
815                 sg_flags |= SG_MITER_TO_SG;
816 
817         sg_miter_start(&miter, sgl, nents, sg_flags);
818 
819         if (!sg_miter_skip(&miter, skip))
820                 return false;
821 
822         while ((offset < buflen) && sg_miter_next(&miter)) {
823                 unsigned int len;
824 
825                 len = min(miter.length, buflen - offset);
826 
827                 if (to_buffer)
828                         memcpy(buf + offset, miter.addr, len);
829                 else
830                         memcpy(miter.addr, buf + offset, len);
831 
832                 offset += len;
833         }
834 
835         sg_miter_stop(&miter);
836 
837         return offset;
838 }
839 EXPORT_SYMBOL(sg_copy_buffer);
840 
841 /**
842  * sg_copy_from_buffer - Copy from a linear buffer to an SG list
843  * @sgl:                 The SG list
844  * @nents:               Number of SG entries
845  * @buf:                 Where to copy from
846  * @buflen:              The number of bytes to copy
847  *
848  * Returns the number of copied bytes.
849  *
850  **/
851 size_t sg_copy_from_buffer(struct scatterlist *sgl, unsigned int nents,
852                            const void *buf, size_t buflen)
853 {
854         return sg_copy_buffer(sgl, nents, (void *)buf, buflen, 0, false);
855 }
856 EXPORT_SYMBOL(sg_copy_from_buffer);
857 
858 /**
859  * sg_copy_to_buffer - Copy from an SG list to a linear buffer
860  * @sgl:                 The SG list
861  * @nents:               Number of SG entries
862  * @buf:                 Where to copy to
863  * @buflen:              The number of bytes to copy
864  *
865  * Returns the number of copied bytes.
866  *
867  **/
868 size_t sg_copy_to_buffer(struct scatterlist *sgl, unsigned int nents,
869                          void *buf, size_t buflen)
870 {
871         return sg_copy_buffer(sgl, nents, buf, buflen, 0, true);
872 }
873 EXPORT_SYMBOL(sg_copy_to_buffer);
874 
875 /**
876  * sg_pcopy_from_buffer - Copy from a linear buffer to an SG list
877  * @sgl:                 The SG list
878  * @nents:               Number of SG entries
879  * @buf:                 Where to copy from
880  * @buflen:              The number of bytes to copy
881  * @skip:                Number of bytes to skip before copying
882  *
883  * Returns the number of copied bytes.
884  *
885  **/
886 size_t sg_pcopy_from_buffer(struct scatterlist *sgl, unsigned int nents,
887                             const void *buf, size_t buflen, off_t skip)
888 {
889         return sg_copy_buffer(sgl, nents, (void *)buf, buflen, skip, false);
890 }
891 EXPORT_SYMBOL(sg_pcopy_from_buffer);
892 
893 /**
894  * sg_pcopy_to_buffer - Copy from an SG list to a linear buffer
895  * @sgl:                 The SG list
896  * @nents:               Number of SG entries
897  * @buf:                 Where to copy to
898  * @buflen:              The number of bytes to copy
899  * @skip:                Number of bytes to skip before copying
900  *
901  * Returns the number of copied bytes.
902  *
903  **/
904 size_t sg_pcopy_to_buffer(struct scatterlist *sgl, unsigned int nents,
905                           void *buf, size_t buflen, off_t skip)
906 {
907         return sg_copy_buffer(sgl, nents, buf, buflen, skip, true);
908 }
909 EXPORT_SYMBOL(sg_pcopy_to_buffer);
910 
911 /**
912  * sg_zero_buffer - Zero-out a part of a SG list
913  * @sgl:                 The SG list
914  * @nents:               Number of SG entries
915  * @buflen:              The number of bytes to zero out
916  * @skip:                Number of bytes to skip before zeroing
917  *
918  * Returns the number of bytes zeroed.
919  **/
920 size_t sg_zero_buffer(struct scatterlist *sgl, unsigned int nents,
921                        size_t buflen, off_t skip)
922 {
923         unsigned int offset = 0;
924         struct sg_mapping_iter miter;
925         unsigned int sg_flags = SG_MITER_ATOMIC | SG_MITER_TO_SG;
926 
927         sg_miter_start(&miter, sgl, nents, sg_flags);
928 
929         if (!sg_miter_skip(&miter, skip))
930                 return false;
931 
932         while (offset < buflen && sg_miter_next(&miter)) {
933                 unsigned int len;
934 
935                 len = min(miter.length, buflen - offset);
936                 memset(miter.addr, 0, len);
937 
938                 offset += len;
939         }
940 
941         sg_miter_stop(&miter);
942         return offset;
943 }
944 EXPORT_SYMBOL(sg_zero_buffer);
945 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp