~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/include/linux/slab.h

Version: ~ [ linux-5.16-rc3 ] ~ [ linux-5.15.5 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.82 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.162 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.218 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.256 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.291 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.293 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /* SPDX-License-Identifier: GPL-2.0 */
  2 /*
  3  * Written by Mark Hemment, 1996 (markhe@nextd.demon.co.uk).
  4  *
  5  * (C) SGI 2006, Christoph Lameter
  6  *      Cleaned up and restructured to ease the addition of alternative
  7  *      implementations of SLAB allocators.
  8  * (C) Linux Foundation 2008-2013
  9  *      Unified interface for all slab allocators
 10  */
 11 
 12 #ifndef _LINUX_SLAB_H
 13 #define _LINUX_SLAB_H
 14 
 15 #include <linux/gfp.h>
 16 #include <linux/overflow.h>
 17 #include <linux/types.h>
 18 #include <linux/workqueue.h>
 19 
 20 
 21 /*
 22  * Flags to pass to kmem_cache_create().
 23  * The ones marked DEBUG are only valid if CONFIG_DEBUG_SLAB is set.
 24  */
 25 /* DEBUG: Perform (expensive) checks on alloc/free */
 26 #define SLAB_CONSISTENCY_CHECKS ((slab_flags_t __force)0x00000100U)
 27 /* DEBUG: Red zone objs in a cache */
 28 #define SLAB_RED_ZONE           ((slab_flags_t __force)0x00000400U)
 29 /* DEBUG: Poison objects */
 30 #define SLAB_POISON             ((slab_flags_t __force)0x00000800U)
 31 /* Align objs on cache lines */
 32 #define SLAB_HWCACHE_ALIGN      ((slab_flags_t __force)0x00002000U)
 33 /* Use GFP_DMA memory */
 34 #define SLAB_CACHE_DMA          ((slab_flags_t __force)0x00004000U)
 35 /* Use GFP_DMA32 memory */
 36 #define SLAB_CACHE_DMA32        ((slab_flags_t __force)0x00008000U)
 37 /* DEBUG: Store the last owner for bug hunting */
 38 #define SLAB_STORE_USER         ((slab_flags_t __force)0x00010000U)
 39 /* Panic if kmem_cache_create() fails */
 40 #define SLAB_PANIC              ((slab_flags_t __force)0x00040000U)
 41 /*
 42  * SLAB_TYPESAFE_BY_RCU - **WARNING** READ THIS!
 43  *
 44  * This delays freeing the SLAB page by a grace period, it does _NOT_
 45  * delay object freeing. This means that if you do kmem_cache_free()
 46  * that memory location is free to be reused at any time. Thus it may
 47  * be possible to see another object there in the same RCU grace period.
 48  *
 49  * This feature only ensures the memory location backing the object
 50  * stays valid, the trick to using this is relying on an independent
 51  * object validation pass. Something like:
 52  *
 53  *  rcu_read_lock()
 54  * again:
 55  *  obj = lockless_lookup(key);
 56  *  if (obj) {
 57  *    if (!try_get_ref(obj)) // might fail for free objects
 58  *      goto again;
 59  *
 60  *    if (obj->key != key) { // not the object we expected
 61  *      put_ref(obj);
 62  *      goto again;
 63  *    }
 64  *  }
 65  *  rcu_read_unlock();
 66  *
 67  * This is useful if we need to approach a kernel structure obliquely,
 68  * from its address obtained without the usual locking. We can lock
 69  * the structure to stabilize it and check it's still at the given address,
 70  * only if we can be sure that the memory has not been meanwhile reused
 71  * for some other kind of object (which our subsystem's lock might corrupt).
 72  *
 73  * rcu_read_lock before reading the address, then rcu_read_unlock after
 74  * taking the spinlock within the structure expected at that address.
 75  *
 76  * Note that SLAB_TYPESAFE_BY_RCU was originally named SLAB_DESTROY_BY_RCU.
 77  */
 78 /* Defer freeing slabs to RCU */
 79 #define SLAB_TYPESAFE_BY_RCU    ((slab_flags_t __force)0x00080000U)
 80 /* Spread some memory over cpuset */
 81 #define SLAB_MEM_SPREAD         ((slab_flags_t __force)0x00100000U)
 82 /* Trace allocations and frees */
 83 #define SLAB_TRACE              ((slab_flags_t __force)0x00200000U)
 84 
 85 /* Flag to prevent checks on free */
 86 #ifdef CONFIG_DEBUG_OBJECTS
 87 # define SLAB_DEBUG_OBJECTS     ((slab_flags_t __force)0x00400000U)
 88 #else
 89 # define SLAB_DEBUG_OBJECTS     0
 90 #endif
 91 
 92 /* Avoid kmemleak tracing */
 93 #define SLAB_NOLEAKTRACE        ((slab_flags_t __force)0x00800000U)
 94 
 95 /* Fault injection mark */
 96 #ifdef CONFIG_FAILSLAB
 97 # define SLAB_FAILSLAB          ((slab_flags_t __force)0x02000000U)
 98 #else
 99 # define SLAB_FAILSLAB          0
100 #endif
101 /* Account to memcg */
102 #ifdef CONFIG_MEMCG_KMEM
103 # define SLAB_ACCOUNT           ((slab_flags_t __force)0x04000000U)
104 #else
105 # define SLAB_ACCOUNT           0
106 #endif
107 
108 #ifdef CONFIG_KASAN
109 #define SLAB_KASAN              ((slab_flags_t __force)0x08000000U)
110 #else
111 #define SLAB_KASAN              0
112 #endif
113 
114 /* The following flags affect the page allocator grouping pages by mobility */
115 /* Objects are reclaimable */
116 #define SLAB_RECLAIM_ACCOUNT    ((slab_flags_t __force)0x00020000U)
117 #define SLAB_TEMPORARY          SLAB_RECLAIM_ACCOUNT    /* Objects are short-lived */
118 /*
119  * ZERO_SIZE_PTR will be returned for zero sized kmalloc requests.
120  *
121  * Dereferencing ZERO_SIZE_PTR will lead to a distinct access fault.
122  *
123  * ZERO_SIZE_PTR can be passed to kfree though in the same way that NULL can.
124  * Both make kfree a no-op.
125  */
126 #define ZERO_SIZE_PTR ((void *)16)
127 
128 #define ZERO_OR_NULL_PTR(x) ((unsigned long)(x) <= \
129                                 (unsigned long)ZERO_SIZE_PTR)
130 
131 #include <linux/kasan.h>
132 
133 struct mem_cgroup;
134 /*
135  * struct kmem_cache related prototypes
136  */
137 void __init kmem_cache_init(void);
138 bool slab_is_available(void);
139 
140 extern bool usercopy_fallback;
141 
142 struct kmem_cache *kmem_cache_create(const char *name, unsigned int size,
143                         unsigned int align, slab_flags_t flags,
144                         void (*ctor)(void *));
145 struct kmem_cache *kmem_cache_create_usercopy(const char *name,
146                         unsigned int size, unsigned int align,
147                         slab_flags_t flags,
148                         unsigned int useroffset, unsigned int usersize,
149                         void (*ctor)(void *));
150 void kmem_cache_destroy(struct kmem_cache *);
151 int kmem_cache_shrink(struct kmem_cache *);
152 
153 void memcg_create_kmem_cache(struct mem_cgroup *, struct kmem_cache *);
154 void memcg_deactivate_kmem_caches(struct mem_cgroup *);
155 void memcg_destroy_kmem_caches(struct mem_cgroup *);
156 
157 /*
158  * Please use this macro to create slab caches. Simply specify the
159  * name of the structure and maybe some flags that are listed above.
160  *
161  * The alignment of the struct determines object alignment. If you
162  * f.e. add ____cacheline_aligned_in_smp to the struct declaration
163  * then the objects will be properly aligned in SMP configurations.
164  */
165 #define KMEM_CACHE(__struct, __flags)                                   \
166                 kmem_cache_create(#__struct, sizeof(struct __struct),   \
167                         __alignof__(struct __struct), (__flags), NULL)
168 
169 /*
170  * To whitelist a single field for copying to/from usercopy, use this
171  * macro instead for KMEM_CACHE() above.
172  */
173 #define KMEM_CACHE_USERCOPY(__struct, __flags, __field)                 \
174                 kmem_cache_create_usercopy(#__struct,                   \
175                         sizeof(struct __struct),                        \
176                         __alignof__(struct __struct), (__flags),        \
177                         offsetof(struct __struct, __field),             \
178                         sizeof_field(struct __struct, __field), NULL)
179 
180 /*
181  * Common kmalloc functions provided by all allocators
182  */
183 void * __must_check __krealloc(const void *, size_t, gfp_t);
184 void * __must_check krealloc(const void *, size_t, gfp_t);
185 void kfree(const void *);
186 void kzfree(const void *);
187 size_t ksize(const void *);
188 
189 #ifdef CONFIG_HAVE_HARDENED_USERCOPY_ALLOCATOR
190 void __check_heap_object(const void *ptr, unsigned long n, struct page *page,
191                         bool to_user);
192 #else
193 static inline void __check_heap_object(const void *ptr, unsigned long n,
194                                        struct page *page, bool to_user) { }
195 #endif
196 
197 /*
198  * Some archs want to perform DMA into kmalloc caches and need a guaranteed
199  * alignment larger than the alignment of a 64-bit integer.
200  * Setting ARCH_KMALLOC_MINALIGN in arch headers allows that.
201  */
202 #if defined(ARCH_DMA_MINALIGN) && ARCH_DMA_MINALIGN > 8
203 #define ARCH_KMALLOC_MINALIGN ARCH_DMA_MINALIGN
204 #define KMALLOC_MIN_SIZE ARCH_DMA_MINALIGN
205 #define KMALLOC_SHIFT_LOW ilog2(ARCH_DMA_MINALIGN)
206 #else
207 #define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long)
208 #endif
209 
210 /*
211  * Setting ARCH_SLAB_MINALIGN in arch headers allows a different alignment.
212  * Intended for arches that get misalignment faults even for 64 bit integer
213  * aligned buffers.
214  */
215 #ifndef ARCH_SLAB_MINALIGN
216 #define ARCH_SLAB_MINALIGN __alignof__(unsigned long long)
217 #endif
218 
219 /*
220  * kmalloc and friends return ARCH_KMALLOC_MINALIGN aligned
221  * pointers. kmem_cache_alloc and friends return ARCH_SLAB_MINALIGN
222  * aligned pointers.
223  */
224 #define __assume_kmalloc_alignment __assume_aligned(ARCH_KMALLOC_MINALIGN)
225 #define __assume_slab_alignment __assume_aligned(ARCH_SLAB_MINALIGN)
226 #define __assume_page_alignment __assume_aligned(PAGE_SIZE)
227 
228 /*
229  * Kmalloc array related definitions
230  */
231 
232 #ifdef CONFIG_SLAB
233 /*
234  * The largest kmalloc size supported by the SLAB allocators is
235  * 32 megabyte (2^25) or the maximum allocatable page order if that is
236  * less than 32 MB.
237  *
238  * WARNING: Its not easy to increase this value since the allocators have
239  * to do various tricks to work around compiler limitations in order to
240  * ensure proper constant folding.
241  */
242 #define KMALLOC_SHIFT_HIGH      ((MAX_ORDER + PAGE_SHIFT - 1) <= 25 ? \
243                                 (MAX_ORDER + PAGE_SHIFT - 1) : 25)
244 #define KMALLOC_SHIFT_MAX       KMALLOC_SHIFT_HIGH
245 #ifndef KMALLOC_SHIFT_LOW
246 #define KMALLOC_SHIFT_LOW       5
247 #endif
248 #endif
249 
250 #ifdef CONFIG_SLUB
251 /*
252  * SLUB directly allocates requests fitting in to an order-1 page
253  * (PAGE_SIZE*2).  Larger requests are passed to the page allocator.
254  */
255 #define KMALLOC_SHIFT_HIGH      (PAGE_SHIFT + 1)
256 #define KMALLOC_SHIFT_MAX       (MAX_ORDER + PAGE_SHIFT - 1)
257 #ifndef KMALLOC_SHIFT_LOW
258 #define KMALLOC_SHIFT_LOW       3
259 #endif
260 #endif
261 
262 #ifdef CONFIG_SLOB
263 /*
264  * SLOB passes all requests larger than one page to the page allocator.
265  * No kmalloc array is necessary since objects of different sizes can
266  * be allocated from the same page.
267  */
268 #define KMALLOC_SHIFT_HIGH      PAGE_SHIFT
269 #define KMALLOC_SHIFT_MAX       (MAX_ORDER + PAGE_SHIFT - 1)
270 #ifndef KMALLOC_SHIFT_LOW
271 #define KMALLOC_SHIFT_LOW       3
272 #endif
273 #endif
274 
275 /* Maximum allocatable size */
276 #define KMALLOC_MAX_SIZE        (1UL << KMALLOC_SHIFT_MAX)
277 /* Maximum size for which we actually use a slab cache */
278 #define KMALLOC_MAX_CACHE_SIZE  (1UL << KMALLOC_SHIFT_HIGH)
279 /* Maximum order allocatable via the slab allocagtor */
280 #define KMALLOC_MAX_ORDER       (KMALLOC_SHIFT_MAX - PAGE_SHIFT)
281 
282 /*
283  * Kmalloc subsystem.
284  */
285 #ifndef KMALLOC_MIN_SIZE
286 #define KMALLOC_MIN_SIZE (1 << KMALLOC_SHIFT_LOW)
287 #endif
288 
289 /*
290  * This restriction comes from byte sized index implementation.
291  * Page size is normally 2^12 bytes and, in this case, if we want to use
292  * byte sized index which can represent 2^8 entries, the size of the object
293  * should be equal or greater to 2^12 / 2^8 = 2^4 = 16.
294  * If minimum size of kmalloc is less than 16, we use it as minimum object
295  * size and give up to use byte sized index.
296  */
297 #define SLAB_OBJ_MIN_SIZE      (KMALLOC_MIN_SIZE < 16 ? \
298                                (KMALLOC_MIN_SIZE) : 16)
299 
300 /*
301  * Whenever changing this, take care of that kmalloc_type() and
302  * create_kmalloc_caches() still work as intended.
303  */
304 enum kmalloc_cache_type {
305         KMALLOC_NORMAL = 0,
306         KMALLOC_RECLAIM,
307 #ifdef CONFIG_ZONE_DMA
308         KMALLOC_DMA,
309 #endif
310         NR_KMALLOC_TYPES
311 };
312 
313 #ifndef CONFIG_SLOB
314 extern struct kmem_cache *
315 kmalloc_caches[NR_KMALLOC_TYPES][KMALLOC_SHIFT_HIGH + 1];
316 
317 static __always_inline enum kmalloc_cache_type kmalloc_type(gfp_t flags)
318 {
319 #ifdef CONFIG_ZONE_DMA
320         /*
321          * The most common case is KMALLOC_NORMAL, so test for it
322          * with a single branch for both flags.
323          */
324         if (likely((flags & (__GFP_DMA | __GFP_RECLAIMABLE)) == 0))
325                 return KMALLOC_NORMAL;
326 
327         /*
328          * At least one of the flags has to be set. If both are, __GFP_DMA
329          * is more important.
330          */
331         return flags & __GFP_DMA ? KMALLOC_DMA : KMALLOC_RECLAIM;
332 #else
333         return flags & __GFP_RECLAIMABLE ? KMALLOC_RECLAIM : KMALLOC_NORMAL;
334 #endif
335 }
336 
337 /*
338  * Figure out which kmalloc slab an allocation of a certain size
339  * belongs to.
340  * 0 = zero alloc
341  * 1 =  65 .. 96 bytes
342  * 2 = 129 .. 192 bytes
343  * n = 2^(n-1)+1 .. 2^n
344  */
345 static __always_inline unsigned int kmalloc_index(size_t size)
346 {
347         if (!size)
348                 return 0;
349 
350         if (size <= KMALLOC_MIN_SIZE)
351                 return KMALLOC_SHIFT_LOW;
352 
353         if (KMALLOC_MIN_SIZE <= 32 && size > 64 && size <= 96)
354                 return 1;
355         if (KMALLOC_MIN_SIZE <= 64 && size > 128 && size <= 192)
356                 return 2;
357         if (size <=          8) return 3;
358         if (size <=         16) return 4;
359         if (size <=         32) return 5;
360         if (size <=         64) return 6;
361         if (size <=        128) return 7;
362         if (size <=        256) return 8;
363         if (size <=        512) return 9;
364         if (size <=       1024) return 10;
365         if (size <=   2 * 1024) return 11;
366         if (size <=   4 * 1024) return 12;
367         if (size <=   8 * 1024) return 13;
368         if (size <=  16 * 1024) return 14;
369         if (size <=  32 * 1024) return 15;
370         if (size <=  64 * 1024) return 16;
371         if (size <= 128 * 1024) return 17;
372         if (size <= 256 * 1024) return 18;
373         if (size <= 512 * 1024) return 19;
374         if (size <= 1024 * 1024) return 20;
375         if (size <=  2 * 1024 * 1024) return 21;
376         if (size <=  4 * 1024 * 1024) return 22;
377         if (size <=  8 * 1024 * 1024) return 23;
378         if (size <=  16 * 1024 * 1024) return 24;
379         if (size <=  32 * 1024 * 1024) return 25;
380         if (size <=  64 * 1024 * 1024) return 26;
381         BUG();
382 
383         /* Will never be reached. Needed because the compiler may complain */
384         return -1;
385 }
386 #endif /* !CONFIG_SLOB */
387 
388 void *__kmalloc(size_t size, gfp_t flags) __assume_kmalloc_alignment __malloc;
389 void *kmem_cache_alloc(struct kmem_cache *, gfp_t flags) __assume_slab_alignment __malloc;
390 void kmem_cache_free(struct kmem_cache *, void *);
391 
392 /*
393  * Bulk allocation and freeing operations. These are accelerated in an
394  * allocator specific way to avoid taking locks repeatedly or building
395  * metadata structures unnecessarily.
396  *
397  * Note that interrupts must be enabled when calling these functions.
398  */
399 void kmem_cache_free_bulk(struct kmem_cache *, size_t, void **);
400 int kmem_cache_alloc_bulk(struct kmem_cache *, gfp_t, size_t, void **);
401 
402 /*
403  * Caller must not use kfree_bulk() on memory not originally allocated
404  * by kmalloc(), because the SLOB allocator cannot handle this.
405  */
406 static __always_inline void kfree_bulk(size_t size, void **p)
407 {
408         kmem_cache_free_bulk(NULL, size, p);
409 }
410 
411 #ifdef CONFIG_NUMA
412 void *__kmalloc_node(size_t size, gfp_t flags, int node) __assume_kmalloc_alignment __malloc;
413 void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node) __assume_slab_alignment __malloc;
414 #else
415 static __always_inline void *__kmalloc_node(size_t size, gfp_t flags, int node)
416 {
417         return __kmalloc(size, flags);
418 }
419 
420 static __always_inline void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t flags, int node)
421 {
422         return kmem_cache_alloc(s, flags);
423 }
424 #endif
425 
426 #ifdef CONFIG_TRACING
427 extern void *kmem_cache_alloc_trace(struct kmem_cache *, gfp_t, size_t) __assume_slab_alignment __malloc;
428 
429 #ifdef CONFIG_NUMA
430 extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
431                                            gfp_t gfpflags,
432                                            int node, size_t size) __assume_slab_alignment __malloc;
433 #else
434 static __always_inline void *
435 kmem_cache_alloc_node_trace(struct kmem_cache *s,
436                               gfp_t gfpflags,
437                               int node, size_t size)
438 {
439         return kmem_cache_alloc_trace(s, gfpflags, size);
440 }
441 #endif /* CONFIG_NUMA */
442 
443 #else /* CONFIG_TRACING */
444 static __always_inline void *kmem_cache_alloc_trace(struct kmem_cache *s,
445                 gfp_t flags, size_t size)
446 {
447         void *ret = kmem_cache_alloc(s, flags);
448 
449         ret = kasan_kmalloc(s, ret, size, flags);
450         return ret;
451 }
452 
453 static __always_inline void *
454 kmem_cache_alloc_node_trace(struct kmem_cache *s,
455                               gfp_t gfpflags,
456                               int node, size_t size)
457 {
458         void *ret = kmem_cache_alloc_node(s, gfpflags, node);
459 
460         ret = kasan_kmalloc(s, ret, size, gfpflags);
461         return ret;
462 }
463 #endif /* CONFIG_TRACING */
464 
465 extern void *kmalloc_order(size_t size, gfp_t flags, unsigned int order) __assume_page_alignment __malloc;
466 
467 #ifdef CONFIG_TRACING
468 extern void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order) __assume_page_alignment __malloc;
469 #else
470 static __always_inline void *
471 kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order)
472 {
473         return kmalloc_order(size, flags, order);
474 }
475 #endif
476 
477 static __always_inline void *kmalloc_large(size_t size, gfp_t flags)
478 {
479         unsigned int order = get_order(size);
480         return kmalloc_order_trace(size, flags, order);
481 }
482 
483 /**
484  * kmalloc - allocate memory
485  * @size: how many bytes of memory are required.
486  * @flags: the type of memory to allocate.
487  *
488  * kmalloc is the normal method of allocating memory
489  * for objects smaller than page size in the kernel.
490  *
491  * The @flags argument may be one of the GFP flags defined at
492  * include/linux/gfp.h and described at
493  * :ref:`Documentation/core-api/mm-api.rst <mm-api-gfp-flags>`
494  *
495  * The recommended usage of the @flags is described at
496  * :ref:`Documentation/core-api/memory-allocation.rst <memory-allocation>`
497  *
498  * Below is a brief outline of the most useful GFP flags
499  *
500  * %GFP_KERNEL
501  *      Allocate normal kernel ram. May sleep.
502  *
503  * %GFP_NOWAIT
504  *      Allocation will not sleep.
505  *
506  * %GFP_ATOMIC
507  *      Allocation will not sleep.  May use emergency pools.
508  *
509  * %GFP_HIGHUSER
510  *      Allocate memory from high memory on behalf of user.
511  *
512  * Also it is possible to set different flags by OR'ing
513  * in one or more of the following additional @flags:
514  *
515  * %__GFP_HIGH
516  *      This allocation has high priority and may use emergency pools.
517  *
518  * %__GFP_NOFAIL
519  *      Indicate that this allocation is in no way allowed to fail
520  *      (think twice before using).
521  *
522  * %__GFP_NORETRY
523  *      If memory is not immediately available,
524  *      then give up at once.
525  *
526  * %__GFP_NOWARN
527  *      If allocation fails, don't issue any warnings.
528  *
529  * %__GFP_RETRY_MAYFAIL
530  *      Try really hard to succeed the allocation but fail
531  *      eventually.
532  */
533 static __always_inline void *kmalloc(size_t size, gfp_t flags)
534 {
535         if (__builtin_constant_p(size)) {
536 #ifndef CONFIG_SLOB
537                 unsigned int index;
538 #endif
539                 if (size > KMALLOC_MAX_CACHE_SIZE)
540                         return kmalloc_large(size, flags);
541 #ifndef CONFIG_SLOB
542                 index = kmalloc_index(size);
543 
544                 if (!index)
545                         return ZERO_SIZE_PTR;
546 
547                 return kmem_cache_alloc_trace(
548                                 kmalloc_caches[kmalloc_type(flags)][index],
549                                 flags, size);
550 #endif
551         }
552         return __kmalloc(size, flags);
553 }
554 
555 /*
556  * Determine size used for the nth kmalloc cache.
557  * return size or 0 if a kmalloc cache for that
558  * size does not exist
559  */
560 static __always_inline unsigned int kmalloc_size(unsigned int n)
561 {
562 #ifndef CONFIG_SLOB
563         if (n > 2)
564                 return 1U << n;
565 
566         if (n == 1 && KMALLOC_MIN_SIZE <= 32)
567                 return 96;
568 
569         if (n == 2 && KMALLOC_MIN_SIZE <= 64)
570                 return 192;
571 #endif
572         return 0;
573 }
574 
575 static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
576 {
577 #ifndef CONFIG_SLOB
578         if (__builtin_constant_p(size) &&
579                 size <= KMALLOC_MAX_CACHE_SIZE) {
580                 unsigned int i = kmalloc_index(size);
581 
582                 if (!i)
583                         return ZERO_SIZE_PTR;
584 
585                 return kmem_cache_alloc_node_trace(
586                                 kmalloc_caches[kmalloc_type(flags)][i],
587                                                 flags, node, size);
588         }
589 #endif
590         return __kmalloc_node(size, flags, node);
591 }
592 
593 struct memcg_cache_array {
594         struct rcu_head rcu;
595         struct kmem_cache *entries[0];
596 };
597 
598 /*
599  * This is the main placeholder for memcg-related information in kmem caches.
600  * Both the root cache and the child caches will have it. For the root cache,
601  * this will hold a dynamically allocated array large enough to hold
602  * information about the currently limited memcgs in the system. To allow the
603  * array to be accessed without taking any locks, on relocation we free the old
604  * version only after a grace period.
605  *
606  * Root and child caches hold different metadata.
607  *
608  * @root_cache: Common to root and child caches.  NULL for root, pointer to
609  *              the root cache for children.
610  *
611  * The following fields are specific to root caches.
612  *
613  * @memcg_caches: kmemcg ID indexed table of child caches.  This table is
614  *              used to index child cachces during allocation and cleared
615  *              early during shutdown.
616  *
617  * @root_caches_node: List node for slab_root_caches list.
618  *
619  * @children:   List of all child caches.  While the child caches are also
620  *              reachable through @memcg_caches, a child cache remains on
621  *              this list until it is actually destroyed.
622  *
623  * The following fields are specific to child caches.
624  *
625  * @memcg:      Pointer to the memcg this cache belongs to.
626  *
627  * @children_node: List node for @root_cache->children list.
628  *
629  * @kmem_caches_node: List node for @memcg->kmem_caches list.
630  */
631 struct memcg_cache_params {
632         struct kmem_cache *root_cache;
633         union {
634                 struct {
635                         struct memcg_cache_array __rcu *memcg_caches;
636                         struct list_head __root_caches_node;
637                         struct list_head children;
638                         bool dying;
639                 };
640                 struct {
641                         struct mem_cgroup *memcg;
642                         struct list_head children_node;
643                         struct list_head kmem_caches_node;
644 
645                         void (*deact_fn)(struct kmem_cache *);
646                         union {
647                                 struct rcu_head deact_rcu_head;
648                                 struct work_struct deact_work;
649                         };
650                 };
651         };
652 };
653 
654 int memcg_update_all_caches(int num_memcgs);
655 
656 /**
657  * kmalloc_array - allocate memory for an array.
658  * @n: number of elements.
659  * @size: element size.
660  * @flags: the type of memory to allocate (see kmalloc).
661  */
662 static inline void *kmalloc_array(size_t n, size_t size, gfp_t flags)
663 {
664         size_t bytes;
665 
666         if (unlikely(check_mul_overflow(n, size, &bytes)))
667                 return NULL;
668         if (__builtin_constant_p(n) && __builtin_constant_p(size))
669                 return kmalloc(bytes, flags);
670         return __kmalloc(bytes, flags);
671 }
672 
673 /**
674  * kcalloc - allocate memory for an array. The memory is set to zero.
675  * @n: number of elements.
676  * @size: element size.
677  * @flags: the type of memory to allocate (see kmalloc).
678  */
679 static inline void *kcalloc(size_t n, size_t size, gfp_t flags)
680 {
681         return kmalloc_array(n, size, flags | __GFP_ZERO);
682 }
683 
684 /*
685  * kmalloc_track_caller is a special version of kmalloc that records the
686  * calling function of the routine calling it for slab leak tracking instead
687  * of just the calling function (confusing, eh?).
688  * It's useful when the call to kmalloc comes from a widely-used standard
689  * allocator where we care about the real place the memory allocation
690  * request comes from.
691  */
692 extern void *__kmalloc_track_caller(size_t, gfp_t, unsigned long);
693 #define kmalloc_track_caller(size, flags) \
694         __kmalloc_track_caller(size, flags, _RET_IP_)
695 
696 static inline void *kmalloc_array_node(size_t n, size_t size, gfp_t flags,
697                                        int node)
698 {
699         size_t bytes;
700 
701         if (unlikely(check_mul_overflow(n, size, &bytes)))
702                 return NULL;
703         if (__builtin_constant_p(n) && __builtin_constant_p(size))
704                 return kmalloc_node(bytes, flags, node);
705         return __kmalloc_node(bytes, flags, node);
706 }
707 
708 static inline void *kcalloc_node(size_t n, size_t size, gfp_t flags, int node)
709 {
710         return kmalloc_array_node(n, size, flags | __GFP_ZERO, node);
711 }
712 
713 
714 #ifdef CONFIG_NUMA
715 extern void *__kmalloc_node_track_caller(size_t, gfp_t, int, unsigned long);
716 #define kmalloc_node_track_caller(size, flags, node) \
717         __kmalloc_node_track_caller(size, flags, node, \
718                         _RET_IP_)
719 
720 #else /* CONFIG_NUMA */
721 
722 #define kmalloc_node_track_caller(size, flags, node) \
723         kmalloc_track_caller(size, flags)
724 
725 #endif /* CONFIG_NUMA */
726 
727 /*
728  * Shortcuts
729  */
730 static inline void *kmem_cache_zalloc(struct kmem_cache *k, gfp_t flags)
731 {
732         return kmem_cache_alloc(k, flags | __GFP_ZERO);
733 }
734 
735 /**
736  * kzalloc - allocate memory. The memory is set to zero.
737  * @size: how many bytes of memory are required.
738  * @flags: the type of memory to allocate (see kmalloc).
739  */
740 static inline void *kzalloc(size_t size, gfp_t flags)
741 {
742         return kmalloc(size, flags | __GFP_ZERO);
743 }
744 
745 /**
746  * kzalloc_node - allocate zeroed memory from a particular memory node.
747  * @size: how many bytes of memory are required.
748  * @flags: the type of memory to allocate (see kmalloc).
749  * @node: memory node from which to allocate
750  */
751 static inline void *kzalloc_node(size_t size, gfp_t flags, int node)
752 {
753         return kmalloc_node(size, flags | __GFP_ZERO, node);
754 }
755 
756 unsigned int kmem_cache_size(struct kmem_cache *s);
757 void __init kmem_cache_init_late(void);
758 
759 #if defined(CONFIG_SMP) && defined(CONFIG_SLAB)
760 int slab_prepare_cpu(unsigned int cpu);
761 int slab_dead_cpu(unsigned int cpu);
762 #else
763 #define slab_prepare_cpu        NULL
764 #define slab_dead_cpu           NULL
765 #endif
766 
767 #endif  /* _LINUX_SLAB_H */
768 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp