~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/mm/slab_common.c

Version: ~ [ linux-5.8 ] ~ [ linux-5.7.12 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.55 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.136 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.191 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.232 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.232 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Slab allocator functions that are independent of the allocator strategy
  3  *
  4  * (C) 2012 Christoph Lameter <cl@linux.com>
  5  */
  6 #include <linux/slab.h>
  7 
  8 #include <linux/mm.h>
  9 #include <linux/poison.h>
 10 #include <linux/interrupt.h>
 11 #include <linux/memory.h>
 12 #include <linux/compiler.h>
 13 #include <linux/module.h>
 14 #include <linux/cpu.h>
 15 #include <linux/uaccess.h>
 16 #include <linux/seq_file.h>
 17 #include <linux/proc_fs.h>
 18 #include <asm/cacheflush.h>
 19 #include <asm/tlbflush.h>
 20 #include <asm/page.h>
 21 #include <linux/memcontrol.h>
 22 
 23 #define CREATE_TRACE_POINTS
 24 #include <trace/events/kmem.h>
 25 
 26 #include "slab.h"
 27 
 28 enum slab_state slab_state;
 29 LIST_HEAD(slab_caches);
 30 DEFINE_MUTEX(slab_mutex);
 31 struct kmem_cache *kmem_cache;
 32 
 33 #ifdef CONFIG_DEBUG_VM
 34 static int kmem_cache_sanity_check(const char *name, size_t size)
 35 {
 36         struct kmem_cache *s = NULL;
 37 
 38         if (!name || in_interrupt() || size < sizeof(void *) ||
 39                 size > KMALLOC_MAX_SIZE) {
 40                 pr_err("kmem_cache_create(%s) integrity check failed\n", name);
 41                 return -EINVAL;
 42         }
 43 
 44         list_for_each_entry(s, &slab_caches, list) {
 45                 char tmp;
 46                 int res;
 47 
 48                 /*
 49                  * This happens when the module gets unloaded and doesn't
 50                  * destroy its slab cache and no-one else reuses the vmalloc
 51                  * area of the module.  Print a warning.
 52                  */
 53                 res = probe_kernel_address(s->name, tmp);
 54                 if (res) {
 55                         pr_err("Slab cache with size %d has lost its name\n",
 56                                s->object_size);
 57                         continue;
 58                 }
 59 
 60 #if !defined(CONFIG_SLUB)
 61                 if (!strcmp(s->name, name)) {
 62                         pr_err("%s (%s): Cache name already exists.\n",
 63                                __func__, name);
 64                         dump_stack();
 65                         s = NULL;
 66                         return -EINVAL;
 67                 }
 68 #endif
 69         }
 70 
 71         WARN_ON(strchr(name, ' '));     /* It confuses parsers */
 72         return 0;
 73 }
 74 #else
 75 static inline int kmem_cache_sanity_check(const char *name, size_t size)
 76 {
 77         return 0;
 78 }
 79 #endif
 80 
 81 #ifdef CONFIG_MEMCG_KMEM
 82 int memcg_update_all_caches(int num_memcgs)
 83 {
 84         struct kmem_cache *s;
 85         int ret = 0;
 86         mutex_lock(&slab_mutex);
 87 
 88         list_for_each_entry(s, &slab_caches, list) {
 89                 if (!is_root_cache(s))
 90                         continue;
 91 
 92                 ret = memcg_update_cache_size(s, num_memcgs);
 93                 /*
 94                  * See comment in memcontrol.c, memcg_update_cache_size:
 95                  * Instead of freeing the memory, we'll just leave the caches
 96                  * up to this point in an updated state.
 97                  */
 98                 if (ret)
 99                         goto out;
100         }
101 
102         memcg_update_array_size(num_memcgs);
103 out:
104         mutex_unlock(&slab_mutex);
105         return ret;
106 }
107 #endif
108 
109 /*
110  * Figure out what the alignment of the objects will be given a set of
111  * flags, a user specified alignment and the size of the objects.
112  */
113 unsigned long calculate_alignment(unsigned long flags,
114                 unsigned long align, unsigned long size)
115 {
116         /*
117          * If the user wants hardware cache aligned objects then follow that
118          * suggestion if the object is sufficiently large.
119          *
120          * The hardware cache alignment cannot override the specified
121          * alignment though. If that is greater then use it.
122          */
123         if (flags & SLAB_HWCACHE_ALIGN) {
124                 unsigned long ralign = cache_line_size();
125                 while (size <= ralign / 2)
126                         ralign /= 2;
127                 align = max(align, ralign);
128         }
129 
130         if (align < ARCH_SLAB_MINALIGN)
131                 align = ARCH_SLAB_MINALIGN;
132 
133         return ALIGN(align, sizeof(void *));
134 }
135 
136 static struct kmem_cache *
137 do_kmem_cache_create(char *name, size_t object_size, size_t size, size_t align,
138                      unsigned long flags, void (*ctor)(void *),
139                      struct mem_cgroup *memcg, struct kmem_cache *root_cache)
140 {
141         struct kmem_cache *s;
142         int err;
143 
144         err = -ENOMEM;
145         s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL);
146         if (!s)
147                 goto out;
148 
149         s->name = name;
150         s->object_size = object_size;
151         s->size = size;
152         s->align = align;
153         s->ctor = ctor;
154 
155         err = memcg_alloc_cache_params(memcg, s, root_cache);
156         if (err)
157                 goto out_free_cache;
158 
159         err = __kmem_cache_create(s, flags);
160         if (err)
161                 goto out_free_cache;
162 
163         s->refcount = 1;
164         list_add(&s->list, &slab_caches);
165 out:
166         if (err)
167                 return ERR_PTR(err);
168         return s;
169 
170 out_free_cache:
171         memcg_free_cache_params(s);
172         kfree(s);
173         goto out;
174 }
175 
176 /*
177  * kmem_cache_create - Create a cache.
178  * @name: A string which is used in /proc/slabinfo to identify this cache.
179  * @size: The size of objects to be created in this cache.
180  * @align: The required alignment for the objects.
181  * @flags: SLAB flags
182  * @ctor: A constructor for the objects.
183  *
184  * Returns a ptr to the cache on success, NULL on failure.
185  * Cannot be called within a interrupt, but can be interrupted.
186  * The @ctor is run when new pages are allocated by the cache.
187  *
188  * The flags are
189  *
190  * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5)
191  * to catch references to uninitialised memory.
192  *
193  * %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check
194  * for buffer overruns.
195  *
196  * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware
197  * cacheline.  This can be beneficial if you're counting cycles as closely
198  * as davem.
199  */
200 struct kmem_cache *
201 kmem_cache_create(const char *name, size_t size, size_t align,
202                   unsigned long flags, void (*ctor)(void *))
203 {
204         struct kmem_cache *s;
205         char *cache_name;
206         int err;
207 
208         get_online_cpus();
209         get_online_mems();
210 
211         mutex_lock(&slab_mutex);
212 
213         err = kmem_cache_sanity_check(name, size);
214         if (err)
215                 goto out_unlock;
216 
217         /*
218          * Some allocators will constraint the set of valid flags to a subset
219          * of all flags. We expect them to define CACHE_CREATE_MASK in this
220          * case, and we'll just provide them with a sanitized version of the
221          * passed flags.
222          */
223         flags &= CACHE_CREATE_MASK;
224 
225         s = __kmem_cache_alias(name, size, align, flags, ctor);
226         if (s)
227                 goto out_unlock;
228 
229         cache_name = kstrdup(name, GFP_KERNEL);
230         if (!cache_name) {
231                 err = -ENOMEM;
232                 goto out_unlock;
233         }
234 
235         s = do_kmem_cache_create(cache_name, size, size,
236                                  calculate_alignment(flags, align, size),
237                                  flags, ctor, NULL, NULL);
238         if (IS_ERR(s)) {
239                 err = PTR_ERR(s);
240                 kfree(cache_name);
241         }
242 
243 out_unlock:
244         mutex_unlock(&slab_mutex);
245 
246         put_online_mems();
247         put_online_cpus();
248 
249         if (err) {
250                 if (flags & SLAB_PANIC)
251                         panic("kmem_cache_create: Failed to create slab '%s'. Error %d\n",
252                                 name, err);
253                 else {
254                         printk(KERN_WARNING "kmem_cache_create(%s) failed with error %d",
255                                 name, err);
256                         dump_stack();
257                 }
258                 return NULL;
259         }
260         return s;
261 }
262 EXPORT_SYMBOL(kmem_cache_create);
263 
264 #ifdef CONFIG_MEMCG_KMEM
265 /*
266  * memcg_create_kmem_cache - Create a cache for a memory cgroup.
267  * @memcg: The memory cgroup the new cache is for.
268  * @root_cache: The parent of the new cache.
269  * @memcg_name: The name of the memory cgroup (used for naming the new cache).
270  *
271  * This function attempts to create a kmem cache that will serve allocation
272  * requests going from @memcg to @root_cache. The new cache inherits properties
273  * from its parent.
274  */
275 struct kmem_cache *memcg_create_kmem_cache(struct mem_cgroup *memcg,
276                                            struct kmem_cache *root_cache,
277                                            const char *memcg_name)
278 {
279         struct kmem_cache *s = NULL;
280         char *cache_name;
281 
282         get_online_cpus();
283         get_online_mems();
284 
285         mutex_lock(&slab_mutex);
286 
287         cache_name = kasprintf(GFP_KERNEL, "%s(%d:%s)", root_cache->name,
288                                memcg_cache_id(memcg), memcg_name);
289         if (!cache_name)
290                 goto out_unlock;
291 
292         s = do_kmem_cache_create(cache_name, root_cache->object_size,
293                                  root_cache->size, root_cache->align,
294                                  root_cache->flags, root_cache->ctor,
295                                  memcg, root_cache);
296         if (IS_ERR(s)) {
297                 kfree(cache_name);
298                 s = NULL;
299         }
300 
301 out_unlock:
302         mutex_unlock(&slab_mutex);
303 
304         put_online_mems();
305         put_online_cpus();
306 
307         return s;
308 }
309 
310 static int memcg_cleanup_cache_params(struct kmem_cache *s)
311 {
312         int rc;
313 
314         if (!s->memcg_params ||
315             !s->memcg_params->is_root_cache)
316                 return 0;
317 
318         mutex_unlock(&slab_mutex);
319         rc = __memcg_cleanup_cache_params(s);
320         mutex_lock(&slab_mutex);
321 
322         return rc;
323 }
324 #else
325 static int memcg_cleanup_cache_params(struct kmem_cache *s)
326 {
327         return 0;
328 }
329 #endif /* CONFIG_MEMCG_KMEM */
330 
331 void slab_kmem_cache_release(struct kmem_cache *s)
332 {
333         kfree(s->name);
334         kmem_cache_free(kmem_cache, s);
335 }
336 
337 void kmem_cache_destroy(struct kmem_cache *s)
338 {
339         get_online_cpus();
340         get_online_mems();
341 
342         mutex_lock(&slab_mutex);
343 
344         s->refcount--;
345         if (s->refcount)
346                 goto out_unlock;
347 
348         if (memcg_cleanup_cache_params(s) != 0)
349                 goto out_unlock;
350 
351         if (__kmem_cache_shutdown(s) != 0) {
352                 printk(KERN_ERR "kmem_cache_destroy %s: "
353                        "Slab cache still has objects\n", s->name);
354                 dump_stack();
355                 goto out_unlock;
356         }
357 
358         list_del(&s->list);
359 
360         mutex_unlock(&slab_mutex);
361         if (s->flags & SLAB_DESTROY_BY_RCU)
362                 rcu_barrier();
363 
364         memcg_free_cache_params(s);
365 #ifdef SLAB_SUPPORTS_SYSFS
366         sysfs_slab_remove(s);
367 #else
368         slab_kmem_cache_release(s);
369 #endif
370         goto out;
371 
372 out_unlock:
373         mutex_unlock(&slab_mutex);
374 out:
375         put_online_mems();
376         put_online_cpus();
377 }
378 EXPORT_SYMBOL(kmem_cache_destroy);
379 
380 /**
381  * kmem_cache_shrink - Shrink a cache.
382  * @cachep: The cache to shrink.
383  *
384  * Releases as many slabs as possible for a cache.
385  * To help debugging, a zero exit status indicates all slabs were released.
386  */
387 int kmem_cache_shrink(struct kmem_cache *cachep)
388 {
389         int ret;
390 
391         get_online_cpus();
392         get_online_mems();
393         ret = __kmem_cache_shrink(cachep);
394         put_online_mems();
395         put_online_cpus();
396         return ret;
397 }
398 EXPORT_SYMBOL(kmem_cache_shrink);
399 
400 int slab_is_available(void)
401 {
402         return slab_state >= UP;
403 }
404 
405 #ifndef CONFIG_SLOB
406 /* Create a cache during boot when no slab services are available yet */
407 void __init create_boot_cache(struct kmem_cache *s, const char *name, size_t size,
408                 unsigned long flags)
409 {
410         int err;
411 
412         s->name = name;
413         s->size = s->object_size = size;
414         s->align = calculate_alignment(flags, ARCH_KMALLOC_MINALIGN, size);
415         err = __kmem_cache_create(s, flags);
416 
417         if (err)
418                 panic("Creation of kmalloc slab %s size=%zu failed. Reason %d\n",
419                                         name, size, err);
420 
421         s->refcount = -1;       /* Exempt from merging for now */
422 }
423 
424 struct kmem_cache *__init create_kmalloc_cache(const char *name, size_t size,
425                                 unsigned long flags)
426 {
427         struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
428 
429         if (!s)
430                 panic("Out of memory when creating slab %s\n", name);
431 
432         create_boot_cache(s, name, size, flags);
433         list_add(&s->list, &slab_caches);
434         s->refcount = 1;
435         return s;
436 }
437 
438 struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1];
439 EXPORT_SYMBOL(kmalloc_caches);
440 
441 #ifdef CONFIG_ZONE_DMA
442 struct kmem_cache *kmalloc_dma_caches[KMALLOC_SHIFT_HIGH + 1];
443 EXPORT_SYMBOL(kmalloc_dma_caches);
444 #endif
445 
446 /*
447  * Conversion table for small slabs sizes / 8 to the index in the
448  * kmalloc array. This is necessary for slabs < 192 since we have non power
449  * of two cache sizes there. The size of larger slabs can be determined using
450  * fls.
451  */
452 static s8 size_index[24] = {
453         3,      /* 8 */
454         4,      /* 16 */
455         5,      /* 24 */
456         5,      /* 32 */
457         6,      /* 40 */
458         6,      /* 48 */
459         6,      /* 56 */
460         6,      /* 64 */
461         1,      /* 72 */
462         1,      /* 80 */
463         1,      /* 88 */
464         1,      /* 96 */
465         7,      /* 104 */
466         7,      /* 112 */
467         7,      /* 120 */
468         7,      /* 128 */
469         2,      /* 136 */
470         2,      /* 144 */
471         2,      /* 152 */
472         2,      /* 160 */
473         2,      /* 168 */
474         2,      /* 176 */
475         2,      /* 184 */
476         2       /* 192 */
477 };
478 
479 static inline int size_index_elem(size_t bytes)
480 {
481         return (bytes - 1) / 8;
482 }
483 
484 /*
485  * Find the kmem_cache structure that serves a given size of
486  * allocation
487  */
488 struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags)
489 {
490         int index;
491 
492         if (unlikely(size > KMALLOC_MAX_SIZE)) {
493                 WARN_ON_ONCE(!(flags & __GFP_NOWARN));
494                 return NULL;
495         }
496 
497         if (size <= 192) {
498                 if (!size)
499                         return ZERO_SIZE_PTR;
500 
501                 index = size_index[size_index_elem(size)];
502         } else
503                 index = fls(size - 1);
504 
505 #ifdef CONFIG_ZONE_DMA
506         if (unlikely((flags & GFP_DMA)))
507                 return kmalloc_dma_caches[index];
508 
509 #endif
510         return kmalloc_caches[index];
511 }
512 
513 /*
514  * Create the kmalloc array. Some of the regular kmalloc arrays
515  * may already have been created because they were needed to
516  * enable allocations for slab creation.
517  */
518 void __init create_kmalloc_caches(unsigned long flags)
519 {
520         int i;
521 
522         /*
523          * Patch up the size_index table if we have strange large alignment
524          * requirements for the kmalloc array. This is only the case for
525          * MIPS it seems. The standard arches will not generate any code here.
526          *
527          * Largest permitted alignment is 256 bytes due to the way we
528          * handle the index determination for the smaller caches.
529          *
530          * Make sure that nothing crazy happens if someone starts tinkering
531          * around with ARCH_KMALLOC_MINALIGN
532          */
533         BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 ||
534                 (KMALLOC_MIN_SIZE & (KMALLOC_MIN_SIZE - 1)));
535 
536         for (i = 8; i < KMALLOC_MIN_SIZE; i += 8) {
537                 int elem = size_index_elem(i);
538 
539                 if (elem >= ARRAY_SIZE(size_index))
540                         break;
541                 size_index[elem] = KMALLOC_SHIFT_LOW;
542         }
543 
544         if (KMALLOC_MIN_SIZE >= 64) {
545                 /*
546                  * The 96 byte size cache is not used if the alignment
547                  * is 64 byte.
548                  */
549                 for (i = 64 + 8; i <= 96; i += 8)
550                         size_index[size_index_elem(i)] = 7;
551 
552         }
553 
554         if (KMALLOC_MIN_SIZE >= 128) {
555                 /*
556                  * The 192 byte sized cache is not used if the alignment
557                  * is 128 byte. Redirect kmalloc to use the 256 byte cache
558                  * instead.
559                  */
560                 for (i = 128 + 8; i <= 192; i += 8)
561                         size_index[size_index_elem(i)] = 8;
562         }
563         for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
564                 if (!kmalloc_caches[i]) {
565                         kmalloc_caches[i] = create_kmalloc_cache(NULL,
566                                                         1 << i, flags);
567                 }
568 
569                 /*
570                  * Caches that are not of the two-to-the-power-of size.
571                  * These have to be created immediately after the
572                  * earlier power of two caches
573                  */
574                 if (KMALLOC_MIN_SIZE <= 32 && !kmalloc_caches[1] && i == 6)
575                         kmalloc_caches[1] = create_kmalloc_cache(NULL, 96, flags);
576 
577                 if (KMALLOC_MIN_SIZE <= 64 && !kmalloc_caches[2] && i == 7)
578                         kmalloc_caches[2] = create_kmalloc_cache(NULL, 192, flags);
579         }
580 
581         /* Kmalloc array is now usable */
582         slab_state = UP;
583 
584         for (i = 0; i <= KMALLOC_SHIFT_HIGH; i++) {
585                 struct kmem_cache *s = kmalloc_caches[i];
586                 char *n;
587 
588                 if (s) {
589                         n = kasprintf(GFP_NOWAIT, "kmalloc-%d", kmalloc_size(i));
590 
591                         BUG_ON(!n);
592                         s->name = n;
593                 }
594         }
595 
596 #ifdef CONFIG_ZONE_DMA
597         for (i = 0; i <= KMALLOC_SHIFT_HIGH; i++) {
598                 struct kmem_cache *s = kmalloc_caches[i];
599 
600                 if (s) {
601                         int size = kmalloc_size(i);
602                         char *n = kasprintf(GFP_NOWAIT,
603                                  "dma-kmalloc-%d", size);
604 
605                         BUG_ON(!n);
606                         kmalloc_dma_caches[i] = create_kmalloc_cache(n,
607                                 size, SLAB_CACHE_DMA | flags);
608                 }
609         }
610 #endif
611 }
612 #endif /* !CONFIG_SLOB */
613 
614 /*
615  * To avoid unnecessary overhead, we pass through large allocation requests
616  * directly to the page allocator. We use __GFP_COMP, because we will need to
617  * know the allocation order to free the pages properly in kfree.
618  */
619 void *kmalloc_order(size_t size, gfp_t flags, unsigned int order)
620 {
621         void *ret;
622         struct page *page;
623 
624         flags |= __GFP_COMP;
625         page = alloc_kmem_pages(flags, order);
626         ret = page ? page_address(page) : NULL;
627         kmemleak_alloc(ret, size, 1, flags);
628         return ret;
629 }
630 EXPORT_SYMBOL(kmalloc_order);
631 
632 #ifdef CONFIG_TRACING
633 void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order)
634 {
635         void *ret = kmalloc_order(size, flags, order);
636         trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << order, flags);
637         return ret;
638 }
639 EXPORT_SYMBOL(kmalloc_order_trace);
640 #endif
641 
642 #ifdef CONFIG_SLABINFO
643 
644 #ifdef CONFIG_SLAB
645 #define SLABINFO_RIGHTS (S_IWUSR | S_IRUSR)
646 #else
647 #define SLABINFO_RIGHTS S_IRUSR
648 #endif
649 
650 void print_slabinfo_header(struct seq_file *m)
651 {
652         /*
653          * Output format version, so at least we can change it
654          * without _too_ many complaints.
655          */
656 #ifdef CONFIG_DEBUG_SLAB
657         seq_puts(m, "slabinfo - version: 2.1 (statistics)\n");
658 #else
659         seq_puts(m, "slabinfo - version: 2.1\n");
660 #endif
661         seq_puts(m, "# name            <active_objs> <num_objs> <objsize> "
662                  "<objperslab> <pagesperslab>");
663         seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
664         seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
665 #ifdef CONFIG_DEBUG_SLAB
666         seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped> "
667                  "<error> <maxfreeable> <nodeallocs> <remotefrees> <alienoverflow>");
668         seq_puts(m, " : cpustat <allochit> <allocmiss> <freehit> <freemiss>");
669 #endif
670         seq_putc(m, '\n');
671 }
672 
673 static void *s_start(struct seq_file *m, loff_t *pos)
674 {
675         loff_t n = *pos;
676 
677         mutex_lock(&slab_mutex);
678         if (!n)
679                 print_slabinfo_header(m);
680 
681         return seq_list_start(&slab_caches, *pos);
682 }
683 
684 void *slab_next(struct seq_file *m, void *p, loff_t *pos)
685 {
686         return seq_list_next(p, &slab_caches, pos);
687 }
688 
689 void slab_stop(struct seq_file *m, void *p)
690 {
691         mutex_unlock(&slab_mutex);
692 }
693 
694 static void
695 memcg_accumulate_slabinfo(struct kmem_cache *s, struct slabinfo *info)
696 {
697         struct kmem_cache *c;
698         struct slabinfo sinfo;
699         int i;
700 
701         if (!is_root_cache(s))
702                 return;
703 
704         for_each_memcg_cache_index(i) {
705                 c = cache_from_memcg_idx(s, i);
706                 if (!c)
707                         continue;
708 
709                 memset(&sinfo, 0, sizeof(sinfo));
710                 get_slabinfo(c, &sinfo);
711 
712                 info->active_slabs += sinfo.active_slabs;
713                 info->num_slabs += sinfo.num_slabs;
714                 info->shared_avail += sinfo.shared_avail;
715                 info->active_objs += sinfo.active_objs;
716                 info->num_objs += sinfo.num_objs;
717         }
718 }
719 
720 int cache_show(struct kmem_cache *s, struct seq_file *m)
721 {
722         struct slabinfo sinfo;
723 
724         memset(&sinfo, 0, sizeof(sinfo));
725         get_slabinfo(s, &sinfo);
726 
727         memcg_accumulate_slabinfo(s, &sinfo);
728 
729         seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d",
730                    cache_name(s), sinfo.active_objs, sinfo.num_objs, s->size,
731                    sinfo.objects_per_slab, (1 << sinfo.cache_order));
732 
733         seq_printf(m, " : tunables %4u %4u %4u",
734                    sinfo.limit, sinfo.batchcount, sinfo.shared);
735         seq_printf(m, " : slabdata %6lu %6lu %6lu",
736                    sinfo.active_slabs, sinfo.num_slabs, sinfo.shared_avail);
737         slabinfo_show_stats(m, s);
738         seq_putc(m, '\n');
739         return 0;
740 }
741 
742 static int s_show(struct seq_file *m, void *p)
743 {
744         struct kmem_cache *s = list_entry(p, struct kmem_cache, list);
745 
746         if (!is_root_cache(s))
747                 return 0;
748         return cache_show(s, m);
749 }
750 
751 /*
752  * slabinfo_op - iterator that generates /proc/slabinfo
753  *
754  * Output layout:
755  * cache-name
756  * num-active-objs
757  * total-objs
758  * object size
759  * num-active-slabs
760  * total-slabs
761  * num-pages-per-slab
762  * + further values on SMP and with statistics enabled
763  */
764 static const struct seq_operations slabinfo_op = {
765         .start = s_start,
766         .next = slab_next,
767         .stop = slab_stop,
768         .show = s_show,
769 };
770 
771 static int slabinfo_open(struct inode *inode, struct file *file)
772 {
773         return seq_open(file, &slabinfo_op);
774 }
775 
776 static const struct file_operations proc_slabinfo_operations = {
777         .open           = slabinfo_open,
778         .read           = seq_read,
779         .write          = slabinfo_write,
780         .llseek         = seq_lseek,
781         .release        = seq_release,
782 };
783 
784 static int __init slab_proc_init(void)
785 {
786         proc_create("slabinfo", SLABINFO_RIGHTS, NULL,
787                                                 &proc_slabinfo_operations);
788         return 0;
789 }
790 module_init(slab_proc_init);
791 #endif /* CONFIG_SLABINFO */
792 
793 static __always_inline void *__do_krealloc(const void *p, size_t new_size,
794                                            gfp_t flags)
795 {
796         void *ret;
797         size_t ks = 0;
798 
799         if (p)
800                 ks = ksize(p);
801 
802         if (ks >= new_size)
803                 return (void *)p;
804 
805         ret = kmalloc_track_caller(new_size, flags);
806         if (ret && p)
807                 memcpy(ret, p, ks);
808 
809         return ret;
810 }
811 
812 /**
813  * __krealloc - like krealloc() but don't free @p.
814  * @p: object to reallocate memory for.
815  * @new_size: how many bytes of memory are required.
816  * @flags: the type of memory to allocate.
817  *
818  * This function is like krealloc() except it never frees the originally
819  * allocated buffer. Use this if you don't want to free the buffer immediately
820  * like, for example, with RCU.
821  */
822 void *__krealloc(const void *p, size_t new_size, gfp_t flags)
823 {
824         if (unlikely(!new_size))
825                 return ZERO_SIZE_PTR;
826 
827         return __do_krealloc(p, new_size, flags);
828 
829 }
830 EXPORT_SYMBOL(__krealloc);
831 
832 /**
833  * krealloc - reallocate memory. The contents will remain unchanged.
834  * @p: object to reallocate memory for.
835  * @new_size: how many bytes of memory are required.
836  * @flags: the type of memory to allocate.
837  *
838  * The contents of the object pointed to are preserved up to the
839  * lesser of the new and old sizes.  If @p is %NULL, krealloc()
840  * behaves exactly like kmalloc().  If @new_size is 0 and @p is not a
841  * %NULL pointer, the object pointed to is freed.
842  */
843 void *krealloc(const void *p, size_t new_size, gfp_t flags)
844 {
845         void *ret;
846 
847         if (unlikely(!new_size)) {
848                 kfree(p);
849                 return ZERO_SIZE_PTR;
850         }
851 
852         ret = __do_krealloc(p, new_size, flags);
853         if (ret && p != ret)
854                 kfree(p);
855 
856         return ret;
857 }
858 EXPORT_SYMBOL(krealloc);
859 
860 /**
861  * kzfree - like kfree but zero memory
862  * @p: object to free memory of
863  *
864  * The memory of the object @p points to is zeroed before freed.
865  * If @p is %NULL, kzfree() does nothing.
866  *
867  * Note: this function zeroes the whole allocated buffer which can be a good
868  * deal bigger than the requested buffer size passed to kmalloc(). So be
869  * careful when using this function in performance sensitive code.
870  */
871 void kzfree(const void *p)
872 {
873         size_t ks;
874         void *mem = (void *)p;
875 
876         if (unlikely(ZERO_OR_NULL_PTR(mem)))
877                 return;
878         ks = ksize(mem);
879         memset(mem, 0, ks);
880         kfree(mem);
881 }
882 EXPORT_SYMBOL(kzfree);
883 
884 /* Tracepoints definitions. */
885 EXPORT_TRACEPOINT_SYMBOL(kmalloc);
886 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc);
887 EXPORT_TRACEPOINT_SYMBOL(kmalloc_node);
888 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc_node);
889 EXPORT_TRACEPOINT_SYMBOL(kfree);
890 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free);
891 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp