~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/mm/memcontrol.c

Version: ~ [ linux-5.10-rc1 ] ~ [ linux-5.9.1 ] ~ [ linux-5.8.16 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.72 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.152 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.202 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.240 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.240 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /* memcontrol.c - Memory Controller
  2  *
  3  * Copyright IBM Corporation, 2007
  4  * Author Balbir Singh <balbir@linux.vnet.ibm.com>
  5  *
  6  * Copyright 2007 OpenVZ SWsoft Inc
  7  * Author: Pavel Emelianov <xemul@openvz.org>
  8  *
  9  * Memory thresholds
 10  * Copyright (C) 2009 Nokia Corporation
 11  * Author: Kirill A. Shutemov
 12  *
 13  * Kernel Memory Controller
 14  * Copyright (C) 2012 Parallels Inc. and Google Inc.
 15  * Authors: Glauber Costa and Suleiman Souhlal
 16  *
 17  * This program is free software; you can redistribute it and/or modify
 18  * it under the terms of the GNU General Public License as published by
 19  * the Free Software Foundation; either version 2 of the License, or
 20  * (at your option) any later version.
 21  *
 22  * This program is distributed in the hope that it will be useful,
 23  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 24  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 25  * GNU General Public License for more details.
 26  */
 27 
 28 #include <linux/res_counter.h>
 29 #include <linux/memcontrol.h>
 30 #include <linux/cgroup.h>
 31 #include <linux/mm.h>
 32 #include <linux/hugetlb.h>
 33 #include <linux/pagemap.h>
 34 #include <linux/smp.h>
 35 #include <linux/page-flags.h>
 36 #include <linux/backing-dev.h>
 37 #include <linux/bit_spinlock.h>
 38 #include <linux/rcupdate.h>
 39 #include <linux/limits.h>
 40 #include <linux/export.h>
 41 #include <linux/mutex.h>
 42 #include <linux/rbtree.h>
 43 #include <linux/slab.h>
 44 #include <linux/swap.h>
 45 #include <linux/swapops.h>
 46 #include <linux/spinlock.h>
 47 #include <linux/eventfd.h>
 48 #include <linux/sort.h>
 49 #include <linux/fs.h>
 50 #include <linux/seq_file.h>
 51 #include <linux/vmalloc.h>
 52 #include <linux/vmpressure.h>
 53 #include <linux/mm_inline.h>
 54 #include <linux/page_cgroup.h>
 55 #include <linux/cpu.h>
 56 #include <linux/oom.h>
 57 #include "internal.h"
 58 #include <net/sock.h>
 59 #include <net/ip.h>
 60 #include <net/tcp_memcontrol.h>
 61 
 62 #include <asm/uaccess.h>
 63 
 64 #include <trace/events/vmscan.h>
 65 
 66 struct cgroup_subsys mem_cgroup_subsys __read_mostly;
 67 EXPORT_SYMBOL(mem_cgroup_subsys);
 68 
 69 #define MEM_CGROUP_RECLAIM_RETRIES      5
 70 static struct mem_cgroup *root_mem_cgroup __read_mostly;
 71 
 72 #ifdef CONFIG_MEMCG_SWAP
 73 /* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */
 74 int do_swap_account __read_mostly;
 75 
 76 /* for remember boot option*/
 77 #ifdef CONFIG_MEMCG_SWAP_ENABLED
 78 static int really_do_swap_account __initdata = 1;
 79 #else
 80 static int really_do_swap_account __initdata = 0;
 81 #endif
 82 
 83 #else
 84 #define do_swap_account         0
 85 #endif
 86 
 87 
 88 /*
 89  * Statistics for memory cgroup.
 90  */
 91 enum mem_cgroup_stat_index {
 92         /*
 93          * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
 94          */
 95         MEM_CGROUP_STAT_CACHE,          /* # of pages charged as cache */
 96         MEM_CGROUP_STAT_RSS,            /* # of pages charged as anon rss */
 97         MEM_CGROUP_STAT_RSS_HUGE,       /* # of pages charged as anon huge */
 98         MEM_CGROUP_STAT_FILE_MAPPED,    /* # of pages charged as file rss */
 99         MEM_CGROUP_STAT_SWAP,           /* # of pages, swapped out */
100         MEM_CGROUP_STAT_NSTATS,
101 };
102 
103 static const char * const mem_cgroup_stat_names[] = {
104         "cache",
105         "rss",
106         "rss_huge",
107         "mapped_file",
108         "swap",
109 };
110 
111 enum mem_cgroup_events_index {
112         MEM_CGROUP_EVENTS_PGPGIN,       /* # of pages paged in */
113         MEM_CGROUP_EVENTS_PGPGOUT,      /* # of pages paged out */
114         MEM_CGROUP_EVENTS_PGFAULT,      /* # of page-faults */
115         MEM_CGROUP_EVENTS_PGMAJFAULT,   /* # of major page-faults */
116         MEM_CGROUP_EVENTS_NSTATS,
117 };
118 
119 static const char * const mem_cgroup_events_names[] = {
120         "pgpgin",
121         "pgpgout",
122         "pgfault",
123         "pgmajfault",
124 };
125 
126 static const char * const mem_cgroup_lru_names[] = {
127         "inactive_anon",
128         "active_anon",
129         "inactive_file",
130         "active_file",
131         "unevictable",
132 };
133 
134 /*
135  * Per memcg event counter is incremented at every pagein/pageout. With THP,
136  * it will be incremated by the number of pages. This counter is used for
137  * for trigger some periodic events. This is straightforward and better
138  * than using jiffies etc. to handle periodic memcg event.
139  */
140 enum mem_cgroup_events_target {
141         MEM_CGROUP_TARGET_THRESH,
142         MEM_CGROUP_TARGET_SOFTLIMIT,
143         MEM_CGROUP_TARGET_NUMAINFO,
144         MEM_CGROUP_NTARGETS,
145 };
146 #define THRESHOLDS_EVENTS_TARGET 128
147 #define SOFTLIMIT_EVENTS_TARGET 1024
148 #define NUMAINFO_EVENTS_TARGET  1024
149 
150 struct mem_cgroup_stat_cpu {
151         long count[MEM_CGROUP_STAT_NSTATS];
152         unsigned long events[MEM_CGROUP_EVENTS_NSTATS];
153         unsigned long nr_page_events;
154         unsigned long targets[MEM_CGROUP_NTARGETS];
155 };
156 
157 struct mem_cgroup_reclaim_iter {
158         /*
159          * last scanned hierarchy member. Valid only if last_dead_count
160          * matches memcg->dead_count of the hierarchy root group.
161          */
162         struct mem_cgroup *last_visited;
163         unsigned long last_dead_count;
164 
165         /* scan generation, increased every round-trip */
166         unsigned int generation;
167 };
168 
169 /*
170  * per-zone information in memory controller.
171  */
172 struct mem_cgroup_per_zone {
173         struct lruvec           lruvec;
174         unsigned long           lru_size[NR_LRU_LISTS];
175 
176         struct mem_cgroup_reclaim_iter reclaim_iter[DEF_PRIORITY + 1];
177 
178         struct rb_node          tree_node;      /* RB tree node */
179         unsigned long long      usage_in_excess;/* Set to the value by which */
180                                                 /* the soft limit is exceeded*/
181         bool                    on_tree;
182         struct mem_cgroup       *memcg;         /* Back pointer, we cannot */
183                                                 /* use container_of        */
184 };
185 
186 struct mem_cgroup_per_node {
187         struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
188 };
189 
190 struct mem_cgroup_lru_info {
191         struct mem_cgroup_per_node *nodeinfo[0];
192 };
193 
194 /*
195  * Cgroups above their limits are maintained in a RB-Tree, independent of
196  * their hierarchy representation
197  */
198 
199 struct mem_cgroup_tree_per_zone {
200         struct rb_root rb_root;
201         spinlock_t lock;
202 };
203 
204 struct mem_cgroup_tree_per_node {
205         struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES];
206 };
207 
208 struct mem_cgroup_tree {
209         struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
210 };
211 
212 static struct mem_cgroup_tree soft_limit_tree __read_mostly;
213 
214 struct mem_cgroup_threshold {
215         struct eventfd_ctx *eventfd;
216         u64 threshold;
217 };
218 
219 /* For threshold */
220 struct mem_cgroup_threshold_ary {
221         /* An array index points to threshold just below or equal to usage. */
222         int current_threshold;
223         /* Size of entries[] */
224         unsigned int size;
225         /* Array of thresholds */
226         struct mem_cgroup_threshold entries[0];
227 };
228 
229 struct mem_cgroup_thresholds {
230         /* Primary thresholds array */
231         struct mem_cgroup_threshold_ary *primary;
232         /*
233          * Spare threshold array.
234          * This is needed to make mem_cgroup_unregister_event() "never fail".
235          * It must be able to store at least primary->size - 1 entries.
236          */
237         struct mem_cgroup_threshold_ary *spare;
238 };
239 
240 /* for OOM */
241 struct mem_cgroup_eventfd_list {
242         struct list_head list;
243         struct eventfd_ctx *eventfd;
244 };
245 
246 static void mem_cgroup_threshold(struct mem_cgroup *memcg);
247 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
248 
249 /*
250  * The memory controller data structure. The memory controller controls both
251  * page cache and RSS per cgroup. We would eventually like to provide
252  * statistics based on the statistics developed by Rik Van Riel for clock-pro,
253  * to help the administrator determine what knobs to tune.
254  *
255  * TODO: Add a water mark for the memory controller. Reclaim will begin when
256  * we hit the water mark. May be even add a low water mark, such that
257  * no reclaim occurs from a cgroup at it's low water mark, this is
258  * a feature that will be implemented much later in the future.
259  */
260 struct mem_cgroup {
261         struct cgroup_subsys_state css;
262         /*
263          * the counter to account for memory usage
264          */
265         struct res_counter res;
266 
267         /* vmpressure notifications */
268         struct vmpressure vmpressure;
269 
270         union {
271                 /*
272                  * the counter to account for mem+swap usage.
273                  */
274                 struct res_counter memsw;
275 
276                 /*
277                  * rcu_freeing is used only when freeing struct mem_cgroup,
278                  * so put it into a union to avoid wasting more memory.
279                  * It must be disjoint from the css field.  It could be
280                  * in a union with the res field, but res plays a much
281                  * larger part in mem_cgroup life than memsw, and might
282                  * be of interest, even at time of free, when debugging.
283                  * So share rcu_head with the less interesting memsw.
284                  */
285                 struct rcu_head rcu_freeing;
286                 /*
287                  * We also need some space for a worker in deferred freeing.
288                  * By the time we call it, rcu_freeing is no longer in use.
289                  */
290                 struct work_struct work_freeing;
291         };
292 
293         /*
294          * the counter to account for kernel memory usage.
295          */
296         struct res_counter kmem;
297         /*
298          * Should the accounting and control be hierarchical, per subtree?
299          */
300         bool use_hierarchy;
301         unsigned long kmem_account_flags; /* See KMEM_ACCOUNTED_*, below */
302 
303         bool            oom_lock;
304         atomic_t        under_oom;
305         atomic_t        oom_wakeups;
306 
307         atomic_t        refcnt;
308 
309         int     swappiness;
310         /* OOM-Killer disable */
311         int             oom_kill_disable;
312 
313         /* set when res.limit == memsw.limit */
314         bool            memsw_is_minimum;
315 
316         /* protect arrays of thresholds */
317         struct mutex thresholds_lock;
318 
319         /* thresholds for memory usage. RCU-protected */
320         struct mem_cgroup_thresholds thresholds;
321 
322         /* thresholds for mem+swap usage. RCU-protected */
323         struct mem_cgroup_thresholds memsw_thresholds;
324 
325         /* For oom notifier event fd */
326         struct list_head oom_notify;
327 
328         /*
329          * Should we move charges of a task when a task is moved into this
330          * mem_cgroup ? And what type of charges should we move ?
331          */
332         unsigned long   move_charge_at_immigrate;
333         /*
334          * set > 0 if pages under this cgroup are moving to other cgroup.
335          */
336         atomic_t        moving_account;
337         /* taken only while moving_account > 0 */
338         spinlock_t      move_lock;
339         /*
340          * percpu counter.
341          */
342         struct mem_cgroup_stat_cpu __percpu *stat;
343         /*
344          * used when a cpu is offlined or other synchronizations
345          * See mem_cgroup_read_stat().
346          */
347         struct mem_cgroup_stat_cpu nocpu_base;
348         spinlock_t pcp_counter_lock;
349 
350         atomic_t        dead_count;
351 #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_INET)
352         struct tcp_memcontrol tcp_mem;
353 #endif
354 #if defined(CONFIG_MEMCG_KMEM)
355         /* analogous to slab_common's slab_caches list. per-memcg */
356         struct list_head memcg_slab_caches;
357         /* Not a spinlock, we can take a lot of time walking the list */
358         struct mutex slab_caches_mutex;
359         /* Index in the kmem_cache->memcg_params->memcg_caches array */
360         int kmemcg_id;
361 #endif
362 
363         int last_scanned_node;
364 #if MAX_NUMNODES > 1
365         nodemask_t      scan_nodes;
366         atomic_t        numainfo_events;
367         atomic_t        numainfo_updating;
368 #endif
369 
370         /*
371          * Per cgroup active and inactive list, similar to the
372          * per zone LRU lists.
373          *
374          * WARNING: This has to be the last element of the struct. Don't
375          * add new fields after this point.
376          */
377         struct mem_cgroup_lru_info info;
378 };
379 
380 static size_t memcg_size(void)
381 {
382         return sizeof(struct mem_cgroup) +
383                 nr_node_ids * sizeof(struct mem_cgroup_per_node *);
384 }
385 
386 /* internal only representation about the status of kmem accounting. */
387 enum {
388         KMEM_ACCOUNTED_ACTIVE = 0, /* accounted by this cgroup itself */
389         KMEM_ACCOUNTED_ACTIVATED, /* static key enabled. */
390         KMEM_ACCOUNTED_DEAD, /* dead memcg with pending kmem charges */
391 };
392 
393 /* We account when limit is on, but only after call sites are patched */
394 #define KMEM_ACCOUNTED_MASK \
395                 ((1 << KMEM_ACCOUNTED_ACTIVE) | (1 << KMEM_ACCOUNTED_ACTIVATED))
396 
397 #ifdef CONFIG_MEMCG_KMEM
398 static inline void memcg_kmem_set_active(struct mem_cgroup *memcg)
399 {
400         set_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags);
401 }
402 
403 static bool memcg_kmem_is_active(struct mem_cgroup *memcg)
404 {
405         return test_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags);
406 }
407 
408 static void memcg_kmem_set_activated(struct mem_cgroup *memcg)
409 {
410         set_bit(KMEM_ACCOUNTED_ACTIVATED, &memcg->kmem_account_flags);
411 }
412 
413 static void memcg_kmem_clear_activated(struct mem_cgroup *memcg)
414 {
415         clear_bit(KMEM_ACCOUNTED_ACTIVATED, &memcg->kmem_account_flags);
416 }
417 
418 static void memcg_kmem_mark_dead(struct mem_cgroup *memcg)
419 {
420         if (test_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags))
421                 set_bit(KMEM_ACCOUNTED_DEAD, &memcg->kmem_account_flags);
422 }
423 
424 static bool memcg_kmem_test_and_clear_dead(struct mem_cgroup *memcg)
425 {
426         return test_and_clear_bit(KMEM_ACCOUNTED_DEAD,
427                                   &memcg->kmem_account_flags);
428 }
429 #endif
430 
431 /* Stuffs for move charges at task migration. */
432 /*
433  * Types of charges to be moved. "move_charge_at_immitgrate" and
434  * "immigrate_flags" are treated as a left-shifted bitmap of these types.
435  */
436 enum move_type {
437         MOVE_CHARGE_TYPE_ANON,  /* private anonymous page and swap of it */
438         MOVE_CHARGE_TYPE_FILE,  /* file page(including tmpfs) and swap of it */
439         NR_MOVE_TYPE,
440 };
441 
442 /* "mc" and its members are protected by cgroup_mutex */
443 static struct move_charge_struct {
444         spinlock_t        lock; /* for from, to */
445         struct mem_cgroup *from;
446         struct mem_cgroup *to;
447         unsigned long immigrate_flags;
448         unsigned long precharge;
449         unsigned long moved_charge;
450         unsigned long moved_swap;
451         struct task_struct *moving_task;        /* a task moving charges */
452         wait_queue_head_t waitq;                /* a waitq for other context */
453 } mc = {
454         .lock = __SPIN_LOCK_UNLOCKED(mc.lock),
455         .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
456 };
457 
458 static bool move_anon(void)
459 {
460         return test_bit(MOVE_CHARGE_TYPE_ANON, &mc.immigrate_flags);
461 }
462 
463 static bool move_file(void)
464 {
465         return test_bit(MOVE_CHARGE_TYPE_FILE, &mc.immigrate_flags);
466 }
467 
468 /*
469  * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
470  * limit reclaim to prevent infinite loops, if they ever occur.
471  */
472 #define MEM_CGROUP_MAX_RECLAIM_LOOPS            100
473 #define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2
474 
475 enum charge_type {
476         MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
477         MEM_CGROUP_CHARGE_TYPE_ANON,
478         MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */
479         MEM_CGROUP_CHARGE_TYPE_DROP,    /* a page was unused swap cache */
480         NR_CHARGE_TYPE,
481 };
482 
483 /* for encoding cft->private value on file */
484 enum res_type {
485         _MEM,
486         _MEMSWAP,
487         _OOM_TYPE,
488         _KMEM,
489 };
490 
491 #define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val))
492 #define MEMFILE_TYPE(val)       ((val) >> 16 & 0xffff)
493 #define MEMFILE_ATTR(val)       ((val) & 0xffff)
494 /* Used for OOM nofiier */
495 #define OOM_CONTROL             (0)
496 
497 /*
498  * Reclaim flags for mem_cgroup_hierarchical_reclaim
499  */
500 #define MEM_CGROUP_RECLAIM_NOSWAP_BIT   0x0
501 #define MEM_CGROUP_RECLAIM_NOSWAP       (1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)
502 #define MEM_CGROUP_RECLAIM_SHRINK_BIT   0x1
503 #define MEM_CGROUP_RECLAIM_SHRINK       (1 << MEM_CGROUP_RECLAIM_SHRINK_BIT)
504 
505 /*
506  * The memcg_create_mutex will be held whenever a new cgroup is created.
507  * As a consequence, any change that needs to protect against new child cgroups
508  * appearing has to hold it as well.
509  */
510 static DEFINE_MUTEX(memcg_create_mutex);
511 
512 static void mem_cgroup_get(struct mem_cgroup *memcg);
513 static void mem_cgroup_put(struct mem_cgroup *memcg);
514 
515 static inline
516 struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *s)
517 {
518         return container_of(s, struct mem_cgroup, css);
519 }
520 
521 /* Some nice accessors for the vmpressure. */
522 struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
523 {
524         if (!memcg)
525                 memcg = root_mem_cgroup;
526         return &memcg->vmpressure;
527 }
528 
529 struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
530 {
531         return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
532 }
533 
534 struct vmpressure *css_to_vmpressure(struct cgroup_subsys_state *css)
535 {
536         return &mem_cgroup_from_css(css)->vmpressure;
537 }
538 
539 static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
540 {
541         return (memcg == root_mem_cgroup);
542 }
543 
544 /* Writing them here to avoid exposing memcg's inner layout */
545 #if defined(CONFIG_INET) && defined(CONFIG_MEMCG_KMEM)
546 
547 void sock_update_memcg(struct sock *sk)
548 {
549         if (mem_cgroup_sockets_enabled) {
550                 struct mem_cgroup *memcg;
551                 struct cg_proto *cg_proto;
552 
553                 BUG_ON(!sk->sk_prot->proto_cgroup);
554 
555                 /* Socket cloning can throw us here with sk_cgrp already
556                  * filled. It won't however, necessarily happen from
557                  * process context. So the test for root memcg given
558                  * the current task's memcg won't help us in this case.
559                  *
560                  * Respecting the original socket's memcg is a better
561                  * decision in this case.
562                  */
563                 if (sk->sk_cgrp) {
564                         BUG_ON(mem_cgroup_is_root(sk->sk_cgrp->memcg));
565                         mem_cgroup_get(sk->sk_cgrp->memcg);
566                         return;
567                 }
568 
569                 rcu_read_lock();
570                 memcg = mem_cgroup_from_task(current);
571                 cg_proto = sk->sk_prot->proto_cgroup(memcg);
572                 if (!mem_cgroup_is_root(memcg) && memcg_proto_active(cg_proto)) {
573                         mem_cgroup_get(memcg);
574                         sk->sk_cgrp = cg_proto;
575                 }
576                 rcu_read_unlock();
577         }
578 }
579 EXPORT_SYMBOL(sock_update_memcg);
580 
581 void sock_release_memcg(struct sock *sk)
582 {
583         if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
584                 struct mem_cgroup *memcg;
585                 WARN_ON(!sk->sk_cgrp->memcg);
586                 memcg = sk->sk_cgrp->memcg;
587                 mem_cgroup_put(memcg);
588         }
589 }
590 
591 struct cg_proto *tcp_proto_cgroup(struct mem_cgroup *memcg)
592 {
593         if (!memcg || mem_cgroup_is_root(memcg))
594                 return NULL;
595 
596         return &memcg->tcp_mem.cg_proto;
597 }
598 EXPORT_SYMBOL(tcp_proto_cgroup);
599 
600 static void disarm_sock_keys(struct mem_cgroup *memcg)
601 {
602         if (!memcg_proto_activated(&memcg->tcp_mem.cg_proto))
603                 return;
604         static_key_slow_dec(&memcg_socket_limit_enabled);
605 }
606 #else
607 static void disarm_sock_keys(struct mem_cgroup *memcg)
608 {
609 }
610 #endif
611 
612 #ifdef CONFIG_MEMCG_KMEM
613 /*
614  * This will be the memcg's index in each cache's ->memcg_params->memcg_caches.
615  * There are two main reasons for not using the css_id for this:
616  *  1) this works better in sparse environments, where we have a lot of memcgs,
617  *     but only a few kmem-limited. Or also, if we have, for instance, 200
618  *     memcgs, and none but the 200th is kmem-limited, we'd have to have a
619  *     200 entry array for that.
620  *
621  *  2) In order not to violate the cgroup API, we would like to do all memory
622  *     allocation in ->create(). At that point, we haven't yet allocated the
623  *     css_id. Having a separate index prevents us from messing with the cgroup
624  *     core for this
625  *
626  * The current size of the caches array is stored in
627  * memcg_limited_groups_array_size.  It will double each time we have to
628  * increase it.
629  */
630 static DEFINE_IDA(kmem_limited_groups);
631 int memcg_limited_groups_array_size;
632 
633 /*
634  * MIN_SIZE is different than 1, because we would like to avoid going through
635  * the alloc/free process all the time. In a small machine, 4 kmem-limited
636  * cgroups is a reasonable guess. In the future, it could be a parameter or
637  * tunable, but that is strictly not necessary.
638  *
639  * MAX_SIZE should be as large as the number of css_ids. Ideally, we could get
640  * this constant directly from cgroup, but it is understandable that this is
641  * better kept as an internal representation in cgroup.c. In any case, the
642  * css_id space is not getting any smaller, and we don't have to necessarily
643  * increase ours as well if it increases.
644  */
645 #define MEMCG_CACHES_MIN_SIZE 4
646 #define MEMCG_CACHES_MAX_SIZE 65535
647 
648 /*
649  * A lot of the calls to the cache allocation functions are expected to be
650  * inlined by the compiler. Since the calls to memcg_kmem_get_cache are
651  * conditional to this static branch, we'll have to allow modules that does
652  * kmem_cache_alloc and the such to see this symbol as well
653  */
654 struct static_key memcg_kmem_enabled_key;
655 EXPORT_SYMBOL(memcg_kmem_enabled_key);
656 
657 static void disarm_kmem_keys(struct mem_cgroup *memcg)
658 {
659         if (memcg_kmem_is_active(memcg)) {
660                 static_key_slow_dec(&memcg_kmem_enabled_key);
661                 ida_simple_remove(&kmem_limited_groups, memcg->kmemcg_id);
662         }
663         /*
664          * This check can't live in kmem destruction function,
665          * since the charges will outlive the cgroup
666          */
667         WARN_ON(res_counter_read_u64(&memcg->kmem, RES_USAGE) != 0);
668 }
669 #else
670 static void disarm_kmem_keys(struct mem_cgroup *memcg)
671 {
672 }
673 #endif /* CONFIG_MEMCG_KMEM */
674 
675 static void disarm_static_keys(struct mem_cgroup *memcg)
676 {
677         disarm_sock_keys(memcg);
678         disarm_kmem_keys(memcg);
679 }
680 
681 static void drain_all_stock_async(struct mem_cgroup *memcg);
682 
683 static struct mem_cgroup_per_zone *
684 mem_cgroup_zoneinfo(struct mem_cgroup *memcg, int nid, int zid)
685 {
686         VM_BUG_ON((unsigned)nid >= nr_node_ids);
687         return &memcg->info.nodeinfo[nid]->zoneinfo[zid];
688 }
689 
690 struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *memcg)
691 {
692         return &memcg->css;
693 }
694 
695 static struct mem_cgroup_per_zone *
696 page_cgroup_zoneinfo(struct mem_cgroup *memcg, struct page *page)
697 {
698         int nid = page_to_nid(page);
699         int zid = page_zonenum(page);
700 
701         return mem_cgroup_zoneinfo(memcg, nid, zid);
702 }
703 
704 static struct mem_cgroup_tree_per_zone *
705 soft_limit_tree_node_zone(int nid, int zid)
706 {
707         return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
708 }
709 
710 static struct mem_cgroup_tree_per_zone *
711 soft_limit_tree_from_page(struct page *page)
712 {
713         int nid = page_to_nid(page);
714         int zid = page_zonenum(page);
715 
716         return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
717 }
718 
719 static void
720 __mem_cgroup_insert_exceeded(struct mem_cgroup *memcg,
721                                 struct mem_cgroup_per_zone *mz,
722                                 struct mem_cgroup_tree_per_zone *mctz,
723                                 unsigned long long new_usage_in_excess)
724 {
725         struct rb_node **p = &mctz->rb_root.rb_node;
726         struct rb_node *parent = NULL;
727         struct mem_cgroup_per_zone *mz_node;
728 
729         if (mz->on_tree)
730                 return;
731 
732         mz->usage_in_excess = new_usage_in_excess;
733         if (!mz->usage_in_excess)
734                 return;
735         while (*p) {
736                 parent = *p;
737                 mz_node = rb_entry(parent, struct mem_cgroup_per_zone,
738                                         tree_node);
739                 if (mz->usage_in_excess < mz_node->usage_in_excess)
740                         p = &(*p)->rb_left;
741                 /*
742                  * We can't avoid mem cgroups that are over their soft
743                  * limit by the same amount
744                  */
745                 else if (mz->usage_in_excess >= mz_node->usage_in_excess)
746                         p = &(*p)->rb_right;
747         }
748         rb_link_node(&mz->tree_node, parent, p);
749         rb_insert_color(&mz->tree_node, &mctz->rb_root);
750         mz->on_tree = true;
751 }
752 
753 static void
754 __mem_cgroup_remove_exceeded(struct mem_cgroup *memcg,
755                                 struct mem_cgroup_per_zone *mz,
756                                 struct mem_cgroup_tree_per_zone *mctz)
757 {
758         if (!mz->on_tree)
759                 return;
760         rb_erase(&mz->tree_node, &mctz->rb_root);
761         mz->on_tree = false;
762 }
763 
764 static void
765 mem_cgroup_remove_exceeded(struct mem_cgroup *memcg,
766                                 struct mem_cgroup_per_zone *mz,
767                                 struct mem_cgroup_tree_per_zone *mctz)
768 {
769         spin_lock(&mctz->lock);
770         __mem_cgroup_remove_exceeded(memcg, mz, mctz);
771         spin_unlock(&mctz->lock);
772 }
773 
774 
775 static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
776 {
777         unsigned long long excess;
778         struct mem_cgroup_per_zone *mz;
779         struct mem_cgroup_tree_per_zone *mctz;
780         int nid = page_to_nid(page);
781         int zid = page_zonenum(page);
782         mctz = soft_limit_tree_from_page(page);
783 
784         /*
785          * Necessary to update all ancestors when hierarchy is used.
786          * because their event counter is not touched.
787          */
788         for (; memcg; memcg = parent_mem_cgroup(memcg)) {
789                 mz = mem_cgroup_zoneinfo(memcg, nid, zid);
790                 excess = res_counter_soft_limit_excess(&memcg->res);
791                 /*
792                  * We have to update the tree if mz is on RB-tree or
793                  * mem is over its softlimit.
794                  */
795                 if (excess || mz->on_tree) {
796                         spin_lock(&mctz->lock);
797                         /* if on-tree, remove it */
798                         if (mz->on_tree)
799                                 __mem_cgroup_remove_exceeded(memcg, mz, mctz);
800                         /*
801                          * Insert again. mz->usage_in_excess will be updated.
802                          * If excess is 0, no tree ops.
803                          */
804                         __mem_cgroup_insert_exceeded(memcg, mz, mctz, excess);
805                         spin_unlock(&mctz->lock);
806                 }
807         }
808 }
809 
810 static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
811 {
812         int node, zone;
813         struct mem_cgroup_per_zone *mz;
814         struct mem_cgroup_tree_per_zone *mctz;
815 
816         for_each_node(node) {
817                 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
818                         mz = mem_cgroup_zoneinfo(memcg, node, zone);
819                         mctz = soft_limit_tree_node_zone(node, zone);
820                         mem_cgroup_remove_exceeded(memcg, mz, mctz);
821                 }
822         }
823 }
824 
825 static struct mem_cgroup_per_zone *
826 __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
827 {
828         struct rb_node *rightmost = NULL;
829         struct mem_cgroup_per_zone *mz;
830 
831 retry:
832         mz = NULL;
833         rightmost = rb_last(&mctz->rb_root);
834         if (!rightmost)
835                 goto done;              /* Nothing to reclaim from */
836 
837         mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node);
838         /*
839          * Remove the node now but someone else can add it back,
840          * we will to add it back at the end of reclaim to its correct
841          * position in the tree.
842          */
843         __mem_cgroup_remove_exceeded(mz->memcg, mz, mctz);
844         if (!res_counter_soft_limit_excess(&mz->memcg->res) ||
845                 !css_tryget(&mz->memcg->css))
846                 goto retry;
847 done:
848         return mz;
849 }
850 
851 static struct mem_cgroup_per_zone *
852 mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
853 {
854         struct mem_cgroup_per_zone *mz;
855 
856         spin_lock(&mctz->lock);
857         mz = __mem_cgroup_largest_soft_limit_node(mctz);
858         spin_unlock(&mctz->lock);
859         return mz;
860 }
861 
862 /*
863  * Implementation Note: reading percpu statistics for memcg.
864  *
865  * Both of vmstat[] and percpu_counter has threshold and do periodic
866  * synchronization to implement "quick" read. There are trade-off between
867  * reading cost and precision of value. Then, we may have a chance to implement
868  * a periodic synchronizion of counter in memcg's counter.
869  *
870  * But this _read() function is used for user interface now. The user accounts
871  * memory usage by memory cgroup and he _always_ requires exact value because
872  * he accounts memory. Even if we provide quick-and-fuzzy read, we always
873  * have to visit all online cpus and make sum. So, for now, unnecessary
874  * synchronization is not implemented. (just implemented for cpu hotplug)
875  *
876  * If there are kernel internal actions which can make use of some not-exact
877  * value, and reading all cpu value can be performance bottleneck in some
878  * common workload, threashold and synchonization as vmstat[] should be
879  * implemented.
880  */
881 static long mem_cgroup_read_stat(struct mem_cgroup *memcg,
882                                  enum mem_cgroup_stat_index idx)
883 {
884         long val = 0;
885         int cpu;
886 
887         get_online_cpus();
888         for_each_online_cpu(cpu)
889                 val += per_cpu(memcg->stat->count[idx], cpu);
890 #ifdef CONFIG_HOTPLUG_CPU
891         spin_lock(&memcg->pcp_counter_lock);
892         val += memcg->nocpu_base.count[idx];
893         spin_unlock(&memcg->pcp_counter_lock);
894 #endif
895         put_online_cpus();
896         return val;
897 }
898 
899 static void mem_cgroup_swap_statistics(struct mem_cgroup *memcg,
900                                          bool charge)
901 {
902         int val = (charge) ? 1 : -1;
903         this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_SWAP], val);
904 }
905 
906 static unsigned long mem_cgroup_read_events(struct mem_cgroup *memcg,
907                                             enum mem_cgroup_events_index idx)
908 {
909         unsigned long val = 0;
910         int cpu;
911 
912         for_each_online_cpu(cpu)
913                 val += per_cpu(memcg->stat->events[idx], cpu);
914 #ifdef CONFIG_HOTPLUG_CPU
915         spin_lock(&memcg->pcp_counter_lock);
916         val += memcg->nocpu_base.events[idx];
917         spin_unlock(&memcg->pcp_counter_lock);
918 #endif
919         return val;
920 }
921 
922 static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
923                                          struct page *page,
924                                          bool anon, int nr_pages)
925 {
926         preempt_disable();
927 
928         /*
929          * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is
930          * counted as CACHE even if it's on ANON LRU.
931          */
932         if (anon)
933                 __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS],
934                                 nr_pages);
935         else
936                 __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_CACHE],
937                                 nr_pages);
938 
939         if (PageTransHuge(page))
940                 __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE],
941                                 nr_pages);
942 
943         /* pagein of a big page is an event. So, ignore page size */
944         if (nr_pages > 0)
945                 __this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGIN]);
946         else {
947                 __this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT]);
948                 nr_pages = -nr_pages; /* for event */
949         }
950 
951         __this_cpu_add(memcg->stat->nr_page_events, nr_pages);
952 
953         preempt_enable();
954 }
955 
956 unsigned long
957 mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
958 {
959         struct mem_cgroup_per_zone *mz;
960 
961         mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
962         return mz->lru_size[lru];
963 }
964 
965 static unsigned long
966 mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, int nid, int zid,
967                         unsigned int lru_mask)
968 {
969         struct mem_cgroup_per_zone *mz;
970         enum lru_list lru;
971         unsigned long ret = 0;
972 
973         mz = mem_cgroup_zoneinfo(memcg, nid, zid);
974 
975         for_each_lru(lru) {
976                 if (BIT(lru) & lru_mask)
977                         ret += mz->lru_size[lru];
978         }
979         return ret;
980 }
981 
982 static unsigned long
983 mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
984                         int nid, unsigned int lru_mask)
985 {
986         u64 total = 0;
987         int zid;
988 
989         for (zid = 0; zid < MAX_NR_ZONES; zid++)
990                 total += mem_cgroup_zone_nr_lru_pages(memcg,
991                                                 nid, zid, lru_mask);
992 
993         return total;
994 }
995 
996 static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
997                         unsigned int lru_mask)
998 {
999         int nid;
1000         u64 total = 0;
1001 
1002         for_each_node_state(nid, N_MEMORY)
1003                 total += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask);
1004         return total;
1005 }
1006 
1007 static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
1008                                        enum mem_cgroup_events_target target)
1009 {
1010         unsigned long val, next;
1011 
1012         val = __this_cpu_read(memcg->stat->nr_page_events);
1013         next = __this_cpu_read(memcg->stat->targets[target]);
1014         /* from time_after() in jiffies.h */
1015         if ((long)next - (long)val < 0) {
1016                 switch (target) {
1017                 case MEM_CGROUP_TARGET_THRESH:
1018                         next = val + THRESHOLDS_EVENTS_TARGET;
1019                         break;
1020                 case MEM_CGROUP_TARGET_SOFTLIMIT:
1021                         next = val + SOFTLIMIT_EVENTS_TARGET;
1022                         break;
1023                 case MEM_CGROUP_TARGET_NUMAINFO:
1024                         next = val + NUMAINFO_EVENTS_TARGET;
1025                         break;
1026                 default:
1027                         break;
1028                 }
1029                 __this_cpu_write(memcg->stat->targets[target], next);
1030                 return true;
1031         }
1032         return false;
1033 }
1034 
1035 /*
1036  * Check events in order.
1037  *
1038  */
1039 static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
1040 {
1041         preempt_disable();
1042         /* threshold event is triggered in finer grain than soft limit */
1043         if (unlikely(mem_cgroup_event_ratelimit(memcg,
1044                                                 MEM_CGROUP_TARGET_THRESH))) {
1045                 bool do_softlimit;
1046                 bool do_numainfo __maybe_unused;
1047 
1048                 do_softlimit = mem_cgroup_event_ratelimit(memcg,
1049                                                 MEM_CGROUP_TARGET_SOFTLIMIT);
1050 #if MAX_NUMNODES > 1
1051                 do_numainfo = mem_cgroup_event_ratelimit(memcg,
1052                                                 MEM_CGROUP_TARGET_NUMAINFO);
1053 #endif
1054                 preempt_enable();
1055 
1056                 mem_cgroup_threshold(memcg);
1057                 if (unlikely(do_softlimit))
1058                         mem_cgroup_update_tree(memcg, page);
1059 #if MAX_NUMNODES > 1
1060                 if (unlikely(do_numainfo))
1061                         atomic_inc(&memcg->numainfo_events);
1062 #endif
1063         } else
1064                 preempt_enable();
1065 }
1066 
1067 struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
1068 {
1069         return mem_cgroup_from_css(
1070                 cgroup_subsys_state(cont, mem_cgroup_subsys_id));
1071 }
1072 
1073 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
1074 {
1075         /*
1076          * mm_update_next_owner() may clear mm->owner to NULL
1077          * if it races with swapoff, page migration, etc.
1078          * So this can be called with p == NULL.
1079          */
1080         if (unlikely(!p))
1081                 return NULL;
1082 
1083         return mem_cgroup_from_css(task_subsys_state(p, mem_cgroup_subsys_id));
1084 }
1085 
1086 struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
1087 {
1088         struct mem_cgroup *memcg = NULL;
1089 
1090         if (!mm)
1091                 return NULL;
1092         /*
1093          * Because we have no locks, mm->owner's may be being moved to other
1094          * cgroup. We use css_tryget() here even if this looks
1095          * pessimistic (rather than adding locks here).
1096          */
1097         rcu_read_lock();
1098         do {
1099                 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
1100                 if (unlikely(!memcg))
1101                         break;
1102         } while (!css_tryget(&memcg->css));
1103         rcu_read_unlock();
1104         return memcg;
1105 }
1106 
1107 /*
1108  * Returns a next (in a pre-order walk) alive memcg (with elevated css
1109  * ref. count) or NULL if the whole root's subtree has been visited.
1110  *
1111  * helper function to be used by mem_cgroup_iter
1112  */
1113 static struct mem_cgroup *__mem_cgroup_iter_next(struct mem_cgroup *root,
1114                 struct mem_cgroup *last_visited)
1115 {
1116         struct cgroup *prev_cgroup, *next_cgroup;
1117 
1118         /*
1119          * Root is not visited by cgroup iterators so it needs an
1120          * explicit visit.
1121          */
1122         if (!last_visited)
1123                 return root;
1124 
1125         prev_cgroup = (last_visited == root) ? NULL
1126                 : last_visited->css.cgroup;
1127 skip_node:
1128         next_cgroup = cgroup_next_descendant_pre(
1129                         prev_cgroup, root->css.cgroup);
1130 
1131         /*
1132          * Even if we found a group we have to make sure it is
1133          * alive. css && !memcg means that the groups should be
1134          * skipped and we should continue the tree walk.
1135          * last_visited css is safe to use because it is
1136          * protected by css_get and the tree walk is rcu safe.
1137          */
1138         if (next_cgroup) {
1139                 struct mem_cgroup *mem = mem_cgroup_from_cont(
1140                                 next_cgroup);
1141                 if (css_tryget(&mem->css))
1142                         return mem;
1143                 else {
1144                         prev_cgroup = next_cgroup;
1145                         goto skip_node;
1146                 }
1147         }
1148 
1149         return NULL;
1150 }
1151 
1152 /**
1153  * mem_cgroup_iter - iterate over memory cgroup hierarchy
1154  * @root: hierarchy root
1155  * @prev: previously returned memcg, NULL on first invocation
1156  * @reclaim: cookie for shared reclaim walks, NULL for full walks
1157  *
1158  * Returns references to children of the hierarchy below @root, or
1159  * @root itself, or %NULL after a full round-trip.
1160  *
1161  * Caller must pass the return value in @prev on subsequent
1162  * invocations for reference counting, or use mem_cgroup_iter_break()
1163  * to cancel a hierarchy walk before the round-trip is complete.
1164  *
1165  * Reclaimers can specify a zone and a priority level in @reclaim to
1166  * divide up the memcgs in the hierarchy among all concurrent
1167  * reclaimers operating on the same zone and priority.
1168  */
1169 struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
1170                                    struct mem_cgroup *prev,
1171                                    struct mem_cgroup_reclaim_cookie *reclaim)
1172 {
1173         struct mem_cgroup *memcg = NULL;
1174         struct mem_cgroup *last_visited = NULL;
1175         unsigned long uninitialized_var(dead_count);
1176 
1177         if (mem_cgroup_disabled())
1178                 return NULL;
1179 
1180         if (!root)
1181                 root = root_mem_cgroup;
1182 
1183         if (prev && !reclaim)
1184                 last_visited = prev;
1185 
1186         if (!root->use_hierarchy && root != root_mem_cgroup) {
1187                 if (prev)
1188                         goto out_css_put;
1189                 return root;
1190         }
1191 
1192         rcu_read_lock();
1193         while (!memcg) {
1194                 struct mem_cgroup_reclaim_iter *uninitialized_var(iter);
1195 
1196                 if (reclaim) {
1197                         int nid = zone_to_nid(reclaim->zone);
1198                         int zid = zone_idx(reclaim->zone);
1199                         struct mem_cgroup_per_zone *mz;
1200 
1201                         mz = mem_cgroup_zoneinfo(root, nid, zid);
1202                         iter = &mz->reclaim_iter[reclaim->priority];
1203                         if (prev && reclaim->generation != iter->generation) {
1204                                 iter->last_visited = NULL;
1205                                 goto out_unlock;
1206                         }
1207 
1208                         /*
1209                          * If the dead_count mismatches, a destruction
1210                          * has happened or is happening concurrently.
1211                          * If the dead_count matches, a destruction
1212                          * might still happen concurrently, but since
1213                          * we checked under RCU, that destruction
1214                          * won't free the object until we release the
1215                          * RCU reader lock.  Thus, the dead_count
1216                          * check verifies the pointer is still valid,
1217                          * css_tryget() verifies the cgroup pointed to
1218                          * is alive.
1219                          */
1220                         dead_count = atomic_read(&root->dead_count);
1221                         if (dead_count == iter->last_dead_count) {
1222                                 smp_rmb();
1223                                 last_visited = iter->last_visited;
1224                                 if (last_visited && last_visited != root &&
1225                                     !css_tryget(&last_visited->css))
1226                                         last_visited = NULL;
1227                         }
1228                 }
1229 
1230                 memcg = __mem_cgroup_iter_next(root, last_visited);
1231 
1232                 if (reclaim) {
1233                         if (last_visited && last_visited != root)
1234                                 css_put(&last_visited->css);
1235 
1236                         iter->last_visited = memcg;
1237                         smp_wmb();
1238                         iter->last_dead_count = dead_count;
1239 
1240                         if (!memcg)
1241                                 iter->generation++;
1242                         else if (!prev && memcg)
1243                                 reclaim->generation = iter->generation;
1244                 }
1245 
1246                 if (prev && !memcg)
1247                         goto out_unlock;
1248         }
1249 out_unlock:
1250         rcu_read_unlock();
1251 out_css_put:
1252         if (prev && prev != root)
1253                 css_put(&prev->css);
1254 
1255         return memcg;
1256 }
1257 
1258 /**
1259  * mem_cgroup_iter_break - abort a hierarchy walk prematurely
1260  * @root: hierarchy root
1261  * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
1262  */
1263 void mem_cgroup_iter_break(struct mem_cgroup *root,
1264                            struct mem_cgroup *prev)
1265 {
1266         if (!root)
1267                 root = root_mem_cgroup;
1268         if (prev && prev != root)
1269                 css_put(&prev->css);
1270 }
1271 
1272 /*
1273  * Iteration constructs for visiting all cgroups (under a tree).  If
1274  * loops are exited prematurely (break), mem_cgroup_iter_break() must
1275  * be used for reference counting.
1276  */
1277 #define for_each_mem_cgroup_tree(iter, root)            \
1278         for (iter = mem_cgroup_iter(root, NULL, NULL);  \
1279              iter != NULL;                              \
1280              iter = mem_cgroup_iter(root, iter, NULL))
1281 
1282 #define for_each_mem_cgroup(iter)                       \
1283         for (iter = mem_cgroup_iter(NULL, NULL, NULL);  \
1284              iter != NULL;                              \
1285              iter = mem_cgroup_iter(NULL, iter, NULL))
1286 
1287 void __mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx)
1288 {
1289         struct mem_cgroup *memcg;
1290 
1291         rcu_read_lock();
1292         memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
1293         if (unlikely(!memcg))
1294                 goto out;
1295 
1296         switch (idx) {
1297         case PGFAULT:
1298                 this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGFAULT]);
1299                 break;
1300         case PGMAJFAULT:
1301                 this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGMAJFAULT]);
1302                 break;
1303         default:
1304                 BUG();
1305         }
1306 out:
1307         rcu_read_unlock();
1308 }
1309 EXPORT_SYMBOL(__mem_cgroup_count_vm_event);
1310 
1311 /**
1312  * mem_cgroup_zone_lruvec - get the lru list vector for a zone and memcg
1313  * @zone: zone of the wanted lruvec
1314  * @memcg: memcg of the wanted lruvec
1315  *
1316  * Returns the lru list vector holding pages for the given @zone and
1317  * @mem.  This can be the global zone lruvec, if the memory controller
1318  * is disabled.
1319  */
1320 struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,
1321                                       struct mem_cgroup *memcg)
1322 {
1323         struct mem_cgroup_per_zone *mz;
1324         struct lruvec *lruvec;
1325 
1326         if (mem_cgroup_disabled()) {
1327                 lruvec = &zone->lruvec;
1328                 goto out;
1329         }
1330 
1331         mz = mem_cgroup_zoneinfo(memcg, zone_to_nid(zone), zone_idx(zone));
1332         lruvec = &mz->lruvec;
1333 out:
1334         /*
1335          * Since a node can be onlined after the mem_cgroup was created,
1336          * we have to be prepared to initialize lruvec->zone here;
1337          * and if offlined then reonlined, we need to reinitialize it.
1338          */
1339         if (unlikely(lruvec->zone != zone))
1340                 lruvec->zone = zone;
1341         return lruvec;
1342 }
1343 
1344 /*
1345  * Following LRU functions are allowed to be used without PCG_LOCK.
1346  * Operations are called by routine of global LRU independently from memcg.
1347  * What we have to take care of here is validness of pc->mem_cgroup.
1348  *
1349  * Changes to pc->mem_cgroup happens when
1350  * 1. charge
1351  * 2. moving account
1352  * In typical case, "charge" is done before add-to-lru. Exception is SwapCache.
1353  * It is added to LRU before charge.
1354  * If PCG_USED bit is not set, page_cgroup is not added to this private LRU.
1355  * When moving account, the page is not on LRU. It's isolated.
1356  */
1357 
1358 /**
1359  * mem_cgroup_page_lruvec - return lruvec for adding an lru page
1360  * @page: the page
1361  * @zone: zone of the page
1362  */
1363 struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct zone *zone)
1364 {
1365         struct mem_cgroup_per_zone *mz;
1366         struct mem_cgroup *memcg;
1367         struct page_cgroup *pc;
1368         struct lruvec *lruvec;
1369 
1370         if (mem_cgroup_disabled()) {
1371                 lruvec = &zone->lruvec;
1372                 goto out;
1373         }
1374 
1375         pc = lookup_page_cgroup(page);
1376         memcg = pc->mem_cgroup;
1377 
1378         /*
1379          * Surreptitiously switch any uncharged offlist page to root:
1380          * an uncharged page off lru does nothing to secure
1381          * its former mem_cgroup from sudden removal.
1382          *
1383          * Our caller holds lru_lock, and PageCgroupUsed is updated
1384          * under page_cgroup lock: between them, they make all uses
1385          * of pc->mem_cgroup safe.
1386          */
1387         if (!PageLRU(page) && !PageCgroupUsed(pc) && memcg != root_mem_cgroup)
1388                 pc->mem_cgroup = memcg = root_mem_cgroup;
1389 
1390         mz = page_cgroup_zoneinfo(memcg, page);
1391         lruvec = &mz->lruvec;
1392 out:
1393         /*
1394          * Since a node can be onlined after the mem_cgroup was created,
1395          * we have to be prepared to initialize lruvec->zone here;
1396          * and if offlined then reonlined, we need to reinitialize it.
1397          */
1398         if (unlikely(lruvec->zone != zone))
1399                 lruvec->zone = zone;
1400         return lruvec;
1401 }
1402 
1403 /**
1404  * mem_cgroup_update_lru_size - account for adding or removing an lru page
1405  * @lruvec: mem_cgroup per zone lru vector
1406  * @lru: index of lru list the page is sitting on
1407  * @nr_pages: positive when adding or negative when removing
1408  *
1409  * This function must be called when a page is added to or removed from an
1410  * lru list.
1411  */
1412 void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
1413                                 int nr_pages)
1414 {
1415         struct mem_cgroup_per_zone *mz;
1416         unsigned long *lru_size;
1417 
1418         if (mem_cgroup_disabled())
1419                 return;
1420 
1421         mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
1422         lru_size = mz->lru_size + lru;
1423         *lru_size += nr_pages;
1424         VM_BUG_ON((long)(*lru_size) < 0);
1425 }
1426 
1427 /*
1428  * Checks whether given mem is same or in the root_mem_cgroup's
1429  * hierarchy subtree
1430  */
1431 bool __mem_cgroup_same_or_subtree(const struct mem_cgroup *root_memcg,
1432                                   struct mem_cgroup *memcg)
1433 {
1434         if (root_memcg == memcg)
1435                 return true;
1436         if (!root_memcg->use_hierarchy || !memcg)
1437                 return false;
1438         return css_is_ancestor(&memcg->css, &root_memcg->css);
1439 }
1440 
1441 static bool mem_cgroup_same_or_subtree(const struct mem_cgroup *root_memcg,
1442                                        struct mem_cgroup *memcg)
1443 {
1444         bool ret;
1445 
1446         rcu_read_lock();
1447         ret = __mem_cgroup_same_or_subtree(root_memcg, memcg);
1448         rcu_read_unlock();
1449         return ret;
1450 }
1451 
1452 int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *memcg)
1453 {
1454         int ret;
1455         struct mem_cgroup *curr = NULL;
1456         struct task_struct *p;
1457 
1458         p = find_lock_task_mm(task);
1459         if (p) {
1460                 curr = try_get_mem_cgroup_from_mm(p->mm);
1461                 task_unlock(p);
1462         } else {
1463                 /*
1464                  * All threads may have already detached their mm's, but the oom
1465                  * killer still needs to detect if they have already been oom
1466                  * killed to prevent needlessly killing additional tasks.
1467                  */
1468                 task_lock(task);
1469                 curr = mem_cgroup_from_task(task);
1470                 if (curr)
1471                         css_get(&curr->css);
1472                 task_unlock(task);
1473         }
1474         if (!curr)
1475                 return 0;
1476         /*
1477          * We should check use_hierarchy of "memcg" not "curr". Because checking
1478          * use_hierarchy of "curr" here make this function true if hierarchy is
1479          * enabled in "curr" and "curr" is a child of "memcg" in *cgroup*
1480          * hierarchy(even if use_hierarchy is disabled in "memcg").
1481          */
1482         ret = mem_cgroup_same_or_subtree(memcg, curr);
1483         css_put(&curr->css);
1484         return ret;
1485 }
1486 
1487 int mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec)
1488 {
1489         unsigned long inactive_ratio;
1490         unsigned long inactive;
1491         unsigned long active;
1492         unsigned long gb;
1493 
1494         inactive = mem_cgroup_get_lru_size(lruvec, LRU_INACTIVE_ANON);
1495         active = mem_cgroup_get_lru_size(lruvec, LRU_ACTIVE_ANON);
1496 
1497         gb = (inactive + active) >> (30 - PAGE_SHIFT);
1498         if (gb)
1499                 inactive_ratio = int_sqrt(10 * gb);
1500         else
1501                 inactive_ratio = 1;
1502 
1503         return inactive * inactive_ratio < active;
1504 }
1505 
1506 #define mem_cgroup_from_res_counter(counter, member)    \
1507         container_of(counter, struct mem_cgroup, member)
1508 
1509 /**
1510  * mem_cgroup_margin - calculate chargeable space of a memory cgroup
1511  * @memcg: the memory cgroup
1512  *
1513  * Returns the maximum amount of memory @mem can be charged with, in
1514  * pages.
1515  */
1516 static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
1517 {
1518         unsigned long long margin;
1519 
1520         margin = res_counter_margin(&memcg->res);
1521         if (do_swap_account)
1522                 margin = min(margin, res_counter_margin(&memcg->memsw));
1523         return margin >> PAGE_SHIFT;
1524 }
1525 
1526 int mem_cgroup_swappiness(struct mem_cgroup *memcg)
1527 {
1528         struct cgroup *cgrp = memcg->css.cgroup;
1529 
1530         /* root ? */
1531         if (cgrp->parent == NULL)
1532                 return vm_swappiness;
1533 
1534         return memcg->swappiness;
1535 }
1536 
1537 /*
1538  * memcg->moving_account is used for checking possibility that some thread is
1539  * calling move_account(). When a thread on CPU-A starts moving pages under
1540  * a memcg, other threads should check memcg->moving_account under
1541  * rcu_read_lock(), like this:
1542  *
1543  *         CPU-A                                    CPU-B
1544  *                                              rcu_read_lock()
1545  *         memcg->moving_account+1              if (memcg->mocing_account)
1546  *                                                   take heavy locks.
1547  *         synchronize_rcu()                    update something.
1548  *                                              rcu_read_unlock()
1549  *         start move here.
1550  */
1551 
1552 /* for quick checking without looking up memcg */
1553 atomic_t memcg_moving __read_mostly;
1554 
1555 static void mem_cgroup_start_move(struct mem_cgroup *memcg)
1556 {
1557         atomic_inc(&memcg_moving);
1558         atomic_inc(&memcg->moving_account);
1559         synchronize_rcu();
1560 }
1561 
1562 static void mem_cgroup_end_move(struct mem_cgroup *memcg)
1563 {
1564         /*
1565          * Now, mem_cgroup_clear_mc() may call this function with NULL.
1566          * We check NULL in callee rather than caller.
1567          */
1568         if (memcg) {
1569                 atomic_dec(&memcg_moving);
1570                 atomic_dec(&memcg->moving_account);
1571         }
1572 }
1573 
1574 /*
1575  * 2 routines for checking "mem" is under move_account() or not.
1576  *
1577  * mem_cgroup_stolen() -  checking whether a cgroup is mc.from or not. This
1578  *                        is used for avoiding races in accounting.  If true,
1579  *                        pc->mem_cgroup may be overwritten.
1580  *
1581  * mem_cgroup_under_move() - checking a cgroup is mc.from or mc.to or
1582  *                        under hierarchy of moving cgroups. This is for
1583  *                        waiting at hith-memory prressure caused by "move".
1584  */
1585 
1586 static bool mem_cgroup_stolen(struct mem_cgroup *memcg)
1587 {
1588         VM_BUG_ON(!rcu_read_lock_held());
1589         return atomic_read(&memcg->moving_account) > 0;
1590 }
1591 
1592 static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
1593 {
1594         struct mem_cgroup *from;
1595         struct mem_cgroup *to;
1596         bool ret = false;
1597         /*
1598          * Unlike task_move routines, we access mc.to, mc.from not under
1599          * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
1600          */
1601         spin_lock(&mc.lock);
1602         from = mc.from;
1603         to = mc.to;
1604         if (!from)
1605                 goto unlock;
1606 
1607         ret = mem_cgroup_same_or_subtree(memcg, from)
1608                 || mem_cgroup_same_or_subtree(memcg, to);
1609 unlock:
1610         spin_unlock(&mc.lock);
1611         return ret;
1612 }
1613 
1614 static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
1615 {
1616         if (mc.moving_task && current != mc.moving_task) {
1617                 if (mem_cgroup_under_move(memcg)) {
1618                         DEFINE_WAIT(wait);
1619                         prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
1620                         /* moving charge context might have finished. */
1621                         if (mc.moving_task)
1622                                 schedule();
1623                         finish_wait(&mc.waitq, &wait);
1624                         return true;
1625                 }
1626         }
1627         return false;
1628 }
1629 
1630 /*
1631  * Take this lock when
1632  * - a code tries to modify page's memcg while it's USED.
1633  * - a code tries to modify page state accounting in a memcg.
1634  * see mem_cgroup_stolen(), too.
1635  */
1636 static void move_lock_mem_cgroup(struct mem_cgroup *memcg,
1637                                   unsigned long *flags)
1638 {
1639         spin_lock_irqsave(&memcg->move_lock, *flags);
1640 }
1641 
1642 static void move_unlock_mem_cgroup(struct mem_cgroup *memcg,
1643                                 unsigned long *flags)
1644 {
1645         spin_unlock_irqrestore(&memcg->move_lock, *flags);
1646 }
1647 
1648 #define K(x) ((x) << (PAGE_SHIFT-10))
1649 /**
1650  * mem_cgroup_print_oom_info: Print OOM information relevant to memory controller.
1651  * @memcg: The memory cgroup that went over limit
1652  * @p: Task that is going to be killed
1653  *
1654  * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1655  * enabled
1656  */
1657 void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
1658 {
1659         struct cgroup *task_cgrp;
1660         struct cgroup *mem_cgrp;
1661         /*
1662          * Need a buffer in BSS, can't rely on allocations. The code relies
1663          * on the assumption that OOM is serialized for memory controller.
1664          * If this assumption is broken, revisit this code.
1665          */
1666         static char memcg_name[PATH_MAX];
1667         int ret;
1668         struct mem_cgroup *iter;
1669         unsigned int i;
1670 
1671         if (!p)
1672                 return;
1673 
1674         rcu_read_lock();
1675 
1676         mem_cgrp = memcg->css.cgroup;
1677         task_cgrp = task_cgroup(p, mem_cgroup_subsys_id);
1678 
1679         ret = cgroup_path(task_cgrp, memcg_name, PATH_MAX);
1680         if (ret < 0) {
1681                 /*
1682                  * Unfortunately, we are unable to convert to a useful name
1683                  * But we'll still print out the usage information
1684                  */
1685                 rcu_read_unlock();
1686                 goto done;
1687         }
1688         rcu_read_unlock();
1689 
1690         pr_info("Task in %s killed", memcg_name);
1691 
1692         rcu_read_lock();
1693         ret = cgroup_path(mem_cgrp, memcg_name, PATH_MAX);
1694         if (ret < 0) {
1695                 rcu_read_unlock();
1696                 goto done;
1697         }
1698         rcu_read_unlock();
1699 
1700         /*
1701          * Continues from above, so we don't need an KERN_ level
1702          */
1703         pr_cont(" as a result of limit of %s\n", memcg_name);
1704 done:
1705 
1706         pr_info("memory: usage %llukB, limit %llukB, failcnt %llu\n",
1707                 res_counter_read_u64(&memcg->res, RES_USAGE) >> 10,
1708                 res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10,
1709                 res_counter_read_u64(&memcg->res, RES_FAILCNT));
1710         pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %llu\n",
1711                 res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10,
1712                 res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10,
1713                 res_counter_read_u64(&memcg->memsw, RES_FAILCNT));
1714         pr_info("kmem: usage %llukB, limit %llukB, failcnt %llu\n",
1715                 res_counter_read_u64(&memcg->kmem, RES_USAGE) >> 10,
1716                 res_counter_read_u64(&memcg->kmem, RES_LIMIT) >> 10,
1717                 res_counter_read_u64(&memcg->kmem, RES_FAILCNT));
1718 
1719         for_each_mem_cgroup_tree(iter, memcg) {
1720                 pr_info("Memory cgroup stats");
1721 
1722                 rcu_read_lock();
1723                 ret = cgroup_path(iter->css.cgroup, memcg_name, PATH_MAX);
1724                 if (!ret)
1725                         pr_cont(" for %s", memcg_name);
1726                 rcu_read_unlock();
1727                 pr_cont(":");
1728 
1729                 for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
1730                         if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
1731                                 continue;
1732                         pr_cont(" %s:%ldKB", mem_cgroup_stat_names[i],
1733                                 K(mem_cgroup_read_stat(iter, i)));
1734                 }
1735 
1736                 for (i = 0; i < NR_LRU_LISTS; i++)
1737                         pr_cont(" %s:%luKB", mem_cgroup_lru_names[i],
1738                                 K(mem_cgroup_nr_lru_pages(iter, BIT(i))));
1739 
1740                 pr_cont("\n");
1741         }
1742 }
1743 
1744 /*
1745  * This function returns the number of memcg under hierarchy tree. Returns
1746  * 1(self count) if no children.
1747  */
1748 static int mem_cgroup_count_children(struct mem_cgroup *memcg)
1749 {
1750         int num = 0;
1751         struct mem_cgroup *iter;
1752 
1753         for_each_mem_cgroup_tree(iter, memcg)
1754                 num++;
1755         return num;
1756 }
1757 
1758 /*
1759  * Return the memory (and swap, if configured) limit for a memcg.
1760  */
1761 static u64 mem_cgroup_get_limit(struct mem_cgroup *memcg)
1762 {
1763         u64 limit;
1764 
1765         limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
1766 
1767         /*
1768          * Do not consider swap space if we cannot swap due to swappiness
1769          */
1770         if (mem_cgroup_swappiness(memcg)) {
1771                 u64 memsw;
1772 
1773                 limit += total_swap_pages << PAGE_SHIFT;
1774                 memsw = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
1775 
1776                 /*
1777                  * If memsw is finite and limits the amount of swap space
1778                  * available to this memcg, return that limit.
1779                  */
1780                 limit = min(limit, memsw);
1781         }
1782 
1783         return limit;
1784 }
1785 
1786 static void mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
1787                                      int order)
1788 {
1789         struct mem_cgroup *iter;
1790         unsigned long chosen_points = 0;
1791         unsigned long totalpages;
1792         unsigned int points = 0;
1793         struct task_struct *chosen = NULL;
1794 
1795         /*
1796          * If current has a pending SIGKILL or is exiting, then automatically
1797          * select it.  The goal is to allow it to allocate so that it may
1798          * quickly exit and free its memory.
1799          */
1800         if (fatal_signal_pending(current) || current->flags & PF_EXITING) {
1801                 set_thread_flag(TIF_MEMDIE);
1802                 return;
1803         }
1804 
1805         check_panic_on_oom(CONSTRAINT_MEMCG, gfp_mask, order, NULL);
1806         totalpages = mem_cgroup_get_limit(memcg) >> PAGE_SHIFT ? : 1;
1807         for_each_mem_cgroup_tree(iter, memcg) {
1808                 struct cgroup *cgroup = iter->css.cgroup;
1809                 struct cgroup_iter it;
1810                 struct task_struct *task;
1811 
1812                 cgroup_iter_start(cgroup, &it);
1813                 while ((task = cgroup_iter_next(cgroup, &it))) {
1814                         switch (oom_scan_process_thread(task, totalpages, NULL,
1815                                                         false)) {
1816                         case OOM_SCAN_SELECT:
1817                                 if (chosen)
1818                                         put_task_struct(chosen);
1819                                 chosen = task;
1820                                 chosen_points = ULONG_MAX;
1821                                 get_task_struct(chosen);
1822                                 /* fall through */
1823                         case OOM_SCAN_CONTINUE:
1824                                 continue;
1825                         case OOM_SCAN_ABORT:
1826                                 cgroup_iter_end(cgroup, &it);
1827                                 mem_cgroup_iter_break(memcg, iter);
1828                                 if (chosen)
1829                                         put_task_struct(chosen);
1830                                 return;
1831                         case OOM_SCAN_OK:
1832                                 break;
1833                         };
1834                         points = oom_badness(task, memcg, NULL, totalpages);
1835                         if (points > chosen_points) {
1836                                 if (chosen)
1837                                         put_task_struct(chosen);
1838                                 chosen = task;
1839                                 chosen_points = points;
1840                                 get_task_struct(chosen);
1841                         }
1842                 }
1843                 cgroup_iter_end(cgroup, &it);
1844         }
1845 
1846         if (!chosen)
1847                 return;
1848         points = chosen_points * 1000 / totalpages;
1849         oom_kill_process(chosen, gfp_mask, order, points, totalpages, memcg,
1850                          NULL, "Memory cgroup out of memory");
1851 }
1852 
1853 static unsigned long mem_cgroup_reclaim(struct mem_cgroup *memcg,
1854                                         gfp_t gfp_mask,
1855                                         unsigned long flags)
1856 {
1857         unsigned long total = 0;
1858         bool noswap = false;
1859         int loop;
1860 
1861         if (flags & MEM_CGROUP_RECLAIM_NOSWAP)
1862                 noswap = true;
1863         if (!(flags & MEM_CGROUP_RECLAIM_SHRINK) && memcg->memsw_is_minimum)
1864                 noswap = true;
1865 
1866         for (loop = 0; loop < MEM_CGROUP_MAX_RECLAIM_LOOPS; loop++) {
1867                 if (loop)
1868                         drain_all_stock_async(memcg);
1869                 total += try_to_free_mem_cgroup_pages(memcg, gfp_mask, noswap);
1870                 /*
1871                  * Allow limit shrinkers, which are triggered directly
1872                  * by userspace, to catch signals and stop reclaim
1873                  * after minimal progress, regardless of the margin.
1874                  */
1875                 if (total && (flags & MEM_CGROUP_RECLAIM_SHRINK))
1876                         break;
1877                 if (mem_cgroup_margin(memcg))
1878                         break;
1879                 /*
1880                  * If nothing was reclaimed after two attempts, there
1881                  * may be no reclaimable pages in this hierarchy.
1882                  */
1883                 if (loop && !total)
1884                         break;
1885         }
1886         return total;
1887 }
1888 
1889 /**
1890  * test_mem_cgroup_node_reclaimable
1891  * @memcg: the target memcg
1892  * @nid: the node ID to be checked.
1893  * @noswap : specify true here if the user wants flle only information.
1894  *
1895  * This function returns whether the specified memcg contains any
1896  * reclaimable pages on a node. Returns true if there are any reclaimable
1897  * pages in the node.
1898  */
1899 static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
1900                 int nid, bool noswap)
1901 {
1902         if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE))
1903                 return true;
1904         if (noswap || !total_swap_pages)
1905                 return false;
1906         if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON))
1907                 return true;
1908         return false;
1909 
1910 }
1911 #if MAX_NUMNODES > 1
1912 
1913 /*
1914  * Always updating the nodemask is not very good - even if we have an empty
1915  * list or the wrong list here, we can start from some node and traverse all
1916  * nodes based on the zonelist. So update the list loosely once per 10 secs.
1917  *
1918  */
1919 static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
1920 {
1921         int nid;
1922         /*
1923          * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
1924          * pagein/pageout changes since the last update.
1925          */
1926         if (!atomic_read(&memcg->numainfo_events))
1927                 return;
1928         if (atomic_inc_return(&memcg->numainfo_updating) > 1)
1929                 return;
1930 
1931         /* make a nodemask where this memcg uses memory from */
1932         memcg->scan_nodes = node_states[N_MEMORY];
1933 
1934         for_each_node_mask(nid, node_states[N_MEMORY]) {
1935 
1936                 if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
1937                         node_clear(nid, memcg->scan_nodes);
1938         }
1939 
1940         atomic_set(&memcg->numainfo_events, 0);
1941         atomic_set(&memcg->numainfo_updating, 0);
1942 }
1943 
1944 /*
1945  * Selecting a node where we start reclaim from. Because what we need is just
1946  * reducing usage counter, start from anywhere is O,K. Considering
1947  * memory reclaim from current node, there are pros. and cons.
1948  *
1949  * Freeing memory from current node means freeing memory from a node which
1950  * we'll use or we've used. So, it may make LRU bad. And if several threads
1951  * hit limits, it will see a contention on a node. But freeing from remote
1952  * node means more costs for memory reclaim because of memory latency.
1953  *
1954  * Now, we use round-robin. Better algorithm is welcomed.
1955  */
1956 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1957 {
1958         int node;
1959 
1960         mem_cgroup_may_update_nodemask(memcg);
1961         node = memcg->last_scanned_node;
1962 
1963         node = next_node(node, memcg->scan_nodes);
1964         if (node == MAX_NUMNODES)
1965                 node = first_node(memcg->scan_nodes);
1966         /*
1967          * We call this when we hit limit, not when pages are added to LRU.
1968          * No LRU may hold pages because all pages are UNEVICTABLE or
1969          * memcg is too small and all pages are not on LRU. In that case,
1970          * we use curret node.
1971          */
1972         if (unlikely(node == MAX_NUMNODES))
1973                 node = numa_node_id();
1974 
1975         memcg->last_scanned_node = node;
1976         return node;
1977 }
1978 
1979 /*
1980  * Check all nodes whether it contains reclaimable pages or not.
1981  * For quick scan, we make use of scan_nodes. This will allow us to skip
1982  * unused nodes. But scan_nodes is lazily updated and may not cotain
1983  * enough new information. We need to do double check.
1984  */
1985 static bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap)
1986 {
1987         int nid;
1988 
1989         /*
1990          * quick check...making use of scan_node.
1991          * We can skip unused nodes.
1992          */
1993         if (!nodes_empty(memcg->scan_nodes)) {
1994                 for (nid = first_node(memcg->scan_nodes);
1995                      nid < MAX_NUMNODES;
1996                      nid = next_node(nid, memcg->scan_nodes)) {
1997 
1998                         if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap))
1999                                 return true;
2000                 }
2001         }
2002         /*
2003          * Check rest of nodes.
2004          */
2005         for_each_node_state(nid, N_MEMORY) {
2006                 if (node_isset(nid, memcg->scan_nodes))
2007                         continue;
2008                 if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap))
2009                         return true;
2010         }
2011         return false;
2012 }
2013 
2014 #else
2015 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
2016 {
2017         return 0;
2018 }
2019 
2020 static bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap)
2021 {
2022         return test_mem_cgroup_node_reclaimable(memcg, 0, noswap);
2023 }
2024 #endif
2025 
2026 static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
2027                                    struct zone *zone,
2028                                    gfp_t gfp_mask,
2029                                    unsigned long *total_scanned)
2030 {
2031         struct mem_cgroup *victim = NULL;
2032         int total = 0;
2033         int loop = 0;
2034         unsigned long excess;
2035         unsigned long nr_scanned;
2036         struct mem_cgroup_reclaim_cookie reclaim = {
2037                 .zone = zone,
2038                 .priority = 0,
2039         };
2040 
2041         excess = res_counter_soft_limit_excess(&root_memcg->res) >> PAGE_SHIFT;
2042 
2043         while (1) {
2044                 victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
2045                 if (!victim) {
2046                         loop++;
2047                         if (loop >= 2) {
2048                                 /*
2049                                  * If we have not been able to reclaim
2050                                  * anything, it might because there are
2051                                  * no reclaimable pages under this hierarchy
2052                                  */
2053                                 if (!total)
2054                                         break;
2055                                 /*
2056                                  * We want to do more targeted reclaim.
2057                                  * excess >> 2 is not to excessive so as to
2058                                  * reclaim too much, nor too less that we keep
2059                                  * coming back to reclaim from this cgroup
2060                                  */
2061                                 if (total >= (excess >> 2) ||
2062                                         (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
2063                                         break;
2064                         }
2065                         continue;
2066                 }
2067                 if (!mem_cgroup_reclaimable(victim, false))
2068                         continue;
2069                 total += mem_cgroup_shrink_node_zone(victim, gfp_mask, false,
2070                                                      zone, &nr_scanned);
2071                 *total_scanned += nr_scanned;
2072                 if (!res_counter_soft_limit_excess(&root_memcg->res))
2073                         break;
2074         }
2075         mem_cgroup_iter_break(root_memcg, victim);
2076         return total;
2077 }
2078 
2079 static DEFINE_SPINLOCK(memcg_oom_lock);
2080 
2081 /*
2082  * Check OOM-Killer is already running under our hierarchy.
2083  * If someone is running, return false.
2084  */
2085 static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
2086 {
2087         struct mem_cgroup *iter, *failed = NULL;
2088 
2089         spin_lock(&memcg_oom_lock);
2090 
2091         for_each_mem_cgroup_tree(iter, memcg) {
2092                 if (iter->oom_lock) {
2093                         /*
2094                          * this subtree of our hierarchy is already locked
2095                          * so we cannot give a lock.
2096                          */
2097                         failed = iter;
2098                         mem_cgroup_iter_break(memcg, iter);
2099                         break;
2100                 } else
2101                         iter->oom_lock = true;
2102         }
2103 
2104         if (failed) {
2105                 /*
2106                  * OK, we failed to lock the whole subtree so we have
2107                  * to clean up what we set up to the failing subtree
2108                  */
2109                 for_each_mem_cgroup_tree(iter, memcg) {
2110                         if (iter == failed) {
2111                                 mem_cgroup_iter_break(memcg, iter);
2112                                 break;
2113                         }
2114                         iter->oom_lock = false;
2115                 }
2116         }
2117 
2118         spin_unlock(&memcg_oom_lock);
2119 
2120         return !failed;
2121 }
2122 
2123 static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
2124 {
2125         struct mem_cgroup *iter;
2126 
2127         spin_lock(&memcg_oom_lock);
2128         for_each_mem_cgroup_tree(iter, memcg)
2129                 iter->oom_lock = false;
2130         spin_unlock(&memcg_oom_lock);
2131 }
2132 
2133 static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
2134 {
2135         struct mem_cgroup *iter;
2136 
2137         for_each_mem_cgroup_tree(iter, memcg)
2138                 atomic_inc(&iter->under_oom);
2139 }
2140 
2141 static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
2142 {
2143         struct mem_cgroup *iter;
2144 
2145         /*
2146          * When a new child is created while the hierarchy is under oom,
2147          * mem_cgroup_oom_lock() may not be called. We have to use
2148          * atomic_add_unless() here.
2149          */
2150         for_each_mem_cgroup_tree(iter, memcg)
2151                 atomic_add_unless(&iter->under_oom, -1, 0);
2152 }
2153 
2154 static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
2155 
2156 struct oom_wait_info {
2157         struct mem_cgroup *memcg;
2158         wait_queue_t    wait;
2159 };
2160 
2161 static int memcg_oom_wake_function(wait_queue_t *wait,
2162         unsigned mode, int sync, void *arg)
2163 {
2164         struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
2165         struct mem_cgroup *oom_wait_memcg;
2166         struct oom_wait_info *oom_wait_info;
2167 
2168         oom_wait_info = container_of(wait, struct oom_wait_info, wait);
2169         oom_wait_memcg = oom_wait_info->memcg;
2170 
2171         /*
2172          * Both of oom_wait_info->memcg and wake_memcg are stable under us.
2173          * Then we can use css_is_ancestor without taking care of RCU.
2174          */
2175         if (!mem_cgroup_same_or_subtree(oom_wait_memcg, wake_memcg)
2176                 && !mem_cgroup_same_or_subtree(wake_memcg, oom_wait_memcg))
2177                 return 0;
2178         return autoremove_wake_function(wait, mode, sync, arg);
2179 }
2180 
2181 static void memcg_wakeup_oom(struct mem_cgroup *memcg)
2182 {
2183         atomic_inc(&memcg->oom_wakeups);
2184         /* for filtering, pass "memcg" as argument. */
2185         __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
2186 }
2187 
2188 static void memcg_oom_recover(struct mem_cgroup *memcg)
2189 {
2190         if (memcg && atomic_read(&memcg->under_oom))
2191                 memcg_wakeup_oom(memcg);
2192 }
2193 
2194 static void mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
2195 {
2196         if (!current->memcg_oom.may_oom)
2197                 return;
2198         /*
2199          * We are in the middle of the charge context here, so we
2200          * don't want to block when potentially sitting on a callstack
2201          * that holds all kinds of filesystem and mm locks.
2202          *
2203          * Also, the caller may handle a failed allocation gracefully
2204          * (like optional page cache readahead) and so an OOM killer
2205          * invocation might not even be necessary.
2206          *
2207          * That's why we don't do anything here except remember the
2208          * OOM context and then deal with it at the end of the page
2209          * fault when the stack is unwound, the locks are released,
2210          * and when we know whether the fault was overall successful.
2211          */
2212         css_get(&memcg->css);
2213         current->memcg_oom.memcg = memcg;
2214         current->memcg_oom.gfp_mask = mask;
2215         current->memcg_oom.order = order;
2216 }
2217 
2218 /**
2219  * mem_cgroup_oom_synchronize - complete memcg OOM handling
2220  * @handle: actually kill/wait or just clean up the OOM state
2221  *
2222  * This has to be called at the end of a page fault if the memcg OOM
2223  * handler was enabled.
2224  *
2225  * Memcg supports userspace OOM handling where failed allocations must
2226  * sleep on a waitqueue until the userspace task resolves the
2227  * situation.  Sleeping directly in the charge context with all kinds
2228  * of locks held is not a good idea, instead we remember an OOM state
2229  * in the task and mem_cgroup_oom_synchronize() has to be called at
2230  * the end of the page fault to complete the OOM handling.
2231  *
2232  * Returns %true if an ongoing memcg OOM situation was detected and
2233  * completed, %false otherwise.
2234  */
2235 bool mem_cgroup_oom_synchronize(bool handle)
2236 {
2237         struct mem_cgroup *memcg = current->memcg_oom.memcg;
2238         struct oom_wait_info owait;
2239         bool locked;
2240 
2241         /* OOM is global, do not handle */
2242         if (!memcg)
2243                 return false;
2244 
2245         if (!handle)
2246                 goto cleanup;
2247 
2248         owait.memcg = memcg;
2249         owait.wait.flags = 0;
2250         owait.wait.func = memcg_oom_wake_function;
2251         owait.wait.private = current;
2252         INIT_LIST_HEAD(&owait.wait.task_list);
2253 
2254         prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
2255         mem_cgroup_mark_under_oom(memcg);
2256 
2257         locked = mem_cgroup_oom_trylock(memcg);
2258 
2259         if (locked)
2260                 mem_cgroup_oom_notify(memcg);
2261 
2262         if (locked && !memcg->oom_kill_disable) {
2263                 mem_cgroup_unmark_under_oom(memcg);
2264                 finish_wait(&memcg_oom_waitq, &owait.wait);
2265                 mem_cgroup_out_of_memory(memcg, current->memcg_oom.gfp_mask,
2266                                          current->memcg_oom.order);
2267         } else {
2268                 schedule();
2269                 mem_cgroup_unmark_under_oom(memcg);
2270                 finish_wait(&memcg_oom_waitq, &owait.wait);
2271         }
2272 
2273         if (locked) {
2274                 mem_cgroup_oom_unlock(memcg);
2275                 /*
2276                  * There is no guarantee that an OOM-lock contender
2277                  * sees the wakeups triggered by the OOM kill
2278                  * uncharges.  Wake any sleepers explicitely.
2279                  */
2280                 memcg_oom_recover(memcg);
2281         }
2282 cleanup:
2283         current->memcg_oom.memcg = NULL;
2284         css_put(&memcg->css);
2285         return true;
2286 }
2287 
2288 /*
2289  * Currently used to update mapped file statistics, but the routine can be
2290  * generalized to update other statistics as well.
2291  *
2292  * Notes: Race condition
2293  *
2294  * We usually use page_cgroup_lock() for accessing page_cgroup member but
2295  * it tends to be costly. But considering some conditions, we doesn't need
2296  * to do so _always_.
2297  *
2298  * Considering "charge", lock_page_cgroup() is not required because all
2299  * file-stat operations happen after a page is attached to radix-tree. There
2300  * are no race with "charge".
2301  *
2302  * Considering "uncharge", we know that memcg doesn't clear pc->mem_cgroup
2303  * at "uncharge" intentionally. So, we always see valid pc->mem_cgroup even
2304  * if there are race with "uncharge". Statistics itself is properly handled
2305  * by flags.
2306  *
2307  * Considering "move", this is an only case we see a race. To make the race
2308  * small, we check mm->moving_account and detect there are possibility of race
2309  * If there is, we take a lock.
2310  */
2311 
2312 void __mem_cgroup_begin_update_page_stat(struct page *page,
2313                                 bool *locked, unsigned long *flags)
2314 {
2315         struct mem_cgroup *memcg;
2316         struct page_cgroup *pc;
2317 
2318         pc = lookup_page_cgroup(page);
2319 again:
2320         memcg = pc->mem_cgroup;
2321         if (unlikely(!memcg || !PageCgroupUsed(pc)))
2322                 return;
2323         /*
2324          * If this memory cgroup is not under account moving, we don't
2325          * need to take move_lock_mem_cgroup(). Because we already hold
2326          * rcu_read_lock(), any calls to move_account will be delayed until
2327          * rcu_read_unlock() if mem_cgroup_stolen() == true.
2328          */
2329         if (!mem_cgroup_stolen(memcg))
2330                 return;
2331 
2332         move_lock_mem_cgroup(memcg, flags);
2333         if (memcg != pc->mem_cgroup || !PageCgroupUsed(pc)) {
2334                 move_unlock_mem_cgroup(memcg, flags);
2335                 goto again;
2336         }
2337         *locked = true;
2338 }
2339 
2340 void __mem_cgroup_end_update_page_stat(struct page *page, unsigned long *flags)
2341 {
2342         struct page_cgroup *pc = lookup_page_cgroup(page);
2343 
2344         /*
2345          * It's guaranteed that pc->mem_cgroup never changes while
2346          * lock is held because a routine modifies pc->mem_cgroup
2347          * should take move_lock_mem_cgroup().
2348          */
2349         move_unlock_mem_cgroup(pc->mem_cgroup, flags);
2350 }
2351 
2352 void mem_cgroup_update_page_stat(struct page *page,
2353                                  enum mem_cgroup_page_stat_item idx, int val)
2354 {
2355         struct mem_cgroup *memcg;
2356         struct page_cgroup *pc = lookup_page_cgroup(page);
2357         unsigned long uninitialized_var(flags);
2358 
2359         if (mem_cgroup_disabled())
2360                 return;
2361 
2362         memcg = pc->mem_cgroup;
2363         if (unlikely(!memcg || !PageCgroupUsed(pc)))
2364                 return;
2365 
2366         switch (idx) {
2367         case MEMCG_NR_FILE_MAPPED:
2368                 idx = MEM_CGROUP_STAT_FILE_MAPPED;
2369                 break;
2370         default:
2371                 BUG();
2372         }
2373 
2374         this_cpu_add(memcg->stat->count[idx], val);
2375 }
2376 
2377 /*
2378  * size of first charge trial. "32" comes from vmscan.c's magic value.
2379  * TODO: maybe necessary to use big numbers in big irons.
2380  */
2381 #define CHARGE_BATCH    32U
2382 struct memcg_stock_pcp {
2383         struct mem_cgroup *cached; /* this never be root cgroup */
2384         unsigned int nr_pages;
2385         struct work_struct work;
2386         unsigned long flags;
2387 #define FLUSHING_CACHED_CHARGE  0
2388 };
2389 static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
2390 static DEFINE_MUTEX(percpu_charge_mutex);
2391 
2392 /**
2393  * consume_stock: Try to consume stocked charge on this cpu.
2394  * @memcg: memcg to consume from.
2395  * @nr_pages: how many pages to charge.
2396  *
2397  * The charges will only happen if @memcg matches the current cpu's memcg
2398  * stock, and at least @nr_pages are available in that stock.  Failure to
2399  * service an allocation will refill the stock.
2400  *
2401  * returns true if successful, false otherwise.
2402  */
2403 static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2404 {
2405         struct memcg_stock_pcp *stock;
2406         bool ret = true;
2407 
2408         if (nr_pages > CHARGE_BATCH)
2409                 return false;
2410 
2411         stock = &get_cpu_var(memcg_stock);
2412         if (memcg == stock->cached && stock->nr_pages >= nr_pages)
2413                 stock->nr_pages -= nr_pages;
2414         else /* need to call res_counter_charge */
2415                 ret = false;
2416         put_cpu_var(memcg_stock);
2417         return ret;
2418 }
2419 
2420 /*
2421  * Returns stocks cached in percpu to res_counter and reset cached information.
2422  */
2423 static void drain_stock(struct memcg_stock_pcp *stock)
2424 {
2425         struct mem_cgroup *old = stock->cached;
2426 
2427         if (stock->nr_pages) {
2428                 unsigned long bytes = stock->nr_pages * PAGE_SIZE;
2429 
2430                 res_counter_uncharge(&old->res, bytes);
2431                 if (do_swap_account)
2432                         res_counter_uncharge(&old->memsw, bytes);
2433                 stock->nr_pages = 0;
2434         }
2435         stock->cached = NULL;
2436 }
2437 
2438 /*
2439  * This must be called under preempt disabled or must be called by
2440  * a thread which is pinned to local cpu.
2441  */
2442 static void drain_local_stock(struct work_struct *dummy)
2443 {
2444         struct memcg_stock_pcp *stock = &__get_cpu_var(memcg_stock);
2445         drain_stock(stock);
2446         clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
2447 }
2448 
2449 static void __init memcg_stock_init(void)
2450 {
2451         int cpu;
2452 
2453         for_each_possible_cpu(cpu) {
2454                 struct memcg_stock_pcp *stock =
2455                                         &per_cpu(memcg_stock, cpu);
2456                 INIT_WORK(&stock->work, drain_local_stock);
2457         }
2458 }
2459 
2460 /*
2461  * Cache charges(val) which is from res_counter, to local per_cpu area.
2462  * This will be consumed by consume_stock() function, later.
2463  */
2464 static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2465 {
2466         struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);
2467 
2468         if (stock->cached != memcg) { /* reset if necessary */
2469                 drain_stock(stock);
2470                 stock->cached = memcg;
2471         }
2472         stock->nr_pages += nr_pages;
2473         put_cpu_var(memcg_stock);
2474 }
2475 
2476 /*
2477  * Drains all per-CPU charge caches for given root_memcg resp. subtree
2478  * of the hierarchy under it. sync flag says whether we should block
2479  * until the work is done.
2480  */
2481 static void drain_all_stock(struct mem_cgroup *root_memcg, bool sync)
2482 {
2483         int cpu, curcpu;
2484 
2485         /* Notify other cpus that system-wide "drain" is running */
2486         get_online_cpus();
2487         curcpu = get_cpu();
2488         for_each_online_cpu(cpu) {
2489                 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
2490                 struct mem_cgroup *memcg;
2491 
2492                 memcg = stock->cached;
2493                 if (!memcg || !stock->nr_pages)
2494                         continue;
2495                 if (!mem_cgroup_same_or_subtree(root_memcg, memcg))
2496                         continue;
2497                 if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
2498                         if (cpu == curcpu)
2499                                 drain_local_stock(&stock->work);
2500                         else
2501                                 schedule_work_on(cpu, &stock->work);
2502                 }
2503         }
2504         put_cpu();
2505 
2506         if (!sync)
2507                 goto out;
2508 
2509         for_each_online_cpu(cpu) {
2510                 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
2511                 if (test_bit(FLUSHING_CACHED_CHARGE, &stock->flags))
2512                         flush_work(&stock->work);
2513         }
2514 out:
2515         put_online_cpus();
2516 }
2517 
2518 /*
2519  * Tries to drain stocked charges in other cpus. This function is asynchronous
2520  * and just put a work per cpu for draining localy on each cpu. Caller can
2521  * expects some charges will be back to res_counter later but cannot wait for
2522  * it.
2523  */
2524 static void drain_all_stock_async(struct mem_cgroup *root_memcg)
2525 {
2526         /*
2527          * If someone calls draining, avoid adding more kworker runs.
2528          */
2529         if (!mutex_trylock(&percpu_charge_mutex))
2530                 return;
2531         drain_all_stock(root_memcg, false);
2532         mutex_unlock(&percpu_charge_mutex);
2533 }
2534 
2535 /* This is a synchronous drain interface. */
2536 static void drain_all_stock_sync(struct mem_cgroup *root_memcg)
2537 {
2538         /* called when force_empty is called */
2539         mutex_lock(&percpu_charge_mutex);
2540         drain_all_stock(root_memcg, true);
2541         mutex_unlock(&percpu_charge_mutex);
2542 }
2543 
2544 /*
2545  * This function drains percpu counter value from DEAD cpu and
2546  * move it to local cpu. Note that this function can be preempted.
2547  */
2548 static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *memcg, int cpu)
2549 {
2550         int i;
2551 
2552         spin_lock(&memcg->pcp_counter_lock);
2553         for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
2554                 long x = per_cpu(memcg->stat->count[i], cpu);
2555 
2556                 per_cpu(memcg->stat->count[i], cpu) = 0;
2557                 memcg->nocpu_base.count[i] += x;
2558         }
2559         for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
2560                 unsigned long x = per_cpu(memcg->stat->events[i], cpu);
2561 
2562                 per_cpu(memcg->stat->events[i], cpu) = 0;
2563                 memcg->nocpu_base.events[i] += x;
2564         }
2565         spin_unlock(&memcg->pcp_counter_lock);
2566 }
2567 
2568 static int __cpuinit memcg_cpu_hotplug_callback(struct notifier_block *nb,
2569                                         unsigned long action,
2570                                         void *hcpu)
2571 {
2572         int cpu = (unsigned long)hcpu;
2573         struct memcg_stock_pcp *stock;
2574         struct mem_cgroup *iter;
2575 
2576         if (action == CPU_ONLINE)
2577                 return NOTIFY_OK;
2578 
2579         if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
2580                 return NOTIFY_OK;
2581 
2582         for_each_mem_cgroup(iter)
2583                 mem_cgroup_drain_pcp_counter(iter, cpu);
2584 
2585         stock = &per_cpu(memcg_stock, cpu);
2586         drain_stock(stock);
2587         return NOTIFY_OK;
2588 }
2589 
2590 
2591 /* See __mem_cgroup_try_charge() for details */
2592 enum {
2593         CHARGE_OK,              /* success */
2594         CHARGE_RETRY,           /* need to retry but retry is not bad */
2595         CHARGE_NOMEM,           /* we can't do more. return -ENOMEM */
2596         CHARGE_WOULDBLOCK,      /* GFP_WAIT wasn't set and no enough res. */
2597 };
2598 
2599 static int mem_cgroup_do_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
2600                                 unsigned int nr_pages, unsigned int min_pages,
2601                                 bool invoke_oom)
2602 {
2603         unsigned long csize = nr_pages * PAGE_SIZE;
2604         struct mem_cgroup *mem_over_limit;
2605         struct res_counter *fail_res;
2606         unsigned long flags = 0;
2607         int ret;
2608 
2609         ret = res_counter_charge(&memcg->res, csize, &fail_res);
2610 
2611         if (likely(!ret)) {
2612                 if (!do_swap_account)
2613                         return CHARGE_OK;
2614                 ret = res_counter_charge(&memcg->memsw, csize, &fail_res);
2615                 if (likely(!ret))
2616                         return CHARGE_OK;
2617 
2618                 res_counter_uncharge(&memcg->res, csize);
2619                 mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw);
2620                 flags |= MEM_CGROUP_RECLAIM_NOSWAP;
2621         } else
2622                 mem_over_limit = mem_cgroup_from_res_counter(fail_res, res);
2623         /*
2624          * Never reclaim on behalf of optional batching, retry with a
2625          * single page instead.
2626          */
2627         if (nr_pages > min_pages)
2628                 return CHARGE_RETRY;
2629 
2630         if (!(gfp_mask & __GFP_WAIT))
2631                 return CHARGE_WOULDBLOCK;
2632 
2633         if (gfp_mask & __GFP_NORETRY)
2634                 return CHARGE_NOMEM;
2635 
2636         ret = mem_cgroup_reclaim(mem_over_limit, gfp_mask, flags);
2637         if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
2638                 return CHARGE_RETRY;
2639         /*
2640          * Even though the limit is exceeded at this point, reclaim
2641          * may have been able to free some pages.  Retry the charge
2642          * before killing the task.
2643          *
2644          * Only for regular pages, though: huge pages are rather
2645          * unlikely to succeed so close to the limit, and we fall back
2646          * to regular pages anyway in case of failure.
2647          */
2648         if (nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER) && ret)
2649                 return CHARGE_RETRY;
2650 
2651         /*
2652          * At task move, charge accounts can be doubly counted. So, it's
2653          * better to wait until the end of task_move if something is going on.
2654          */
2655         if (mem_cgroup_wait_acct_move(mem_over_limit))
2656                 return CHARGE_RETRY;
2657 
2658         if (invoke_oom)
2659                 mem_cgroup_oom(mem_over_limit, gfp_mask, get_order(csize));
2660 
2661         return CHARGE_NOMEM;
2662 }
2663 
2664 /*
2665  * __mem_cgroup_try_charge() does
2666  * 1. detect memcg to be charged against from passed *mm and *ptr,
2667  * 2. update res_counter
2668  * 3. call memory reclaim if necessary.
2669  *
2670  * In some special case, if the task is fatal, fatal_signal_pending() or
2671  * has TIF_MEMDIE, this function returns -EINTR while writing root_mem_cgroup
2672  * to *ptr. There are two reasons for this. 1: fatal threads should quit as soon
2673  * as possible without any hazards. 2: all pages should have a valid
2674  * pc->mem_cgroup. If mm is NULL and the caller doesn't pass a valid memcg
2675  * pointer, that is treated as a charge to root_mem_cgroup.
2676  *
2677  * So __mem_cgroup_try_charge() will return
2678  *  0       ...  on success, filling *ptr with a valid memcg pointer.
2679  *  -ENOMEM ...  charge failure because of resource limits.
2680  *  -EINTR  ...  if thread is fatal. *ptr is filled with root_mem_cgroup.
2681  *
2682  * Unlike the exported interface, an "oom" parameter is added. if oom==true,
2683  * the oom-killer can be invoked.
2684  */
2685 static int __mem_cgroup_try_charge(struct mm_struct *mm,
2686                                    gfp_t gfp_mask,
2687                                    unsigned int nr_pages,
2688                                    struct mem_cgroup **ptr,
2689                                    bool oom)
2690 {
2691         unsigned int batch = max(CHARGE_BATCH, nr_pages);
2692         int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
2693         struct mem_cgroup *memcg = NULL;
2694         int ret;
2695 
2696         /*
2697          * Unlike gloval-vm's OOM-kill, we're not in memory shortage
2698          * in system level. So, allow to go ahead dying process in addition to
2699          * MEMDIE process.
2700          */
2701         if (unlikely(test_thread_flag(TIF_MEMDIE)
2702                      || fatal_signal_pending(current)))
2703                 goto bypass;
2704 
2705         if (unlikely(task_in_memcg_oom(current)))
2706                 goto bypass;
2707 
2708         /*
2709          * We always charge the cgroup the mm_struct belongs to.
2710          * The mm_struct's mem_cgroup changes on task migration if the
2711          * thread group leader migrates. It's possible that mm is not
2712          * set, if so charge the root memcg (happens for pagecache usage).
2713          */
2714         if (!*ptr && !mm)
2715                 *ptr = root_mem_cgroup;
2716 again:
2717         if (*ptr) { /* css should be a valid one */
2718                 memcg = *ptr;
2719                 if (mem_cgroup_is_root(memcg))
2720                         goto done;
2721                 if (consume_stock(memcg, nr_pages))
2722                         goto done;
2723                 css_get(&memcg->css);
2724         } else {
2725                 struct task_struct *p;
2726 
2727                 rcu_read_lock();
2728                 p = rcu_dereference(mm->owner);
2729                 /*
2730                  * Because we don't have task_lock(), "p" can exit.
2731                  * In that case, "memcg" can point to root or p can be NULL with
2732                  * race with swapoff. Then, we have small risk of mis-accouning.
2733                  * But such kind of mis-account by race always happens because
2734                  * we don't have cgroup_mutex(). It's overkill and we allo that
2735                  * small race, here.
2736                  * (*) swapoff at el will charge against mm-struct not against
2737                  * task-struct. So, mm->owner can be NULL.
2738                  */
2739                 memcg = mem_cgroup_from_task(p);
2740                 if (!memcg)
2741                         memcg = root_mem_cgroup;
2742                 if (mem_cgroup_is_root(memcg)) {
2743                         rcu_read_unlock();
2744                         goto done;
2745                 }
2746                 if (consume_stock(memcg, nr_pages)) {
2747                         /*
2748                          * It seems dagerous to access memcg without css_get().
2749                          * But considering how consume_stok works, it's not
2750                          * necessary. If consume_stock success, some charges
2751                          * from this memcg are cached on this cpu. So, we
2752                          * don't need to call css_get()/css_tryget() before
2753                          * calling consume_stock().
2754                          */
2755                         rcu_read_unlock();
2756                         goto done;
2757                 }
2758                 /* after here, we may be blocked. we need to get refcnt */
2759                 if (!css_tryget(&memcg->css)) {
2760                         rcu_read_unlock();
2761                         goto again;
2762                 }
2763                 rcu_read_unlock();
2764         }
2765 
2766         do {
2767                 bool invoke_oom = oom && !nr_oom_retries;
2768 
2769                 /* If killed, bypass charge */
2770                 if (fatal_signal_pending(current)) {
2771                         css_put(&memcg->css);
2772                         goto bypass;
2773                 }
2774 
2775                 ret = mem_cgroup_do_charge(memcg, gfp_mask, batch,
2776                                            nr_pages, invoke_oom);
2777                 switch (ret) {
2778                 case CHARGE_OK:
2779                         break;
2780                 case CHARGE_RETRY: /* not in OOM situation but retry */
2781                         batch = nr_pages;
2782                         css_put(&memcg->css);
2783                         memcg = NULL;
2784                         goto again;
2785                 case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */
2786                         css_put(&memcg->css);
2787                         goto nomem;
2788                 case CHARGE_NOMEM: /* OOM routine works */
2789                         if (!oom || invoke_oom) {
2790                                 css_put(&memcg->css);
2791                                 goto nomem;
2792                         }
2793                         nr_oom_retries--;
2794                         break;
2795                 }
2796         } while (ret != CHARGE_OK);
2797 
2798         if (batch > nr_pages)
2799                 refill_stock(memcg, batch - nr_pages);
2800         css_put(&memcg->css);
2801 done:
2802         *ptr = memcg;
2803         return 0;
2804 nomem:
2805         *ptr = NULL;
2806         return -ENOMEM;
2807 bypass:
2808         *ptr = root_mem_cgroup;
2809         return -EINTR;
2810 }
2811 
2812 /*
2813  * Somemtimes we have to undo a charge we got by try_charge().
2814  * This function is for that and do uncharge, put css's refcnt.
2815  * gotten by try_charge().
2816  */
2817 static void __mem_cgroup_cancel_charge(struct mem_cgroup *memcg,
2818                                        unsigned int nr_pages)
2819 {
2820         if (!mem_cgroup_is_root(memcg)) {
2821                 unsigned long bytes = nr_pages * PAGE_SIZE;
2822 
2823                 res_counter_uncharge(&memcg->res, bytes);
2824                 if (do_swap_account)
2825                         res_counter_uncharge(&memcg->memsw, bytes);
2826         }
2827 }
2828 
2829 /*
2830  * Cancel chrages in this cgroup....doesn't propagate to parent cgroup.
2831  * This is useful when moving usage to parent cgroup.
2832  */
2833 static void __mem_cgroup_cancel_local_charge(struct mem_cgroup *memcg,
2834                                         unsigned int nr_pages)
2835 {
2836         unsigned long bytes = nr_pages * PAGE_SIZE;
2837 
2838         if (mem_cgroup_is_root(memcg))
2839                 return;
2840 
2841         res_counter_uncharge_until(&memcg->res, memcg->res.parent, bytes);
2842         if (do_swap_account)
2843                 res_counter_uncharge_until(&memcg->memsw,
2844                                                 memcg->memsw.parent, bytes);
2845 }
2846 
2847 /*
2848  * A helper function to get mem_cgroup from ID. must be called under
2849  * rcu_read_lock().  The caller is responsible for calling css_tryget if
2850  * the mem_cgroup is used for charging. (dropping refcnt from swap can be
2851  * called against removed memcg.)
2852  */
2853 static struct mem_cgroup *mem_cgroup_lookup(unsigned short id)
2854 {
2855         struct cgroup_subsys_state *css;
2856 
2857         /* ID 0 is unused ID */
2858         if (!id)
2859                 return NULL;
2860         css = css_lookup(&mem_cgroup_subsys, id);
2861         if (!css)
2862                 return NULL;
2863         return mem_cgroup_from_css(css);
2864 }
2865 
2866 struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
2867 {
2868         struct mem_cgroup *memcg = NULL;
2869         struct page_cgroup *pc;
2870         unsigned short id;
2871         swp_entry_t ent;
2872 
2873         VM_BUG_ON(!PageLocked(page));
2874 
2875         pc = lookup_page_cgroup(page);
2876         lock_page_cgroup(pc);
2877         if (PageCgroupUsed(pc)) {
2878                 memcg = pc->mem_cgroup;
2879                 if (memcg && !css_tryget(&memcg->css))
2880                         memcg = NULL;
2881         } else if (PageSwapCache(page)) {
2882                 ent.val = page_private(page);
2883                 id = lookup_swap_cgroup_id(ent);
2884                 rcu_read_lock();
2885                 memcg = mem_cgroup_lookup(id);
2886                 if (memcg && !css_tryget(&memcg->css))
2887                         memcg = NULL;
2888                 rcu_read_unlock();
2889         }
2890         unlock_page_cgroup(pc);
2891         return memcg;
2892 }
2893 
2894 static void __mem_cgroup_commit_charge(struct mem_cgroup *memcg,
2895                                        struct page *page,
2896                                        unsigned int nr_pages,
2897                                        enum charge_type ctype,
2898                                        bool lrucare)
2899 {
2900         struct page_cgroup *pc = lookup_page_cgroup(page);
2901         struct zone *uninitialized_var(zone);
2902         struct lruvec *lruvec;
2903         bool was_on_lru = false;
2904         bool anon;
2905 
2906         lock_page_cgroup(pc);
2907         VM_BUG_ON(PageCgroupUsed(pc));
2908         /*
2909          * we don't need page_cgroup_lock about tail pages, becase they are not
2910          * accessed by any other context at this point.
2911          */
2912 
2913         /*
2914          * In some cases, SwapCache and FUSE(splice_buf->radixtree), the page
2915          * may already be on some other mem_cgroup's LRU.  Take care of it.
2916          */
2917         if (lrucare) {
2918                 zone = page_zone(page);
2919                 spin_lock_irq(&zone->lru_lock);
2920                 if (PageLRU(page)) {
2921                         lruvec = mem_cgroup_zone_lruvec(zone, pc->mem_cgroup);
2922                         ClearPageLRU(page);
2923                         del_page_from_lru_list(page, lruvec, page_lru(page));
2924                         was_on_lru = true;
2925                 }
2926         }
2927 
2928         pc->mem_cgroup = memcg;
2929         /*
2930          * We access a page_cgroup asynchronously without lock_page_cgroup().
2931          * Especially when a page_cgroup is taken from a page, pc->mem_cgroup
2932          * is accessed after testing USED bit. To make pc->mem_cgroup visible
2933          * before USED bit, we need memory barrier here.
2934          * See mem_cgroup_add_lru_list(), etc.
2935          */
2936         smp_wmb();
2937         SetPageCgroupUsed(pc);
2938 
2939         if (lrucare) {
2940                 if (was_on_lru) {
2941                         lruvec = mem_cgroup_zone_lruvec(zone, pc->mem_cgroup);
2942                         VM_BUG_ON(PageLRU(page));
2943                         SetPageLRU(page);
2944                         add_page_to_lru_list(page, lruvec, page_lru(page));
2945                 }
2946                 spin_unlock_irq(&zone->lru_lock);
2947         }
2948 
2949         if (ctype == MEM_CGROUP_CHARGE_TYPE_ANON)
2950                 anon = true;
2951         else
2952                 anon = false;
2953 
2954         mem_cgroup_charge_statistics(memcg, page, anon, nr_pages);
2955         unlock_page_cgroup(pc);
2956 
2957         /*
2958          * "charge_statistics" updated event counter. Then, check it.
2959          * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree.
2960          * if they exceeds softlimit.
2961          */
2962         memcg_check_events(memcg, page);
2963 }
2964 
2965 static DEFINE_MUTEX(set_limit_mutex);
2966 
2967 #ifdef CONFIG_MEMCG_KMEM
2968 static inline bool memcg_can_account_kmem(struct mem_cgroup *memcg)
2969 {
2970         return !mem_cgroup_disabled() && !mem_cgroup_is_root(memcg) &&
2971                 (memcg->kmem_account_flags & KMEM_ACCOUNTED_MASK);
2972 }
2973 
2974 /*
2975  * This is a bit cumbersome, but it is rarely used and avoids a backpointer
2976  * in the memcg_cache_params struct.
2977  */
2978 static struct kmem_cache *memcg_params_to_cache(struct memcg_cache_params *p)
2979 {
2980         struct kmem_cache *cachep;
2981 
2982         VM_BUG_ON(p->is_root_cache);
2983         cachep = p->root_cache;
2984         return cachep->memcg_params->memcg_caches[memcg_cache_id(p->memcg)];
2985 }
2986 
2987 #ifdef CONFIG_SLABINFO
2988 static int mem_cgroup_slabinfo_read(struct cgroup *cont, struct cftype *cft,
2989                                         struct seq_file *m)
2990 {
2991         struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
2992         struct memcg_cache_params *params;
2993 
2994         if (!memcg_can_account_kmem(memcg))
2995                 return -EIO;
2996 
2997         print_slabinfo_header(m);
2998 
2999         mutex_lock(&memcg->slab_caches_mutex);
3000         list_for_each_entry(params, &memcg->memcg_slab_caches, list)
3001                 cache_show(memcg_params_to_cache(params), m);
3002         mutex_unlock(&memcg->slab_caches_mutex);
3003 
3004         return 0;
3005 }
3006 #endif
3007 
3008 static int memcg_charge_kmem(struct mem_cgroup *memcg, gfp_t gfp, u64 size)
3009 {
3010         struct res_counter *fail_res;
3011         struct mem_cgroup *_memcg;
3012         int ret = 0;
3013         bool may_oom;
3014 
3015         ret = res_counter_charge(&memcg->kmem, size, &fail_res);
3016         if (ret)
3017                 return ret;
3018 
3019         /*
3020          * Conditions under which we can wait for the oom_killer. Those are
3021          * the same conditions tested by the core page allocator
3022          */
3023         may_oom = (gfp & __GFP_FS) && !(gfp & __GFP_NORETRY);
3024 
3025         _memcg = memcg;
3026         ret = __mem_cgroup_try_charge(NULL, gfp, size >> PAGE_SHIFT,
3027                                       &_memcg, may_oom);
3028 
3029         if (ret == -EINTR)  {
3030                 /*
3031                  * __mem_cgroup_try_charge() chosed to bypass to root due to
3032                  * OOM kill or fatal signal.  Since our only options are to
3033                  * either fail the allocation or charge it to this cgroup, do
3034                  * it as a temporary condition. But we can't fail. From a
3035                  * kmem/slab perspective, the cache has already been selected,
3036                  * by mem_cgroup_kmem_get_cache(), so it is too late to change
3037                  * our minds.
3038                  *
3039                  * This condition will only trigger if the task entered
3040                  * memcg_charge_kmem in a sane state, but was OOM-killed during
3041                  * __mem_cgroup_try_charge() above. Tasks that were already
3042                  * dying when the allocation triggers should have been already
3043                  * directed to the root cgroup in memcontrol.h
3044                  */
3045                 res_counter_charge_nofail(&memcg->res, size, &fail_res);
3046                 if (do_swap_account)
3047                         res_counter_charge_nofail(&memcg->memsw, size,
3048                                                   &fail_res);
3049                 ret = 0;
3050         } else if (ret)
3051                 res_counter_uncharge(&memcg->kmem, size);
3052 
3053         return ret;
3054 }
3055 
3056 static void memcg_uncharge_kmem(struct mem_cgroup *memcg, u64 size)
3057 {
3058         res_counter_uncharge(&memcg->res, size);
3059         if (do_swap_account)
3060                 res_counter_uncharge(&memcg->memsw, size);
3061 
3062         /* Not down to 0 */
3063         if (res_counter_uncharge(&memcg->kmem, size))
3064                 return;
3065 
3066         if (memcg_kmem_test_and_clear_dead(memcg))
3067                 mem_cgroup_put(memcg);
3068 }
3069 
3070 void memcg_cache_list_add(struct mem_cgroup *memcg, struct kmem_cache *cachep)
3071 {
3072         if (!memcg)
3073                 return;
3074 
3075         mutex_lock(&memcg->slab_caches_mutex);
3076         list_add(&cachep->memcg_params->list, &memcg->memcg_slab_caches);
3077         mutex_unlock(&memcg->slab_caches_mutex);
3078 }
3079 
3080 /*
3081  * helper for acessing a memcg's index. It will be used as an index in the
3082  * child cache array in kmem_cache, and also to derive its name. This function
3083  * will return -1 when this is not a kmem-limited memcg.
3084  */
3085 int memcg_cache_id(struct mem_cgroup *memcg)
3086 {
3087         return memcg ? memcg->kmemcg_id : -1;
3088 }
3089 
3090 /*
3091  * This ends up being protected by the set_limit mutex, during normal
3092  * operation, because that is its main call site.
3093  *
3094  * But when we create a new cache, we can call this as well if its parent
3095  * is kmem-limited. That will have to hold set_limit_mutex as well.
3096  */
3097 int memcg_update_cache_sizes(struct mem_cgroup *memcg)
3098 {
3099         int num, ret;
3100 
3101         num = ida_simple_get(&kmem_limited_groups,
3102                                 0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
3103         if (num < 0)
3104                 return num;
3105         /*
3106          * After this point, kmem_accounted (that we test atomically in
3107          * the beginning of this conditional), is no longer 0. This
3108          * guarantees only one process will set the following boolean
3109          * to true. We don't need test_and_set because we're protected
3110          * by the set_limit_mutex anyway.
3111          */
3112         memcg_kmem_set_activated(memcg);
3113 
3114         ret = memcg_update_all_caches(num+1);
3115         if (ret) {
3116                 ida_simple_remove(&kmem_limited_groups, num);
3117                 memcg_kmem_clear_activated(memcg);
3118                 return ret;
3119         }
3120 
3121         memcg->kmemcg_id = num;
3122         INIT_LIST_HEAD(&memcg->memcg_slab_caches);
3123         mutex_init(&memcg->slab_caches_mutex);
3124         return 0;
3125 }
3126 
3127 static size_t memcg_caches_array_size(int num_groups)
3128 {
3129         ssize_t size;
3130         if (num_groups <= 0)
3131                 return 0;
3132 
3133         size = 2 * num_groups;
3134         if (size < MEMCG_CACHES_MIN_SIZE)
3135                 size = MEMCG_CACHES_MIN_SIZE;
3136         else if (size > MEMCG_CACHES_MAX_SIZE)
3137                 size = MEMCG_CACHES_MAX_SIZE;
3138 
3139         return size;
3140 }
3141 
3142 /*
3143  * We should update the current array size iff all caches updates succeed. This
3144  * can only be done from the slab side. The slab mutex needs to be held when
3145  * calling this.
3146  */
3147 void memcg_update_array_size(int num)
3148 {
3149         if (num > memcg_limited_groups_array_size)
3150                 memcg_limited_groups_array_size = memcg_caches_array_size(num);
3151 }
3152 
3153 static void kmem_cache_destroy_work_func(struct work_struct *w);
3154 
3155 int memcg_update_cache_size(struct kmem_cache *s, int num_groups)
3156 {
3157         struct memcg_cache_params *cur_params = s->memcg_params;
3158 
3159         VM_BUG_ON(s->memcg_params && !s->memcg_params->is_root_cache);
3160 
3161         if (num_groups > memcg_limited_groups_array_size) {
3162                 int i;
3163                 ssize_t size = memcg_caches_array_size(num_groups);
3164 
3165                 size *= sizeof(void *);
3166                 size += sizeof(struct memcg_cache_params);
3167 
3168                 s->memcg_params = kzalloc(size, GFP_KERNEL);
3169                 if (!s->memcg_params) {
3170                         s->memcg_params = cur_params;
3171                         return -ENOMEM;
3172                 }
3173 
3174                 s->memcg_params->is_root_cache = true;
3175 
3176                 /*
3177                  * There is the chance it will be bigger than
3178                  * memcg_limited_groups_array_size, if we failed an allocation
3179                  * in a cache, in which case all caches updated before it, will
3180                  * have a bigger array.
3181                  *
3182                  * But if that is the case, the data after
3183                  * memcg_limited_groups_array_size is certainly unused
3184                  */
3185                 for (i = 0; i < memcg_limited_groups_array_size; i++) {
3186                         if (!cur_params->memcg_caches[i])
3187                                 continue;
3188                         s->memcg_params->memcg_caches[i] =
3189                                                 cur_params->memcg_caches[i];
3190                 }
3191 
3192                 /*
3193                  * Ideally, we would wait until all caches succeed, and only
3194                  * then free the old one. But this is not worth the extra
3195                  * pointer per-cache we'd have to have for this.
3196                  *
3197                  * It is not a big deal if some caches are left with a size
3198                  * bigger than the others. And all updates will reset this
3199                  * anyway.
3200                  */
3201                 kfree(cur_params);
3202         }
3203         return 0;
3204 }
3205 
3206 int memcg_register_cache(struct mem_cgroup *memcg, struct kmem_cache *s,
3207                          struct kmem_cache *root_cache)
3208 {
3209         size_t size = sizeof(struct memcg_cache_params);
3210 
3211         if (!memcg_kmem_enabled())
3212                 return 0;
3213 
3214         if (!memcg)
3215                 size += memcg_limited_groups_array_size * sizeof(void *);
3216 
3217         s->memcg_params = kzalloc(size, GFP_KERNEL);
3218         if (!s->memcg_params)
3219                 return -ENOMEM;
3220 
3221         if (memcg) {
3222                 s->memcg_params->memcg = memcg;
3223                 s->memcg_params->root_cache = root_cache;
3224                 INIT_WORK(&s->memcg_params->destroy,
3225                                 kmem_cache_destroy_work_func);
3226         } else
3227                 s->memcg_params->is_root_cache = true;
3228 
3229         return 0;
3230 }
3231 
3232 void memcg_release_cache(struct kmem_cache *s)
3233 {
3234         struct kmem_cache *root;
3235         struct mem_cgroup *memcg;
3236         int id;
3237 
3238         /*
3239          * This happens, for instance, when a root cache goes away before we
3240          * add any memcg.
3241          */
3242         if (!s->memcg_params)
3243                 return;
3244 
3245         if (s->memcg_params->is_root_cache)
3246                 goto out;
3247 
3248         memcg = s->memcg_params->memcg;
3249         id  = memcg_cache_id(memcg);
3250 
3251         root = s->memcg_params->root_cache;
3252         root->memcg_params->memcg_caches[id] = NULL;
3253 
3254         mutex_lock(&memcg->slab_caches_mutex);
3255         list_del(&s->memcg_params->list);
3256         mutex_unlock(&memcg->slab_caches_mutex);
3257 
3258         mem_cgroup_put(memcg);
3259 out:
3260         kfree(s->memcg_params);
3261 }
3262 
3263 /*
3264  * During the creation a new cache, we need to disable our accounting mechanism
3265  * altogether. This is true even if we are not creating, but rather just
3266  * enqueing new caches to be created.
3267  *
3268  * This is because that process will trigger allocations; some visible, like
3269  * explicit kmallocs to auxiliary data structures, name strings and internal
3270  * cache structures; some well concealed, like INIT_WORK() that can allocate
3271  * objects during debug.
3272  *
3273  * If any allocation happens during memcg_kmem_get_cache, we will recurse back
3274  * to it. This may not be a bounded recursion: since the first cache creation
3275  * failed to complete (waiting on the allocation), we'll just try to create the
3276  * cache again, failing at the same point.
3277  *
3278  * memcg_kmem_get_cache is prepared to abort after seeing a positive count of
3279  * memcg_kmem_skip_account. So we enclose anything that might allocate memory
3280  * inside the following two functions.
3281  */
3282 static inline void memcg_stop_kmem_account(void)
3283 {
3284         VM_BUG_ON(!current->mm);
3285         current->memcg_kmem_skip_account++;
3286 }
3287 
3288 static inline void memcg_resume_kmem_account(void)
3289 {
3290         VM_BUG_ON(!current->mm);
3291         current->memcg_kmem_skip_account--;
3292 }
3293 
3294 static void kmem_cache_destroy_work_func(struct work_struct *w)
3295 {
3296         struct kmem_cache *cachep;
3297         struct memcg_cache_params *p;
3298 
3299         p = container_of(w, struct memcg_cache_params, destroy);
3300 
3301         cachep = memcg_params_to_cache(p);
3302 
3303         /*
3304          * If we get down to 0 after shrink, we could delete right away.
3305          * However, memcg_release_pages() already puts us back in the workqueue
3306          * in that case. If we proceed deleting, we'll get a dangling
3307          * reference, and removing the object from the workqueue in that case
3308          * is unnecessary complication. We are not a fast path.
3309          *
3310          * Note that this case is fundamentally different from racing with
3311          * shrink_slab(): if memcg_cgroup_destroy_cache() is called in
3312          * kmem_cache_shrink, not only we would be reinserting a dead cache
3313          * into the queue, but doing so from inside the worker racing to
3314          * destroy it.
3315          *
3316          * So if we aren't down to zero, we'll just schedule a worker and try
3317          * again
3318          */
3319         if (atomic_read(&cachep->memcg_params->nr_pages) != 0) {
3320                 kmem_cache_shrink(cachep);
3321                 if (atomic_read(&cachep->memcg_params->nr_pages) == 0)
3322                         return;
3323         } else
3324                 kmem_cache_destroy(cachep);
3325 }
3326 
3327 void mem_cgroup_destroy_cache(struct kmem_cache *cachep)
3328 {
3329         if (!cachep->memcg_params->dead)
3330                 return;
3331 
3332         /*
3333          * There are many ways in which we can get here.
3334          *
3335          * We can get to a memory-pressure situation while the delayed work is
3336          * still pending to run. The vmscan shrinkers can then release all
3337          * cache memory and get us to destruction. If this is the case, we'll
3338          * be executed twice, which is a bug (the second time will execute over
3339          * bogus data). In this case, cancelling the work should be fine.
3340          *
3341          * But we can also get here from the worker itself, if
3342          * kmem_cache_shrink is enough to shake all the remaining objects and
3343          * get the page count to 0. In this case, we'll deadlock if we try to
3344          * cancel the work (the worker runs with an internal lock held, which
3345          * is the same lock we would hold for cancel_work_sync().)
3346          *
3347          * Since we can't possibly know who got us here, just refrain from
3348          * running if there is already work pending
3349          */
3350         if (work_pending(&cachep->memcg_params->destroy))
3351                 return;
3352         /*
3353          * We have to defer the actual destroying to a workqueue, because
3354          * we might currently be in a context that cannot sleep.
3355          */
3356         schedule_work(&cachep->memcg_params->destroy);
3357 }
3358 
3359 /*
3360  * This lock protects updaters, not readers. We want readers to be as fast as
3361  * they can, and they will either see NULL or a valid cache value. Our model
3362  * allow them to see NULL, in which case the root memcg will be selected.
3363  *
3364  * We need this lock because multiple allocations to the same cache from a non
3365  * will span more than one worker. Only one of them can create the cache.
3366  */
3367 static DEFINE_MUTEX(memcg_cache_mutex);
3368 
3369 /*
3370  * Called with memcg_cache_mutex held
3371  */
3372 static struct kmem_cache *kmem_cache_dup(struct mem_cgroup *memcg,
3373                                          struct kmem_cache *s)
3374 {
3375         struct kmem_cache *new;
3376         static char *tmp_name = NULL;
3377 
3378         lockdep_assert_held(&memcg_cache_mutex);
3379 
3380         /*
3381          * kmem_cache_create_memcg duplicates the given name and
3382          * cgroup_name for this name requires RCU context.
3383          * This static temporary buffer is used to prevent from
3384          * pointless shortliving allocation.
3385          */
3386         if (!tmp_name) {
3387                 tmp_name = kmalloc(PATH_MAX, GFP_KERNEL);
3388                 if (!tmp_name)
3389                         return NULL;
3390         }
3391 
3392         rcu_read_lock();
3393         snprintf(tmp_name, PATH_MAX, "%s(%d:%s)", s->name,
3394                          memcg_cache_id(memcg), cgroup_name(memcg->css.cgroup));
3395         rcu_read_unlock();
3396 
3397         new = kmem_cache_create_memcg(memcg, tmp_name, s->object_size, s->align,
3398                                       (s->flags & ~SLAB_PANIC), s->ctor, s);
3399 
3400         if (new)
3401                 new->allocflags |= __GFP_KMEMCG;
3402 
3403         return new;
3404 }
3405 
3406 static struct kmem_cache *memcg_create_kmem_cache(struct mem_cgroup *memcg,
3407                                                   struct kmem_cache *cachep)
3408 {
3409         struct kmem_cache *new_cachep;
3410         int idx;
3411 
3412         BUG_ON(!memcg_can_account_kmem(memcg));
3413 
3414         idx = memcg_cache_id(memcg);
3415 
3416         mutex_lock(&memcg_cache_mutex);
3417         new_cachep = cachep->memcg_params->memcg_caches[idx];
3418         if (new_cachep)
3419                 goto out;
3420 
3421         new_cachep = kmem_cache_dup(memcg, cachep);
3422         if (new_cachep == NULL) {
3423                 new_cachep = cachep;
3424                 goto out;
3425         }
3426 
3427         mem_cgroup_get(memcg);
3428         atomic_set(&new_cachep->memcg_params->nr_pages , 0);
3429 
3430         cachep->memcg_params->memcg_caches[idx] = new_cachep;
3431         /*
3432          * the readers won't lock, make sure everybody sees the updated value,
3433          * so they won't put stuff in the queue again for no reason
3434          */
3435         wmb();
3436 out:
3437         mutex_unlock(&memcg_cache_mutex);
3438         return new_cachep;
3439 }
3440 
3441 void kmem_cache_destroy_memcg_children(struct kmem_cache *s)
3442 {
3443         struct kmem_cache *c;
3444         int i;
3445 
3446         if (!s->memcg_params)
3447                 return;
3448         if (!s->memcg_params->is_root_cache)
3449                 return;
3450 
3451         /*
3452          * If the cache is being destroyed, we trust that there is no one else
3453          * requesting objects from it. Even if there are, the sanity checks in
3454          * kmem_cache_destroy should caught this ill-case.
3455          *
3456          * Still, we don't want anyone else freeing memcg_caches under our
3457          * noses, which can happen if a new memcg comes to life. As usual,
3458          * we'll take the set_limit_mutex to protect ourselves against this.
3459          */
3460         mutex_lock(&set_limit_mutex);
3461         for (i = 0; i < memcg_limited_groups_array_size; i++) {
3462                 c = s->memcg_params->memcg_caches[i];
3463                 if (!c)
3464                         continue;
3465 
3466                 /*
3467                  * We will now manually delete the caches, so to avoid races
3468                  * we need to cancel all pending destruction workers and
3469                  * proceed with destruction ourselves.
3470                  *
3471                  * kmem_cache_destroy() will call kmem_cache_shrink internally,
3472                  * and that could spawn the workers again: it is likely that
3473                  * the cache still have active pages until this very moment.
3474                  * This would lead us back to mem_cgroup_destroy_cache.
3475                  *
3476                  * But that will not execute at all if the "dead" flag is not
3477                  * set, so flip it down to guarantee we are in control.
3478                  */
3479                 c->memcg_params->dead = false;
3480                 cancel_work_sync(&c->memcg_params->destroy);
3481                 kmem_cache_destroy(c);
3482         }
3483         mutex_unlock(&set_limit_mutex);
3484 }
3485 
3486 struct create_work {
3487         struct mem_cgroup *memcg;
3488         struct kmem_cache *cachep;
3489         struct work_struct work;
3490 };
3491 
3492 static void mem_cgroup_destroy_all_caches(struct mem_cgroup *memcg)
3493 {
3494         struct kmem_cache *cachep;
3495         struct memcg_cache_params *params;
3496 
3497         if (!memcg_kmem_is_active(memcg))
3498                 return;
3499 
3500         mutex_lock(&memcg->slab_caches_mutex);
3501         list_for_each_entry(params, &memcg->memcg_slab_caches, list) {
3502                 cachep = memcg_params_to_cache(params);
3503                 cachep->memcg_params->dead = true;
3504                 schedule_work(&cachep->memcg_params->destroy);
3505         }
3506         mutex_unlock(&memcg->slab_caches_mutex);
3507 }
3508 
3509 static void memcg_create_cache_work_func(struct work_struct *w)
3510 {
3511         struct create_work *cw;
3512 
3513         cw = container_of(w, struct create_work, work);
3514         memcg_create_kmem_cache(cw->memcg, cw->cachep);
3515         /* Drop the reference gotten when we enqueued. */
3516         css_put(&cw->memcg->css);
3517         kfree(cw);
3518 }
3519 
3520 /*
3521  * Enqueue the creation of a per-memcg kmem_cache.
3522  */
3523 static void __memcg_create_cache_enqueue(struct mem_cgroup *memcg,
3524                                          struct kmem_cache *cachep)
3525 {
3526         struct create_work *cw;
3527 
3528         cw = kmalloc(sizeof(struct create_work), GFP_NOWAIT);
3529         if (cw == NULL) {
3530                 css_put(&memcg->css);
3531                 return;
3532         }
3533 
3534         cw->memcg = memcg;
3535         cw->cachep = cachep;
3536 
3537         INIT_WORK(&cw->work, memcg_create_cache_work_func);
3538         schedule_work(&cw->work);
3539 }
3540 
3541 static void memcg_create_cache_enqueue(struct mem_cgroup *memcg,
3542                                        struct kmem_cache *cachep)
3543 {
3544         /*
3545          * We need to stop accounting when we kmalloc, because if the
3546          * corresponding kmalloc cache is not yet created, the first allocation
3547          * in __memcg_create_cache_enqueue will recurse.
3548          *
3549          * However, it is better to enclose the whole function. Depending on
3550          * the debugging options enabled, INIT_WORK(), for instance, can
3551          * trigger an allocation. This too, will make us recurse. Because at
3552          * this point we can't allow ourselves back into memcg_kmem_get_cache,
3553          * the safest choice is to do it like this, wrapping the whole function.
3554          */
3555         memcg_stop_kmem_account();
3556         __memcg_create_cache_enqueue(memcg, cachep);
3557         memcg_resume_kmem_account();
3558 }
3559 /*
3560  * Return the kmem_cache we're supposed to use for a slab allocation.
3561  * We try to use the current memcg's version of the cache.
3562  *
3563  * If the cache does not exist yet, if we are the first user of it,
3564  * we either create it immediately, if possible, or create it asynchronously
3565  * in a workqueue.
3566  * In the latter case, we will let the current allocation go through with
3567  * the original cache.
3568  *
3569  * Can't be called in interrupt context or from kernel threads.
3570  * This function needs to be called with rcu_read_lock() held.
3571  */
3572 struct kmem_cache *__memcg_kmem_get_cache(struct kmem_cache *cachep,
3573                                           gfp_t gfp)
3574 {
3575         struct mem_cgroup *memcg;
3576         int idx;
3577 
3578         VM_BUG_ON(!cachep->memcg_params);
3579         VM_BUG_ON(!cachep->memcg_params->is_root_cache);
3580 
3581         if (!current->mm || current->memcg_kmem_skip_account)
3582                 return cachep;
3583 
3584         rcu_read_lock();
3585         memcg = mem_cgroup_from_task(rcu_dereference(current->mm->owner));
3586 
3587         if (!memcg_can_account_kmem(memcg))
3588                 goto out;
3589 
3590         idx = memcg_cache_id(memcg);
3591 
3592         /*
3593          * barrier to mare sure we're always seeing the up to date value.  The
3594          * code updating memcg_caches will issue a write barrier to match this.
3595          */
3596         read_barrier_depends();
3597         if (likely(cachep->memcg_params->memcg_caches[idx])) {
3598                 cachep = cachep->memcg_params->memcg_caches[idx];
3599                 goto out;
3600         }
3601 
3602         /* The corresponding put will be done in the workqueue. */
3603         if (!css_tryget(&memcg->css))
3604                 goto out;
3605         rcu_read_unlock();
3606 
3607         /*
3608          * If we are in a safe context (can wait, and not in interrupt
3609          * context), we could be be predictable and return right away.
3610          * This would guarantee that the allocation being performed
3611          * already belongs in the new cache.
3612          *
3613          * However, there are some clashes that can arrive from locking.
3614          * For instance, because we acquire the slab_mutex while doing
3615          * kmem_cache_dup, this means no further allocation could happen
3616          * with the slab_mutex held.
3617          *
3618          * Also, because cache creation issue get_online_cpus(), this
3619          * creates a lock chain: memcg_slab_mutex -> cpu_hotplug_mutex,
3620          * that ends up reversed during cpu hotplug. (cpuset allocates
3621          * a bunch of GFP_KERNEL memory during cpuup). Due to all that,
3622          * better to defer everything.
3623          */
3624         memcg_create_cache_enqueue(memcg, cachep);
3625         return cachep;
3626 out:
3627         rcu_read_unlock();
3628         return cachep;
3629 }
3630 EXPORT_SYMBOL(__memcg_kmem_get_cache);
3631 
3632 /*
3633  * We need to verify if the allocation against current->mm->owner's memcg is
3634  * possible for the given order. But the page is not allocated yet, so we'll
3635  * need a further commit step to do the final arrangements.
3636  *
3637  * It is possible for the task to switch cgroups in this mean time, so at
3638  * commit time, we can't rely on task conversion any longer.  We'll then use
3639  * the handle argument to return to the caller which cgroup we should commit
3640  * against. We could also return the memcg directly and avoid the pointer
3641  * passing, but a boolean return value gives better semantics considering
3642  * the compiled-out case as well.
3643  *
3644  * Returning true means the allocation is possible.
3645  */
3646 bool
3647 __memcg_kmem_newpage_charge(gfp_t gfp, struct mem_cgroup **_memcg, int order)
3648 {
3649         struct mem_cgroup *memcg;
3650         int ret;
3651 
3652         *_memcg = NULL;
3653         memcg = try_get_mem_cgroup_from_mm(current->mm);
3654 
3655         /*
3656          * very rare case described in mem_cgroup_from_task. Unfortunately there
3657          * isn't much we can do without complicating this too much, and it would
3658          * be gfp-dependent anyway. Just let it go
3659          */
3660         if (unlikely(!memcg))
3661                 return true;
3662 
3663         if (!memcg_can_account_kmem(memcg)) {
3664                 css_put(&memcg->css);
3665                 return true;
3666         }
3667 
3668         ret = memcg_charge_kmem(memcg, gfp, PAGE_SIZE << order);
3669         if (!ret)
3670                 *_memcg = memcg;
3671 
3672         css_put(&memcg->css);
3673         return (ret == 0);
3674 }
3675 
3676 void __memcg_kmem_commit_charge(struct page *page, struct mem_cgroup *memcg,
3677                               int order)
3678 {
3679         struct page_cgroup *pc;
3680 
3681         VM_BUG_ON(mem_cgroup_is_root(memcg));
3682 
3683         /* The page allocation failed. Revert */
3684         if (!page) {
3685                 memcg_uncharge_kmem(memcg, PAGE_SIZE << order);
3686                 return;
3687         }
3688 
3689         pc = lookup_page_cgroup(page);
3690         lock_page_cgroup(pc);
3691         pc->mem_cgroup = memcg;
3692         SetPageCgroupUsed(pc);
3693         unlock_page_cgroup(pc);
3694 }
3695 
3696 void __memcg_kmem_uncharge_pages(struct page *page, int order)
3697 {
3698         struct mem_cgroup *memcg = NULL;
3699         struct page_cgroup *pc;
3700 
3701 
3702         pc = lookup_page_cgroup(page);
3703         /*
3704          * Fast unlocked return. Theoretically might have changed, have to
3705          * check again after locking.
3706          */
3707         if (!PageCgroupUsed(pc))
3708                 return;
3709 
3710         lock_page_cgroup(pc);
3711         if (PageCgroupUsed(pc)) {
3712                 memcg = pc->mem_cgroup;
3713                 ClearPageCgroupUsed(pc);
3714         }
3715         unlock_page_cgroup(pc);
3716 
3717         /*
3718          * We trust that only if there is a memcg associated with the page, it
3719          * is a valid allocation
3720          */
3721         if (!memcg)
3722                 return;
3723 
3724         VM_BUG_ON(mem_cgroup_is_root(memcg));
3725         memcg_uncharge_kmem(memcg, PAGE_SIZE << order);
3726 }
3727 #else
3728 static inline void mem_cgroup_destroy_all_caches(struct mem_cgroup *memcg)
3729 {
3730 }
3731 #endif /* CONFIG_MEMCG_KMEM */
3732 
3733 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
3734 
3735 #define PCGF_NOCOPY_AT_SPLIT (1 << PCG_LOCK | 1 << PCG_MIGRATION)
3736 /*
3737  * Because tail pages are not marked as "used", set it. We're under
3738  * zone->lru_lock, 'splitting on pmd' and compound_lock.
3739  * charge/uncharge will be never happen and move_account() is done under
3740  * compound_lock(), so we don't have to take care of races.
3741  */
3742 void mem_cgroup_split_huge_fixup(struct page *head)
3743 {
3744         struct page_cgroup *head_pc = lookup_page_cgroup(head);
3745         struct page_cgroup *pc;
3746         struct mem_cgroup *memcg;
3747         int i;
3748 
3749         if (mem_cgroup_disabled())
3750                 return;
3751 
3752         memcg = head_pc->mem_cgroup;
3753         for (i = 1; i < HPAGE_PMD_NR; i++) {
3754                 pc = head_pc + i;
3755                 pc->mem_cgroup = memcg;
3756                 smp_wmb();/* see __commit_charge() */
3757                 pc->flags = head_pc->flags & ~PCGF_NOCOPY_AT_SPLIT;
3758         }
3759         __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE],
3760                        HPAGE_PMD_NR);
3761 }
3762 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
3763 
3764 /**
3765  * mem_cgroup_move_account - move account of the page
3766  * @page: the page
3767  * @nr_pages: number of regular pages (>1 for huge pages)
3768  * @pc: page_cgroup of the page.
3769  * @from: mem_cgroup which the page is moved from.
3770  * @to: mem_cgroup which the page is moved to. @from != @to.
3771  *
3772  * The caller must confirm following.
3773  * - page is not on LRU (isolate_page() is useful.)
3774  * - compound_lock is held when nr_pages > 1
3775  *
3776  * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
3777  * from old cgroup.
3778  */
3779 static int mem_cgroup_move_account(struct page *page,
3780                                    unsigned int nr_pages,
3781                                    struct page_cgroup *pc,
3782                                    struct mem_cgroup *from,
3783                                    struct mem_cgroup *to)
3784 {
3785         unsigned long flags;
3786         int ret;
3787         bool anon = PageAnon(page);
3788 
3789         VM_BUG_ON(from == to);
3790         VM_BUG_ON(PageLRU(page));
3791         /*
3792          * The page is isolated from LRU. So, collapse function
3793          * will not handle this page. But page splitting can happen.
3794          * Do this check under compound_page_lock(). The caller should
3795          * hold it.
3796          */
3797         ret = -EBUSY;
3798         if (nr_pages > 1 && !PageTransHuge(page))
3799                 goto out;
3800 
3801         lock_page_cgroup(pc);
3802 
3803         ret = -EINVAL;
3804         if (!PageCgroupUsed(pc) || pc->mem_cgroup != from)
3805                 goto unlock;
3806 
3807         move_lock_mem_cgroup(from, &flags);
3808 
3809         if (!anon && page_mapped(page)) {
3810                 /* Update mapped_file data for mem_cgroup */
3811                 preempt_disable();
3812                 __this_cpu_dec(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
3813                 __this_cpu_inc(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
3814                 preempt_enable();
3815         }
3816         mem_cgroup_charge_statistics(from, page, anon, -nr_pages);
3817 
3818         /* caller should have done css_get */
3819         pc->mem_cgroup = to;
3820         mem_cgroup_charge_statistics(to, page, anon, nr_pages);
3821         move_unlock_mem_cgroup(from, &flags);
3822         ret = 0;
3823 unlock:
3824         unlock_page_cgroup(pc);
3825         /*
3826          * check events
3827          */
3828         memcg_check_events(to, page);
3829         memcg_check_events(from, page);
3830 out:
3831         return ret;
3832 }
3833 
3834 /**
3835  * mem_cgroup_move_parent - moves page to the parent group
3836  * @page: the page to move
3837  * @pc: page_cgroup of the page
3838  * @child: page's cgroup
3839  *
3840  * move charges to its parent or the root cgroup if the group has no
3841  * parent (aka use_hierarchy==0).
3842  * Although this might fail (get_page_unless_zero, isolate_lru_page or
3843  * mem_cgroup_move_account fails) the failure is always temporary and
3844  * it signals a race with a page removal/uncharge or migration. In the
3845  * first case the page is on the way out and it will vanish from the LRU
3846  * on the next attempt and the call should be retried later.
3847  * Isolation from the LRU fails only if page has been isolated from
3848  * the LRU since we looked at it and that usually means either global
3849  * reclaim or migration going on. The page will either get back to the
3850  * LRU or vanish.
3851  * Finaly mem_cgroup_move_account fails only if the page got uncharged
3852  * (!PageCgroupUsed) or moved to a different group. The page will
3853  * disappear in the next attempt.
3854  */
3855 static int mem_cgroup_move_parent(struct page *page,
3856                                   struct page_cgroup *pc,
3857                                   struct mem_cgroup *child)
3858 {
3859         struct mem_cgroup *parent;
3860         unsigned int nr_pages;
3861         unsigned long uninitialized_var(flags);
3862         int ret;
3863 
3864         VM_BUG_ON(mem_cgroup_is_root(child));
3865 
3866         ret = -EBUSY;
3867         if (!get_page_unless_zero(page))
3868                 goto out;
3869         if (isolate_lru_page(page))
3870                 goto put;
3871 
3872         nr_pages = hpage_nr_pages(page);
3873 
3874         parent = parent_mem_cgroup(child);
3875         /*
3876          * If no parent, move charges to root cgroup.
3877          */
3878         if (!parent)
3879                 parent = root_mem_cgroup;
3880 
3881         if (nr_pages > 1) {
3882                 VM_BUG_ON(!PageTransHuge(page));
3883                 flags = compound_lock_irqsave(page);
3884         }
3885 
3886         ret = mem_cgroup_move_account(page, nr_pages,
3887                                 pc, child, parent);
3888         if (!ret)
3889                 __mem_cgroup_cancel_local_charge(child, nr_pages);
3890 
3891         if (nr_pages > 1)
3892                 compound_unlock_irqrestore(page, flags);
3893         putback_lru_page(page);
3894 put:
3895         put_page(page);
3896 out:
3897         return ret;
3898 }
3899 
3900 /*
3901  * Charge the memory controller for page usage.
3902  * Return
3903  * 0 if the charge was successful
3904  * < 0 if the cgroup is over its limit
3905  */
3906 static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
3907                                 gfp_t gfp_mask, enum charge_type ctype)
3908 {
3909         struct mem_cgroup *memcg = NULL;
3910         unsigned int nr_pages = 1;
3911         bool oom = true;
3912         int ret;
3913 
3914         if (PageTransHuge(page)) {
3915                 nr_pages <<= compound_order(page);
3916                 VM_BUG_ON(!PageTransHuge(page));
3917                 /*
3918                  * Never OOM-kill a process for a huge page.  The
3919                  * fault handler will fall back to regular pages.
3920                  */
3921                 oom = false;
3922         }
3923 
3924         ret = __mem_cgroup_try_charge(mm, gfp_mask, nr_pages, &memcg, oom);
3925         if (ret == -ENOMEM)
3926                 return ret;
3927         __mem_cgroup_commit_charge(memcg, page, nr_pages, ctype, false);
3928         return 0;
3929 }
3930 
3931 int mem_cgroup_newpage_charge(struct page *page,
3932                               struct mm_struct *mm, gfp_t gfp_mask)
3933 {
3934         if (mem_cgroup_disabled())
3935                 return 0;
3936         VM_BUG_ON(page_mapped(page));
3937         VM_BUG_ON(page->mapping && !PageAnon(page));
3938         VM_BUG_ON(!mm);
3939         return mem_cgroup_charge_common(page, mm, gfp_mask,
3940                                         MEM_CGROUP_CHARGE_TYPE_ANON);
3941 }
3942 
3943 /*
3944  * While swap-in, try_charge -> commit or cancel, the page is locked.
3945  * And when try_charge() successfully returns, one refcnt to memcg without
3946  * struct page_cgroup is acquired. This refcnt will be consumed by
3947  * "commit()" or removed by "cancel()"
3948  */
3949 static int __mem_cgroup_try_charge_swapin(struct mm_struct *mm,
3950                                           struct page *page,
3951                                           gfp_t mask,
3952                                           struct mem_cgroup **memcgp)
3953 {
3954         struct mem_cgroup *memcg;
3955         struct page_cgroup *pc;
3956         int ret;
3957 
3958         pc = lookup_page_cgroup(page);
3959         /*
3960          * Every swap fault against a single page tries to charge the
3961          * page, bail as early as possible.  shmem_unuse() encounters
3962          * already charged pages, too.  The USED bit is protected by
3963          * the page lock, which serializes swap cache removal, which
3964          * in turn serializes uncharging.
3965          */
3966         if (PageCgroupUsed(pc))
3967                 return 0;
3968         if (!do_swap_account)
3969                 goto charge_cur_mm;
3970         memcg = try_get_mem_cgroup_from_page(page);
3971         if (!memcg)
3972                 goto charge_cur_mm;
3973         *memcgp = memcg;
3974         ret = __mem_cgroup_try_charge(NULL, mask, 1, memcgp, true);
3975         css_put(&memcg->css);
3976         if (ret == -EINTR)
3977                 ret = 0;
3978         return ret;
3979 charge_cur_mm:
3980         ret = __mem_cgroup_try_charge(mm, mask, 1, memcgp, true);
3981         if (ret == -EINTR)
3982                 ret = 0;
3983         return ret;
3984 }
3985 
3986 int mem_cgroup_try_charge_swapin(struct mm_struct *mm, struct page *page,
3987                                  gfp_t gfp_mask, struct mem_cgroup **memcgp)
3988 {
3989         *memcgp = NULL;
3990         if (mem_cgroup_disabled())
3991                 return 0;
3992         /*
3993          * A racing thread's fault, or swapoff, may have already
3994          * updated the pte, and even removed page from swap cache: in
3995          * those cases unuse_pte()'s pte_same() test will fail; but
3996          * there's also a KSM case which does need to charge the page.
3997          */
3998         if (!PageSwapCache(page)) {
3999                 int ret;
4000 
4001                 ret = __mem_cgroup_try_charge(mm, gfp_mask, 1, memcgp, true);
4002                 if (ret == -EINTR)
4003                         ret = 0;
4004                 return ret;
4005         }
4006         return __mem_cgroup_try_charge_swapin(mm, page, gfp_mask, memcgp);
4007 }
4008 
4009 void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *memcg)
4010 {
4011         if (mem_cgroup_disabled())
4012                 return;
4013         if (!memcg)
4014                 return;
4015         __mem_cgroup_cancel_charge(memcg, 1);
4016 }
4017 
4018 static void
4019 __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *memcg,
4020                                         enum charge_type ctype)
4021 {
4022         if (mem_cgroup_disabled())
4023                 return;
4024         if (!memcg)
4025                 return;
4026 
4027         __mem_cgroup_commit_charge(memcg, page, 1, ctype, true);
4028         /*
4029          * Now swap is on-memory. This means this page may be
4030          * counted both as mem and swap....double count.
4031          * Fix it by uncharging from memsw. Basically, this SwapCache is stable
4032          * under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page()
4033          * may call delete_from_swap_cache() before reach here.
4034          */
4035         if (do_swap_account && PageSwapCache(page)) {
4036                 swp_entry_t ent = {.val = page_private(page)};
4037                 mem_cgroup_uncharge_swap(ent);
4038         }
4039 }
4040 
4041 void mem_cgroup_commit_charge_swapin(struct page *page,
4042                                      struct mem_cgroup *memcg)
4043 {
4044         __mem_cgroup_commit_charge_swapin(page, memcg,
4045                                           MEM_CGROUP_CHARGE_TYPE_ANON);
4046 }
4047 
4048 int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
4049                                 gfp_t gfp_mask)
4050 {
4051         struct mem_cgroup *memcg = NULL;
4052         enum charge_type type = MEM_CGROUP_CHARGE_TYPE_CACHE;
4053         int ret;
4054 
4055         if (mem_cgroup_disabled())
4056                 return 0;
4057         if (PageCompound(page))
4058                 return 0;
4059 
4060         if (!PageSwapCache(page))
4061                 ret = mem_cgroup_charge_common(page, mm, gfp_mask, type);
4062         else { /* page is swapcache/shmem */
4063                 ret = __mem_cgroup_try_charge_swapin(mm, page,
4064                                                      gfp_mask, &memcg);
4065                 if (!ret)
4066                         __mem_cgroup_commit_charge_swapin(page, memcg, type);
4067         }
4068         return ret;
4069 }
4070 
4071 static void mem_cgroup_do_uncharge(struct mem_cgroup *memcg,
4072                                    unsigned int nr_pages,
4073                                    const enum charge_type ctype)
4074 {
4075         struct memcg_batch_info *batch = NULL;
4076         bool uncharge_memsw = true;
4077 
4078         /* If swapout, usage of swap doesn't decrease */
4079         if (!do_swap_account || ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
4080                 uncharge_memsw = false;
4081 
4082         batch = &current->memcg_batch;
4083         /*
4084          * In usual, we do css_get() when we remember memcg pointer.
4085          * But in this case, we keep res->usage until end of a series of
4086          * uncharges. Then, it's ok to ignore memcg's refcnt.
4087          */
4088         if (!batch->memcg)
4089                 batch->memcg = memcg;
4090         /*
4091          * do_batch > 0 when unmapping pages or inode invalidate/truncate.
4092          * In those cases, all pages freed continuously can be expected to be in
4093          * the same cgroup and we have chance to coalesce uncharges.
4094          * But we do uncharge one by one if this is killed by OOM(TIF_MEMDIE)
4095          * because we want to do uncharge as soon as possible.
4096          */
4097 
4098         if (!batch->do_batch || test_thread_flag(TIF_MEMDIE))
4099                 goto direct_uncharge;
4100 
4101         if (nr_pages > 1)
4102                 goto direct_uncharge;
4103 
4104         /*
4105          * In typical case, batch->memcg == mem. This means we can
4106          * merge a series of uncharges to an uncharge of res_counter.
4107          * If not, we uncharge res_counter ony by one.
4108          */
4109         if (batch->memcg != memcg)
4110                 goto direct_uncharge;
4111         /* remember freed charge and uncharge it later */
4112         batch->nr_pages++;
4113         if (uncharge_memsw)
4114                 batch->memsw_nr_pages++;
4115         return;
4116 direct_uncharge:
4117         res_counter_uncharge(&memcg->res, nr_pages * PAGE_SIZE);
4118         if (uncharge_memsw)
4119                 res_counter_uncharge(&memcg->memsw, nr_pages * PAGE_SIZE);
4120         if (unlikely(batch->memcg != memcg))
4121                 memcg_oom_recover(memcg);
4122 }
4123 
4124 /*
4125  * uncharge if !page_mapped(page)
4126  */
4127 static struct mem_cgroup *
4128 __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype,
4129                              bool end_migration)
4130 {
4131         struct mem_cgroup *memcg = NULL;
4132         unsigned int nr_pages = 1;
4133         struct page_cgroup *pc;
4134         bool anon;
4135 
4136         if (mem_cgroup_disabled())
4137                 return NULL;
4138 
4139         if (PageTransHuge(page)) {
4140                 nr_pages <<= compound_order(page);
4141                 VM_BUG_ON(!PageTransHuge(page));
4142         }
4143         /*
4144          * Check if our page_cgroup is valid
4145          */
4146         pc = lookup_page_cgroup(page);
4147         if (unlikely(!PageCgroupUsed(pc)))
4148                 return NULL;
4149 
4150         lock_page_cgroup(pc);
4151 
4152         memcg = pc->mem_cgroup;
4153 
4154         if (!PageCgroupUsed(pc))
4155                 goto unlock_out;
4156 
4157         anon = PageAnon(page);
4158 
4159         switch (ctype) {
4160         case MEM_CGROUP_CHARGE_TYPE_ANON:
4161                 /*
4162                  * Generally PageAnon tells if it's the anon statistics to be
4163                  * updated; but sometimes e.g. mem_cgroup_uncharge_page() is
4164                  * used before page reached the stage of being marked PageAnon.
4165                  */
4166                 anon = true;
4167                 /* fallthrough */
4168         case MEM_CGROUP_CHARGE_TYPE_DROP:
4169                 /* See mem_cgroup_prepare_migration() */
4170                 if (page_mapped(page))
4171                         goto unlock_out;
4172                 /*
4173                  * Pages under migration may not be uncharged.  But
4174                  * end_migration() /must/ be the one uncharging the
4175                  * unused post-migration page and so it has to call
4176                  * here with the migration bit still set.  See the
4177                  * res_counter handling below.
4178                  */
4179                 if (!end_migration && PageCgroupMigration(pc))
4180                         goto unlock_out;
4181                 break;
4182         case MEM_CGROUP_CHARGE_TYPE_SWAPOUT:
4183                 if (!PageAnon(page)) {  /* Shared memory */
4184                         if (page->mapping && !page_is_file_cache(page))
4185                                 goto unlock_out;
4186                 } else if (page_mapped(page)) /* Anon */
4187                                 goto unlock_out;
4188                 break;
4189         default:
4190                 break;
4191         }
4192 
4193         mem_cgroup_charge_statistics(memcg, page, anon, -nr_pages);
4194 
4195         ClearPageCgroupUsed(pc);
4196         /*
4197          * pc->mem_cgroup is not cleared here. It will be accessed when it's
4198          * freed from LRU. This is safe because uncharged page is expected not
4199          * to be reused (freed soon). Exception is SwapCache, it's handled by
4200          * special functions.
4201          */
4202 
4203         unlock_page_cgroup(pc);
4204         /*
4205          * even after unlock, we have memcg->res.usage here and this memcg
4206          * will never be freed.
4207          */
4208         memcg_check_events(memcg, page);
4209         if (do_swap_account && ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) {
4210                 mem_cgroup_swap_statistics(memcg, true);
4211                 mem_cgroup_get(memcg);
4212         }
4213         /*
4214          * Migration does not charge the res_counter for the
4215          * replacement page, so leave it alone when phasing out the
4216          * page that is unused after the migration.
4217          */
4218         if (!end_migration && !mem_cgroup_is_root(memcg))
4219                 mem_cgroup_do_uncharge(memcg, nr_pages, ctype);
4220 
4221         return memcg;
4222 
4223 unlock_out:
4224         unlock_page_cgroup(pc);
4225         return NULL;
4226 }
4227 
4228 void mem_cgroup_uncharge_page(struct page *page)
4229 {
4230         /* early check. */
4231         if (page_mapped(page))
4232                 return;
4233         VM_BUG_ON(page->mapping && !PageAnon(page));
4234         /*
4235          * If the page is in swap cache, uncharge should be deferred
4236          * to the swap path, which also properly accounts swap usage
4237          * and handles memcg lifetime.
4238          *
4239          * Note that this check is not stable and reclaim may add the
4240          * page to swap cache at any time after this.  However, if the
4241          * page is not in swap cache by the time page->mapcount hits
4242          * 0, there won't be any page table references to the swap
4243          * slot, and reclaim will free it and not actually write the
4244          * page to disk.
4245          */
4246         if (PageSwapCache(page))
4247                 return;
4248         __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_ANON, false);
4249 }
4250 
4251 void mem_cgroup_uncharge_cache_page(struct page *page)
4252 {
4253         VM_BUG_ON(page_mapped(page));
4254         VM_BUG_ON(page->mapping);
4255         __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE, false);
4256 }
4257 
4258 /*
4259  * Batch_start/batch_end is called in unmap_page_range/invlidate/trucate.
4260  * In that cases, pages are freed continuously and we can expect pages
4261  * are in the same memcg. All these calls itself limits the number of
4262  * pages freed at once, then uncharge_start/end() is called properly.
4263  * This may be called prural(2) times in a context,
4264  */
4265 
4266 void mem_cgroup_uncharge_start(void)
4267 {
4268         current->memcg_batch.do_batch++;
4269         /* We can do nest. */
4270         if (current->memcg_batch.do_batch == 1) {
4271                 current->memcg_batch.memcg = NULL;
4272                 current->memcg_batch.nr_pages = 0;
4273                 current->memcg_batch.memsw_nr_pages = 0;
4274         }
4275 }
4276 
4277 void mem_cgroup_uncharge_end(void)
4278 {
4279         struct memcg_batch_info *batch = &current->memcg_batch;
4280 
4281         if (!batch->do_batch)
4282                 return;
4283 
4284         batch->do_batch--;
4285         if (batch->do_batch) /* If stacked, do nothing. */
4286                 return;
4287 
4288         if (!batch->memcg)
4289                 return;
4290         /*
4291          * This "batch->memcg" is valid without any css_get/put etc...
4292          * bacause we hide charges behind us.
4293          */
4294         if (batch->nr_pages)
4295                 res_counter_uncharge(&batch->memcg->res,
4296                                      batch->nr_pages * PAGE_SIZE);
4297         if (batch->memsw_nr_pages)
4298                 res_counter_uncharge(&batch->memcg->memsw,
4299                                      batch->memsw_nr_pages * PAGE_SIZE);
4300         memcg_oom_recover(batch->memcg);
4301         /* forget this pointer (for sanity check) */
4302         batch->memcg = NULL;
4303 }
4304 
4305 #ifdef CONFIG_SWAP
4306 /*
4307  * called after __delete_from_swap_cache() and drop "page" account.
4308  * memcg information is recorded to swap_cgroup of "ent"
4309  */
4310 void
4311 mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout)
4312 {
4313         struct mem_cgroup *memcg;
4314         int ctype = MEM_CGROUP_CHARGE_TYPE_SWAPOUT;
4315 
4316         if (!swapout) /* this was a swap cache but the swap is unused ! */
4317                 ctype = MEM_CGROUP_CHARGE_TYPE_DROP;
4318 
4319         memcg = __mem_cgroup_uncharge_common(page, ctype, false);
4320 
4321         /*
4322          * record memcg information,  if swapout && memcg != NULL,
4323          * mem_cgroup_get() was called in uncharge().
4324          */
4325         if (do_swap_account && swapout && memcg)
4326                 swap_cgroup_record(ent, css_id(&memcg->css));
4327 }
4328 #endif
4329 
4330 #ifdef CONFIG_MEMCG_SWAP
4331 /*
4332  * called from swap_entry_free(). remove record in swap_cgroup and
4333  * uncharge "memsw" account.
4334  */
4335 void mem_cgroup_uncharge_swap(swp_entry_t ent)
4336 {
4337         struct mem_cgroup *memcg;
4338         unsigned short id;
4339 
4340         if (!do_swap_account)
4341                 return;
4342 
4343         id = swap_cgroup_record(ent, 0);
4344         rcu_read_lock();
4345         memcg = mem_cgroup_lookup(id);
4346         if (memcg) {
4347                 /*
4348                  * We uncharge this because swap is freed.
4349                  * This memcg can be obsolete one. We avoid calling css_tryget
4350                  */
4351                 if (!mem_cgroup_is_root(memcg))
4352                         res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
4353                 mem_cgroup_swap_statistics(memcg, false);
4354                 mem_cgroup_put(memcg);
4355         }
4356         rcu_read_unlock();
4357 }
4358 
4359 /**
4360  * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
4361  * @entry: swap entry to be moved
4362  * @from:  mem_cgroup which the entry is moved from
4363  * @to:  mem_cgroup which the entry is moved to
4364  *
4365  * It succeeds only when the swap_cgroup's record for this entry is the same
4366  * as the mem_cgroup's id of @from.
4367  *
4368  * Returns 0 on success, -EINVAL on failure.
4369  *
4370  * The caller must have charged to @to, IOW, called res_counter_charge() about
4371  * both res and memsw, and called css_get().
4372  */
4373 static int mem_cgroup_move_swap_account(swp_entry_t entry,
4374                                 struct mem_cgroup *from, struct mem_cgroup *to)
4375 {
4376         unsigned short old_id, new_id;
4377 
4378         old_id = css_id(&from->css);
4379         new_id = css_id(&to->css);
4380 
4381         if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
4382                 mem_cgroup_swap_statistics(from, false);
4383                 mem_cgroup_swap_statistics(to, true);
4384                 /*
4385                  * This function is only called from task migration context now.
4386                  * It postpones res_counter and refcount handling till the end
4387                  * of task migration(mem_cgroup_clear_mc()) for performance
4388                  * improvement. But we cannot postpone mem_cgroup_get(to)
4389                  * because if the process that has been moved to @to does
4390                  * swap-in, the refcount of @to might be decreased to 0.
4391                  */
4392                 mem_cgroup_get(to);
4393                 return 0;
4394         }
4395         return -EINVAL;
4396 }
4397 #else
4398 static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
4399                                 struct mem_cgroup *from, struct mem_cgroup *to)
4400 {
4401         return -EINVAL;
4402 }
4403 #endif
4404 
4405 /*
4406  * Before starting migration, account PAGE_SIZE to mem_cgroup that the old
4407  * page belongs to.
4408  */
4409 void mem_cgroup_prepare_migration(struct page *page, struct page *newpage,
4410                                   struct mem_cgroup **memcgp)
4411 {
4412         struct mem_cgroup *memcg = NULL;
4413         unsigned int nr_pages = 1;
4414         struct page_cgroup *pc;
4415         enum charge_type ctype;
4416 
4417         *memcgp = NULL;
4418 
4419         if (mem_cgroup_disabled())
4420                 return;
4421 
4422         if (PageTransHuge(page))
4423                 nr_pages <<= compound_order(page);
4424 
4425         pc = lookup_page_cgroup(page);
4426         lock_page_cgroup(pc);
4427         if (PageCgroupUsed(pc)) {
4428                 memcg = pc->mem_cgroup;
4429                 css_get(&memcg->css);
4430                 /*
4431                  * At migrating an anonymous page, its mapcount goes down
4432                  * to 0 and uncharge() will be called. But, even if it's fully
4433                  * unmapped, migration may fail and this page has to be
4434                  * charged again. We set MIGRATION flag here and delay uncharge
4435                  * until end_migration() is called
4436                  *
4437                  * Corner Case Thinking
4438                  * A)
4439                  * When the old page was mapped as Anon and it's unmap-and-freed
4440                  * while migration was ongoing.
4441                  * If unmap finds the old page, uncharge() of it will be delayed
4442                  * until end_migration(). If unmap finds a new page, it's
4443                  * uncharged when it make mapcount to be 1->0. If unmap code
4444                  * finds swap_migration_entry, the new page will not be mapped
4445                  * and end_migration() will find it(mapcount==0).
4446                  *
4447                  * B)
4448                  * When the old page was mapped but migraion fails, the kernel
4449                  * remaps it. A charge for it is kept by MIGRATION flag even
4450                  * if mapcount goes down to 0. We can do remap successfully
4451                  * without charging it again.
4452                  *
4453                  * C)
4454                  * The "old" page is under lock_page() until the end of
4455                  * migration, so, the old page itself will not be swapped-out.
4456                  * If the new page is swapped out before end_migraton, our
4457                  * hook to usual swap-out path will catch the event.
4458                  */
4459                 if (PageAnon(page))
4460                         SetPageCgroupMigration(pc);
4461         }
4462         unlock_page_cgroup(pc);
4463         /*
4464          * If the page is not charged at this point,
4465          * we return here.
4466          */
4467         if (!memcg)
4468                 return;
4469 
4470         *memcgp = memcg;
4471         /*
4472          * We charge new page before it's used/mapped. So, even if unlock_page()
4473          * is called before end_migration, we can catch all events on this new
4474          * page. In the case new page is migrated but not remapped, new page's
4475          * mapcount will be finally 0 and we call uncharge in end_migration().
4476          */
4477         if (PageAnon(page))
4478                 ctype = MEM_CGROUP_CHARGE_TYPE_ANON;
4479         else
4480                 ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
4481         /*
4482          * The page is committed to the memcg, but it's not actually
4483          * charged to the res_counter since we plan on replacing the
4484          * old one and only one page is going to be left afterwards.
4485          */
4486         __mem_cgroup_commit_charge(memcg, newpage, nr_pages, ctype, false);
4487 }
4488 
4489 /* remove redundant charge if migration failed*/
4490 void mem_cgroup_end_migration(struct mem_cgroup *memcg,
4491         struct page *oldpage, struct page *newpage, bool migration_ok)
4492 {
4493         struct page *used, *unused;
4494         struct page_cgroup *pc;
4495         bool anon;
4496 
4497         if (!memcg)
4498                 return;
4499 
4500         if (!migration_ok) {
4501                 used = oldpage;
4502                 unused = newpage;
4503         } else {
4504                 used = newpage;
4505                 unused = oldpage;
4506         }
4507         anon = PageAnon(used);
4508         __mem_cgroup_uncharge_common(unused,
4509                                      anon ? MEM_CGROUP_CHARGE_TYPE_ANON
4510                                      : MEM_CGROUP_CHARGE_TYPE_CACHE,
4511                                      true);
4512         css_put(&memcg->css);
4513         /*
4514          * We disallowed uncharge of pages under migration because mapcount
4515          * of the page goes down to zero, temporarly.
4516          * Clear the flag and check the page should be charged.
4517          */
4518         pc = lookup_page_cgroup(oldpage);
4519         lock_page_cgroup(pc);
4520         ClearPageCgroupMigration(pc);
4521         unlock_page_cgroup(pc);
4522 
4523         /*
4524          * If a page is a file cache, radix-tree replacement is very atomic
4525          * and we can skip this check. When it was an Anon page, its mapcount
4526          * goes down to 0. But because we added MIGRATION flage, it's not
4527          * uncharged yet. There are several case but page->mapcount check
4528          * and USED bit check in mem_cgroup_uncharge_page() will do enough
4529          * check. (see prepare_charge() also)
4530          */
4531         if (anon)
4532                 mem_cgroup_uncharge_page(used);
4533 }
4534 
4535 /*
4536  * At replace page cache, newpage is not under any memcg but it's on
4537  * LRU. So, this function doesn't touch res_counter but handles LRU
4538  * in correct way. Both pages are locked so we cannot race with uncharge.
4539  */
4540 void mem_cgroup_replace_page_cache(struct page *oldpage,
4541                                   struct page *newpage)
4542 {
4543         struct mem_cgroup *memcg = NULL;
4544         struct page_cgroup *pc;
4545         enum charge_type type = MEM_CGROUP_CHARGE_TYPE_CACHE;
4546 
4547         if (mem_cgroup_disabled())
4548                 return;
4549 
4550         pc = lookup_page_cgroup(oldpage);
4551         /* fix accounting on old pages */
4552         lock_page_cgroup(pc);
4553         if (PageCgroupUsed(pc)) {
4554                 memcg = pc->mem_cgroup;
4555                 mem_cgroup_charge_statistics(memcg, oldpage, false, -1);
4556                 ClearPageCgroupUsed(pc);
4557         }
4558         unlock_page_cgroup(pc);
4559 
4560         /*
4561          * When called from shmem_replace_page(), in some cases the
4562          * oldpage has already been charged, and in some cases not.
4563          */
4564         if (!memcg)
4565                 return;
4566         /*
4567          * Even if newpage->mapping was NULL before starting replacement,
4568          * the newpage may be on LRU(or pagevec for LRU) already. We lock
4569          * LRU while we overwrite pc->mem_cgroup.
4570          */
4571         __mem_cgroup_commit_charge(memcg, newpage, 1, type, true);
4572 }
4573 
4574 #ifdef CONFIG_DEBUG_VM
4575 static struct page_cgroup *lookup_page_cgroup_used(struct page *page)
4576 {
4577         struct page_cgroup *pc;
4578 
4579         pc = lookup_page_cgroup(page);
4580         /*
4581          * Can be NULL while feeding pages into the page allocator for
4582          * the first time, i.e. during boot or memory hotplug;
4583          * or when mem_cgroup_disabled().
4584          */
4585         if (likely(pc) && PageCgroupUsed(pc))
4586                 return pc;
4587         return NULL;
4588 }
4589 
4590 bool mem_cgroup_bad_page_check(struct page *page)
4591 {
4592         if (mem_cgroup_disabled())
4593                 return false;
4594 
4595         return lookup_page_cgroup_used(page) != NULL;
4596 }
4597 
4598 void mem_cgroup_print_bad_page(struct page *page)
4599 {
4600         struct page_cgroup *pc;
4601 
4602         pc = lookup_page_cgroup_used(page);
4603         if (pc) {
4604                 pr_alert("pc:%p pc->flags:%lx pc->mem_cgroup:%p\n",
4605                          pc, pc->flags, pc->mem_cgroup);
4606         }
4607 }
4608 #endif
4609 
4610 static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
4611                                 unsigned long long val)
4612 {
4613         int retry_count;
4614         u64 memswlimit, memlimit;
4615         int ret = 0;
4616         int children = mem_cgroup_count_children(memcg);
4617         u64 curusage, oldusage;
4618         int enlarge;
4619 
4620         /*
4621          * For keeping hierarchical_reclaim simple, how long we should retry
4622          * is depends on callers. We set our retry-count to be function
4623          * of # of children which we should visit in this loop.
4624          */
4625         retry_count = MEM_CGROUP_RECLAIM_RETRIES * children;
4626 
4627         oldusage = res_counter_read_u64(&memcg->res, RES_USAGE);
4628 
4629         enlarge = 0;
4630         while (retry_count) {
4631                 if (signal_pending(current)) {
4632                         ret = -EINTR;
4633                         break;
4634                 }
4635                 /*
4636                  * Rather than hide all in some function, I do this in
4637                  * open coded manner. You see what this really does.
4638                  * We have to guarantee memcg->res.limit <= memcg->memsw.limit.
4639                  */
4640                 mutex_lock(&set_limit_mutex);
4641                 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
4642                 if (memswlimit < val) {
4643                         ret = -EINVAL;
4644                         mutex_unlock(&set_limit_mutex);
4645                         break;
4646                 }
4647 
4648                 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
4649                 if (memlimit < val)
4650                         enlarge = 1;
4651 
4652                 ret = res_counter_set_limit(&memcg->res, val);
4653                 if (!ret) {
4654                         if (memswlimit == val)
4655                                 memcg->memsw_is_minimum = true;
4656                         else
4657                                 memcg->memsw_is_minimum = false;
4658                 }
4659                 mutex_unlock(&set_limit_mutex);
4660 
4661                 if (!ret)
4662                         break;
4663 
4664                 mem_cgroup_reclaim(memcg, GFP_KERNEL,
4665                                    MEM_CGROUP_RECLAIM_SHRINK);
4666                 curusage = res_counter_read_u64(&memcg->res, RES_USAGE);
4667                 /* Usage is reduced ? */
4668                 if (curusage >= oldusage)
4669                         retry_count--;
4670                 else
4671                         oldusage = curusage;
4672         }
4673         if (!ret && enlarge)
4674                 memcg_oom_recover(memcg);
4675 
4676         return ret;
4677 }
4678 
4679 static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
4680                                         unsigned long long val)
4681 {
4682         int retry_count;
4683         u64 memlimit, memswlimit, oldusage, curusage;
4684         int children = mem_cgroup_count_children(memcg);
4685         int ret = -EBUSY;
4686         int enlarge = 0;
4687 
4688         /* see mem_cgroup_resize_res_limit */
4689         retry_count = children * MEM_CGROUP_RECLAIM_RETRIES;
4690         oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
4691         while (retry_count) {
4692                 if (signal_pending(current)) {
4693                         ret = -EINTR;
4694                         break;
4695                 }
4696                 /*
4697                  * Rather than hide all in some function, I do this in
4698                  * open coded manner. You see what this really does.
4699                  * We have to guarantee memcg->res.limit <= memcg->memsw.limit.
4700                  */
4701                 mutex_lock(&set_limit_mutex);
4702                 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
4703                 if (memlimit > val) {
4704                         ret = -EINVAL;
4705                         mutex_unlock(&set_limit_mutex);
4706                         break;
4707                 }
4708                 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
4709                 if (memswlimit < val)
4710                         enlarge = 1;
4711                 ret = res_counter_set_limit(&memcg->memsw, val);
4712                 if (!ret) {
4713                         if (memlimit == val)
4714                                 memcg->memsw_is_minimum = true;
4715                         else
4716                                 memcg->memsw_is_minimum = false;
4717                 }
4718                 mutex_unlock(&set_limit_mutex);
4719 
4720                 if (!ret)
4721                         break;
4722 
4723                 mem_cgroup_reclaim(memcg, GFP_KERNEL,
4724                                    MEM_CGROUP_RECLAIM_NOSWAP |
4725                                    MEM_CGROUP_RECLAIM_SHRINK);
4726                 curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
4727                 /* Usage is reduced ? */
4728                 if (curusage >= oldusage)
4729                         retry_count--;
4730                 else
4731                         oldusage = curusage;
4732         }
4733         if (!ret && enlarge)
4734                 memcg_oom_recover(memcg);
4735         return ret;
4736 }
4737 
4738 unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
4739                                             gfp_t gfp_mask,
4740                                             unsigned long *total_scanned)
4741 {
4742         unsigned long nr_reclaimed = 0;
4743         struct mem_cgroup_per_zone *mz, *next_mz = NULL;
4744         unsigned long reclaimed;
4745         int loop = 0;
4746         struct mem_cgroup_tree_per_zone *mctz;
4747         unsigned long long excess;
4748         unsigned long nr_scanned;
4749 
4750         if (order > 0)
4751                 return 0;
4752 
4753         mctz = soft_limit_tree_node_zone(zone_to_nid(zone), zone_idx(zone));
4754         /*
4755          * This loop can run a while, specially if mem_cgroup's continuously
4756          * keep exceeding their soft limit and putting the system under
4757          * pressure
4758          */
4759         do {
4760                 if (next_mz)
4761                         mz = next_mz;
4762                 else
4763                         mz = mem_cgroup_largest_soft_limit_node(mctz);
4764                 if (!mz)
4765                         break;
4766 
4767                 nr_scanned = 0;
4768                 reclaimed = mem_cgroup_soft_reclaim(mz->memcg, zone,
4769                                                     gfp_mask, &nr_scanned);
4770                 nr_reclaimed += reclaimed;
4771                 *total_scanned += nr_scanned;
4772                 spin_lock(&mctz->lock);
4773 
4774                 /*
4775                  * If we failed to reclaim anything from this memory cgroup
4776                  * it is time to move on to the next cgroup
4777                  */
4778                 next_mz = NULL;
4779                 if (!reclaimed) {
4780                         do {
4781                                 /*
4782                                  * Loop until we find yet another one.
4783                                  *
4784                                  * By the time we get the soft_limit lock
4785                                  * again, someone might have aded the
4786                                  * group back on the RB tree. Iterate to
4787                                  * make sure we get a different mem.
4788                                  * mem_cgroup_largest_soft_limit_node returns
4789                                  * NULL if no other cgroup is present on
4790                                  * the tree
4791                                  */
4792                                 next_mz =
4793                                 __mem_cgroup_largest_soft_limit_node(mctz);
4794                                 if (next_mz == mz)
4795                                         css_put(&next_mz->memcg->css);
4796                                 else /* next_mz == NULL or other memcg */
4797                                         break;
4798                         } while (1);
4799                 }
4800                 __mem_cgroup_remove_exceeded(mz->memcg, mz, mctz);
4801                 excess = res_counter_soft_limit_excess(&mz->memcg->res);
4802                 /*
4803                  * One school of thought says that we should not add
4804                  * back the node to the tree if reclaim returns 0.
4805                  * But our reclaim could return 0, simply because due
4806                  * to priority we are exposing a smaller subset of
4807                  * memory to reclaim from. Consider this as a longer
4808                  * term TODO.
4809                  */
4810                 /* If excess == 0, no tree ops */
4811                 __mem_cgroup_insert_exceeded(mz->memcg, mz, mctz, excess);
4812                 spin_unlock(&mctz->lock);
4813                 css_put(&mz->memcg->css);
4814                 loop++;
4815                 /*
4816                  * Could not reclaim anything and there are no more
4817                  * mem cgroups to try or we seem to be looping without
4818                  * reclaiming anything.
4819                  */
4820                 if (!nr_reclaimed &&
4821                         (next_mz == NULL ||
4822                         loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
4823                         break;
4824         } while (!nr_reclaimed);
4825         if (next_mz)
4826                 css_put(&next_mz->memcg->css);
4827         return nr_reclaimed;
4828 }
4829 
4830 /**
4831  * mem_cgroup_force_empty_list - clears LRU of a group
4832  * @memcg: group to clear
4833  * @node: NUMA node
4834  * @zid: zone id
4835  * @lru: lru to to clear
4836  *
4837  * Traverse a specified page_cgroup list and try to drop them all.  This doesn't
4838  * reclaim the pages page themselves - pages are moved to the parent (or root)
4839  * group.
4840  */
4841 static void mem_cgroup_force_empty_list(struct mem_cgroup *memcg,
4842                                 int node, int zid, enum lru_list lru)
4843 {
4844         struct lruvec *lruvec;
4845         unsigned long flags;
4846         struct list_head *list;
4847         struct page *busy;
4848         struct zone *zone;
4849 
4850         zone = &NODE_DATA(node)->node_zones[zid];
4851         lruvec = mem_cgroup_zone_lruvec(zone, memcg);
4852         list = &lruvec->lists[lru];
4853 
4854         busy = NULL;
4855         do {
4856                 struct page_cgroup *pc;
4857                 struct page *page;
4858 
4859                 spin_lock_irqsave(&zone->lru_lock, flags);
4860                 if (list_empty(list)) {
4861                         spin_unlock_irqrestore(&zone->lru_lock, flags);
4862                         break;
4863                 }
4864                 page = list_entry(list->prev, struct page, lru);
4865                 if (busy == page) {
4866                         list_move(&page->lru, list);
4867                         busy = NULL;
4868                         spin_unlock_irqrestore(&zone->lru_lock, flags);
4869                         continue;
4870                 }
4871                 spin_unlock_irqrestore(&zone->lru_lock, flags);
4872 
4873                 pc = lookup_page_cgroup(page);
4874 
4875                 if (mem_cgroup_move_parent(page, pc, memcg)) {
4876                         /* found lock contention or "pc" is obsolete. */
4877                         busy = page;
4878                         cond_resched();
4879                 } else
4880                         busy = NULL;
4881         } while (!list_empty(list));
4882 }
4883 
4884 /*
4885  * make mem_cgroup's charge to be 0 if there is no task by moving
4886  * all the charges and pages to the parent.
4887  * This enables deleting this mem_cgroup.
4888  *
4889  * Caller is responsible for holding css reference on the memcg.
4890  */
4891 static void mem_cgroup_reparent_charges(struct mem_cgroup *memcg)
4892 {
4893         int node, zid;
4894         u64 usage;
4895 
4896         do {
4897                 /* This is for making all *used* pages to be on LRU. */
4898                 lru_add_drain_all();
4899                 drain_all_stock_sync(memcg);
4900                 mem_cgroup_start_move(memcg);
4901                 for_each_node_state(node, N_MEMORY) {
4902                         for (zid = 0; zid < MAX_NR_ZONES; zid++) {
4903                                 enum lru_list lru;
4904                                 for_each_lru(lru) {
4905                                         mem_cgroup_force_empty_list(memcg,
4906                                                         node, zid, lru);
4907                                 }
4908                         }
4909                 }
4910                 mem_cgroup_end_move(memcg);
4911                 memcg_oom_recover(memcg);
4912                 cond_resched();
4913 
4914                 /*
4915                  * Kernel memory may not necessarily be trackable to a specific
4916                  * process. So they are not migrated, and therefore we can't
4917                  * expect their value to drop to 0 here.
4918                  * Having res filled up with kmem only is enough.
4919                  *
4920                  * This is a safety check because mem_cgroup_force_empty_list
4921                  * could have raced with mem_cgroup_replace_page_cache callers
4922                  * so the lru seemed empty but the page could have been added
4923                  * right after the check. RES_USAGE should be safe as we always
4924                  * charge before adding to the LRU.
4925                  */
4926                 usage = res_counter_read_u64(&memcg->res, RES_USAGE) -
4927                         res_counter_read_u64(&memcg->kmem, RES_USAGE);
4928         } while (usage > 0);
4929 }
4930 
4931 /*
4932  * This mainly exists for tests during the setting of set of use_hierarchy.
4933  * Since this is the very setting we are changing, the current hierarchy value
4934  * is meaningless
4935  */
4936 static inline bool __memcg_has_children(struct mem_cgroup *memcg)
4937 {
4938         struct cgroup *pos;
4939 
4940         /* bounce at first found */
4941         cgroup_for_each_child(pos, memcg->css.cgroup)
4942                 return true;
4943         return false;
4944 }
4945 
4946 /*
4947  * Must be called with memcg_create_mutex held, unless the cgroup is guaranteed
4948  * to be already dead (as in mem_cgroup_force_empty, for instance).  This is
4949  * from mem_cgroup_count_children(), in the sense that we don't really care how
4950  * many children we have; we only need to know if we have any.  It also counts
4951  * any memcg without hierarchy as infertile.
4952  */
4953 static inline bool memcg_has_children(struct mem_cgroup *memcg)
4954 {
4955         return memcg->use_hierarchy && __memcg_has_children(memcg);
4956 }
4957 
4958 /*
4959  * Reclaims as many pages from the given memcg as possible and moves
4960  * the rest to the parent.
4961  *
4962  * Caller is responsible for holding css reference for memcg.
4963  */
4964 static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
4965 {
4966         int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
4967         struct cgroup *cgrp = memcg->css.cgroup;
4968 
4969         /* returns EBUSY if there is a task or if we come here twice. */
4970         if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children))
4971                 return -EBUSY;
4972 
4973         /* we call try-to-free pages for make this cgroup empty */
4974         lru_add_drain_all();
4975         /* try to free all pages in this cgroup */
4976         while (nr_retries && res_counter_read_u64(&memcg->res, RES_USAGE) > 0) {
4977                 int progress;
4978 
4979                 if (signal_pending(current))
4980                         return -EINTR;
4981 
4982                 progress = try_to_free_mem_cgroup_pages(memcg, GFP_KERNEL,
4983                                                 false);
4984                 if (!progress) {
4985                         nr_retries--;
4986                         /* maybe some writeback is necessary */
4987                         congestion_wait(BLK_RW_ASYNC, HZ/10);
4988                 }
4989 
4990         }
4991         lru_add_drain();
4992         mem_cgroup_reparent_charges(memcg);
4993 
4994         return 0;
4995 }
4996 
4997 static int mem_cgroup_force_empty_write(struct cgroup *cont, unsigned int event)
4998 {
4999         struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
5000         int ret;
5001 
5002         if (mem_cgroup_is_root(memcg))
5003                 return -EINVAL;
5004         css_get(&memcg->css);
5005         ret = mem_cgroup_force_empty(memcg);
5006         css_put(&memcg->css);
5007 
5008         return ret;
5009 }
5010 
5011 
5012 static u64 mem_cgroup_hierarchy_read(struct cgroup *cont, struct cftype *cft)
5013 {
5014         return mem_cgroup_from_cont(cont)->use_hierarchy;
5015 }
5016 
5017 static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft,
5018                                         u64 val)
5019 {
5020         int retval = 0;
5021         struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
5022         struct cgroup *parent = cont->parent;
5023         struct mem_cgroup *parent_memcg = NULL;
5024 
5025         if (parent)
5026                 parent_memcg = mem_cgroup_from_cont(parent);
5027 
5028         mutex_lock(&memcg_create_mutex);
5029 
5030         if (memcg->use_hierarchy == val)
5031                 goto out;
5032 
5033         /*
5034          * If parent's use_hierarchy is set, we can't make any modifications
5035          * in the child subtrees. If it is unset, then the change can
5036          * occur, provided the current cgroup has no children.
5037          *
5038          * For the root cgroup, parent_mem is NULL, we allow value to be
5039          * set if there are no children.
5040          */
5041         if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
5042                                 (val == 1 || val == 0)) {
5043                 if (!__memcg_has_children(memcg))
5044                         memcg->use_hierarchy = val;
5045                 else
5046                         retval = -EBUSY;
5047         } else
5048                 retval = -EINVAL;
5049 
5050 out:
5051         mutex_unlock(&memcg_create_mutex);
5052 
5053         return retval;
5054 }
5055 
5056 
5057 static unsigned long mem_cgroup_recursive_stat(struct mem_cgroup *memcg,
5058                                                enum mem_cgroup_stat_index idx)
5059 {
5060         struct mem_cgroup *iter;
5061         long val = 0;
5062 
5063         /* Per-cpu values can be negative, use a signed accumulator */
5064         for_each_mem_cgroup_tree(iter, memcg)
5065                 val += mem_cgroup_read_stat(iter, idx);
5066 
5067         if (val < 0) /* race ? */
5068                 val = 0;
5069         return val;
5070 }
5071 
5072 static inline u64 mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
5073 {
5074         u64 val;
5075 
5076         if (!mem_cgroup_is_root(memcg)) {
5077                 if (!swap)
5078                         return res_counter_read_u64(&memcg->res, RES_USAGE);
5079                 else
5080                         return res_counter_read_u64(&memcg->memsw, RES_USAGE);
5081         }
5082 
5083         /*
5084          * Transparent hugepages are still accounted for in MEM_CGROUP_STAT_RSS
5085          * as well as in MEM_CGROUP_STAT_RSS_HUGE.
5086          */
5087         val = mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_CACHE);
5088         val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_RSS);
5089 
5090         if (swap)
5091                 val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_SWAP);
5092 
5093         return val << PAGE_SHIFT;
5094 }
5095 
5096 static ssize_t mem_cgroup_read(struct cgroup *cont, struct cftype *cft,
5097                                struct file *file, char __user *buf,
5098                                size_t nbytes, loff_t *ppos)
5099 {
5100         struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
5101         char str[64];
5102         u64 val;
5103         int name, len;
5104         enum res_type type;
5105 
5106         type = MEMFILE_TYPE(cft->private);
5107         name = MEMFILE_ATTR(cft->private);
5108 
5109         switch (type) {
5110         case _MEM:
5111                 if (name == RES_USAGE)
5112                         val = mem_cgroup_usage(memcg, false);
5113                 else
5114                         val = res_counter_read_u64(&memcg->res, name);
5115                 break;
5116         case _MEMSWAP:
5117                 if (name == RES_USAGE)
5118                         val = mem_cgroup_usage(memcg, true);
5119                 else
5120                         val = res_counter_read_u64(&memcg->memsw, name);
5121                 break;
5122         case _KMEM:
5123                 val = res_counter_read_u64(&memcg->kmem, name);
5124                 break;
5125         default:
5126                 BUG();
5127         }
5128 
5129         len = scnprintf(str, sizeof(str), "%llu\n", (unsigned long long)val);
5130         return simple_read_from_buffer(buf, nbytes, ppos, str, len);
5131 }
5132 
5133 static int memcg_update_kmem_limit(struct cgroup *cont, u64 val)
5134 {
5135         int ret = -EINVAL;
5136 #ifdef CONFIG_MEMCG_KMEM
5137         struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
5138         /*
5139          * For simplicity, we won't allow this to be disabled.  It also can't
5140          * be changed if the cgroup has children already, or if tasks had
5141          * already joined.
5142          *
5143          * If tasks join before we set the limit, a person looking at
5144          * kmem.usage_in_bytes will have no way to determine when it took
5145          * place, which makes the value quite meaningless.
5146          *
5147          * After it first became limited, changes in the value of the limit are
5148          * of course permitted.
5149          */
5150         mutex_lock(&memcg_create_mutex);
5151         mutex_lock(&set_limit_mutex);
5152         if (!memcg->kmem_account_flags && val != RESOURCE_MAX) {
5153                 if (cgroup_task_count(cont) || memcg_has_children(memcg)) {
5154                         ret = -EBUSY;
5155                         goto out;
5156                 }
5157                 ret = res_counter_set_limit(&memcg->kmem, val);
5158                 VM_BUG_ON(ret);
5159 
5160                 ret = memcg_update_cache_sizes(memcg);
5161                 if (ret) {
5162                         res_counter_set_limit(&memcg->kmem, RESOURCE_MAX);
5163                         goto out;
5164                 }
5165                 static_key_slow_inc(&memcg_kmem_enabled_key);
5166                 /*
5167                  * setting the active bit after the inc will guarantee no one
5168                  * starts accounting before all call sites are patched
5169                  */
5170                 memcg_kmem_set_active(memcg);
5171 
5172                 /*
5173                  * kmem charges can outlive the cgroup. In the case of slab
5174                  * pages, for instance, a page contain objects from various
5175                  * processes, so it is unfeasible to migrate them away. We
5176                  * need to reference count the memcg because of that.
5177                  */
5178                 mem_cgroup_get(memcg);
5179         } else
5180                 ret = res_counter_set_limit(&memcg->kmem, val);
5181 out:
5182         mutex_unlock(&set_limit_mutex);
5183         mutex_unlock(&memcg_create_mutex);
5184 #endif
5185         return ret;
5186 }
5187 
5188 #ifdef CONFIG_MEMCG_KMEM
5189 static int memcg_propagate_kmem(struct mem_cgroup *memcg)
5190 {
5191         int ret = 0;
5192         struct mem_cgroup *parent = parent_mem_cgroup(memcg);
5193         if (!parent)
5194                 goto out;
5195 
5196         memcg->kmem_account_flags = parent->kmem_account_flags;
5197         /*
5198          * When that happen, we need to disable the static branch only on those
5199          * memcgs that enabled it. To achieve this, we would be forced to
5200          * complicate the code by keeping track of which memcgs were the ones
5201          * that actually enabled limits, and which ones got it from its
5202          * parents.
5203          *
5204          * It is a lot simpler just to do static_key_slow_inc() on every child
5205          * that is accounted.
5206          */
5207         if (!memcg_kmem_is_active(memcg))
5208                 goto out;
5209 
5210         /*
5211          * destroy(), called if we fail, will issue static_key_slow_inc() and
5212          * mem_cgroup_put() if kmem is enabled. We have to either call them
5213          * unconditionally, or clear the KMEM_ACTIVE flag. I personally find
5214          * this more consistent, since it always leads to the same destroy path
5215          */
5216         mem_cgroup_get(memcg);
5217         static_key_slow_inc(&memcg_kmem_enabled_key);
5218 
5219         mutex_lock(&set_limit_mutex);
5220         ret = memcg_update_cache_sizes(memcg);
5221         mutex_unlock(&set_limit_mutex);
5222 out:
5223         return ret;
5224 }
5225 #endif /* CONFIG_MEMCG_KMEM */
5226 
5227 /*
5228  * The user of this function is...
5229  * RES_LIMIT.
5230  */
5231 static int mem_cgroup_write(struct cgroup *cont, struct cftype *cft,
5232                             const char *buffer)
5233 {
5234         struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
5235         enum res_type type;
5236         int name;
5237         unsigned long long val;
5238         int ret;
5239 
5240         type = MEMFILE_TYPE(cft->private);
5241         name = MEMFILE_ATTR(cft->private);
5242 
5243         switch (name) {
5244         case RES_LIMIT:
5245                 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
5246                         ret = -EINVAL;
5247                         break;
5248                 }
5249                 /* This function does all necessary parse...reuse it */
5250                 ret = res_counter_memparse_write_strategy(buffer, &val);
5251                 if (ret)
5252                         break;
5253                 if (type == _MEM)
5254                         ret = mem_cgroup_resize_limit(memcg, val);
5255                 else if (type == _MEMSWAP)
5256                         ret = mem_cgroup_resize_memsw_limit(memcg, val);
5257                 else if (type == _KMEM)
5258                         ret = memcg_update_kmem_limit(cont, val);
5259                 else
5260                         return -EINVAL;
5261                 break;
5262         case RES_SOFT_LIMIT:
5263                 ret = res_counter_memparse_write_strategy(buffer, &val);
5264                 if (ret)
5265                         break;
5266                 /*
5267                  * For memsw, soft limits are hard to implement in terms
5268                  * of semantics, for now, we support soft limits for
5269                  * control without swap
5270                  */
5271                 if (type == _MEM)
5272                         ret = res_counter_set_soft_limit(&memcg->res, val);
5273                 else
5274                         ret = -EINVAL;
5275                 break;
5276         default:
5277                 ret = -EINVAL; /* should be BUG() ? */
5278                 break;
5279         }
5280         return ret;
5281 }
5282 
5283 static void memcg_get_hierarchical_limit(struct mem_cgroup *memcg,
5284                 unsigned long long *mem_limit, unsigned long long *memsw_limit)
5285 {
5286         struct cgroup *cgroup;
5287         unsigned long long min_limit, min_memsw_limit, tmp;
5288 
5289         min_limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
5290         min_memsw_limit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
5291         cgroup = memcg->css.cgroup;
5292         if (!memcg->use_hierarchy)
5293                 goto out;
5294 
5295         while (cgroup->parent) {
5296                 cgroup = cgroup->parent;
5297                 memcg = mem_cgroup_from_cont(cgroup);
5298                 if (!memcg->use_hierarchy)
5299                         break;
5300                 tmp = res_counter_read_u64(&memcg->res, RES_LIMIT);
5301                 min_limit = min(min_limit, tmp);
5302                 tmp = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
5303                 min_memsw_limit = min(min_memsw_limit, tmp);
5304         }
5305 out:
5306         *mem_limit = min_limit;
5307         *memsw_limit = min_memsw_limit;
5308 }
5309 
5310 static int mem_cgroup_reset(struct cgroup *cont, unsigned int event)
5311 {
5312         struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
5313         int name;
5314         enum res_type type;
5315 
5316         type = MEMFILE_TYPE(event);
5317         name = MEMFILE_ATTR(event);
5318 
5319         switch (name) {
5320         case RES_MAX_USAGE:
5321                 if (type == _MEM)
5322                         res_counter_reset_max(&memcg->res);
5323                 else if (type == _MEMSWAP)
5324                         res_counter_reset_max(&memcg->memsw);
5325                 else if (type == _KMEM)
5326                         res_counter_reset_max(&memcg->kmem);
5327                 else
5328                         return -EINVAL;
5329                 break;
5330         case RES_FAILCNT:
5331                 if (type == _MEM)
5332                         res_counter_reset_failcnt(&memcg->res);
5333                 else if (type == _MEMSWAP)
5334                         res_counter_reset_failcnt(&memcg->memsw);
5335                 else if (type == _KMEM)
5336                         res_counter_reset_failcnt(&memcg->kmem);
5337                 else
5338                         return -EINVAL;
5339                 break;
5340         }
5341 
5342         return 0;
5343 }
5344 
5345 static u64 mem_cgroup_move_charge_read(struct cgroup *cgrp,
5346                                         struct cftype *cft)
5347 {
5348         return mem_cgroup_from_cont(cgrp)->move_charge_at_immigrate;
5349 }
5350 
5351 #ifdef CONFIG_MMU
5352 static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
5353                                         struct cftype *cft, u64 val)
5354 {
5355         struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
5356 
5357         if (val >= (1 << NR_MOVE_TYPE))
5358                 return -EINVAL;
5359 
5360         /*
5361          * No kind of locking is needed in here, because ->can_attach() will
5362          * check this value once in the beginning of the process, and then carry
5363          * on with stale data. This means that changes to this value will only
5364          * affect task migrations starting after the change.
5365          */
5366         memcg->move_charge_at_immigrate = val;
5367         return 0;
5368 }
5369 #else
5370 static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
5371                                         struct cftype *cft, u64 val)
5372 {
5373         return -ENOSYS;
5374 }
5375 #endif
5376 
5377 #ifdef CONFIG_NUMA
5378 static int memcg_numa_stat_show(struct cgroup *cont, struct cftype *cft,
5379                                       struct seq_file *m)
5380 {
5381         int nid;
5382         unsigned long total_nr, file_nr, anon_nr, unevictable_nr;
5383         unsigned long node_nr;
5384         struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
5385 
5386         total_nr = mem_cgroup_nr_lru_pages(memcg, LRU_ALL);
5387         seq_printf(m, "total=%lu", total_nr);
5388         for_each_node_state(nid, N_MEMORY) {
5389                 node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL);
5390                 seq_printf(m, " N%d=%lu", nid, node_nr);
5391         }
5392         seq_putc(m, '\n');
5393 
5394         file_nr = mem_cgroup_nr_lru_pages(memcg, LRU_ALL_FILE);
5395         seq_printf(m, "file=%lu", file_nr);
5396         for_each_node_state(nid, N_MEMORY) {
5397                 node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
5398                                 LRU_ALL_FILE);
5399                 seq_printf(m, " N%d=%lu", nid, node_nr);
5400         }
5401         seq_putc(m, '\n');
5402 
5403         anon_nr = mem_cgroup_nr_lru_pages(memcg, LRU_ALL_ANON);
5404         seq_printf(m, "anon=%lu", anon_nr);
5405         for_each_node_state(nid, N_MEMORY) {
5406                 node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
5407                                 LRU_ALL_ANON);
5408                 seq_printf(m, " N%d=%lu", nid, node_nr);
5409         }
5410         seq_putc(m, '\n');
5411 
5412         unevictable_nr = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_UNEVICTABLE));
5413         seq_printf(m, "unevictable=%lu", unevictable_nr);
5414         for_each_node_state(nid, N_MEMORY) {
5415                 node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
5416                                 BIT(LRU_UNEVICTABLE));
5417                 seq_printf(m, " N%d=%lu", nid, node_nr);
5418         }
5419         seq_putc(m, '\n');
5420         return 0;
5421 }
5422 #endif /* CONFIG_NUMA */
5423 
5424 static inline void mem_cgroup_lru_names_not_uptodate(void)
5425 {
5426         BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_lru_names) != NR_LRU_LISTS);
5427 }
5428 
5429 static int memcg_stat_show(struct cgroup *cont, struct cftype *cft,
5430                                  struct seq_file *m)
5431 {
5432         struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
5433         struct mem_cgroup *mi;
5434         unsigned int i;
5435 
5436         for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
5437                 if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
5438                         continue;
5439                 seq_printf(m, "%s %ld\n", mem_cgroup_stat_names[i],
5440                            mem_cgroup_read_stat(memcg, i) * PAGE_SIZE);
5441         }
5442 
5443         for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++)
5444                 seq_printf(m, "%s %lu\n", mem_cgroup_events_names[i],
5445                            mem_cgroup_read_events(memcg, i));
5446 
5447         for (i = 0; i < NR_LRU_LISTS; i++)
5448                 seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i],
5449                            mem_cgroup_nr_lru_pages(memcg, BIT(i)) * PAGE_SIZE);
5450 
5451         /* Hierarchical information */
5452         {
5453                 unsigned long long limit, memsw_limit;
5454                 memcg_get_hierarchical_limit(memcg, &limit, &memsw_limit);
5455                 seq_printf(m, "hierarchical_memory_limit %llu\n", limit);
5456                 if (do_swap_account)
5457                         seq_printf(m, "hierarchical_memsw_limit %llu\n",
5458                                    memsw_limit);
5459         }
5460 
5461         for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
5462                 long long val = 0;
5463 
5464                 if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
5465                         continue;
5466                 for_each_mem_cgroup_tree(mi, memcg)
5467                         val += mem_cgroup_read_stat(mi, i) * PAGE_SIZE;
5468                 seq_printf(m, "total_%s %lld\n", mem_cgroup_stat_names[i], val);
5469         }
5470 
5471         for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
5472                 unsigned long long val = 0;
5473 
5474                 for_each_mem_cgroup_tree(mi, memcg)
5475                         val += mem_cgroup_read_events(mi, i);
5476                 seq_printf(m, "total_%s %llu\n",
5477                            mem_cgroup_events_names[i], val);
5478         }
5479 
5480         for (i = 0; i < NR_LRU_LISTS; i++) {
5481                 unsigned long long val = 0;
5482 
5483                 for_each_mem_cgroup_tree(mi, memcg)
5484                         val += mem_cgroup_nr_lru_pages(mi, BIT(i)) * PAGE_SIZE;
5485                 seq_printf(m, "total_%s %llu\n", mem_cgroup_lru_names[i], val);
5486         }
5487 
5488 #ifdef CONFIG_DEBUG_VM
5489         {
5490                 int nid, zid;
5491                 struct mem_cgroup_per_zone *mz;
5492                 struct zone_reclaim_stat *rstat;
5493                 unsigned long recent_rotated[2] = {0, 0};
5494                 unsigned long recent_scanned[2] = {0, 0};
5495 
5496                 for_each_online_node(nid)
5497                         for (zid = 0; zid < MAX_NR_ZONES; zid++) {
5498                                 mz = mem_cgroup_zoneinfo(memcg, nid, zid);
5499                                 rstat = &mz->lruvec.reclaim_stat;
5500 
5501                                 recent_rotated[0] += rstat->recent_rotated[0];
5502                                 recent_rotated[1] += rstat->recent_rotated[1];
5503                                 recent_scanned[0] += rstat->recent_scanned[0];
5504                                 recent_scanned[1] += rstat->recent_scanned[1];
5505                         }
5506                 seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]);
5507                 seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]);
5508                 seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]);
5509                 seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]);
5510         }
5511 #endif
5512 
5513         return 0;
5514 }
5515 
5516 static u64 mem_cgroup_swappiness_read(struct cgroup *cgrp, struct cftype *cft)
5517 {
5518         struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
5519 
5520         return mem_cgroup_swappiness(memcg);
5521 }
5522 
5523 static int mem_cgroup_swappiness_write(struct cgroup *cgrp, struct cftype *cft,
5524                                        u64 val)
5525 {
5526         struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
5527         struct mem_cgroup *parent;
5528 
5529         if (val > 100)
5530                 return -EINVAL;
5531 
5532         if (cgrp->parent == NULL)
5533                 return -EINVAL;
5534 
5535         parent = mem_cgroup_from_cont(cgrp->parent);
5536 
5537         mutex_lock(&memcg_create_mutex);
5538 
5539         /* If under hierarchy, only empty-root can set this value */
5540         if ((parent->use_hierarchy) || memcg_has_children(memcg)) {
5541                 mutex_unlock(&memcg_create_mutex);
5542                 return -EINVAL;
5543         }
5544 
5545         memcg->swappiness = val;
5546 
5547         mutex_unlock(&memcg_create_mutex);
5548 
5549         return 0;
5550 }
5551 
5552 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
5553 {
5554         struct mem_cgroup_threshold_ary *t;
5555         u64 usage;
5556         int i;
5557 
5558         rcu_read_lock();
5559         if (!swap)
5560                 t = rcu_dereference(memcg->thresholds.primary);
5561         else
5562                 t = rcu_dereference(memcg->memsw_thresholds.primary);
5563 
5564         if (!t)
5565                 goto unlock;
5566 
5567         usage = mem_cgroup_usage(memcg, swap);
5568 
5569         /*
5570          * current_threshold points to threshold just below or equal to usage.
5571          * If it's not true, a threshold was crossed after last
5572          * call of __mem_cgroup_threshold().
5573          */
5574         i = t->current_threshold;
5575 
5576         /*
5577          * Iterate backward over array of thresholds starting from
5578          * current_threshold and check if a threshold is crossed.
5579          * If none of thresholds below usage is crossed, we read
5580          * only one element of the array here.
5581          */
5582         for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
5583                 eventfd_signal(t->entries[i].eventfd, 1);
5584 
5585         /* i = current_threshold + 1 */
5586         i++;
5587 
5588         /*
5589          * Iterate forward over array of thresholds starting from
5590          * current_threshold+1 and check if a threshold is crossed.
5591          * If none of thresholds above usage is crossed, we read
5592          * only one element of the array here.
5593          */
5594         for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
5595                 eventfd_signal(t->entries[i].eventfd, 1);
5596 
5597         /* Update current_threshold */
5598         t->current_threshold = i - 1;
5599 unlock:
5600         rcu_read_unlock();
5601 }
5602 
5603 static void mem_cgroup_threshold(struct mem_cgroup *memcg)
5604 {
5605         while (memcg) {
5606                 __mem_cgroup_threshold(memcg, false);
5607                 if (do_swap_account)
5608                         __mem_cgroup_threshold(memcg, true);
5609 
5610                 memcg = parent_mem_cgroup(memcg);
5611         }
5612 }
5613 
5614 static int compare_thresholds(const void *a, const void *b)
5615 {
5616         const struct mem_cgroup_threshold *_a = a;
5617         const struct mem_cgroup_threshold *_b = b;
5618 
5619         if (_a->threshold > _b->threshold)
5620                 return 1;
5621 
5622         if (_a->threshold < _b->threshold)
5623                 return -1;
5624 
5625         return 0;
5626 }
5627 
5628 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
5629 {
5630         struct mem_cgroup_eventfd_list *ev;
5631 
5632         list_for_each_entry(ev, &memcg->oom_notify, list)
5633                 eventfd_signal(ev->eventfd, 1);
5634         return 0;
5635 }
5636 
5637 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
5638 {
5639         struct mem_cgroup *iter;
5640 
5641         for_each_mem_cgroup_tree(iter, memcg)
5642                 mem_cgroup_oom_notify_cb(iter);
5643 }
5644 
5645 static int mem_cgroup_usage_register_event(struct cgroup *cgrp,
5646         struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
5647 {
5648         struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
5649         struct mem_cgroup_thresholds *thresholds;
5650         struct mem_cgroup_threshold_ary *new;
5651         enum res_type type = MEMFILE_TYPE(cft->private);
5652         u64 threshold, usage;
5653         int i, size, ret;
5654 
5655         ret = res_counter_memparse_write_strategy(args, &threshold);
5656         if (ret)
5657                 return ret;
5658 
5659         mutex_lock(&memcg->thresholds_lock);
5660 
5661         if (type == _MEM)
5662                 thresholds = &memcg->thresholds;
5663         else if (type == _MEMSWAP)
5664                 thresholds = &memcg->memsw_thresholds;
5665         else
5666                 BUG();
5667 
5668         usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
5669 
5670         /* Check if a threshold crossed before adding a new one */
5671         if (thresholds->primary)
5672                 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
5673 
5674         size = thresholds->primary ? thresholds->primary->size + 1 : 1;
5675 
5676         /* Allocate memory for new array of thresholds */
5677         new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
5678                         GFP_KERNEL);
5679         if (!new) {
5680                 ret = -ENOMEM;
5681                 goto unlock;
5682         }
5683         new->size = size;
5684 
5685         /* Copy thresholds (if any) to new array */
5686         if (thresholds->primary) {
5687                 memcpy(new->entries, thresholds->primary->entries, (size - 1) *
5688                                 sizeof(struct mem_cgroup_threshold));
5689         }
5690 
5691         /* Add new threshold */
5692         new->entries[size - 1].eventfd = eventfd;
5693         new->entries[size - 1].threshold = threshold;
5694 
5695         /* Sort thresholds. Registering of new threshold isn't time-critical */
5696         sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
5697                         compare_thresholds, NULL);
5698 
5699         /* Find current threshold */
5700         new->current_threshold = -1;
5701         for (i = 0; i < size; i++) {
5702                 if (new->entries[i].threshold <= usage) {
5703                         /*
5704                          * new->current_threshold will not be used until
5705                          * rcu_assign_pointer(), so it's safe to increment
5706                          * it here.
5707                          */
5708                         ++new->current_threshold;
5709                 } else
5710                         break;
5711         }
5712 
5713         /* Free old spare buffer and save old primary buffer as spare */
5714         kfree(thresholds->spare);
5715         thresholds->spare = thresholds->primary;
5716 
5717         rcu_assign_pointer(thresholds->primary, new);
5718 
5719         /* To be sure that nobody uses thresholds */
5720         synchronize_rcu();
5721 
5722 unlock:
5723         mutex_unlock(&memcg->thresholds_lock);
5724 
5725         return ret;
5726 }
5727 
5728 static void mem_cgroup_usage_unregister_event(struct cgroup *cgrp,
5729         struct cftype *cft, struct eventfd_ctx *eventfd)
5730 {
5731         struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
5732         struct mem_cgroup_thresholds *thresholds;
5733         struct mem_cgroup_threshold_ary *new;
5734         enum res_type type = MEMFILE_TYPE(cft->private);
5735         u64 usage;
5736         int i, j, size;
5737 
5738         mutex_lock(&memcg->thresholds_lock);
5739         if (type == _MEM)
5740                 thresholds = &memcg->thresholds;
5741         else if (type == _MEMSWAP)
5742                 thresholds = &memcg->memsw_thresholds;
5743         else
5744                 BUG();
5745 
5746         if (!thresholds->primary)
5747                 goto unlock;
5748 
5749         usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
5750 
5751         /* Check if a threshold crossed before removing */
5752         __mem_cgroup_threshold(memcg, type == _MEMSWAP);
5753 
5754         /* Calculate new number of threshold */
5755         size = 0;
5756         for (i = 0; i < thresholds->primary->size; i++) {
5757                 if (thresholds->primary->entries[i].eventfd != eventfd)
5758                         size++;
5759         }
5760 
5761         new = thresholds->spare;
5762 
5763         /* Set thresholds array to NULL if we don't have thresholds */
5764         if (!size) {
5765                 kfree(new);
5766                 new = NULL;
5767                 goto swap_buffers;
5768         }
5769 
5770         new->size = size;
5771 
5772         /* Copy thresholds and find current threshold */
5773         new->current_threshold = -1;
5774         for (i = 0, j = 0; i < thresholds->primary->size; i++) {
5775                 if (thresholds->primary->entries[i].eventfd == eventfd)
5776                         continue;
5777 
5778                 new->entries[j] = thresholds->primary->entries[i];
5779                 if (new->entries[j].threshold <= usage) {
5780                         /*
5781                          * new->current_threshold will not be used
5782                          * until rcu_assign_pointer(), so it's safe to increment
5783                          * it here.
5784                          */
5785                         ++new->current_threshold;
5786                 }
5787                 j++;
5788         }
5789 
5790 swap_buffers:
5791         /* Swap primary and spare array */
5792         thresholds->spare = thresholds->primary;
5793 
5794         rcu_assign_pointer(thresholds->primary, new);
5795 
5796         /* To be sure that nobody uses thresholds */
5797         synchronize_rcu();
5798 
5799         /* If all events are unregistered, free the spare array */
5800         if (!new) {
5801                 kfree(thresholds->spare);
5802                 thresholds->spare = NULL;
5803         }
5804 unlock:
5805         mutex_unlock(&memcg->thresholds_lock);
5806 }
5807 
5808 static int mem_cgroup_oom_register_event(struct cgroup *cgrp,
5809         struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
5810 {
5811         struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
5812         struct mem_cgroup_eventfd_list *event;
5813         enum res_type type = MEMFILE_TYPE(cft->private);
5814 
5815         BUG_ON(type != _OOM_TYPE);
5816         event = kmalloc(sizeof(*event), GFP_KERNEL);
5817         if (!event)
5818                 return -ENOMEM;
5819 
5820         spin_lock(&memcg_oom_lock);
5821 
5822         event->eventfd = eventfd;
5823         list_add(&event->list, &memcg->oom_notify);
5824 
5825         /* already in OOM ? */
5826         if (atomic_read(&memcg->under_oom))
5827                 eventfd_signal(eventfd, 1);
5828         spin_unlock(&memcg_oom_lock);
5829 
5830         return 0;
5831 }
5832 
5833 static void mem_cgroup_oom_unregister_event(struct cgroup *cgrp,
5834         struct cftype *cft, struct eventfd_ctx *eventfd)
5835 {
5836         struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
5837         struct mem_cgroup_eventfd_list *ev, *tmp;
5838         enum res_type type = MEMFILE_TYPE(cft->private);
5839 
5840         BUG_ON(type != _OOM_TYPE);
5841 
5842         spin_lock(&memcg_oom_lock);
5843 
5844         list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
5845                 if (ev->eventfd == eventfd) {
5846                         list_del(&ev->list);
5847                         kfree(ev);
5848                 }
5849         }
5850 
5851         spin_unlock(&memcg_oom_lock);
5852 }
5853 
5854 static int mem_cgroup_oom_control_read(struct cgroup *cgrp,
5855         struct cftype *cft,  struct cgroup_map_cb *cb)
5856 {
5857         struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
5858 
5859         cb->fill(cb, "oom_kill_disable", memcg->oom_kill_disable);
5860 
5861         if (atomic_read(&memcg->under_oom))
5862                 cb->fill(cb, "under_oom", 1);
5863         else
5864                 cb->fill(cb, "under_oom", 0);
5865         return 0;
5866 }
5867 
5868 static int mem_cgroup_oom_control_write(struct cgroup *cgrp,
5869         struct cftype *cft, u64 val)
5870 {
5871         struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
5872         struct mem_cgroup *parent;
5873 
5874         /* cannot set to root cgroup and only 0 and 1 are allowed */
5875         if (!cgrp->parent || !((val == 0) || (val == 1)))
5876                 return -EINVAL;
5877 
5878         parent = mem_cgroup_from_cont(cgrp->parent);
5879 
5880         mutex_lock(&memcg_create_mutex);
5881         /* oom-kill-disable is a flag for subhierarchy. */
5882         if ((parent->use_hierarchy) || memcg_has_children(memcg)) {
5883                 mutex_unlock(&memcg_create_mutex);
5884                 return -EINVAL;
5885         }
5886         memcg->oom_kill_disable = val;
5887         if (!val)
5888                 memcg_oom_recover(memcg);
5889         mutex_unlock(&memcg_create_mutex);
5890         return 0;
5891 }
5892 
5893 #ifdef CONFIG_MEMCG_KMEM
5894 static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
5895 {
5896         int ret;
5897 
5898         memcg->kmemcg_id = -1;
5899         ret = memcg_propagate_kmem(memcg);
5900         if (ret)
5901                 return ret;
5902 
5903         return mem_cgroup_sockets_init(memcg, ss);
5904 }
5905 
5906 static void kmem_cgroup_destroy(struct mem_cgroup *memcg)
5907 {
5908         mem_cgroup_sockets_destroy(memcg);
5909 
5910         memcg_kmem_mark_dead(memcg);
5911 
5912         if (res_counter_read_u64(&memcg->kmem, RES_USAGE) != 0)
5913                 return;
5914 
5915         /*
5916          * Charges already down to 0, undo mem_cgroup_get() done in the charge
5917          * path here, being careful not to race with memcg_uncharge_kmem: it is
5918          * possible that the charges went down to 0 between mark_dead and the
5919          * res_counter read, so in that case, we don't need the put
5920          */
5921         if (memcg_kmem_test_and_clear_dead(memcg))
5922                 mem_cgroup_put(memcg);
5923 }
5924 #else
5925 static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
5926 {
5927         return 0;
5928 }
5929 
5930 static void kmem_cgroup_destroy(struct mem_cgroup *memcg)
5931 {
5932 }
5933 #endif
5934 
5935 static struct cftype mem_cgroup_files[] = {
5936         {
5937                 .name = "usage_in_bytes",
5938                 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
5939                 .read = mem_cgroup_read,
5940                 .register_event = mem_cgroup_usage_register_event,
5941                 .unregister_event = mem_cgroup_usage_unregister_event,
5942         },
5943         {
5944                 .name = "max_usage_in_bytes",
5945                 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
5946                 .trigger = mem_cgroup_reset,
5947                 .read = mem_cgroup_read,
5948         },
5949         {
5950                 .name = "limit_in_bytes",
5951                 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
5952                 .write_string = mem_cgroup_write,
5953                 .read = mem_cgroup_read,
5954         },
5955         {
5956                 .name = "soft_limit_in_bytes",
5957                 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
5958                 .write_string = mem_cgroup_write,
5959                 .read = mem_cgroup_read,
5960         },
5961         {
5962                 .name = "failcnt",
5963                 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
5964                 .trigger = mem_cgroup_reset,
5965                 .read = mem_cgroup_read,
5966         },
5967         {
5968                 .name = "stat",
5969                 .read_seq_string = memcg_stat_show,
5970         },
5971         {
5972                 .name = "force_empty",
5973                 .trigger = mem_cgroup_force_empty_write,
5974         },
5975         {
5976                 .name = "use_hierarchy",
5977                 .flags = CFTYPE_INSANE,
5978                 .write_u64 = mem_cgroup_hierarchy_write,
5979                 .read_u64 = mem_cgroup_hierarchy_read,
5980         },
5981         {
5982                 .name = "swappiness",
5983                 .read_u64 = mem_cgroup_swappiness_read,
5984                 .write_u64 = mem_cgroup_swappiness_write,
5985         },
5986         {
5987                 .name = "move_charge_at_immigrate",
5988                 .read_u64 = mem_cgroup_move_charge_read,
5989                 .write_u64 = mem_cgroup_move_charge_write,
5990         },
5991         {
5992                 .name = "oom_control",
5993                 .read_map = mem_cgroup_oom_control_read,
5994                 .write_u64 = mem_cgroup_oom_control_write,
5995                 .register_event = mem_cgroup_oom_register_event,
5996                 .unregister_event = mem_cgroup_oom_unregister_event,
5997                 .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
5998         },
5999         {
6000                 .name = "pressure_level",
6001                 .register_event = vmpressure_register_event,
6002                 .unregister_event = vmpressure_unregister_event,
6003         },
6004 #ifdef CONFIG_NUMA
6005         {
6006                 .name = "numa_stat",
6007                 .read_seq_string = memcg_numa_stat_show,
6008         },
6009 #endif
6010 #ifdef CONFIG_MEMCG_KMEM
6011         {
6012                 .name = "kmem.limit_in_bytes",
6013                 .private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
6014                 .write_string = mem_cgroup_write,
6015                 .read = mem_cgroup_read,
6016         },
6017         {
6018                 .name = "kmem.usage_in_bytes",
6019                 .private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
6020                 .read = mem_cgroup_read,
6021         },
6022         {
6023                 .name = "kmem.failcnt",
6024                 .private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
6025                 .trigger = mem_cgroup_reset,
6026                 .read = mem_cgroup_read,
6027         },
6028         {
6029                 .name = "kmem.max_usage_in_bytes",
6030                 .private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
6031                 .trigger = mem_cgroup_reset,
6032                 .read = mem_cgroup_read,
6033         },
6034 #ifdef CONFIG_SLABINFO
6035         {
6036                 .name = "kmem.slabinfo",
6037                 .read_seq_string = mem_cgroup_slabinfo_read,
6038         },
6039 #endif
6040 #endif
6041         { },    /* terminate */
6042 };
6043 
6044 #ifdef CONFIG_MEMCG_SWAP
6045 static struct cftype memsw_cgroup_files[] = {
6046         {
6047                 .name = "memsw.usage_in_bytes",
6048                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
6049                 .read = mem_cgroup_read,
6050                 .register_event = mem_cgroup_usage_register_event,
6051                 .unregister_event = mem_cgroup_usage_unregister_event,
6052         },
6053         {
6054                 .name = "memsw.max_usage_in_bytes",
6055                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
6056                 .trigger = mem_cgroup_reset,
6057                 .read = mem_cgroup_read,
6058         },
6059         {
6060                 .name = "memsw.limit_in_bytes",
6061                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
6062                 .write_string = mem_cgroup_write,
6063                 .read = mem_cgroup_read,
6064         },
6065         {
6066                 .name = "memsw.failcnt",
6067                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
6068                 .trigger = mem_cgroup_reset,
6069                 .read = mem_cgroup_read,
6070         },
6071         { },    /* terminate */
6072 };
6073 #endif
6074 static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
6075 {
6076         struct mem_cgroup_per_node *pn;
6077         struct mem_cgroup_per_zone *mz;
6078         int zone, tmp = node;
6079         /*
6080          * This routine is called against possible nodes.
6081          * But it's BUG to call kmalloc() against offline node.
6082          *
6083          * TODO: this routine can waste much memory for nodes which will
6084          *       never be onlined. It's better to use memory hotplug callback
6085          *       function.
6086          */
6087         if (!node_state(node, N_NORMAL_MEMORY))
6088                 tmp = -1;
6089         pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
6090         if (!pn)
6091                 return 1;
6092 
6093         for (zone = 0; zone < MAX_NR_ZONES; zone++) {
6094                 mz = &pn->zoneinfo[zone];
6095                 lruvec_init(&mz->lruvec);
6096                 mz->usage_in_excess = 0;
6097                 mz->on_tree = false;
6098                 mz->memcg = memcg;
6099         }
6100         memcg->info.nodeinfo[node] = pn;
6101         return 0;
6102 }
6103 
6104 static void free_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
6105 {
6106         kfree(memcg->info.nodeinfo[node]);
6107 }
6108 
6109 static struct mem_cgroup *mem_cgroup_alloc(void)
6110 {
6111         struct mem_cgroup *memcg;
6112         size_t size = memcg_size();
6113 
6114         /* Can be very big if nr_node_ids is very big */
6115         if (size < PAGE_SIZE)
6116                 memcg = kzalloc(size, GFP_KERNEL);
6117         else
6118                 memcg = vzalloc(size);
6119 
6120         if (!memcg)
6121                 return NULL;
6122 
6123         memcg->stat = alloc_percpu(struct mem_cgroup_stat_cpu);
6124         if (!memcg->stat)
6125                 goto out_free;
6126         spin_lock_init(&memcg->pcp_counter_lock);
6127         return memcg;
6128 
6129 out_free:
6130         if (size < PAGE_SIZE)
6131                 kfree(memcg);
6132         else
6133                 vfree(memcg);
6134         return NULL;
6135 }
6136 
6137 /*
6138  * At destroying mem_cgroup, references from swap_cgroup can remain.
6139  * (scanning all at force_empty is too costly...)
6140  *
6141  * Instead of clearing all references at force_empty, we remember
6142  * the number of reference from swap_cgroup and free mem_cgroup when
6143  * it goes down to 0.
6144  *
6145  * Removal of cgroup itself succeeds regardless of refs from swap.
6146  */
6147 
6148 static void __mem_cgroup_free(struct mem_cgroup *memcg)
6149 {
6150         int node;
6151         size_t size = memcg_size();
6152 
6153         mem_cgroup_remove_from_trees(memcg);
6154         free_css_id(&mem_cgroup_subsys, &memcg->css);
6155 
6156         for_each_node(node)
6157                 free_mem_cgroup_per_zone_info(memcg, node);
6158 
6159         free_percpu(memcg->stat);
6160 
6161         /*
6162          * We need to make sure that (at least for now), the jump label
6163          * destruction code runs outside of the cgroup lock. This is because
6164          * get_online_cpus(), which is called from the static_branch update,
6165          * can't be called inside the cgroup_lock. cpusets are the ones
6166          * enforcing this dependency, so if they ever change, we might as well.
6167          *
6168          * schedule_work() will guarantee this happens. Be careful if you need
6169          * to move this code around, and make sure it is outside
6170          * the cgroup_lock.
6171          */
6172         disarm_static_keys(memcg);
6173         if (size < PAGE_SIZE)
6174                 kfree(memcg);
6175         else
6176                 vfree(memcg);
6177 }
6178 
6179 
6180 /*
6181  * Helpers for freeing a kmalloc()ed/vzalloc()ed mem_cgroup by RCU,
6182  * but in process context.  The work_freeing structure is overlaid
6183  * on the rcu_freeing structure, which itself is overlaid on memsw.
6184  */
6185 static void free_work(struct work_struct *work)
6186 {
6187         struct mem_cgroup *memcg;
6188 
6189         memcg = container_of(work, struct mem_cgroup, work_freeing);
6190         __mem_cgroup_free(memcg);
6191 }
6192 
6193 static void free_rcu(struct rcu_head *rcu_head)
6194 {
6195         struct mem_cgroup *memcg;
6196 
6197         memcg = container_of(rcu_head, struct mem_cgroup, rcu_freeing);
6198         INIT_WORK(&memcg->work_freeing, free_work);
6199         schedule_work(&memcg->work_freeing);
6200 }
6201 
6202 static void mem_cgroup_get(struct mem_cgroup *memcg)
6203 {
6204         atomic_inc(&memcg->refcnt);
6205 }
6206 
6207 static void __mem_cgroup_put(struct mem_cgroup *memcg, int count)
6208 {
6209         if (atomic_sub_and_test(count, &memcg->refcnt)) {
6210                 struct mem_cgroup *parent = parent_mem_cgroup(memcg);
6211                 call_rcu(&memcg->rcu_freeing, free_rcu);
6212                 if (parent)
6213                         mem_cgroup_put(parent);
6214         }
6215 }
6216 
6217 static void mem_cgroup_put(struct mem_cgroup *memcg)
6218 {
6219         __mem_cgroup_put(memcg, 1);
6220 }
6221 
6222 /*
6223  * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled.
6224  */
6225 struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
6226 {
6227         if (!memcg->res.parent)
6228                 return NULL;
6229         return mem_cgroup_from_res_counter(memcg->res.parent, res);
6230 }
6231 EXPORT_SYMBOL(parent_mem_cgroup);
6232 
6233 static void __init mem_cgroup_soft_limit_tree_init(void)
6234 {
6235         struct mem_cgroup_tree_per_node *rtpn;
6236         struct mem_cgroup_tree_per_zone *rtpz;
6237         int tmp, node, zone;
6238 
6239         for_each_node(node) {
6240                 tmp = node;
6241                 if (!node_state(node, N_NORMAL_MEMORY))
6242                         tmp = -1;
6243                 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, tmp);
6244                 BUG_ON(!rtpn);
6245 
6246                 soft_limit_tree.rb_tree_per_node[node] = rtpn;
6247 
6248                 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
6249                         rtpz = &rtpn->rb_tree_per_zone[zone];
6250                         rtpz->rb_root = RB_ROOT;
6251                         spin_lock_init(&rtpz->lock);
6252                 }
6253         }
6254 }
6255 
6256 static struct cgroup_subsys_state * __ref
6257 mem_cgroup_css_alloc(struct cgroup *cont)
6258 {
6259         struct mem_cgroup *memcg;
6260         long error = -ENOMEM;
6261         int node;
6262 
6263         memcg = mem_cgroup_alloc();
6264         if (!memcg)
6265                 return ERR_PTR(error);
6266 
6267         for_each_node(node)
6268                 if (alloc_mem_cgroup_per_zone_info(memcg, node))
6269                         goto free_out;
6270 
6271         /* root ? */
6272         if (cont->parent == NULL) {
6273                 root_mem_cgroup = memcg;
6274                 res_counter_init(&memcg->res, NULL);
6275                 res_counter_init(&memcg->memsw, NULL);
6276                 res_counter_init(&memcg->kmem, NULL);
6277         }
6278 
6279         memcg->last_scanned_node = MAX_NUMNODES;
6280         INIT_LIST_HEAD(&memcg->oom_notify);
6281         atomic_set(&memcg->refcnt, 1);
6282         memcg->move_charge_at_immigrate = 0;
6283         mutex_init(&memcg->thresholds_lock);
6284         spin_lock_init(&memcg->move_lock);
6285         vmpressure_init(&memcg->vmpressure);
6286 
6287         return &memcg->css;
6288 
6289 free_out:
6290         __mem_cgroup_free(memcg);
6291         return ERR_PTR(error);
6292 }
6293 
6294 static int
6295 mem_cgroup_css_online(struct cgroup *cont)
6296 {
6297         struct mem_cgroup *memcg, *parent;
6298         int error = 0;
6299 
6300         if (!cont->parent)
6301                 return 0;
6302 
6303         mutex_lock(&memcg_create_mutex);
6304         memcg = mem_cgroup_from_cont(cont);
6305         parent = mem_cgroup_from_cont(cont->parent);
6306 
6307         memcg->use_hierarchy = parent->use_hierarchy;
6308         memcg->oom_kill_disable = parent->oom_kill_disable;
6309         memcg->swappiness = mem_cgroup_swappiness(parent);
6310 
6311         if (parent->use_hierarchy) {
6312                 res_counter_init(&memcg->res, &parent->res);
6313                 res_counter_init(&memcg->memsw, &parent->memsw);
6314                 res_counter_init(&memcg->kmem, &parent->kmem);
6315 
6316                 /*
6317                  * We increment refcnt of the parent to ensure that we can
6318                  * safely access it on res_counter_charge/uncharge.
6319                  * This refcnt will be decremented when freeing this
6320                  * mem_cgroup(see mem_cgroup_put).
6321                  */
6322                 mem_cgroup_get(parent);
6323         } else {
6324                 res_counter_init(&memcg->res, NULL);
6325                 res_counter_init(&memcg->memsw, NULL);
6326                 res_counter_init(&memcg->kmem, NULL);
6327                 /*
6328                  * Deeper hierachy with use_hierarchy == false doesn't make
6329                  * much sense so let cgroup subsystem know about this
6330                  * unfortunate state in our controller.