~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/mm/memcontrol.c

Version: ~ [ linux-5.13-rc1 ] ~ [ linux-5.12.2 ] ~ [ linux-5.11.19 ] ~ [ linux-5.10.35 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.117 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.190 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.232 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.268 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.268 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /* memcontrol.c - Memory Controller
  2  *
  3  * Copyright IBM Corporation, 2007
  4  * Author Balbir Singh <balbir@linux.vnet.ibm.com>
  5  *
  6  * Copyright 2007 OpenVZ SWsoft Inc
  7  * Author: Pavel Emelianov <xemul@openvz.org>
  8  *
  9  * Memory thresholds
 10  * Copyright (C) 2009 Nokia Corporation
 11  * Author: Kirill A. Shutemov
 12  *
 13  * Kernel Memory Controller
 14  * Copyright (C) 2012 Parallels Inc. and Google Inc.
 15  * Authors: Glauber Costa and Suleiman Souhlal
 16  *
 17  * This program is free software; you can redistribute it and/or modify
 18  * it under the terms of the GNU General Public License as published by
 19  * the Free Software Foundation; either version 2 of the License, or
 20  * (at your option) any later version.
 21  *
 22  * This program is distributed in the hope that it will be useful,
 23  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 24  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 25  * GNU General Public License for more details.
 26  */
 27 
 28 #include <linux/res_counter.h>
 29 #include <linux/memcontrol.h>
 30 #include <linux/cgroup.h>
 31 #include <linux/mm.h>
 32 #include <linux/hugetlb.h>
 33 #include <linux/pagemap.h>
 34 #include <linux/smp.h>
 35 #include <linux/page-flags.h>
 36 #include <linux/backing-dev.h>
 37 #include <linux/bit_spinlock.h>
 38 #include <linux/rcupdate.h>
 39 #include <linux/limits.h>
 40 #include <linux/export.h>
 41 #include <linux/mutex.h>
 42 #include <linux/rbtree.h>
 43 #include <linux/slab.h>
 44 #include <linux/swap.h>
 45 #include <linux/swapops.h>
 46 #include <linux/spinlock.h>
 47 #include <linux/eventfd.h>
 48 #include <linux/sort.h>
 49 #include <linux/fs.h>
 50 #include <linux/seq_file.h>
 51 #include <linux/vmalloc.h>
 52 #include <linux/vmpressure.h>
 53 #include <linux/mm_inline.h>
 54 #include <linux/page_cgroup.h>
 55 #include <linux/cpu.h>
 56 #include <linux/oom.h>
 57 #include <linux/lockdep.h>
 58 #include "internal.h"
 59 #include <net/sock.h>
 60 #include <net/ip.h>
 61 #include <net/tcp_memcontrol.h>
 62 
 63 #include <asm/uaccess.h>
 64 
 65 #include <trace/events/vmscan.h>
 66 
 67 struct cgroup_subsys mem_cgroup_subsys __read_mostly;
 68 EXPORT_SYMBOL(mem_cgroup_subsys);
 69 
 70 #define MEM_CGROUP_RECLAIM_RETRIES      5
 71 static struct mem_cgroup *root_mem_cgroup __read_mostly;
 72 
 73 #ifdef CONFIG_MEMCG_SWAP
 74 /* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */
 75 int do_swap_account __read_mostly;
 76 
 77 /* for remember boot option*/
 78 #ifdef CONFIG_MEMCG_SWAP_ENABLED
 79 static int really_do_swap_account __initdata = 1;
 80 #else
 81 static int really_do_swap_account __initdata = 0;
 82 #endif
 83 
 84 #else
 85 #define do_swap_account         0
 86 #endif
 87 
 88 
 89 static const char * const mem_cgroup_stat_names[] = {
 90         "cache",
 91         "rss",
 92         "rss_huge",
 93         "mapped_file",
 94         "writeback",
 95         "swap",
 96 };
 97 
 98 enum mem_cgroup_events_index {
 99         MEM_CGROUP_EVENTS_PGPGIN,       /* # of pages paged in */
100         MEM_CGROUP_EVENTS_PGPGOUT,      /* # of pages paged out */
101         MEM_CGROUP_EVENTS_PGFAULT,      /* # of page-faults */
102         MEM_CGROUP_EVENTS_PGMAJFAULT,   /* # of major page-faults */
103         MEM_CGROUP_EVENTS_NSTATS,
104 };
105 
106 static const char * const mem_cgroup_events_names[] = {
107         "pgpgin",
108         "pgpgout",
109         "pgfault",
110         "pgmajfault",
111 };
112 
113 static const char * const mem_cgroup_lru_names[] = {
114         "inactive_anon",
115         "active_anon",
116         "inactive_file",
117         "active_file",
118         "unevictable",
119 };
120 
121 /*
122  * Per memcg event counter is incremented at every pagein/pageout. With THP,
123  * it will be incremated by the number of pages. This counter is used for
124  * for trigger some periodic events. This is straightforward and better
125  * than using jiffies etc. to handle periodic memcg event.
126  */
127 enum mem_cgroup_events_target {
128         MEM_CGROUP_TARGET_THRESH,
129         MEM_CGROUP_TARGET_SOFTLIMIT,
130         MEM_CGROUP_TARGET_NUMAINFO,
131         MEM_CGROUP_NTARGETS,
132 };
133 #define THRESHOLDS_EVENTS_TARGET 128
134 #define SOFTLIMIT_EVENTS_TARGET 1024
135 #define NUMAINFO_EVENTS_TARGET  1024
136 
137 struct mem_cgroup_stat_cpu {
138         long count[MEM_CGROUP_STAT_NSTATS];
139         unsigned long events[MEM_CGROUP_EVENTS_NSTATS];
140         unsigned long nr_page_events;
141         unsigned long targets[MEM_CGROUP_NTARGETS];
142 };
143 
144 struct mem_cgroup_reclaim_iter {
145         /*
146          * last scanned hierarchy member. Valid only if last_dead_count
147          * matches memcg->dead_count of the hierarchy root group.
148          */
149         struct mem_cgroup *last_visited;
150         unsigned long last_dead_count;
151 
152         /* scan generation, increased every round-trip */
153         unsigned int generation;
154 };
155 
156 /*
157  * per-zone information in memory controller.
158  */
159 struct mem_cgroup_per_zone {
160         struct lruvec           lruvec;
161         unsigned long           lru_size[NR_LRU_LISTS];
162 
163         struct mem_cgroup_reclaim_iter reclaim_iter[DEF_PRIORITY + 1];
164 
165         struct rb_node          tree_node;      /* RB tree node */
166         unsigned long long      usage_in_excess;/* Set to the value by which */
167                                                 /* the soft limit is exceeded*/
168         bool                    on_tree;
169         struct mem_cgroup       *memcg;         /* Back pointer, we cannot */
170                                                 /* use container_of        */
171 };
172 
173 struct mem_cgroup_per_node {
174         struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
175 };
176 
177 /*
178  * Cgroups above their limits are maintained in a RB-Tree, independent of
179  * their hierarchy representation
180  */
181 
182 struct mem_cgroup_tree_per_zone {
183         struct rb_root rb_root;
184         spinlock_t lock;
185 };
186 
187 struct mem_cgroup_tree_per_node {
188         struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES];
189 };
190 
191 struct mem_cgroup_tree {
192         struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
193 };
194 
195 static struct mem_cgroup_tree soft_limit_tree __read_mostly;
196 
197 struct mem_cgroup_threshold {
198         struct eventfd_ctx *eventfd;
199         u64 threshold;
200 };
201 
202 /* For threshold */
203 struct mem_cgroup_threshold_ary {
204         /* An array index points to threshold just below or equal to usage. */
205         int current_threshold;
206         /* Size of entries[] */
207         unsigned int size;
208         /* Array of thresholds */
209         struct mem_cgroup_threshold entries[0];
210 };
211 
212 struct mem_cgroup_thresholds {
213         /* Primary thresholds array */
214         struct mem_cgroup_threshold_ary *primary;
215         /*
216          * Spare threshold array.
217          * This is needed to make mem_cgroup_unregister_event() "never fail".
218          * It must be able to store at least primary->size - 1 entries.
219          */
220         struct mem_cgroup_threshold_ary *spare;
221 };
222 
223 /* for OOM */
224 struct mem_cgroup_eventfd_list {
225         struct list_head list;
226         struct eventfd_ctx *eventfd;
227 };
228 
229 static void mem_cgroup_threshold(struct mem_cgroup *memcg);
230 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
231 
232 /*
233  * The memory controller data structure. The memory controller controls both
234  * page cache and RSS per cgroup. We would eventually like to provide
235  * statistics based on the statistics developed by Rik Van Riel for clock-pro,
236  * to help the administrator determine what knobs to tune.
237  *
238  * TODO: Add a water mark for the memory controller. Reclaim will begin when
239  * we hit the water mark. May be even add a low water mark, such that
240  * no reclaim occurs from a cgroup at it's low water mark, this is
241  * a feature that will be implemented much later in the future.
242  */
243 struct mem_cgroup {
244         struct cgroup_subsys_state css;
245         /*
246          * the counter to account for memory usage
247          */
248         struct res_counter res;
249 
250         /* vmpressure notifications */
251         struct vmpressure vmpressure;
252 
253         /* css_online() has been completed */
254         int initialized;
255 
256         /*
257          * the counter to account for mem+swap usage.
258          */
259         struct res_counter memsw;
260 
261         /*
262          * the counter to account for kernel memory usage.
263          */
264         struct res_counter kmem;
265         /*
266          * Should the accounting and control be hierarchical, per subtree?
267          */
268         bool use_hierarchy;
269         unsigned long kmem_account_flags; /* See KMEM_ACCOUNTED_*, below */
270 
271         bool            oom_lock;
272         atomic_t        under_oom;
273         atomic_t        oom_wakeups;
274 
275         int     swappiness;
276         /* OOM-Killer disable */
277         int             oom_kill_disable;
278 
279         /* set when res.limit == memsw.limit */
280         bool            memsw_is_minimum;
281 
282         /* protect arrays of thresholds */
283         struct mutex thresholds_lock;
284 
285         /* thresholds for memory usage. RCU-protected */
286         struct mem_cgroup_thresholds thresholds;
287 
288         /* thresholds for mem+swap usage. RCU-protected */
289         struct mem_cgroup_thresholds memsw_thresholds;
290 
291         /* For oom notifier event fd */
292         struct list_head oom_notify;
293 
294         /*
295          * Should we move charges of a task when a task is moved into this
296          * mem_cgroup ? And what type of charges should we move ?
297          */
298         unsigned long move_charge_at_immigrate;
299         /*
300          * set > 0 if pages under this cgroup are moving to other cgroup.
301          */
302         atomic_t        moving_account;
303         /* taken only while moving_account > 0 */
304         spinlock_t      move_lock;
305         /*
306          * percpu counter.
307          */
308         struct mem_cgroup_stat_cpu __percpu *stat;
309         /*
310          * used when a cpu is offlined or other synchronizations
311          * See mem_cgroup_read_stat().
312          */
313         struct mem_cgroup_stat_cpu nocpu_base;
314         spinlock_t pcp_counter_lock;
315 
316         atomic_t        dead_count;
317 #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_INET)
318         struct tcp_memcontrol tcp_mem;
319 #endif
320 #if defined(CONFIG_MEMCG_KMEM)
321         /* analogous to slab_common's slab_caches list. per-memcg */
322         struct list_head memcg_slab_caches;
323         /* Not a spinlock, we can take a lot of time walking the list */
324         struct mutex slab_caches_mutex;
325         /* Index in the kmem_cache->memcg_params->memcg_caches array */
326         int kmemcg_id;
327 #endif
328 
329         int last_scanned_node;
330 #if MAX_NUMNODES > 1
331         nodemask_t      scan_nodes;
332         atomic_t        numainfo_events;
333         atomic_t        numainfo_updating;
334 #endif
335 
336         struct mem_cgroup_per_node *nodeinfo[0];
337         /* WARNING: nodeinfo must be the last member here */
338 };
339 
340 static size_t memcg_size(void)
341 {
342         return sizeof(struct mem_cgroup) +
343                 nr_node_ids * sizeof(struct mem_cgroup_per_node *);
344 }
345 
346 /* internal only representation about the status of kmem accounting. */
347 enum {
348         KMEM_ACCOUNTED_ACTIVE = 0, /* accounted by this cgroup itself */
349         KMEM_ACCOUNTED_ACTIVATED, /* static key enabled. */
350         KMEM_ACCOUNTED_DEAD, /* dead memcg with pending kmem charges */
351 };
352 
353 /* We account when limit is on, but only after call sites are patched */
354 #define KMEM_ACCOUNTED_MASK \
355                 ((1 << KMEM_ACCOUNTED_ACTIVE) | (1 << KMEM_ACCOUNTED_ACTIVATED))
356 
357 #ifdef CONFIG_MEMCG_KMEM
358 static inline void memcg_kmem_set_active(struct mem_cgroup *memcg)
359 {
360         set_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags);
361 }
362 
363 static bool memcg_kmem_is_active(struct mem_cgroup *memcg)
364 {
365         return test_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags);
366 }
367 
368 static void memcg_kmem_set_activated(struct mem_cgroup *memcg)
369 {
370         set_bit(KMEM_ACCOUNTED_ACTIVATED, &memcg->kmem_account_flags);
371 }
372 
373 static void memcg_kmem_clear_activated(struct mem_cgroup *memcg)
374 {
375         clear_bit(KMEM_ACCOUNTED_ACTIVATED, &memcg->kmem_account_flags);
376 }
377 
378 static void memcg_kmem_mark_dead(struct mem_cgroup *memcg)
379 {
380         /*
381          * Our caller must use css_get() first, because memcg_uncharge_kmem()
382          * will call css_put() if it sees the memcg is dead.
383          */
384         smp_wmb();
385         if (test_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags))
386                 set_bit(KMEM_ACCOUNTED_DEAD, &memcg->kmem_account_flags);
387 }
388 
389 static bool memcg_kmem_test_and_clear_dead(struct mem_cgroup *memcg)
390 {
391         return test_and_clear_bit(KMEM_ACCOUNTED_DEAD,
392                                   &memcg->kmem_account_flags);
393 }
394 #endif
395 
396 /* Stuffs for move charges at task migration. */
397 /*
398  * Types of charges to be moved. "move_charge_at_immitgrate" and
399  * "immigrate_flags" are treated as a left-shifted bitmap of these types.
400  */
401 enum move_type {
402         MOVE_CHARGE_TYPE_ANON,  /* private anonymous page and swap of it */
403         MOVE_CHARGE_TYPE_FILE,  /* file page(including tmpfs) and swap of it */
404         NR_MOVE_TYPE,
405 };
406 
407 /* "mc" and its members are protected by cgroup_mutex */
408 static struct move_charge_struct {
409         spinlock_t        lock; /* for from, to */
410         struct mem_cgroup *from;
411         struct mem_cgroup *to;
412         unsigned long immigrate_flags;
413         unsigned long precharge;
414         unsigned long moved_charge;
415         unsigned long moved_swap;
416         struct task_struct *moving_task;        /* a task moving charges */
417         wait_queue_head_t waitq;                /* a waitq for other context */
418 } mc = {
419         .lock = __SPIN_LOCK_UNLOCKED(mc.lock),
420         .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
421 };
422 
423 static bool move_anon(void)
424 {
425         return test_bit(MOVE_CHARGE_TYPE_ANON, &mc.immigrate_flags);
426 }
427 
428 static bool move_file(void)
429 {
430         return test_bit(MOVE_CHARGE_TYPE_FILE, &mc.immigrate_flags);
431 }
432 
433 /*
434  * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
435  * limit reclaim to prevent infinite loops, if they ever occur.
436  */
437 #define MEM_CGROUP_MAX_RECLAIM_LOOPS            100
438 #define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2
439 
440 enum charge_type {
441         MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
442         MEM_CGROUP_CHARGE_TYPE_ANON,
443         MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */
444         MEM_CGROUP_CHARGE_TYPE_DROP,    /* a page was unused swap cache */
445         NR_CHARGE_TYPE,
446 };
447 
448 /* for encoding cft->private value on file */
449 enum res_type {
450         _MEM,
451         _MEMSWAP,
452         _OOM_TYPE,
453         _KMEM,
454 };
455 
456 #define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val))
457 #define MEMFILE_TYPE(val)       ((val) >> 16 & 0xffff)
458 #define MEMFILE_ATTR(val)       ((val) & 0xffff)
459 /* Used for OOM nofiier */
460 #define OOM_CONTROL             (0)
461 
462 /*
463  * Reclaim flags for mem_cgroup_hierarchical_reclaim
464  */
465 #define MEM_CGROUP_RECLAIM_NOSWAP_BIT   0x0
466 #define MEM_CGROUP_RECLAIM_NOSWAP       (1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)
467 #define MEM_CGROUP_RECLAIM_SHRINK_BIT   0x1
468 #define MEM_CGROUP_RECLAIM_SHRINK       (1 << MEM_CGROUP_RECLAIM_SHRINK_BIT)
469 
470 /*
471  * The memcg_create_mutex will be held whenever a new cgroup is created.
472  * As a consequence, any change that needs to protect against new child cgroups
473  * appearing has to hold it as well.
474  */
475 static DEFINE_MUTEX(memcg_create_mutex);
476 
477 struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *s)
478 {
479         return s ? container_of(s, struct mem_cgroup, css) : NULL;
480 }
481 
482 /* Some nice accessors for the vmpressure. */
483 struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
484 {
485         if (!memcg)
486                 memcg = root_mem_cgroup;
487         return &memcg->vmpressure;
488 }
489 
490 struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
491 {
492         return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
493 }
494 
495 struct vmpressure *css_to_vmpressure(struct cgroup_subsys_state *css)
496 {
497         return &mem_cgroup_from_css(css)->vmpressure;
498 }
499 
500 static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
501 {
502         return (memcg == root_mem_cgroup);
503 }
504 
505 /* Writing them here to avoid exposing memcg's inner layout */
506 #if defined(CONFIG_INET) && defined(CONFIG_MEMCG_KMEM)
507 
508 void sock_update_memcg(struct sock *sk)
509 {
510         if (mem_cgroup_sockets_enabled) {
511                 struct mem_cgroup *memcg;
512                 struct cg_proto *cg_proto;
513 
514                 BUG_ON(!sk->sk_prot->proto_cgroup);
515 
516                 /* Socket cloning can throw us here with sk_cgrp already
517                  * filled. It won't however, necessarily happen from
518                  * process context. So the test for root memcg given
519                  * the current task's memcg won't help us in this case.
520                  *
521                  * Respecting the original socket's memcg is a better
522                  * decision in this case.
523                  */
524                 if (sk->sk_cgrp) {
525                         BUG_ON(mem_cgroup_is_root(sk->sk_cgrp->memcg));
526                         css_get(&sk->sk_cgrp->memcg->css);
527                         return;
528                 }
529 
530                 rcu_read_lock();
531                 memcg = mem_cgroup_from_task(current);
532                 cg_proto = sk->sk_prot->proto_cgroup(memcg);
533                 if (!mem_cgroup_is_root(memcg) &&
534                     memcg_proto_active(cg_proto) && css_tryget(&memcg->css)) {
535                         sk->sk_cgrp = cg_proto;
536                 }
537                 rcu_read_unlock();
538         }
539 }
540 EXPORT_SYMBOL(sock_update_memcg);
541 
542 void sock_release_memcg(struct sock *sk)
543 {
544         if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
545                 struct mem_cgroup *memcg;
546                 WARN_ON(!sk->sk_cgrp->memcg);
547                 memcg = sk->sk_cgrp->memcg;
548                 css_put(&sk->sk_cgrp->memcg->css);
549         }
550 }
551 
552 struct cg_proto *tcp_proto_cgroup(struct mem_cgroup *memcg)
553 {
554         if (!memcg || mem_cgroup_is_root(memcg))
555                 return NULL;
556 
557         return &memcg->tcp_mem.cg_proto;
558 }
559 EXPORT_SYMBOL(tcp_proto_cgroup);
560 
561 static void disarm_sock_keys(struct mem_cgroup *memcg)
562 {
563         if (!memcg_proto_activated(&memcg->tcp_mem.cg_proto))
564                 return;
565         static_key_slow_dec(&memcg_socket_limit_enabled);
566 }
567 #else
568 static void disarm_sock_keys(struct mem_cgroup *memcg)
569 {
570 }
571 #endif
572 
573 #ifdef CONFIG_MEMCG_KMEM
574 /*
575  * This will be the memcg's index in each cache's ->memcg_params->memcg_caches.
576  * There are two main reasons for not using the css_id for this:
577  *  1) this works better in sparse environments, where we have a lot of memcgs,
578  *     but only a few kmem-limited. Or also, if we have, for instance, 200
579  *     memcgs, and none but the 200th is kmem-limited, we'd have to have a
580  *     200 entry array for that.
581  *
582  *  2) In order not to violate the cgroup API, we would like to do all memory
583  *     allocation in ->create(). At that point, we haven't yet allocated the
584  *     css_id. Having a separate index prevents us from messing with the cgroup
585  *     core for this
586  *
587  * The current size of the caches array is stored in
588  * memcg_limited_groups_array_size.  It will double each time we have to
589  * increase it.
590  */
591 static DEFINE_IDA(kmem_limited_groups);
592 int memcg_limited_groups_array_size;
593 
594 /*
595  * MIN_SIZE is different than 1, because we would like to avoid going through
596  * the alloc/free process all the time. In a small machine, 4 kmem-limited
597  * cgroups is a reasonable guess. In the future, it could be a parameter or
598  * tunable, but that is strictly not necessary.
599  *
600  * MAX_SIZE should be as large as the number of css_ids. Ideally, we could get
601  * this constant directly from cgroup, but it is understandable that this is
602  * better kept as an internal representation in cgroup.c. In any case, the
603  * css_id space is not getting any smaller, and we don't have to necessarily
604  * increase ours as well if it increases.
605  */
606 #define MEMCG_CACHES_MIN_SIZE 4
607 #define MEMCG_CACHES_MAX_SIZE 65535
608 
609 /*
610  * A lot of the calls to the cache allocation functions are expected to be
611  * inlined by the compiler. Since the calls to memcg_kmem_get_cache are
612  * conditional to this static branch, we'll have to allow modules that does
613  * kmem_cache_alloc and the such to see this symbol as well
614  */
615 struct static_key memcg_kmem_enabled_key;
616 EXPORT_SYMBOL(memcg_kmem_enabled_key);
617 
618 static void disarm_kmem_keys(struct mem_cgroup *memcg)
619 {
620         if (memcg_kmem_is_active(memcg)) {
621                 static_key_slow_dec(&memcg_kmem_enabled_key);
622                 ida_simple_remove(&kmem_limited_groups, memcg->kmemcg_id);
623         }
624         /*
625          * This check can't live in kmem destruction function,
626          * since the charges will outlive the cgroup
627          */
628         WARN_ON(res_counter_read_u64(&memcg->kmem, RES_USAGE) != 0);
629 }
630 #else
631 static void disarm_kmem_keys(struct mem_cgroup *memcg)
632 {
633 }
634 #endif /* CONFIG_MEMCG_KMEM */
635 
636 static void disarm_static_keys(struct mem_cgroup *memcg)
637 {
638         disarm_sock_keys(memcg);
639         disarm_kmem_keys(memcg);
640 }
641 
642 static void drain_all_stock_async(struct mem_cgroup *memcg);
643 
644 static struct mem_cgroup_per_zone *
645 mem_cgroup_zoneinfo(struct mem_cgroup *memcg, int nid, int zid)
646 {
647         VM_BUG_ON((unsigned)nid >= nr_node_ids);
648         return &memcg->nodeinfo[nid]->zoneinfo[zid];
649 }
650 
651 struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *memcg)
652 {
653         return &memcg->css;
654 }
655 
656 static struct mem_cgroup_per_zone *
657 page_cgroup_zoneinfo(struct mem_cgroup *memcg, struct page *page)
658 {
659         int nid = page_to_nid(page);
660         int zid = page_zonenum(page);
661 
662         return mem_cgroup_zoneinfo(memcg, nid, zid);
663 }
664 
665 static struct mem_cgroup_tree_per_zone *
666 soft_limit_tree_node_zone(int nid, int zid)
667 {
668         return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
669 }
670 
671 static struct mem_cgroup_tree_per_zone *
672 soft_limit_tree_from_page(struct page *page)
673 {
674         int nid = page_to_nid(page);
675         int zid = page_zonenum(page);
676 
677         return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
678 }
679 
680 static void
681 __mem_cgroup_insert_exceeded(struct mem_cgroup *memcg,
682                                 struct mem_cgroup_per_zone *mz,
683                                 struct mem_cgroup_tree_per_zone *mctz,
684                                 unsigned long long new_usage_in_excess)
685 {
686         struct rb_node **p = &mctz->rb_root.rb_node;
687         struct rb_node *parent = NULL;
688         struct mem_cgroup_per_zone *mz_node;
689 
690         if (mz->on_tree)
691                 return;
692 
693         mz->usage_in_excess = new_usage_in_excess;
694         if (!mz->usage_in_excess)
695                 return;
696         while (*p) {
697                 parent = *p;
698                 mz_node = rb_entry(parent, struct mem_cgroup_per_zone,
699                                         tree_node);
700                 if (mz->usage_in_excess < mz_node->usage_in_excess)
701                         p = &(*p)->rb_left;
702                 /*
703                  * We can't avoid mem cgroups that are over their soft
704                  * limit by the same amount
705                  */
706                 else if (mz->usage_in_excess >= mz_node->usage_in_excess)
707                         p = &(*p)->rb_right;
708         }
709         rb_link_node(&mz->tree_node, parent, p);
710         rb_insert_color(&mz->tree_node, &mctz->rb_root);
711         mz->on_tree = true;
712 }
713 
714 static void
715 __mem_cgroup_remove_exceeded(struct mem_cgroup *memcg,
716                                 struct mem_cgroup_per_zone *mz,
717                                 struct mem_cgroup_tree_per_zone *mctz)
718 {
719         if (!mz->on_tree)
720                 return;
721         rb_erase(&mz->tree_node, &mctz->rb_root);
722         mz->on_tree = false;
723 }
724 
725 static void
726 mem_cgroup_remove_exceeded(struct mem_cgroup *memcg,
727                                 struct mem_cgroup_per_zone *mz,
728                                 struct mem_cgroup_tree_per_zone *mctz)
729 {
730         spin_lock(&mctz->lock);
731         __mem_cgroup_remove_exceeded(memcg, mz, mctz);
732         spin_unlock(&mctz->lock);
733 }
734 
735 
736 static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
737 {
738         unsigned long long excess;
739         struct mem_cgroup_per_zone *mz;
740         struct mem_cgroup_tree_per_zone *mctz;
741         int nid = page_to_nid(page);
742         int zid = page_zonenum(page);
743         mctz = soft_limit_tree_from_page(page);
744 
745         /*
746          * Necessary to update all ancestors when hierarchy is used.
747          * because their event counter is not touched.
748          */
749         for (; memcg; memcg = parent_mem_cgroup(memcg)) {
750                 mz = mem_cgroup_zoneinfo(memcg, nid, zid);
751                 excess = res_counter_soft_limit_excess(&memcg->res);
752                 /*
753                  * We have to update the tree if mz is on RB-tree or
754                  * mem is over its softlimit.
755                  */
756                 if (excess || mz->on_tree) {
757                         spin_lock(&mctz->lock);
758                         /* if on-tree, remove it */
759                         if (mz->on_tree)
760                                 __mem_cgroup_remove_exceeded(memcg, mz, mctz);
761                         /*
762                          * Insert again. mz->usage_in_excess will be updated.
763                          * If excess is 0, no tree ops.
764                          */
765                         __mem_cgroup_insert_exceeded(memcg, mz, mctz, excess);
766                         spin_unlock(&mctz->lock);
767                 }
768         }
769 }
770 
771 static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
772 {
773         int node, zone;
774         struct mem_cgroup_per_zone *mz;
775         struct mem_cgroup_tree_per_zone *mctz;
776 
777         for_each_node(node) {
778                 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
779                         mz = mem_cgroup_zoneinfo(memcg, node, zone);
780                         mctz = soft_limit_tree_node_zone(node, zone);
781                         mem_cgroup_remove_exceeded(memcg, mz, mctz);
782                 }
783         }
784 }
785 
786 static struct mem_cgroup_per_zone *
787 __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
788 {
789         struct rb_node *rightmost = NULL;
790         struct mem_cgroup_per_zone *mz;
791 
792 retry:
793         mz = NULL;
794         rightmost = rb_last(&mctz->rb_root);
795         if (!rightmost)
796                 goto done;              /* Nothing to reclaim from */
797 
798         mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node);
799         /*
800          * Remove the node now but someone else can add it back,
801          * we will to add it back at the end of reclaim to its correct
802          * position in the tree.
803          */
804         __mem_cgroup_remove_exceeded(mz->memcg, mz, mctz);
805         if (!res_counter_soft_limit_excess(&mz->memcg->res) ||
806                 !css_tryget(&mz->memcg->css))
807                 goto retry;
808 done:
809         return mz;
810 }
811 
812 static struct mem_cgroup_per_zone *
813 mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
814 {
815         struct mem_cgroup_per_zone *mz;
816 
817         spin_lock(&mctz->lock);
818         mz = __mem_cgroup_largest_soft_limit_node(mctz);
819         spin_unlock(&mctz->lock);
820         return mz;
821 }
822 
823 /*
824  * Implementation Note: reading percpu statistics for memcg.
825  *
826  * Both of vmstat[] and percpu_counter has threshold and do periodic
827  * synchronization to implement "quick" read. There are trade-off between
828  * reading cost and precision of value. Then, we may have a chance to implement
829  * a periodic synchronizion of counter in memcg's counter.
830  *
831  * But this _read() function is used for user interface now. The user accounts
832  * memory usage by memory cgroup and he _always_ requires exact value because
833  * he accounts memory. Even if we provide quick-and-fuzzy read, we always
834  * have to visit all online cpus and make sum. So, for now, unnecessary
835  * synchronization is not implemented. (just implemented for cpu hotplug)
836  *
837  * If there are kernel internal actions which can make use of some not-exact
838  * value, and reading all cpu value can be performance bottleneck in some
839  * common workload, threashold and synchonization as vmstat[] should be
840  * implemented.
841  */
842 static long mem_cgroup_read_stat(struct mem_cgroup *memcg,
843                                  enum mem_cgroup_stat_index idx)
844 {
845         long val = 0;
846         int cpu;
847 
848         get_online_cpus();
849         for_each_online_cpu(cpu)
850                 val += per_cpu(memcg->stat->count[idx], cpu);
851 #ifdef CONFIG_HOTPLUG_CPU
852         spin_lock(&memcg->pcp_counter_lock);
853         val += memcg->nocpu_base.count[idx];
854         spin_unlock(&memcg->pcp_counter_lock);
855 #endif
856         put_online_cpus();
857         return val;
858 }
859 
860 static void mem_cgroup_swap_statistics(struct mem_cgroup *memcg,
861                                          bool charge)
862 {
863         int val = (charge) ? 1 : -1;
864         this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_SWAP], val);
865 }
866 
867 static unsigned long mem_cgroup_read_events(struct mem_cgroup *memcg,
868                                             enum mem_cgroup_events_index idx)
869 {
870         unsigned long val = 0;
871         int cpu;
872 
873         get_online_cpus();
874         for_each_online_cpu(cpu)
875                 val += per_cpu(memcg->stat->events[idx], cpu);
876 #ifdef CONFIG_HOTPLUG_CPU
877         spin_lock(&memcg->pcp_counter_lock);
878         val += memcg->nocpu_base.events[idx];
879         spin_unlock(&memcg->pcp_counter_lock);
880 #endif
881         put_online_cpus();
882         return val;
883 }
884 
885 static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
886                                          struct page *page,
887                                          bool anon, int nr_pages)
888 {
889         preempt_disable();
890 
891         /*
892          * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is
893          * counted as CACHE even if it's on ANON LRU.
894          */
895         if (anon)
896                 __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS],
897                                 nr_pages);
898         else
899                 __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_CACHE],
900                                 nr_pages);
901 
902         if (PageTransHuge(page))
903                 __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE],
904                                 nr_pages);
905 
906         /* pagein of a big page is an event. So, ignore page size */
907         if (nr_pages > 0)
908                 __this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGIN]);
909         else {
910                 __this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT]);
911                 nr_pages = -nr_pages; /* for event */
912         }
913 
914         __this_cpu_add(memcg->stat->nr_page_events, nr_pages);
915 
916         preempt_enable();
917 }
918 
919 unsigned long
920 mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
921 {
922         struct mem_cgroup_per_zone *mz;
923 
924         mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
925         return mz->lru_size[lru];
926 }
927 
928 static unsigned long
929 mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, int nid, int zid,
930                         unsigned int lru_mask)
931 {
932         struct mem_cgroup_per_zone *mz;
933         enum lru_list lru;
934         unsigned long ret = 0;
935 
936         mz = mem_cgroup_zoneinfo(memcg, nid, zid);
937 
938         for_each_lru(lru) {
939                 if (BIT(lru) & lru_mask)
940                         ret += mz->lru_size[lru];
941         }
942         return ret;
943 }
944 
945 static unsigned long
946 mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
947                         int nid, unsigned int lru_mask)
948 {
949         u64 total = 0;
950         int zid;
951 
952         for (zid = 0; zid < MAX_NR_ZONES; zid++)
953                 total += mem_cgroup_zone_nr_lru_pages(memcg,
954                                                 nid, zid, lru_mask);
955 
956         return total;
957 }
958 
959 static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
960                         unsigned int lru_mask)
961 {
962         int nid;
963         u64 total = 0;
964 
965         for_each_node_state(nid, N_MEMORY)
966                 total += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask);
967         return total;
968 }
969 
970 static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
971                                        enum mem_cgroup_events_target target)
972 {
973         unsigned long val, next;
974 
975         val = __this_cpu_read(memcg->stat->nr_page_events);
976         next = __this_cpu_read(memcg->stat->targets[target]);
977         /* from time_after() in jiffies.h */
978         if ((long)next - (long)val < 0) {
979                 switch (target) {
980                 case MEM_CGROUP_TARGET_THRESH:
981                         next = val + THRESHOLDS_EVENTS_TARGET;
982                         break;
983                 case MEM_CGROUP_TARGET_SOFTLIMIT:
984                         next = val + SOFTLIMIT_EVENTS_TARGET;
985                         break;
986                 case MEM_CGROUP_TARGET_NUMAINFO:
987                         next = val + NUMAINFO_EVENTS_TARGET;
988                         break;
989                 default:
990                         break;
991                 }
992                 __this_cpu_write(memcg->stat->targets[target], next);
993                 return true;
994         }
995         return false;
996 }
997 
998 /*
999  * Check events in order.
1000  *
1001  */
1002 static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
1003 {
1004         preempt_disable();
1005         /* threshold event is triggered in finer grain than soft limit */
1006         if (unlikely(mem_cgroup_event_ratelimit(memcg,
1007                                                 MEM_CGROUP_TARGET_THRESH))) {
1008                 bool do_softlimit;
1009                 bool do_numainfo __maybe_unused;
1010 
1011                 do_softlimit = mem_cgroup_event_ratelimit(memcg,
1012                                                 MEM_CGROUP_TARGET_SOFTLIMIT);
1013 #if MAX_NUMNODES > 1
1014                 do_numainfo = mem_cgroup_event_ratelimit(memcg,
1015                                                 MEM_CGROUP_TARGET_NUMAINFO);
1016 #endif
1017                 preempt_enable();
1018 
1019                 mem_cgroup_threshold(memcg);
1020                 if (unlikely(do_softlimit))
1021                         mem_cgroup_update_tree(memcg, page);
1022 #if MAX_NUMNODES > 1
1023                 if (unlikely(do_numainfo))
1024                         atomic_inc(&memcg->numainfo_events);
1025 #endif
1026         } else
1027                 preempt_enable();
1028 }
1029 
1030 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
1031 {
1032         /*
1033          * mm_update_next_owner() may clear mm->owner to NULL
1034          * if it races with swapoff, page migration, etc.
1035          * So this can be called with p == NULL.
1036          */
1037         if (unlikely(!p))
1038                 return NULL;
1039 
1040         return mem_cgroup_from_css(task_css(p, mem_cgroup_subsys_id));
1041 }
1042 
1043 struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
1044 {
1045         struct mem_cgroup *memcg = NULL;
1046 
1047         if (!mm)
1048                 return NULL;
1049         /*
1050          * Because we have no locks, mm->owner's may be being moved to other
1051          * cgroup. We use css_tryget() here even if this looks
1052          * pessimistic (rather than adding locks here).
1053          */
1054         rcu_read_lock();
1055         do {
1056                 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
1057                 if (unlikely(!memcg))
1058                         break;
1059         } while (!css_tryget(&memcg->css));
1060         rcu_read_unlock();
1061         return memcg;
1062 }
1063 
1064 /*
1065  * Returns a next (in a pre-order walk) alive memcg (with elevated css
1066  * ref. count) or NULL if the whole root's subtree has been visited.
1067  *
1068  * helper function to be used by mem_cgroup_iter
1069  */
1070 static struct mem_cgroup *__mem_cgroup_iter_next(struct mem_cgroup *root,
1071                 struct mem_cgroup *last_visited)
1072 {
1073         struct cgroup_subsys_state *prev_css, *next_css;
1074 
1075         prev_css = last_visited ? &last_visited->css : NULL;
1076 skip_node:
1077         next_css = css_next_descendant_pre(prev_css, &root->css);
1078 
1079         /*
1080          * Even if we found a group we have to make sure it is
1081          * alive. css && !memcg means that the groups should be
1082          * skipped and we should continue the tree walk.
1083          * last_visited css is safe to use because it is
1084          * protected by css_get and the tree walk is rcu safe.
1085          *
1086          * We do not take a reference on the root of the tree walk
1087          * because we might race with the root removal when it would
1088          * be the only node in the iterated hierarchy and mem_cgroup_iter
1089          * would end up in an endless loop because it expects that at
1090          * least one valid node will be returned. Root cannot disappear
1091          * because caller of the iterator should hold it already so
1092          * skipping css reference should be safe.
1093          */
1094         if (next_css) {
1095                 struct mem_cgroup *memcg = mem_cgroup_from_css(next_css);
1096 
1097                 if (next_css == &root->css)
1098                         return memcg;
1099 
1100                 if (css_tryget(next_css)) {
1101                         if (memcg->initialized) {
1102                                 /*
1103                                  * Make sure the memcg is initialized:
1104                                  * mem_cgroup_css_online() orders the the
1105                                  * initialization against setting the flag.
1106                                  */
1107                                 smp_rmb();
1108                                 return memcg;
1109                         }
1110                         css_put(next_css);
1111                 }
1112 
1113                 prev_css = next_css;
1114                 goto skip_node;
1115         }
1116 
1117         return NULL;
1118 }
1119 
1120 static void mem_cgroup_iter_invalidate(struct mem_cgroup *root)
1121 {
1122         /*
1123          * When a group in the hierarchy below root is destroyed, the
1124          * hierarchy iterator can no longer be trusted since it might
1125          * have pointed to the destroyed group.  Invalidate it.
1126          */
1127         atomic_inc(&root->dead_count);
1128 }
1129 
1130 static struct mem_cgroup *
1131 mem_cgroup_iter_load(struct mem_cgroup_reclaim_iter *iter,
1132                      struct mem_cgroup *root,
1133                      int *sequence)
1134 {
1135         struct mem_cgroup *position = NULL;
1136         /*
1137          * A cgroup destruction happens in two stages: offlining and
1138          * release.  They are separated by a RCU grace period.
1139          *
1140          * If the iterator is valid, we may still race with an
1141          * offlining.  The RCU lock ensures the object won't be
1142          * released, tryget will fail if we lost the race.
1143          */
1144         *sequence = atomic_read(&root->dead_count);
1145         if (iter->last_dead_count == *sequence) {
1146                 smp_rmb();
1147                 position = iter->last_visited;
1148 
1149                 /*
1150                  * We cannot take a reference to root because we might race
1151                  * with root removal and returning NULL would end up in
1152                  * an endless loop on the iterator user level when root
1153                  * would be returned all the time.
1154                  */
1155                 if (position && position != root &&
1156                                 !css_tryget(&position->css))
1157                         position = NULL;
1158         }
1159         return position;
1160 }
1161 
1162 static void mem_cgroup_iter_update(struct mem_cgroup_reclaim_iter *iter,
1163                                    struct mem_cgroup *last_visited,
1164                                    struct mem_cgroup *new_position,
1165                                    struct mem_cgroup *root,
1166                                    int sequence)
1167 {
1168         /* root reference counting symmetric to mem_cgroup_iter_load */
1169         if (last_visited && last_visited != root)
1170                 css_put(&last_visited->css);
1171         /*
1172          * We store the sequence count from the time @last_visited was
1173          * loaded successfully instead of rereading it here so that we
1174          * don't lose destruction events in between.  We could have
1175          * raced with the destruction of @new_position after all.
1176          */
1177         iter->last_visited = new_position;
1178         smp_wmb();
1179         iter->last_dead_count = sequence;
1180 }
1181 
1182 /**
1183  * mem_cgroup_iter - iterate over memory cgroup hierarchy
1184  * @root: hierarchy root
1185  * @prev: previously returned memcg, NULL on first invocation
1186  * @reclaim: cookie for shared reclaim walks, NULL for full walks
1187  *
1188  * Returns references to children of the hierarchy below @root, or
1189  * @root itself, or %NULL after a full round-trip.
1190  *
1191  * Caller must pass the return value in @prev on subsequent
1192  * invocations for reference counting, or use mem_cgroup_iter_break()
1193  * to cancel a hierarchy walk before the round-trip is complete.
1194  *
1195  * Reclaimers can specify a zone and a priority level in @reclaim to
1196  * divide up the memcgs in the hierarchy among all concurrent
1197  * reclaimers operating on the same zone and priority.
1198  */
1199 struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
1200                                    struct mem_cgroup *prev,
1201                                    struct mem_cgroup_reclaim_cookie *reclaim)
1202 {
1203         struct mem_cgroup *memcg = NULL;
1204         struct mem_cgroup *last_visited = NULL;
1205 
1206         if (mem_cgroup_disabled())
1207                 return NULL;
1208 
1209         if (!root)
1210                 root = root_mem_cgroup;
1211 
1212         if (prev && !reclaim)
1213                 last_visited = prev;
1214 
1215         if (!root->use_hierarchy && root != root_mem_cgroup) {
1216                 if (prev)
1217                         goto out_css_put;
1218                 return root;
1219         }
1220 
1221         rcu_read_lock();
1222         while (!memcg) {
1223                 struct mem_cgroup_reclaim_iter *uninitialized_var(iter);
1224                 int uninitialized_var(seq);
1225 
1226                 if (reclaim) {
1227                         int nid = zone_to_nid(reclaim->zone);
1228                         int zid = zone_idx(reclaim->zone);
1229                         struct mem_cgroup_per_zone *mz;
1230 
1231                         mz = mem_cgroup_zoneinfo(root, nid, zid);
1232                         iter = &mz->reclaim_iter[reclaim->priority];
1233                         if (prev && reclaim->generation != iter->generation) {
1234                                 iter->last_visited = NULL;
1235                                 goto out_unlock;
1236                         }
1237 
1238                         last_visited = mem_cgroup_iter_load(iter, root, &seq);
1239                 }
1240 
1241                 memcg = __mem_cgroup_iter_next(root, last_visited);
1242 
1243                 if (reclaim) {
1244                         mem_cgroup_iter_update(iter, last_visited, memcg, root,
1245                                         seq);
1246 
1247                         if (!memcg)
1248                                 iter->generation++;
1249                         else if (!prev && memcg)
1250                                 reclaim->generation = iter->generation;
1251                 }
1252 
1253                 if (prev && !memcg)
1254                         goto out_unlock;
1255         }
1256 out_unlock:
1257         rcu_read_unlock();
1258 out_css_put:
1259         if (prev && prev != root)
1260                 css_put(&prev->css);
1261 
1262         return memcg;
1263 }
1264 
1265 /**
1266  * mem_cgroup_iter_break - abort a hierarchy walk prematurely
1267  * @root: hierarchy root
1268  * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
1269  */
1270 void mem_cgroup_iter_break(struct mem_cgroup *root,
1271                            struct mem_cgroup *prev)
1272 {
1273         if (!root)
1274                 root = root_mem_cgroup;
1275         if (prev && prev != root)
1276                 css_put(&prev->css);
1277 }
1278 
1279 /*
1280  * Iteration constructs for visiting all cgroups (under a tree).  If
1281  * loops are exited prematurely (break), mem_cgroup_iter_break() must
1282  * be used for reference counting.
1283  */
1284 #define for_each_mem_cgroup_tree(iter, root)            \
1285         for (iter = mem_cgroup_iter(root, NULL, NULL);  \
1286              iter != NULL;                              \
1287              iter = mem_cgroup_iter(root, iter, NULL))
1288 
1289 #define for_each_mem_cgroup(iter)                       \
1290         for (iter = mem_cgroup_iter(NULL, NULL, NULL);  \
1291              iter != NULL;                              \
1292              iter = mem_cgroup_iter(NULL, iter, NULL))
1293 
1294 void __mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx)
1295 {
1296         struct mem_cgroup *memcg;
1297 
1298         rcu_read_lock();
1299         memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
1300         if (unlikely(!memcg))
1301                 goto out;
1302 
1303         switch (idx) {
1304         case PGFAULT:
1305                 this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGFAULT]);
1306                 break;
1307         case PGMAJFAULT:
1308                 this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGMAJFAULT]);
1309                 break;
1310         default:
1311                 BUG();
1312         }
1313 out:
1314         rcu_read_unlock();
1315 }
1316 EXPORT_SYMBOL(__mem_cgroup_count_vm_event);
1317 
1318 /**
1319  * mem_cgroup_zone_lruvec - get the lru list vector for a zone and memcg
1320  * @zone: zone of the wanted lruvec
1321  * @memcg: memcg of the wanted lruvec
1322  *
1323  * Returns the lru list vector holding pages for the given @zone and
1324  * @mem.  This can be the global zone lruvec, if the memory controller
1325  * is disabled.
1326  */
1327 struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,
1328                                       struct mem_cgroup *memcg)
1329 {
1330         struct mem_cgroup_per_zone *mz;
1331         struct lruvec *lruvec;
1332 
1333         if (mem_cgroup_disabled()) {
1334                 lruvec = &zone->lruvec;
1335                 goto out;
1336         }
1337 
1338         mz = mem_cgroup_zoneinfo(memcg, zone_to_nid(zone), zone_idx(zone));
1339         lruvec = &mz->lruvec;
1340 out:
1341         /*
1342          * Since a node can be onlined after the mem_cgroup was created,
1343          * we have to be prepared to initialize lruvec->zone here;
1344          * and if offlined then reonlined, we need to reinitialize it.
1345          */
1346         if (unlikely(lruvec->zone != zone))
1347                 lruvec->zone = zone;
1348         return lruvec;
1349 }
1350 
1351 /*
1352  * Following LRU functions are allowed to be used without PCG_LOCK.
1353  * Operations are called by routine of global LRU independently from memcg.
1354  * What we have to take care of here is validness of pc->mem_cgroup.
1355  *
1356  * Changes to pc->mem_cgroup happens when
1357  * 1. charge
1358  * 2. moving account
1359  * In typical case, "charge" is done before add-to-lru. Exception is SwapCache.
1360  * It is added to LRU before charge.
1361  * If PCG_USED bit is not set, page_cgroup is not added to this private LRU.
1362  * When moving account, the page is not on LRU. It's isolated.
1363  */
1364 
1365 /**
1366  * mem_cgroup_page_lruvec - return lruvec for adding an lru page
1367  * @page: the page
1368  * @zone: zone of the page
1369  */
1370 struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct zone *zone)
1371 {
1372         struct mem_cgroup_per_zone *mz;
1373         struct mem_cgroup *memcg;
1374         struct page_cgroup *pc;
1375         struct lruvec *lruvec;
1376 
1377         if (mem_cgroup_disabled()) {
1378                 lruvec = &zone->lruvec;
1379                 goto out;
1380         }
1381 
1382         pc = lookup_page_cgroup(page);
1383         memcg = pc->mem_cgroup;
1384 
1385         /*
1386          * Surreptitiously switch any uncharged offlist page to root:
1387          * an uncharged page off lru does nothing to secure
1388          * its former mem_cgroup from sudden removal.
1389          *
1390          * Our caller holds lru_lock, and PageCgroupUsed is updated
1391          * under page_cgroup lock: between them, they make all uses
1392          * of pc->mem_cgroup safe.
1393          */
1394         if (!PageLRU(page) && !PageCgroupUsed(pc) && memcg != root_mem_cgroup)
1395                 pc->mem_cgroup = memcg = root_mem_cgroup;
1396 
1397         mz = page_cgroup_zoneinfo(memcg, page);
1398         lruvec = &mz->lruvec;
1399 out:
1400         /*
1401          * Since a node can be onlined after the mem_cgroup was created,
1402          * we have to be prepared to initialize lruvec->zone here;
1403          * and if offlined then reonlined, we need to reinitialize it.
1404          */
1405         if (unlikely(lruvec->zone != zone))
1406                 lruvec->zone = zone;
1407         return lruvec;
1408 }
1409 
1410 /**
1411  * mem_cgroup_update_lru_size - account for adding or removing an lru page
1412  * @lruvec: mem_cgroup per zone lru vector
1413  * @lru: index of lru list the page is sitting on
1414  * @nr_pages: positive when adding or negative when removing
1415  *
1416  * This function must be called when a page is added to or removed from an
1417  * lru list.
1418  */
1419 void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
1420                                 int nr_pages)
1421 {
1422         struct mem_cgroup_per_zone *mz;
1423         unsigned long *lru_size;
1424 
1425         if (mem_cgroup_disabled())
1426                 return;
1427 
1428         mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
1429         lru_size = mz->lru_size + lru;
1430         *lru_size += nr_pages;
1431         VM_BUG_ON((long)(*lru_size) < 0);
1432 }
1433 
1434 /*
1435  * Checks whether given mem is same or in the root_mem_cgroup's
1436  * hierarchy subtree
1437  */
1438 bool __mem_cgroup_same_or_subtree(const struct mem_cgroup *root_memcg,
1439                                   struct mem_cgroup *memcg)
1440 {
1441         if (root_memcg == memcg)
1442                 return true;
1443         if (!root_memcg->use_hierarchy || !memcg)
1444                 return false;
1445         return css_is_ancestor(&memcg->css, &root_memcg->css);
1446 }
1447 
1448 static bool mem_cgroup_same_or_subtree(const struct mem_cgroup *root_memcg,
1449                                        struct mem_cgroup *memcg)
1450 {
1451         bool ret;
1452 
1453         rcu_read_lock();
1454         ret = __mem_cgroup_same_or_subtree(root_memcg, memcg);
1455         rcu_read_unlock();
1456         return ret;
1457 }
1458 
1459 bool task_in_mem_cgroup(struct task_struct *task,
1460                         const struct mem_cgroup *memcg)
1461 {
1462         struct mem_cgroup *curr = NULL;
1463         struct task_struct *p;
1464         bool ret;
1465 
1466         p = find_lock_task_mm(task);
1467         if (p) {
1468                 curr = try_get_mem_cgroup_from_mm(p->mm);
1469                 task_unlock(p);
1470         } else {
1471                 /*
1472                  * All threads may have already detached their mm's, but the oom
1473                  * killer still needs to detect if they have already been oom
1474                  * killed to prevent needlessly killing additional tasks.
1475                  */
1476                 rcu_read_lock();
1477                 curr = mem_cgroup_from_task(task);
1478                 if (curr)
1479                         css_get(&curr->css);
1480                 rcu_read_unlock();
1481         }
1482         if (!curr)
1483                 return false;
1484         /*
1485          * We should check use_hierarchy of "memcg" not "curr". Because checking
1486          * use_hierarchy of "curr" here make this function true if hierarchy is
1487          * enabled in "curr" and "curr" is a child of "memcg" in *cgroup*
1488          * hierarchy(even if use_hierarchy is disabled in "memcg").
1489          */
1490         ret = mem_cgroup_same_or_subtree(memcg, curr);
1491         css_put(&curr->css);
1492         return ret;
1493 }
1494 
1495 int mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec)
1496 {
1497         unsigned long inactive_ratio;
1498         unsigned long inactive;
1499         unsigned long active;
1500         unsigned long gb;
1501 
1502         inactive = mem_cgroup_get_lru_size(lruvec, LRU_INACTIVE_ANON);
1503         active = mem_cgroup_get_lru_size(lruvec, LRU_ACTIVE_ANON);
1504 
1505         gb = (inactive + active) >> (30 - PAGE_SHIFT);
1506         if (gb)
1507                 inactive_ratio = int_sqrt(10 * gb);
1508         else
1509                 inactive_ratio = 1;
1510 
1511         return inactive * inactive_ratio < active;
1512 }
1513 
1514 #define mem_cgroup_from_res_counter(counter, member)    \
1515         container_of(counter, struct mem_cgroup, member)
1516 
1517 /**
1518  * mem_cgroup_margin - calculate chargeable space of a memory cgroup
1519  * @memcg: the memory cgroup
1520  *
1521  * Returns the maximum amount of memory @mem can be charged with, in
1522  * pages.
1523  */
1524 static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
1525 {
1526         unsigned long long margin;
1527 
1528         margin = res_counter_margin(&memcg->res);
1529         if (do_swap_account)
1530                 margin = min(margin, res_counter_margin(&memcg->memsw));
1531         return margin >> PAGE_SHIFT;
1532 }
1533 
1534 int mem_cgroup_swappiness(struct mem_cgroup *memcg)
1535 {
1536         /* root ? */
1537         if (!css_parent(&memcg->css))
1538                 return vm_swappiness;
1539 
1540         return memcg->swappiness;
1541 }
1542 
1543 /*
1544  * memcg->moving_account is used for checking possibility that some thread is
1545  * calling move_account(). When a thread on CPU-A starts moving pages under
1546  * a memcg, other threads should check memcg->moving_account under
1547  * rcu_read_lock(), like this:
1548  *
1549  *         CPU-A                                    CPU-B
1550  *                                              rcu_read_lock()
1551  *         memcg->moving_account+1              if (memcg->mocing_account)
1552  *                                                   take heavy locks.
1553  *         synchronize_rcu()                    update something.
1554  *                                              rcu_read_unlock()
1555  *         start move here.
1556  */
1557 
1558 /* for quick checking without looking up memcg */
1559 atomic_t memcg_moving __read_mostly;
1560 
1561 static void mem_cgroup_start_move(struct mem_cgroup *memcg)
1562 {
1563         atomic_inc(&memcg_moving);
1564         atomic_inc(&memcg->moving_account);
1565         synchronize_rcu();
1566 }
1567 
1568 static void mem_cgroup_end_move(struct mem_cgroup *memcg)
1569 {
1570         /*
1571          * Now, mem_cgroup_clear_mc() may call this function with NULL.
1572          * We check NULL in callee rather than caller.
1573          */
1574         if (memcg) {
1575                 atomic_dec(&memcg_moving);
1576                 atomic_dec(&memcg->moving_account);
1577         }
1578 }
1579 
1580 /*
1581  * 2 routines for checking "mem" is under move_account() or not.
1582  *
1583  * mem_cgroup_stolen() -  checking whether a cgroup is mc.from or not. This
1584  *                        is used for avoiding races in accounting.  If true,
1585  *                        pc->mem_cgroup may be overwritten.
1586  *
1587  * mem_cgroup_under_move() - checking a cgroup is mc.from or mc.to or
1588  *                        under hierarchy of moving cgroups. This is for
1589  *                        waiting at hith-memory prressure caused by "move".
1590  */
1591 
1592 static bool mem_cgroup_stolen(struct mem_cgroup *memcg)
1593 {
1594         VM_BUG_ON(!rcu_read_lock_held());
1595         return atomic_read(&memcg->moving_account) > 0;
1596 }
1597 
1598 static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
1599 {
1600         struct mem_cgroup *from;
1601         struct mem_cgroup *to;
1602         bool ret = false;
1603         /*
1604          * Unlike task_move routines, we access mc.to, mc.from not under
1605          * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
1606          */
1607         spin_lock(&mc.lock);
1608         from = mc.from;
1609         to = mc.to;
1610         if (!from)
1611                 goto unlock;
1612 
1613         ret = mem_cgroup_same_or_subtree(memcg, from)
1614                 || mem_cgroup_same_or_subtree(memcg, to);
1615 unlock:
1616         spin_unlock(&mc.lock);
1617         return ret;
1618 }
1619 
1620 static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
1621 {
1622         if (mc.moving_task && current != mc.moving_task) {
1623                 if (mem_cgroup_under_move(memcg)) {
1624                         DEFINE_WAIT(wait);
1625                         prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
1626                         /* moving charge context might have finished. */
1627                         if (mc.moving_task)
1628                                 schedule();
1629                         finish_wait(&mc.waitq, &wait);
1630                         return true;
1631                 }
1632         }
1633         return false;
1634 }
1635 
1636 /*
1637  * Take this lock when
1638  * - a code tries to modify page's memcg while it's USED.
1639  * - a code tries to modify page state accounting in a memcg.
1640  * see mem_cgroup_stolen(), too.
1641  */
1642 static void move_lock_mem_cgroup(struct mem_cgroup *memcg,
1643                                   unsigned long *flags)
1644 {
1645         spin_lock_irqsave(&memcg->move_lock, *flags);
1646 }
1647 
1648 static void move_unlock_mem_cgroup(struct mem_cgroup *memcg,
1649                                 unsigned long *flags)
1650 {
1651         spin_unlock_irqrestore(&memcg->move_lock, *flags);
1652 }
1653 
1654 #define K(x) ((x) << (PAGE_SHIFT-10))
1655 /**
1656  * mem_cgroup_print_oom_info: Print OOM information relevant to memory controller.
1657  * @memcg: The memory cgroup that went over limit
1658  * @p: Task that is going to be killed
1659  *
1660  * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1661  * enabled
1662  */
1663 void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
1664 {
1665         struct cgroup *task_cgrp;
1666         struct cgroup *mem_cgrp;
1667         /*
1668          * Need a buffer in BSS, can't rely on allocations. The code relies
1669          * on the assumption that OOM is serialized for memory controller.
1670          * If this assumption is broken, revisit this code.
1671          */
1672         static char memcg_name[PATH_MAX];
1673         int ret;
1674         struct mem_cgroup *iter;
1675         unsigned int i;
1676 
1677         if (!p)
1678                 return;
1679 
1680         rcu_read_lock();
1681 
1682         mem_cgrp = memcg->css.cgroup;
1683         task_cgrp = task_cgroup(p, mem_cgroup_subsys_id);
1684 
1685         ret = cgroup_path(task_cgrp, memcg_name, PATH_MAX);
1686         if (ret < 0) {
1687                 /*
1688                  * Unfortunately, we are unable to convert to a useful name
1689                  * But we'll still print out the usage information
1690                  */
1691                 rcu_read_unlock();
1692                 goto done;
1693         }
1694         rcu_read_unlock();
1695 
1696         pr_info("Task in %s killed", memcg_name);
1697 
1698         rcu_read_lock();
1699         ret = cgroup_path(mem_cgrp, memcg_name, PATH_MAX);
1700         if (ret < 0) {
1701                 rcu_read_unlock();
1702                 goto done;
1703         }
1704         rcu_read_unlock();
1705 
1706         /*
1707          * Continues from above, so we don't need an KERN_ level
1708          */
1709         pr_cont(" as a result of limit of %s\n", memcg_name);
1710 done:
1711 
1712         pr_info("memory: usage %llukB, limit %llukB, failcnt %llu\n",
1713                 res_counter_read_u64(&memcg->res, RES_USAGE) >> 10,
1714                 res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10,
1715                 res_counter_read_u64(&memcg->res, RES_FAILCNT));
1716         pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %llu\n",
1717                 res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10,
1718                 res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10,
1719                 res_counter_read_u64(&memcg->memsw, RES_FAILCNT));
1720         pr_info("kmem: usage %llukB, limit %llukB, failcnt %llu\n",
1721                 res_counter_read_u64(&memcg->kmem, RES_USAGE) >> 10,
1722                 res_counter_read_u64(&memcg->kmem, RES_LIMIT) >> 10,
1723                 res_counter_read_u64(&memcg->kmem, RES_FAILCNT));
1724 
1725         for_each_mem_cgroup_tree(iter, memcg) {
1726                 pr_info("Memory cgroup stats");
1727 
1728                 rcu_read_lock();
1729                 ret = cgroup_path(iter->css.cgroup, memcg_name, PATH_MAX);
1730                 if (!ret)
1731                         pr_cont(" for %s", memcg_name);
1732                 rcu_read_unlock();
1733                 pr_cont(":");
1734 
1735                 for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
1736                         if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
1737                                 continue;
1738                         pr_cont(" %s:%ldKB", mem_cgroup_stat_names[i],
1739                                 K(mem_cgroup_read_stat(iter, i)));
1740                 }
1741 
1742                 for (i = 0; i < NR_LRU_LISTS; i++)
1743                         pr_cont(" %s:%luKB", mem_cgroup_lru_names[i],
1744                                 K(mem_cgroup_nr_lru_pages(iter, BIT(i))));
1745 
1746                 pr_cont("\n");
1747         }
1748 }
1749 
1750 /*
1751  * This function returns the number of memcg under hierarchy tree. Returns
1752  * 1(self count) if no children.
1753  */
1754 static int mem_cgroup_count_children(struct mem_cgroup *memcg)
1755 {
1756         int num = 0;
1757         struct mem_cgroup *iter;
1758 
1759         for_each_mem_cgroup_tree(iter, memcg)
1760                 num++;
1761         return num;
1762 }
1763 
1764 /*
1765  * Return the memory (and swap, if configured) limit for a memcg.
1766  */
1767 static u64 mem_cgroup_get_limit(struct mem_cgroup *memcg)
1768 {
1769         u64 limit;
1770 
1771         limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
1772 
1773         /*
1774          * Do not consider swap space if we cannot swap due to swappiness
1775          */
1776         if (mem_cgroup_swappiness(memcg)) {
1777                 u64 memsw;
1778 
1779                 limit += total_swap_pages << PAGE_SHIFT;
1780                 memsw = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
1781 
1782                 /*
1783                  * If memsw is finite and limits the amount of swap space
1784                  * available to this memcg, return that limit.
1785                  */
1786                 limit = min(limit, memsw);
1787         }
1788 
1789         return limit;
1790 }
1791 
1792 static void mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
1793                                      int order)
1794 {
1795         struct mem_cgroup *iter;
1796         unsigned long chosen_points = 0;
1797         unsigned long totalpages;
1798         unsigned int points = 0;
1799         struct task_struct *chosen = NULL;
1800 
1801         /*
1802          * If current has a pending SIGKILL or is exiting, then automatically
1803          * select it.  The goal is to allow it to allocate so that it may
1804          * quickly exit and free its memory.
1805          */
1806         if (fatal_signal_pending(current) || current->flags & PF_EXITING) {
1807                 set_thread_flag(TIF_MEMDIE);
1808                 return;
1809         }
1810 
1811         check_panic_on_oom(CONSTRAINT_MEMCG, gfp_mask, order, NULL);
1812         totalpages = mem_cgroup_get_limit(memcg) >> PAGE_SHIFT ? : 1;
1813         for_each_mem_cgroup_tree(iter, memcg) {
1814                 struct css_task_iter it;
1815                 struct task_struct *task;
1816 
1817                 css_task_iter_start(&iter->css, &it);
1818                 while ((task = css_task_iter_next(&it))) {
1819                         switch (oom_scan_process_thread(task, totalpages, NULL,
1820                                                         false)) {
1821                         case OOM_SCAN_SELECT:
1822                                 if (chosen)
1823                                         put_task_struct(chosen);
1824                                 chosen = task;
1825                                 chosen_points = ULONG_MAX;
1826                                 get_task_struct(chosen);
1827                                 /* fall through */
1828                         case OOM_SCAN_CONTINUE:
1829                                 continue;
1830                         case OOM_SCAN_ABORT:
1831                                 css_task_iter_end(&it);
1832                                 mem_cgroup_iter_break(memcg, iter);
1833                                 if (chosen)
1834                                         put_task_struct(chosen);
1835                                 return;
1836                         case OOM_SCAN_OK:
1837                                 break;
1838                         };
1839                         points = oom_badness(task, memcg, NULL, totalpages);
1840                         if (!points || points < chosen_points)
1841                                 continue;
1842                         /* Prefer thread group leaders for display purposes */
1843                         if (points == chosen_points &&
1844                             thread_group_leader(chosen))
1845                                 continue;
1846 
1847                         if (chosen)
1848                                 put_task_struct(chosen);
1849                         chosen = task;
1850                         chosen_points = points;
1851                         get_task_struct(chosen);
1852                 }
1853                 css_task_iter_end(&it);
1854         }
1855 
1856         if (!chosen)
1857                 return;
1858         points = chosen_points * 1000 / totalpages;
1859         oom_kill_process(chosen, gfp_mask, order, points, totalpages, memcg,
1860                          NULL, "Memory cgroup out of memory");
1861 }
1862 
1863 static unsigned long mem_cgroup_reclaim(struct mem_cgroup *memcg,
1864                                         gfp_t gfp_mask,
1865                                         unsigned long flags)
1866 {
1867         unsigned long total = 0;
1868         bool noswap = false;
1869         int loop;
1870 
1871         if (flags & MEM_CGROUP_RECLAIM_NOSWAP)
1872                 noswap = true;
1873         if (!(flags & MEM_CGROUP_RECLAIM_SHRINK) && memcg->memsw_is_minimum)
1874                 noswap = true;
1875 
1876         for (loop = 0; loop < MEM_CGROUP_MAX_RECLAIM_LOOPS; loop++) {
1877                 if (loop)
1878                         drain_all_stock_async(memcg);
1879                 total += try_to_free_mem_cgroup_pages(memcg, gfp_mask, noswap);
1880                 /*
1881                  * Allow limit shrinkers, which are triggered directly
1882                  * by userspace, to catch signals and stop reclaim
1883                  * after minimal progress, regardless of the margin.
1884                  */
1885                 if (total && (flags & MEM_CGROUP_RECLAIM_SHRINK))
1886                         break;
1887                 if (mem_cgroup_margin(memcg))
1888                         break;
1889                 /*
1890                  * If nothing was reclaimed after two attempts, there
1891                  * may be no reclaimable pages in this hierarchy.
1892                  */
1893                 if (loop && !total)
1894                         break;
1895         }
1896         return total;
1897 }
1898 
1899 /**
1900  * test_mem_cgroup_node_reclaimable
1901  * @memcg: the target memcg
1902  * @nid: the node ID to be checked.
1903  * @noswap : specify true here if the user wants flle only information.
1904  *
1905  * This function returns whether the specified memcg contains any
1906  * reclaimable pages on a node. Returns true if there are any reclaimable
1907  * pages in the node.
1908  */
1909 static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
1910                 int nid, bool noswap)
1911 {
1912         if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE))
1913                 return true;
1914         if (noswap || !total_swap_pages)
1915                 return false;
1916         if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON))
1917                 return true;
1918         return false;
1919 
1920 }
1921 #if MAX_NUMNODES > 1
1922 
1923 /*
1924  * Always updating the nodemask is not very good - even if we have an empty
1925  * list or the wrong list here, we can start from some node and traverse all
1926  * nodes based on the zonelist. So update the list loosely once per 10 secs.
1927  *
1928  */
1929 static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
1930 {
1931         int nid;
1932         /*
1933          * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
1934          * pagein/pageout changes since the last update.
1935          */
1936         if (!atomic_read(&memcg->numainfo_events))
1937                 return;
1938         if (atomic_inc_return(&memcg->numainfo_updating) > 1)
1939                 return;
1940 
1941         /* make a nodemask where this memcg uses memory from */
1942         memcg->scan_nodes = node_states[N_MEMORY];
1943 
1944         for_each_node_mask(nid, node_states[N_MEMORY]) {
1945 
1946                 if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
1947                         node_clear(nid, memcg->scan_nodes);
1948         }
1949 
1950         atomic_set(&memcg->numainfo_events, 0);
1951         atomic_set(&memcg->numainfo_updating, 0);
1952 }
1953 
1954 /*
1955  * Selecting a node where we start reclaim from. Because what we need is just
1956  * reducing usage counter, start from anywhere is O,K. Considering
1957  * memory reclaim from current node, there are pros. and cons.
1958  *
1959  * Freeing memory from current node means freeing memory from a node which
1960  * we'll use or we've used. So, it may make LRU bad. And if several threads
1961  * hit limits, it will see a contention on a node. But freeing from remote
1962  * node means more costs for memory reclaim because of memory latency.
1963  *
1964  * Now, we use round-robin. Better algorithm is welcomed.
1965  */
1966 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1967 {
1968         int node;
1969 
1970         mem_cgroup_may_update_nodemask(memcg);
1971         node = memcg->last_scanned_node;
1972 
1973         node = next_node(node, memcg->scan_nodes);
1974         if (node == MAX_NUMNODES)
1975                 node = first_node(memcg->scan_nodes);
1976         /*
1977          * We call this when we hit limit, not when pages are added to LRU.
1978          * No LRU may hold pages because all pages are UNEVICTABLE or
1979          * memcg is too small and all pages are not on LRU. In that case,
1980          * we use curret node.
1981          */
1982         if (unlikely(node == MAX_NUMNODES))
1983                 node = numa_node_id();
1984 
1985         memcg->last_scanned_node = node;
1986         return node;
1987 }
1988 
1989 /*
1990  * Check all nodes whether it contains reclaimable pages or not.
1991  * For quick scan, we make use of scan_nodes. This will allow us to skip
1992  * unused nodes. But scan_nodes is lazily updated and may not cotain
1993  * enough new information. We need to do double check.
1994  */
1995 static bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap)
1996 {
1997         int nid;
1998 
1999         /*
2000          * quick check...making use of scan_node.
2001          * We can skip unused nodes.
2002          */
2003         if (!nodes_empty(memcg->scan_nodes)) {
2004                 for (nid = first_node(memcg->scan_nodes);
2005                      nid < MAX_NUMNODES;
2006                      nid = next_node(nid, memcg->scan_nodes)) {
2007 
2008                         if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap))
2009                                 return true;
2010                 }
2011         }
2012         /*
2013          * Check rest of nodes.
2014          */
2015         for_each_node_state(nid, N_MEMORY) {
2016                 if (node_isset(nid, memcg->scan_nodes))
2017                         continue;
2018                 if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap))
2019                         return true;
2020         }
2021         return false;
2022 }
2023 
2024 #else
2025 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
2026 {
2027         return 0;
2028 }
2029 
2030 static bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap)
2031 {
2032         return test_mem_cgroup_node_reclaimable(memcg, 0, noswap);
2033 }
2034 #endif
2035 
2036 static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
2037                                    struct zone *zone,
2038                                    gfp_t gfp_mask,
2039                                    unsigned long *total_scanned)
2040 {
2041         struct mem_cgroup *victim = NULL;
2042         int total = 0;
2043         int loop = 0;
2044         unsigned long excess;
2045         unsigned long nr_scanned;
2046         struct mem_cgroup_reclaim_cookie reclaim = {
2047                 .zone = zone,
2048                 .priority = 0,
2049         };
2050 
2051         excess = res_counter_soft_limit_excess(&root_memcg->res) >> PAGE_SHIFT;
2052 
2053         while (1) {
2054                 victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
2055                 if (!victim) {
2056                         loop++;
2057                         if (loop >= 2) {
2058                                 /*
2059                                  * If we have not been able to reclaim
2060                                  * anything, it might because there are
2061                                  * no reclaimable pages under this hierarchy
2062                                  */
2063                                 if (!total)
2064                                         break;
2065                                 /*
2066                                  * We want to do more targeted reclaim.
2067                                  * excess >> 2 is not to excessive so as to
2068                                  * reclaim too much, nor too less that we keep
2069                                  * coming back to reclaim from this cgroup
2070                                  */
2071                                 if (total >= (excess >> 2) ||
2072                                         (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
2073                                         break;
2074                         }
2075                         continue;
2076                 }
2077                 if (!mem_cgroup_reclaimable(victim, false))
2078                         continue;
2079                 total += mem_cgroup_shrink_node_zone(victim, gfp_mask, false,
2080                                                      zone, &nr_scanned);
2081                 *total_scanned += nr_scanned;
2082                 if (!res_counter_soft_limit_excess(&root_memcg->res))
2083                         break;
2084         }
2085         mem_cgroup_iter_break(root_memcg, victim);
2086         return total;
2087 }
2088 
2089 #ifdef CONFIG_LOCKDEP
2090 static struct lockdep_map memcg_oom_lock_dep_map = {
2091         .name = "memcg_oom_lock",
2092 };
2093 #endif
2094 
2095 static DEFINE_SPINLOCK(memcg_oom_lock);
2096 
2097 /*
2098  * Check OOM-Killer is already running under our hierarchy.
2099  * If someone is running, return false.
2100  */
2101 static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
2102 {
2103         struct mem_cgroup *iter, *failed = NULL;
2104 
2105         spin_lock(&memcg_oom_lock);
2106 
2107         for_each_mem_cgroup_tree(iter, memcg) {
2108                 if (iter->oom_lock) {
2109                         /*
2110                          * this subtree of our hierarchy is already locked
2111                          * so we cannot give a lock.
2112                          */
2113                         failed = iter;
2114                         mem_cgroup_iter_break(memcg, iter);
2115                         break;
2116                 } else
2117                         iter->oom_lock = true;
2118         }
2119 
2120         if (failed) {
2121                 /*
2122                  * OK, we failed to lock the whole subtree so we have
2123                  * to clean up what we set up to the failing subtree
2124                  */
2125                 for_each_mem_cgroup_tree(iter, memcg) {
2126                         if (iter == failed) {
2127                                 mem_cgroup_iter_break(memcg, iter);
2128                                 break;
2129                         }
2130                         iter->oom_lock = false;
2131                 }
2132         } else
2133                 mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_);
2134 
2135         spin_unlock(&memcg_oom_lock);
2136 
2137         return !failed;
2138 }
2139 
2140 static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
2141 {
2142         struct mem_cgroup *iter;
2143 
2144         spin_lock(&memcg_oom_lock);
2145         mutex_release(&memcg_oom_lock_dep_map, 1, _RET_IP_);
2146         for_each_mem_cgroup_tree(iter, memcg)
2147                 iter->oom_lock = false;
2148         spin_unlock(&memcg_oom_lock);
2149 }
2150 
2151 static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
2152 {
2153         struct mem_cgroup *iter;
2154 
2155         for_each_mem_cgroup_tree(iter, memcg)
2156                 atomic_inc(&iter->under_oom);
2157 }
2158 
2159 static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
2160 {
2161         struct mem_cgroup *iter;
2162 
2163         /*
2164          * When a new child is created while the hierarchy is under oom,
2165          * mem_cgroup_oom_lock() may not be called. We have to use
2166          * atomic_add_unless() here.
2167          */
2168         for_each_mem_cgroup_tree(iter, memcg)
2169                 atomic_add_unless(&iter->under_oom, -1, 0);
2170 }
2171 
2172 static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
2173 
2174 struct oom_wait_info {
2175         struct mem_cgroup *memcg;
2176         wait_queue_t    wait;
2177 };
2178 
2179 static int memcg_oom_wake_function(wait_queue_t *wait,
2180         unsigned mode, int sync, void *arg)
2181 {
2182         struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
2183         struct mem_cgroup *oom_wait_memcg;
2184         struct oom_wait_info *oom_wait_info;
2185 
2186         oom_wait_info = container_of(wait, struct oom_wait_info, wait);
2187         oom_wait_memcg = oom_wait_info->memcg;
2188 
2189         /*
2190          * Both of oom_wait_info->memcg and wake_memcg are stable under us.
2191          * Then we can use css_is_ancestor without taking care of RCU.
2192          */
2193         if (!mem_cgroup_same_or_subtree(oom_wait_memcg, wake_memcg)
2194                 && !mem_cgroup_same_or_subtree(wake_memcg, oom_wait_memcg))
2195                 return 0;
2196         return autoremove_wake_function(wait, mode, sync, arg);
2197 }
2198 
2199 static void memcg_wakeup_oom(struct mem_cgroup *memcg)
2200 {
2201         atomic_inc(&memcg->oom_wakeups);
2202         /* for filtering, pass "memcg" as argument. */
2203         __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
2204 }
2205 
2206 static void memcg_oom_recover(struct mem_cgroup *memcg)
2207 {
2208         if (memcg && atomic_read(&memcg->under_oom))
2209                 memcg_wakeup_oom(memcg);
2210 }
2211 
2212 static void mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
2213 {
2214         if (!current->memcg_oom.may_oom)
2215                 return;
2216         /*
2217          * We are in the middle of the charge context here, so we
2218          * don't want to block when potentially sitting on a callstack
2219          * that holds all kinds of filesystem and mm locks.
2220          *
2221          * Also, the caller may handle a failed allocation gracefully
2222          * (like optional page cache readahead) and so an OOM killer
2223          * invocation might not even be necessary.
2224          *
2225          * That's why we don't do anything here except remember the
2226          * OOM context and then deal with it at the end of the page
2227          * fault when the stack is unwound, the locks are released,
2228          * and when we know whether the fault was overall successful.
2229          */
2230         css_get(&memcg->css);
2231         current->memcg_oom.memcg = memcg;
2232         current->memcg_oom.gfp_mask = mask;
2233         current->memcg_oom.order = order;
2234 }
2235 
2236 /**
2237  * mem_cgroup_oom_synchronize - complete memcg OOM handling
2238  * @handle: actually kill/wait or just clean up the OOM state
2239  *
2240  * This has to be called at the end of a page fault if the memcg OOM
2241  * handler was enabled.
2242  *
2243  * Memcg supports userspace OOM handling where failed allocations must
2244  * sleep on a waitqueue until the userspace task resolves the
2245  * situation.  Sleeping directly in the charge context with all kinds
2246  * of locks held is not a good idea, instead we remember an OOM state
2247  * in the task and mem_cgroup_oom_synchronize() has to be called at
2248  * the end of the page fault to complete the OOM handling.
2249  *
2250  * Returns %true if an ongoing memcg OOM situation was detected and
2251  * completed, %false otherwise.
2252  */
2253 bool mem_cgroup_oom_synchronize(bool handle)
2254 {
2255         struct mem_cgroup *memcg = current->memcg_oom.memcg;
2256         struct oom_wait_info owait;
2257         bool locked;
2258 
2259         /* OOM is global, do not handle */
2260         if (!memcg)
2261                 return false;
2262 
2263         if (!handle)
2264                 goto cleanup;
2265 
2266         owait.memcg = memcg;
2267         owait.wait.flags = 0;
2268         owait.wait.func = memcg_oom_wake_function;
2269         owait.wait.private = current;
2270         INIT_LIST_HEAD(&owait.wait.task_list);
2271 
2272         prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
2273         mem_cgroup_mark_under_oom(memcg);
2274 
2275         locked = mem_cgroup_oom_trylock(memcg);
2276 
2277         if (locked)
2278                 mem_cgroup_oom_notify(memcg);
2279 
2280         if (locked && !memcg->oom_kill_disable) {
2281                 mem_cgroup_unmark_under_oom(memcg);
2282                 finish_wait(&memcg_oom_waitq, &owait.wait);
2283                 mem_cgroup_out_of_memory(memcg, current->memcg_oom.gfp_mask,
2284                                          current->memcg_oom.order);
2285         } else {
2286                 schedule();
2287                 mem_cgroup_unmark_under_oom(memcg);
2288                 finish_wait(&memcg_oom_waitq, &owait.wait);
2289         }
2290 
2291         if (locked) {
2292                 mem_cgroup_oom_unlock(memcg);
2293                 /*
2294                  * There is no guarantee that an OOM-lock contender
2295                  * sees the wakeups triggered by the OOM kill
2296                  * uncharges.  Wake any sleepers explicitely.
2297                  */
2298                 memcg_oom_recover(memcg);
2299         }
2300 cleanup:
2301         current->memcg_oom.memcg = NULL;
2302         css_put(&memcg->css);
2303         return true;
2304 }
2305 
2306 /*
2307  * Currently used to update mapped file statistics, but the routine can be
2308  * generalized to update other statistics as well.
2309  *
2310  * Notes: Race condition
2311  *
2312  * We usually use page_cgroup_lock() for accessing page_cgroup member but
2313  * it tends to be costly. But considering some conditions, we doesn't need
2314  * to do so _always_.
2315  *
2316  * Considering "charge", lock_page_cgroup() is not required because all
2317  * file-stat operations happen after a page is attached to radix-tree. There
2318  * are no race with "charge".
2319  *
2320  * Considering "uncharge", we know that memcg doesn't clear pc->mem_cgroup
2321  * at "uncharge" intentionally. So, we always see valid pc->mem_cgroup even
2322  * if there are race with "uncharge". Statistics itself is properly handled
2323  * by flags.
2324  *
2325  * Considering "move", this is an only case we see a race. To make the race
2326  * small, we check mm->moving_account and detect there are possibility of race
2327  * If there is, we take a lock.
2328  */
2329 
2330 void __mem_cgroup_begin_update_page_stat(struct page *page,
2331                                 bool *locked, unsigned long *flags)
2332 {
2333         struct mem_cgroup *memcg;
2334         struct page_cgroup *pc;
2335 
2336         pc = lookup_page_cgroup(page);
2337 again:
2338         memcg = pc->mem_cgroup;
2339         if (unlikely(!memcg || !PageCgroupUsed(pc)))
2340                 return;
2341         /*
2342          * If this memory cgroup is not under account moving, we don't
2343          * need to take move_lock_mem_cgroup(). Because we already hold
2344          * rcu_read_lock(), any calls to move_account will be delayed until
2345          * rcu_read_unlock() if mem_cgroup_stolen() == true.
2346          */
2347         if (!mem_cgroup_stolen(memcg))
2348                 return;
2349 
2350         move_lock_mem_cgroup(memcg, flags);
2351         if (memcg != pc->mem_cgroup || !PageCgroupUsed(pc)) {
2352                 move_unlock_mem_cgroup(memcg, flags);
2353                 goto again;
2354         }
2355         *locked = true;
2356 }
2357 
2358 void __mem_cgroup_end_update_page_stat(struct page *page, unsigned long *flags)
2359 {
2360         struct page_cgroup *pc = lookup_page_cgroup(page);
2361 
2362         /*
2363          * It's guaranteed that pc->mem_cgroup never changes while
2364          * lock is held because a routine modifies pc->mem_cgroup
2365          * should take move_lock_mem_cgroup().
2366          */
2367         move_unlock_mem_cgroup(pc->mem_cgroup, flags);
2368 }
2369 
2370 void mem_cgroup_update_page_stat(struct page *page,
2371                                  enum mem_cgroup_stat_index idx, int val)
2372 {
2373         struct mem_cgroup *memcg;
2374         struct page_cgroup *pc = lookup_page_cgroup(page);
2375         unsigned long uninitialized_var(flags);
2376 
2377         if (mem_cgroup_disabled())
2378                 return;
2379 
2380         VM_BUG_ON(!rcu_read_lock_held());
2381         memcg = pc->mem_cgroup;
2382         if (unlikely(!memcg || !PageCgroupUsed(pc)))
2383                 return;
2384 
2385         this_cpu_add(memcg->stat->count[idx], val);
2386 }
2387 
2388 /*
2389  * size of first charge trial. "32" comes from vmscan.c's magic value.
2390  * TODO: maybe necessary to use big numbers in big irons.
2391  */
2392 #define CHARGE_BATCH    32U
2393 struct memcg_stock_pcp {
2394         struct mem_cgroup *cached; /* this never be root cgroup */
2395         unsigned int nr_pages;
2396         struct work_struct work;
2397         unsigned long flags;
2398 #define FLUSHING_CACHED_CHARGE  0
2399 };
2400 static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
2401 static DEFINE_MUTEX(percpu_charge_mutex);
2402 
2403 /**
2404  * consume_stock: Try to consume stocked charge on this cpu.
2405  * @memcg: memcg to consume from.
2406  * @nr_pages: how many pages to charge.
2407  *
2408  * The charges will only happen if @memcg matches the current cpu's memcg
2409  * stock, and at least @nr_pages are available in that stock.  Failure to
2410  * service an allocation will refill the stock.
2411  *
2412  * returns true if successful, false otherwise.
2413  */
2414 static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2415 {
2416         struct memcg_stock_pcp *stock;
2417         bool ret = true;
2418 
2419         if (nr_pages > CHARGE_BATCH)
2420                 return false;
2421 
2422         stock = &get_cpu_var(memcg_stock);
2423         if (memcg == stock->cached && stock->nr_pages >= nr_pages)
2424                 stock->nr_pages -= nr_pages;
2425         else /* need to call res_counter_charge */
2426                 ret = false;
2427         put_cpu_var(memcg_stock);
2428         return ret;
2429 }
2430 
2431 /*
2432  * Returns stocks cached in percpu to res_counter and reset cached information.
2433  */
2434 static void drain_stock(struct memcg_stock_pcp *stock)
2435 {
2436         struct mem_cgroup *old = stock->cached;
2437 
2438         if (stock->nr_pages) {
2439                 unsigned long bytes = stock->nr_pages * PAGE_SIZE;
2440 
2441                 res_counter_uncharge(&old->res, bytes);
2442                 if (do_swap_account)
2443                         res_counter_uncharge(&old->memsw, bytes);
2444                 stock->nr_pages = 0;
2445         }
2446         stock->cached = NULL;
2447 }
2448 
2449 /*
2450  * This must be called under preempt disabled or must be called by
2451  * a thread which is pinned to local cpu.
2452  */
2453 static void drain_local_stock(struct work_struct *dummy)
2454 {
2455         struct memcg_stock_pcp *stock = &__get_cpu_var(memcg_stock);
2456         drain_stock(stock);
2457         clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
2458 }
2459 
2460 static void __init memcg_stock_init(void)
2461 {
2462         int cpu;
2463 
2464         for_each_possible_cpu(cpu) {
2465                 struct memcg_stock_pcp *stock =
2466                                         &per_cpu(memcg_stock, cpu);
2467                 INIT_WORK(&stock->work, drain_local_stock);
2468         }
2469 }
2470 
2471 /*
2472  * Cache charges(val) which is from res_counter, to local per_cpu area.
2473  * This will be consumed by consume_stock() function, later.
2474  */
2475 static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2476 {
2477         struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);
2478 
2479         if (stock->cached != memcg) { /* reset if necessary */
2480                 drain_stock(stock);
2481                 stock->cached = memcg;
2482         }
2483         stock->nr_pages += nr_pages;
2484         put_cpu_var(memcg_stock);
2485 }
2486 
2487 /*
2488  * Drains all per-CPU charge caches for given root_memcg resp. subtree
2489  * of the hierarchy under it. sync flag says whether we should block
2490  * until the work is done.
2491  */
2492 static void drain_all_stock(struct mem_cgroup *root_memcg, bool sync)
2493 {
2494         int cpu, curcpu;
2495 
2496         /* Notify other cpus that system-wide "drain" is running */
2497         get_online_cpus();
2498         curcpu = get_cpu();
2499         for_each_online_cpu(cpu) {
2500                 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
2501                 struct mem_cgroup *memcg;
2502 
2503                 memcg = stock->cached;
2504                 if (!memcg || !stock->nr_pages)
2505                         continue;
2506                 if (!mem_cgroup_same_or_subtree(root_memcg, memcg))
2507                         continue;
2508                 if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
2509                         if (cpu == curcpu)
2510                                 drain_local_stock(&stock->work);
2511                         else
2512                                 schedule_work_on(cpu, &stock->work);
2513                 }
2514         }
2515         put_cpu();
2516 
2517         if (!sync)
2518                 goto out;
2519 
2520         for_each_online_cpu(cpu) {
2521                 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
2522                 if (test_bit(FLUSHING_CACHED_CHARGE, &stock->flags))
2523                         flush_work(&stock->work);
2524         }
2525 out:
2526         put_online_cpus();
2527 }
2528 
2529 /*
2530  * Tries to drain stocked charges in other cpus. This function is asynchronous
2531  * and just put a work per cpu for draining localy on each cpu. Caller can
2532  * expects some charges will be back to res_counter later but cannot wait for
2533  * it.
2534  */
2535 static void drain_all_stock_async(struct mem_cgroup *root_memcg)
2536 {
2537         /*
2538          * If someone calls draining, avoid adding more kworker runs.
2539          */
2540         if (!mutex_trylock(&percpu_charge_mutex))
2541                 return;
2542         drain_all_stock(root_memcg, false);
2543         mutex_unlock(&percpu_charge_mutex);
2544 }
2545 
2546 /* This is a synchronous drain interface. */
2547 static void drain_all_stock_sync(struct mem_cgroup *root_memcg)
2548 {
2549         /* called when force_empty is called */
2550         mutex_lock(&percpu_charge_mutex);
2551         drain_all_stock(root_memcg, true);
2552         mutex_unlock(&percpu_charge_mutex);
2553 }
2554 
2555 /*
2556  * This function drains percpu counter value from DEAD cpu and
2557  * move it to local cpu. Note that this function can be preempted.
2558  */
2559 static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *memcg, int cpu)
2560 {
2561         int i;
2562 
2563         spin_lock(&memcg->pcp_counter_lock);
2564         for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
2565                 long x = per_cpu(memcg->stat->count[i], cpu);
2566 
2567                 per_cpu(memcg->stat->count[i], cpu) = 0;
2568                 memcg->nocpu_base.count[i] += x;
2569         }
2570         for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
2571                 unsigned long x = per_cpu(memcg->stat->events[i], cpu);
2572 
2573                 per_cpu(memcg->stat->events[i], cpu) = 0;
2574                 memcg->nocpu_base.events[i] += x;
2575         }
2576         spin_unlock(&memcg->pcp_counter_lock);
2577 }
2578 
2579 static int memcg_cpu_hotplug_callback(struct notifier_block *nb,
2580                                         unsigned long action,
2581                                         void *hcpu)
2582 {
2583         int cpu = (unsigned long)hcpu;
2584         struct memcg_stock_pcp *stock;
2585         struct mem_cgroup *iter;
2586 
2587         if (action == CPU_ONLINE)
2588                 return NOTIFY_OK;
2589 
2590         if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
2591                 return NOTIFY_OK;
2592 
2593         for_each_mem_cgroup(iter)
2594                 mem_cgroup_drain_pcp_counter(iter, cpu);
2595 
2596         stock = &per_cpu(memcg_stock, cpu);
2597         drain_stock(stock);
2598         return NOTIFY_OK;
2599 }
2600 
2601 
2602 /* See __mem_cgroup_try_charge() for details */
2603 enum {
2604         CHARGE_OK,              /* success */
2605         CHARGE_RETRY,           /* need to retry but retry is not bad */
2606         CHARGE_NOMEM,           /* we can't do more. return -ENOMEM */
2607         CHARGE_WOULDBLOCK,      /* GFP_WAIT wasn't set and no enough res. */
2608 };
2609 
2610 static int mem_cgroup_do_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
2611                                 unsigned int nr_pages, unsigned int min_pages,
2612                                 bool invoke_oom)
2613 {
2614         unsigned long csize = nr_pages * PAGE_SIZE;
2615         struct mem_cgroup *mem_over_limit;
2616         struct res_counter *fail_res;
2617         unsigned long flags = 0;
2618         int ret;
2619 
2620         ret = res_counter_charge(&memcg->res, csize, &fail_res);
2621 
2622         if (likely(!ret)) {
2623                 if (!do_swap_account)
2624                         return CHARGE_OK;
2625                 ret = res_counter_charge(&memcg->memsw, csize, &fail_res);
2626                 if (likely(!ret))
2627                         return CHARGE_OK;
2628 
2629                 res_counter_uncharge(&memcg->res, csize);
2630                 mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw);
2631                 flags |= MEM_CGROUP_RECLAIM_NOSWAP;
2632         } else
2633                 mem_over_limit = mem_cgroup_from_res_counter(fail_res, res);
2634         /*
2635          * Never reclaim on behalf of optional batching, retry with a
2636          * single page instead.
2637          */
2638         if (nr_pages > min_pages)
2639                 return CHARGE_RETRY;
2640 
2641         if (!(gfp_mask & __GFP_WAIT))
2642                 return CHARGE_WOULDBLOCK;
2643 
2644         if (gfp_mask & __GFP_NORETRY)
2645                 return CHARGE_NOMEM;
2646 
2647         ret = mem_cgroup_reclaim(mem_over_limit, gfp_mask, flags);
2648         if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
2649                 return CHARGE_RETRY;
2650         /*
2651          * Even though the limit is exceeded at this point, reclaim
2652          * may have been able to free some pages.  Retry the charge
2653          * before killing the task.
2654          *
2655          * Only for regular pages, though: huge pages are rather
2656          * unlikely to succeed so close to the limit, and we fall back
2657          * to regular pages anyway in case of failure.
2658          */
2659         if (nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER) && ret)
2660                 return CHARGE_RETRY;
2661 
2662         /*
2663          * At task move, charge accounts can be doubly counted. So, it's
2664          * better to wait until the end of task_move if something is going on.
2665          */
2666         if (mem_cgroup_wait_acct_move(mem_over_limit))
2667                 return CHARGE_RETRY;
2668 
2669         if (invoke_oom)
2670                 mem_cgroup_oom(mem_over_limit, gfp_mask, get_order(csize));
2671 
2672         return CHARGE_NOMEM;
2673 }
2674 
2675 /*
2676  * __mem_cgroup_try_charge() does
2677  * 1. detect memcg to be charged against from passed *mm and *ptr,
2678  * 2. update res_counter
2679  * 3. call memory reclaim if necessary.
2680  *
2681  * In some special case, if the task is fatal, fatal_signal_pending() or
2682  * has TIF_MEMDIE, this function returns -EINTR while writing root_mem_cgroup
2683  * to *ptr. There are two reasons for this. 1: fatal threads should quit as soon
2684  * as possible without any hazards. 2: all pages should have a valid
2685  * pc->mem_cgroup. If mm is NULL and the caller doesn't pass a valid memcg
2686  * pointer, that is treated as a charge to root_mem_cgroup.
2687  *
2688  * So __mem_cgroup_try_charge() will return
2689  *  0       ...  on success, filling *ptr with a valid memcg pointer.
2690  *  -ENOMEM ...  charge failure because of resource limits.
2691  *  -EINTR  ...  if thread is fatal. *ptr is filled with root_mem_cgroup.
2692  *
2693  * Unlike the exported interface, an "oom" parameter is added. if oom==true,
2694  * the oom-killer can be invoked.
2695  */
2696 static int __mem_cgroup_try_charge(struct mm_struct *mm,
2697                                    gfp_t gfp_mask,
2698                                    unsigned int nr_pages,
2699                                    struct mem_cgroup **ptr,
2700                                    bool oom)
2701 {
2702         unsigned int batch = max(CHARGE_BATCH, nr_pages);
2703         int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
2704         struct mem_cgroup *memcg = NULL;
2705         int ret;
2706 
2707         /*
2708          * Unlike gloval-vm's OOM-kill, we're not in memory shortage
2709          * in system level. So, allow to go ahead dying process in addition to
2710          * MEMDIE process.
2711          */
2712         if (unlikely(test_thread_flag(TIF_MEMDIE)
2713                      || fatal_signal_pending(current)
2714                      || current->flags & PF_EXITING))
2715                 goto bypass;
2716 
2717         if (unlikely(task_in_memcg_oom(current)))
2718                 goto nomem;
2719 
2720         if (gfp_mask & __GFP_NOFAIL)
2721                 oom = false;
2722 
2723         /*
2724          * We always charge the cgroup the mm_struct belongs to.
2725          * The mm_struct's mem_cgroup changes on task migration if the
2726          * thread group leader migrates. It's possible that mm is not
2727          * set, if so charge the root memcg (happens for pagecache usage).
2728          */
2729         if (!*ptr && !mm)
2730                 *ptr = root_mem_cgroup;
2731 again:
2732         if (*ptr) { /* css should be a valid one */
2733                 memcg = *ptr;
2734                 if (mem_cgroup_is_root(memcg))
2735                         goto done;
2736                 if (consume_stock(memcg, nr_pages))
2737                         goto done;
2738                 css_get(&memcg->css);
2739         } else {
2740                 struct task_struct *p;
2741 
2742                 rcu_read_lock();
2743                 p = rcu_dereference(mm->owner);
2744                 /*
2745                  * Because we don't have task_lock(), "p" can exit.
2746                  * In that case, "memcg" can point to root or p can be NULL with
2747                  * race with swapoff. Then, we have small risk of mis-accouning.
2748                  * But such kind of mis-account by race always happens because
2749                  * we don't have cgroup_mutex(). It's overkill and we allo that
2750                  * small race, here.
2751                  * (*) swapoff at el will charge against mm-struct not against
2752                  * task-struct. So, mm->owner can be NULL.
2753                  */
2754                 memcg = mem_cgroup_from_task(p);
2755                 if (!memcg)
2756                         memcg = root_mem_cgroup;
2757                 if (mem_cgroup_is_root(memcg)) {
2758                         rcu_read_unlock();
2759                         goto done;
2760                 }
2761                 if (consume_stock(memcg, nr_pages)) {
2762                         /*
2763                          * It seems dagerous to access memcg without css_get().
2764                          * But considering how consume_stok works, it's not
2765                          * necessary. If consume_stock success, some charges
2766                          * from this memcg are cached on this cpu. So, we
2767                          * don't need to call css_get()/css_tryget() before
2768                          * calling consume_stock().
2769                          */
2770                         rcu_read_unlock();
2771                         goto done;
2772                 }
2773                 /* after here, we may be blocked. we need to get refcnt */
2774                 if (!css_tryget(&memcg->css)) {
2775                         rcu_read_unlock();
2776                         goto again;
2777                 }
2778                 rcu_read_unlock();
2779         }
2780 
2781         do {
2782                 bool invoke_oom = oom && !nr_oom_retries;
2783 
2784                 /* If killed, bypass charge */
2785                 if (fatal_signal_pending(current)) {
2786                         css_put(&memcg->css);
2787                         goto bypass;
2788                 }
2789 
2790                 ret = mem_cgroup_do_charge(memcg, gfp_mask, batch,
2791                                            nr_pages, invoke_oom);
2792                 switch (ret) {
2793                 case CHARGE_OK:
2794                         break;
2795                 case CHARGE_RETRY: /* not in OOM situation but retry */
2796                         batch = nr_pages;
2797                         css_put(&memcg->css);
2798                         memcg = NULL;
2799                         goto again;
2800                 case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */
2801                         css_put(&memcg->css);
2802                         goto nomem;
2803                 case CHARGE_NOMEM: /* OOM routine works */
2804                         if (!oom || invoke_oom) {
2805                                 css_put(&memcg->css);
2806                                 goto nomem;
2807                         }
2808                         nr_oom_retries--;
2809                         break;
2810                 }
2811         } while (ret != CHARGE_OK);
2812 
2813         if (batch > nr_pages)
2814                 refill_stock(memcg, batch - nr_pages);
2815         css_put(&memcg->css);
2816 done:
2817         *ptr = memcg;
2818         return 0;
2819 nomem:
2820         if (!(gfp_mask & __GFP_NOFAIL)) {
2821                 *ptr = NULL;
2822                 return -ENOMEM;
2823         }
2824 bypass:
2825         *ptr = root_mem_cgroup;
2826         return -EINTR;
2827 }
2828 
2829 /*
2830  * Somemtimes we have to undo a charge we got by try_charge().
2831  * This function is for that and do uncharge, put css's refcnt.
2832  * gotten by try_charge().
2833  */
2834 static void __mem_cgroup_cancel_charge(struct mem_cgroup *memcg,
2835                                        unsigned int nr_pages)
2836 {
2837         if (!mem_cgroup_is_root(memcg)) {
2838                 unsigned long bytes = nr_pages * PAGE_SIZE;
2839 
2840                 res_counter_uncharge(&memcg->res, bytes);
2841                 if (do_swap_account)
2842                         res_counter_uncharge(&memcg->memsw, bytes);
2843         }
2844 }
2845 
2846 /*
2847  * Cancel chrages in this cgroup....doesn't propagate to parent cgroup.
2848  * This is useful when moving usage to parent cgroup.
2849  */
2850 static void __mem_cgroup_cancel_local_charge(struct mem_cgroup *memcg,
2851                                         unsigned int nr_pages)
2852 {
2853         unsigned long bytes = nr_pages * PAGE_SIZE;
2854 
2855         if (mem_cgroup_is_root(memcg))
2856                 return;
2857 
2858         res_counter_uncharge_until(&memcg->res, memcg->res.parent, bytes);
2859         if (do_swap_account)
2860                 res_counter_uncharge_until(&memcg->memsw,
2861                                                 memcg->memsw.parent, bytes);
2862 }
2863 
2864 /*
2865  * A helper function to get mem_cgroup from ID. must be called under
2866  * rcu_read_lock().  The caller is responsible for calling css_tryget if
2867  * the mem_cgroup is used for charging. (dropping refcnt from swap can be
2868  * called against removed memcg.)
2869  */
2870 static struct mem_cgroup *mem_cgroup_lookup(unsigned short id)
2871 {
2872         struct cgroup_subsys_state *css;
2873 
2874         /* ID 0 is unused ID */
2875         if (!id)
2876                 return NULL;
2877         css = css_lookup(&mem_cgroup_subsys, id);
2878         if (!css)
2879                 return NULL;
2880         return mem_cgroup_from_css(css);
2881 }
2882 
2883 struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
2884 {
2885         struct mem_cgroup *memcg = NULL;
2886         struct page_cgroup *pc;
2887         unsigned short id;
2888         swp_entry_t ent;
2889 
2890         VM_BUG_ON(!PageLocked(page));
2891 
2892         pc = lookup_page_cgroup(page);
2893         lock_page_cgroup(pc);
2894         if (PageCgroupUsed(pc)) {
2895                 memcg = pc->mem_cgroup;
2896                 if (memcg && !css_tryget(&memcg->css))
2897                         memcg = NULL;
2898         } else if (PageSwapCache(page)) {
2899                 ent.val = page_private(page);
2900                 id = lookup_swap_cgroup_id(ent);
2901                 rcu_read_lock();
2902                 memcg = mem_cgroup_lookup(id);
2903                 if (memcg && !css_tryget(&memcg->css))
2904                         memcg = NULL;
2905                 rcu_read_unlock();
2906         }
2907         unlock_page_cgroup(pc);
2908         return memcg;
2909 }
2910 
2911 static void __mem_cgroup_commit_charge(struct mem_cgroup *memcg,
2912                                        struct page *page,
2913                                        unsigned int nr_pages,
2914                                        enum charge_type ctype,
2915                                        bool lrucare)
2916 {
2917         struct page_cgroup *pc = lookup_page_cgroup(page);
2918         struct zone *uninitialized_var(zone);
2919         struct lruvec *lruvec;
2920         bool was_on_lru = false;
2921         bool anon;
2922 
2923         lock_page_cgroup(pc);
2924         VM_BUG_ON(PageCgroupUsed(pc));
2925         /*
2926          * we don't need page_cgroup_lock about tail pages, becase they are not
2927          * accessed by any other context at this point.
2928          */
2929 
2930         /*
2931          * In some cases, SwapCache and FUSE(splice_buf->radixtree), the page
2932          * may already be on some other mem_cgroup's LRU.  Take care of it.
2933          */
2934         if (lrucare) {
2935                 zone = page_zone(page);
2936                 spin_lock_irq(&zone->lru_lock);
2937                 if (PageLRU(page)) {
2938                         lruvec = mem_cgroup_zone_lruvec(zone, pc->mem_cgroup);
2939                         ClearPageLRU(page);
2940                         del_page_from_lru_list(page, lruvec, page_lru(page));
2941                         was_on_lru = true;
2942                 }
2943         }
2944 
2945         pc->mem_cgroup = memcg;
2946         /*
2947          * We access a page_cgroup asynchronously without lock_page_cgroup().
2948          * Especially when a page_cgroup is taken from a page, pc->mem_cgroup
2949          * is accessed after testing USED bit. To make pc->mem_cgroup visible
2950          * before USED bit, we need memory barrier here.
2951          * See mem_cgroup_add_lru_list(), etc.
2952          */
2953         smp_wmb();
2954         SetPageCgroupUsed(pc);
2955 
2956         if (lrucare) {
2957                 if (was_on_lru) {
2958                         lruvec = mem_cgroup_zone_lruvec(zone, pc->mem_cgroup);
2959                         VM_BUG_ON(PageLRU(page));
2960                         SetPageLRU(page);
2961                         add_page_to_lru_list(page, lruvec, page_lru(page));
2962                 }
2963                 spin_unlock_irq(&zone->lru_lock);
2964         }
2965 
2966         if (ctype == MEM_CGROUP_CHARGE_TYPE_ANON)
2967                 anon = true;
2968         else
2969                 anon = false;
2970 
2971         mem_cgroup_charge_statistics(memcg, page, anon, nr_pages);
2972         unlock_page_cgroup(pc);
2973 
2974         /*
2975          * "charge_statistics" updated event counter. Then, check it.
2976          * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree.
2977          * if they exceeds softlimit.
2978          */
2979         memcg_check_events(memcg, page);
2980 }
2981 
2982 static DEFINE_MUTEX(set_limit_mutex);
2983 
2984 #ifdef CONFIG_MEMCG_KMEM
2985 static inline bool memcg_can_account_kmem(struct mem_cgroup *memcg)
2986 {
2987         return !mem_cgroup_disabled() && !mem_cgroup_is_root(memcg) &&
2988                 (memcg->kmem_account_flags & KMEM_ACCOUNTED_MASK);
2989 }
2990 
2991 /*
2992  * This is a bit cumbersome, but it is rarely used and avoids a backpointer
2993  * in the memcg_cache_params struct.
2994  */
2995 static struct kmem_cache *memcg_params_to_cache(struct memcg_cache_params *p)
2996 {
2997         struct kmem_cache *cachep;
2998 
2999         VM_BUG_ON(p->is_root_cache);
3000         cachep = p->root_cache;
3001         return cachep->memcg_params->memcg_caches[memcg_cache_id(p->memcg)];
3002 }
3003 
3004 #ifdef CONFIG_SLABINFO
3005 static int mem_cgroup_slabinfo_read(struct cgroup_subsys_state *css,
3006                                     struct cftype *cft, struct seq_file *m)
3007 {
3008         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3009         struct memcg_cache_params *params;
3010 
3011         if (!memcg_can_account_kmem(memcg))
3012                 return -EIO;
3013 
3014         print_slabinfo_header(m);
3015 
3016         mutex_lock(&memcg->slab_caches_mutex);
3017         list_for_each_entry(params, &memcg->memcg_slab_caches, list)
3018                 cache_show(memcg_params_to_cache(params), m);
3019         mutex_unlock(&memcg->slab_caches_mutex);
3020 
3021         return 0;
3022 }
3023 #endif
3024 
3025 static int memcg_charge_kmem(struct mem_cgroup *memcg, gfp_t gfp, u64 size)
3026 {
3027         struct res_counter *fail_res;
3028         struct mem_cgroup *_memcg;
3029         int ret = 0;
3030         bool may_oom;
3031 
3032         ret = res_counter_charge(&memcg->kmem, size, &fail_res);
3033         if (ret)
3034                 return ret;
3035 
3036         /*
3037          * Conditions under which we can wait for the oom_killer. Those are
3038          * the same conditions tested by the core page allocator
3039          */
3040         may_oom = (gfp & __GFP_FS) && !(gfp & __GFP_NORETRY);
3041 
3042         _memcg = memcg;
3043         ret = __mem_cgroup_try_charge(NULL, gfp, size >> PAGE_SHIFT,
3044                                       &_memcg, may_oom);
3045 
3046         if (ret == -EINTR)  {
3047                 /*
3048                  * __mem_cgroup_try_charge() chosed to bypass to root due to
3049                  * OOM kill or fatal signal.  Since our only options are to
3050                  * either fail the allocation or charge it to this cgroup, do
3051                  * it as a temporary condition. But we can't fail. From a
3052                  * kmem/slab perspective, the cache has already been selected,
3053                  * by mem_cgroup_kmem_get_cache(), so it is too late to change
3054                  * our minds.
3055                  *
3056                  * This condition will only trigger if the task entered
3057                  * memcg_charge_kmem in a sane state, but was OOM-killed during
3058                  * __mem_cgroup_try_charge() above. Tasks that were already
3059                  * dying when the allocation triggers should have been already
3060                  * directed to the root cgroup in memcontrol.h
3061                  */
3062                 res_counter_charge_nofail(&memcg->res, size, &fail_res);
3063                 if (do_swap_account)
3064                         res_counter_charge_nofail(&memcg->memsw, size,
3065                                                   &fail_res);
3066                 ret = 0;
3067         } else if (ret)
3068                 res_counter_uncharge(&memcg->kmem, size);
3069 
3070         return ret;
3071 }
3072 
3073 static void memcg_uncharge_kmem(struct mem_cgroup *memcg, u64 size)
3074 {
3075         res_counter_uncharge(&memcg->res, size);
3076         if (do_swap_account)
3077                 res_counter_uncharge(&memcg->memsw, size);
3078 
3079         /* Not down to 0 */
3080         if (res_counter_uncharge(&memcg->kmem, size))
3081                 return;
3082 
3083         /*
3084          * Releases a reference taken in kmem_cgroup_css_offline in case
3085          * this last uncharge is racing with the offlining code or it is
3086          * outliving the memcg existence.
3087          *
3088          * The memory barrier imposed by test&clear is paired with the
3089          * explicit one in memcg_kmem_mark_dead().
3090          */
3091         if (memcg_kmem_test_and_clear_dead(memcg))
3092                 css_put(&memcg->css);
3093 }
3094 
3095 void memcg_cache_list_add(struct mem_cgroup *memcg, struct kmem_cache *cachep)
3096 {
3097         if (!memcg)
3098                 return;
3099 
3100         mutex_lock(&memcg->slab_caches_mutex);
3101         list_add(&cachep->memcg_params->list, &memcg->memcg_slab_caches);
3102         mutex_unlock(&memcg->slab_caches_mutex);
3103 }
3104 
3105 /*
3106  * helper for acessing a memcg's index. It will be used as an index in the
3107  * child cache array in kmem_cache, and also to derive its name. This function
3108  * will return -1 when this is not a kmem-limited memcg.
3109  */
3110 int memcg_cache_id(struct mem_cgroup *memcg)
3111 {
3112         return memcg ? memcg->kmemcg_id : -1;
3113 }
3114 
3115 /*
3116  * This ends up being protected by the set_limit mutex, during normal
3117  * operation, because that is its main call site.
3118  *
3119  * But when we create a new cache, we can call this as well if its parent
3120  * is kmem-limited. That will have to hold set_limit_mutex as well.
3121  */
3122 int memcg_update_cache_sizes(struct mem_cgroup *memcg)
3123 {
3124         int num, ret;
3125 
3126         num = ida_simple_get(&kmem_limited_groups,
3127                                 0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
3128         if (num < 0)
3129                 return num;
3130         /*
3131          * After this point, kmem_accounted (that we test atomically in
3132          * the beginning of this conditional), is no longer 0. This
3133          * guarantees only one process will set the following boolean
3134          * to true. We don't need test_and_set because we're protected
3135          * by the set_limit_mutex anyway.
3136          */
3137         memcg_kmem_set_activated(memcg);
3138 
3139         ret = memcg_update_all_caches(num+1);
3140         if (ret) {
3141                 ida_simple_remove(&kmem_limited_groups, num);
3142                 memcg_kmem_clear_activated(memcg);
3143                 return ret;
3144         }
3145 
3146         memcg->kmemcg_id = num;
3147         INIT_LIST_HEAD(&memcg->memcg_slab_caches);
3148         mutex_init(&memcg->slab_caches_mutex);
3149         return 0;
3150 }
3151 
3152 static size_t memcg_caches_array_size(int num_groups)
3153 {
3154         ssize_t size;
3155         if (num_groups <= 0)
3156                 return 0;
3157 
3158         size = 2 * num_groups;
3159         if (size < MEMCG_CACHES_MIN_SIZE)
3160                 size = MEMCG_CACHES_MIN_SIZE;
3161         else if (size > MEMCG_CACHES_MAX_SIZE)
3162                 size = MEMCG_CACHES_MAX_SIZE;
3163 
3164         return size;
3165 }
3166 
3167 /*
3168  * We should update the current array size iff all caches updates succeed. This
3169  * can only be done from the slab side. The slab mutex needs to be held when
3170  * calling this.
3171  */
3172 void memcg_update_array_size(int num)
3173 {
3174         if (num > memcg_limited_groups_array_size)
3175                 memcg_limited_groups_array_size = memcg_caches_array_size(num);
3176 }
3177 
3178 static void kmem_cache_destroy_work_func(struct work_struct *w);
3179 
3180 int memcg_update_cache_size(struct kmem_cache *s, int num_groups)
3181 {
3182         struct memcg_cache_params *cur_params = s->memcg_params;
3183 
3184         VM_BUG_ON(s->memcg_params && !s->memcg_params->is_root_cache);
3185 
3186         if (num_groups > memcg_limited_groups_array_size) {
3187                 int i;
3188                 ssize_t size = memcg_caches_array_size(num_groups);
3189 
3190                 size *= sizeof(void *);
3191                 size += offsetof(struct memcg_cache_params, memcg_caches);
3192 
3193                 s->memcg_params = kzalloc(size, GFP_KERNEL);
3194                 if (!s->memcg_params) {
3195                         s->memcg_params = cur_params;
3196                         return -ENOMEM;
3197                 }
3198 
3199                 s->memcg_params->is_root_cache = true;
3200 
3201                 /*
3202                  * There is the chance it will be bigger than
3203                  * memcg_limited_groups_array_size, if we failed an allocation
3204                  * in a cache, in which case all caches updated before it, will
3205                  * have a bigger array.
3206                  *
3207                  * But if that is the case, the data after
3208                  * memcg_limited_groups_array_size is certainly unused
3209                  */
3210                 for (i = 0; i < memcg_limited_groups_array_size; i++) {
3211                         if (!cur_params->memcg_caches[i])
3212                                 continue;
3213                         s->memcg_params->memcg_caches[i] =
3214                                                 cur_params->memcg_caches[i];
3215                 }
3216 
3217                 /*
3218                  * Ideally, we would wait until all caches succeed, and only
3219                  * then free the old one. But this is not worth the extra
3220                  * pointer per-cache we'd have to have for this.
3221                  *
3222                  * It is not a big deal if some caches are left with a size
3223                  * bigger than the others. And all updates will reset this
3224                  * anyway.
3225                  */
3226                 kfree(cur_params);
3227         }
3228         return 0;
3229 }
3230 
3231 int memcg_register_cache(struct mem_cgroup *memcg, struct kmem_cache *s,
3232                          struct kmem_cache *root_cache)
3233 {
3234         size_t size;
3235 
3236         if (!memcg_kmem_enabled())
3237                 return 0;
3238 
3239         if (!memcg) {
3240                 size = offsetof(struct memcg_cache_params, memcg_caches);
3241                 size += memcg_limited_groups_array_size * sizeof(void *);
3242         } else
3243                 size = sizeof(struct memcg_cache_params);
3244 
3245         s->memcg_params = kzalloc(size, GFP_KERNEL);
3246         if (!s->memcg_params)
3247                 return -ENOMEM;
3248 
3249         if (memcg) {
3250                 s->memcg_params->memcg = memcg;
3251                 s->memcg_params->root_cache = root_cache;
3252                 INIT_WORK(&s->memcg_params->destroy,
3253                                 kmem_cache_destroy_work_func);
3254         } else
3255                 s->memcg_params->is_root_cache = true;
3256 
3257         return 0;
3258 }
3259 
3260 void memcg_release_cache(struct kmem_cache *s)
3261 {
3262         struct kmem_cache *root;
3263         struct mem_cgroup *memcg;
3264         int id;
3265 
3266         /*
3267          * This happens, for instance, when a root cache goes away before we
3268          * add any memcg.
3269          */
3270         if (!s->memcg_params)
3271                 return;
3272 
3273         if (s->memcg_params->is_root_cache)
3274                 goto out;
3275 
3276         memcg = s->memcg_params->memcg;
3277         id  = memcg_cache_id(memcg);
3278 
3279         root = s->memcg_params->root_cache;
3280         root->memcg_params->memcg_caches[id] = NULL;
3281 
3282         mutex_lock(&memcg->slab_caches_mutex);
3283         list_del(&s->memcg_params->list);
3284         mutex_unlock(&memcg->slab_caches_mutex);
3285 
3286         css_put(&memcg->css);
3287 out:
3288         kfree(s->memcg_params);
3289 }
3290 
3291 /*
3292  * During the creation a new cache, we need to disable our accounting mechanism
3293  * altogether. This is true even if we are not creating, but rather just
3294  * enqueing new caches to be created.
3295  *
3296  * This is because that process will trigger allocations; some visible, like
3297  * explicit kmallocs to auxiliary data structures, name strings and internal
3298  * cache structures; some well concealed, like INIT_WORK() that can allocate
3299  * objects during debug.
3300  *
3301  * If any allocation happens during memcg_kmem_get_cache, we will recurse back
3302  * to it. This may not be a bounded recursion: since the first cache creation
3303  * failed to complete (waiting on the allocation), we'll just try to create the
3304  * cache again, failing at the same point.
3305  *
3306  * memcg_kmem_get_cache is prepared to abort after seeing a positive count of
3307  * memcg_kmem_skip_account. So we enclose anything that might allocate memory
3308  * inside the following two functions.
3309  */
3310 static inline void memcg_stop_kmem_account(void)
3311 {
3312         VM_BUG_ON(!current->mm);
3313         current->memcg_kmem_skip_account++;
3314 }
3315 
3316 static inline void memcg_resume_kmem_account(void)
3317 {
3318         VM_BUG_ON(!current->mm);
3319         current->memcg_kmem_skip_account--;
3320 }
3321 
3322 static void kmem_cache_destroy_work_func(struct work_struct *w)
3323 {
3324         struct kmem_cache *cachep;
3325         struct memcg_cache_params *p;
3326 
3327         p = container_of(w, struct memcg_cache_params, destroy);
3328 
3329         cachep = memcg_params_to_cache(p);
3330 
3331         /*
3332          * If we get down to 0 after shrink, we could delete right away.
3333          * However, memcg_release_pages() already puts us back in the workqueue
3334          * in that case. If we proceed deleting, we'll get a dangling
3335          * reference, and removing the object from the workqueue in that case
3336          * is unnecessary complication. We are not a fast path.
3337          *
3338          * Note that this case is fundamentally different from racing with
3339          * shrink_slab(): if memcg_cgroup_destroy_cache() is called in
3340          * kmem_cache_shrink, not only we would be reinserting a dead cache
3341          * into the queue, but doing so from inside the worker racing to
3342          * destroy it.
3343          *
3344          * So if we aren't down to zero, we'll just schedule a worker and try
3345          * again
3346          */
3347         if (atomic_read(&cachep->memcg_params->nr_pages) != 0) {
3348                 kmem_cache_shrink(cachep);
3349                 if (atomic_read(&cachep->memcg_params->nr_pages) == 0)
3350                         return;
3351         } else
3352                 kmem_cache_destroy(cachep);
3353 }
3354 
3355 void mem_cgroup_destroy_cache(struct kmem_cache *cachep)
3356 {
3357         if (!cachep->memcg_params->dead)
3358                 return;
3359 
3360         /*
3361          * There are many ways in which we can get here.
3362          *
3363          * We can get to a memory-pressure situation while the delayed work is
3364          * still pending to run. The vmscan shrinkers can then release all
3365          * cache memory and get us to destruction. If this is the case, we'll
3366          * be executed twice, which is a bug (the second time will execute over
3367          * bogus data). In this case, cancelling the work should be fine.
3368          *
3369          * But we can also get here from the worker itself, if
3370          * kmem_cache_shrink is enough to shake all the remaining objects and
3371          * get the page count to 0. In this case, we'll deadlock if we try to
3372          * cancel the work (the worker runs with an internal lock held, which
3373          * is the same lock we would hold for cancel_work_sync().)
3374          *
3375          * Since we can't possibly know who got us here, just refrain from
3376          * running if there is already work pending
3377          */
3378         if (work_pending(&cachep->memcg_params->destroy))
3379                 return;
3380         /*
3381          * We have to defer the actual destroying to a workqueue, because
3382          * we might currently be in a context that cannot sleep.
3383          */
3384         schedule_work(&cachep->memcg_params->destroy);
3385 }
3386 
3387 /*
3388  * This lock protects updaters, not readers. We want readers to be as fast as
3389  * they can, and they will either see NULL or a valid cache value. Our model
3390  * allow them to see NULL, in which case the root memcg will be selected.
3391  *
3392  * We need this lock because multiple allocations to the same cache from a non
3393  * will span more than one worker. Only one of them can create the cache.
3394  */
3395 static DEFINE_MUTEX(memcg_cache_mutex);
3396 
3397 /*
3398  * Called with memcg_cache_mutex held
3399  */
3400 static struct kmem_cache *kmem_cache_dup(struct mem_cgroup *memcg,
3401                                          struct kmem_cache *s)
3402 {
3403         struct kmem_cache *new;
3404         static char *tmp_name = NULL;
3405 
3406         lockdep_assert_held(&memcg_cache_mutex);
3407 
3408         /*
3409          * kmem_cache_create_memcg duplicates the given name and
3410          * cgroup_name for this name requires RCU context.
3411          * This static temporary buffer is used to prevent from
3412          * pointless shortliving allocation.
3413          */
3414         if (!tmp_name) {
3415                 tmp_name = kmalloc(PATH_MAX, GFP_KERNEL);
3416                 if (!tmp_name)
3417                         return NULL;
3418         }
3419 
3420         rcu_read_lock();
3421         snprintf(tmp_name, PATH_MAX, "%s(%d:%s)", s->name,
3422                          memcg_cache_id(memcg), cgroup_name(memcg->css.cgroup));
3423         rcu_read_unlock();
3424 
3425         new = kmem_cache_create_memcg(memcg, tmp_name, s->object_size, s->align,
3426                                       (s->flags & ~SLAB_PANIC), s->ctor, s);
3427 
3428         if (new)
3429                 new->allocflags |= __GFP_KMEMCG;
3430 
3431         return new;
3432 }
3433 
3434 static struct kmem_cache *memcg_create_kmem_cache(struct mem_cgroup *memcg,
3435                                                   struct kmem_cache *cachep)
3436 {
3437         struct kmem_cache *new_cachep;
3438         int idx;
3439 
3440         BUG_ON(!memcg_can_account_kmem(memcg));
3441 
3442         idx = memcg_cache_id(memcg);
3443 
3444         mutex_lock(&memcg_cache_mutex);
3445         new_cachep = cachep->memcg_params->memcg_caches[idx];
3446         if (new_cachep) {
3447                 css_put(&memcg->css);
3448                 goto out;
3449         }
3450 
3451         new_cachep = kmem_cache_dup(memcg, cachep);
3452         if (new_cachep == NULL) {
3453                 new_cachep = cachep;
3454                 css_put(&memcg->css);
3455                 goto out;
3456         }
3457 
3458         atomic_set(&new_cachep->memcg_params->nr_pages , 0);
3459 
3460         cachep->memcg_params->memcg_caches[idx] = new_cachep;
3461         /*
3462          * the readers won't lock, make sure everybody sees the updated value,
3463          * so they won't put stuff in the queue again for no reason
3464          */
3465         wmb();
3466 out:
3467         mutex_unlock(&memcg_cache_mutex);
3468         return new_cachep;
3469 }
3470 
3471 void kmem_cache_destroy_memcg_children(struct kmem_cache *s)
3472 {
3473         struct kmem_cache *c;
3474         int i;
3475 
3476         if (!s->memcg_params)
3477                 return;
3478         if (!s->memcg_params->is_root_cache)
3479                 return;
3480 
3481         /*
3482          * If the cache is being destroyed, we trust that there is no one else
3483          * requesting objects from it. Even if there are, the sanity checks in
3484          * kmem_cache_destroy should caught this ill-case.
3485          *
3486          * Still, we don't want anyone else freeing memcg_caches under our
3487          * noses, which can happen if a new memcg comes to life. As usual,
3488          * we'll take the set_limit_mutex to protect ourselves against this.
3489          */
3490         mutex_lock(&set_limit_mutex);
3491         for (i = 0; i < memcg_limited_groups_array_size; i++) {
3492                 c = s->memcg_params->memcg_caches[i];
3493                 if (!c)
3494                         continue;
3495 
3496                 /*
3497                  * We will now manually delete the caches, so to avoid races
3498                  * we need to cancel all pending destruction workers and
3499                  * proceed with destruction ourselves.
3500                  *
3501                  * kmem_cache_destroy() will call kmem_cache_shrink internally,
3502                  * and that could spawn the workers again: it is likely that
3503                  * the cache still have active pages until this very moment.
3504                  * This would lead us back to mem_cgroup_destroy_cache.
3505                  *
3506                  * But that will not execute at all if the "dead" flag is not
3507                  * set, so flip it down to guarantee we are in control.
3508                  */
3509                 c->memcg_params->dead = false;
3510                 cancel_work_sync(&c->memcg_params->destroy);
3511                 kmem_cache_destroy(c);
3512         }
3513         mutex_unlock(&set_limit_mutex);
3514 }
3515 
3516 struct create_work {
3517         struct mem_cgroup *memcg;
3518         struct kmem_cache *cachep;
3519         struct work_struct work;
3520 };
3521 
3522 static void mem_cgroup_destroy_all_caches(struct mem_cgroup *memcg)
3523 {
3524         struct kmem_cache *cachep;
3525         struct memcg_cache_params *params;
3526 
3527         if (!memcg_kmem_is_active(memcg))
3528                 return;
3529 
3530         mutex_lock(&memcg->slab_caches_mutex);
3531         list_for_each_entry(params, &memcg->memcg_slab_caches, list) {
3532                 cachep = memcg_params_to_cache(params);
3533                 cachep->memcg_params->dead = true;
3534                 schedule_work(&cachep->memcg_params->destroy);
3535         }
3536         mutex_unlock(&memcg->slab_caches_mutex);
3537 }
3538 
3539 static void memcg_create_cache_work_func(struct work_struct *w)
3540 {
3541         struct create_work *cw;
3542 
3543         cw = container_of(w, struct create_work, work);
3544         memcg_create_kmem_cache(cw->memcg, cw->cachep);
3545         kfree(cw);
3546 }
3547 
3548 /*
3549  * Enqueue the creation of a per-memcg kmem_cache.
3550  */
3551 static void __memcg_create_cache_enqueue(struct mem_cgroup *memcg,
3552                                          struct kmem_cache *cachep)
3553 {
3554         struct create_work *cw;
3555 
3556         cw = kmalloc(sizeof(struct create_work), GFP_NOWAIT);
3557         if (cw == NULL) {
3558                 css_put(&memcg->css);
3559                 return;
3560         }
3561 
3562         cw->memcg = memcg;
3563         cw->cachep = cachep;
3564 
3565         INIT_WORK(&cw->work, memcg_create_cache_work_func);
3566         schedule_work(&cw->work);
3567 }
3568 
3569 static void memcg_create_cache_enqueue(struct mem_cgroup *memcg,
3570                                        struct kmem_cache *cachep)
3571 {
3572         /*
3573          * We need to stop accounting when we kmalloc, because if the
3574          * corresponding kmalloc cache is not yet created, the first allocation
3575          * in __memcg_create_cache_enqueue will recurse.
3576          *
3577          * However, it is better to enclose the whole function. Depending on
3578          * the debugging options enabled, INIT_WORK(), for instance, can
3579          * trigger an allocation. This too, will make us recurse. Because at
3580          * this point we can't allow ourselves back into memcg_kmem_get_cache,
3581          * the safest choice is to do it like this, wrapping the whole function.
3582          */
3583         memcg_stop_kmem_account();
3584         __memcg_create_cache_enqueue(memcg, cachep);
3585         memcg_resume_kmem_account();
3586 }
3587 /*
3588  * Return the kmem_cache we're supposed to use for a slab allocation.
3589  * We try to use the current memcg's version of the cache.
3590  *
3591  * If the cache does not exist yet, if we are the first user of it,
3592  * we either create it immediately, if possible, or create it asynchronously
3593  * in a workqueue.
3594  * In the latter case, we will let the current allocation go through with
3595  * the original cache.
3596  *
3597  * Can't be called in interrupt context or from kernel threads.
3598  * This function needs to be called with rcu_read_lock() held.
3599  */
3600 struct kmem_cache *__memcg_kmem_get_cache(struct kmem_cache *cachep,
3601                                           gfp_t gfp)
3602 {
3603         struct mem_cgroup *memcg;
3604         int idx;
3605 
3606         VM_BUG_ON(!cachep->memcg_params);
3607         VM_BUG_ON(!cachep->memcg_params->is_root_cache);
3608 
3609         if (!current->mm || current->memcg_kmem_skip_account)
3610                 return cachep;
3611 
3612         rcu_read_lock();
3613         memcg = mem_cgroup_from_task(rcu_dereference(current->mm->owner));
3614 
3615         if (!memcg_can_account_kmem(memcg))
3616                 goto out;
3617 
3618         idx = memcg_cache_id(memcg);
3619 
3620         /*
3621          * barrier to mare sure we're always seeing the up to date value.  The
3622          * code updating memcg_caches will issue a write barrier to match this.
3623          */
3624         read_barrier_depends();
3625         if (likely(cachep->memcg_params->memcg_caches[idx])) {
3626                 cachep = cachep->memcg_params->memcg_caches[idx];
3627                 goto out;
3628         }
3629 
3630         /* The corresponding put will be done in the workqueue. */
3631         if (!css_tryget(&memcg->css))
3632                 goto out;
3633         rcu_read_unlock();
3634 
3635         /*
3636          * If we are in a safe context (can wait, and not in interrupt
3637          * context), we could be be predictable and return right away.
3638          * This would guarantee that the allocation being performed
3639          * already belongs in the new cache.
3640          *
3641          * However, there are some clashes that can arrive from locking.
3642          * For instance, because we acquire the slab_mutex while doing
3643          * kmem_cache_dup, this means no further allocation could happen
3644          * with the slab_mutex held.
3645          *
3646          * Also, because cache creation issue get_online_cpus(), this
3647          * creates a lock chain: memcg_slab_mutex -> cpu_hotplug_mutex,
3648          * that ends up reversed during cpu hotplug. (cpuset allocates
3649          * a bunch of GFP_KERNEL memory during cpuup). Due to all that,
3650          * better to defer everything.
3651          */
3652         memcg_create_cache_enqueue(memcg, cachep);
3653         return cachep;
3654 out:
3655         rcu_read_unlock();
3656         return cachep;
3657 }
3658 EXPORT_SYMBOL(__memcg_kmem_get_cache);
3659 
3660 /*
3661  * We need to verify if the allocation against current->mm->owner's memcg is
3662  * possible for the given order. But the page is not allocated yet, so we'll
3663  * need a further commit step to do the final arrangements.
3664  *
3665  * It is possible for the task to switch cgroups in this mean time, so at
3666  * commit time, we can't rely on task conversion any longer.  We'll then use
3667  * the handle argument to return to the caller which cgroup we should commit
3668  * against. We could also return the memcg directly and avoid the pointer
3669  * passing, but a boolean return value gives better semantics considering
3670  * the compiled-out case as well.
3671  *
3672  * Returning true means the allocation is possible.
3673  */
3674 bool
3675 __memcg_kmem_newpage_charge(gfp_t gfp, struct mem_cgroup **_memcg, int order)
3676 {
3677         struct mem_cgroup *memcg;
3678         int ret;
3679 
3680         *_memcg = NULL;
3681 
3682         /*
3683          * Disabling accounting is only relevant for some specific memcg
3684          * internal allocations. Therefore we would initially not have such
3685          * check here, since direct calls to the page allocator that are marked
3686          * with GFP_KMEMCG only happen outside memcg core. We are mostly
3687          * concerned with cache allocations, and by having this test at
3688          * memcg_kmem_get_cache, we are already able to relay the allocation to
3689          * the root cache and bypass the memcg cache altogether.
3690          *
3691          * There is one exception, though: the SLUB allocator does not create
3692          * large order caches, but rather service large kmallocs directly from
3693          * the page allocator. Therefore, the following sequence when backed by
3694          * the SLUB allocator:
3695          *
3696          *      memcg_stop_kmem_account();
3697          *      kmalloc(<large_number>)
3698          *      memcg_resume_kmem_account();
3699          *
3700          * would effectively ignore the fact that we should skip accounting,
3701          * since it will drive us directly to this function without passing
3702          * through the cache selector memcg_kmem_get_cache. Such large
3703          * allocations are extremely rare but can happen, for instance, for the
3704          * cache arrays. We bring this test here.
3705          */
3706         if (!current->mm || current->memcg_kmem_skip_account)
3707                 return true;
3708 
3709         memcg = try_get_mem_cgroup_from_mm(current->mm);
3710 
3711         /*
3712          * very rare case described in mem_cgroup_from_task. Unfortunately there
3713          * isn't much we can do without complicating this too much, and it would
3714          * be gfp-dependent anyway. Just let it go
3715          */
3716         if (unlikely(!memcg))
3717                 return true;
3718 
3719         if (!memcg_can_account_kmem(memcg)) {
3720                 css_put(&memcg->css);
3721                 return true;
3722         }
3723 
3724         ret = memcg_charge_kmem(memcg, gfp, PAGE_SIZE << order);
3725         if (!ret)
3726                 *_memcg = memcg;
3727 
3728         css_put(&memcg->css);
3729         return (ret == 0);
3730 }
3731 
3732 void __memcg_kmem_commit_charge(struct page *page, struct mem_cgroup *memcg,
3733                               int order)
3734 {
3735         struct page_cgroup *pc;
3736 
3737         VM_BUG_ON(mem_cgroup_is_root(memcg));
3738 
3739         /* The page allocation failed. Revert */
3740         if (!page) {
3741                 memcg_uncharge_kmem(memcg, PAGE_SIZE << order);
3742                 return;
3743         }
3744 
3745         pc = lookup_page_cgroup(page);
3746         lock_page_cgroup(pc);
3747         pc->mem_cgroup = memcg;
3748         SetPageCgroupUsed(pc);
3749         unlock_page_cgroup(pc);
3750 }
3751 
3752 void __memcg_kmem_uncharge_pages(struct page *page, int order)
3753 {
3754         struct mem_cgroup *memcg = NULL;
3755         struct page_cgroup *pc;
3756 
3757 
3758         pc = lookup_page_cgroup(page);
3759         /*
3760          * Fast unlocked return. Theoretically might have changed, have to
3761          * check again after locking.
3762          */
3763         if (!PageCgroupUsed(pc))
3764                 return;
3765 
3766         lock_page_cgroup(pc);
3767         if (PageCgroupUsed(pc)) {
3768                 memcg = pc->mem_cgroup;
3769                 ClearPageCgroupUsed(pc);
3770         }
3771         unlock_page_cgroup(pc);
3772 
3773         /*
3774          * We trust that only if there is a memcg associated with the page, it
3775          * is a valid allocation
3776          */
3777         if (!memcg)
3778                 return;
3779 
3780         VM_BUG_ON(mem_cgroup_is_root(memcg));
3781         memcg_uncharge_kmem(memcg, PAGE_SIZE << order);
3782 }
3783 #else
3784 static inline void mem_cgroup_destroy_all_caches(struct mem_cgroup *memcg)
3785 {
3786 }
3787 #endif /* CONFIG_MEMCG_KMEM */
3788 
3789 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
3790 
3791 #define PCGF_NOCOPY_AT_SPLIT (1 << PCG_LOCK | 1 << PCG_MIGRATION)
3792 /*
3793  * Because tail pages are not marked as "used", set it. We're under
3794  * zone->lru_lock, 'splitting on pmd' and compound_lock.
3795  * charge/uncharge will be never happen and move_account() is done under
3796  * compound_lock(), so we don't have to take care of races.
3797  */
3798 void mem_cgroup_split_huge_fixup(struct page *head)
3799 {
3800         struct page_cgroup *head_pc = lookup_page_cgroup(head);
3801         struct page_cgroup *pc;
3802         struct mem_cgroup *memcg;
3803         int i;
3804 
3805         if (mem_cgroup_disabled())
3806                 return;
3807 
3808         memcg = head_pc->mem_cgroup;
3809         for (i = 1; i < HPAGE_PMD_NR; i++) {
3810                 pc = head_pc + i;
3811                 pc->mem_cgroup = memcg;
3812                 smp_wmb();/* see __commit_charge() */
3813                 pc->flags = head_pc->flags & ~PCGF_NOCOPY_AT_SPLIT;
3814         }
3815         __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE],
3816                        HPAGE_PMD_NR);
3817 }
3818 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
3819 
3820 static inline
3821 void mem_cgroup_move_account_page_stat(struct mem_cgroup *from,
3822                                         struct mem_cgroup *to,
3823                                         unsigned int nr_pages,
3824                                         enum mem_cgroup_stat_index idx)
3825 {
3826         /* Update stat data for mem_cgroup */
3827         preempt_disable();
3828         __this_cpu_sub(from->stat->count[idx], nr_pages);
3829         __this_cpu_add(to->stat->count[idx], nr_pages);
3830         preempt_enable();
3831 }
3832 
3833 /**
3834  * mem_cgroup_move_account - move account of the page
3835  * @page: the page
3836  * @nr_pages: number of regular pages (>1 for huge pages)
3837  * @pc: page_cgroup of the page.
3838  * @from: mem_cgroup which the page is moved from.
3839  * @to: mem_cgroup which the page is moved to. @from != @to.
3840  *
3841  * The caller must confirm following.
3842  * - page is not on LRU (isolate_page() is useful.)
3843  * - compound_lock is held when nr_pages > 1
3844  *
3845  * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
3846  * from old cgroup.
3847  */
3848 static int mem_cgroup_move_account(struct page *page,
3849                                    unsigned int nr_pages,
3850                                    struct page_cgroup *pc,
3851                                    struct mem_cgroup *from,
3852                                    struct mem_cgroup *to)
3853 {
3854         unsigned long flags;
3855         int ret;
3856         bool anon = PageAnon(page);
3857 
3858         VM_BUG_ON(from == to);
3859         VM_BUG_ON(PageLRU(page));
3860         /*
3861          * The page is isolated from LRU. So, collapse function
3862          * will not handle this page. But page splitting can happen.
3863          * Do this check under compound_page_lock(). The caller should
3864          * hold it.
3865          */
3866         ret = -EBUSY;
3867         if (nr_pages > 1 && !PageTransHuge(page))
3868                 goto out;
3869 
3870         lock_page_cgroup(pc);
3871 
3872         ret = -EINVAL;
3873         if (!PageCgroupUsed(pc) || pc->mem_cgroup != from)
3874                 goto unlock;
3875 
3876         move_lock_mem_cgroup(from, &flags);
3877 
3878         if (!anon && page_mapped(page))
3879                 mem_cgroup_move_account_page_stat(from, to, nr_pages,
3880                         MEM_CGROUP_STAT_FILE_MAPPED);
3881 
3882         if (PageWriteback(page))
3883                 mem_cgroup_move_account_page_stat(from, to, nr_pages,
3884                         MEM_CGROUP_STAT_WRITEBACK);
3885 
3886         mem_cgroup_charge_statistics(from, page, anon, -nr_pages);
3887 
3888         /* caller should have done css_get */
3889         pc->mem_cgroup = to;
3890         mem_cgroup_charge_statistics(to, page, anon, nr_pages);
3891         move_unlock_mem_cgroup(from, &flags);
3892         ret = 0;
3893 unlock:
3894         unlock_page_cgroup(pc);
3895         /*
3896          * check events
3897          */
3898         memcg_check_events(to, page);
3899         memcg_check_events(from, page);
3900 out:
3901         return ret;
3902 }
3903 
3904 /**
3905  * mem_cgroup_move_parent - moves page to the parent group
3906  * @page: the page to move
3907  * @pc: page_cgroup of the page
3908  * @child: page's cgroup
3909  *
3910  * move charges to its parent or the root cgroup if the group has no
3911  * parent (aka use_hierarchy==0).
3912  * Although this might fail (get_page_unless_zero, isolate_lru_page or
3913  * mem_cgroup_move_account fails) the failure is always temporary and
3914  * it signals a race with a page removal/uncharge or migration. In the
3915  * first case the page is on the way out and it will vanish from the LRU
3916  * on the next attempt and the call should be retried later.
3917  * Isolation from the LRU fails only if page has been isolated from
3918  * the LRU since we looked at it and that usually means either global
3919  * reclaim or migration going on. The page will either get back to the
3920  * LRU or vanish.
3921  * Finaly mem_cgroup_move_account fails only if the page got uncharged
3922  * (!PageCgroupUsed) or moved to a different group. The page will
3923  * disappear in the next attempt.
3924  */
3925 static int mem_cgroup_move_parent(struct page *page,
3926                                   struct page_cgroup *pc,
3927                                   struct mem_cgroup *child)
3928 {
3929         struct mem_cgroup *parent;
3930         unsigned int nr_pages;
3931         unsigned long uninitialized_var(flags);
3932         int ret;
3933 
3934         VM_BUG_ON(mem_cgroup_is_root(child));
3935 
3936         ret = -EBUSY;
3937         if (!get_page_unless_zero(page))
3938                 goto out;
3939         if (isolate_lru_page(page))
3940                 goto put;
3941 
3942         nr_pages = hpage_nr_pages(page);
3943 
3944         parent = parent_mem_cgroup(child);
3945         /*
3946          * If no parent, move charges to root cgroup.
3947          */
3948         if (!parent)
3949                 parent = root_mem_cgroup;
3950 
3951         if (nr_pages > 1) {
3952                 VM_BUG_ON(!PageTransHuge(page));
3953                 flags = compound_lock_irqsave(page);
3954         }
3955 
3956         ret = mem_cgroup_move_account(page, nr_pages,
3957                                 pc, child, parent);
3958         if (!ret)
3959                 __mem_cgroup_cancel_local_charge(child, nr_pages);
3960 
3961         if (nr_pages > 1)
3962                 compound_unlock_irqrestore(page, flags);
3963         putback_lru_page(page);
3964 put:
3965         put_page(page);
3966 out:
3967         return ret;
3968 }
3969 
3970 /*
3971  * Charge the memory controller for page usage.
3972  * Return
3973  * 0 if the charge was successful
3974  * < 0 if the cgroup is over its limit
3975  */
3976 static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
3977                                 gfp_t gfp_mask, enum charge_type ctype)
3978 {
3979         struct mem_cgroup *memcg = NULL;
3980         unsigned int nr_pages = 1;
3981         bool oom = true;
3982         int ret;
3983 
3984         if (PageTransHuge(page)) {
3985                 nr_pages <<= compound_order(page);
3986                 VM_BUG_ON(!PageTransHuge(page));
3987                 /*
3988                  * Never OOM-kill a process for a huge page.  The
3989                  * fault handler will fall back to regular pages.
3990                  */
3991                 oom = false;
3992         }
3993 
3994         ret = __mem_cgroup_try_charge(mm, gfp_mask, nr_pages, &memcg, oom);
3995         if (ret == -ENOMEM)
3996                 return ret;
3997         __mem_cgroup_commit_charge(memcg, page, nr_pages, ctype, false);
3998         return 0;
3999 }
4000 
4001 int mem_cgroup_newpage_charge(struct page *page,
4002                               struct mm_struct *mm, gfp_t gfp_mask)
4003 {
4004         if (mem_cgroup_disabled())
4005                 return 0;
4006         VM_BUG_ON(page_mapped(page));
4007         VM_BUG_ON(page->mapping && !PageAnon(page));
4008         VM_BUG_ON(!mm);
4009         return mem_cgroup_charge_common(page, mm, gfp_mask,
4010                                         MEM_CGROUP_CHARGE_TYPE_ANON);
4011 }
4012 
4013 /*
4014  * While swap-in, try_charge -> commit or cancel, the page is locked.
4015  * And when try_charge() successfully returns, one refcnt to memcg without
4016  * struct page_cgroup is acquired. This refcnt will be consumed by
4017  * "commit()" or removed by "cancel()"
4018  */
4019 static int __mem_cgroup_try_charge_swapin(struct mm_struct *mm,
4020                                           struct page *page,
4021                                           gfp_t mask,
4022                                           struct mem_cgroup **memcgp)
4023 {
4024         struct mem_cgroup *memcg;
4025         struct page_cgroup *pc;
4026         int ret;
4027 
4028         pc = lookup_page_cgroup(page);
4029         /*
4030          * Every swap fault against a single page tries to charge the
4031          * page, bail as early as possible.  shmem_unuse() encounters
4032          * already charged pages, too.  The USED bit is protected by
4033          * the page lock, which serializes swap cache removal, which
4034          * in turn serializes uncharging.
4035          */
4036         if (PageCgroupUsed(pc))
4037                 return 0;
4038         if (!do_swap_account)
4039                 goto charge_cur_mm;
4040         memcg = try_get_mem_cgroup_from_page(page);
4041         if (!memcg)
4042                 goto charge_cur_mm;
4043         *memcgp = memcg;
4044         ret = __mem_cgroup_try_charge(NULL, mask, 1, memcgp, true);
4045         css_put(&memcg->css);
4046         if (ret == -EINTR)
4047                 ret = 0;
4048         return ret;
4049 charge_cur_mm:
4050         ret = __mem_cgroup_try_charge(mm, mask, 1, memcgp, true);
4051         if (ret == -EINTR)
4052                 ret = 0;
4053         return ret;
4054 }
4055 
4056 int mem_cgroup_try_charge_swapin(struct mm_struct *mm, struct page *page,
4057                                  gfp_t gfp_mask, struct mem_cgroup **memcgp)
4058 {
4059         *memcgp = NULL;
4060         if (mem_cgroup_disabled())
4061                 return 0;
4062         /*
4063          * A racing thread's fault, or swapoff, may have already
4064          * updated the pte, and even removed page from swap cache: in
4065          * those cases unuse_pte()'s pte_same() test will fail; but
4066          * there's also a KSM case which does need to charge the page.
4067          */
4068         if (!PageSwapCache(page)) {
4069                 int ret;
4070 
4071                 ret = __mem_cgroup_try_charge(mm, gfp_mask, 1, memcgp, true);
4072                 if (ret == -EINTR)
4073                         ret = 0;
4074                 return ret;
4075         }
4076         return __mem_cgroup_try_charge_swapin(mm, page, gfp_mask, memcgp);
4077 }
4078 
4079 void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *memcg)
4080 {
4081         if (mem_cgroup_disabled())
4082                 return;
4083         if (!memcg)
4084                 return;
4085         __mem_cgroup_cancel_charge(memcg, 1);
4086 }
4087 
4088 static void
4089 __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *memcg,
4090                                         enum charge_type ctype)
4091 {
4092         if (mem_cgroup_disabled())
4093                 return;
4094         if (!memcg)
4095                 return;
4096 
4097         __mem_cgroup_commit_charge(memcg, page, 1, ctype, true);
4098         /*
4099          * Now swap is on-memory. This means this page may be
4100          * counted both as mem and swap....double count.
4101          * Fix it by uncharging from memsw. Basically, this SwapCache is stable
4102          * under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page()
4103          * may call delete_from_swap_cache() before reach here.
4104          */
4105         if (do_swap_account && PageSwapCache(page)) {
4106                 swp_entry_t ent = {.val = page_private(page)};
4107                 mem_cgroup_uncharge_swap(ent);
4108         }
4109 }
4110 
4111 void mem_cgroup_commit_charge_swapin(struct page *page,
4112                                      struct mem_cgroup *memcg)
4113 {
4114         __mem_cgroup_commit_charge_swapin(page, memcg,
4115                                           MEM_CGROUP_CHARGE_TYPE_ANON);
4116 }
4117 
4118 int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
4119                                 gfp_t gfp_mask)
4120 {
4121         struct mem_cgroup *memcg = NULL;
4122         enum charge_type type = MEM_CGROUP_CHARGE_TYPE_CACHE;
4123         int ret;
4124 
4125         if (mem_cgroup_disabled())
4126                 return 0;
4127         if (PageCompound(page))
4128                 return 0;
4129 
4130         if (!PageSwapCache(page))
4131                 ret = mem_cgroup_charge_common(page, mm, gfp_mask, type);
4132         else { /* page is swapcache/shmem */
4133                 ret = __mem_cgroup_try_charge_swapin(mm, page,
4134                                                      gfp_mask, &memcg);
4135                 if (!ret)
4136                         __mem_cgroup_commit_charge_swapin(page, memcg, type);
4137         }
4138         return ret;
4139 }
4140 
4141 static void mem_cgroup_do_uncharge(struct mem_cgroup *memcg,
4142                                    unsigned int nr_pages,
4143                                    const enum charge_type ctype)
4144 {
4145         struct memcg_batch_info *batch = NULL;
4146         bool uncharge_memsw = true;
4147 
4148         /* If swapout, usage of swap doesn't decrease */
4149         if (!do_swap_account || ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
4150                 uncharge_memsw = false;
4151 
4152         batch = &current->memcg_batch;
4153         /*
4154          * In usual, we do css_get() when we remember memcg pointer.
4155          * But in this case, we keep res->usage until end of a series of
4156          * uncharges. Then, it's ok to ignore memcg's refcnt.
4157          */
4158         if (!batch->memcg)
4159                 batch->memcg = memcg;
4160         /*
4161          * do_batch > 0 when unmapping pages or inode invalidate/truncate.
4162          * In those cases, all pages freed continuously can be expected to be in
4163          * the same cgroup and we have chance to coalesce uncharges.
4164          * But we do uncharge one by one if this is killed by OOM(TIF_MEMDIE)
4165          * because we want to do uncharge as soon as possible.
4166          */
4167 
4168         if (!batch->do_batch || test_thread_flag(TIF_MEMDIE))
4169                 goto direct_uncharge;
4170 
4171         if (nr_pages > 1)
4172                 goto direct_uncharge;
4173 
4174         /*
4175          * In typical case, batch->memcg == mem. This means we can
4176          * merge a series of uncharges to an uncharge of res_counter.
4177          * If not, we uncharge res_counter ony by one.
4178          */
4179         if (batch->memcg != memcg)
4180                 goto direct_uncharge;
4181         /* remember freed charge and uncharge it later */
4182         batch->nr_pages++;
4183         if (uncharge_memsw)
4184                 batch->memsw_nr_pages++;
4185         return;
4186 direct_uncharge:
4187         res_counter_uncharge(&memcg->res, nr_pages * PAGE_SIZE);
4188         if (uncharge_memsw)
4189                 res_counter_uncharge(&memcg->memsw, nr_pages * PAGE_SIZE);
4190         if (unlikely(batch->memcg != memcg))
4191                 memcg_oom_recover(memcg);
4192 }
4193 
4194 /*
4195  * uncharge if !page_mapped(page)
4196  */
4197 static struct mem_cgroup *
4198 __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype,
4199                              bool end_migration)
4200 {
4201         struct mem_cgroup *memcg = NULL;
4202         unsigned int nr_pages = 1;
4203         struct page_cgroup *pc;
4204         bool anon;
4205 
4206         if (mem_cgroup_disabled())
4207                 return NULL;
4208 
4209         if (PageTransHuge(page)) {
4210                 nr_pages <<= compound_order(page);
4211                 VM_BUG_ON(!PageTransHuge(page));
4212         }
4213         /*
4214          * Check if our page_cgroup is valid
4215          */
4216         pc = lookup_page_cgroup(page);
4217         if (unlikely(!PageCgroupUsed(pc)))
4218                 return NULL;
4219 
4220         lock_page_cgroup(pc);
4221 
4222         memcg = pc->mem_cgroup;
4223 
4224         if (!PageCgroupUsed(pc))
4225                 goto unlock_out;
4226 
4227         anon = PageAnon(page);
4228 
4229         switch (ctype) {
4230         case MEM_CGROUP_CHARGE_TYPE_ANON:
4231                 /*
4232                  * Generally PageAnon tells if it's the anon statistics to be
4233                  * updated; but sometimes e.g. mem_cgroup_uncharge_page() is
4234                  * used before page reached the stage of being marked PageAnon.
4235                  */
4236                 anon = true;
4237                 /* fallthrough */
4238         case MEM_CGROUP_CHARGE_TYPE_DROP:
4239                 /* See mem_cgroup_prepare_migration() */
4240                 if (page_mapped(page))
4241                         goto unlock_out;
4242                 /*
4243                  * Pages under migration may not be uncharged.  But
4244                  * end_migration() /must/ be the one uncharging the
4245                  * unused post-migration page and so it has to call
4246                  * here with the migration bit still set.  See the
4247                  * res_counter handling below.
4248                  */
4249                 if (!end_migration && PageCgroupMigration(pc))
4250                         goto unlock_out;
4251                 break;
4252         case MEM_CGROUP_CHARGE_TYPE_SWAPOUT:
4253                 if (!PageAnon(page)) {  /* Shared memory */
4254                         if (page->mapping && !page_is_file_cache(page))
4255                                 goto unlock_out;
4256                 } else if (page_mapped(page)) /* Anon */
4257                                 goto unlock_out;
4258                 break;
4259         default:
4260                 break;
4261         }
4262 
4263         mem_cgroup_charge_statistics(memcg, page, anon, -nr_pages);
4264 
4265         ClearPageCgroupUsed(pc);
4266         /*
4267          * pc->mem_cgroup is not cleared here. It will be accessed when it's
4268          * freed from LRU. This is safe because uncharged page is expected not
4269          * to be reused (freed soon). Exception is SwapCache, it's handled by
4270          * special functions.
4271          */
4272 
4273         unlock_page_cgroup(pc);
4274         /*
4275          * even after unlock, we have memcg->res.usage here and this memcg
4276          * will never be freed, so it's safe to call css_get().
4277          */
4278         memcg_check_events(memcg, page);
4279         if (do_swap_account && ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) {
4280                 mem_cgroup_swap_statistics(memcg, true);
4281                 css_get(&memcg->css);
4282         }
4283         /*
4284          * Migration does not charge the res_counter for the
4285          * replacement page, so leave it alone when phasing out the
4286          * page that is unused after the migration.
4287          */
4288         if (!end_migration && !mem_cgroup_is_root(memcg))
4289                 mem_cgroup_do_uncharge(memcg, nr_pages, ctype);
4290 
4291         return memcg;
4292 
4293 unlock_out:
4294         unlock_page_cgroup(pc);
4295         return NULL;
4296 }
4297 
4298 void mem_cgroup_uncharge_page(struct page *page)
4299 {
4300         /* early check. */
4301         if (page_mapped(page))
4302                 return;
4303         VM_BUG_ON(page->mapping && !PageAnon(page));
4304         /*
4305          * If the page is in swap cache, uncharge should be deferred
4306          * to the swap path, which also properly accounts swap usage
4307          * and handles memcg lifetime.
4308          *
4309          * Note that this check is not stable and reclaim may add the
4310          * page to swap cache at any time after this.  However, if the
4311          * page is not in swap cache by the time page->mapcount hits
4312          * 0, there won't be any page table references to the swap
4313          * slot, and reclaim will free it and not actually write the
4314          * page to disk.
4315          */
4316         if (PageSwapCache(page))
4317                 return;
4318         __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_ANON, false);
4319 }
4320 
4321 void mem_cgroup_uncharge_cache_page(struct page *page)
4322 {
4323         VM_BUG_ON(page_mapped(page));
4324         VM_BUG_ON(page->mapping);
4325         __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE, false);
4326 }
4327 
4328 /*
4329  * Batch_start/batch_end is called in unmap_page_range/invlidate/trucate.
4330  * In that cases, pages are freed continuously and we can expect pages
4331  * are in the same memcg. All these calls itself limits the number of
4332  * pages freed at once, then uncharge_start/end() is called properly.
4333  * This may be called prural(2) times in a context,
4334  */
4335 
4336 void mem_cgroup_uncharge_start(void)
4337 {
4338         current->memcg_batch.do_batch++;
4339         /* We can do nest. */
4340         if (current->memcg_batch.do_batch == 1) {
4341                 current->memcg_batch.memcg = NULL;
4342                 current->memcg_batch.nr_pages = 0;
4343                 current->memcg_batch.memsw_nr_pages = 0;
4344         }
4345 }
4346 
4347 void mem_cgroup_uncharge_end(void)
4348 {
4349         struct memcg_batch_info *batch = &current->memcg_batch;
4350 
4351         if (!batch->do_batch)
4352                 return;
4353 
4354         batch->do_batch--;
4355         if (batch->do_batch) /* If stacked, do nothing. */
4356                 return;
4357 
4358         if (!batch->memcg)
4359                 return;
4360         /*
4361          * This "batch->memcg" is valid without any css_get/put etc...
4362          * bacause we hide charges behind us.
4363          */
4364         if (batch->nr_pages)
4365                 res_counter_uncharge(&batch->memcg->res,
4366                                      batch->nr_pages * PAGE_SIZE);
4367         if (batch->memsw_nr_pages)
4368                 res_counter_uncharge(&batch->memcg->memsw,
4369                                      batch->memsw_nr_pages * PAGE_SIZE);
4370         memcg_oom_recover(batch->memcg);
4371         /* forget this pointer (for sanity check) */
4372         batch->memcg = NULL;
4373 }
4374 
4375 #ifdef CONFIG_SWAP
4376 /*
4377  * called after __delete_from_swap_cache() and drop "page" account.
4378  * memcg information is recorded to swap_cgroup of "ent"
4379  */
4380 void
4381 mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout)
4382 {
4383         struct mem_cgroup *memcg;
4384         int ctype = MEM_CGROUP_CHARGE_TYPE_SWAPOUT;
4385 
4386         if (!swapout) /* this was a swap cache but the swap is unused ! */
4387                 ctype = MEM_CGROUP_CHARGE_TYPE_DROP;
4388 
4389         memcg = __mem_cgroup_uncharge_common(page, ctype, false);
4390 
4391         /*
4392          * record memcg information,  if swapout && memcg != NULL,
4393          * css_get() was called in uncharge().
4394          */
4395         if (do_swap_account && swapout && memcg)
4396                 swap_cgroup_record(ent, css_id(&memcg->css));
4397 }
4398 #endif
4399 
4400 #ifdef CONFIG_MEMCG_SWAP
4401 /*
4402  * called from swap_entry_free(). remove record in swap_cgroup and
4403  * uncharge "memsw" account.
4404  */
4405 void mem_cgroup_uncharge_swap(swp_entry_t ent)
4406 {
4407         struct mem_cgroup *memcg;
4408         unsigned short id;
4409 
4410         if (!do_swap_account)
4411                 return;
4412 
4413         id = swap_cgroup_record(ent, 0);
4414         rcu_read_lock();
4415         memcg = mem_cgroup_lookup(id);
4416         if (memcg) {
4417                 /*
4418                  * We uncharge this because swap is freed.
4419                  * This memcg can be obsolete one. We avoid calling css_tryget
4420                  */
4421                 if (!mem_cgroup_is_root(memcg))
4422                         res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
4423                 mem_cgroup_swap_statistics(memcg, false);
4424                 css_put(&memcg->css);
4425         }
4426         rcu_read_unlock();
4427 }
4428 
4429 /**
4430  * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
4431  * @entry: swap entry to be moved
4432  * @from:  mem_cgroup which the entry is moved from
4433  * @to:  mem_cgroup which the entry is moved to
4434  *
4435  * It succeeds only when the swap_cgroup's record for this entry is the same
4436  * as the mem_cgroup's id of @from.
4437  *
4438  * Returns 0 on success, -EINVAL on failure.
4439  *
4440  * The caller must have charged to @to, IOW, called res_counter_charge() about
4441  * both res and memsw, and called css_get().
4442  */
4443 static int mem_cgroup_move_swap_account(swp_entry_t entry,
4444                                 struct mem_cgroup *from, struct mem_cgroup *to)
4445 {
4446         unsigned short old_id, new_id;
4447 
4448         old_id = css_id(&from->css);
4449         new_id = css_id(&to->css);
4450 
4451         if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
4452                 mem_cgroup_swap_statistics(from, false);
4453                 mem_cgroup_swap_statistics(to, true);
4454                 /*
4455                  * This function is only called from task migration context now.
4456                  * It postpones res_counter and refcount handling till the end
4457                  * of task migration(mem_cgroup_clear_mc()) for performance
4458                  * improvement. But we cannot postpone css_get(to)  because if
4459                  * the process that has been moved to @to does swap-in, the
4460                  * refcount of @to might be decreased to 0.
4461                  *
4462                  * We are in attach() phase, so the cgroup is guaranteed to be
4463                  * alive, so we can just call css_get().
4464                  */
4465                 css_get(&to->css);
4466                 return 0;
4467         }
4468         return -EINVAL;
4469 }
4470 #else
4471 static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
4472                                 struct mem_cgroup *from, struct mem_cgroup *to)
4473 {
4474         return -EINVAL;
4475 }
4476 #endif
4477 
4478 /*
4479  * Before starting migration, account PAGE_SIZE to mem_cgroup that the old
4480  * page belongs to.
4481  */
4482 void mem_cgroup_prepare_migration(struct page *page, struct page *newpage,
4483                                   struct mem_cgroup **memcgp)
4484 {
4485         struct mem_cgroup *memcg = NULL;
4486         unsigned int nr_pages = 1;
4487         struct page_cgroup *pc;
4488         enum charge_type ctype;
4489 
4490         *memcgp = NULL;
4491 
4492         if (mem_cgroup_disabled())
4493                 return;
4494 
4495         if (PageTransHuge(page))
4496                 nr_pages <<= compound_order(page);
4497 
4498         pc = lookup_page_cgroup(page);
4499         lock_page_cgroup(pc);
4500         if (PageCgroupUsed(pc)) {
4501                 memcg = pc->mem_cgroup;
4502                 css_get(&memcg->css);
4503                 /*
4504                  * At migrating an anonymous page, its mapcount goes down
4505                  * to 0 and uncharge() will be called. But, even if it's fully
4506                  * unmapped, migration may fail and this page has to be
4507                  * charged again. We set MIGRATION flag here and delay uncharge
4508                  * until end_migration() is called
4509                  *
4510                  * Corner Case Thinking
4511                  * A)
4512                  * When the old page was mapped as Anon and it's unmap-and-freed
4513                  * while migration was ongoing.
4514                  * If unmap finds the old page, uncharge() of it will be delayed
4515                  * until end_migration(). If unmap finds a new page, it's
4516                  * uncharged when it make mapcount to be 1->0. If unmap code
4517                  * finds swap_migration_entry, the new page will not be mapped
4518                  * and end_migration() will find it(mapcount==0).
4519                  *
4520                  * B)
4521                  * When the old page was mapped but migraion fails, the kernel
4522                  * remaps it. A charge for it is kept by MIGRATION flag even
4523                  * if mapcount goes down to 0. We can do remap successfully
4524                  * without charging it again.
4525                  *
4526                  * C)
4527                  * The "old" page is under lock_page() until the end of
4528                  * migration, so, the old page itself will not be swapped-out.
4529                  * If the new page is swapped out before end_migraton, our
4530                  * hook to usual swap-out path will catch the event.
4531                  */
4532                 if (PageAnon(page))
4533                         SetPageCgroupMigration(pc);
4534         }
4535         unlock_page_cgroup(pc);
4536         /*
4537          * If the page is not charged at this point,
4538          * we return here.
4539          */
4540         if (!memcg)
4541                 return;
4542 
4543         *memcgp = memcg;
4544         /*
4545          * We charge new page before it's used/mapped. So, even if unlock_page()
4546          * is called before end_migration, we can catch all events on this new
4547          * page. In the case new page is migrated but not remapped, new page's
4548          * mapcount will be finally 0 and we call uncharge in end_migration().
4549          */
4550         if (PageAnon(page))
4551                 ctype = MEM_CGROUP_CHARGE_TYPE_ANON;
4552         else
4553                 ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
4554         /*
4555          * The page is committed to the memcg, but it's not actually
4556          * charged to the res_counter since we plan on replacing the
4557          * old one and only one page is going to be left afterwards.
4558          */
4559         __mem_cgroup_commit_charge(memcg, newpage, nr_pages, ctype, false);
4560 }
4561 
4562 /* remove redundant charge if migration failed*/
4563 void mem_cgroup_end_migration(struct mem_cgroup *memcg,
4564         struct page *oldpage, struct page *newpage, bool migration_ok)
4565 {
4566         struct page *used, *unused;
4567         struct page_cgroup *pc;
4568         bool anon;
4569 
4570         if (!memcg)
4571                 return;
4572 
4573         if (!migration_ok) {
4574                 used = oldpage;
4575                 unused = newpage;
4576         } else {
4577                 used = newpage;
4578                 unused = oldpage;
4579         }
4580         anon = PageAnon(used);
4581         __mem_cgroup_uncharge_common(unused,
4582                                      anon ? MEM_CGROUP_CHARGE_TYPE_ANON
4583                                      : MEM_CGROUP_CHARGE_TYPE_CACHE,
4584                                      true);
4585         css_put(&memcg->css);
4586         /*
4587          * We disallowed uncharge of pages under migration because mapcount
4588          * of the page goes down to zero, temporarly.
4589          * Clear the flag and check the page should be charged.
4590          */
4591         pc = lookup_page_cgroup(oldpage);
4592         lock_page_cgroup(pc);
4593         ClearPageCgroupMigration(pc);
4594         unlock_page_cgroup(pc);
4595 
4596         /*
4597          * If a page is a file cache, radix-tree replacement is very atomic
4598          * and we can skip this check. When it was an Anon page, its mapcount
4599          * goes down to 0. But because we added MIGRATION flage, it's not
4600          * uncharged yet. There are several case but page->mapcount check
4601          * and USED bit check in mem_cgroup_uncharge_page() will do enough
4602          * check. (see prepare_charge() also)
4603          */
4604         if (anon)
4605                 mem_cgroup_uncharge_page(used);
4606 }
4607 
4608 /*
4609  * At replace page cache, newpage is not under any memcg but it's on
4610  * LRU. So, this function doesn't touch res_counter but handles LRU
4611  * in correct way. Both pages are locked so we cannot race with uncharge.
4612  */
4613 void mem_cgroup_replace_page_cache(struct page *oldpage,
4614                                   struct page *newpage)
4615 {
4616         struct mem_cgroup *memcg = NULL;
4617         struct page_cgroup *pc;
4618         enum charge_type type = MEM_CGROUP_CHARGE_TYPE_CACHE;
4619 
4620         if (mem_cgroup_disabled())
4621                 return;
4622 
4623         pc = lookup_page_cgroup(oldpage);
4624         /* fix accounting on old pages */
4625         lock_page_cgroup(pc);
4626         if (PageCgroupUsed(pc)) {
4627                 memcg = pc->mem_cgroup;
4628                 mem_cgroup_charge_statistics(memcg, oldpage, false, -1);
4629                 ClearPageCgroupUsed(pc);
4630         }
4631         unlock_page_cgroup(pc);
4632 
4633         /*
4634          * When called from shmem_replace_page(), in some cases the
4635          * oldpage has already been charged, and in some cases not.
4636          */
4637         if (!memcg)
4638                 return;
4639         /*
4640          * Even if newpage->mapping was NULL before starting replacement,
4641          * the newpage may be on LRU(or pagevec for LRU) already. We lock
4642          * LRU while we overwrite pc->mem_cgroup.
4643          */
4644         __mem_cgroup_commit_charge(memcg, newpage, 1, type, true);
4645 }
4646 
4647 #ifdef CONFIG_DEBUG_VM
4648 static struct page_cgroup *lookup_page_cgroup_used(struct page *page)
4649 {
4650         struct page_cgroup *pc;
4651 
4652         pc = lookup_page_cgroup(page);
4653         /*
4654          * Can be NULL while feeding pages into the page allocator for
4655          * the first time, i.e. during boot or memory hotplug;
4656          * or when mem_cgroup_disabled().
4657          */
4658         if (likely(pc) && PageCgroupUsed(pc))
4659                 return pc;
4660         return NULL;
4661 }
4662 
4663 bool mem_cgroup_bad_page_check(struct page *page)
4664 {
4665         if (mem_cgroup_disabled())
4666                 return false;
4667 
4668         return lookup_page_cgroup_used(page) != NULL;
4669 }
4670 
4671 void mem_cgroup_print_bad_page(struct page *page)
4672 {
4673         struct page_cgroup *pc;
4674 
4675         pc = lookup_page_cgroup_used(page);
4676         if (pc) {
4677                 pr_alert("pc:%p pc->flags:%lx pc->mem_cgroup:%p\n",
4678                          pc, pc->flags, pc->mem_cgroup);
4679         }
4680 }
4681 #endif
4682 
4683 static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
4684                                 unsigned long long val)
4685 {
4686         int retry_count;
4687         u64 memswlimit, memlimit;
4688         int ret = 0;
4689         int children = mem_cgroup_count_children(memcg);
4690         u64 curusage, oldusage;
4691         int enlarge;
4692 
4693         /*
4694          * For keeping hierarchical_reclaim simple, how long we should retry
4695          * is depends on callers. We set our retry-count to be function
4696          * of # of children which we should visit in this loop.
4697          */
4698         retry_count = MEM_CGROUP_RECLAIM_RETRIES * children;
4699 
4700         oldusage = res_counter_read_u64(&memcg->res, RES_USAGE);
4701 
4702         enlarge = 0;
4703         while (retry_count) {
4704                 if (signal_pending(current)) {
4705                         ret = -EINTR;
4706                         break;
4707                 }
4708                 /*
4709                  * Rather than hide all in some function, I do this in
4710                  * open coded manner. You see what this really does.
4711                  * We have to guarantee memcg->res.limit <= memcg->memsw.limit.
4712                  */
4713                 mutex_lock(&set_limit_mutex);
4714                 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
4715                 if (memswlimit < val) {
4716                         ret = -EINVAL;
4717                         mutex_unlock(&set_limit_mutex);
4718                         break;
4719                 }
4720 
4721                 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
4722                 if (memlimit < val)
4723                         enlarge = 1;
4724 
4725                 ret = res_counter_set_limit(&memcg->res, val);
4726                 if (!ret) {
4727                         if (memswlimit == val)
4728                                 memcg->memsw_is_minimum = true;
4729                         else
4730                                 memcg->memsw_is_minimum = false;
4731                 }
4732                 mutex_unlock(&set_limit_mutex);
4733 
4734                 if (!ret)
4735                         break;
4736 
4737                 mem_cgroup_reclaim(memcg, GFP_KERNEL,
4738                                    MEM_CGROUP_RECLAIM_SHRINK);
4739                 curusage = res_counter_read_u64(&memcg->res, RES_USAGE);
4740                 /* Usage is reduced ? */
4741                 if (curusage >= oldusage)
4742                         retry_count--;
4743                 else
4744                         oldusage = curusage;
4745         }
4746         if (!ret && enlarge)
4747                 memcg_oom_recover(memcg);
4748 
4749         return ret;
4750 }
4751 
4752 static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
4753                                         unsigned long long val)
4754 {
4755         int retry_count;
4756         u64 memlimit, memswlimit, oldusage, curusage;
4757         int children = mem_cgroup_count_children(memcg);
4758         int ret = -EBUSY;
4759         int enlarge = 0;
4760 
4761         /* see mem_cgroup_resize_res_limit */
4762         retry_count = children * MEM_CGROUP_RECLAIM_RETRIES;
4763         oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
4764         while (retry_count) {
4765                 if (signal_pending(current)) {
4766                         ret = -EINTR;
4767                         break;
4768                 }
4769                 /*
4770                  * Rather than hide all in some function, I do this in
4771                  * open coded manner. You see what this really does.
4772                  * We have to guarantee memcg->res.limit <= memcg->memsw.limit.
4773                  */
4774                 mutex_lock(&set_limit_mutex);
4775                 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
4776                 if (memlimit > val) {
4777                         ret = -EINVAL;
4778                         mutex_unlock(&set_limit_mutex);
4779                         break;
4780                 }
4781                 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
4782                 if (memswlimit < val)
4783                         enlarge = 1;
4784                 ret = res_counter_set_limit(&memcg->memsw, val);
4785                 if (!ret) {
4786                         if (memlimit == val)
4787                                 memcg->memsw_is_minimum = true;
4788                         else
4789                                 memcg->memsw_is_minimum = false;
4790                 }
4791                 mutex_unlock(&set_limit_mutex);
4792 
4793                 if (!ret)
4794                         break;
4795 
4796                 mem_cgroup_reclaim(memcg, GFP_KERNEL,
4797                                    MEM_CGROUP_RECLAIM_NOSWAP |
4798                                    MEM_CGROUP_RECLAIM_SHRINK);
4799                 curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
4800                 /* Usage is reduced ? */
4801                 if (curusage >= oldusage)
4802                         retry_count--;
4803                 else
4804                         oldusage = curusage;
4805         }
4806         if (!ret && enlarge)
4807                 memcg_oom_recover(memcg);
4808         return ret;
4809 }
4810 
4811 unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
4812                                             gfp_t gfp_mask,
4813                                             unsigned long *total_scanned)
4814 {
4815         unsigned long nr_reclaimed = 0;
4816         struct mem_cgroup_per_zone *mz, *next_mz = NULL;
4817         unsigned long reclaimed;
4818         int loop = 0;
4819         struct mem_cgroup_tree_per_zone *mctz;
4820         unsigned long long excess;
4821         unsigned long nr_scanned;
4822 
4823         if (order > 0)
4824                 return 0;
4825 
4826         mctz = soft_limit_tree_node_zone(zone_to_nid(zone), zone_idx(zone));
4827         /*
4828          * This loop can run a while, specially if mem_cgroup's continuously
4829          * keep exceeding their soft limit and putting the system under
4830          * pressure
4831          */
4832         do {
4833                 if (next_mz)
4834                         mz = next_mz;
4835                 else
4836                         mz = mem_cgroup_largest_soft_limit_node(mctz);
4837                 if (!mz)
4838                         break;
4839 
4840                 nr_scanned = 0;
4841                 reclaimed = mem_cgroup_soft_reclaim(mz->memcg, zone,
4842                                                     gfp_mask, &nr_scanned);
4843                 nr_reclaimed += reclaimed;
4844                 *total_scanned += nr_scanned;
4845                 spin_lock(&mctz->lock);
4846 
4847                 /*
4848                  * If we failed to reclaim anything from this memory cgroup
4849                  * it is time to move on to the next cgroup
4850                  */
4851                 next_mz = NULL;
4852                 if (!reclaimed) {
4853                         do {
4854                                 /*
4855                                  * Loop until we find yet another one.
4856                                  *
4857                                  * By the time we get the soft_limit lock
4858                                  * again, someone might have aded the
4859                                  * group back on the RB tree. Iterate to
4860                                  * make sure we get a different mem.
4861                                  * mem_cgroup_largest_soft_limit_node returns
4862                                  * NULL if no other cgroup is present on
4863                                  * the tree
4864                                  */
4865                                 next_mz =
4866                                 __mem_cgroup_largest_soft_limit_node(mctz);
4867                                 if (next_mz == mz)
4868                                         css_put(&next_mz->memcg->css);
4869                                 else /* next_mz == NULL or other memcg */
4870                                         break;
4871                         } while (1);
4872                 }
4873                 __mem_cgroup_remove_exceeded(mz->memcg, mz, mctz);
4874                 excess = res_counter_soft_limit_excess(&mz->memcg->res);
4875                 /*
4876                  * One school of thought says that we should not add
4877                  * back the node to the tree if reclaim returns 0.
4878                  * But our reclaim could return 0, simply because due
4879                  * to priority we are exposing a smaller subset of
4880                  * memory to reclaim from. Consider this as a longer
4881                  * term TODO.
4882                  */
4883                 /* If excess == 0, no tree ops */
4884                 __mem_cgroup_insert_exceeded(mz->memcg, mz, mctz, excess);
4885                 spin_unlock(&mctz->lock);
4886                 css_put(&mz->memcg->css);
4887                 loop++;
4888                 /*
4889                  * Could not reclaim anything and there are no more
4890                  * mem cgroups to try or we seem to be looping without
4891                  * reclaiming anything.
4892                  */
4893                 if (!nr_reclaimed &&
4894                         (next_mz == NULL ||
4895                         loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
4896                         break;
4897         } while (!nr_reclaimed);
4898         if (next_mz)
4899                 css_put(&next_mz->memcg->css);
4900         return nr_reclaimed;
4901 }
4902 
4903 /**
4904  * mem_cgroup_force_empty_list - clears LRU of a group
4905  * @memcg: group to clear
4906  * @node: NUMA node
4907  * @zid: zone id
4908  * @lru: lru to to clear
4909  *
4910  * Traverse a specified page_cgroup list and try to drop them all.  This doesn't
4911  * reclaim the pages page themselves - pages are moved to the parent (or root)
4912  * group.
4913  */
4914 static void mem_cgroup_force_empty_list(struct mem_cgroup *memcg,
4915                                 int node, int zid, enum lru_list lru)
4916 {
4917         struct lruvec *lruvec;
4918         unsigned long flags;
4919         struct list_head *list;
4920         struct page *busy;
4921         struct zone *zone;
4922 
4923         zone = &NODE_DATA(node)->node_zones[zid];
4924         lruvec = mem_cgroup_zone_lruvec(zone, memcg);
4925         list = &lruvec->lists[lru];
4926 
4927         busy = NULL;
4928         do {
4929                 struct page_cgroup *pc;
4930                 struct page *page;
4931 
4932                 spin_lock_irqsave(&zone->lru_lock, flags);
4933                 if (list_empty(list)) {
4934                         spin_unlock_irqrestore(&zone->lru_lock, flags);
4935                         break;
4936                 }
4937                 page = list_entry(list->prev, struct page, lru);
4938                 if (busy == page) {
4939                         list_move(&page->lru, list);
4940                         busy = NULL;
4941                         spin_unlock_irqrestore(&zone->lru_lock, flags);
4942                         continue;
4943                 }
4944                 spin_unlock_irqrestore(&zone->lru_lock, flags);
4945 
4946                 pc = lookup_page_cgroup(page);
4947 
4948                 if (mem_cgroup_move_parent(page, pc, memcg)) {
4949                         /* found lock contention or "pc" is obsolete. */
4950                         busy = page;
4951                         cond_resched();
4952                 } else
4953                         busy = NULL;
4954         } while (!list_empty(list));
4955 }
4956 
4957 /*
4958  * make mem_cgroup's charge to be 0 if there is no task by moving
4959  * all the charges and pages to the parent.
4960  * This enables deleting this mem_cgroup.
4961  *
4962  * Caller is responsible for holding css reference on the memcg.
4963  */
4964 static void mem_cgroup_reparent_charges(struct mem_cgroup *memcg)
4965 {
4966         int node, zid;
4967         u64 usage;
4968 
4969         do {
4970                 /* This is for making all *used* pages to be on LRU. */
4971                 lru_add_drain_all();
4972                 drain_all_stock_sync(memcg);
4973                 mem_cgroup_start_move(memcg);
4974                 for_each_node_state(node, N_MEMORY) {
4975                         for (zid = 0; zid < MAX_NR_ZONES; zid++) {
4976                                 enum lru_list lru;
4977                                 for_each_lru(lru) {
4978                                         mem_cgroup_force_empty_list(memcg,
4979                                                         node, zid, lru);
4980                                 }
4981                         }
4982                 }
4983                 mem_cgroup_end_move(memcg);
4984                 memcg_oom_recover(memcg);
4985                 cond_resched();
4986 
4987                 /*
4988                  * Kernel memory may not necessarily be trackable to a specific
4989                  * process. So they are not migrated, and therefore we can't
4990                  * expect their value to drop to 0 here.
4991                  * Having res filled up with kmem only is enough.
4992                  *
4993                  * This is a safety check because mem_cgroup_force_empty_list
4994                  * could have raced with mem_cgroup_replace_page_cache callers
4995                  * so the lru seemed empty but the page could have been added
4996                  * right after the check. RES_USAGE should be safe as we always
4997                  * charge before adding to the LRU.
4998                  */
4999                 usage = res_counter_read_u64(&memcg->res, RES_USAGE) -
5000                         res_counter_read_u64(&memcg->kmem, RES_USAGE);
5001         } while (usage > 0);
5002 }
5003 
5004 static inline bool memcg_has_children(struct mem_cgroup *memcg)
5005 {
5006         lockdep_assert_held(&memcg_create_mutex);
5007         /*
5008          * The lock does not prevent addition or deletion to the list
5009          * of children, but it prevents a new child from being
5010          * initialized based on this parent in css_online(), so it's
5011          * enough to decide whether hierarchically inherited
5012          * attributes can still be changed or not.
5013          */
5014         return memcg->use_hierarchy &&
5015                 !list_empty(&memcg->css.cgroup->children);
5016 }
5017 
5018 /*
5019  * Reclaims as many pages from the given memcg as possible and moves
5020  * the rest to the parent.
5021  *
5022  * Caller is responsible for holding css reference for memcg.
5023  */
5024 static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
5025 {
5026         int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
5027         struct cgroup *cgrp = memcg->css.cgroup;
5028 
5029         /* returns EBUSY if there is a task or if we come here twice. */
5030         if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children))
5031                 return -EBUSY;
5032 
5033         /* we call try-to-free pages for make this cgroup empty */
5034         lru_add_drain_all();
5035         /* try to free all pages in this cgroup */
5036         while (nr_retries && res_counter_read_u64(&memcg->res, RES_USAGE) > 0) {
5037                 int progress;
5038 
5039                 if (signal_pending(current))
5040                         return -EINTR;
5041 
5042                 progress = try_to_free_mem_cgroup_pages(memcg, GFP_KERNEL,
5043                                                 false);
5044                 if (!progress) {
5045                         nr_retries--;
5046                         /* maybe some writeback is necessary */
5047                         congestion_wait(BLK_RW_ASYNC, HZ/10);
5048                 }
5049 
5050         }
5051         lru_add_drain();
5052         mem_cgroup_reparent_charges(memcg);
5053 
5054         return 0;
5055 }
5056 
5057 static int mem_cgroup_force_empty_write(struct cgroup_subsys_state *css,
5058                                         unsigned int event)
5059 {
5060         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5061 
5062         if (mem_cgroup_is_root(memcg))
5063                 return -EINVAL;
5064         return mem_cgroup_force_empty(memcg);
5065 }
5066 
5067 static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css,
5068                                      struct cftype *cft)
5069 {
5070         return mem_cgroup_from_css(css)->use_hierarchy;
5071 }
5072 
5073 static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
5074                                       struct cftype *cft, u64 val)
5075 {
5076         int retval = 0;
5077         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5078         struct mem_cgroup *parent_memcg = mem_cgroup_from_css(css_parent(&memcg->css));
5079 
5080         mutex_lock(&memcg_create_mutex);
5081 
5082         if (memcg->use_hierarchy == val)
5083                 goto out;
5084 
5085         /*
5086          * If parent's use_hierarchy is set, we can't make any modifications
5087          * in the child subtrees. If it is unset, then the change can
5088          * occur, provided the current cgroup has no children.
5089          *
5090          * For the root cgroup, parent_mem is NULL, we allow value to be
5091          * set if there are no children.
5092          */
5093         if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
5094                                 (val == 1 || val == 0)) {
5095                 if (list_empty(&memcg->css.cgroup->children))
5096                         memcg->use_hierarchy = val;
5097                 else
5098                         retval = -EBUSY;
5099         } else
5100                 retval = -EINVAL;
5101 
5102 out:
5103         mutex_unlock(&memcg_create_mutex);
5104 
5105         return retval;
5106 }
5107 
5108 
5109 static unsigned long mem_cgroup_recursive_stat(struct mem_cgroup *memcg,
5110                                                enum mem_cgroup_stat_index idx)
5111 {
5112         struct mem_cgroup *iter;
5113         long val = 0;
5114 
5115         /* Per-cpu values can be negative, use a signed accumulator */
5116         for_each_mem_cgroup_tree(iter, memcg)
5117                 val += mem_cgroup_read_stat(iter, idx);
5118 
5119         if (val < 0) /* race ? */
5120                 val = 0;
5121         return val;
5122 }
5123 
5124 static inline u64 mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
5125 {
5126         u64 val;
5127 
5128         if (!mem_cgroup_is_root(memcg)) {
5129                 if (!swap)
5130                         return res_counter_read_u64(&memcg->res, RES_USAGE);
5131                 else
5132                         return res_counter_read_u64(&memcg->memsw, RES_USAGE);
5133         }
5134 
5135         /*
5136          * Transparent hugepages are still accounted for in MEM_CGROUP_STAT_RSS
5137          * as well as in MEM_CGROUP_STAT_RSS_HUGE.
5138          */
5139         val = mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_CACHE);
5140         val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_RSS);
5141 
5142         if (swap)
5143                 val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_SWAP);
5144 
5145         return val << PAGE_SHIFT;
5146 }
5147 
5148 static ssize_t mem_cgroup_read(struct cgroup_subsys_state *css,
5149                                struct cftype *cft, struct file *file,
5150                                char __user *buf, size_t nbytes, loff_t *ppos)
5151 {
5152         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5153         char str[64];
5154         u64 val;
5155         int name, len;
5156         enum res_type type;
5157 
5158         type = MEMFILE_TYPE(cft->private);
5159         name = MEMFILE_ATTR(cft->private);
5160 
5161         switch (type) {
5162         case _MEM:
5163                 if (name == RES_USAGE)
5164                         val = mem_cgroup_usage(memcg, false);
5165                 else
5166                         val = res_counter_read_u64(&memcg->res, name);
5167                 break;
5168         case _MEMSWAP:
5169                 if (name == RES_USAGE)
5170                         val = mem_cgroup_usage(memcg, true);
5171                 else
5172                         val = res_counter_read_u64(&memcg->memsw, name);
5173                 break;
5174         case _KMEM:
5175                 val = res_counter_read_u64(&memcg->kmem, name);
5176                 break;
5177         default:
5178                 BUG();
5179         }
5180 
5181         len = scnprintf(str, sizeof(str), "%llu\n", (unsigned long long)val);
5182         return simple_read_from_buffer(buf, nbytes, ppos, str, len);
5183 }
5184 
5185 static int memcg_update_kmem_limit(struct cgroup_subsys_state *css, u64 val)
5186 {
5187         int ret = -EINVAL;
5188 #ifdef CONFIG_MEMCG_KMEM
5189         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5190         /*
5191          * For simplicity, we won't allow this to be disabled.  It also can't
5192          * be changed if the cgroup has children already, or if tasks had
5193          * already joined.
5194          *
5195          * If tasks join before we set the limit, a person looking at
5196          * kmem.usage_in_bytes will have no way to determine when it took
5197          * place, which makes the value quite meaningless.
5198          *
5199          * After it first became limited, changes in the value of the limit are
5200          * of course permitted.
5201          */
5202         mutex_lock(&memcg_create_mutex);
5203         mutex_lock(&set_limit_mutex);
5204         if (!memcg->kmem_account_flags && val != RES_COUNTER_MAX) {
5205                 if (cgroup_task_count(css->cgroup) || memcg_has_children(memcg)) {
5206                         ret = -EBUSY;
5207                         goto out;
5208                 }
5209                 ret = res_counter_set_limit(&memcg->kmem, val);
5210                 VM_BUG_ON(ret);
5211 
5212                 ret = memcg_update_cache_sizes(memcg);
5213                 if (ret) {
5214                         res_counter_set_limit(&memcg->kmem, RES_COUNTER_MAX);
5215                         goto out;
5216                 }
5217                 static_key_slow_inc(&memcg_kmem_enabled_key);
5218                 /*
5219                  * setting the active bit after the inc will guarantee no one
5220                  * starts accounting before all call sites are patched
5221                  */
5222                 memcg_kmem_set_active(memcg);
5223         } else
5224                 ret = res_counter_set_limit(&memcg->kmem, val);
5225 out:
5226         mutex_unlock(&set_limit_mutex);
5227         mutex_unlock(&memcg_create_mutex);
5228 #endif
5229         return ret;
5230 }
5231 
5232 #ifdef CONFIG_MEMCG_KMEM
5233 static int memcg_propagate_kmem(struct mem_cgroup *memcg)
5234 {
5235         int ret = 0;
5236         struct mem_cgroup *parent = parent_mem_cgroup(memcg);
5237         if (!parent)
5238                 goto out;
5239 
5240         memcg->kmem_account_flags = parent->kmem_account_flags;
5241         /*
5242          * When that happen, we need to disable the static branch only on those
5243          * memcgs that enabled it. To achieve this, we would be forced to
5244          * complicate the code by keeping track of which memcgs were the ones
5245          * that actually enabled limits, and which ones got it from its
5246          * parents.
5247          *
5248          * It is a lot simpler just to do static_key_slow_inc() on every child
5249          * that is accounted.
5250          */
5251         if (!memcg_kmem_is_active(memcg))
5252                 goto out;
5253 
5254         /*
5255          * __mem_cgroup_free() will issue static_key_slow_dec() because this
5256          * memcg is active already. If the later initialization fails then the
5257          * cgroup core triggers the cleanup so we do not have to do it here.
5258          */
5259         static_key_slow_inc(&memcg_kmem_enabled_key);
5260 
5261         mutex_lock(&set_limit_mutex);
5262         memcg_stop_kmem_account();
5263         ret = memcg_update_cache_sizes(memcg);
5264         memcg_resume_kmem_account();
5265         mutex_unlock(&set_limit_mutex);
5266 out:
5267         return ret;
5268 }
5269 #endif /* CONFIG_MEMCG_KMEM */
5270 
5271 /*
5272  * The user of this function is...
5273  * RES_LIMIT.
5274  */
5275 static int mem_cgroup_write(struct cgroup_subsys_state *css, struct cftype *cft,
5276                             const char *buffer)
5277 {
5278         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5279         enum res_type type;
5280         int name;
5281         unsigned long long val;
5282         int ret;
5283 
5284         type = MEMFILE_TYPE(cft->private);
5285         name = MEMFILE_ATTR(cft->private);
5286 
5287         switch (name) {
5288         case RES_LIMIT:
5289                 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
5290                         ret = -EINVAL;
5291                         break;
5292                 }
5293                 /* This function does all necessary parse...reuse it */
5294                 ret = res_counter_memparse_write_strategy(buffer, &val);
5295                 if (ret)
5296                         break;
5297                 if (type == _MEM)
5298                         ret = mem_cgroup_resize_limit(memcg, val);
5299                 else if (type == _MEMSWAP)
5300                         ret = mem_cgroup_resize_memsw_limit(memcg, val);
5301                 else if (type == _KMEM)
5302                         ret = memcg_update_kmem_limit(css, val);
5303                 else
5304                         return -EINVAL;
5305                 break;
5306         case RES_SOFT_LIMIT:
5307                 ret = res_counter_memparse_write_strategy(buffer, &val);
5308                 if (ret)
5309                         break;
5310                 /*
5311                  * For memsw, soft limits are hard to implement in terms
5312                  * of semantics, for now, we support soft limits for
5313                  * control without swap
5314                  */
5315                 if (type == _MEM)
5316                         ret = res_counter_set_soft_limit(&memcg->res, val);
5317                 else
5318                         ret = -EINVAL;
5319                 break;
5320         default:
5321                 ret = -EINVAL; /* should be BUG() ? */
5322                 break;
5323         }
5324         return ret;
5325 }
5326 
5327 static void memcg_get_hierarchical_limit(struct mem_cgroup *memcg,
5328                 unsigned long long *mem_limit, unsigned long long *memsw_limit)
5329 {
5330         unsigned long long min_limit, min_memsw_limit, tmp;
5331 
5332         min_limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
5333         min_memsw_limit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
5334         if (!memcg->use_hierarchy)
5335                 goto out;
5336 
5337         while (css_parent(&memcg->css)) {
5338                 memcg = mem_cgroup_from_css(css_parent(&memcg->css));
5339                 if (!memcg->use_hierarchy)
5340                         break;
5341                 tmp = res_counter_read_u64(&memcg->res, RES_LIMIT);
5342                 min_limit = min(min_limit, tmp);
5343                 tmp = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
5344                 min_memsw_limit = min(min_memsw_limit, tmp);
5345         }
5346 out:
5347         *mem_limit = min_limit;
5348         *memsw_limit = min_memsw_limit;
5349 }
5350 
5351 static int mem_cgroup_reset(struct cgroup_subsys_state *css, unsigned int event)
5352 {
5353         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5354         int name;
5355         enum res_type type;
5356 
5357         type = MEMFILE_TYPE(event);
5358         name = MEMFILE_ATTR(event);
5359 
5360         switch (name) {
5361         case RES_MAX_USAGE:
5362                 if (type == _MEM)
5363                         res_counter_reset_max(&memcg->res);
5364                 else if (type == _MEMSWAP)
5365                         res_counter_reset_max(&memcg->memsw);
5366                 else if (type == _KMEM)
5367                         res_counter_reset_max(&memcg->kmem);
5368                 else
5369                         return -EINVAL;
5370                 break;
5371         case RES_FAILCNT:
5372                 if (type == _MEM)
5373                         res_counter_reset_failcnt(&memcg->res);
5374                 else if (type == _MEMSWAP)
5375                         res_counter_reset_failcnt(&memcg->memsw);
5376                 else if (type == _KMEM)
5377                         res_counter_reset_failcnt(&memcg->kmem);
5378                 else
5379                         return -EINVAL;
5380                 break;
5381         }
5382 
5383         return 0;
5384 }
5385 
5386 static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
5387                                         struct cftype *cft)
5388 {
5389         return mem_cgroup_from_css(css)->move_charge_at_immigrate;
5390 }
5391 
5392 #ifdef CONFIG_MMU
5393 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
5394                                         struct cftype *cft, u64 val)
5395 {
5396         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5397 
5398         if (val >= (1 << NR_MOVE_TYPE))
5399                 return -EINVAL;
5400 
5401         /*
5402          * No kind of locking is needed in here, because ->can_attach() will
5403          * check this value once in the beginning of the process, and then carry
5404          * on with stale data. This means that changes to this value will only
5405          * affect task migrations starting after the change.
5406          */
5407         memcg->move_charge_at_immigrate = val;
5408         return 0;
5409 }
5410 #else
5411 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
5412                                         struct cftype *cft, u64 val)
5413 {
5414         return -ENOSYS;
5415 }
5416 #endif
5417 
5418 #ifdef CONFIG_NUMA
5419 static int memcg_numa_stat_show(struct cgroup_subsys_state *css,
5420                                 struct cftype *cft, struct seq_file *m)
5421 {
5422         int nid;
5423         unsigned long total_nr, file_nr, anon_nr, unevictable_nr;
5424         unsigned long node_nr;
5425         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5426 
5427         total_nr = mem_cgroup_nr_lru_pages(memcg, LRU_ALL);
5428         seq_printf(m, "total=%lu", total_nr);
5429         for_each_node_state(nid, N_MEMORY) {
5430                 node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL);
5431                 seq_printf(m, " N%d=%lu", nid, node_nr);
5432         }
5433         seq_putc(m, '\n');
5434 
5435         file_nr = mem_cgroup_nr_lru_pages(memcg, LRU_ALL_FILE);
5436         seq_printf(m, "file=%lu", file_nr);
5437         for_each_node_state(nid, N_MEMORY) {
5438                 node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
5439                                 LRU_ALL_FILE);
5440                 seq_printf(m, " N%d=%lu", nid, node_nr);
5441         }
5442         seq_putc(m, '\n');
5443 
5444         anon_nr = mem_cgroup_nr_lru_pages(memcg, LRU_ALL_ANON);
5445         seq_printf(m, "anon=%lu", anon_nr);
5446         for_each_node_state(nid, N_MEMORY) {
5447                 node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
5448                                 LRU_ALL_ANON);
5449                 seq_printf(m, " N%d=%lu", nid, node_nr);
5450         }
5451         seq_putc(m, '\n');
5452 
5453         unevictable_nr = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_UNEVICTABLE));
5454         seq_printf(m, "unevictable=%lu", unevictable_nr);
5455         for_each_node_state(nid, N_MEMORY) {
5456                 node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
5457                                 BIT(LRU_UNEVICTABLE));
5458                 seq_printf(m, " N%d=%lu", nid, node_nr);
5459         }
5460         seq_putc(m, '\n');
5461         return 0;
5462 }
5463 #endif /* CONFIG_NUMA */
5464 
5465 static inline void mem_cgroup_lru_names_not_uptodate(void)
5466 {
5467         BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_lru_names) != NR_LRU_LISTS);
5468 }
5469 
5470 static int memcg_stat_show(struct cgroup_subsys_state *css, struct cftype *cft,
5471                                  struct seq_file *m)
5472 {
5473         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5474         struct mem_cgroup *mi;
5475         unsigned int i;
5476 
5477         for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
5478                 if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
5479                         continue;
5480                 seq_printf(m, "%s %ld\n", mem_cgroup_stat_names[i],
5481                            mem_cgroup_read_stat(memcg, i) * PAGE_SIZE);
5482         }
5483 
5484         for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++)
5485                 seq_printf(m, "%s %lu\n", mem_cgroup_events_names[i],
5486                            mem_cgroup_read_events(memcg, i));
5487 
5488         for (i = 0; i < NR_LRU_LISTS; i++)
5489                 seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i],
5490                            mem_cgroup_nr_lru_pages(memcg, BIT(i)) * PAGE_SIZE);
5491 
5492         /* Hierarchical information */
5493         {
5494                 unsigned long long limit, memsw_limit;
5495                 memcg_get_hierarchical_limit(memcg, &limit, &memsw_limit);
5496                 seq_printf(m, "hierarchical_memory_limit %llu\n", limit);
5497                 if (do_swap_account)
5498                         seq_printf(m, "hierarchical_memsw_limit %llu\n",
5499                                    memsw_limit);
5500         }
5501 
5502         for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
5503                 long long val = 0;
5504 
5505                 if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
5506                         continue;
5507                 for_each_mem_cgroup_tree(mi, memcg)
5508                         val += mem_cgroup_read_stat(mi, i) * PAGE_SIZE;
5509                 seq_printf(m, "total_%s %lld\n", mem_cgroup_stat_names[i], val);
5510         }
5511 
5512         for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
5513                 unsigned long long val = 0;
5514 
5515                 for_each_mem_cgroup_tree(mi, memcg)
5516                         val += mem_cgroup_read_events(mi, i);
5517                 seq_printf(m, "total_%s %llu\n",
5518                            mem_cgroup_events_names[i], val);
5519         }
5520 
5521         for (i = 0; i < NR_LRU_LISTS; i++) {
5522                 unsigned long long val = 0;
5523 
5524                 for_each_mem_cgroup_tree(mi, memcg)
5525                         val += mem_cgroup_nr_lru_pages(mi, BIT(i)) * PAGE_SIZE;
5526                 seq_printf(m, "total_%s %llu\n", mem_cgroup_lru_names[i], val);
5527         }
5528 
5529 #ifdef CONFIG_DEBUG_VM
5530         {
5531                 int nid, zid;
5532                 struct mem_cgroup_per_zone *mz;
5533                 struct zone_reclaim_stat *rstat;
5534                 unsigned long recent_rotated[2] = {0, 0};
5535                 unsigned long recent_scanned[2] = {0, 0};
5536 
5537                 for_each_online_node(nid)
5538                         for (zid = 0; zid < MAX_NR_ZONES; zid++) {
5539                                 mz = mem_cgroup_zoneinfo(memcg, nid, zid);
5540                                 rstat = &mz->lruvec.reclaim_stat;
5541 
5542                                 recent_rotated[0] += rstat->recent_rotated[0];
5543                                 recent_rotated[1] += rstat->recent_rotated[1];
5544                                 recent_scanned[0] += rstat->recent_scanned[0];
5545                                 recent_scanned[1] += rstat->recent_scanned[1];
5546                         }
5547                 seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]);
5548                 seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]);
5549                 seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]);
5550                 seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]);
5551         }
5552 #endif
5553 
5554         return 0;
5555 }
5556 
5557 static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css,
5558                                       struct cftype *cft)
5559 {
5560         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5561 
5562         return mem_cgroup_swappiness(memcg);
5563 }
5564 
5565 static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
5566                                        struct cftype *cft, u64 val)
5567 {
5568         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5569         struct mem_cgroup *parent = mem_cgroup_from_css(css_parent(&memcg->css));
5570 
5571         if (val > 100 || !parent)
5572                 return -EINVAL;
5573 
5574         mutex_lock(&memcg_create_mutex);
5575 
5576         /* If under hierarchy, only empty-root can set this value */
5577         if ((parent->use_hierarchy) || memcg_has_children(memcg)) {
5578                 mutex_unlock(&memcg_create_mutex);
5579                 return -EINVAL;
5580         }
5581 
5582         memcg->swappiness = val;
5583 
5584         mutex_unlock(&memcg_create_mutex);
5585 
5586         return 0;
5587 }
5588 
5589 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
5590 {
5591         struct mem_cgroup_threshold_ary *t;
5592         u64 usage;
5593         int i;
5594 
5595         rcu_read_lock();
5596         if (!swap)
5597                 t = rcu_dereference(memcg->thresholds.primary);
5598         else
5599                 t = rcu_dereference(memcg->memsw_thresholds.primary);
5600 
5601         if (!t)
5602                 goto unlock;
5603 
5604         usage = mem_cgroup_usage(memcg, swap);
5605 
5606         /*
5607          * current_threshold points to threshold just below or equal to usage.
5608          * If it's not true, a threshold was crossed after last
5609          * call of __mem_cgroup_threshold().
5610          */
5611         i = t->current_threshold;
5612 
5613         /*
5614          * Iterate backward over array of thresholds starting from
5615          * current_threshold and check if a threshold is crossed.
5616          * If none of thresholds below usage is crossed, we read
5617          * only one element of the array here.
5618          */
5619         for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
5620                 eventfd_signal(t->entries[i].eventfd, 1);
5621 
5622         /* i = current_threshold + 1 */
5623         i++;
5624 
5625         /*
5626          * Iterate forward over array of thresholds starting from
5627          * current_threshold+1 and check if a threshold is crossed.
5628          * If none of thresholds above usage is crossed, we read
5629          * only one element of the array here.
5630          */
5631         for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
5632                 eventfd_signal(t->entries[i].eventfd, 1);
5633 
5634         /* Update current_threshold */
5635         t->current_threshold = i - 1;
5636 unlock:
5637         rcu_read_unlock();
5638 }
5639 
5640 static void mem_cgroup_threshold(struct mem_cgroup *memcg)
5641 {
5642         while (memcg) {
5643                 __mem_cgroup_threshold(memcg, false);
5644                 if (do_swap_account)
5645                         __mem_cgroup_threshold(memcg, true);
5646 
5647                 memcg = parent_mem_cgroup(memcg);
5648         }
5649 }
5650 
5651 static int compare_thresholds(const void *a, const void *b)
5652 {
5653         const struct mem_cgroup_threshold *_a = a;
5654         const struct mem_cgroup_threshold *_b = b;
5655 
5656         if (_a->threshold > _b->threshold)
5657                 return 1;
5658 
5659         if (_a->threshold < _b->threshold)
5660                 return -1;
5661 
5662         return 0;
5663 }
5664 
5665 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
5666 {
5667         struct mem_cgroup_eventfd_list *ev;
5668 
5669         spin_lock(&memcg_oom_lock);
5670 
5671         list_for_each_entry(ev, &memcg->oom_notify, list)
5672                 eventfd_signal(ev->eventfd, 1);
5673 
5674         spin_unlock(&memcg_oom_lock);
5675         return 0;
5676 }
5677 
5678 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
5679 {
5680         struct mem_cgroup *iter;
5681 
5682         for_each_mem_cgroup_tree(iter, memcg)
5683                 mem_cgroup_oom_notify_cb(iter);
5684 }
5685 
5686 static int mem_cgroup_usage_register_event(struct cgroup_subsys_state *css,
5687         struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
5688 {
5689         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5690         struct mem_cgroup_thresholds *thresholds;
5691         struct mem_cgroup_threshold_ary *new;
5692         enum res_type type = MEMFILE_TYPE(cft->private);
5693         u64 threshold, usage;
5694         int i, size, ret;
5695 
5696         ret = res_counter_memparse_write_strategy(args, &threshold);
5697         if (ret)
5698                 return ret;
5699 
5700         mutex_lock(&memcg->thresholds_lock);
5701 
5702         if (type == _MEM)
5703                 thresholds = &memcg->thresholds;
5704         else if (type == _MEMSWAP)
5705                 thresholds = &memcg->memsw_thresholds;
5706         else
5707                 BUG();
5708 
5709         usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
5710 
5711         /* Check if a threshold crossed before adding a new one */
5712         if (thresholds->primary)
5713                 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
5714 
5715         size = thresholds->primary ? thresholds->primary->size + 1 : 1;
5716 
5717         /* Allocate memory for new array of thresholds */
5718         new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
5719                         GFP_KERNEL);
5720         if (!new) {
5721                 ret = -ENOMEM;
5722                 goto unlock;
5723         }
5724         new->size = size;
5725 
5726         /* Copy thresholds (if any) to new array */
5727         if (thresholds->primary) {
5728                 memcpy(new->entries, thresholds->primary->entries, (size - 1) *
5729                                 sizeof(struct mem_cgroup_threshold));
5730         }
5731 
5732         /* Add new threshold */
5733         new->entries[size - 1].eventfd = eventfd;
5734         new->entries[size - 1].threshold = threshold;
5735 
5736         /* Sort thresholds. Registering of new threshold isn't time-critical */
5737         sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
5738                         compare_thresholds, NULL);
5739 
5740         /* Find current threshold */
5741         new->current_threshold = -1;
5742         for (i = 0; i < size; i++) {
5743                 if (new->entries[i].threshold <= usage) {
5744                         /*
5745                          * new->current_threshold will not be used until
5746                          * rcu_assign_pointer(), so it's safe to increment
5747                          * it here.
5748                          */
5749                         ++new->current_threshold;
5750                 } else
5751                         break;
5752         }
5753 
5754         /* Free old spare buffer and save old primary buffer as spare */
5755         kfree(thresholds->spare);
5756         thresholds->spare = thresholds->primary;
5757 
5758         rcu_assign_pointer(thresholds->primary, new);
5759 
5760         /* To be sure that nobody uses thresholds */
5761         synchronize_rcu();
5762 
5763 unlock:
5764         mutex_unlock(&memcg->thresholds_lock);
5765 
5766         return ret;
5767 }
5768 
5769 static void mem_cgroup_usage_unregister_event(struct cgroup_subsys_state *css,
5770         struct cftype *cft, struct eventfd_ctx *eventfd)
5771 {
5772         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5773         struct mem_cgroup_thresholds *thresholds;
5774         struct mem_cgroup_threshold_ary *new;
5775         enum res_type type = MEMFILE_TYPE(cft->private);
5776         u64 usage;
5777         int i, j, size;
5778 
5779         mutex_lock(&memcg->thresholds_lock);
5780         if (type == _MEM)
5781                 thresholds = &memcg->thresholds;
5782         else if (type == _MEMSWAP)
5783                 thresholds = &memcg->memsw_thresholds;
5784         else
5785                 BUG();
5786 
5787         if (!thresholds->primary)
5788                 goto unlock;
5789 
5790         usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
5791 
5792         /* Check if a threshold crossed before removing */
5793         __mem_cgroup_threshold(memcg, type == _MEMSWAP);
5794 
5795         /* Calculate new number of threshold */
5796         size = 0;
5797         for (i = 0; i < thresholds->primary->size; i++) {
5798                 if (thresholds->primary->entries[i].eventfd != eventfd)
5799                         size++;
5800         }
5801 
5802         new = thresholds->spare;
5803 
5804         /* Set thresholds array to NULL if we don't have thresholds */
5805         if (!size) {
5806                 kfree(new);
5807                 new = NULL;
5808                 goto swap_buffers;
5809         }
5810 
5811         new->size = size;
5812 
5813         /* Copy thresholds and find current threshold */
5814         new->current_threshold = -1;
5815         for (i = 0, j = 0; i < thresholds->primary->size; i++) {
5816                 if (thresholds->primary->entries[i].eventfd == eventfd)
5817                         continue;
5818 
5819                 new->entries[j] = thresholds->primary->entries[i];
5820                 if (new->entries[j].threshold <= usage) {
5821                         /*
5822                          * new->current_threshold will not be used
5823                          * until rcu_assign_pointer(), so it's safe to increment
5824                          * it here.
5825                          */
5826                         ++new->current_threshold;
5827                 }
5828                 j++;
5829         }
5830 
5831 swap_buffers:
5832         /* Swap primary and spare array */
5833         thresholds->spare = thresholds->primary;
5834 
5835         rcu_assign_pointer(thresholds->primary, new);
5836 
5837         /* To be sure that nobody uses thresholds */
5838         synchronize_rcu();
5839 
5840         /* If all events are unregistered, free the spare array */
5841         if (!new) {
5842                 kfree(thresholds->spare);
5843                 thresholds->spare = NULL;
5844         }
5845 unlock:
5846         mutex_unlock(&memcg->thresholds_lock);
5847 }
5848 
5849 static int mem_cgroup_oom_register_event(struct cgroup_subsys_state *css,
5850         struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
5851 {
5852         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5853         struct mem_cgroup_eventfd_list *event;
5854         enum res_type type = MEMFILE_TYPE(cft->private);
5855 
5856         BUG_ON(type != _OOM_TYPE);
5857         event = kmalloc(sizeof(*event), GFP_KERNEL);
5858         if (!event)
5859                 return -ENOMEM;
5860 
5861         spin_lock(&memcg_oom_lock);
5862 
5863         event->eventfd = eventfd;
5864         list_add(&event->list, &memcg->oom_notify);
5865 
5866         /* already in OOM ? */
5867         if (atomic_read(&memcg->under_oom))
5868                 eventfd_signal(eventfd, 1);
5869         spin_unlock(&memcg_oom_lock);
5870 
5871         return 0;
5872 }
5873 
5874 static void mem_cgroup_oom_unregister_event(struct cgroup_subsys_state *css,
5875         struct cftype *cft, struct eventfd_ctx *eventfd)
5876 {
5877         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5878         struct mem_cgroup_eventfd_list *ev, *tmp;
5879         enum res_type type = MEMFILE_TYPE(cft->private);
5880 
5881         BUG_ON(type != _OOM_TYPE);
5882 
5883         spin_lock(&memcg_oom_lock);
5884 
5885         list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
5886                 if (ev->eventfd == eventfd) {
5887                         list_del(&ev->list);
5888                         kfree(ev);
5889                 }
5890         }
5891 
5892         spin_unlock(&memcg_oom_lock);
5893 }
5894 
5895 static int mem_cgroup_oom_control_read(struct cgroup_subsys_state *css,
5896         struct cftype *cft,  struct cgroup_map_cb *cb)
5897 {
5898         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5899 
5900         cb->fill(cb, "oom_kill_disable", memcg->oom_kill_disable);
5901 
5902         if (atomic_read(&memcg->under_oom))
5903                 cb->fill(cb, "under_oom", 1);
5904         else
5905                 cb->fill(cb, "under_oom", 0);
5906         return 0;
5907 }
5908 
5909 static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
5910         struct cftype *cft, u64 val)
5911 {
5912         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5913         struct mem_cgroup *parent = mem_cgroup_from_css(css_parent(&memcg->css));
5914 
5915         /* cannot set to root cgroup and only 0 and 1 are allowed */
5916         if (!parent || !((val == 0) || (val == 1)))
5917                 return -EINVAL;
5918 
5919         mutex_lock(&memcg_create_mutex);
5920         /* oom-kill-disable is a flag for subhierarchy. */
5921         if ((parent->use_hierarchy) || memcg_has_children(memcg)) {
5922                 mutex_unlock(&memcg_create_mutex);
5923                 return -EINVAL;
5924         }
5925         memcg->oom_kill_disable = val;
5926         if (!val)
5927                 memcg_oom_recover(memcg);
5928         mutex_unlock(&memcg_create_mutex);
5929         return 0;
5930 }
5931 
5932 #ifdef CONFIG_MEMCG_KMEM
5933 static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
5934 {
5935         int ret;
5936 
5937         memcg->kmemcg_id = -1;
5938         ret = memcg_propagate_kmem(memcg);
5939         if (ret)
5940                 return ret;
5941 
5942         return mem_cgroup_sockets_init(memcg, ss);
5943 }
5944 
5945 static void memcg_destroy_kmem(struct mem_cgroup *memcg)
5946 {
5947         mem_cgroup_sockets_destroy(memcg);
5948 }
5949 
5950 static void kmem_cgroup_css_offline(struct mem_cgroup *memcg)
5951 {
5952         if (!memcg_kmem_is_active(memcg))
5953                 return;
5954 
5955         /*
5956          * kmem charges can outlive the cgroup. In the case of slab
5957          * pages, for instance, a page contain objects from various
5958          * processes. As we prevent from taking a reference for every
5959          * such allocation we have to be careful when doing uncharge
5960          * (see memcg_uncharge_kmem) and here during offlining.
5961          *
5962          * The idea is that that only the _last_ uncharge which sees
5963          * the dead memcg will drop the last reference. An additional
5964          * reference is taken here before the group is marked dead
5965          * which is then paired with css_put during uncharge resp. here.
5966          *
5967          * Although this might sound strange as this path is called from
5968          * css_offline() when the referencemight have dropped down to 0
5969          * and shouldn't be incremented anymore (css_tryget would fail)
5970          * we do not have other options because of the kmem allocations
5971          * lifetime.
5972          */
5973         css_get(&memcg->css);
5974 
5975         memcg_kmem_mark_dead(memcg);
5976 
5977         if (res_counter_read_u64(&memcg->kmem, RES_USAGE) != 0)
5978                 return;
5979 
5980         if (memcg_kmem_test_and_clear_dead(memcg))
5981                 css_put(&memcg->css);
5982 }
5983 #else
5984 static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
5985 {
5986         return 0;
5987 }
5988 
5989 static void memcg_destroy_kmem(struct mem_cgroup *memcg)
5990 {
5991 }
5992 
5993 static void kmem_cgroup_css_offline(struct mem_cgroup *memcg)
5994 {
5995 }
5996 #endif
5997 
5998 static struct cftype mem_cgroup_files[] = {
5999         {
6000                 .name = "usage_in_bytes",
6001                 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
6002                 .read = mem_cgroup_read,
6003                 .register_event = mem_cgroup_usage_register_event,
6004                 .unregister_event = mem_cgroup_usage_unregister_event,
6005         },
6006         {
6007                 .name = "max_usage_in_bytes",
6008                 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
6009                 .trigger = mem_cgroup_reset,
6010                 .read = mem_cgroup_read,
6011         },
6012         {
6013                 .name = "limit_in_bytes",
6014                 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
6015                 .write_string = mem_cgroup_write,
6016                 .read = mem_cgroup_read,
6017         },
6018         {
6019                 .name = "soft_limit_in_bytes",
6020                 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
6021                 .write_string = mem_cgroup_write,
6022                 .read = mem_cgroup_read,
6023         },
6024         {
6025                 .name = "failcnt",
6026                 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
6027                 .trigger = mem_cgroup_reset,
6028                 .read = mem_cgroup_read,
6029         },
6030         {
6031                 .name = "stat",
6032                 .read_seq_string = memcg_stat_show,
6033         },
6034         {
6035                 .name = "force_empty",
6036                 .trigger = mem_cgroup_force_empty_write,
6037         },
6038         {
6039                 .name = "use_hierarchy",
6040                 .flags = CFTYPE_INSANE,
6041                 .write_u64 = mem_cgroup_hierarchy_write,
6042                 .read_u64 = mem_cgroup_hierarchy_read,
6043         },
6044         {
6045                 .name = "swappiness",
6046                 .read_u64 = mem_cgroup_swappiness_read,
6047                 .write_u64 = mem_cgroup_swappiness_write,
6048         },
6049         {
6050                 .name = "move_charge_at_immigrate",
6051                 .read_u64 = mem_cgroup_move_charge_read,
6052                 .write_u64 = mem_cgroup_move_charge_write,
6053         },
6054         {
6055                 .name = "oom_control",
6056                 .read_map = mem_cgroup_oom_control_read,
6057                 .write_u64 = mem_cgroup_oom_control_write,
6058                 .register_event = mem_cgroup_oom_register_event,
6059                 .unregister_event = mem_cgroup_oom_unregister_event,
6060                 .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
6061         },
6062         {
6063                 .name = "pressure_level",
6064                 .register_event = vmpressure_register_event,
6065                 .unregister_event = vmpressure_unregister_event,
6066         },
6067 #ifdef CONFIG_NUMA
6068         {
6069                 .name = "numa_stat",
6070                 .read_seq_string = memcg_numa_stat_show,
6071         },
6072 #endif
6073 #ifdef CONFIG_MEMCG_KMEM
6074         {
6075                 .name = "kmem.limit_in_bytes",
6076                 .private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
6077                 .write_string = mem_cgroup_write,
6078                 .read = mem_cgroup_read,
6079         },
6080         {
6081                 .name = "kmem.usage_in_bytes",
6082                 .private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
6083                 .read = mem_cgroup_read,
6084         },
6085         {
6086                 .name = "kmem.failcnt",
6087                 .private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
6088                 .trigger = mem_cgroup_reset,
6089                 .read = mem_cgroup_read,
6090         },
6091         {
6092                 .name = "kmem.max_usage_in_bytes",
6093                 .private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
6094                 .trigger = mem_cgroup_reset,
6095                 .read = mem_cgroup_read,
6096         },
6097 #ifdef CONFIG_SLABINFO
6098         {
6099                 .name = "kmem.slabinfo",
6100                 .read_seq_string = mem_cgroup_slabinfo_read,
6101         },
6102 #endif
6103 #endif
6104         { },    /* terminate */
6105 };
6106 
6107 #ifdef CONFIG_MEMCG_SWAP
6108 static struct cftype memsw_cgroup_files[] = {
6109         {
6110                 .name = "memsw.usage_in_bytes",
6111                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
6112                 .read = mem_cgroup_read,
6113                 .register_event = mem_cgroup_usage_register_event,
6114                 .unregister_event = mem_cgroup_usage_unregister_event,
6115         },
6116         {
6117                 .name = "memsw.max_usage_in_bytes",
6118                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
6119                 .trigger = mem_cgroup_reset,
6120                 .read = mem_cgroup_read,
6121         },
6122         {
6123                 .name = "memsw.limit_in_bytes",
6124                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
6125                 .write_string = mem_cgroup_write,
6126                 .read = mem_cgroup_read,
6127         },
6128         {
6129                 .name = "memsw.failcnt",
6130                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
6131                 .trigger = mem_cgroup_reset,
6132                 .read = mem_cgroup_read,
6133         },
6134         { },    /* terminate */
6135 };
6136 #endif
6137 static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
6138 {
6139         struct mem_cgroup_per_node *pn;
6140         struct mem_cgroup_per_zone *mz;
6141         int zone, tmp = node;
6142         /*
6143          * This routine is called against possible nodes.
6144          * But it's BUG to call kmalloc() against offline node.
6145          *
6146          * TODO: this routine can waste much memory for nodes which will
6147          *       never be onlined. It's better to use memory hotplug callback
6148          *       function.
6149          */
6150         if (!node_state(node, N_NORMAL_MEMORY))
6151                 tmp = -1;
6152         pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
6153         if (!pn)
6154                 return 1;
6155 
6156         for (zone = 0; zone < MAX_NR_ZONES; zone++) {
6157                 mz = &pn->zoneinfo[zone];
6158                 lruvec_init(&mz->lruvec);
6159                 mz->usage_in_excess = 0;
6160                 mz->on_tree = false;
6161                 mz->memcg = memcg;
6162         }
6163         memcg->nodeinfo[node] = pn;
6164         return 0;
6165 }
6166 
6167 static void free_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
6168 {
6169         kfree(memcg->nodeinfo[node]);
6170 }
6171 
6172 static struct mem_cgroup *mem_cgroup_alloc(void)
6173 {
6174         struct mem_cgroup *memcg;
6175         size_t size = memcg_size();
6176 
6177         /* Can be very big if nr_node_ids is very big */
6178         if (size < PAGE_SIZE)
6179                 memcg = kzalloc(size, GFP_KERNEL);
6180         else
6181                 memcg = vzalloc(size);
6182 
6183         if (!memcg)
6184                 return NULL;
6185 
6186         memcg->stat = alloc_percpu(struct mem_cgroup_stat_cpu);
6187         if (!memcg->stat)
6188                 goto out_free;
6189         spin_lock_init(&memcg->pcp_counter_lock);
6190         return memcg;
6191 
6192 out_free:
6193         if (size < PAGE_SIZE)
6194                 kfree(memcg);
6195         else
6196                 vfree(memcg);
6197         return NULL;
6198 }
6199 
6200 /*
6201  * At destroying mem_cgroup, references from swap_cgroup can remain.
6202  * (scanning all at force_empty is too costly...)
6203  *
6204  * Instead of clearing all references at force_empty, we remember
6205  * the number of reference from swap_cgroup and free mem_cgroup when
6206  * it goes down to 0.
6207  *
6208  * Removal of cgroup itself succeeds regardless of refs from swap.
6209  */
6210 
6211 static void __mem_cgroup_free(struct mem_cgroup *memcg)
6212 {
6213         int node;
6214         size_t size = memcg_size();
6215 
6216         mem_cgroup_remove_from_trees(memcg);
6217         free_css_id(&mem_cgroup_subsys, &memcg->css);
6218 
6219         for_each_node(node)
6220                 free_mem_cgroup_per_zone_info(memcg, node);
6221 
6222         free_percpu(memcg->stat);
6223 
6224         /*
6225          * We need to make sure that (at least for now), the jump label
6226          * destruction code runs outside of the cgroup lock. This is because
6227          * get_online_cpus(), which is called from the static_branch update,
6228          * can't be called inside the cgroup_lock. cpusets are the ones
6229          * enforcing this dependency, so if they ever change, we might as well.
6230          *
6231          * schedule_work() will guarantee this happens. Be careful if you need
6232          * to move this code around, and make sure it is outside
6233          * the cgroup_lock.
6234          */
6235         disarm_static_keys(memcg);
6236         if (size < PAGE_SIZE)
6237                 kfree(memcg);
6238         else
6239                 vfree(memcg);
6240 }
6241 
6242 /*
6243  * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled.
6244  */
6245 struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
6246 {
6247         if (!memcg->res.parent)
6248                 return NULL;
6249         return mem_cgroup_from_res_counter(memcg->res.parent, res);
6250 }
6251 EXPORT_SYMBOL(parent_mem_cgroup);
6252 
6253 static void __init mem_cgroup_soft_limit_tree_init(void)
6254 {
6255         struct mem_cgroup_tree_per_node *rtpn;
6256         struct mem_cgroup_tree_per_zone *rtpz;
6257         int tmp, node, zone;
6258 
6259         for_each_node(node) {
6260                 tmp = node;
6261                 if (!node_state(node, N_NORMAL_MEMORY))
6262                         tmp = -1;
6263                 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, tmp);
6264                 BUG_ON(!rtpn);
6265 
6266                 soft_limit_tree.rb_tree_per_node[node] = rtpn;
6267 
6268                 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
6269                         rtpz = &rtpn->rb_tree_per_zone[zone];
6270                         rtpz->rb_root = RB_ROOT;
6271                         spin_lock_init(&rtpz->lock);
6272                 }
6273         }
6274 }
6275 
6276 static struct cgroup_subsys_state * __ref
6277 mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
6278 {
6279         struct mem_cgroup *memcg;
6280         long error = -ENOMEM;
6281         int node;
6282 
6283         memcg = mem_cgroup_alloc();
6284         if (!memcg)
6285                 return ERR_PTR(error);
6286 
6287         for_each_node(node)
6288                 if (alloc_mem_cgroup_per_zone_info(memcg, node))
6289                         goto free_out;
6290 
6291         /* root ? */
6292         if (parent_css == NULL) {
6293                 root_mem_cgroup = memcg;
6294                 res_counter_init(&memcg->res, NULL);
6295                 res_counter_init(&memcg->memsw, NULL);
6296                 res_counter_init(&memcg->kmem, NULL);
6297         }
6298 
6299         memcg->last_scanned_node = MAX_NUMNODES;
6300         INIT_LIST_HEAD(&memcg->oom_notify);
6301         memcg->move_charge_at_immigrate = 0;
6302         mutex_init(&memcg->thresholds_lock);
6303         spin_lock_init(&memcg->move_lock);
6304         vmpressure_init(&memcg->vmpressure);
6305 
6306         return &memcg->css;
6307 
6308 free_out:
6309         __mem_cgroup_free(memcg);
6310         return ERR_PTR(error);
6311 }
6312 
6313 static int
6314 mem_cgroup_css_online(struct cgroup_subsys_state *css)
6315 {
6316         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
6317         struct mem_cgroup *parent = mem_cgroup_from_css(css_parent(css));
6318         int error = 0;
6319 
6320         if (!parent)
6321                 return 0;
6322 
6323         mutex_lock(&memcg_create_mutex);
6324 
6325         memcg->use_hierarchy = parent->use_hierarchy;
6326         memcg->oom_kill_disable = parent->oom_kill_disable;
6327         memcg->swappiness = mem_cgroup_swappiness(parent);
6328 
6329         if (parent->use_hierarchy) {
6330                 res_counter_init(&memcg->res, &parent->res);
6331                 res_counter_init(&memcg->memsw, &parent->memsw);
6332                 res_counter_init(&memcg->kmem, &parent->kmem);
6333 
6334                 /*
6335                  * No need to take a reference to the parent because cgroup
6336                  * core guarantees its existence.
6337                  */
6338         } else {
6339                 res_counter_init(&memcg->res, NULL);
6340                 res_counter_init(&memcg->memsw, NULL);
6341                 res_counter_init(&memcg->kmem, NULL);
6342                 /*
6343                  * Deeper hierachy with use_hierarchy == false doesn't make
6344                  * much sense so let cgroup subsystem know about this
6345                  * unfortunate state in our controller.
6346                  */
6347                 if (parent != root_mem_cgroup)
6348                         mem_cgroup_subsys.broken_hierarchy = true;
6349         }
6350 
6351         error = memcg_init_kmem(memcg, &mem_cgroup_subsys);
6352         mutex_unlock(&memcg_create_mutex);
6353 
6354         if (!error) {
6355                 /*
6356                  * Make sure the memcg is initialized: mem_cgroup_iter()
6357                  * orders reading memcg->initialized against its callers
6358                  * reading the memcg members.
6359                  */
6360                 smp_wmb();
6361                 memcg->initialized = 1;
6362         }
6363         return error;
6364 }
6365 
6366 /*
6367  * Announce all parents that a group from their hierarchy is gone.
6368  */
6369 static void mem_cgroup_invalidate_reclaim_iterators(struct mem_cgroup *memcg)
6370 {
6371         struct mem_cgroup *parent = memcg;
6372 
6373         while ((parent = parent_mem_cgroup(parent)))
6374                 mem_cgroup_iter_invalidate(parent);
6375 
6376         /*
6377          * if the root memcg is not hierarchical we have to check it
6378          * explicitely.
6379          */
6380         if (!root_mem_cgroup->use_hierarchy)
6381                 mem_cgroup_iter_invalidate(root_mem_cgroup);
6382 }
6383 
6384 static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
6385 {
6386         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
6387         struct cgroup_subsys_state *iter;
6388 
6389         kmem_cgroup_css_offline(memcg);
6390 
6391         mem_cgroup_invalidate_reclaim_iterators(memcg);
6392 
6393         /*
6394          * This requires that offlining is serialized.  Right now that is
6395          * guaranteed because css_killed_work_fn() holds the cgroup_mutex.
6396          */
6397         rcu_read_lock();
6398         css_for_each_descendant_post(iter, css) {
6399                 rcu_read_unlock();
6400                 mem_cgroup_reparent_charges(mem_cgroup_from_css(iter));
6401                 rcu_read_lock();
6402         }
6403         rcu_read_unlock();
6404 
6405         mem_cgroup_destroy_all_caches(memcg);
6406         vmpressure_cleanup(&memcg->vmpressure);
6407 }
6408 
6409 static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
6410 {
6411         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
6412         /*
6413          * XXX: css_offline() would be where we should reparent all
6414          * memory to prepare the cgroup for destruction.  However,
6415          * memcg does not do css_tryget() and res_counter charging
6416          * under the same RCU lock region, which means that charging
6417          * could race with offlining.  Offlining only happens to
6418          * cgroups with no tasks in them but charges can show up
6419          * without any tasks from the swapin path when the target
6420          * memcg is looked up from the swapout record and not from the
6421          * current task as it usually is.  A race like this can leak
6422          * charges and put pages with stale cgroup pointers into
6423          * circulation:
6424          *
6425          * #0                        #1
6426          *                           lookup_swap_cgroup_id()
6427          *                           rcu_read_lock()
6428          *                           mem_cgroup_lookup()
6429          *                           css_tryget()
6430          *                           rcu_read_unlock()
6431          * disable css_tryget()
6432          * call_rcu()
6433          *   offline_css()
6434          *     reparent_charges()
6435          *                           res_counter_charge()
6436          *                           css_put()
6437          *                             css_free()
6438          *                           pc->mem_cgroup = dead memcg
6439          *                           add page to lru
6440          *
6441          * The bulk of the charges are still moved in offline_css() to
6442          * avoid pinning a lot of pages in case a long-term reference
6443          * like a swapout record is deferring the css_free() to long
6444          * after offlining.  But this makes sure we catch any charges
6445          * made after offlining:
6446          */
6447         mem_cgroup_reparent_charges(memcg);
6448 
6449         memcg_destroy_kmem(memcg);
6450         __mem_cgroup_free(memcg);
6451 }
6452 
6453 #ifdef CONFIG_MMU
6454 /* Handlers for move charge at task migration. */
6455 #define PRECHARGE_COUNT_AT_ONCE 256
6456 static int mem_cgroup_do_precharge(unsigned long count)
6457 {
6458         int ret = 0;
6459         int batch_count = PRECHARGE_COUNT_AT_ONCE;
6460         struct mem_cgroup *memcg = mc.to;
6461 
6462         if (mem_cgroup_is_root(memcg)) {
6463                 mc.precharge += count;
6464                 /* we don't need css_get for root */
6465                 return ret;
6466         }
6467         /* try to charge at once */
6468         if (count > 1) {
6469                 struct res_counter *dummy;
6470                 /*
6471                  * "memcg" cannot be under rmdir() because we've already checked
6472                  * by cgroup_lock_live_cgroup() that it is not removed and we
6473                  * are still under the same cgroup_mutex. So we can postpone
6474                  * css_get().
6475                  */
6476                 if (res_counter_charge(&memcg->res, PAGE_SIZE * count, &dummy))
6477                         goto one_by_one;
6478                 if (do_swap_account && res_counter_charge(&memcg->memsw,
6479                                                 PAGE_SIZE * count, &dummy)) {
6480                         res_counter_uncharge(&memcg->res, PAGE_SIZE * count);
6481                         goto one_by_one;
6482                 }
6483                 mc.precharge += count;
6484                 return ret;
6485         }
6486 one_by_one:
6487         /* fall back to one by one charge */
6488         while (count--) {
6489                 if (signal_pending(current)) {
6490                         ret = -EINTR;
6491                         break;
6492                 }
6493                 if (!batch_count--) {
6494                         batch_count = PRECHARGE_COUNT_AT_ONCE;
6495                         cond_resched();
6496                 }
6497                 ret = __mem_cgroup_try_charge(NULL,
6498                                         GFP_KERNEL, 1, &memcg, false);
6499                 if (ret)
6500                         /* mem_cgroup_clear_mc() will do uncharge later */
6501                         return ret;
6502                 mc.precharge++;
6503         }
6504         return ret;
6505 }
6506 
6507 /**
6508  * get_mctgt_type - get target type of moving charge
6509  * @vma: the vma the pte to be checked belongs
6510  * @addr: the address corresponding to the pte to be checked
6511  * @ptent: the pte to be checked
6512  * @target: the pointer the target page or swap ent will be stored(can be NULL)
6513  *
6514  * Returns
6515  *   0(MC_TARGET_NONE): if the pte is not a target for move charge.
6516  *   1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
6517  *     move charge. if @target is not NULL, the page is stored in target->page
6518  *     with extra refcnt got(Callers should handle it).
6519  *   2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
6520  *     target for charge migration. if @target is not NULL, the entry is stored
6521  *     in target->ent.
6522  *
6523  * Called with pte lock held.