~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/mm/memcontrol.c

Version: ~ [ linux-4.19-rc7 ] ~ [ linux-4.18.12 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.74 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.131 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.159 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.123 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.59 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.31.14 ] ~ [ linux-2.6.30.10 ] ~ [ linux-2.6.29.6 ] ~ [ linux-2.6.28.10 ] ~ [ linux-2.6.27.62 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /* memcontrol.c - Memory Controller
  2  *
  3  * Copyright IBM Corporation, 2007
  4  * Author Balbir Singh <balbir@linux.vnet.ibm.com>
  5  *
  6  * Copyright 2007 OpenVZ SWsoft Inc
  7  * Author: Pavel Emelianov <xemul@openvz.org>
  8  *
  9  * Memory thresholds
 10  * Copyright (C) 2009 Nokia Corporation
 11  * Author: Kirill A. Shutemov
 12  *
 13  * Kernel Memory Controller
 14  * Copyright (C) 2012 Parallels Inc. and Google Inc.
 15  * Authors: Glauber Costa and Suleiman Souhlal
 16  *
 17  * Native page reclaim
 18  * Charge lifetime sanitation
 19  * Lockless page tracking & accounting
 20  * Unified hierarchy configuration model
 21  * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner
 22  *
 23  * This program is free software; you can redistribute it and/or modify
 24  * it under the terms of the GNU General Public License as published by
 25  * the Free Software Foundation; either version 2 of the License, or
 26  * (at your option) any later version.
 27  *
 28  * This program is distributed in the hope that it will be useful,
 29  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 30  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 31  * GNU General Public License for more details.
 32  */
 33 
 34 #include <linux/page_counter.h>
 35 #include <linux/memcontrol.h>
 36 #include <linux/cgroup.h>
 37 #include <linux/mm.h>
 38 #include <linux/sched/mm.h>
 39 #include <linux/shmem_fs.h>
 40 #include <linux/hugetlb.h>
 41 #include <linux/pagemap.h>
 42 #include <linux/smp.h>
 43 #include <linux/page-flags.h>
 44 #include <linux/backing-dev.h>
 45 #include <linux/bit_spinlock.h>
 46 #include <linux/rcupdate.h>
 47 #include <linux/limits.h>
 48 #include <linux/export.h>
 49 #include <linux/mutex.h>
 50 #include <linux/rbtree.h>
 51 #include <linux/slab.h>
 52 #include <linux/swap.h>
 53 #include <linux/swapops.h>
 54 #include <linux/spinlock.h>
 55 #include <linux/eventfd.h>
 56 #include <linux/poll.h>
 57 #include <linux/sort.h>
 58 #include <linux/fs.h>
 59 #include <linux/seq_file.h>
 60 #include <linux/vmpressure.h>
 61 #include <linux/mm_inline.h>
 62 #include <linux/swap_cgroup.h>
 63 #include <linux/cpu.h>
 64 #include <linux/oom.h>
 65 #include <linux/lockdep.h>
 66 #include <linux/file.h>
 67 #include <linux/tracehook.h>
 68 #include "internal.h"
 69 #include <net/sock.h>
 70 #include <net/ip.h>
 71 #include "slab.h"
 72 
 73 #include <linux/uaccess.h>
 74 
 75 #include <trace/events/vmscan.h>
 76 
 77 struct cgroup_subsys memory_cgrp_subsys __read_mostly;
 78 EXPORT_SYMBOL(memory_cgrp_subsys);
 79 
 80 struct mem_cgroup *root_mem_cgroup __read_mostly;
 81 
 82 #define MEM_CGROUP_RECLAIM_RETRIES      5
 83 
 84 /* Socket memory accounting disabled? */
 85 static bool cgroup_memory_nosocket;
 86 
 87 /* Kernel memory accounting disabled? */
 88 static bool cgroup_memory_nokmem;
 89 
 90 /* Whether the swap controller is active */
 91 #ifdef CONFIG_MEMCG_SWAP
 92 int do_swap_account __read_mostly;
 93 #else
 94 #define do_swap_account         0
 95 #endif
 96 
 97 /* Whether legacy memory+swap accounting is active */
 98 static bool do_memsw_account(void)
 99 {
100         return !cgroup_subsys_on_dfl(memory_cgrp_subsys) && do_swap_account;
101 }
102 
103 static const char *const mem_cgroup_lru_names[] = {
104         "inactive_anon",
105         "active_anon",
106         "inactive_file",
107         "active_file",
108         "unevictable",
109 };
110 
111 #define THRESHOLDS_EVENTS_TARGET 128
112 #define SOFTLIMIT_EVENTS_TARGET 1024
113 #define NUMAINFO_EVENTS_TARGET  1024
114 
115 /*
116  * Cgroups above their limits are maintained in a RB-Tree, independent of
117  * their hierarchy representation
118  */
119 
120 struct mem_cgroup_tree_per_node {
121         struct rb_root rb_root;
122         struct rb_node *rb_rightmost;
123         spinlock_t lock;
124 };
125 
126 struct mem_cgroup_tree {
127         struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
128 };
129 
130 static struct mem_cgroup_tree soft_limit_tree __read_mostly;
131 
132 /* for OOM */
133 struct mem_cgroup_eventfd_list {
134         struct list_head list;
135         struct eventfd_ctx *eventfd;
136 };
137 
138 /*
139  * cgroup_event represents events which userspace want to receive.
140  */
141 struct mem_cgroup_event {
142         /*
143          * memcg which the event belongs to.
144          */
145         struct mem_cgroup *memcg;
146         /*
147          * eventfd to signal userspace about the event.
148          */
149         struct eventfd_ctx *eventfd;
150         /*
151          * Each of these stored in a list by the cgroup.
152          */
153         struct list_head list;
154         /*
155          * register_event() callback will be used to add new userspace
156          * waiter for changes related to this event.  Use eventfd_signal()
157          * on eventfd to send notification to userspace.
158          */
159         int (*register_event)(struct mem_cgroup *memcg,
160                               struct eventfd_ctx *eventfd, const char *args);
161         /*
162          * unregister_event() callback will be called when userspace closes
163          * the eventfd or on cgroup removing.  This callback must be set,
164          * if you want provide notification functionality.
165          */
166         void (*unregister_event)(struct mem_cgroup *memcg,
167                                  struct eventfd_ctx *eventfd);
168         /*
169          * All fields below needed to unregister event when
170          * userspace closes eventfd.
171          */
172         poll_table pt;
173         wait_queue_head_t *wqh;
174         wait_queue_entry_t wait;
175         struct work_struct remove;
176 };
177 
178 static void mem_cgroup_threshold(struct mem_cgroup *memcg);
179 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
180 
181 /* Stuffs for move charges at task migration. */
182 /*
183  * Types of charges to be moved.
184  */
185 #define MOVE_ANON       0x1U
186 #define MOVE_FILE       0x2U
187 #define MOVE_MASK       (MOVE_ANON | MOVE_FILE)
188 
189 /* "mc" and its members are protected by cgroup_mutex */
190 static struct move_charge_struct {
191         spinlock_t        lock; /* for from, to */
192         struct mm_struct  *mm;
193         struct mem_cgroup *from;
194         struct mem_cgroup *to;
195         unsigned long flags;
196         unsigned long precharge;
197         unsigned long moved_charge;
198         unsigned long moved_swap;
199         struct task_struct *moving_task;        /* a task moving charges */
200         wait_queue_head_t waitq;                /* a waitq for other context */
201 } mc = {
202         .lock = __SPIN_LOCK_UNLOCKED(mc.lock),
203         .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
204 };
205 
206 /*
207  * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
208  * limit reclaim to prevent infinite loops, if they ever occur.
209  */
210 #define MEM_CGROUP_MAX_RECLAIM_LOOPS            100
211 #define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2
212 
213 enum charge_type {
214         MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
215         MEM_CGROUP_CHARGE_TYPE_ANON,
216         MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */
217         MEM_CGROUP_CHARGE_TYPE_DROP,    /* a page was unused swap cache */
218         NR_CHARGE_TYPE,
219 };
220 
221 /* for encoding cft->private value on file */
222 enum res_type {
223         _MEM,
224         _MEMSWAP,
225         _OOM_TYPE,
226         _KMEM,
227         _TCP,
228 };
229 
230 #define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val))
231 #define MEMFILE_TYPE(val)       ((val) >> 16 & 0xffff)
232 #define MEMFILE_ATTR(val)       ((val) & 0xffff)
233 /* Used for OOM nofiier */
234 #define OOM_CONTROL             (0)
235 
236 /*
237  * Iteration constructs for visiting all cgroups (under a tree).  If
238  * loops are exited prematurely (break), mem_cgroup_iter_break() must
239  * be used for reference counting.
240  */
241 #define for_each_mem_cgroup_tree(iter, root)            \
242         for (iter = mem_cgroup_iter(root, NULL, NULL);  \
243              iter != NULL;                              \
244              iter = mem_cgroup_iter(root, iter, NULL))
245 
246 #define for_each_mem_cgroup(iter)                       \
247         for (iter = mem_cgroup_iter(NULL, NULL, NULL);  \
248              iter != NULL;                              \
249              iter = mem_cgroup_iter(NULL, iter, NULL))
250 
251 /* Some nice accessors for the vmpressure. */
252 struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
253 {
254         if (!memcg)
255                 memcg = root_mem_cgroup;
256         return &memcg->vmpressure;
257 }
258 
259 struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
260 {
261         return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
262 }
263 
264 #ifdef CONFIG_MEMCG_KMEM
265 /*
266  * This will be the memcg's index in each cache's ->memcg_params.memcg_caches.
267  * The main reason for not using cgroup id for this:
268  *  this works better in sparse environments, where we have a lot of memcgs,
269  *  but only a few kmem-limited. Or also, if we have, for instance, 200
270  *  memcgs, and none but the 200th is kmem-limited, we'd have to have a
271  *  200 entry array for that.
272  *
273  * The current size of the caches array is stored in memcg_nr_cache_ids. It
274  * will double each time we have to increase it.
275  */
276 static DEFINE_IDA(memcg_cache_ida);
277 int memcg_nr_cache_ids;
278 
279 /* Protects memcg_nr_cache_ids */
280 static DECLARE_RWSEM(memcg_cache_ids_sem);
281 
282 void memcg_get_cache_ids(void)
283 {
284         down_read(&memcg_cache_ids_sem);
285 }
286 
287 void memcg_put_cache_ids(void)
288 {
289         up_read(&memcg_cache_ids_sem);
290 }
291 
292 /*
293  * MIN_SIZE is different than 1, because we would like to avoid going through
294  * the alloc/free process all the time. In a small machine, 4 kmem-limited
295  * cgroups is a reasonable guess. In the future, it could be a parameter or
296  * tunable, but that is strictly not necessary.
297  *
298  * MAX_SIZE should be as large as the number of cgrp_ids. Ideally, we could get
299  * this constant directly from cgroup, but it is understandable that this is
300  * better kept as an internal representation in cgroup.c. In any case, the
301  * cgrp_id space is not getting any smaller, and we don't have to necessarily
302  * increase ours as well if it increases.
303  */
304 #define MEMCG_CACHES_MIN_SIZE 4
305 #define MEMCG_CACHES_MAX_SIZE MEM_CGROUP_ID_MAX
306 
307 /*
308  * A lot of the calls to the cache allocation functions are expected to be
309  * inlined by the compiler. Since the calls to memcg_kmem_get_cache are
310  * conditional to this static branch, we'll have to allow modules that does
311  * kmem_cache_alloc and the such to see this symbol as well
312  */
313 DEFINE_STATIC_KEY_FALSE(memcg_kmem_enabled_key);
314 EXPORT_SYMBOL(memcg_kmem_enabled_key);
315 
316 struct workqueue_struct *memcg_kmem_cache_wq;
317 
318 static int memcg_shrinker_map_size;
319 static DEFINE_MUTEX(memcg_shrinker_map_mutex);
320 
321 static void memcg_free_shrinker_map_rcu(struct rcu_head *head)
322 {
323         kvfree(container_of(head, struct memcg_shrinker_map, rcu));
324 }
325 
326 static int memcg_expand_one_shrinker_map(struct mem_cgroup *memcg,
327                                          int size, int old_size)
328 {
329         struct memcg_shrinker_map *new, *old;
330         int nid;
331 
332         lockdep_assert_held(&memcg_shrinker_map_mutex);
333 
334         for_each_node(nid) {
335                 old = rcu_dereference_protected(
336                         mem_cgroup_nodeinfo(memcg, nid)->shrinker_map, true);
337                 /* Not yet online memcg */
338                 if (!old)
339                         return 0;
340 
341                 new = kvmalloc(sizeof(*new) + size, GFP_KERNEL);
342                 if (!new)
343                         return -ENOMEM;
344 
345                 /* Set all old bits, clear all new bits */
346                 memset(new->map, (int)0xff, old_size);
347                 memset((void *)new->map + old_size, 0, size - old_size);
348 
349                 rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, new);
350                 call_rcu(&old->rcu, memcg_free_shrinker_map_rcu);
351         }
352 
353         return 0;
354 }
355 
356 static void memcg_free_shrinker_maps(struct mem_cgroup *memcg)
357 {
358         struct mem_cgroup_per_node *pn;
359         struct memcg_shrinker_map *map;
360         int nid;
361 
362         if (mem_cgroup_is_root(memcg))
363                 return;
364 
365         for_each_node(nid) {
366                 pn = mem_cgroup_nodeinfo(memcg, nid);
367                 map = rcu_dereference_protected(pn->shrinker_map, true);
368                 if (map)
369                         kvfree(map);
370                 rcu_assign_pointer(pn->shrinker_map, NULL);
371         }
372 }
373 
374 static int memcg_alloc_shrinker_maps(struct mem_cgroup *memcg)
375 {
376         struct memcg_shrinker_map *map;
377         int nid, size, ret = 0;
378 
379         if (mem_cgroup_is_root(memcg))
380                 return 0;
381 
382         mutex_lock(&memcg_shrinker_map_mutex);
383         size = memcg_shrinker_map_size;
384         for_each_node(nid) {
385                 map = kvzalloc(sizeof(*map) + size, GFP_KERNEL);
386                 if (!map) {
387                         memcg_free_shrinker_maps(memcg);
388                         ret = -ENOMEM;
389                         break;
390                 }
391                 rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, map);
392         }
393         mutex_unlock(&memcg_shrinker_map_mutex);
394 
395         return ret;
396 }
397 
398 int memcg_expand_shrinker_maps(int new_id)
399 {
400         int size, old_size, ret = 0;
401         struct mem_cgroup *memcg;
402 
403         size = DIV_ROUND_UP(new_id + 1, BITS_PER_LONG) * sizeof(unsigned long);
404         old_size = memcg_shrinker_map_size;
405         if (size <= old_size)
406                 return 0;
407 
408         mutex_lock(&memcg_shrinker_map_mutex);
409         if (!root_mem_cgroup)
410                 goto unlock;
411 
412         for_each_mem_cgroup(memcg) {
413                 if (mem_cgroup_is_root(memcg))
414                         continue;
415                 ret = memcg_expand_one_shrinker_map(memcg, size, old_size);
416                 if (ret)
417                         goto unlock;
418         }
419 unlock:
420         if (!ret)
421                 memcg_shrinker_map_size = size;
422         mutex_unlock(&memcg_shrinker_map_mutex);
423         return ret;
424 }
425 
426 void memcg_set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id)
427 {
428         if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) {
429                 struct memcg_shrinker_map *map;
430 
431                 rcu_read_lock();
432                 map = rcu_dereference(memcg->nodeinfo[nid]->shrinker_map);
433                 /* Pairs with smp mb in shrink_slab() */
434                 smp_mb__before_atomic();
435                 set_bit(shrinker_id, map->map);
436                 rcu_read_unlock();
437         }
438 }
439 
440 #else /* CONFIG_MEMCG_KMEM */
441 static int memcg_alloc_shrinker_maps(struct mem_cgroup *memcg)
442 {
443         return 0;
444 }
445 static void memcg_free_shrinker_maps(struct mem_cgroup *memcg) { }
446 #endif /* CONFIG_MEMCG_KMEM */
447 
448 /**
449  * mem_cgroup_css_from_page - css of the memcg associated with a page
450  * @page: page of interest
451  *
452  * If memcg is bound to the default hierarchy, css of the memcg associated
453  * with @page is returned.  The returned css remains associated with @page
454  * until it is released.
455  *
456  * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup
457  * is returned.
458  */
459 struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page)
460 {
461         struct mem_cgroup *memcg;
462 
463         memcg = page->mem_cgroup;
464 
465         if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
466                 memcg = root_mem_cgroup;
467 
468         return &memcg->css;
469 }
470 
471 /**
472  * page_cgroup_ino - return inode number of the memcg a page is charged to
473  * @page: the page
474  *
475  * Look up the closest online ancestor of the memory cgroup @page is charged to
476  * and return its inode number or 0 if @page is not charged to any cgroup. It
477  * is safe to call this function without holding a reference to @page.
478  *
479  * Note, this function is inherently racy, because there is nothing to prevent
480  * the cgroup inode from getting torn down and potentially reallocated a moment
481  * after page_cgroup_ino() returns, so it only should be used by callers that
482  * do not care (such as procfs interfaces).
483  */
484 ino_t page_cgroup_ino(struct page *page)
485 {
486         struct mem_cgroup *memcg;
487         unsigned long ino = 0;
488 
489         rcu_read_lock();
490         memcg = READ_ONCE(page->mem_cgroup);
491         while (memcg && !(memcg->css.flags & CSS_ONLINE))
492                 memcg = parent_mem_cgroup(memcg);
493         if (memcg)
494                 ino = cgroup_ino(memcg->css.cgroup);
495         rcu_read_unlock();
496         return ino;
497 }
498 
499 static struct mem_cgroup_per_node *
500 mem_cgroup_page_nodeinfo(struct mem_cgroup *memcg, struct page *page)
501 {
502         int nid = page_to_nid(page);
503 
504         return memcg->nodeinfo[nid];
505 }
506 
507 static struct mem_cgroup_tree_per_node *
508 soft_limit_tree_node(int nid)
509 {
510         return soft_limit_tree.rb_tree_per_node[nid];
511 }
512 
513 static struct mem_cgroup_tree_per_node *
514 soft_limit_tree_from_page(struct page *page)
515 {
516         int nid = page_to_nid(page);
517 
518         return soft_limit_tree.rb_tree_per_node[nid];
519 }
520 
521 static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz,
522                                          struct mem_cgroup_tree_per_node *mctz,
523                                          unsigned long new_usage_in_excess)
524 {
525         struct rb_node **p = &mctz->rb_root.rb_node;
526         struct rb_node *parent = NULL;
527         struct mem_cgroup_per_node *mz_node;
528         bool rightmost = true;
529 
530         if (mz->on_tree)
531                 return;
532 
533         mz->usage_in_excess = new_usage_in_excess;
534         if (!mz->usage_in_excess)
535                 return;
536         while (*p) {
537                 parent = *p;
538                 mz_node = rb_entry(parent, struct mem_cgroup_per_node,
539                                         tree_node);
540                 if (mz->usage_in_excess < mz_node->usage_in_excess) {
541                         p = &(*p)->rb_left;
542                         rightmost = false;
543                 }
544 
545                 /*
546                  * We can't avoid mem cgroups that are over their soft
547                  * limit by the same amount
548                  */
549                 else if (mz->usage_in_excess >= mz_node->usage_in_excess)
550                         p = &(*p)->rb_right;
551         }
552 
553         if (rightmost)
554                 mctz->rb_rightmost = &mz->tree_node;
555 
556         rb_link_node(&mz->tree_node, parent, p);
557         rb_insert_color(&mz->tree_node, &mctz->rb_root);
558         mz->on_tree = true;
559 }
560 
561 static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
562                                          struct mem_cgroup_tree_per_node *mctz)
563 {
564         if (!mz->on_tree)
565                 return;
566 
567         if (&mz->tree_node == mctz->rb_rightmost)
568                 mctz->rb_rightmost = rb_prev(&mz->tree_node);
569 
570         rb_erase(&mz->tree_node, &mctz->rb_root);
571         mz->on_tree = false;
572 }
573 
574 static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
575                                        struct mem_cgroup_tree_per_node *mctz)
576 {
577         unsigned long flags;
578 
579         spin_lock_irqsave(&mctz->lock, flags);
580         __mem_cgroup_remove_exceeded(mz, mctz);
581         spin_unlock_irqrestore(&mctz->lock, flags);
582 }
583 
584 static unsigned long soft_limit_excess(struct mem_cgroup *memcg)
585 {
586         unsigned long nr_pages = page_counter_read(&memcg->memory);
587         unsigned long soft_limit = READ_ONCE(memcg->soft_limit);
588         unsigned long excess = 0;
589 
590         if (nr_pages > soft_limit)
591                 excess = nr_pages - soft_limit;
592 
593         return excess;
594 }
595 
596 static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
597 {
598         unsigned long excess;
599         struct mem_cgroup_per_node *mz;
600         struct mem_cgroup_tree_per_node *mctz;
601 
602         mctz = soft_limit_tree_from_page(page);
603         if (!mctz)
604                 return;
605         /*
606          * Necessary to update all ancestors when hierarchy is used.
607          * because their event counter is not touched.
608          */
609         for (; memcg; memcg = parent_mem_cgroup(memcg)) {
610                 mz = mem_cgroup_page_nodeinfo(memcg, page);
611                 excess = soft_limit_excess(memcg);
612                 /*
613                  * We have to update the tree if mz is on RB-tree or
614                  * mem is over its softlimit.
615                  */
616                 if (excess || mz->on_tree) {
617                         unsigned long flags;
618 
619                         spin_lock_irqsave(&mctz->lock, flags);
620                         /* if on-tree, remove it */
621                         if (mz->on_tree)
622                                 __mem_cgroup_remove_exceeded(mz, mctz);
623                         /*
624                          * Insert again. mz->usage_in_excess will be updated.
625                          * If excess is 0, no tree ops.
626                          */
627                         __mem_cgroup_insert_exceeded(mz, mctz, excess);
628                         spin_unlock_irqrestore(&mctz->lock, flags);
629                 }
630         }
631 }
632 
633 static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
634 {
635         struct mem_cgroup_tree_per_node *mctz;
636         struct mem_cgroup_per_node *mz;
637         int nid;
638 
639         for_each_node(nid) {
640                 mz = mem_cgroup_nodeinfo(memcg, nid);
641                 mctz = soft_limit_tree_node(nid);
642                 if (mctz)
643                         mem_cgroup_remove_exceeded(mz, mctz);
644         }
645 }
646 
647 static struct mem_cgroup_per_node *
648 __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
649 {
650         struct mem_cgroup_per_node *mz;
651 
652 retry:
653         mz = NULL;
654         if (!mctz->rb_rightmost)
655                 goto done;              /* Nothing to reclaim from */
656 
657         mz = rb_entry(mctz->rb_rightmost,
658                       struct mem_cgroup_per_node, tree_node);
659         /*
660          * Remove the node now but someone else can add it back,
661          * we will to add it back at the end of reclaim to its correct
662          * position in the tree.
663          */
664         __mem_cgroup_remove_exceeded(mz, mctz);
665         if (!soft_limit_excess(mz->memcg) ||
666             !css_tryget_online(&mz->memcg->css))
667                 goto retry;
668 done:
669         return mz;
670 }
671 
672 static struct mem_cgroup_per_node *
673 mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
674 {
675         struct mem_cgroup_per_node *mz;
676 
677         spin_lock_irq(&mctz->lock);
678         mz = __mem_cgroup_largest_soft_limit_node(mctz);
679         spin_unlock_irq(&mctz->lock);
680         return mz;
681 }
682 
683 static unsigned long memcg_sum_events(struct mem_cgroup *memcg,
684                                       int event)
685 {
686         return atomic_long_read(&memcg->events[event]);
687 }
688 
689 static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
690                                          struct page *page,
691                                          bool compound, int nr_pages)
692 {
693         /*
694          * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is
695          * counted as CACHE even if it's on ANON LRU.
696          */
697         if (PageAnon(page))
698                 __mod_memcg_state(memcg, MEMCG_RSS, nr_pages);
699         else {
700                 __mod_memcg_state(memcg, MEMCG_CACHE, nr_pages);
701                 if (PageSwapBacked(page))
702                         __mod_memcg_state(memcg, NR_SHMEM, nr_pages);
703         }
704 
705         if (compound) {
706                 VM_BUG_ON_PAGE(!PageTransHuge(page), page);
707                 __mod_memcg_state(memcg, MEMCG_RSS_HUGE, nr_pages);
708         }
709 
710         /* pagein of a big page is an event. So, ignore page size */
711         if (nr_pages > 0)
712                 __count_memcg_events(memcg, PGPGIN, 1);
713         else {
714                 __count_memcg_events(memcg, PGPGOUT, 1);
715                 nr_pages = -nr_pages; /* for event */
716         }
717 
718         __this_cpu_add(memcg->stat_cpu->nr_page_events, nr_pages);
719 }
720 
721 unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
722                                            int nid, unsigned int lru_mask)
723 {
724         struct lruvec *lruvec = mem_cgroup_lruvec(NODE_DATA(nid), memcg);
725         unsigned long nr = 0;
726         enum lru_list lru;
727 
728         VM_BUG_ON((unsigned)nid >= nr_node_ids);
729 
730         for_each_lru(lru) {
731                 if (!(BIT(lru) & lru_mask))
732                         continue;
733                 nr += mem_cgroup_get_lru_size(lruvec, lru);
734         }
735         return nr;
736 }
737 
738 static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
739                         unsigned int lru_mask)
740 {
741         unsigned long nr = 0;
742         int nid;
743 
744         for_each_node_state(nid, N_MEMORY)
745                 nr += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask);
746         return nr;
747 }
748 
749 static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
750                                        enum mem_cgroup_events_target target)
751 {
752         unsigned long val, next;
753 
754         val = __this_cpu_read(memcg->stat_cpu->nr_page_events);
755         next = __this_cpu_read(memcg->stat_cpu->targets[target]);
756         /* from time_after() in jiffies.h */
757         if ((long)(next - val) < 0) {
758                 switch (target) {
759                 case MEM_CGROUP_TARGET_THRESH:
760                         next = val + THRESHOLDS_EVENTS_TARGET;
761                         break;
762                 case MEM_CGROUP_TARGET_SOFTLIMIT:
763                         next = val + SOFTLIMIT_EVENTS_TARGET;
764                         break;
765                 case MEM_CGROUP_TARGET_NUMAINFO:
766                         next = val + NUMAINFO_EVENTS_TARGET;
767                         break;
768                 default:
769                         break;
770                 }
771                 __this_cpu_write(memcg->stat_cpu->targets[target], next);
772                 return true;
773         }
774         return false;
775 }
776 
777 /*
778  * Check events in order.
779  *
780  */
781 static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
782 {
783         /* threshold event is triggered in finer grain than soft limit */
784         if (unlikely(mem_cgroup_event_ratelimit(memcg,
785                                                 MEM_CGROUP_TARGET_THRESH))) {
786                 bool do_softlimit;
787                 bool do_numainfo __maybe_unused;
788 
789                 do_softlimit = mem_cgroup_event_ratelimit(memcg,
790                                                 MEM_CGROUP_TARGET_SOFTLIMIT);
791 #if MAX_NUMNODES > 1
792                 do_numainfo = mem_cgroup_event_ratelimit(memcg,
793                                                 MEM_CGROUP_TARGET_NUMAINFO);
794 #endif
795                 mem_cgroup_threshold(memcg);
796                 if (unlikely(do_softlimit))
797                         mem_cgroup_update_tree(memcg, page);
798 #if MAX_NUMNODES > 1
799                 if (unlikely(do_numainfo))
800                         atomic_inc(&memcg->numainfo_events);
801 #endif
802         }
803 }
804 
805 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
806 {
807         /*
808          * mm_update_next_owner() may clear mm->owner to NULL
809          * if it races with swapoff, page migration, etc.
810          * So this can be called with p == NULL.
811          */
812         if (unlikely(!p))
813                 return NULL;
814 
815         return mem_cgroup_from_css(task_css(p, memory_cgrp_id));
816 }
817 EXPORT_SYMBOL(mem_cgroup_from_task);
818 
819 /**
820  * get_mem_cgroup_from_mm: Obtain a reference on given mm_struct's memcg.
821  * @mm: mm from which memcg should be extracted. It can be NULL.
822  *
823  * Obtain a reference on mm->memcg and returns it if successful. Otherwise
824  * root_mem_cgroup is returned. However if mem_cgroup is disabled, NULL is
825  * returned.
826  */
827 struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
828 {
829         struct mem_cgroup *memcg;
830 
831         if (mem_cgroup_disabled())
832                 return NULL;
833 
834         rcu_read_lock();
835         do {
836                 /*
837                  * Page cache insertions can happen withou an
838                  * actual mm context, e.g. during disk probing
839                  * on boot, loopback IO, acct() writes etc.
840                  */
841                 if (unlikely(!mm))
842                         memcg = root_mem_cgroup;
843                 else {
844                         memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
845                         if (unlikely(!memcg))
846                                 memcg = root_mem_cgroup;
847                 }
848         } while (!css_tryget_online(&memcg->css));
849         rcu_read_unlock();
850         return memcg;
851 }
852 EXPORT_SYMBOL(get_mem_cgroup_from_mm);
853 
854 /**
855  * get_mem_cgroup_from_page: Obtain a reference on given page's memcg.
856  * @page: page from which memcg should be extracted.
857  *
858  * Obtain a reference on page->memcg and returns it if successful. Otherwise
859  * root_mem_cgroup is returned.
860  */
861 struct mem_cgroup *get_mem_cgroup_from_page(struct page *page)
862 {
863         struct mem_cgroup *memcg = page->mem_cgroup;
864 
865         if (mem_cgroup_disabled())
866                 return NULL;
867 
868         rcu_read_lock();
869         if (!memcg || !css_tryget_online(&memcg->css))
870                 memcg = root_mem_cgroup;
871         rcu_read_unlock();
872         return memcg;
873 }
874 EXPORT_SYMBOL(get_mem_cgroup_from_page);
875 
876 /**
877  * If current->active_memcg is non-NULL, do not fallback to current->mm->memcg.
878  */
879 static __always_inline struct mem_cgroup *get_mem_cgroup_from_current(void)
880 {
881         if (unlikely(current->active_memcg)) {
882                 struct mem_cgroup *memcg = root_mem_cgroup;
883 
884                 rcu_read_lock();
885                 if (css_tryget_online(&current->active_memcg->css))
886                         memcg = current->active_memcg;
887                 rcu_read_unlock();
888                 return memcg;
889         }
890         return get_mem_cgroup_from_mm(current->mm);
891 }
892 
893 /**
894  * mem_cgroup_iter - iterate over memory cgroup hierarchy
895  * @root: hierarchy root
896  * @prev: previously returned memcg, NULL on first invocation
897  * @reclaim: cookie for shared reclaim walks, NULL for full walks
898  *
899  * Returns references to children of the hierarchy below @root, or
900  * @root itself, or %NULL after a full round-trip.
901  *
902  * Caller must pass the return value in @prev on subsequent
903  * invocations for reference counting, or use mem_cgroup_iter_break()
904  * to cancel a hierarchy walk before the round-trip is complete.
905  *
906  * Reclaimers can specify a node and a priority level in @reclaim to
907  * divide up the memcgs in the hierarchy among all concurrent
908  * reclaimers operating on the same node and priority.
909  */
910 struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
911                                    struct mem_cgroup *prev,
912                                    struct mem_cgroup_reclaim_cookie *reclaim)
913 {
914         struct mem_cgroup_reclaim_iter *uninitialized_var(iter);
915         struct cgroup_subsys_state *css = NULL;
916         struct mem_cgroup *memcg = NULL;
917         struct mem_cgroup *pos = NULL;
918 
919         if (mem_cgroup_disabled())
920                 return NULL;
921 
922         if (!root)
923                 root = root_mem_cgroup;
924 
925         if (prev && !reclaim)
926                 pos = prev;
927 
928         if (!root->use_hierarchy && root != root_mem_cgroup) {
929                 if (prev)
930                         goto out;
931                 return root;
932         }
933 
934         rcu_read_lock();
935 
936         if (reclaim) {
937                 struct mem_cgroup_per_node *mz;
938 
939                 mz = mem_cgroup_nodeinfo(root, reclaim->pgdat->node_id);
940                 iter = &mz->iter[reclaim->priority];
941 
942                 if (prev && reclaim->generation != iter->generation)
943                         goto out_unlock;
944 
945                 while (1) {
946                         pos = READ_ONCE(iter->position);
947                         if (!pos || css_tryget(&pos->css))
948                                 break;
949                         /*
950                          * css reference reached zero, so iter->position will
951                          * be cleared by ->css_released. However, we should not
952                          * rely on this happening soon, because ->css_released
953                          * is called from a work queue, and by busy-waiting we
954                          * might block it. So we clear iter->position right
955                          * away.
956                          */
957                         (void)cmpxchg(&iter->position, pos, NULL);
958                 }
959         }
960 
961         if (pos)
962                 css = &pos->css;
963 
964         for (;;) {
965                 css = css_next_descendant_pre(css, &root->css);
966                 if (!css) {
967                         /*
968                          * Reclaimers share the hierarchy walk, and a
969                          * new one might jump in right at the end of
970                          * the hierarchy - make sure they see at least
971                          * one group and restart from the beginning.
972                          */
973                         if (!prev)
974                                 continue;
975                         break;
976                 }
977 
978                 /*
979                  * Verify the css and acquire a reference.  The root
980                  * is provided by the caller, so we know it's alive
981                  * and kicking, and don't take an extra reference.
982                  */
983                 memcg = mem_cgroup_from_css(css);
984 
985                 if (css == &root->css)
986                         break;
987 
988                 if (css_tryget(css))
989                         break;
990 
991                 memcg = NULL;
992         }
993 
994         if (reclaim) {
995                 /*
996                  * The position could have already been updated by a competing
997                  * thread, so check that the value hasn't changed since we read
998                  * it to avoid reclaiming from the same cgroup twice.
999                  */
1000                 (void)cmpxchg(&iter->position, pos, memcg);
1001 
1002                 if (pos)
1003                         css_put(&pos->css);
1004 
1005                 if (!memcg)
1006                         iter->generation++;
1007                 else if (!prev)
1008                         reclaim->generation = iter->generation;
1009         }
1010 
1011 out_unlock:
1012         rcu_read_unlock();
1013 out:
1014         if (prev && prev != root)
1015                 css_put(&prev->css);
1016 
1017         return memcg;
1018 }
1019 
1020 /**
1021  * mem_cgroup_iter_break - abort a hierarchy walk prematurely
1022  * @root: hierarchy root
1023  * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
1024  */
1025 void mem_cgroup_iter_break(struct mem_cgroup *root,
1026                            struct mem_cgroup *prev)
1027 {
1028         if (!root)
1029                 root = root_mem_cgroup;
1030         if (prev && prev != root)
1031                 css_put(&prev->css);
1032 }
1033 
1034 static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg)
1035 {
1036         struct mem_cgroup *memcg = dead_memcg;
1037         struct mem_cgroup_reclaim_iter *iter;
1038         struct mem_cgroup_per_node *mz;
1039         int nid;
1040         int i;
1041 
1042         for (; memcg; memcg = parent_mem_cgroup(memcg)) {
1043                 for_each_node(nid) {
1044                         mz = mem_cgroup_nodeinfo(memcg, nid);
1045                         for (i = 0; i <= DEF_PRIORITY; i++) {
1046                                 iter = &mz->iter[i];
1047                                 cmpxchg(&iter->position,
1048                                         dead_memcg, NULL);
1049                         }
1050                 }
1051         }
1052 }
1053 
1054 /**
1055  * mem_cgroup_scan_tasks - iterate over tasks of a memory cgroup hierarchy
1056  * @memcg: hierarchy root
1057  * @fn: function to call for each task
1058  * @arg: argument passed to @fn
1059  *
1060  * This function iterates over tasks attached to @memcg or to any of its
1061  * descendants and calls @fn for each task. If @fn returns a non-zero
1062  * value, the function breaks the iteration loop and returns the value.
1063  * Otherwise, it will iterate over all tasks and return 0.
1064  *
1065  * This function must not be called for the root memory cgroup.
1066  */
1067 int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
1068                           int (*fn)(struct task_struct *, void *), void *arg)
1069 {
1070         struct mem_cgroup *iter;
1071         int ret = 0;
1072 
1073         BUG_ON(memcg == root_mem_cgroup);
1074 
1075         for_each_mem_cgroup_tree(iter, memcg) {
1076                 struct css_task_iter it;
1077                 struct task_struct *task;
1078 
1079                 css_task_iter_start(&iter->css, 0, &it);
1080                 while (!ret && (task = css_task_iter_next(&it)))
1081                         ret = fn(task, arg);
1082                 css_task_iter_end(&it);
1083                 if (ret) {
1084                         mem_cgroup_iter_break(memcg, iter);
1085                         break;
1086                 }
1087         }
1088         return ret;
1089 }
1090 
1091 /**
1092  * mem_cgroup_page_lruvec - return lruvec for isolating/putting an LRU page
1093  * @page: the page
1094  * @pgdat: pgdat of the page
1095  *
1096  * This function is only safe when following the LRU page isolation
1097  * and putback protocol: the LRU lock must be held, and the page must
1098  * either be PageLRU() or the caller must have isolated/allocated it.
1099  */
1100 struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct pglist_data *pgdat)
1101 {
1102         struct mem_cgroup_per_node *mz;
1103         struct mem_cgroup *memcg;
1104         struct lruvec *lruvec;
1105 
1106         if (mem_cgroup_disabled()) {
1107                 lruvec = &pgdat->lruvec;
1108                 goto out;
1109         }
1110 
1111         memcg = page->mem_cgroup;
1112         /*
1113          * Swapcache readahead pages are added to the LRU - and
1114          * possibly migrated - before they are charged.
1115          */
1116         if (!memcg)
1117                 memcg = root_mem_cgroup;
1118 
1119         mz = mem_cgroup_page_nodeinfo(memcg, page);
1120         lruvec = &mz->lruvec;
1121 out:
1122         /*
1123          * Since a node can be onlined after the mem_cgroup was created,
1124          * we have to be prepared to initialize lruvec->zone here;
1125          * and if offlined then reonlined, we need to reinitialize it.
1126          */
1127         if (unlikely(lruvec->pgdat != pgdat))
1128                 lruvec->pgdat = pgdat;
1129         return lruvec;
1130 }
1131 
1132 /**
1133  * mem_cgroup_update_lru_size - account for adding or removing an lru page
1134  * @lruvec: mem_cgroup per zone lru vector
1135  * @lru: index of lru list the page is sitting on
1136  * @zid: zone id of the accounted pages
1137  * @nr_pages: positive when adding or negative when removing
1138  *
1139  * This function must be called under lru_lock, just before a page is added
1140  * to or just after a page is removed from an lru list (that ordering being
1141  * so as to allow it to check that lru_size 0 is consistent with list_empty).
1142  */
1143 void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
1144                                 int zid, int nr_pages)
1145 {
1146         struct mem_cgroup_per_node *mz;
1147         unsigned long *lru_size;
1148         long size;
1149 
1150         if (mem_cgroup_disabled())
1151                 return;
1152 
1153         mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
1154         lru_size = &mz->lru_zone_size[zid][lru];
1155 
1156         if (nr_pages < 0)
1157                 *lru_size += nr_pages;
1158 
1159         size = *lru_size;
1160         if (WARN_ONCE(size < 0,
1161                 "%s(%p, %d, %d): lru_size %ld\n",
1162                 __func__, lruvec, lru, nr_pages, size)) {
1163                 VM_BUG_ON(1);
1164                 *lru_size = 0;
1165         }
1166 
1167         if (nr_pages > 0)
1168                 *lru_size += nr_pages;
1169 }
1170 
1171 bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg)
1172 {
1173         struct mem_cgroup *task_memcg;
1174         struct task_struct *p;
1175         bool ret;
1176 
1177         p = find_lock_task_mm(task);
1178         if (p) {
1179                 task_memcg = get_mem_cgroup_from_mm(p->mm);
1180                 task_unlock(p);
1181         } else {
1182                 /*
1183                  * All threads may have already detached their mm's, but the oom
1184                  * killer still needs to detect if they have already been oom
1185                  * killed to prevent needlessly killing additional tasks.
1186                  */
1187                 rcu_read_lock();
1188                 task_memcg = mem_cgroup_from_task(task);
1189                 css_get(&task_memcg->css);
1190                 rcu_read_unlock();
1191         }
1192         ret = mem_cgroup_is_descendant(task_memcg, memcg);
1193         css_put(&task_memcg->css);
1194         return ret;
1195 }
1196 
1197 /**
1198  * mem_cgroup_margin - calculate chargeable space of a memory cgroup
1199  * @memcg: the memory cgroup
1200  *
1201  * Returns the maximum amount of memory @mem can be charged with, in
1202  * pages.
1203  */
1204 static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
1205 {
1206         unsigned long margin = 0;
1207         unsigned long count;
1208         unsigned long limit;
1209 
1210         count = page_counter_read(&memcg->memory);
1211         limit = READ_ONCE(memcg->memory.max);
1212         if (count < limit)
1213                 margin = limit - count;
1214 
1215         if (do_memsw_account()) {
1216                 count = page_counter_read(&memcg->memsw);
1217                 limit = READ_ONCE(memcg->memsw.max);
1218                 if (count <= limit)
1219                         margin = min(margin, limit - count);
1220                 else
1221                         margin = 0;
1222         }
1223 
1224         return margin;
1225 }
1226 
1227 /*
1228  * A routine for checking "mem" is under move_account() or not.
1229  *
1230  * Checking a cgroup is mc.from or mc.to or under hierarchy of
1231  * moving cgroups. This is for waiting at high-memory pressure
1232  * caused by "move".
1233  */
1234 static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
1235 {
1236         struct mem_cgroup *from;
1237         struct mem_cgroup *to;
1238         bool ret = false;
1239         /*
1240          * Unlike task_move routines, we access mc.to, mc.from not under
1241          * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
1242          */
1243         spin_lock(&mc.lock);
1244         from = mc.from;
1245         to = mc.to;
1246         if (!from)
1247                 goto unlock;
1248 
1249         ret = mem_cgroup_is_descendant(from, memcg) ||
1250                 mem_cgroup_is_descendant(to, memcg);
1251 unlock:
1252         spin_unlock(&mc.lock);
1253         return ret;
1254 }
1255 
1256 static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
1257 {
1258         if (mc.moving_task && current != mc.moving_task) {
1259                 if (mem_cgroup_under_move(memcg)) {
1260                         DEFINE_WAIT(wait);
1261                         prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
1262                         /* moving charge context might have finished. */
1263                         if (mc.moving_task)
1264                                 schedule();
1265                         finish_wait(&mc.waitq, &wait);
1266                         return true;
1267                 }
1268         }
1269         return false;
1270 }
1271 
1272 static const unsigned int memcg1_stats[] = {
1273         MEMCG_CACHE,
1274         MEMCG_RSS,
1275         MEMCG_RSS_HUGE,
1276         NR_SHMEM,
1277         NR_FILE_MAPPED,
1278         NR_FILE_DIRTY,
1279         NR_WRITEBACK,
1280         MEMCG_SWAP,
1281 };
1282 
1283 static const char *const memcg1_stat_names[] = {
1284         "cache",
1285         "rss",
1286         "rss_huge",
1287         "shmem",
1288         "mapped_file",
1289         "dirty",
1290         "writeback",
1291         "swap",
1292 };
1293 
1294 #define K(x) ((x) << (PAGE_SHIFT-10))
1295 /**
1296  * mem_cgroup_print_oom_info: Print OOM information relevant to memory controller.
1297  * @memcg: The memory cgroup that went over limit
1298  * @p: Task that is going to be killed
1299  *
1300  * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1301  * enabled
1302  */
1303 void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
1304 {
1305         struct mem_cgroup *iter;
1306         unsigned int i;
1307 
1308         rcu_read_lock();
1309 
1310         if (p) {
1311                 pr_info("Task in ");
1312                 pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id));
1313                 pr_cont(" killed as a result of limit of ");
1314         } else {
1315                 pr_info("Memory limit reached of cgroup ");
1316         }
1317 
1318         pr_cont_cgroup_path(memcg->css.cgroup);
1319         pr_cont("\n");
1320 
1321         rcu_read_unlock();
1322 
1323         pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n",
1324                 K((u64)page_counter_read(&memcg->memory)),
1325                 K((u64)memcg->memory.max), memcg->memory.failcnt);
1326         pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n",
1327                 K((u64)page_counter_read(&memcg->memsw)),
1328                 K((u64)memcg->memsw.max), memcg->memsw.failcnt);
1329         pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n",
1330                 K((u64)page_counter_read(&memcg->kmem)),
1331                 K((u64)memcg->kmem.max), memcg->kmem.failcnt);
1332 
1333         for_each_mem_cgroup_tree(iter, memcg) {
1334                 pr_info("Memory cgroup stats for ");
1335                 pr_cont_cgroup_path(iter->css.cgroup);
1336                 pr_cont(":");
1337 
1338                 for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
1339                         if (memcg1_stats[i] == MEMCG_SWAP && !do_swap_account)
1340                                 continue;
1341                         pr_cont(" %s:%luKB", memcg1_stat_names[i],
1342                                 K(memcg_page_state(iter, memcg1_stats[i])));
1343                 }
1344 
1345                 for (i = 0; i < NR_LRU_LISTS; i++)
1346                         pr_cont(" %s:%luKB", mem_cgroup_lru_names[i],
1347                                 K(mem_cgroup_nr_lru_pages(iter, BIT(i))));
1348 
1349                 pr_cont("\n");
1350         }
1351 }
1352 
1353 /*
1354  * Return the memory (and swap, if configured) limit for a memcg.
1355  */
1356 unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg)
1357 {
1358         unsigned long max;
1359 
1360         max = memcg->memory.max;
1361         if (mem_cgroup_swappiness(memcg)) {
1362                 unsigned long memsw_max;
1363                 unsigned long swap_max;
1364 
1365                 memsw_max = memcg->memsw.max;
1366                 swap_max = memcg->swap.max;
1367                 swap_max = min(swap_max, (unsigned long)total_swap_pages);
1368                 max = min(max + swap_max, memsw_max);
1369         }
1370         return max;
1371 }
1372 
1373 static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
1374                                      int order)
1375 {
1376         struct oom_control oc = {
1377                 .zonelist = NULL,
1378                 .nodemask = NULL,
1379                 .memcg = memcg,
1380                 .gfp_mask = gfp_mask,
1381                 .order = order,
1382         };
1383         bool ret;
1384 
1385         mutex_lock(&oom_lock);
1386         ret = out_of_memory(&oc);
1387         mutex_unlock(&oom_lock);
1388         return ret;
1389 }
1390 
1391 #if MAX_NUMNODES > 1
1392 
1393 /**
1394  * test_mem_cgroup_node_reclaimable
1395  * @memcg: the target memcg
1396  * @nid: the node ID to be checked.
1397  * @noswap : specify true here if the user wants flle only information.
1398  *
1399  * This function returns whether the specified memcg contains any
1400  * reclaimable pages on a node. Returns true if there are any reclaimable
1401  * pages in the node.
1402  */
1403 static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
1404                 int nid, bool noswap)
1405 {
1406         if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE))
1407                 return true;
1408         if (noswap || !total_swap_pages)
1409                 return false;
1410         if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON))
1411                 return true;
1412         return false;
1413 
1414 }
1415 
1416 /*
1417  * Always updating the nodemask is not very good - even if we have an empty
1418  * list or the wrong list here, we can start from some node and traverse all
1419  * nodes based on the zonelist. So update the list loosely once per 10 secs.
1420  *
1421  */
1422 static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
1423 {
1424         int nid;
1425         /*
1426          * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
1427          * pagein/pageout changes since the last update.
1428          */
1429         if (!atomic_read(&memcg->numainfo_events))
1430                 return;
1431         if (atomic_inc_return(&memcg->numainfo_updating) > 1)
1432                 return;
1433 
1434         /* make a nodemask where this memcg uses memory from */
1435         memcg->scan_nodes = node_states[N_MEMORY];
1436 
1437         for_each_node_mask(nid, node_states[N_MEMORY]) {
1438 
1439                 if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
1440                         node_clear(nid, memcg->scan_nodes);
1441         }
1442 
1443         atomic_set(&memcg->numainfo_events, 0);
1444         atomic_set(&memcg->numainfo_updating, 0);
1445 }
1446 
1447 /*
1448  * Selecting a node where we start reclaim from. Because what we need is just
1449  * reducing usage counter, start from anywhere is O,K. Considering
1450  * memory reclaim from current node, there are pros. and cons.
1451  *
1452  * Freeing memory from current node means freeing memory from a node which
1453  * we'll use or we've used. So, it may make LRU bad. And if several threads
1454  * hit limits, it will see a contention on a node. But freeing from remote
1455  * node means more costs for memory reclaim because of memory latency.
1456  *
1457  * Now, we use round-robin. Better algorithm is welcomed.
1458  */
1459 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1460 {
1461         int node;
1462 
1463         mem_cgroup_may_update_nodemask(memcg);
1464         node = memcg->last_scanned_node;
1465 
1466         node = next_node_in(node, memcg->scan_nodes);
1467         /*
1468          * mem_cgroup_may_update_nodemask might have seen no reclaimmable pages
1469          * last time it really checked all the LRUs due to rate limiting.
1470          * Fallback to the current node in that case for simplicity.
1471          */
1472         if (unlikely(node == MAX_NUMNODES))
1473                 node = numa_node_id();
1474 
1475         memcg->last_scanned_node = node;
1476         return node;
1477 }
1478 #else
1479 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1480 {
1481         return 0;
1482 }
1483 #endif
1484 
1485 static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
1486                                    pg_data_t *pgdat,
1487                                    gfp_t gfp_mask,
1488                                    unsigned long *total_scanned)
1489 {
1490         struct mem_cgroup *victim = NULL;
1491         int total = 0;
1492         int loop = 0;
1493         unsigned long excess;
1494         unsigned long nr_scanned;
1495         struct mem_cgroup_reclaim_cookie reclaim = {
1496                 .pgdat = pgdat,
1497                 .priority = 0,
1498         };
1499 
1500         excess = soft_limit_excess(root_memcg);
1501 
1502         while (1) {
1503                 victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
1504                 if (!victim) {
1505                         loop++;
1506                         if (loop >= 2) {
1507                                 /*
1508                                  * If we have not been able to reclaim
1509                                  * anything, it might because there are
1510                                  * no reclaimable pages under this hierarchy
1511                                  */
1512                                 if (!total)
1513                                         break;
1514                                 /*
1515                                  * We want to do more targeted reclaim.
1516                                  * excess >> 2 is not to excessive so as to
1517                                  * reclaim too much, nor too less that we keep
1518                                  * coming back to reclaim from this cgroup
1519                                  */
1520                                 if (total >= (excess >> 2) ||
1521                                         (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
1522                                         break;
1523                         }
1524                         continue;
1525                 }
1526                 total += mem_cgroup_shrink_node(victim, gfp_mask, false,
1527                                         pgdat, &nr_scanned);
1528                 *total_scanned += nr_scanned;
1529                 if (!soft_limit_excess(root_memcg))
1530                         break;
1531         }
1532         mem_cgroup_iter_break(root_memcg, victim);
1533         return total;
1534 }
1535 
1536 #ifdef CONFIG_LOCKDEP
1537 static struct lockdep_map memcg_oom_lock_dep_map = {
1538         .name = "memcg_oom_lock",
1539 };
1540 #endif
1541 
1542 static DEFINE_SPINLOCK(memcg_oom_lock);
1543 
1544 /*
1545  * Check OOM-Killer is already running under our hierarchy.
1546  * If someone is running, return false.
1547  */
1548 static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
1549 {
1550         struct mem_cgroup *iter, *failed = NULL;
1551 
1552         spin_lock(&memcg_oom_lock);
1553 
1554         for_each_mem_cgroup_tree(iter, memcg) {
1555                 if (iter->oom_lock) {
1556                         /*
1557                          * this subtree of our hierarchy is already locked
1558                          * so we cannot give a lock.
1559                          */
1560                         failed = iter;
1561                         mem_cgroup_iter_break(memcg, iter);
1562                         break;
1563                 } else
1564                         iter->oom_lock = true;
1565         }
1566 
1567         if (failed) {
1568                 /*
1569                  * OK, we failed to lock the whole subtree so we have
1570                  * to clean up what we set up to the failing subtree
1571                  */
1572                 for_each_mem_cgroup_tree(iter, memcg) {
1573                         if (iter == failed) {
1574                                 mem_cgroup_iter_break(memcg, iter);
1575                                 break;
1576                         }
1577                         iter->oom_lock = false;
1578                 }
1579         } else
1580                 mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_);
1581 
1582         spin_unlock(&memcg_oom_lock);
1583 
1584         return !failed;
1585 }
1586 
1587 static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
1588 {
1589         struct mem_cgroup *iter;
1590 
1591         spin_lock(&memcg_oom_lock);
1592         mutex_release(&memcg_oom_lock_dep_map, 1, _RET_IP_);
1593         for_each_mem_cgroup_tree(iter, memcg)
1594                 iter->oom_lock = false;
1595         spin_unlock(&memcg_oom_lock);
1596 }
1597 
1598 static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
1599 {
1600         struct mem_cgroup *iter;
1601 
1602         spin_lock(&memcg_oom_lock);
1603         for_each_mem_cgroup_tree(iter, memcg)
1604                 iter->under_oom++;
1605         spin_unlock(&memcg_oom_lock);
1606 }
1607 
1608 static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
1609 {
1610         struct mem_cgroup *iter;
1611 
1612         /*
1613          * When a new child is created while the hierarchy is under oom,
1614          * mem_cgroup_oom_lock() may not be called. Watch for underflow.
1615          */
1616         spin_lock(&memcg_oom_lock);
1617         for_each_mem_cgroup_tree(iter, memcg)
1618                 if (iter->under_oom > 0)
1619                         iter->under_oom--;
1620         spin_unlock(&memcg_oom_lock);
1621 }
1622 
1623 static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
1624 
1625 struct oom_wait_info {
1626         struct mem_cgroup *memcg;
1627         wait_queue_entry_t      wait;
1628 };
1629 
1630 static int memcg_oom_wake_function(wait_queue_entry_t *wait,
1631         unsigned mode, int sync, void *arg)
1632 {
1633         struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
1634         struct mem_cgroup *oom_wait_memcg;
1635         struct oom_wait_info *oom_wait_info;
1636 
1637         oom_wait_info = container_of(wait, struct oom_wait_info, wait);
1638         oom_wait_memcg = oom_wait_info->memcg;
1639 
1640         if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) &&
1641             !mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg))
1642                 return 0;
1643         return autoremove_wake_function(wait, mode, sync, arg);
1644 }
1645 
1646 static void memcg_oom_recover(struct mem_cgroup *memcg)
1647 {
1648         /*
1649          * For the following lockless ->under_oom test, the only required
1650          * guarantee is that it must see the state asserted by an OOM when
1651          * this function is called as a result of userland actions
1652          * triggered by the notification of the OOM.  This is trivially
1653          * achieved by invoking mem_cgroup_mark_under_oom() before
1654          * triggering notification.
1655          */
1656         if (memcg && memcg->under_oom)
1657                 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
1658 }
1659 
1660 enum oom_status {
1661         OOM_SUCCESS,
1662         OOM_FAILED,
1663         OOM_ASYNC,
1664         OOM_SKIPPED
1665 };
1666 
1667 static enum oom_status mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
1668 {
1669         if (order > PAGE_ALLOC_COSTLY_ORDER)
1670                 return OOM_SKIPPED;
1671 
1672         /*
1673          * We are in the middle of the charge context here, so we
1674          * don't want to block when potentially sitting on a callstack
1675          * that holds all kinds of filesystem and mm locks.
1676          *
1677          * cgroup1 allows disabling the OOM killer and waiting for outside
1678          * handling until the charge can succeed; remember the context and put
1679          * the task to sleep at the end of the page fault when all locks are
1680          * released.
1681          *
1682          * On the other hand, in-kernel OOM killer allows for an async victim
1683          * memory reclaim (oom_reaper) and that means that we are not solely
1684          * relying on the oom victim to make a forward progress and we can
1685          * invoke the oom killer here.
1686          *
1687          * Please note that mem_cgroup_out_of_memory might fail to find a
1688          * victim and then we have to bail out from the charge path.
1689          */
1690         if (memcg->oom_kill_disable) {
1691                 if (!current->in_user_fault)
1692                         return OOM_SKIPPED;
1693                 css_get(&memcg->css);
1694                 current->memcg_in_oom = memcg;
1695                 current->memcg_oom_gfp_mask = mask;
1696                 current->memcg_oom_order = order;
1697 
1698                 return OOM_ASYNC;
1699         }
1700 
1701         if (mem_cgroup_out_of_memory(memcg, mask, order))
1702                 return OOM_SUCCESS;
1703 
1704         return OOM_FAILED;
1705 }
1706 
1707 /**
1708  * mem_cgroup_oom_synchronize - complete memcg OOM handling
1709  * @handle: actually kill/wait or just clean up the OOM state
1710  *
1711  * This has to be called at the end of a page fault if the memcg OOM
1712  * handler was enabled.
1713  *
1714  * Memcg supports userspace OOM handling where failed allocations must
1715  * sleep on a waitqueue until the userspace task resolves the
1716  * situation.  Sleeping directly in the charge context with all kinds
1717  * of locks held is not a good idea, instead we remember an OOM state
1718  * in the task and mem_cgroup_oom_synchronize() has to be called at
1719  * the end of the page fault to complete the OOM handling.
1720  *
1721  * Returns %true if an ongoing memcg OOM situation was detected and
1722  * completed, %false otherwise.
1723  */
1724 bool mem_cgroup_oom_synchronize(bool handle)
1725 {
1726         struct mem_cgroup *memcg = current->memcg_in_oom;
1727         struct oom_wait_info owait;
1728         bool locked;
1729 
1730         /* OOM is global, do not handle */
1731         if (!memcg)
1732                 return false;
1733 
1734         if (!handle)
1735                 goto cleanup;
1736 
1737         owait.memcg = memcg;
1738         owait.wait.flags = 0;
1739         owait.wait.func = memcg_oom_wake_function;
1740         owait.wait.private = current;
1741         INIT_LIST_HEAD(&owait.wait.entry);
1742 
1743         prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
1744         mem_cgroup_mark_under_oom(memcg);
1745 
1746         locked = mem_cgroup_oom_trylock(memcg);
1747 
1748         if (locked)
1749                 mem_cgroup_oom_notify(memcg);
1750 
1751         if (locked && !memcg->oom_kill_disable) {
1752                 mem_cgroup_unmark_under_oom(memcg);
1753                 finish_wait(&memcg_oom_waitq, &owait.wait);
1754                 mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask,
1755                                          current->memcg_oom_order);
1756         } else {
1757                 schedule();
1758                 mem_cgroup_unmark_under_oom(memcg);
1759                 finish_wait(&memcg_oom_waitq, &owait.wait);
1760         }
1761 
1762         if (locked) {
1763                 mem_cgroup_oom_unlock(memcg);
1764                 /*
1765                  * There is no guarantee that an OOM-lock contender
1766                  * sees the wakeups triggered by the OOM kill
1767                  * uncharges.  Wake any sleepers explicitely.
1768                  */
1769                 memcg_oom_recover(memcg);
1770         }
1771 cleanup:
1772         current->memcg_in_oom = NULL;
1773         css_put(&memcg->css);
1774         return true;
1775 }
1776 
1777 /**
1778  * mem_cgroup_get_oom_group - get a memory cgroup to clean up after OOM
1779  * @victim: task to be killed by the OOM killer
1780  * @oom_domain: memcg in case of memcg OOM, NULL in case of system-wide OOM
1781  *
1782  * Returns a pointer to a memory cgroup, which has to be cleaned up
1783  * by killing all belonging OOM-killable tasks.
1784  *
1785  * Caller has to call mem_cgroup_put() on the returned non-NULL memcg.
1786  */
1787 struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim,
1788                                             struct mem_cgroup *oom_domain)
1789 {
1790         struct mem_cgroup *oom_group = NULL;
1791         struct mem_cgroup *memcg;
1792 
1793         if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
1794                 return NULL;
1795 
1796         if (!oom_domain)
1797                 oom_domain = root_mem_cgroup;
1798 
1799         rcu_read_lock();
1800 
1801         memcg = mem_cgroup_from_task(victim);
1802         if (memcg == root_mem_cgroup)
1803                 goto out;
1804 
1805         /*
1806          * Traverse the memory cgroup hierarchy from the victim task's
1807          * cgroup up to the OOMing cgroup (or root) to find the
1808          * highest-level memory cgroup with oom.group set.
1809          */
1810         for (; memcg; memcg = parent_mem_cgroup(memcg)) {
1811                 if (memcg->oom_group)
1812                         oom_group = memcg;
1813 
1814                 if (memcg == oom_domain)
1815                         break;
1816         }
1817 
1818         if (oom_group)
1819                 css_get(&oom_group->css);
1820 out:
1821         rcu_read_unlock();
1822 
1823         return oom_group;
1824 }
1825 
1826 void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
1827 {
1828         pr_info("Tasks in ");
1829         pr_cont_cgroup_path(memcg->css.cgroup);
1830         pr_cont(" are going to be killed due to memory.oom.group set\n");
1831 }
1832 
1833 /**
1834  * lock_page_memcg - lock a page->mem_cgroup binding
1835  * @page: the page
1836  *
1837  * This function protects unlocked LRU pages from being moved to
1838  * another cgroup.
1839  *
1840  * It ensures lifetime of the returned memcg. Caller is responsible
1841  * for the lifetime of the page; __unlock_page_memcg() is available
1842  * when @page might get freed inside the locked section.
1843  */
1844 struct mem_cgroup *lock_page_memcg(struct page *page)
1845 {
1846         struct mem_cgroup *memcg;
1847         unsigned long flags;
1848 
1849         /*
1850          * The RCU lock is held throughout the transaction.  The fast
1851          * path can get away without acquiring the memcg->move_lock
1852          * because page moving starts with an RCU grace period.
1853          *
1854          * The RCU lock also protects the memcg from being freed when
1855          * the page state that is going to change is the only thing
1856          * preventing the page itself from being freed. E.g. writeback
1857          * doesn't hold a page reference and relies on PG_writeback to
1858          * keep off truncation, migration and so forth.
1859          */
1860         rcu_read_lock();
1861 
1862         if (mem_cgroup_disabled())
1863                 return NULL;
1864 again:
1865         memcg = page->mem_cgroup;
1866         if (unlikely(!memcg))
1867                 return NULL;
1868 
1869         if (atomic_read(&memcg->moving_account) <= 0)
1870                 return memcg;
1871 
1872         spin_lock_irqsave(&memcg->move_lock, flags);
1873         if (memcg != page->mem_cgroup) {
1874                 spin_unlock_irqrestore(&memcg->move_lock, flags);
1875                 goto again;
1876         }
1877 
1878         /*
1879          * When charge migration first begins, we can have locked and
1880          * unlocked page stat updates happening concurrently.  Track
1881          * the task who has the lock for unlock_page_memcg().
1882          */
1883         memcg->move_lock_task = current;
1884         memcg->move_lock_flags = flags;
1885 
1886         return memcg;
1887 }
1888 EXPORT_SYMBOL(lock_page_memcg);
1889 
1890 /**
1891  * __unlock_page_memcg - unlock and unpin a memcg
1892  * @memcg: the memcg
1893  *
1894  * Unlock and unpin a memcg returned by lock_page_memcg().
1895  */
1896 void __unlock_page_memcg(struct mem_cgroup *memcg)
1897 {
1898         if (memcg && memcg->move_lock_task == current) {
1899                 unsigned long flags = memcg->move_lock_flags;
1900 
1901                 memcg->move_lock_task = NULL;
1902                 memcg->move_lock_flags = 0;
1903 
1904                 spin_unlock_irqrestore(&memcg->move_lock, flags);
1905         }
1906 
1907         rcu_read_unlock();
1908 }
1909 
1910 /**
1911  * unlock_page_memcg - unlock a page->mem_cgroup binding
1912  * @page: the page
1913  */
1914 void unlock_page_memcg(struct page *page)
1915 {
1916         __unlock_page_memcg(page->mem_cgroup);
1917 }
1918 EXPORT_SYMBOL(unlock_page_memcg);
1919 
1920 struct memcg_stock_pcp {
1921         struct mem_cgroup *cached; /* this never be root cgroup */
1922         unsigned int nr_pages;
1923         struct work_struct work;
1924         unsigned long flags;
1925 #define FLUSHING_CACHED_CHARGE  0
1926 };
1927 static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
1928 static DEFINE_MUTEX(percpu_charge_mutex);
1929 
1930 /**
1931  * consume_stock: Try to consume stocked charge on this cpu.
1932  * @memcg: memcg to consume from.
1933  * @nr_pages: how many pages to charge.
1934  *
1935  * The charges will only happen if @memcg matches the current cpu's memcg
1936  * stock, and at least @nr_pages are available in that stock.  Failure to
1937  * service an allocation will refill the stock.
1938  *
1939  * returns true if successful, false otherwise.
1940  */
1941 static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
1942 {
1943         struct memcg_stock_pcp *stock;
1944         unsigned long flags;
1945         bool ret = false;
1946 
1947         if (nr_pages > MEMCG_CHARGE_BATCH)
1948                 return ret;
1949 
1950         local_irq_save(flags);
1951 
1952         stock = this_cpu_ptr(&memcg_stock);
1953         if (memcg == stock->cached && stock->nr_pages >= nr_pages) {
1954                 stock->nr_pages -= nr_pages;
1955                 ret = true;
1956         }
1957 
1958         local_irq_restore(flags);
1959 
1960         return ret;
1961 }
1962 
1963 /*
1964  * Returns stocks cached in percpu and reset cached information.
1965  */
1966 static void drain_stock(struct memcg_stock_pcp *stock)
1967 {
1968         struct mem_cgroup *old = stock->cached;
1969 
1970         if (stock->nr_pages) {
1971                 page_counter_uncharge(&old->memory, stock->nr_pages);
1972                 if (do_memsw_account())
1973                         page_counter_uncharge(&old->memsw, stock->nr_pages);
1974                 css_put_many(&old->css, stock->nr_pages);
1975                 stock->nr_pages = 0;
1976         }
1977         stock->cached = NULL;
1978 }
1979 
1980 static void drain_local_stock(struct work_struct *dummy)
1981 {
1982         struct memcg_stock_pcp *stock;
1983         unsigned long flags;
1984 
1985         /*
1986          * The only protection from memory hotplug vs. drain_stock races is
1987          * that we always operate on local CPU stock here with IRQ disabled
1988          */
1989         local_irq_save(flags);
1990 
1991         stock = this_cpu_ptr(&memcg_stock);
1992         drain_stock(stock);
1993         clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
1994 
1995         local_irq_restore(flags);
1996 }
1997 
1998 /*
1999  * Cache charges(val) to local per_cpu area.
2000  * This will be consumed by consume_stock() function, later.
2001  */
2002 static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2003 {
2004         struct memcg_stock_pcp *stock;
2005         unsigned long flags;
2006 
2007         local_irq_save(flags);
2008 
2009         stock = this_cpu_ptr(&memcg_stock);
2010         if (stock->cached != memcg) { /* reset if necessary */
2011                 drain_stock(stock);
2012                 stock->cached = memcg;
2013         }
2014         stock->nr_pages += nr_pages;
2015 
2016         if (stock->nr_pages > MEMCG_CHARGE_BATCH)
2017                 drain_stock(stock);
2018 
2019         local_irq_restore(flags);
2020 }
2021 
2022 /*
2023  * Drains all per-CPU charge caches for given root_memcg resp. subtree
2024  * of the hierarchy under it.
2025  */
2026 static void drain_all_stock(struct mem_cgroup *root_memcg)
2027 {
2028         int cpu, curcpu;
2029 
2030         /* If someone's already draining, avoid adding running more workers. */
2031         if (!mutex_trylock(&percpu_charge_mutex))
2032                 return;
2033         /*
2034          * Notify other cpus that system-wide "drain" is running
2035          * We do not care about races with the cpu hotplug because cpu down
2036          * as well as workers from this path always operate on the local
2037          * per-cpu data. CPU up doesn't touch memcg_stock at all.
2038          */
2039         curcpu = get_cpu();
2040         for_each_online_cpu(cpu) {
2041                 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
2042                 struct mem_cgroup *memcg;
2043 
2044                 memcg = stock->cached;
2045                 if (!memcg || !stock->nr_pages || !css_tryget(&memcg->css))
2046                         continue;
2047                 if (!mem_cgroup_is_descendant(memcg, root_memcg)) {
2048                         css_put(&memcg->css);
2049                         continue;
2050                 }
2051                 if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
2052                         if (cpu == curcpu)
2053                                 drain_local_stock(&stock->work);
2054                         else
2055                                 schedule_work_on(cpu, &stock->work);
2056                 }
2057                 css_put(&memcg->css);
2058         }
2059         put_cpu();
2060         mutex_unlock(&percpu_charge_mutex);
2061 }
2062 
2063 static int memcg_hotplug_cpu_dead(unsigned int cpu)
2064 {
2065         struct memcg_stock_pcp *stock;
2066         struct mem_cgroup *memcg;
2067 
2068         stock = &per_cpu(memcg_stock, cpu);
2069         drain_stock(stock);
2070 
2071         for_each_mem_cgroup(memcg) {
2072                 int i;
2073 
2074                 for (i = 0; i < MEMCG_NR_STAT; i++) {
2075                         int nid;
2076                         long x;
2077 
2078                         x = this_cpu_xchg(memcg->stat_cpu->count[i], 0);
2079                         if (x)
2080                                 atomic_long_add(x, &memcg->stat[i]);
2081 
2082                         if (i >= NR_VM_NODE_STAT_ITEMS)
2083                                 continue;
2084 
2085                         for_each_node(nid) {
2086                                 struct mem_cgroup_per_node *pn;
2087 
2088                                 pn = mem_cgroup_nodeinfo(memcg, nid);
2089                                 x = this_cpu_xchg(pn->lruvec_stat_cpu->count[i], 0);
2090                                 if (x)
2091                                         atomic_long_add(x, &pn->lruvec_stat[i]);
2092                         }
2093                 }
2094 
2095                 for (i = 0; i < NR_VM_EVENT_ITEMS; i++) {
2096                         long x;
2097 
2098                         x = this_cpu_xchg(memcg->stat_cpu->events[i], 0);
2099                         if (x)
2100                                 atomic_long_add(x, &memcg->events[i]);
2101                 }
2102         }
2103 
2104         return 0;
2105 }
2106 
2107 static void reclaim_high(struct mem_cgroup *memcg,
2108                          unsigned int nr_pages,
2109                          gfp_t gfp_mask)
2110 {
2111         do {
2112                 if (page_counter_read(&memcg->memory) <= memcg->high)
2113                         continue;
2114                 memcg_memory_event(memcg, MEMCG_HIGH);
2115                 try_to_free_mem_cgroup_pages(memcg, nr_pages, gfp_mask, true);
2116         } while ((memcg = parent_mem_cgroup(memcg)));
2117 }
2118 
2119 static void high_work_func(struct work_struct *work)
2120 {
2121         struct mem_cgroup *memcg;
2122 
2123         memcg = container_of(work, struct mem_cgroup, high_work);
2124         reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL);
2125 }
2126 
2127 /*
2128  * Scheduled by try_charge() to be executed from the userland return path
2129  * and reclaims memory over the high limit.
2130  */
2131 void mem_cgroup_handle_over_high(void)
2132 {
2133         unsigned int nr_pages = current->memcg_nr_pages_over_high;
2134         struct mem_cgroup *memcg;
2135 
2136         if (likely(!nr_pages))
2137                 return;
2138 
2139         memcg = get_mem_cgroup_from_mm(current->mm);
2140         reclaim_high(memcg, nr_pages, GFP_KERNEL);
2141         css_put(&memcg->css);
2142         current->memcg_nr_pages_over_high = 0;
2143 }
2144 
2145 static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
2146                       unsigned int nr_pages)
2147 {
2148         unsigned int batch = max(MEMCG_CHARGE_BATCH, nr_pages);
2149         int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
2150         struct mem_cgroup *mem_over_limit;
2151         struct page_counter *counter;
2152         unsigned long nr_reclaimed;
2153         bool may_swap = true;
2154         bool drained = false;
2155         bool oomed = false;
2156         enum oom_status oom_status;
2157 
2158         if (mem_cgroup_is_root(memcg))
2159                 return 0;
2160 retry:
2161         if (consume_stock(memcg, nr_pages))
2162                 return 0;
2163 
2164         if (!do_memsw_account() ||
2165             page_counter_try_charge(&memcg->memsw, batch, &counter)) {
2166                 if (page_counter_try_charge(&memcg->memory, batch, &counter))
2167                         goto done_restock;
2168                 if (do_memsw_account())
2169                         page_counter_uncharge(&memcg->memsw, batch);
2170                 mem_over_limit = mem_cgroup_from_counter(counter, memory);
2171         } else {
2172                 mem_over_limit = mem_cgroup_from_counter(counter, memsw);
2173                 may_swap = false;
2174         }
2175 
2176         if (batch > nr_pages) {
2177                 batch = nr_pages;
2178                 goto retry;
2179         }
2180 
2181         /*
2182          * Unlike in global OOM situations, memcg is not in a physical
2183          * memory shortage.  Allow dying and OOM-killed tasks to
2184          * bypass the last charges so that they can exit quickly and
2185          * free their memory.
2186          */
2187         if (unlikely(tsk_is_oom_victim(current) ||
2188                      fatal_signal_pending(current) ||
2189                      current->flags & PF_EXITING))
2190                 goto force;
2191 
2192         /*
2193          * Prevent unbounded recursion when reclaim operations need to
2194          * allocate memory. This might exceed the limits temporarily,
2195          * but we prefer facilitating memory reclaim and getting back
2196          * under the limit over triggering OOM kills in these cases.
2197          */
2198         if (unlikely(current->flags & PF_MEMALLOC))
2199                 goto force;
2200 
2201         if (unlikely(task_in_memcg_oom(current)))
2202                 goto nomem;
2203 
2204         if (!gfpflags_allow_blocking(gfp_mask))
2205                 goto nomem;
2206 
2207         memcg_memory_event(mem_over_limit, MEMCG_MAX);
2208 
2209         nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages,
2210                                                     gfp_mask, may_swap);
2211 
2212         if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
2213                 goto retry;
2214 
2215         if (!drained) {
2216                 drain_all_stock(mem_over_limit);
2217                 drained = true;
2218                 goto retry;
2219         }
2220 
2221         if (gfp_mask & __GFP_NORETRY)
2222                 goto nomem;
2223         /*
2224          * Even though the limit is exceeded at this point, reclaim
2225          * may have been able to free some pages.  Retry the charge
2226          * before killing the task.
2227          *
2228          * Only for regular pages, though: huge pages are rather
2229          * unlikely to succeed so close to the limit, and we fall back
2230          * to regular pages anyway in case of failure.
2231          */
2232         if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER))
2233                 goto retry;
2234         /*
2235          * At task move, charge accounts can be doubly counted. So, it's
2236          * better to wait until the end of task_move if something is going on.
2237          */
2238         if (mem_cgroup_wait_acct_move(mem_over_limit))
2239                 goto retry;
2240 
2241         if (nr_retries--)
2242                 goto retry;
2243 
2244         if (gfp_mask & __GFP_RETRY_MAYFAIL && oomed)
2245                 goto nomem;
2246 
2247         if (gfp_mask & __GFP_NOFAIL)
2248                 goto force;
2249 
2250         if (fatal_signal_pending(current))
2251                 goto force;
2252 
2253         memcg_memory_event(mem_over_limit, MEMCG_OOM);
2254 
2255         /*
2256          * keep retrying as long as the memcg oom killer is able to make
2257          * a forward progress or bypass the charge if the oom killer
2258          * couldn't make any progress.
2259          */
2260         oom_status = mem_cgroup_oom(mem_over_limit, gfp_mask,
2261                        get_order(nr_pages * PAGE_SIZE));
2262         switch (oom_status) {
2263         case OOM_SUCCESS:
2264                 nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
2265                 oomed = true;
2266                 goto retry;
2267         case OOM_FAILED:
2268                 goto force;
2269         default:
2270                 goto nomem;
2271         }
2272 nomem:
2273         if (!(gfp_mask & __GFP_NOFAIL))
2274                 return -ENOMEM;
2275 force:
2276         /*
2277          * The allocation either can't fail or will lead to more memory
2278          * being freed very soon.  Allow memory usage go over the limit
2279          * temporarily by force charging it.
2280          */
2281         page_counter_charge(&memcg->memory, nr_pages);
2282         if (do_memsw_account())
2283                 page_counter_charge(&memcg->memsw, nr_pages);
2284         css_get_many(&memcg->css, nr_pages);
2285 
2286         return 0;
2287 
2288 done_restock:
2289         css_get_many(&memcg->css, batch);
2290         if (batch > nr_pages)
2291                 refill_stock(memcg, batch - nr_pages);
2292 
2293         /*
2294          * If the hierarchy is above the normal consumption range, schedule
2295          * reclaim on returning to userland.  We can perform reclaim here
2296          * if __GFP_RECLAIM but let's always punt for simplicity and so that
2297          * GFP_KERNEL can consistently be used during reclaim.  @memcg is
2298          * not recorded as it most likely matches current's and won't
2299          * change in the meantime.  As high limit is checked again before
2300          * reclaim, the cost of mismatch is negligible.
2301          */
2302         do {
2303                 if (page_counter_read(&memcg->memory) > memcg->high) {
2304                         /* Don't bother a random interrupted task */
2305                         if (in_interrupt()) {
2306                                 schedule_work(&memcg->high_work);
2307                                 break;
2308                         }
2309                         current->memcg_nr_pages_over_high += batch;
2310                         set_notify_resume(current);
2311                         break;
2312                 }
2313         } while ((memcg = parent_mem_cgroup(memcg)));
2314 
2315         return 0;
2316 }
2317 
2318 static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
2319 {
2320         if (mem_cgroup_is_root(memcg))
2321                 return;
2322 
2323         page_counter_uncharge(&memcg->memory, nr_pages);
2324         if (do_memsw_account())
2325                 page_counter_uncharge(&memcg->memsw, nr_pages);
2326 
2327         css_put_many(&memcg->css, nr_pages);
2328 }
2329 
2330 static void lock_page_lru(struct page *page, int *isolated)
2331 {
2332         struct zone *zone = page_zone(page);
2333 
2334         spin_lock_irq(zone_lru_lock(zone));
2335         if (PageLRU(page)) {
2336                 struct lruvec *lruvec;
2337 
2338                 lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);
2339                 ClearPageLRU(page);
2340                 del_page_from_lru_list(page, lruvec, page_lru(page));
2341                 *isolated = 1;
2342         } else
2343                 *isolated = 0;
2344 }
2345 
2346 static void unlock_page_lru(struct page *page, int isolated)
2347 {
2348         struct zone *zone = page_zone(page);
2349 
2350         if (isolated) {
2351                 struct lruvec *lruvec;
2352 
2353                 lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);
2354                 VM_BUG_ON_PAGE(PageLRU(page), page);
2355                 SetPageLRU(page);
2356                 add_page_to_lru_list(page, lruvec, page_lru(page));
2357         }
2358         spin_unlock_irq(zone_lru_lock(zone));
2359 }
2360 
2361 static void commit_charge(struct page *page, struct mem_cgroup *memcg,
2362                           bool lrucare)
2363 {
2364         int isolated;
2365 
2366         VM_BUG_ON_PAGE(page->mem_cgroup, page);
2367 
2368         /*
2369          * In some cases, SwapCache and FUSE(splice_buf->radixtree), the page
2370          * may already be on some other mem_cgroup's LRU.  Take care of it.
2371          */
2372         if (lrucare)
2373                 lock_page_lru(page, &isolated);
2374 
2375         /*
2376          * Nobody should be changing or seriously looking at
2377          * page->mem_cgroup at this point:
2378          *
2379          * - the page is uncharged
2380          *
2381          * - the page is off-LRU
2382          *
2383          * - an anonymous fault has exclusive page access, except for
2384          *   a locked page table
2385          *
2386          * - a page cache insertion, a swapin fault, or a migration
2387          *   have the page locked
2388          */
2389         page->mem_cgroup = memcg;
2390 
2391         if (lrucare)
2392                 unlock_page_lru(page, isolated);
2393 }
2394 
2395 #ifdef CONFIG_MEMCG_KMEM
2396 static int memcg_alloc_cache_id(void)
2397 {
2398         int id, size;
2399         int err;
2400 
2401         id = ida_simple_get(&memcg_cache_ida,
2402                             0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
2403         if (id < 0)
2404                 return id;
2405 
2406         if (id < memcg_nr_cache_ids)
2407                 return id;
2408 
2409         /*
2410          * There's no space for the new id in memcg_caches arrays,
2411          * so we have to grow them.
2412          */
2413         down_write(&memcg_cache_ids_sem);
2414 
2415         size = 2 * (id + 1);
2416         if (size < MEMCG_CACHES_MIN_SIZE)
2417                 size = MEMCG_CACHES_MIN_SIZE;
2418         else if (size > MEMCG_CACHES_MAX_SIZE)
2419                 size = MEMCG_CACHES_MAX_SIZE;
2420 
2421         err = memcg_update_all_caches(size);
2422         if (!err)
2423                 err = memcg_update_all_list_lrus(size);
2424         if (!err)
2425                 memcg_nr_cache_ids = size;
2426 
2427         up_write(&memcg_cache_ids_sem);
2428 
2429         if (err) {
2430                 ida_simple_remove(&memcg_cache_ida, id);
2431                 return err;
2432         }
2433         return id;
2434 }
2435 
2436 static void memcg_free_cache_id(int id)
2437 {
2438         ida_simple_remove(&memcg_cache_ida, id);
2439 }
2440 
2441 struct memcg_kmem_cache_create_work {
2442         struct mem_cgroup *memcg;
2443         struct kmem_cache *cachep;
2444         struct work_struct work;
2445 };
2446 
2447 static void memcg_kmem_cache_create_func(struct work_struct *w)
2448 {
2449         struct memcg_kmem_cache_create_work *cw =
2450                 container_of(w, struct memcg_kmem_cache_create_work, work);
2451         struct mem_cgroup *memcg = cw->memcg;
2452         struct kmem_cache *cachep = cw->cachep;
2453 
2454         memcg_create_kmem_cache(memcg, cachep);
2455 
2456         css_put(&memcg->css);
2457         kfree(cw);
2458 }
2459 
2460 /*
2461  * Enqueue the creation of a per-memcg kmem_cache.
2462  */
2463 static void __memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,
2464                                                struct kmem_cache *cachep)
2465 {
2466         struct memcg_kmem_cache_create_work *cw;
2467 
2468         cw = kmalloc(sizeof(*cw), GFP_NOWAIT | __GFP_NOWARN);
2469         if (!cw)
2470                 return;
2471 
2472         css_get(&memcg->css);
2473 
2474         cw->memcg = memcg;
2475         cw->cachep = cachep;
2476         INIT_WORK(&cw->work, memcg_kmem_cache_create_func);
2477 
2478         queue_work(memcg_kmem_cache_wq, &cw->work);
2479 }
2480 
2481 static void memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,
2482                                              struct kmem_cache *cachep)
2483 {
2484         /*
2485          * We need to stop accounting when we kmalloc, because if the
2486          * corresponding kmalloc cache is not yet created, the first allocation
2487          * in __memcg_schedule_kmem_cache_create will recurse.
2488          *
2489          * However, it is better to enclose the whole function. Depending on
2490          * the debugging options enabled, INIT_WORK(), for instance, can
2491          * trigger an allocation. This too, will make us recurse. Because at
2492          * this point we can't allow ourselves back into memcg_kmem_get_cache,
2493          * the safest choice is to do it like this, wrapping the whole function.
2494          */
2495         current->memcg_kmem_skip_account = 1;
2496         __memcg_schedule_kmem_cache_create(memcg, cachep);
2497         current->memcg_kmem_skip_account = 0;
2498 }
2499 
2500 static inline bool memcg_kmem_bypass(void)
2501 {
2502         if (in_interrupt() || !current->mm || (current->flags & PF_KTHREAD))
2503                 return true;
2504         return false;
2505 }
2506 
2507 /**
2508  * memcg_kmem_get_cache: select the correct per-memcg cache for allocation
2509  * @cachep: the original global kmem cache
2510  *
2511  * Return the kmem_cache we're supposed to use for a slab allocation.
2512  * We try to use the current memcg's version of the cache.
2513  *
2514  * If the cache does not exist yet, if we are the first user of it, we
2515  * create it asynchronously in a workqueue and let the current allocation
2516  * go through with the original cache.
2517  *
2518  * This function takes a reference to the cache it returns to assure it
2519  * won't get destroyed while we are working with it. Once the caller is
2520  * done with it, memcg_kmem_put_cache() must be called to release the
2521  * reference.
2522  */
2523 struct kmem_cache *memcg_kmem_get_cache(struct kmem_cache *cachep)
2524 {
2525         struct mem_cgroup *memcg;
2526         struct kmem_cache *memcg_cachep;
2527         int kmemcg_id;
2528 
2529         VM_BUG_ON(!is_root_cache(cachep));
2530 
2531         if (memcg_kmem_bypass())
2532                 return cachep;
2533 
2534         if (current->memcg_kmem_skip_account)
2535                 return cachep;
2536 
2537         memcg = get_mem_cgroup_from_current();
2538         kmemcg_id = READ_ONCE(memcg->kmemcg_id);
2539         if (kmemcg_id < 0)
2540                 goto out;
2541 
2542         memcg_cachep = cache_from_memcg_idx(cachep, kmemcg_id);
2543         if (likely(memcg_cachep))
2544                 return memcg_cachep;
2545 
2546         /*
2547          * If we are in a safe context (can wait, and not in interrupt
2548          * context), we could be be predictable and return right away.
2549          * This would guarantee that the allocation being performed
2550          * already belongs in the new cache.
2551          *
2552          * However, there are some clashes that can arrive from locking.
2553          * For instance, because we acquire the slab_mutex while doing
2554          * memcg_create_kmem_cache, this means no further allocation
2555          * could happen with the slab_mutex held. So it's better to
2556          * defer everything.
2557          */
2558         memcg_schedule_kmem_cache_create(memcg, cachep);
2559 out:
2560         css_put(&memcg->css);
2561         return cachep;
2562 }
2563 
2564 /**
2565  * memcg_kmem_put_cache: drop reference taken by memcg_kmem_get_cache
2566  * @cachep: the cache returned by memcg_kmem_get_cache
2567  */
2568 void memcg_kmem_put_cache(struct kmem_cache *cachep)
2569 {
2570         if (!is_root_cache(cachep))
2571                 css_put(&cachep->memcg_params.memcg->css);
2572 }
2573 
2574 /**
2575  * memcg_kmem_charge_memcg: charge a kmem page
2576  * @page: page to charge
2577  * @gfp: reclaim mode
2578  * @order: allocation order
2579  * @memcg: memory cgroup to charge
2580  *
2581  * Returns 0 on success, an error code on failure.
2582  */
2583 int memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order,
2584                             struct mem_cgroup *memcg)
2585 {
2586         unsigned int nr_pages = 1 << order;
2587         struct page_counter *counter;
2588         int ret;
2589 
2590         ret = try_charge(memcg, gfp, nr_pages);
2591         if (ret)
2592                 return ret;
2593 
2594         if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
2595             !page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) {
2596                 cancel_charge(memcg, nr_pages);
2597                 return -ENOMEM;
2598         }
2599 
2600         page->mem_cgroup = memcg;
2601 
2602         return 0;
2603 }
2604 
2605 /**
2606  * memcg_kmem_charge: charge a kmem page to the current memory cgroup
2607  * @page: page to charge
2608  * @gfp: reclaim mode
2609  * @order: allocation order
2610  *
2611  * Returns 0 on success, an error code on failure.
2612  */
2613 int memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
2614 {
2615         struct mem_cgroup *memcg;
2616         int ret = 0;
2617 
2618         if (memcg_kmem_bypass())
2619                 return 0;
2620 
2621         memcg = get_mem_cgroup_from_current();
2622         if (!mem_cgroup_is_root(memcg)) {
2623                 ret = memcg_kmem_charge_memcg(page, gfp, order, memcg);
2624                 if (!ret)
2625                         __SetPageKmemcg(page);
2626         }
2627         css_put(&memcg->css);
2628         return ret;
2629 }
2630 /**
2631  * memcg_kmem_uncharge: uncharge a kmem page
2632  * @page: page to uncharge
2633  * @order: allocation order
2634  */
2635 void memcg_kmem_uncharge(struct page *page, int order)
2636 {
2637         struct mem_cgroup *memcg = page->mem_cgroup;
2638         unsigned int nr_pages = 1 << order;
2639 
2640         if (!memcg)
2641                 return;
2642 
2643         VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page);
2644 
2645         if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
2646                 page_counter_uncharge(&memcg->kmem, nr_pages);
2647 
2648         page_counter_uncharge(&memcg->memory, nr_pages);
2649         if (do_memsw_account())
2650                 page_counter_uncharge(&memcg->memsw, nr_pages);
2651 
2652         page->mem_cgroup = NULL;
2653 
2654         /* slab pages do not have PageKmemcg flag set */
2655         if (PageKmemcg(page))
2656                 __ClearPageKmemcg(page);
2657 
2658         css_put_many(&memcg->css, nr_pages);
2659 }
2660 #endif /* CONFIG_MEMCG_KMEM */
2661 
2662 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
2663 
2664 /*
2665  * Because tail pages are not marked as "used", set it. We're under
2666  * zone_lru_lock and migration entries setup in all page mappings.
2667  */
2668 void mem_cgroup_split_huge_fixup(struct page *head)
2669 {
2670         int i;
2671 
2672         if (mem_cgroup_disabled())
2673                 return;
2674 
2675         for (i = 1; i < HPAGE_PMD_NR; i++)
2676                 head[i].mem_cgroup = head->mem_cgroup;
2677 
2678         __mod_memcg_state(head->mem_cgroup, MEMCG_RSS_HUGE, -HPAGE_PMD_NR);
2679 }
2680 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
2681 
2682 #ifdef CONFIG_MEMCG_SWAP
2683 /**
2684  * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
2685  * @entry: swap entry to be moved
2686  * @from:  mem_cgroup which the entry is moved from
2687  * @to:  mem_cgroup which the entry is moved to
2688  *
2689  * It succeeds only when the swap_cgroup's record for this entry is the same
2690  * as the mem_cgroup's id of @from.
2691  *
2692  * Returns 0 on success, -EINVAL on failure.
2693  *
2694  * The caller must have charged to @to, IOW, called page_counter_charge() about
2695  * both res and memsw, and called css_get().
2696  */
2697 static int mem_cgroup_move_swap_account(swp_entry_t entry,
2698                                 struct mem_cgroup *from, struct mem_cgroup *to)
2699 {
2700         unsigned short old_id, new_id;
2701 
2702         old_id = mem_cgroup_id(from);
2703         new_id = mem_cgroup_id(to);
2704 
2705         if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
2706                 mod_memcg_state(from, MEMCG_SWAP, -1);
2707                 mod_memcg_state(to, MEMCG_SWAP, 1);
2708                 return 0;
2709         }
2710         return -EINVAL;
2711 }
2712 #else
2713 static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
2714                                 struct mem_cgroup *from, struct mem_cgroup *to)
2715 {
2716         return -EINVAL;
2717 }
2718 #endif
2719 
2720 static DEFINE_MUTEX(memcg_max_mutex);
2721 
2722 static int mem_cgroup_resize_max(struct mem_cgroup *memcg,
2723                                  unsigned long max, bool memsw)
2724 {
2725         bool enlarge = false;
2726         bool drained = false;
2727         int ret;
2728         bool limits_invariant;
2729         struct page_counter *counter = memsw ? &memcg->memsw : &memcg->memory;
2730 
2731         do {
2732                 if (signal_pending(current)) {
2733                         ret = -EINTR;
2734                         break;
2735                 }
2736 
2737                 mutex_lock(&memcg_max_mutex);
2738                 /*
2739                  * Make sure that the new limit (memsw or memory limit) doesn't
2740                  * break our basic invariant rule memory.max <= memsw.max.
2741                  */
2742                 limits_invariant = memsw ? max >= memcg->memory.max :
2743                                            max <= memcg->memsw.max;
2744                 if (!limits_invariant) {
2745                         mutex_unlock(&memcg_max_mutex);
2746                         ret = -EINVAL;
2747                         break;
2748                 }
2749                 if (max > counter->max)
2750                         enlarge = true;
2751                 ret = page_counter_set_max(counter, max);
2752                 mutex_unlock(&memcg_max_mutex);
2753 
2754                 if (!ret)
2755                         break;
2756 
2757                 if (!drained) {
2758                         drain_all_stock(memcg);
2759                         drained = true;
2760                         continue;
2761                 }
2762 
2763                 if (!try_to_free_mem_cgroup_pages(memcg, 1,
2764                                         GFP_KERNEL, !memsw)) {
2765                         ret = -EBUSY;
2766                         break;
2767                 }
2768         } while (true);
2769 
2770         if (!ret && enlarge)
2771                 memcg_oom_recover(memcg);
2772 
2773         return ret;
2774 }
2775 
2776 unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
2777                                             gfp_t gfp_mask,
2778                                             unsigned long *total_scanned)
2779 {
2780         unsigned long nr_reclaimed = 0;
2781         struct mem_cgroup_per_node *mz, *next_mz = NULL;
2782         unsigned long reclaimed;
2783         int loop = 0;
2784         struct mem_cgroup_tree_per_node *mctz;
2785         unsigned long excess;
2786         unsigned long nr_scanned;
2787 
2788         if (order > 0)
2789                 return 0;
2790 
2791         mctz = soft_limit_tree_node(pgdat->node_id);
2792 
2793         /*
2794          * Do not even bother to check the largest node if the root
2795          * is empty. Do it lockless to prevent lock bouncing. Races
2796          * are acceptable as soft limit is best effort anyway.
2797          */
2798         if (!mctz || RB_EMPTY_ROOT(&mctz->rb_root))
2799                 return 0;
2800 
2801         /*
2802          * This loop can run a while, specially if mem_cgroup's continuously
2803          * keep exceeding their soft limit and putting the system under
2804          * pressure
2805          */
2806         do {
2807                 if (next_mz)
2808                         mz = next_mz;
2809                 else
2810                         mz = mem_cgroup_largest_soft_limit_node(mctz);
2811                 if (!mz)
2812                         break;
2813 
2814                 nr_scanned = 0;
2815                 reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat,
2816                                                     gfp_mask, &nr_scanned);
2817                 nr_reclaimed += reclaimed;
2818                 *total_scanned += nr_scanned;
2819                 spin_lock_irq(&mctz->lock);
2820                 __mem_cgroup_remove_exceeded(mz, mctz);
2821 
2822                 /*
2823                  * If we failed to reclaim anything from this memory cgroup
2824                  * it is time to move on to the next cgroup
2825                  */
2826                 next_mz = NULL;
2827                 if (!reclaimed)
2828                         next_mz = __mem_cgroup_largest_soft_limit_node(mctz);
2829 
2830                 excess = soft_limit_excess(mz->memcg);
2831                 /*
2832                  * One school of thought says that we should not add
2833                  * back the node to the tree if reclaim returns 0.
2834                  * But our reclaim could return 0, simply because due
2835                  * to priority we are exposing a smaller subset of
2836                  * memory to reclaim from. Consider this as a longer
2837                  * term TODO.
2838                  */
2839                 /* If excess == 0, no tree ops */
2840                 __mem_cgroup_insert_exceeded(mz, mctz, excess);
2841                 spin_unlock_irq(&mctz->lock);
2842                 css_put(&mz->memcg->css);
2843                 loop++;
2844                 /*
2845                  * Could not reclaim anything and there are no more
2846                  * mem cgroups to try or we seem to be looping without
2847                  * reclaiming anything.
2848                  */
2849                 if (!nr_reclaimed &&
2850                         (next_mz == NULL ||
2851                         loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
2852                         break;
2853         } while (!nr_reclaimed);
2854         if (next_mz)
2855                 css_put(&next_mz->memcg->css);
2856         return nr_reclaimed;
2857 }
2858 
2859 /*
2860  * Test whether @memcg has children, dead or alive.  Note that this
2861  * function doesn't care whether @memcg has use_hierarchy enabled and
2862  * returns %true if there are child csses according to the cgroup
2863  * hierarchy.  Testing use_hierarchy is the caller's responsiblity.
2864  */
2865 static inline bool memcg_has_children(struct mem_cgroup *memcg)
2866 {
2867         bool ret;
2868 
2869         rcu_read_lock();
2870         ret = css_next_child(NULL, &memcg->css);
2871         rcu_read_unlock();
2872         return ret;
2873 }
2874 
2875 /*
2876  * Reclaims as many pages from the given memcg as possible.
2877  *
2878  * Caller is responsible for holding css reference for memcg.
2879  */
2880 static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
2881 {
2882         int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
2883 
2884         /* we call try-to-free pages for make this cgroup empty */
2885         lru_add_drain_all();
2886 
2887         drain_all_stock(memcg);
2888 
2889         /* try to free all pages in this cgroup */
2890         while (nr_retries && page_counter_read(&memcg->memory)) {
2891                 int progress;
2892 
2893                 if (signal_pending(current))
2894                         return -EINTR;
2895 
2896                 progress = try_to_free_mem_cgroup_pages(memcg, 1,
2897                                                         GFP_KERNEL, true);
2898                 if (!progress) {
2899                         nr_retries--;
2900                         /* maybe some writeback is necessary */
2901                         congestion_wait(BLK_RW_ASYNC, HZ/10);
2902                 }
2903 
2904         }
2905 
2906         return 0;
2907 }
2908 
2909 static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of,
2910                                             char *buf, size_t nbytes,
2911                                             loff_t off)
2912 {
2913         struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
2914 
2915         if (mem_cgroup_is_root(memcg))
2916                 return -EINVAL;
2917         return mem_cgroup_force_empty(memcg) ?: nbytes;
2918 }
2919 
2920 static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css,
2921                                      struct cftype *cft)
2922 {
2923         return mem_cgroup_from_css(css)->use_hierarchy;
2924 }
2925 
2926 static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
2927                                       struct cftype *cft, u64 val)
2928 {
2929         int retval = 0;
2930         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
2931         struct mem_cgroup *parent_memcg = mem_cgroup_from_css(memcg->css.parent);
2932 
2933         if (memcg->use_hierarchy == val)
2934                 return 0;
2935 
2936         /*
2937          * If parent's use_hierarchy is set, we can't make any modifications
2938          * in the child subtrees. If it is unset, then the change can
2939          * occur, provided the current cgroup has no children.
2940          *
2941          * For the root cgroup, parent_mem is NULL, we allow value to be
2942          * set if there are no children.
2943          */
2944         if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
2945                                 (val == 1 || val == 0)) {
2946                 if (!memcg_has_children(memcg))
2947                         memcg->use_hierarchy = val;
2948                 else
2949                         retval = -EBUSY;
2950         } else
2951                 retval = -EINVAL;
2952 
2953         return retval;
2954 }
2955 
2956 struct accumulated_stats {
2957         unsigned long stat[MEMCG_NR_STAT];
2958         unsigned long events[NR_VM_EVENT_ITEMS];
2959         unsigned long lru_pages[NR_LRU_LISTS];
2960         const unsigned int *stats_array;
2961         const unsigned int *events_array;
2962         int stats_size;
2963         int events_size;
2964 };
2965 
2966 static void accumulate_memcg_tree(struct mem_cgroup *memcg,
2967                                   struct accumulated_stats *acc)
2968 {
2969         struct mem_cgroup *mi;
2970         int i;
2971 
2972         for_each_mem_cgroup_tree(mi, memcg) {
2973                 for (i = 0; i < acc->stats_size; i++)
2974                         acc->stat[i] += memcg_page_state(mi,
2975                                 acc->stats_array ? acc->stats_array[i] : i);
2976 
2977                 for (i = 0; i < acc->events_size; i++)
2978                         acc->events[i] += memcg_sum_events(mi,
2979                                 acc->events_array ? acc->events_array[i] : i);
2980 
2981                 for (i = 0; i < NR_LRU_LISTS; i++)
2982                         acc->lru_pages[i] +=
2983                                 mem_cgroup_nr_lru_pages(mi, BIT(i));
2984         }
2985 }
2986 
2987 static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
2988 {
2989         unsigned long val = 0;
2990 
2991         if (mem_cgroup_is_root(memcg)) {
2992                 struct mem_cgroup *iter;
2993 
2994                 for_each_mem_cgroup_tree(iter, memcg) {
2995                         val += memcg_page_state(iter, MEMCG_CACHE);
2996                         val += memcg_page_state(iter, MEMCG_RSS);
2997                         if (swap)
2998                                 val += memcg_page_state(iter, MEMCG_SWAP);
2999                 }
3000         } else {
3001                 if (!swap)
3002                         val = page_counter_read(&memcg->memory);
3003                 else
3004                         val = page_counter_read(&memcg->memsw);
3005         }
3006         return val;
3007 }
3008 
3009 enum {
3010         RES_USAGE,
3011         RES_LIMIT,
3012         RES_MAX_USAGE,
3013         RES_FAILCNT,
3014         RES_SOFT_LIMIT,
3015 };
3016 
3017 static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
3018                                struct cftype *cft)
3019 {
3020         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3021         struct page_counter *counter;
3022 
3023         switch (MEMFILE_TYPE(cft->private)) {
3024         case _MEM:
3025                 counter = &memcg->memory;
3026                 break;
3027         case _MEMSWAP:
3028                 counter = &memcg->memsw;
3029                 break;
3030         case _KMEM:
3031                 counter = &memcg->kmem;
3032                 break;
3033         case _TCP:
3034                 counter = &memcg->tcpmem;
3035                 break;
3036         default:
3037                 BUG();
3038         }
3039 
3040         switch (MEMFILE_ATTR(cft->private)) {
3041         case RES_USAGE:
3042                 if (counter == &memcg->memory)
3043                         return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE;
3044                 if (counter == &memcg->memsw)
3045                         return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE;
3046                 return (u64)page_counter_read(counter) * PAGE_SIZE;
3047         case RES_LIMIT:
3048                 return (u64)counter->max * PAGE_SIZE;
3049         case RES_MAX_USAGE:
3050                 return (u64)counter->watermark * PAGE_SIZE;
3051         case RES_FAILCNT:
3052                 return counter->failcnt;
3053         case RES_SOFT_LIMIT:
3054                 return (u64)memcg->soft_limit * PAGE_SIZE;
3055         default:
3056                 BUG();
3057         }
3058 }
3059 
3060 #ifdef CONFIG_MEMCG_KMEM
3061 static int memcg_online_kmem(struct mem_cgroup *memcg)
3062 {
3063         int memcg_id;
3064 
3065         if (cgroup_memory_nokmem)
3066                 return 0;
3067 
3068         BUG_ON(memcg->kmemcg_id >= 0);
3069         BUG_ON(memcg->kmem_state);
3070 
3071         memcg_id = memcg_alloc_cache_id();
3072         if (memcg_id < 0)
3073                 return memcg_id;
3074 
3075         static_branch_inc(&memcg_kmem_enabled_key);
3076         /*
3077          * A memory cgroup is considered kmem-online as soon as it gets
3078          * kmemcg_id. Setting the id after enabling static branching will
3079          * guarantee no one starts accounting before all call sites are
3080          * patched.
3081          */
3082         memcg->kmemcg_id = memcg_id;
3083         memcg->kmem_state = KMEM_ONLINE;
3084         INIT_LIST_HEAD(&memcg->kmem_caches);
3085 
3086         return 0;
3087 }
3088 
3089 static void memcg_offline_kmem(struct mem_cgroup *memcg)
3090 {
3091         struct cgroup_subsys_state *css;
3092         struct mem_cgroup *parent, *child;
3093         int kmemcg_id;
3094 
3095         if (memcg->kmem_state != KMEM_ONLINE)
3096                 return;
3097         /*
3098          * Clear the online state before clearing memcg_caches array
3099          * entries. The slab_mutex in memcg_deactivate_kmem_caches()
3100          * guarantees that no cache will be created for this cgroup
3101          * after we are done (see memcg_create_kmem_cache()).
3102          */
3103         memcg->kmem_state = KMEM_ALLOCATED;
3104 
3105         memcg_deactivate_kmem_caches(memcg);
3106 
3107         kmemcg_id = memcg->kmemcg_id;
3108         BUG_ON(kmemcg_id < 0);
3109 
3110         parent = parent_mem_cgroup(memcg);
3111         if (!parent)
3112                 parent = root_mem_cgroup;
3113 
3114         /*
3115          * Change kmemcg_id of this cgroup and all its descendants to the
3116          * parent's id, and then move all entries from this cgroup's list_lrus
3117          * to ones of the parent. After we have finished, all list_lrus
3118          * corresponding to this cgroup are guaranteed to remain empty. The
3119          * ordering is imposed by list_lru_node->lock taken by
3120          * memcg_drain_all_list_lrus().
3121          */
3122         rcu_read_lock(); /* can be called from css_free w/o cgroup_mutex */
3123         css_for_each_descendant_pre(css, &memcg->css) {
3124                 child = mem_cgroup_from_css(css);
3125                 BUG_ON(child->kmemcg_id != kmemcg_id);
3126                 child->kmemcg_id = parent->kmemcg_id;
3127                 if (!memcg->use_hierarchy)
3128                         break;
3129         }
3130         rcu_read_unlock();
3131 
3132         memcg_drain_all_list_lrus(kmemcg_id, parent);
3133 
3134         memcg_free_cache_id(kmemcg_id);
3135 }
3136 
3137 static void memcg_free_kmem(struct mem_cgroup *memcg)
3138 {
3139         /* css_alloc() failed, offlining didn't happen */
3140         if (unlikely(memcg->kmem_state == KMEM_ONLINE))
3141                 memcg_offline_kmem(memcg);
3142 
3143         if (memcg->kmem_state == KMEM_ALLOCATED) {
3144                 memcg_destroy_kmem_caches(memcg);
3145                 static_branch_dec(&memcg_kmem_enabled_key);
3146                 WARN_ON(page_counter_read(&memcg->kmem));
3147         }
3148 }
3149 #else
3150 static int memcg_online_kmem(struct mem_cgroup *memcg)
3151 {
3152         return 0;
3153 }
3154 static void memcg_offline_kmem(struct mem_cgroup *memcg)
3155 {
3156 }
3157 static void memcg_free_kmem(struct mem_cgroup *memcg)
3158 {
3159 }
3160 #endif /* CONFIG_MEMCG_KMEM */
3161 
3162 static int memcg_update_kmem_max(struct mem_cgroup *memcg,
3163                                  unsigned long max)
3164 {
3165         int ret;
3166 
3167         mutex_lock(&memcg_max_mutex);
3168         ret = page_counter_set_max(&memcg->kmem, max);
3169         mutex_unlock(&memcg_max_mutex);
3170         return ret;
3171 }
3172 
3173 static int memcg_update_tcp_max(struct mem_cgroup *memcg, unsigned long max)
3174 {
3175         int ret;
3176 
3177         mutex_lock(&memcg_max_mutex);
3178 
3179         ret = page_counter_set_max(&memcg->tcpmem, max);
3180         if (ret)
3181                 goto out;
3182 
3183         if (!memcg->tcpmem_active) {
3184                 /*
3185                  * The active flag needs to be written after the static_key
3186                  * update. This is what guarantees that the socket activation
3187                  * function is the last one to run. See mem_cgroup_sk_alloc()
3188                  * for details, and note that we don't mark any socket as
3189                  * belonging to this memcg until that flag is up.
3190                  *
3191                  * We need to do this, because static_keys will span multiple
3192                  * sites, but we can't control their order. If we mark a socket
3193                  * as accounted, but the accounting functions are not patched in
3194                  * yet, we'll lose accounting.
3195                  *
3196                  * We never race with the readers in mem_cgroup_sk_alloc(),
3197                  * because when this value change, the code to process it is not
3198                  * patched in yet.
3199                  */
3200                 static_branch_inc(&memcg_sockets_enabled_key);
3201                 memcg->tcpmem_active = true;
3202         }
3203 out:
3204         mutex_unlock(&memcg_max_mutex);
3205         return ret;
3206 }
3207 
3208 /*
3209  * The user of this function is...
3210  * RES_LIMIT.
3211  */
3212 static ssize_t mem_cgroup_write(struct kernfs_open_file *of,
3213                                 char *buf, size_t nbytes, loff_t off)
3214 {
3215         struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3216         unsigned long nr_pages;
3217         int ret;
3218 
3219         buf = strstrip(buf);
3220         ret = page_counter_memparse(buf, "-1", &nr_pages);
3221         if (ret)
3222                 return ret;
3223 
3224         switch (MEMFILE_ATTR(of_cft(of)->private)) {
3225         case RES_LIMIT:
3226                 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
3227                         ret = -EINVAL;
3228                         break;
3229                 }
3230                 switch (MEMFILE_TYPE(of_cft(of)->private)) {
3231                 case _MEM:
3232                         ret = mem_cgroup_resize_max(memcg, nr_pages, false);
3233                         break;
3234                 case _MEMSWAP:
3235                         ret = mem_cgroup_resize_max(memcg, nr_pages, true);
3236                         break;
3237                 case _KMEM:
3238                         ret = memcg_update_kmem_max(memcg, nr_pages);
3239                         break;
3240                 case _TCP:
3241                         ret = memcg_update_tcp_max(memcg, nr_pages);
3242                         break;
3243                 }
3244                 break;
3245         case RES_SOFT_LIMIT:
3246                 memcg->soft_limit = nr_pages;
3247                 ret = 0;
3248                 break;
3249         }
3250         return ret ?: nbytes;
3251 }
3252 
3253 static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf,
3254                                 size_t nbytes, loff_t off)
3255 {
3256         struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3257         struct page_counter *counter;
3258 
3259         switch (MEMFILE_TYPE(of_cft(of)->private)) {
3260         case _MEM:
3261                 counter = &memcg->memory;
3262                 break;
3263         case _MEMSWAP:
3264                 counter = &memcg->memsw;
3265                 break;
3266         case _KMEM:
3267                 counter = &memcg->kmem;
3268                 break;
3269         case _TCP:
3270                 counter = &memcg->tcpmem;
3271                 break;
3272         default:
3273                 BUG();
3274         }
3275 
3276         switch (MEMFILE_ATTR(of_cft(of)->private)) {
3277         case RES_MAX_USAGE:
3278                 page_counter_reset_watermark(counter);
3279                 break;
3280         case RES_FAILCNT:
3281                 counter->failcnt = 0;
3282                 break;
3283         default:
3284                 BUG();
3285         }
3286 
3287         return nbytes;
3288 }
3289 
3290 static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
3291                                         struct cftype *cft)
3292 {
3293         return mem_cgroup_from_css(css)->move_charge_at_immigrate;
3294 }
3295 
3296 #ifdef CONFIG_MMU
3297 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3298                                         struct cftype *cft, u64 val)
3299 {
3300         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3301 
3302         if (val & ~MOVE_MASK)
3303                 return -EINVAL;
3304 
3305         /*
3306          * No kind of locking is needed in here, because ->can_attach() will
3307          * check this value once in the beginning of the process, and then carry
3308          * on with stale data. This means that changes to this value will only
3309          * affect task migrations starting after the change.
3310          */
3311         memcg->move_charge_at_immigrate = val;
3312         return 0;
3313 }
3314 #else
3315 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3316                                         struct cftype *cft, u64 val)
3317 {
3318         return -ENOSYS;
3319 }
3320 #endif
3321 
3322 #ifdef CONFIG_NUMA
3323 static int memcg_numa_stat_show(struct seq_file *m, void *v)
3324 {
3325         struct numa_stat {
3326                 const char *name;
3327                 unsigned int lru_mask;
3328         };
3329 
3330         static const struct numa_stat stats[] = {
3331                 { "total", LRU_ALL },
3332                 { "file", LRU_ALL_FILE },
3333                 { "anon", LRU_ALL_ANON },
3334                 { "unevictable", BIT(LRU_UNEVICTABLE) },
3335         };
3336         const struct numa_stat *stat;
3337         int nid;
3338         unsigned long nr;
3339         struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
3340 
3341         for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
3342                 nr = mem_cgroup_nr_lru_pages(memcg, stat->lru_mask);
3343                 seq_printf(m, "%s=%lu", stat->name, nr);
3344                 for_each_node_state(nid, N_MEMORY) {
3345                         nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
3346                                                           stat->lru_mask);
3347                         seq_printf(m, " N%d=%lu", nid, nr);
3348                 }
3349                 seq_putc(m, '\n');
3350         }
3351 
3352         for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
3353                 struct mem_cgroup *iter;
3354 
3355                 nr = 0;
3356                 for_each_mem_cgroup_tree(iter, memcg)
3357                         nr += mem_cgroup_nr_lru_pages(iter, stat->lru_mask);
3358                 seq_printf(m, "hierarchical_%s=%lu", stat->name, nr);
3359                 for_each_node_state(nid, N_MEMORY) {
3360                         nr = 0;
3361                         for_each_mem_cgroup_tree(iter, memcg)
3362                                 nr += mem_cgroup_node_nr_lru_pages(
3363                                         iter, nid, stat->lru_mask);
3364                         seq_printf(m, " N%d=%lu", nid, nr);
3365                 }
3366                 seq_putc(m, '\n');
3367         }
3368 
3369         return 0;
3370 }
3371 #endif /* CONFIG_NUMA */
3372 
3373 /* Universal VM events cgroup1 shows, original sort order */
3374 static const unsigned int memcg1_events[] = {
3375         PGPGIN,
3376         PGPGOUT,
3377         PGFAULT,
3378         PGMAJFAULT,
3379 };
3380 
3381 static const char *const memcg1_event_names[] = {
3382         "pgpgin",
3383         "pgpgout",
3384         "pgfault",
3385         "pgmajfault",
3386 };
3387 
3388 static int memcg_stat_show(struct seq_file *m, void *v)
3389 {
3390         struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
3391         unsigned long memory, memsw;
3392         struct mem_cgroup *mi;
3393         unsigned int i;
3394         struct accumulated_stats acc;
3395 
3396         BUILD_BUG_ON(ARRAY_SIZE(memcg1_stat_names) != ARRAY_SIZE(memcg1_stats));
3397         BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_lru_names) != NR_LRU_LISTS);
3398 
3399         for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
3400                 if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
3401                         continue;
3402                 seq_printf(m, "%s %lu\n", memcg1_stat_names[i],
3403                            memcg_page_state(memcg, memcg1_stats[i]) *
3404                            PAGE_SIZE);
3405         }
3406 
3407         for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
3408                 seq_printf(m, "%s %lu\n", memcg1_event_names[i],
3409                            memcg_sum_events(memcg, memcg1_events[i]));
3410 
3411         for (i = 0; i < NR_LRU_LISTS; i++)
3412                 seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i],
3413                            mem_cgroup_nr_lru_pages(memcg, BIT(i)) * PAGE_SIZE);
3414 
3415         /* Hierarchical information */
3416         memory = memsw = PAGE_COUNTER_MAX;
3417         for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) {
3418                 memory = min(memory, mi->memory.max);
3419                 memsw = min(memsw, mi->memsw.max);
3420         }
3421         seq_printf(m, "hierarchical_memory_limit %llu\n",
3422                    (u64)memory * PAGE_SIZE);
3423         if (do_memsw_account())
3424                 seq_printf(m, "hierarchical_memsw_limit %llu\n",
3425                            (u64)memsw * PAGE_SIZE);
3426 
3427         memset(&acc, 0, sizeof(acc));
3428         acc.stats_size = ARRAY_SIZE(memcg1_stats);
3429         acc.stats_array = memcg1_stats;
3430         acc.events_size = ARRAY_SIZE(memcg1_events);
3431         acc.events_array = memcg1_events;
3432         accumulate_memcg_tree(memcg, &acc);
3433 
3434         for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
3435                 if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
3436                         continue;
3437                 seq_printf(m, "total_%s %llu\n", memcg1_stat_names[i],
3438                            (u64)acc.stat[i] * PAGE_SIZE);
3439         }
3440 
3441         for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
3442                 seq_printf(m, "total_%s %llu\n", memcg1_event_names[i],
3443                            (u64)acc.events[i]);
3444 
3445         for (i = 0; i < NR_LRU_LISTS; i++)
3446                 seq_printf(m, "total_%s %llu\n", mem_cgroup_lru_names[i],
3447                            (u64)acc.lru_pages[i] * PAGE_SIZE);
3448 
3449 #ifdef CONFIG_DEBUG_VM
3450         {
3451                 pg_data_t *pgdat;
3452                 struct mem_cgroup_per_node *mz;
3453                 struct zone_reclaim_stat *rstat;
3454                 unsigned long recent_rotated[2] = {0, 0};
3455                 unsigned long recent_scanned[2] = {0, 0};
3456 
3457                 for_each_online_pgdat(pgdat) {
3458                         mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
3459                         rstat = &mz->lruvec.reclaim_stat;
3460 
3461                         recent_rotated[0] += rstat->recent_rotated[0];
3462                         recent_rotated[1] += rstat->recent_rotated[1];
3463                         recent_scanned[0] += rstat->recent_scanned[0];
3464                         recent_scanned[1] += rstat->recent_scanned[1];
3465                 }
3466                 seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]);
3467                 seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]);
3468                 seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]);
3469                 seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]);
3470         }
3471 #endif
3472 
3473         return 0;
3474 }
3475 
3476 static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css,
3477                                       struct cftype *cft)
3478 {
3479         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3480 
3481         return mem_cgroup_swappiness(memcg);
3482 }
3483 
3484 static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
3485                                        struct cftype *cft, u64 val)
3486 {
3487         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3488 
3489         if (val > 100)
3490                 return -EINVAL;
3491 
3492         if (css->parent)
3493                 memcg->swappiness = val;
3494         else
3495                 vm_swappiness = val;
3496 
3497         return 0;
3498 }
3499 
3500 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
3501 {
3502         struct mem_cgroup_threshold_ary *t;
3503         unsigned long usage;
3504         int i;
3505 
3506         rcu_read_lock();
3507         if (!swap)
3508                 t = rcu_dereference(memcg->thresholds.primary);
3509         else
3510                 t = rcu_dereference(memcg->memsw_thresholds.primary);
3511 
3512         if (!t)
3513                 goto unlock;
3514 
3515         usage = mem_cgroup_usage(memcg, swap);
3516 
3517         /*
3518          * current_threshold points to threshold just below or equal to usage.
3519          * If it's not true, a threshold was crossed after last
3520          * call of __mem_cgroup_threshold().
3521          */
3522         i = t->current_threshold;
3523 
3524         /*
3525          * Iterate backward over array of thresholds starting from
3526          * current_threshold and check if a threshold is crossed.
3527          * If none of thresholds below usage is crossed, we read
3528          * only one element of the array here.
3529          */
3530         for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
3531                 eventfd_signal(t->entries[i].eventfd, 1);
3532 
3533         /* i = current_threshold + 1 */
3534         i++;
3535 
3536         /*
3537          * Iterate forward over array of thresholds starting from
3538          * current_threshold+1 and check if a threshold is crossed.
3539          * If none of thresholds above usage is crossed, we read
3540          * only one element of the array here.
3541          */
3542         for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
3543                 eventfd_signal(t->entries[i].eventfd, 1);
3544 
3545         /* Update current_threshold */
3546         t->current_threshold = i - 1;
3547 unlock:
3548         rcu_read_unlock();
3549 }
3550 
3551 static void mem_cgroup_threshold(struct mem_cgroup *memcg)
3552 {
3553         while (memcg) {
3554                 __mem_cgroup_threshold(memcg, false);
3555                 if (do_memsw_account())
3556                         __mem_cgroup_threshold(memcg, true);
3557 
3558                 memcg = parent_mem_cgroup(memcg);
3559         }
3560 }
3561 
3562 static int compare_thresholds(const void *a, const void *b)
3563 {
3564         const struct mem_cgroup_threshold *_a = a;
3565         const struct mem_cgroup_threshold *_b = b;
3566 
3567         if (_a->threshold > _b->threshold)
3568                 return 1;
3569 
3570         if (_a->threshold < _b->threshold)
3571                 return -1;
3572 
3573         return 0;
3574 }
3575 
3576 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
3577 {
3578         struct mem_cgroup_eventfd_list *ev;
3579 
3580         spin_lock(&memcg_oom_lock);
3581 
3582         list_for_each_entry(ev, &memcg->oom_notify, list)
3583                 eventfd_signal(ev->eventfd, 1);
3584 
3585         spin_unlock(&memcg_oom_lock);
3586         return 0;
3587 }
3588 
3589 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
3590 {
3591         struct mem_cgroup *iter;
3592 
3593         for_each_mem_cgroup_tree(iter, memcg)
3594                 mem_cgroup_oom_notify_cb(iter);
3595 }
3596 
3597 static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
3598         struct eventfd_ctx *eventfd, const char *args, enum res_type type)
3599 {
3600         struct mem_cgroup_thresholds *thresholds;
3601         struct mem_cgroup_threshold_ary *new;
3602         unsigned long threshold;
3603         unsigned long usage;
3604         int i, size, ret;
3605 
3606         ret = page_counter_memparse(args, "-1", &threshold);
3607         if (ret)
3608                 return ret;
3609 
3610         mutex_lock(&memcg->thresholds_lock);
3611 
3612         if (type == _MEM) {
3613                 thresholds = &memcg->thresholds;
3614                 usage = mem_cgroup_usage(memcg, false);
3615         } else if (type == _MEMSWAP) {
3616                 thresholds = &memcg->memsw_thresholds;
3617                 usage = mem_cgroup_usage(memcg, true);
3618         } else
3619                 BUG();
3620 
3621         /* Check if a threshold crossed before adding a new one */
3622         if (thresholds->primary)
3623                 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
3624 
3625         size = thresholds->primary ? thresholds->primary->size + 1 : 1;
3626 
3627         /* Allocate memory for new array of thresholds */
3628         new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
3629                         GFP_KERNEL);
3630         if (!new) {
3631                 ret = -ENOMEM;
3632                 goto unlock;
3633         }
3634         new->size = size;
3635 
3636         /* Copy thresholds (if any) to new array */
3637         if (thresholds->primary) {
3638                 memcpy(new->entries, thresholds->primary->entries, (size - 1) *
3639                                 sizeof(struct mem_cgroup_threshold));
3640         }
3641 
3642         /* Add new threshold */
3643         new->entries[size - 1].eventfd = eventfd;
3644         new->entries[size - 1].threshold = threshold;
3645 
3646         /* Sort thresholds. Registering of new threshold isn't time-critical */
3647         sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
3648                         compare_thresholds, NULL);
3649 
3650         /* Find current threshold */
3651         new->current_threshold = -1;
3652         for (i = 0; i < size; i++) {
3653                 if (new->entries[i].threshold <= usage) {
3654                         /*
3655                          * new->current_threshold will not be used until
3656                          * rcu_assign_pointer(), so it's safe to increment
3657                          * it here.
3658                          */
3659                         ++new->current_threshold;
3660                 } else
3661                         break;
3662         }
3663 
3664         /* Free old spare buffer and save old primary buffer as spare */
3665         kfree(thresholds->spare);
3666         thresholds->spare = thresholds->primary;
3667 
3668         rcu_assign_pointer(thresholds->primary, new);
3669 
3670         /* To be sure that nobody uses thresholds */
3671         synchronize_rcu();
3672 
3673 unlock:
3674         mutex_unlock(&memcg->thresholds_lock);
3675 
3676         return ret;
3677 }
3678 
3679 static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
3680         struct eventfd_ctx *eventfd, const char *args)
3681 {
3682         return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM);
3683 }
3684 
3685 static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg,
3686         struct eventfd_ctx *eventfd, const char *args)
3687 {
3688         return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP);
3689 }
3690 
3691 static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
3692         struct eventfd_ctx *eventfd, enum res_type type)
3693 {
3694         struct mem_cgroup_thresholds *thresholds;
3695         struct mem_cgroup_threshold_ary *new;
3696         unsigned long usage;
3697         int i, j, size;
3698 
3699         mutex_lock(&memcg->thresholds_lock);
3700 
3701         if (type == _MEM) {
3702                 thresholds = &memcg->thresholds;
3703                 usage = mem_cgroup_usage(memcg, false);
3704         } else if (type == _MEMSWAP) {
3705                 thresholds = &memcg->memsw_thresholds;
3706                 usage = mem_cgroup_usage(memcg, true);
3707         } else
3708                 BUG();
3709 
3710         if (!thresholds->primary)
3711                 goto unlock;
3712 
3713         /* Check if a threshold crossed before removing */
3714         __mem_cgroup_threshold(memcg, type == _MEMSWAP);
3715 
3716         /* Calculate new number of threshold */
3717         size = 0;
3718         for (i = 0; i < thresholds->primary->size; i++) {
3719                 if (thresholds->primary->entries[i].eventfd != eventfd)
3720                         size++;
3721         }
3722 
3723         new = thresholds->spare;
3724 
3725         /* Set thresholds array to NULL if we don't have thresholds */
3726         if (!size) {
3727                 kfree(new);
3728                 new = NULL;
3729                 goto swap_buffers;
3730         }
3731 
3732         new->size = size;
3733 
3734         /* Copy thresholds and find current threshold */
3735         new->current_threshold = -1;
3736         for (i = 0, j = 0; i < thresholds->primary->size; i++) {
3737                 if (thresholds->primary->entries[i].eventfd == eventfd)
3738                         continue;
3739 
3740                 new->entries[j] = thresholds->primary->entries[i];
3741                 if (new->entries[j].threshold <= usage) {
3742                         /*
3743                          * new->current_threshold will not be used
3744                          * until rcu_assign_pointer(), so it's safe to increment
3745                          * it here.
3746                          */
3747                         ++new->current_threshold;
3748                 }
3749                 j++;
3750         }
3751 
3752 swap_buffers:
3753         /* Swap primary and spare array */
3754         thresholds->spare = thresholds->primary;
3755 
3756         rcu_assign_pointer(thresholds->primary, new);
3757 
3758         /* To be sure that nobody uses thresholds */
3759         synchronize_rcu();
3760 
3761         /* If all events are unregistered, free the spare array */
3762         if (!new) {
3763                 kfree(thresholds->spare);
3764                 thresholds->spare = NULL;
3765         }
3766 unlock:
3767         mutex_unlock(&memcg->thresholds_lock);
3768 }
3769 
3770 static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
3771         struct eventfd_ctx *eventfd)
3772 {
3773         return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM);
3774 }
3775 
3776 static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
3777         struct eventfd_ctx *eventfd)
3778 {
3779         return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP);
3780 }
3781 
3782 static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg,
3783         struct eventfd_ctx *eventfd, const char *args)
3784 {
3785         struct mem_cgroup_eventfd_list *event;
3786 
3787         event = kmalloc(sizeof(*event), GFP_KERNEL);
3788         if (!event)
3789                 return -ENOMEM;
3790 
3791         spin_lock(&memcg_oom_lock);
3792 
3793         event->eventfd = eventfd;
3794         list_add(&event->list, &memcg->oom_notify);
3795 
3796         /* already in OOM ? */
3797         if (memcg->under_oom)
3798                 eventfd_signal(eventfd, 1);
3799         spin_unlock(&memcg_oom_lock);
3800 
3801         return 0;
3802 }
3803 
3804 static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg,
3805         struct eventfd_ctx *eventfd)
3806 {
3807         struct mem_cgroup_eventfd_list *ev, *tmp;
3808 
3809         spin_lock(&memcg_oom_lock);
3810 
3811         list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
3812                 if (ev->eventfd == eventfd) {
3813                         list_del(&ev->list);
3814                         kfree(ev);
3815                 }
3816         }
3817 
3818         spin_unlock(&memcg_oom_lock);
3819 }
3820 
3821 static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v)
3822 {
3823         struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(sf));
3824 
3825         seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable);
3826         seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom);
3827         seq_printf(sf, "oom_kill %lu\n",
3828                    atomic_long_read(&memcg->memory_events[MEMCG_OOM_KILL]));
3829         return 0;
3830 }
3831 
3832 static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
3833         struct cftype *cft, u64 val)
3834 {
3835         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3836 
3837         /* cannot set to root cgroup and only 0 and 1 are allowed */
3838         if (!css->parent || !((val == 0) || (val == 1)))
3839                 return -EINVAL;
3840 
3841         memcg->oom_kill_disable = val;
3842         if (!val)
3843                 memcg_oom_recover(memcg);
3844 
3845         return 0;
3846 }
3847 
3848 #ifdef CONFIG_CGROUP_WRITEBACK
3849 
3850 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
3851 {
3852         return wb_domain_init(&memcg->cgwb_domain, gfp);
3853 }
3854 
3855 static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
3856 {
3857         wb_domain_exit(&memcg->cgwb_domain);
3858 }
3859 
3860 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
3861 {
3862         wb_domain_size_changed(&memcg->cgwb_domain);
3863 }
3864 
3865 struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
3866 {
3867         struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
3868 
3869         if (!memcg->css.parent)
3870                 return NULL;
3871 
3872         return &memcg->cgwb_domain;
3873 }
3874 
3875 /**
3876  * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg
3877  * @wb: bdi_writeback in question
3878  * @pfilepages: out parameter for number of file pages
3879  * @pheadroom: out parameter for number of allocatable pages according to memcg
3880  * @pdirty: out parameter for number of dirty pages
3881  * @pwriteback: out parameter for number of pages under writeback
3882  *
3883  * Determine the numbers of file, headroom, dirty, and writeback pages in
3884  * @wb's memcg.  File, dirty and writeback are self-explanatory.  Headroom
3885  * is a bit more involved.
3886  *
3887  * A memcg's headroom is "min(max, high) - used".  In the hierarchy, the
3888  * headroom is calculated as the lowest headroom of itself and the
3889  * ancestors.  Note that this doesn't consider the actual amount of
3890  * available memory in the system.  The caller should further cap
3891  * *@pheadroom accordingly.
3892  */
3893 void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
3894                          unsigned long *pheadroom, unsigned long *pdirty,
3895                          unsigned long *pwriteback)
3896 {
3897         struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
3898         struct mem_cgroup *parent;
3899 
3900         *pdirty = memcg_page_state(memcg, NR_FILE_DIRTY);
3901 
3902         /* this should eventually include NR_UNSTABLE_NFS */
3903         *pwriteback = memcg_page_state(memcg, NR_WRITEBACK);
3904         *pfilepages = mem_cgroup_nr_lru_pages(memcg, (1 << LRU_INACTIVE_FILE) |
3905                                                      (1 << LRU_ACTIVE_FILE));
3906         *pheadroom = PAGE_COUNTER_MAX;
3907 
3908         while ((parent = parent_mem_cgroup(memcg))) {
3909                 unsigned long ceiling = min(memcg->memory.max, memcg->high);
3910                 unsigned long used = page_counter_read(&memcg->memory);
3911 
3912                 *pheadroom = min(*pheadroom, ceiling - min(ceiling, used));
3913                 memcg = parent;
3914         }
3915 }
3916 
3917 #else   /* CONFIG_CGROUP_WRITEBACK */
3918 
3919 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
3920 {
3921         return 0;
3922 }
3923 
3924 static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
3925 {
3926 }
3927 
3928 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
3929 {
3930 }
3931 
3932 #endif  /* CONFIG_CGROUP_WRITEBACK */
3933 
3934 /*
3935  * DO NOT USE IN NEW FILES.
3936  *
3937  * "cgroup.event_control" implementation.
3938  *
3939  * This is way over-engineered.  It tries to support fully configurable
3940  * events for each user.  Such level of flexibility is completely
3941  * unnecessary especially in the light of the planned unified hierarchy.
3942  *
3943  * Please deprecate this and replace with something simpler if at all
3944  * possible.
3945  */
3946 
3947 /*
3948  * Unregister event and free resources.
3949  *
3950  * Gets called from workqueue.
3951  */
3952 static void memcg_event_remove(struct work_struct *work)
3953 {
3954         struct mem_cgroup_event *event =
3955                 container_of(work, struct mem_cgroup_event, remove);
3956         struct mem_cgroup *memcg = event->memcg;
3957 
3958         remove_wait_queue(event->wqh, &event->wait);
3959 
3960         event->unregister_event(memcg, event->eventfd);
3961 
3962         /* Notify userspace the event is going away. */
3963         eventfd_signal(event->eventfd, 1);
3964 
3965         eventfd_ctx_put(event->eventfd);
3966         kfree(event);
3967         css_put(&memcg->css);
3968 }
3969 
3970 /*
3971  * Gets called on EPOLLHUP on eventfd when user closes it.
3972  *
3973  * Called with wqh->lock held and interrupts disabled.
3974  */
3975 static int memcg_event_wake(wait_queue_entry_t *wait, unsigned mode,
3976                             int sync, void *key)
3977 {
3978         struct mem_cgroup_event *event =
3979                 container_of(wait, struct mem_cgroup_event, wait);
3980         struct mem_cgroup *memcg = event->memcg;
3981         __poll_t flags = key_to_poll(key);
3982 
3983         if (flags & EPOLLHUP) {
3984                 /*
3985                  * If the event has been detached at cgroup removal, we
3986                  * can simply return knowing the other side will cleanup
3987                  * for us.
3988                  *
3989                  * We can't race against event freeing since the other
3990                  * side will require wqh->lock via remove_wait_queue(),
3991                  * which we hold.
3992                  */
3993                 spin_lock(&memcg->event_list_lock);
3994                 if (!list_empty(&event->list)) {
3995                         list_del_init(&event->list);
3996                         /*
3997                          * We are in atomic context, but cgroup_event_remove()
3998                          * may sleep, so we have to call it in workqueue.
3999                          */
4000                         schedule_work(&event->remove);
4001                 }
4002                 spin_unlock(&memcg->event_list_lock);
4003         }
4004 
4005         return 0;
4006 }
4007 
4008 static void memcg_event_ptable_queue_proc(struct file *file,
4009                 wait_queue_head_t *wqh, poll_table *pt)
4010 {
4011         struct mem_cgroup_event *event =
4012                 container_of(pt, struct mem_cgroup_event, pt);
4013 
4014         event->wqh = wqh;
4015         add_wait_queue(wqh, &event->wait);
4016 }
4017 
4018 /*
4019  * DO NOT USE IN NEW FILES.
4020  *
4021  * Parse input and register new cgroup event handler.
4022  *
4023  * Input must be in format '<event_fd> <control_fd> <args>'.
4024  * Interpretation of args is defined by control file implementation.
4025  */
4026 static ssize_t memcg_write_event_control(struct kernfs_open_file *of,
4027                                          char *buf, size_t nbytes, loff_t off)
4028 {
4029         struct cgroup_subsys_state *css = of_css(of);
4030         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4031         struct mem_cgroup_event *event;
4032         struct cgroup_subsys_state *cfile_css;
4033         unsigned int efd, cfd;
4034         struct fd efile;
4035         struct fd cfile;
4036         const char *name;
4037         char *endp;
4038         int ret;
4039 
4040         buf = strstrip(buf);
4041 
4042         efd = simple_strtoul(buf, &endp, 10);
4043         if (*endp != ' ')
4044                 return -EINVAL;
4045         buf = endp + 1;
4046 
4047         cfd = simple_strtoul(buf, &endp, 10);
4048         if ((*endp != ' ') && (*endp != '\0'))
4049                 return -EINVAL;
4050         buf = endp + 1;
4051 
4052         event = kzalloc(sizeof(*event), GFP_KERNEL);
4053         if (!event)
4054                 return -ENOMEM;
4055 
4056         event->memcg = memcg;
4057         INIT_LIST_HEAD(&event->list);
4058         init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc);
4059         init_waitqueue_func_entry(&event->wait, memcg_event_wake);
4060         INIT_WORK(&event->remove, memcg_event_remove);
4061 
4062         efile = fdget(efd);
4063         if (!efile.file) {
4064                 ret = -EBADF;
4065                 goto out_kfree;
4066         }
4067 
4068         event->eventfd = eventfd_ctx_fileget(efile.file);
4069         if (IS_ERR(event->eventfd)) {
4070                 ret = PTR_ERR(event->eventfd);
4071                 goto out_put_efile;
4072         }
4073 
4074         cfile = fdget(cfd);
4075         if (!cfile.file) {
4076                 ret = -EBADF;
4077                 goto out_put_eventfd;
4078         }
4079 
4080         /* the process need read permission on control file */
4081         /* AV: shouldn't we check that it's been opened for read instead? */
4082         ret = inode_permission(file_inode(cfile.file), MAY_READ);
4083         if (ret < 0)
4084                 goto out_put_cfile;
4085 
4086         /*
4087          * Determine the event callbacks and set them in @event.  This used
4088          * to be done via struct cftype but cgroup core no longer knows
4089          * about these events.  The following is crude but the whole thing
4090          * is for compatibility anyway.
4091          *
4092          * DO NOT ADD NEW FILES.
4093          */
4094         name = cfile.file->f_path.dentry->d_name.name;
4095 
4096         if (!strcmp(name, "memory.usage_in_bytes")) {
4097                 event->register_event = mem_cgroup_usage_register_event;
4098                 event->unregister_event = mem_cgroup_usage_unregister_event;
4099         } else if (!strcmp(name, "memory.oom_control")) {
4100                 event->register_event = mem_cgroup_oom_register_event;
4101                 event->unregister_event = mem_cgroup_oom_unregister_event;
4102         } else if (!strcmp(name, "memory.pressure_level")) {
4103                 event->register_event = vmpressure_register_event;
4104                 event->unregister_event = vmpressure_unregister_event;
4105         } else if (!strcmp(name, "memory.memsw.usage_in_bytes")) {
4106                 event->register_event = memsw_cgroup_usage_register_event;
4107                 event->unregister_event = memsw_cgroup_usage_unregister_event;
4108         } else {
4109                 ret = -EINVAL;
4110                 goto out_put_cfile;
4111         }
4112 
4113         /*
4114          * Verify @cfile should belong to @css.  Also, remaining events are
4115          * automatically removed on cgroup destruction but the removal is
4116          * asynchronous, so take an extra ref on @css.
4117          */
4118         cfile_css = css_tryget_online_from_dir(cfile.file->f_path.dentry->d_parent,
4119                                                &memory_cgrp_subsys);
4120         ret = -EINVAL;
4121         if (IS_ERR(cfile_css))
4122                 goto out_put_cfile;
4123         if (cfile_css != css) {
4124                 css_put(cfile_css);
4125                 goto out_put_cfile;
4126         }
4127 
4128         ret = event->register_event(memcg, event->eventfd, buf);
4129         if (ret)
4130                 goto out_put_css;
4131 
4132         vfs_poll(efile.file, &event->pt);
4133 
4134         spin_lock(&memcg->event_list_lock);
4135         list_add(&event->list, &memcg->event_list);
4136         spin_unlock(&memcg->event_list_lock);
4137 
4138         fdput(cfile);
4139         fdput(efile);
4140 
4141         return nbytes;
4142 
4143 out_put_css:
4144         css_put(css);
4145 out_put_cfile:
4146         fdput(cfile);
4147 out_put_eventfd:
4148         eventfd_ctx_put(event->eventfd);
4149 out_put_efile:
4150         fdput(efile);
4151 out_kfree:
4152         kfree(event);
4153 
4154         return ret;
4155 }
4156 
4157 static struct cftype mem_cgroup_legacy_files[] = {
4158         {
4159                 .name = "usage_in_bytes",
4160                 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
4161                 .read_u64 = mem_cgroup_read_u64,
4162         },
4163         {
4164                 .name = "max_usage_in_bytes",
4165                 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
4166                 .write = mem_cgroup_reset,
4167                 .read_u64 = mem_cgroup_read_u64,
4168         },
4169         {
4170                 .name = "limit_in_bytes",
4171                 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
4172                 .write = mem_cgroup_write,
4173                 .read_u64 = mem_cgroup_read_u64,
4174         },
4175         {
4176                 .name = "soft_limit_in_bytes",
4177                 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
4178                 .write = mem_cgroup_write,
4179                 .read_u64 = mem_cgroup_read_u64,
4180         },
4181         {
4182                 .name = "failcnt",
4183                 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
4184                 .write = mem_cgroup_reset,
4185                 .read_u64 = mem_cgroup_read_u64,
4186         },
4187         {
4188                 .name = "stat",
4189                 .seq_show = memcg_stat_show,
4190         },
4191         {
4192                 .name = "force_empty",
4193                 .write = mem_cgroup_force_empty_write,
4194         },
4195         {
4196                 .name = "use_hierarchy",
4197                 .write_u64 = mem_cgroup_hierarchy_write,
4198                 .read_u64 = mem_cgroup_hierarchy_read,
4199         },
4200         {
4201                 .name = "cgroup.event_control",         /* XXX: for compat */
4202                 .write = memcg_write_event_control,
4203                 .flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE,
4204         },
4205         {
4206                 .name = "swappiness",
4207                 .read_u64 = mem_cgroup_swappiness_read,
4208                 .write_u64 = mem_cgroup_swappiness_write,
4209         },
4210         {
4211                 .name = "move_charge_at_immigrate",
4212                 .read_u64 = mem_cgroup_move_charge_read,
4213                 .write_u64 = mem_cgroup_move_charge_write,
4214         },
4215         {
4216                 .name = "oom_control",
4217                 .seq_show = mem_cgroup_oom_control_read,
4218                 .write_u64 = mem_cgroup_oom_control_write,
4219                 .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
4220         },
4221         {
4222                 .name = "pressure_level",
4223         },
4224 #ifdef CONFIG_NUMA
4225         {
4226                 .name = "numa_stat",
4227                 .seq_show = memcg_numa_stat_show,
4228         },
4229 #endif
4230         {
4231                 .name = "kmem.limit_in_bytes",
4232                 .private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
4233                 .write = mem_cgroup_write,
4234                 .read_u64 = mem_cgroup_read_u64,
4235         },
4236         {
4237                 .name = "kmem.usage_in_bytes",
4238                 .private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
4239                 .read_u64 = mem_cgroup_read_u64,
4240         },
4241         {
4242                 .name = "kmem.failcnt",
4243                 .private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
4244                 .write = mem_cgroup_reset,
4245                 .read_u64 = mem_cgroup_read_u64,
4246         },
4247         {
4248                 .name = "kmem.max_usage_in_bytes",
4249                 .private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
4250                 .write = mem_cgroup_reset,
4251                 .read_u64 = mem_cgroup_read_u64,
4252         },
4253 #if defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG)
4254         {
4255                 .name = "kmem.slabinfo",
4256                 .seq_start = memcg_slab_start,
4257                 .seq_next = memcg_slab_next,
4258                 .seq_stop = memcg_slab_stop,
4259                 .seq_show = memcg_slab_show,
4260         },
4261 #endif
4262         {
4263                 .name = "kmem.tcp.limit_in_bytes",
4264                 .private = MEMFILE_PRIVATE(_TCP, RES_LIMIT),
4265                 .write = mem_cgroup_write,
4266                 .read_u64 = mem_cgroup_read_u64,
4267         },
4268         {
4269                 .name = "kmem.tcp.usage_in_bytes",
4270                 .private = MEMFILE_PRIVATE(_TCP, RES_USAGE),
4271                 .read_u64 = mem_cgroup_read_u64,
4272         },
4273         {
4274                 .name = "kmem.tcp.failcnt",
4275                 .private = MEMFILE_PRIVATE(_TCP, RES_FAILCNT),
4276                 .write = mem_cgroup_reset,
4277                 .read_u64 = mem_cgroup_read_u64,
4278         },
4279         {
4280                 .name = "kmem.tcp.max_usage_in_bytes",
4281                 .private = MEMFILE_PRIVATE(_TCP, RES_MAX_USAGE),
4282                 .write = mem_cgroup_reset,
4283                 .read_u64 = mem_cgroup_read_u64,
4284         },
4285         { },    /* terminate */
4286 };
4287 
4288 /*
4289  * Private memory cgroup IDR
4290  *
4291  * Swap-out records and page cache shadow entries need to store memcg
4292  * references in constrained space, so we maintain an ID space that is
4293  * limited to 16 bit (MEM_CGROUP_ID_MAX), limiting the total number of
4294  * memory-controlled cgroups to 64k.
4295  *
4296  * However, there usually are many references to the oflline CSS after
4297  * the cgroup has been destroyed, such as page cache or reclaimable
4298  * slab objects, that don't need to hang on to the ID. We want to keep
4299  * those dead CSS from occupying IDs, or we might quickly exhaust the
4300  * relatively small ID space and prevent the creation of new cgroups
4301  * even when there are much fewer than 64k cgroups - possibly none.
4302  *
4303  * Maintain a private 16-bit ID space for memcg, and allow the ID to
4304  * be freed and recycled when it's no longer needed, which is usually
4305  * when the CSS is offlined.
4306  *
4307  * The only exception to that are records of swapped out tmpfs/shmem
4308  * pages that need to be attributed to live ancestors on swapin. But
4309  * those references are manageable from userspace.
4310  */
4311 
4312 static DEFINE_IDR(mem_cgroup_idr);
4313 
4314 static void mem_cgroup_id_remove(struct mem_cgroup *memcg)
4315 {
4316         if (memcg->id.id > 0) {
4317                 idr_remove(&mem_cgroup_idr, memcg->id.id);
4318                 memcg->id.id = 0;
4319         }
4320 }
4321 
4322 static void mem_cgroup_id_get_many(struct mem_cgroup *memcg, unsigned int n)
4323 {
4324         VM_BUG_ON(atomic_read(&memcg->id.ref) <= 0);
4325         atomic_add(n, &memcg->id.ref);
4326 }
4327 
4328 static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n)
4329 {
4330         VM_BUG_ON(atomic_read(&memcg->id.ref) < n);
4331         if (atomic_sub_and_test(n, &memcg->id.ref)) {
4332                 mem_cgroup_id_remove(memcg);
4333 
4334                 /* Memcg ID pins CSS */
4335                 css_put(&memcg->css);
4336         }
4337 }
4338 
4339 static inline void mem_cgroup_id_get(struct mem_cgroup *memcg)
4340 {
4341         mem_cgroup_id_get_many(memcg, 1);
4342 }
4343 
4344 static inline void mem_cgroup_id_put(struct mem_cgroup *memcg)
4345 {
4346         mem_cgroup_id_put_many(memcg, 1);
4347 }
4348 
4349 /**
4350  * mem_cgroup_from_id - look up a memcg from a memcg id
4351  * @id: the memcg id to look up
4352  *
4353  * Caller must hold rcu_read_lock().
4354  */
4355 struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
4356 {
4357         WARN_ON_ONCE(!rcu_read_lock_held());
4358         return idr_find(&mem_cgroup_idr, id);
4359 }
4360 
4361 static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
4362 {
4363         struct mem_cgroup_per_node *pn;
4364         int tmp = node;
4365         /*
4366          * This routine is called against possible nodes.
4367          * But it's BUG to call kmalloc() against offline node.
4368          *
4369          * TODO: this routine can waste much memory for nodes which will
4370          *       never be onlined. It's better to use memory hotplug callback
4371          *       function.
4372          */
4373         if (!node_state(node, N_NORMAL_MEMORY))
4374                 tmp = -1;
4375         pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
4376         if (!pn)
4377                 return 1;
4378 
4379         pn->lruvec_stat_cpu = alloc_percpu(struct lruvec_stat);
4380         if (!pn->lruvec_stat_cpu) {
4381                 kfree(pn);
4382                 return 1;
4383         }
4384 
4385         lruvec_init(&pn->lruvec);
4386         pn->usage_in_excess = 0;
4387         pn->on_tree = false;
4388         pn->memcg = memcg;
4389 
4390         memcg->nodeinfo[node] = pn;
4391         return 0;
4392 }
4393 
4394 static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
4395 {
4396         struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
4397 
4398         if (!pn)
4399                 return;
4400 
4401         free_percpu(pn->lruvec_stat_cpu);
4402         kfree(pn);
4403 }
4404 
4405 static void __mem_cgroup_free(struct mem_cgroup *memcg)
4406 {
4407         int node;
4408 
4409         for_each_node(node)
4410                 free_mem_cgroup_per_node_info(memcg, node);
4411         free_percpu(memcg->stat_cpu);
4412         kfree(memcg);
4413 }
4414 
4415 static void mem_cgroup_free(struct mem_cgroup *memcg)
4416 {
4417         memcg_wb_domain_exit(memcg);
4418         __mem_cgroup_free(memcg);
4419 }
4420 
4421 static struct mem_cgroup *mem_cgroup_alloc(void)
4422 {
4423         struct mem_cgroup *memcg;
4424         size_t size;
4425         int node;
4426 
4427         size = sizeof(struct mem_cgroup);
4428         size += nr_node_ids * sizeof(struct mem_cgroup_per_node *);
4429 
4430         memcg = kzalloc(size, GFP_KERNEL);
4431         if (!memcg)
4432                 return NULL;
4433 
4434         memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL,
4435                                  1, MEM_CGROUP_ID_MAX,
4436                                  GFP_KERNEL);
4437         if (memcg->id.id < 0)
4438                 goto fail;
4439 
4440         memcg->stat_cpu = alloc_percpu(struct mem_cgroup_stat_cpu);
4441         if (!memcg->stat_cpu)
4442                 goto fail;
4443 
4444         for_each_node(node)
4445                 if (alloc_mem_cgroup_per_node_info(memcg, node))
4446                         goto fail;
4447 
4448         if (memcg_wb_domain_init(memcg, GFP_KERNEL))
4449                 goto fail;
4450 
4451         INIT_WORK(&memcg->high_work, high_work_func);
4452         memcg->last_scanned_node = MAX_NUMNODES;
4453         INIT_LIST_HEAD(&memcg->oom_notify);
4454         mutex_init(&memcg->thresholds_lock);
4455         spin_lock_init(&memcg->move_lock);
4456         vmpressure_init(&memcg->vmpressure);
4457         INIT_LIST_HEAD(&memcg->event_list);
4458         spin_lock_init(&memcg->event_list_lock);
4459         memcg->socket_pressure = jiffies;
4460 #ifdef CONFIG_MEMCG_KMEM
4461         memcg->kmemcg_id = -1;
4462 #endif
4463 #ifdef CONFIG_CGROUP_WRITEBACK
4464         INIT_LIST_HEAD(&memcg->cgwb_list);
4465 #endif
4466         idr_replace(&mem_cgroup_idr, memcg, memcg->id.id);
4467         return memcg;
4468 fail:
4469         mem_cgroup_id_remove(memcg);
4470         __mem_cgroup_free(memcg);
4471         return NULL;
4472 }
4473 
4474 static struct cgroup_subsys_state * __ref
4475 mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
4476 {
4477         struct mem_cgroup *parent = mem_cgroup_from_css(parent_css);
4478         struct mem_cgroup *memcg;
4479         long error = -ENOMEM;
4480 
4481         memcg = mem_cgroup_alloc();
4482         if (!memcg)
4483                 return ERR_PTR(error);
4484 
4485         memcg->high = PAGE_COUNTER_MAX;
4486         memcg->soft_limit = PAGE_COUNTER_MAX;
4487         if (parent) {
4488                 memcg->swappiness = mem_cgroup_swappiness(parent);
4489                 memcg->oom_kill_disable = parent->oom_kill_disable;
4490         }
4491         if (parent && parent->use_hierarchy) {
4492                 memcg->use_hierarchy = true;
4493                 page_counter_init(&memcg->memory, &parent->memory);
4494                 page_counter_init(&memcg->swap, &parent->swap);
4495                 page_counter_init(&memcg->memsw, &parent->memsw);
4496                 page_counter_init(&memcg->kmem, &parent->kmem);
4497                 page_counter_init(&memcg->tcpmem, &parent->tcpmem);
4498         } else {
4499                 page_counter_init(&memcg->memory, NULL);
4500                 page_counter_init(&memcg->swap, NULL);
4501                 page_counter_init(&memcg->memsw, NULL);
4502                 page_counter_init(&memcg->kmem, NULL);
4503                 page_counter_init(&memcg->tcpmem, NULL);
4504                 /*
4505                  * Deeper hierachy with use_hierarchy == false doesn't make
4506                  * much sense so let cgroup subsystem know about this
4507                  * unfortunate state in our controller.
4508                  */
4509                 if (parent != root_mem_cgroup)
4510                         memory_cgrp_subsys.broken_hierarchy = true;
4511         }
4512 
4513         /* The following stuff does not apply to the root */
4514         if (!parent) {
4515                 root_mem_cgroup = memcg;
4516                 return &memcg->css;
4517         }
4518 
4519         error = memcg_online_kmem(memcg);
4520         if (error)
4521                 goto fail;
4522 
4523         if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
4524                 static_branch_inc(&memcg_sockets_enabled_key);
4525 
4526         return &memcg->css;
4527 fail:
4528         mem_cgroup_id_remove(memcg);
4529         mem_cgroup_free(memcg);
4530         return ERR_PTR(-ENOMEM);
4531 }
4532 
4533 static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
4534 {
4535         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4536 
4537         /*
4538          * A memcg must be visible for memcg_expand_shrinker_maps()
4539          * by the time the maps are allocated. So, we allocate maps
4540          * here, when for_each_mem_cgroup() can't skip it.
4541          */
4542         if (memcg_alloc_shrinker_maps(memcg)) {
4543                 mem_cgroup_id_remove(memcg);
4544                 return -ENOMEM;
4545         }
4546 
4547         /* Online state pins memcg ID, memcg ID pins CSS */
4548         atomic_set(&memcg->id.ref, 1);
4549         css_get(css);
4550         return 0;
4551 }
4552 
4553 static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
4554 {
4555         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4556         struct mem_cgroup_event *event, *tmp;
4557 
4558         /*
4559          * Unregister events and notify userspace.
4560          * Notify userspace about cgroup removing only after rmdir of cgroup
4561          * directory to avoid race between userspace and kernelspace.
4562          */
4563         spin_lock(&memcg->event_list_lock);
4564         list_for_each_entry_safe(event, tmp, &memcg->event_list, list) {
4565                 list_del_init(&event->list);
4566                 schedule_work(&event->remove);
4567         }
4568         spin_unlock(&memcg->event_list_lock);
4569 
4570         page_counter_set_min(&memcg->memory, 0);
4571         page_counter_set_low(&memcg->memory, 0);
4572 
4573         memcg_offline_kmem(memcg);
4574         wb_memcg_offline(memcg);
4575 
4576         mem_cgroup_id_put(memcg);
4577 }
4578 
4579 static void mem_cgroup_css_released(struct cgroup_subsys_state *css)
4580 {
4581         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4582 
4583         invalidate_reclaim_iterators(memcg);
4584 }
4585 
4586 static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
4587 {
4588         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4589 
4590         if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
4591                 static_branch_dec(&memcg_sockets_enabled_key);
4592 
4593         if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active)
4594                 static_branch_dec(&memcg_sockets_enabled_key);
4595 
4596         vmpressure_cleanup(&memcg->vmpressure);
4597         cancel_work_sync(&memcg->high_work);
4598         mem_cgroup_remove_from_trees(memcg);
4599         memcg_free_shrinker_maps(memcg);
4600         memcg_free_kmem(memcg);
4601         mem_cgroup_free(memcg);
4602 }
4603 
4604 /**
4605  * mem_cgroup_css_reset - reset the states of a mem_cgroup
4606  * @css: the target css
4607  *
4608  * Reset the states of the mem_cgroup associated with @css.  This is
4609  * invoked when the userland requests disabling on the default hierarchy
4610  * but the memcg is pinned through dependency.  The memcg should stop
4611  * applying policies and should revert to the vanilla state as it may be
4612  * made visible again.
4613  *
4614  * The current implementation only resets the essential configurations.
4615  * This needs to be expanded to cover all the visible parts.
4616  */
4617 static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
4618 {
4619         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4620 
4621         page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX);
4622         page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX);
4623         page_counter_set_max(&memcg->memsw, PAGE_COUNTER_MAX);
4624         page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX);
4625         page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX);
4626         page_counter_set_min(&memcg->memory, 0);
4627         page_counter_set_low(&memcg->memory, 0);
4628         memcg->high = PAGE_COUNTER_MAX;
4629         memcg->soft_limit = PAGE_COUNTER_MAX;
4630         memcg_wb_domain_size_changed(memcg);
4631 }
4632 
4633 #ifdef CONFIG_MMU
4634 /* Handlers for move charge at task migration. */
4635 static int mem_cgroup_do_precharge(unsigned long count)
4636 {
4637         int ret;
4638 
4639         /* Try a single bulk charge without reclaim first, kswapd may wake */
4640         ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_DIRECT_RECLAIM, count);
4641         if (!ret) {
4642                 mc.precharge += count;
4643                 return ret;
4644         }
4645 
4646         /* Try charges one by one with reclaim, but do not retry */
4647         while (count--) {
4648                 ret = try_charge(mc.to, GFP_KERNEL | __GFP_NORETRY, 1);
4649                 if (ret)
4650                         return ret;
4651                 mc.precharge++;
4652                 cond_resched();
4653         }
4654         return 0;
4655 }
4656 
4657 union mc_target {
4658         struct page     *page;
4659         swp_entry_t     ent;
4660 };
4661 
4662 enum mc_target_type {
4663         MC_TARGET_NONE = 0,
4664         MC_TARGET_PAGE,
4665         MC_TARGET_SWAP,
4666         MC_TARGET_DEVICE,
4667 };
4668 
4669 static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
4670                                                 unsigned long addr, pte_t ptent)
4671 {
4672         struct page *page = _vm_normal_page(vma, addr, ptent, true);
4673 
4674         if (!page || !page_mapped(page))
4675                 return NULL;
4676         if (PageAnon(page)) {
4677                 if (!(mc.flags & MOVE_ANON))
4678                         return NULL;
4679         } else {
4680                 if (!(mc.flags & MOVE_FILE))
4681                         return NULL;
4682         }
4683         if (!get_page_unless_zero(page))
4684                 return NULL;
4685 
4686         return page;
4687 }
4688 
4689 #if defined(CONFIG_SWAP) || defined(CONFIG_DEVICE_PRIVATE)
4690 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
4691                         pte_t ptent, swp_entry_t *entry)
4692 {
4693         struct page *page = NULL;
4694         swp_entry_t ent = pte_to_swp_entry(ptent);
4695 
4696         if (!(mc.flags & MOVE_ANON) || non_swap_entry(ent))
4697                 return NULL;
4698 
4699         /*
4700          * Handle MEMORY_DEVICE_PRIVATE which are ZONE_DEVICE page belonging to
4701          * a device and because they are not accessible by CPU they are store
4702          * as special swap entry in the CPU page table.
4703          */
4704         if (is_device_private_entry(ent)) {
4705                 page = device_private_entry_to_page(ent);
4706                 /*
4707                  * MEMORY_DEVICE_PRIVATE means ZONE_DEVICE page and which have
4708                  * a refcount of 1 when free (unlike normal page)
4709                  */
4710                 if (!page_ref_add_unless(page, 1, 1))
4711                         return NULL;
4712                 return page;
4713         }
4714 
4715         /*
4716          * Because lookup_swap_cache() updates some statistics counter,
4717          * we call find_get_page() with swapper_space directly.
4718          */
4719         page = find_get_page(swap_address_space(ent), swp_offset(ent));
4720         if (do_memsw_account())
4721                 entry->val = ent.val;
4722 
4723         return page;
4724 }
4725 #else
4726 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
4727                         pte_t ptent, swp_entry_t *entry)
4728 {
4729         return NULL;
4730 }
4731 #endif
4732 
4733 static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
4734                         unsigned long addr, pte_t ptent, swp_entry_t *entry)
4735 {
4736         struct page *page = NULL;
4737         struct address_space *mapping;
4738         pgoff_t pgoff;
4739 
4740         if (!vma->vm_file) /* anonymous vma */
4741                 return NULL;
4742         if (!(mc.flags & MOVE_FILE))
4743                 return NULL;
4744 
4745         mapping = vma->vm_file->f_mapping;
4746         pgoff = linear_page_index(vma, addr);
4747 
4748         /* page is moved even if it's not RSS of this task(page-faulted). */
4749 #ifdef CONFIG_SWAP
4750         /* shmem/tmpfs may report page out on swap: account for that too. */
4751         if (shmem_mapping(mapping)) {
4752                 page = find_get_entry(mapping, pgoff);
4753                 if (radix_tree_exceptional_entry(page)) {
4754                         swp_entry_t swp = radix_to_swp_entry(page);
4755                         if (do_memsw_account())
4756                                 *entry = swp;
4757                         page = find_get_page(swap_address_space(swp),
4758                                              swp_offset(swp));
4759                 }
4760         } else
4761                 page = find_get_page(mapping, pgoff);
4762 #else
4763         page = find_get_page(mapping, pgoff);
4764 #endif
4765         return page;
4766 }
4767 
4768 /**
4769  * mem_cgroup_move_account - move account of the page
4770  * @page: the page
4771  * @compound: charge the page as compound or small page
4772  * @from: mem_cgroup which the page is moved from.
4773  * @to: mem_cgroup which the page is moved to. @from != @to.
4774  *
4775  * The caller must make sure the page is not on LRU (isolate_page() is useful.)
4776  *
4777  * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
4778  * from old cgroup.
4779  */
4780 static int mem_cgroup_move_account(struct page *page,
4781                                    bool compound,
4782                                    struct mem_cgroup *from,
4783                                    struct mem_cgroup *to)
4784 {
4785         unsigned long flags;
4786         unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
4787         int ret;
4788         bool anon;
4789 
4790         VM_BUG_ON(from == to);
4791         VM_BUG_ON_PAGE(PageLRU(page), page);
4792         VM_BUG_ON(compound && !PageTransHuge(page));
4793 
4794         /*
4795          * Prevent mem_cgroup_migrate() from looking at
4796          * page->mem_cgroup of its source page while we change it.
4797          */
4798         ret = -EBUSY;
4799         if (!trylock_page(page))
4800                 goto out;
4801 
4802         ret = -EINVAL;
4803         if (page->mem_cgroup != from)
4804                 goto out_unlock;
4805 
4806         anon = PageAnon(page);
4807 
4808         spin_lock_irqsave(&from->move_lock, flags);
4809 
4810         if (!anon && page_mapped(page)) {
4811                 __mod_memcg_state(from, NR_FILE_MAPPED, -nr_pages);
4812                 __mod_memcg_state(to, NR_FILE_MAPPED, nr_pages);
4813         }
4814 
4815         /*
4816          * move_lock grabbed above and caller set from->moving_account, so
4817          * mod_memcg_page_state will serialize updates to PageDirty.
4818          * So mapping should be stable for dirty pages.
4819          */
4820         if (!anon && PageDirty(page)) {
4821                 struct address_space *mapping = page_mapping(page);
4822 
4823                 if (mapping_cap_account_dirty(mapping)) {
4824                         __mod_memcg_state(from, NR_FILE_DIRTY, -nr_pages);
4825                         __mod_memcg_state(to, NR_FILE_DIRTY, nr_pages);
4826                 }
4827         }
4828 
4829         if (PageWriteback(page)) {
4830                 __mod_memcg_state(from, NR_WRITEBACK, -nr_pages);
4831                 __mod_memcg_state(to, NR_WRITEBACK, nr_pages);
4832         }
4833 
4834         /*
4835          * It is safe to change page->mem_cgroup here because the page
4836          * is referenced, charged, and isolated - we can't race with
4837          * uncharging, charging, migration, or LRU putback.
4838          */
4839 
4840         /* caller should have done css_get */
4841         page->mem_cgroup = to;
4842         spin_unlock_irqrestore(&from->move_lock, flags);
4843 
4844         ret = 0;
4845 
4846         local_irq_disable();
4847         mem_cgroup_charge_statistics(to, page, compound, nr_pages);
4848         memcg_check_events(to, page);
4849         mem_cgroup_charge_statistics(from, page, compound, -nr_pages);
4850         memcg_check_events(from, page);
4851         local_irq_enable();
4852 out_unlock:
4853         unlock_page(page);
4854 out:
4855         return ret;
4856 }
4857 
4858 /**
4859  * get_mctgt_type - get target type of moving charge
4860  * @vma: the vma the pte to be checked belongs
4861  * @addr: the address corresponding to the pte to be checked
4862  * @ptent: the pte to be checked
4863  * @target: the pointer the target page or swap ent will be stored(can be NULL)
4864  *
4865  * Returns
4866  *   0(MC_TARGET_NONE): if the pte is not a target for move charge.
4867  *   1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
4868  *     move charge. if @target is not NULL, the page is stored in target->page
4869  *     with extra refcnt got(Callers should handle it).
4870  *   2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
4871  *     target for charge migration. if @target is not NULL, the entry is stored
4872  *     in target->ent.
4873  *   3(MC_TARGET_DEVICE): like MC_TARGET_PAGE  but page is MEMORY_DEVICE_PUBLIC
4874  *     or MEMORY_DEVICE_PRIVATE (so ZONE_DEVICE page and thus not on the lru).
4875  *     For now we such page is charge like a regular page would be as for all
4876  *     intent and purposes it is just special memory taking the place of a
4877  *     regular page.
4878  *
4879  *     See Documentations/vm/hmm.txt and include/linux/hmm.h
4880  *
4881  * Called with pte lock held.
4882  */
4883 
4884 static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
4885                 unsigned long addr, pte_t ptent, union mc_target *target)
4886 {
4887         struct page *page = NULL;
4888         enum mc_target_type ret = MC_TARGET_NONE;
4889         swp_entry_t ent = { .val = 0 };
4890 
4891         if (pte_present(ptent))
4892                 page = mc_handle_present_pte(vma, addr, ptent);
4893         else if (is_swap_pte(ptent))
4894                 page = mc_handle_swap_pte(vma, ptent, &ent);
4895         else if (pte_none(ptent))
4896                 page = mc_handle_file_pte(vma, addr, ptent, &ent);
4897 
4898         if (!page && !ent.val)
4899                 return ret;
4900         if (page) {
4901                 /*
4902                  * Do only loose check w/o serialization.
4903                  * mem_cgroup_move_account() checks the page is valid or
4904                  * not under LRU exclusion.
4905                  */
4906                 if (page->mem_cgroup == mc.from) {
4907                         ret = MC_TARGET_PAGE;
4908                         if (is_device_private_page(page) ||
4909                             is_device_public_page(page))
4910                                 ret = MC_TARGET_DEVICE;
4911                         if (target)
4912                                 target->page = page;
4913                 }
4914                 if (!ret || !target)
4915                         put_page(page);
4916         }
4917         /*
4918          * There is a swap entry and a page doesn't exist or isn't charged.
4919          * But we cannot move a tail-page in a THP.
4920          */
4921         if (ent.val && !ret && (!page || !PageTransCompound(page)) &&
4922             mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) {
4923                 ret = MC_TARGET_SWAP;
4924                 if (target)
4925                         target->ent = ent;
4926         }
4927         return ret;
4928 }
4929 
4930 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4931 /*
4932  * We don't consider PMD mapped swapping or file mapped pages because THP does
4933  * not support them for now.
4934  * Caller should make sure that pmd_trans_huge(pmd) is true.
4935  */
4936 static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
4937                 unsigned long addr, pmd_t pmd, union mc_target *target)
4938 {
4939         struct page *page = NULL;
4940         enum mc_target_type ret = MC_TARGET_NONE;
4941 
4942         if (unlikely(is_swap_pmd(pmd))) {
4943                 VM_BUG_ON(thp_migration_supported() &&
4944                                   !is_pmd_migration_entry(pmd));
4945                 return ret;
4946         }
4947         page = pmd_page(pmd);
4948         VM_BUG_ON_PAGE(!page || !PageHead(page), page);
4949         if (!(mc.flags & MOVE_ANON))
4950                 return ret;
4951         if (page->mem_cgroup == mc.from) {
4952                 ret = MC_TARGET_PAGE;
4953                 if (target) {
4954                         get_page(page);
4955                         target->page = page;
4956                 }
4957         }
4958         return ret;
4959 }
4960 #else
4961 static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
4962                 unsigned long addr, pmd_t pmd, union mc_target *target)
4963 {
4964         return MC_TARGET_NONE;
4965 }
4966 #endif
4967 
4968 static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
4969                                         unsigned long addr, unsigned long end,
4970                                         struct mm_walk *walk)
4971 {
4972         struct vm_area_struct *vma = walk->vma;
4973         pte_t *pte;
4974         spinlock_t *ptl;
4975 
4976         ptl = pmd_trans_huge_lock(pmd, vma);
4977         if (ptl) {
4978                 /*
4979                  * Note their can not be MC_TARGET_DEVICE for now as we do not
4980                  * support transparent huge page with MEMORY_DEVICE_PUBLIC or
4981                  * MEMORY_DEVICE_PRIVATE but this might change.
4982                  */
4983                 if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
4984                         mc.precharge += HPAGE_PMD_NR;
4985                 spin_unlock(ptl);
4986                 return 0;
4987         }
4988 
4989         if (pmd_trans_unstable(pmd))
4990                 return 0;
4991         pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
4992         for (; addr != end; pte++, addr += PAGE_SIZE)
4993                 if (get_mctgt_type(vma, addr, *pte, NULL))
4994                         mc.precharge++; /* increment precharge temporarily */
4995         pte_unmap_unlock(pte - 1, ptl);
4996         cond_resched();
4997 
4998         return 0;
4999 }
5000 
5001 static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
5002 {
5003         unsigned long precharge;
5004 
5005         struct mm_walk mem_cgroup_count_precharge_walk = {
5006                 .pmd_entry = mem_cgroup_count_precharge_pte_range,
5007                 .mm = mm,
5008         };
5009         down_read(&mm->mmap_sem);
5010         walk_page_range(0, mm->highest_vm_end,
5011                         &mem_cgroup_count_precharge_walk);
5012         up_read(&mm->mmap_sem);
5013 
5014         precharge = mc.precharge;
5015         mc.precharge = 0;
5016 
5017         return precharge;
5018 }
5019 
5020 static int mem_cgroup_precharge_mc(struct mm_struct *mm)
5021 {
5022         unsigned long precharge = mem_cgroup_count_precharge(mm);
5023 
5024         VM_BUG_ON(mc.moving_task);
5025         mc.moving_task = current;
5026         return mem_cgroup_do_precharge(precharge);
5027 }
5028 
5029 /* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
5030 static void __mem_cgroup_clear_mc(void)
5031 {
5032         struct mem_cgroup *from = mc.from;
5033         struct mem_cgroup *to = mc.to;
5034 
5035         /* we must uncharge all the leftover precharges from mc.to */
5036         if (mc.precharge) {
5037                 cancel_charge(mc.to, mc.precharge);
5038                 mc.precharge = 0;
5039         }
5040         /*
5041          * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
5042          * we must uncharge here.
5043          */
5044         if (mc.moved_charge) {
5045                 cancel_charge(mc.from, mc.moved_charge);
5046                 mc.moved_charge = 0;
5047         }
5048         /* we must fixup refcnts and charges */
5049         if (mc.moved_swap) {
5050                 /* uncharge swap account from the old cgroup */
5051                 if (!mem_cgroup_is_root(mc.from))
5052                         page_counter_uncharge(&mc.from->memsw, mc.moved_swap);
5053 
5054                 mem_cgroup_id_put_many(mc.from, mc.moved_swap);
5055 
5056                 /*
5057                  * we charged both to->memory and to->memsw, so we
5058                  * should uncharge to->memory.
5059                  */
5060                 if (!mem_cgroup_is_root(mc.to))
5061                         page_counter_uncharge(&mc.to->memory, mc.moved_swap);
5062 
5063                 mem_cgroup_id_get_many(mc.to, mc.moved_swap);
5064                 css_put_many(&mc.to->css, mc.moved_swap);
5065 
5066                 mc.moved_swap = 0;
5067         }
5068         memcg_oom_recover(from);
5069         memcg_oom_recover(to);
5070         wake_up_all(&mc.waitq);
5071 }
5072 
5073 static void mem_cgroup_clear_mc(void)
5074 {
5075         struct mm_struct *mm = mc.mm;
5076 
5077         /*
5078          * we must clear moving_task before waking up waiters at the end of
5079          * task migration.
5080          */
5081         mc.moving_task = NULL;
5082         __mem_cgroup_clear_mc();
5083         spin_lock(&mc.lock);
5084         mc.from = NULL;
5085         mc.to = NULL;
5086         mc.mm = NULL;
5087         spin_unlock(&mc.lock);
5088 
5089         mmput(mm);
5090 }
5091 
5092 static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
5093 {
5094         struct cgroup_subsys_state *css;
5095         struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */
5096         struct mem_cgroup *from;
5097         struct task_struct *leader, *p;
5098         struct mm_struct *mm;
5099         unsigned long move_flags;
5100         int ret = 0;
5101 
5102         /* charge immigration isn't supported on the default hierarchy */
5103         if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
5104                 return 0;
5105 
5106         /*
5107          * Multi-process migrations only happen on the default hierarchy
5108          * where charge immigration is not used.  Perform charge
5109          * immigration if @tset contains a leader and whine if there are
5110          * multiple.
5111          */
5112         p = NULL;
5113         cgroup_taskset_for_each_leader(leader, css, tset) {
5114                 WARN_ON_ONCE(p);
5115                 p = leader;
5116                 memcg = mem_cgroup_from_css(css);
5117         }
5118         if (!p)
5119                 return 0;
5120 
5121         /*
5122          * We are now commited to this value whatever it is. Changes in this
5123          * tunable will only affect upcoming migrations, not the current one.
5124          * So we need to save it, and keep it going.
5125          */
5126         move_flags = READ_ONCE(memcg->move_charge_at_immigrate);
5127         if (!move_flags)
5128                 return 0;
5129 
5130         from = mem_cgroup_from_task(p);
5131 
5132         VM_BUG_ON(from == memcg);
5133 
5134         mm = get_task_mm(p);
5135         if (!mm)
5136                 return 0;
5137         /* We move charges only when we move a owner of the mm */
5138         if (mm->owner == p) {
5139                 VM_BUG_ON(mc.from);
5140                 VM_BUG_ON(mc.to);
5141                 VM_BUG_ON(mc.precharge);
5142                 VM_BUG_ON(mc.moved_charge);
5143                 VM_BUG_ON(mc.moved_swap);
5144 
5145                 spin_lock(&mc.lock);
5146                 mc.mm = mm;
5147                 mc.from = from;
5148                 mc.to = memcg;
5149                 mc.flags = move_flags;
5150                 spin_unlock(&mc.lock);
5151                 /* We set mc.moving_task later */
5152 
5153                 ret = mem_cgroup_precharge_mc(mm);
5154                 if (ret)
5155                         mem_cgroup_clear_mc();
5156         } else {
5157                 mmput(mm);
5158         }
5159         return ret;
5160 }
5161 
5162 static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
5163 {
5164         if (mc.to)
5165                 mem_cgroup_clear_mc();
5166 }
5167 
5168 static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
5169                                 unsigned long addr, unsigned long end,
5170                                 struct mm_walk *walk)
5171 {
5172         int ret = 0;
5173         struct vm_area_struct *vma = walk->vma;
5174         pte_t *pte;
5175         spinlock_t *ptl;
5176         enum mc_target_type target_type;
5177         union mc_target target;
5178         struct page *page;
5179 
5180         ptl = pmd_trans_huge_lock(pmd, vma);
5181         if (ptl) {
5182                 if (mc.precharge < HPAGE_PMD_NR) {
5183                         spin_unlock(ptl);
5184                         return 0;
5185                 }
5186                 target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
5187                 if (target_type == MC_TARGET_PAGE) {
5188                         page = target.page;
5189                         if (!isolate_lru_page(page)) {
5190                                 if (!mem_cgroup_move_account(page, true,
5191                                                              mc.from, mc.to)) {
5192                                         mc.precharge -= HPAGE_PMD_NR;
5193                                         mc.moved_charge += HPAGE_PMD_NR;
5194                                 }
5195                                 putback_lru_page(page);
5196                         }
5197                         put_page(page);
5198                 } else if (target_type == MC_TARGET_DEVICE) {
5199                         page = target.page;
5200                         if (!mem_cgroup_move_account(page, true,
5201                                                      mc.from, mc.to)) {
5202                                 mc.precharge -= HPAGE_PMD_NR;
5203                                 mc.moved_charge += HPAGE_PMD_NR;
5204                         }
5205                         put_page(page);
5206                 }
5207                 spin_unlock(ptl);
5208                 return 0;
5209         }
5210 
5211         if (pmd_trans_unstable(pmd))
5212                 return 0;
5213 retry:
5214         pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
5215         for (; addr != end; addr += PAGE_SIZE) {
5216                 pte_t ptent = *(pte++);
5217                 bool device = false;
5218                 swp_entry_t ent;
5219 
5220                 if (!mc.precharge)
5221                         break;
5222 
5223                 switch (get_mctgt_type(vma, addr, ptent, &target)) {
5224                 case MC_TARGET_DEVICE:
5225                         device = true;
5226                         /* fall through */
5227                 case MC_TARGET_PAGE:
5228                         page = target.page;
5229                         /*
5230                          * We can have a part of the split pmd here. Moving it
5231                          * can be done but it would be too convoluted so simply
5232                          * ignore such a partial THP and keep it in original
5233                          * memcg. There should be somebody mapping the head.
5234                          */
5235                         if (PageTransCompound(page))
5236                                 goto put;
5237                         if (!device && isolate_lru_page(page))
5238                                 goto put;
5239                         if (!mem_cgroup_move_account(page, false,
5240                                                 mc.from, mc.to)) {
5241                                 mc.precharge--;
5242                                 /* we uncharge from mc.from later. */
5243                                 mc.moved_charge++;
5244                         }
5245                         if (!device)
5246                                 putback_lru_page(page);
5247 put:                    /* get_mctgt_type() gets the page */
5248                         put_page(page);
5249                         break;
5250                 case MC_TARGET_SWAP:
5251                         ent = target.ent;
5252                         if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
5253                                 mc.precharge--;
5254                                 /* we fixup refcnts and charges later. */
5255                                 mc.moved_swap++;
5256                         }
5257                         break;
5258                 default:
5259                         break;
5260                 }
5261         }
5262         pte_unmap_unlock(pte - 1, ptl);
5263         cond_resched();
5264 
5265         if (addr != end) {
5266                 /*
5267                  * We have consumed all precharges we got in can_attach().
5268                  * We try charge one by one, but don't do any additional
5269                  * charges to mc.to if we have failed in charge once in attach()
5270                  * phase.
5271                  */
5272                 ret = mem_cgroup_do_precharge(1);
5273                 if (!ret)
5274                         goto retry;
5275         }
5276 
5277         return ret;
5278 }
5279 
5280 static void mem_cgroup_move_charge(void)
5281 {
5282         struct mm_walk mem_cgroup_move_charge_walk = {
5283                 .pmd_entry = mem_cgroup_move_charge_pte_range,
5284                 .mm = mc.mm,
5285         };
5286 
5287         lru_add_drain_all();
5288         /*
5289          * Signal lock_page_memcg() to take the memcg's move_lock
5290          * while we're moving its pages to another memcg. Then wait
5291          * for already started RCU-only updates to finish.
5292          */
5293         atomic_inc(&mc.from->moving_account);
5294         synchronize_rcu();
5295 retry:
5296         if (unlikely(!down_read_trylock(&mc.mm->mmap_sem))) {
5297                 /*
5298                  * Someone who are holding the mmap_sem might be waiting in
5299                  * waitq. So we cancel all extra charges, wake up all waiters,
5300                  * and retry. Because we cancel precharges, we might not be able
5301                  * to move enough charges, but moving charge is a best-effort
5302                  * feature anyway, so it wouldn't be a big problem.
5303                  */
5304                 __mem_cgroup_clear_mc();
5305                 cond_resched();
5306                 goto retry;
5307         }
5308         /*
5309          * When we have consumed all precharges and failed in doing
5310          * additional charge, the page walk just aborts.
5311          */
5312         walk_page_range(0, mc.mm->highest_vm_end, &mem_cgroup_move_charge_walk);
5313 
5314         up_read(&mc.mm->mmap_sem);
5315         atomic_dec(&mc.from->moving_account);
5316 }
5317 
5318 static void mem_cgroup_move_task(void)
5319 {
5320         if (mc.to) {
5321                 mem_cgroup_move_charge();
5322                 mem_cgroup_clear_mc();
5323         }
5324 }
5325 #else   /* !CONFIG_MMU */
5326 static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
5327 {
5328         return 0;
5329 }
5330 static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
5331 {
5332 }
5333 static void mem_cgroup_move_task(void)
5334 {
5335 }
5336 #endif
5337 
5338 /*
5339  * Cgroup retains root cgroups across [un]mount cycles making it necessary
5340  * to verify whether we're attached to the default hierarchy on each mount
5341  * attempt.
5342  */
5343 static void mem_cgroup_bind(struct cgroup_subsys_state *root_css)
5344 {
5345         /*
5346          * use_hierarchy is forced on the default hierarchy.  cgroup core
5347          * guarantees that @root doesn't have any children, so turning it
5348          * on for the root memcg is enough.
5349          */
5350         if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
5351                 root_mem_cgroup->use_hierarchy = true;
5352         else
5353                 root_mem_cgroup->use_hierarchy = false;
5354 }
5355 
5356 static u64 memory_current_read(struct cgroup_subsys_state *css,
5357                                struct cftype *cft)
5358 {
5359         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5360 
5361         return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE;
5362 }
5363 
5364 static int memory_min_show(struct seq_file *m, void *v)
5365 {
5366         struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5367         unsigned long min = READ_ONCE(memcg->memory.min);
5368 
5369         if (min == PAGE_COUNTER_MAX)
5370                 seq_puts(m, "max\n");
5371         else
5372                 seq_printf(m, "%llu\n", (u64)min * PAGE_SIZE);
5373 
5374         return 0;
5375 }
5376 
5377 static ssize_t memory_min_write(struct kernfs_open_file *of,
5378                                 char *buf, size_t nbytes, loff_t off)
5379 {
5380         struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5381         unsigned long min;
5382         int err;
5383 
5384         buf = strstrip(buf);
5385         err = page_counter_memparse(buf, "max", &min);
5386         if (err)
5387                 return err;
5388 
5389         page_counter_set_min(&memcg->memory, min);
5390 
5391         return nbytes;
5392 }
5393 
5394 static int memory_low_show(struct seq_file *m, void *v)
5395 {
5396         struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5397         unsigned long low = READ_ONCE(memcg->memory.low);
5398 
5399         if (low == PAGE_COUNTER_MAX)
5400                 seq_puts(m, "max\n");
5401         else
5402                 seq_printf(m, "%llu\n", (u64)low * PAGE_SIZE);
5403 
5404         return 0;
5405 }
5406 
5407 static ssize_t memory_low_write(struct kernfs_open_file *of,
5408                                 char *buf, size_t nbytes, loff_t off)
5409 {
5410         struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5411         unsigned long low;
5412         int err;
5413 
5414         buf = strstrip(buf);
5415         err = page_counter_memparse(buf, "max", &low);
5416         if (err)
5417                 return err;
5418 
5419         page_counter_set_low(&memcg->memory, low);
5420 
5421         return nbytes;
5422 }
5423 
5424 static int memory_high_show(struct seq_file *m, void *v)
5425 {
5426         struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5427         unsigned long high = READ_ONCE(memcg->high);
5428 
5429         if (high == PAGE_COUNTER_MAX)
5430                 seq_puts(m, "max\n");
5431         else
5432                 seq_printf(m, "%llu\n", (u64)high * PAGE_SIZE);
5433 
5434         return 0;
5435 }
5436 
5437 static ssize_t memory_high_write(struct kernfs_open_file *of,
5438                                  char *buf, size_t nbytes, loff_t off)
5439 {
5440         struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5441         unsigned long nr_pages;
5442         unsigned long high;
5443         int err;
5444 
5445         buf = strstrip(buf);
5446         err = page_counter_memparse(buf, "max", &high);
5447         if (err)
5448                 return err;
5449 
5450         memcg->high = high;
5451 
5452         nr_pages = page_counter_read(&memcg->memory);
5453         if (nr_pages > high)
5454                 try_to_free_mem_cgroup_pages(memcg, nr_pages - high,
5455                                              GFP_KERNEL, true);
5456 
5457         memcg_wb_domain_size_changed(memcg);
5458         return nbytes;
5459 }
5460 
5461 static int memory_max_show(struct seq_file *m, void *v)
5462 {
5463         struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5464         unsigned long max = READ_ONCE(memcg->memory.max);
5465 
5466         if (max == PAGE_COUNTER_MAX)
5467                 seq_puts(m, "max\n");
5468         else
5469                 seq_printf(m, "%llu\n", (u64)max * PAGE_SIZE);
5470 
5471         return 0;
5472 }
5473 
5474 static ssize_t memory_max_write(struct kernfs_open_file *of,
5475                                 char *buf, size_t nbytes, loff_t off)
5476 {
5477         struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5478         unsigned int nr_reclaims = MEM_CGROUP_RECLAIM_RETRIES;
5479         bool drained = false;
5480         unsigned long max;
5481         int err;
5482 
5483         buf = strstrip(buf);
5484         err = page_counter_memparse(buf, "max", &max);
5485         if (err)
5486                 return err;
5487 
5488         xchg(&memcg->memory.max, max);
5489 
5490         for (;;) {
5491                 unsigned long nr_pages = page_counter_read(&memcg->memory);
5492 
5493                 if (nr_pages <= max)
5494                         break;
5495 
5496                 if (signal_pending(current)) {
5497                         err = -EINTR;
5498                         break;
5499                 }
5500 
5501                 if (!drained) {
5502                         drain_all_stock(memcg);
5503                         drained = true;
5504                         continue;
5505                 }
5506 
5507                 if (nr_reclaims) {
5508                         if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max,
5509                                                           GFP_KERNEL, true))
5510                                 nr_reclaims--;
5511                         continue;
5512                 }
5513 
5514                 memcg_memory_event(memcg, MEMCG_OOM);
5515                 if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0))
5516                         break;
5517         }
5518 
5519         memcg_wb_domain_size_changed(memcg);
5520         return nbytes;
5521 }
5522 
5523 static int memory_events_show(struct seq_file *m, void *v)
5524 {
5525         struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5526 
5527         seq_printf(m, "low %lu\n",
5528                    atomic_long_read(&memcg->memory_events[MEMCG_LOW]));
5529         seq_printf(m, "high %lu\n",
5530                    atomic_long_read(&memcg->memory_events[MEMCG_HIGH]));
5531         seq_printf(m, "max %lu\n",
5532                    atomic_long_read(&memcg->memory_events[MEMCG_MAX]));
5533         seq_printf(m, "oom %lu\n",
5534                    atomic_long_read(&memcg->memory_events[MEMCG_OOM]));
5535         seq_printf(m, "oom_kill %lu\n",
5536                    atomic_long_read(&memcg->memory_events[MEMCG_OOM_KILL]));
5537 
5538         return 0;
5539 }
5540 
5541 static int memory_stat_show(struct seq_file *m, void *v)
5542 {
5543         struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5544         struct accumulated_stats acc;
5545         int i;
5546 
5547         /*
5548          * Provide statistics on the state of the memory subsystem as
5549          * well as cumulative event counters that show past behavior.
5550          *
5551          * This list is ordered following a combination of these gradients:
5552          * 1) generic big picture -> specifics and details
5553          * 2) reflecting userspace activity -> reflecting kernel heuristics
5554          *
5555          * Current memory state:
5556          */
5557 
5558         memset(&acc, 0, sizeof(acc));
5559         acc.stats_size = MEMCG_NR_STAT;
5560         acc.events_size = NR_VM_EVENT_ITEMS;
5561         accumulate_memcg_tree(memcg, &acc);
5562 
5563         seq_printf(m, "anon %llu\n",
5564                    (u64)acc.stat[MEMCG_RSS] * PAGE_SIZE);
5565         seq_printf(m, "file %llu\n",
5566                    (u64)acc.stat[MEMCG_CACHE] * PAGE_SIZE);
5567         seq_printf(m, "kernel_stack %llu\n",
5568                    (u64)acc.stat[MEMCG_KERNEL_STACK_KB] * 1024);
5569         seq_printf(m, "slab %llu\n",
5570                    (u64)(acc.stat[NR_SLAB_RECLAIMABLE] +
5571                          acc.stat[NR_SLAB_UNRECLAIMABLE]) * PAGE_SIZE);
5572         seq_printf(m, "sock %llu\n",
5573                    (u64)acc.stat[MEMCG_SOCK] * PAGE_SIZE);
5574 
5575         seq_printf(m, "shmem %llu\n",
5576                    (u64)acc.stat[NR_SHMEM] * PAGE_SIZE);
5577         seq_printf(m, "file_mapped %llu\n",
5578                    (u64)acc.stat[NR_FILE_MAPPED] * PAGE_SIZE);
5579         seq_printf(m, "file_dirty %llu\n",
5580                    (u64)acc.stat[NR_FILE_DIRTY] * PAGE_SIZE);
5581         seq_printf(m, "file_writeback %llu\n",
5582                    (u64)acc.stat[NR_WRITEBACK] * PAGE_SIZE);
5583 
5584         for (i = 0; i < NR_LRU_LISTS; i++)
5585                 seq_printf(m, "%s %llu\n", mem_cgroup_lru_names[i],
5586                            (u64)acc.lru_pages[i] * PAGE_SIZE);
5587 
5588         seq_printf(m, "slab_reclaimable %llu\n",
5589                    (u64)acc.stat[NR_SLAB_RECLAIMABLE] * PAGE_SIZE);
5590         seq_printf(m, "slab_unreclaimable %llu\n",
5591                    (u64)acc.stat[NR_SLAB_UNRECLAIMABLE] * PAGE_SIZE);
5592 
5593         /* Accumulated memory events */
5594 
5595         seq_printf(m, "pgfault %lu\n", acc.events[PGFAULT]);
5596         seq_printf(m, "pgmajfault %lu\n", acc.events[PGMAJFAULT]);
5597 
5598         seq_printf(m, "pgrefill %lu\n", acc.events[PGREFILL]);
5599         seq_printf(m, "pgscan %lu\n", acc.events[PGSCAN_KSWAPD] +
5600                    acc.events[PGSCAN_DIRECT]);
5601         seq_printf(m, "pgsteal %lu\n", acc.events[PGSTEAL_KSWAPD] +
5602                    acc.events[PGSTEAL_DIRECT]);
5603         seq_printf(m, "pgactivate %lu\n", acc.events[PGACTIVATE]);
5604         seq_printf(m, "pgdeactivate %lu\n", acc.events[PGDEACTIVATE]);
5605         seq_printf(m, "pglazyfree %lu\n", acc.events[PGLAZYFREE]);
5606         seq_printf(m, "pglazyfreed %lu\n", acc.events[PGLAZYFREED]);
5607 
5608         seq_printf(m, "workingset_refault %lu\n",
5609                    acc.stat[WORKINGSET_REFAULT]);
5610         seq_printf(m, "workingset_activate %lu\n",
5611                    acc.stat[WORKINGSET_ACTIVATE]);
5612         seq_printf(m, "workingset_nodereclaim %lu\n",
5613                    acc.stat[WORKINGSET_NODERECLAIM]);
5614 
5615         return 0;
5616 }
5617 
5618 static int memory_oom_group_show(struct seq_file *m, void *v)
5619 {
5620         struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5621 
5622         seq_printf(m, "%d\n", memcg->oom_group);
5623 
5624         return 0;
5625 }
5626 
5627 static ssize_t memory_oom_group_write(struct kernfs_open_file *of,
5628                                       char *buf, size_t nbytes, loff_t off)
5629 {
5630         struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5631         int ret, oom_group;
5632 
5633         buf = strstrip(buf);
5634         if (!buf)
5635                 return -EINVAL;
5636 
5637         ret = kstrtoint(buf, 0, &oom_group);
5638         if (ret)
5639                 return ret;
5640 
5641         if (oom_group != 0 && oom_group != 1)
5642                 return -EINVAL;
5643 
5644         memcg->oom_group = oom_group;
5645 
5646         return nbytes;
5647 }
5648 
5649 static struct cftype memory_files[] = {
5650         {
5651                 .name = "current",
5652                 .flags = CFTYPE_NOT_ON_ROOT,
5653                 .read_u64 = memory_current_read,
5654         },
5655         {
5656                 .name = "min",
5657                 .flags = CFTYPE_NOT_ON_ROOT,
5658                 .seq_show = memory_min_show,
5659                 .write = memory_min_write,
5660         },
5661         {
5662                 .name = "low",
5663                 .flags = CFTYPE_NOT_ON_ROOT,
5664                 .seq_show = memory_low_show,
5665                 .write = memory_low_write,
5666         },
5667         {
5668                 .name = "high",
5669                 .flags = CFTYPE_NOT_ON_ROOT,
5670                 .seq_show = memory_high_show,
5671                 .write = memory_high_write,
5672         },
5673         {
5674                 .name = "max",
5675                 .flags = CFTYPE_NOT_ON_ROOT,
5676                 .seq_show = memory_max_show,
5677                 .write = memory_max_write,
5678         },
5679         {
5680                 .name = "events",
5681                 .flags = CFTYPE_NOT_ON_ROOT,
5682                 .file_offset = offsetof(struct mem_cgroup, events_file),
5683                 .seq_show = memory_events_show,
5684         },
5685         {
5686                 .name = "stat",
5687                 .flags = CFTYPE_NOT_ON_ROOT,
5688                 .seq_show = memory_stat_show,
5689         },
5690         {
5691                 .name = "oom.group",
5692                 .flags = CFTYPE_NOT_ON_ROOT | CFTYPE_NS_DELEGATABLE,
5693                 .seq_show = memory_oom_group_show,
5694                 .write = memory_oom_group_write,
5695         },
5696         { }     /* terminate */
5697 };
5698 
5699 struct cgroup_subsys memory_cgrp_subsys = {
5700         .css_alloc = mem_cgroup_css_alloc,
5701         .css_online = mem_cgroup_css_online,
5702         .css_offline = mem_cgroup_css_offline,
5703         .css_released = mem_cgroup_css_released,
5704         .css_free = mem_cgroup_css_free,
5705         .css_reset = mem_cgroup_css_reset,
5706         .can_attach = mem_cgroup_can_attach,
5707         .cancel_attach = mem_cgroup_cancel_attach,
5708         .post_attach = mem_cgroup_move_task,
5709         .bind = mem_cgroup_bind,
5710         .dfl_cftypes = memory_files,
5711         .legacy_cftypes = mem_cgroup_legacy_files,
5712         .early_init = 0,
5713 };
5714 
5715 /**
5716  * mem_cgroup_protected - check if memory consumption is in the normal range
5717  * @root: the top ancestor of the sub-tree being checked
5718  * @memcg: the memory cgroup to check
5719  *
5720  * WARNING: This function is not stateless! It can only be used as part
5721  *          of a top-down tree iteration, not for isolated queries.
5722  *
5723  * Returns one of the following:
5724  *   MEMCG_PROT_NONE: cgroup memory is not protected
5725  *   MEMCG_PROT_LOW: cgroup memory is protected as long there is
5726  *     an unprotected supply of reclaimable memory from other cgroups.
5727  *   MEMCG_PROT_MIN: cgroup memory is protected
5728  *
5729  * @root is exclusive; it is never protected when looked at directly
5730  *
5731  * To provide a proper hierarchical behavior, effective memory.min/low values
5732  * are used. Below is the description of how effective memory.low is calculated.
5733  * Effective memory.min values is calculated in the same way.
5734  *
5735  * Effective memory.low is always equal or less than the original memory.low.
5736  * If there is no memory.low overcommittment (which is always true for
5737  * top-level memory cgroups), these two values are equal.
5738  * Otherwise, it's a part of parent's effective memory.low,
5739  * calculated as a cgroup's memory.low usage divided by sum of sibling's
5740  * memory.low usages, where memory.low usage is the size of actually
5741  * protected memory.
5742  *
5743  *                                             low_usage
5744  * elow = min( memory.low, parent->elow * ------------------ ),
5745  *                                        siblings_low_usage
5746  *
5747  *             | memory.current, if memory.current < memory.low
5748  * low_usage = |
5749                | 0, otherwise.
5750  *
5751  *
5752  * Such definition of the effective memory.low provides the expected
5753  * hierarchical behavior: parent's memory.low value is limiting
5754  * children, unprotected memory is reclaimed first and cgroups,
5755  * which are not using their guarantee do not affect actual memory
5756  * distribution.
5757  *
5758  * For example, if there are memcgs A, A/B, A/C, A/D and A/E:
5759  *
5760  *     A      A/memory.low = 2G, A/memory.current = 6G
5761  *    //\\
5762  *   BC  DE   B/memory.low = 3G  B/memory.current = 2G
5763  *            C/memory.low = 1G  C/memory.current = 2G
5764  *            D/memory.low = 0   D/memory.current = 2G
5765  *            E/memory.low = 10G E/memory.current = 0
5766  *
5767  * and the memory pressure is applied, the following memory distribution
5768  * is expected (approximately):
5769  *
5770  *     A/memory.current = 2G
5771  *
5772  *     B/memory.current = 1.3G
5773  *     C/memory.current = 0.6G
5774  *     D/memory.current = 0
5775  *     E/memory.current = 0
5776  *
5777  * These calculations require constant tracking of the actual low usages
5778  * (see propagate_protected_usage()), as well as recursive calculation of
5779  * effective memory.low values. But as we do call mem_cgroup_protected()
5780  * path for each memory cgroup top-down from the reclaim,
5781  * it's possible to optimize this part, and save calculated elow
5782  * for next usage. This part is intentionally racy, but it's ok,
5783  * as memory.low is a best-effort mechanism.
5784  */
5785 enum mem_cgroup_protection mem_cgroup_protected(struct mem_cgroup *root,
5786                                                 struct mem_cgroup *memcg)
5787 {
5788         struct mem_cgroup *parent;
5789         unsigned long emin, parent_emin;
5790         unsigned long elow, parent_elow;
5791         unsigned long usage;
5792 
5793         if (mem_cgroup_disabled())
5794                 return MEMCG_PROT_NONE;
5795 
5796         if (!root)
5797                 root = root_mem_cgroup;
5798         if (memcg == root)
5799                 return MEMCG_PROT_NONE;
5800 
5801         usage = page_counter_read(&memcg->memory);
5802         if (!usage)
5803                 return MEMCG_PROT_NONE;
5804 
5805         emin = memcg->memory.min;
5806         elow = memcg->memory.low;
5807 
5808         parent = parent_mem_cgroup(memcg);
5809         /* No parent means a non-hierarchical mode on v1 memcg */
5810         if (!parent)
5811                 return MEMCG_PROT_NONE;
5812 
5813         if (parent == root)
5814                 goto exit;
5815 
5816         parent_emin = READ_ONCE(parent->memory.emin);
5817         emin = min(emin, parent_emin);
5818         if (emin && parent_emin) {
5819                 unsigned long min_usage, siblings_min_usage;
5820 
5821                 min_usage = min(usage, memcg->memory.min);
5822                 siblings_min_usage = atomic_long_read(
5823                         &parent->memory.children_min_usage);
5824 
5825                 if (min_usage && siblings_min_usage)
5826                         emin = min(emin, parent_emin * min_usage /
5827                                    siblings_min_usage);
5828         }
5829 
5830         parent_elow = READ_ONCE(parent->memory.elow);
5831         elow = min(elow, parent_elow);
5832         if (elow && parent_elow) {
5833                 unsigned long low_usage, siblings_low_usage;
5834 
5835                 low_usage = min(usage, memcg->memory.low);
5836                 siblings_low_usage = atomic_long_read(
5837                         &parent->memory.children_low_usage);
5838 
5839                 if (low_usage && siblings_low_usage)
5840                         elow = min(elow, parent_elow * low_usage /
5841                                    siblings_low_usage);
5842         }
5843 
5844 exit:
5845         memcg->memory.emin = emin;
5846         memcg->memory.elow = elow;
5847 
5848         if (usage <= emin)
5849                 return MEMCG_PROT_MIN;
5850         else if (usage <= elow)
5851                 return MEMCG_PROT_LOW;
5852         else
5853                 return MEMCG_PROT_NONE;
5854 }
5855 
5856 /**
5857  * mem_cgroup_try_charge - try charging a page
5858  * @page: page to charge
5859  * @mm: mm context of the victim
5860  * @gfp_mask: reclaim mode
5861  * @memcgp: charged memcg return
5862  * @compound: charge the page as compound or small page
5863  *
5864  * Try to charge @page to the memcg that @mm belongs to, reclaiming
5865  * pages according to @gfp_mask if necessary.
5866  *
5867  * Returns 0 on success, with *@memcgp pointing to the charged memcg.
5868  * Otherwise, an error code is returned.
5869  *
5870  * After page->mapping has been set up, the caller must finalize the
5871  * charge with mem_cgroup_commit_charge().  Or abort the transaction
5872  * with mem_cgroup_cancel_charge() in case page instantiation fails.
5873  */
5874 int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
5875                           gfp_t gfp_mask, struct mem_cgroup **memcgp,
5876                           bool compound)
5877 {
5878         struct mem_cgroup *memcg = NULL;
5879         unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
5880         int ret = 0;
5881 
5882         if (mem_cgroup_disabled())
5883                 goto out;
5884 
5885         if (PageSwapCache(page)) {
5886                 /*
5887                  * Every swap fault against a single page tries to charge the
5888                  * page, bail as early as possible.  shmem_unuse() encounters
5889                  * already charged pages, too.  The USED bit is protected by
5890                  * the page lock, which serializes swap cache removal, which
5891                  * in turn serializes uncharging.
5892                  */
5893                 VM_BUG_ON_PAGE(!PageLocked(page), page);
5894                 if (compound_head(page)->mem_cgroup)
5895                         goto out;
5896 
5897                 if (do_swap_account) {
5898                         swp_entry_t ent = { .val = page_private(page), };
5899                         unsigned short id = lookup_swap_cgroup_id(ent);
5900 
5901                         rcu_read_lock();
5902                         memcg = mem_cgroup_from_id(id);
5903                         if (memcg && !css_tryget_online(&memcg->css))
5904                                 memcg = NULL;
5905                         rcu_read_unlock();
5906                 }
5907         }
5908 
5909         if (!memcg)
5910                 memcg = get_mem_cgroup_from_mm(mm);
5911 
5912         ret = try_charge(memcg, gfp_mask, nr_pages);
5913 
5914         css_put(&memcg->css);
5915 out:
5916         *memcgp = memcg;
5917         return ret;
5918 }
5919 
5920 int mem_cgroup_try_charge_delay(struct page *page, struct mm_struct *mm,
5921                           gfp_t gfp_mask, struct mem_cgroup **memcgp,
5922                           bool compound)
5923 {
5924         struct mem_cgroup *memcg;
5925         int ret;
5926 
5927         ret = mem_cgroup_try_charge(page, mm, gfp_mask, memcgp, compound);
5928         memcg = *memcgp;
5929         mem_cgroup_throttle_swaprate(memcg, page_to_nid(page), gfp_mask);
5930         return ret;
5931 }
5932 
5933 /**
5934  * mem_cgroup_commit_charge - commit a page charge
5935  * @page: page to charge
5936  * @memcg: memcg to charge the page to
5937  * @lrucare: page might be on LRU already
5938  * @compound: charge the page as compound or small page
5939  *
5940  * Finalize a charge transaction started by mem_cgroup_try_charge(),
5941  * after page->mapping has been set up.  This must happen atomically
5942  * as part of the page instantiation, i.e. under the page table lock
5943  * for anonymous pages, under the page lock for page and swap cache.
5944  *
5945  * In addition, the page must not be on the LRU during the commit, to
5946  * prevent racing with task migration.  If it might be, use @lrucare.
5947  *
5948  * Use mem_cgroup_cancel_charge() to cancel the transaction instead.
5949  */
5950 void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
5951                               bool lrucare, bool compound)
5952 {
5953         unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
5954 
5955         VM_BUG_ON_PAGE(!page->mapping, page);
5956         VM_BUG_ON_PAGE(PageLRU(page) && !lrucare, page);
5957 
5958         if (mem_cgroup_disabled())
5959                 return;
5960         /*
5961          * Swap faults will attempt to charge the same page multiple
5962          * times.  But reuse_swap_page() might have removed the page
5963          * from swapcache already, so we can't check PageSwapCache().
5964          */
5965         if (!memcg)
5966                 return;
5967 
5968         commit_charge(page, memcg, lrucare);
5969 
5970         local_irq_disable();
5971         mem_cgroup_charge_statistics(memcg, page, compound, nr_pages);
5972         memcg_check_events(memcg, page);
5973         local_irq_enable();
5974 
5975         if (do_memsw_account() && PageSwapCache(page)) {
5976                 swp_entry_t entry = { .val = page_private(page) };
5977                 /*
5978                  * The swap entry might not get freed for a long time,
5979                  * let's not wait for it.  The page already received a
5980                  * memory+swap charge, drop the swap entry duplicate.
5981                  */
5982                 mem_cgroup_uncharge_swap(entry, nr_pages);
5983         }
5984 }
5985 
5986 /**
5987  * mem_cgroup_cancel_charge - cancel a page charge
5988  * @page: page to charge
5989  * @memcg: memcg to charge the page to
5990  * @compound: charge the page as compound or small page
5991  *
5992  * Cancel a charge transaction started by mem_cgroup_try_charge().
5993  */
5994 void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg,
5995                 bool compound)
5996 {
5997         unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
5998 
5999         if (mem_cgroup_disabled())
6000                 return;
6001         /*
6002          * Swap faults will attempt to charge the same page multiple
6003          * times.  But reuse_swap_page() might have removed the page
6004          * from swapcache already, so we can't check PageSwapCache().
6005          */
6006         if (!memcg)
6007                 return;
6008 
6009         cancel_charge(memcg, nr_pages);
6010 }
6011 
6012 struct uncharge_gather {
6013         struct mem_cgroup *memcg;
6014         unsigned long pgpgout;
6015         unsigned long nr_anon;
6016         unsigned long nr_file;
6017         unsigned long nr_kmem;
6018         unsigned long nr_huge;
6019         unsigned long nr_shmem;
6020         struct page *dummy_page;
6021 };
6022 
6023 static inline void uncharge_gather_clear(struct uncharge_gather *ug)
6024 {
6025         memset(ug, 0, sizeof(*ug));
6026 }
6027 
6028 static void uncharge_batch(const struct uncharge_gather *ug)
6029 {
6030         unsigned long nr_pages = ug->nr_anon + ug->nr_file + ug->nr_kmem;
6031         unsigned long flags;
6032 
6033         if (!mem_cgroup_is_root(ug->memcg)) {
6034                 page_counter_uncharge(&ug->memcg->memory, nr_pages);
6035                 if (do_memsw_account())
6036                         page_counter_uncharge(&ug->memcg->memsw, nr_pages);
6037                 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && ug->nr_kmem)
6038                         page_counter_uncharge(&ug->memcg->kmem, ug->nr_kmem);
6039                 memcg_oom_recover(ug->memcg);
6040         }
6041 
6042         local_irq_save(flags);
6043         __mod_memcg_state(ug->memcg, MEMCG_RSS, -ug->nr_anon);
6044         __mod_memcg_state(ug->memcg, MEMCG_CACHE, -ug->nr_file);
6045         __mod_memcg_state(ug->memcg, MEMCG_RSS_HUGE, -ug->nr_huge);
6046         __mod_memcg_state(ug->memcg, NR_SHMEM, -ug->nr_shmem);
6047         __count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout);
6048         __this_cpu_add(ug->memcg->stat_cpu->nr_page_events, nr_pages);
6049         memcg_check_events(ug->memcg, ug->dummy_page);
6050         local_irq_restore(flags);
6051 
6052         if (!mem_cgroup_is_root(ug->memcg))
6053                 css_put_many(&ug->memcg->css, nr_pages);
6054 }
6055 
6056 static void uncharge_page(struct page *page, struct uncharge_gather *ug)
6057 {
6058         VM_BUG_ON_PAGE(PageLRU(page), page);
6059         VM_BUG_ON_PAGE(page_count(page) && !is_zone_device_page(page) &&
6060                         !PageHWPoison(page) , page);
6061 
6062         if (!page->mem_cgroup)
6063                 return;
6064 
6065         /*
6066          * Nobody should be changing or seriously looking at
6067          * page->mem_cgroup at this point, we have fully
6068          * exclusive access to the page.
6069          */
6070 
6071         if (ug->memcg != page->mem_cgroup) {
6072                 if (ug->memcg) {
6073                         uncharge_batch(ug);
6074                         uncharge_gather_clear(ug);
6075                 }
6076                 ug->memcg = page->mem_cgroup;
6077         }
6078 
6079         if (!PageKmemcg(page)) {
6080                 unsigned int nr_pages = 1;
6081 
6082                 if (PageTransHuge(page)) {
6083                         nr_pages <<= compound_order(page);
6084                         ug->nr_huge += nr_pages;
6085                 }
6086                 if (PageAnon(page))
6087                         ug->nr_anon += nr_pages;
6088                 else {
6089                         ug->nr_file += nr_pages;
6090                         if (PageSwapBacked(page))
6091                                 ug->nr_shmem += nr_pages;
6092                 }
6093                 ug->pgpgout++;
6094         } else {
6095                 ug->nr_kmem += 1 << compound_order(page);
6096                 __ClearPageKmemcg(page);
6097         }
6098 
6099         ug->dummy_page = page;
6100         page->mem_cgroup = NULL;
6101 }
6102 
6103 static void uncharge_list(struct list_head *page_list)
6104 {
6105         struct uncharge_gather ug;
6106         struct list_head *next;
6107 
6108         uncharge_gather_clear(&ug);
6109 
6110         /*
6111          * Note that the list can be a single page->lru; hence the
6112          * do-while loop instead of a simple list_for_each_entry().
6113          */
6114         next = page_list->next;
6115         do {
6116                 struct page *page;
6117 
6118                 page = list_entry(next, struct page, lru);
6119                 next = page->lru.next;
6120 
6121                 uncharge_page(page, &ug);
6122         } while (next != page_list);
6123 
6124         if (ug.memcg)
6125                 uncharge_batch(&ug);
6126 }
6127 
6128 /**
6129  * mem_cgroup_uncharge - uncharge a page
6130  * @page: page to uncharge
6131  *
6132  * Uncharge a page previously charged with mem_cgroup_try_charge() and
6133  * mem_cgroup_commit_charge().
6134  */
6135 void mem_cgroup_uncharge(struct page *page)
6136 {
6137         struct uncharge_gather ug;
6138 
6139         if (mem_cgroup_disabled())
6140                 return;
6141 
6142         /* Don't touch page->lru of any random page, pre-check: */
6143         if (!page->mem_cgroup)
6144                 return;
6145 
6146         uncharge_gather_clear(&ug);
6147         uncharge_page(page, &ug);
6148         uncharge_batch(&ug);
6149 }
6150 
6151 /**
6152  * mem_cgroup_uncharge_list - uncharge a list of page
6153  * @page_list: list of pages to uncharge
6154  *
6155  * Uncharge a list of pages previously charged with
6156  * mem_cgroup_try_charge() and mem_cgroup_commit_charge().
6157  */
6158 void mem_cgroup_uncharge_list(struct list_head *page_list)
6159 {
6160         if (mem_cgroup_disabled())
6161                 return;
6162 
6163         if (!list_empty(page_list))
6164                 uncharge_list(page_list);
6165 }
6166 
6167 /**
6168  * mem_cgroup_migrate - charge a page's replacement
6169  * @oldpage: currently circulating page
6170  * @newpage: replacement page
6171  *
6172  * Charge @newpage as a replacement page for @oldpage. @oldpage will
6173  * be uncharged upon free.
6174  *
6175  * Both pages must be locked, @newpage->mapping must be set up.
6176  */
6177 void mem_cgroup_migrate(struct page *oldpage, struct page *newpage)
6178 {
6179         struct mem_cgroup *memcg;
6180         unsigned int nr_pages;
6181         bool compound;
6182         unsigned long flags;
6183 
6184         VM_BUG_ON_PAGE(!PageLocked(oldpage), oldpage);
6185         VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
6186         VM_BUG_ON_PAGE(PageAnon(oldpage) != PageAnon(newpage), newpage);
6187         VM_BUG_ON_PAGE(PageTransHuge(oldpage) != PageTransHuge(newpage),
6188                        newpage);
6189 
6190         if (mem_cgroup_disabled())
6191                 return;
6192 
6193         /* Page cache replacement: new page already charged? */
6194         if (newpage->mem_cgroup)
6195                 return;
6196 
6197         /* Swapcache readahead pages can get replaced before being charged */
6198         memcg = oldpage->mem_cgroup;
6199         if (!memcg)
6200                 return;
6201 
6202         /* Force-charge the new page. The old one will be freed soon */
6203         compound = PageTransHuge(newpage);
6204         nr_pages = compound ? hpage_nr_pages(newpage) : 1;
6205 
6206         page_counter_charge(&memcg->memory, nr_pages);
6207         if (do_memsw_account())
6208                 page_counter_charge(&memcg->memsw, nr_pages);
6209         css_get_many(&memcg->css, nr_pages);
6210 
6211         commit_charge(newpage, memcg, false);
6212 
6213         local_irq_save(flags);
6214         mem_cgroup_charge_statistics(memcg, newpage, compound, nr_pages);
6215         memcg_check_events(memcg, newpage);
6216         local_irq_restore(flags);
6217 }
6218 
6219 DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key);
6220 EXPORT_SYMBOL(memcg_sockets_enabled_key);
6221 
6222 void mem_cgroup_sk_alloc(struct sock *sk)
6223 {
6224         struct mem_cgroup *memcg;
6225 
6226         if (!mem_cgroup_sockets_enabled)
6227                 return;
6228 
6229         /*
6230          * Socket cloning can throw us here with sk_memcg already
6231          * filled. It won't however, necessarily happen from
6232          * process context. So the test for root memcg given
6233          * the current task's memcg won't help us in this case.
6234          *
6235          * Respecting the original socket's memcg is a better
6236          * decision in this case.
6237          */
6238         if (sk->sk_memcg) {
6239                 css_get(&sk->sk_memcg->css);
6240                 return;
6241         }
6242 
6243         rcu_read_lock();
6244         memcg = mem_cgroup_from_task(current);
6245         if (memcg == root_mem_cgroup)
6246                 goto out;
6247         if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active)
6248                 goto out;
6249         if (css_tryget_online(&memcg->css))
6250                 sk->sk_memcg = memcg;
6251 out:
6252         rcu_read_unlock();
6253 }
6254 
6255 void mem_cgroup_sk_free(struct sock *sk)
6256 {
6257         if (sk->sk_memcg)
6258                 css_put(&sk->sk_memcg->css);
6259 }
6260 
6261 /**
6262  * mem_cgroup_charge_skmem - charge socket memory
6263  * @memcg: memcg to charge
6264  * @nr_pages: number of pages to charge
6265  *
6266  * Charges @nr_pages to @memcg. Returns %true if the charge fit within
6267  * @memcg's configured limit, %false if the charge had to be forced.
6268  */
6269 bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
6270 {
6271         gfp_t gfp_mask = GFP_KERNEL;
6272 
6273         if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
6274                 struct page_counter *fail;
6275 
6276                 if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) {
6277                         memcg->tcpmem_pressure = 0;
6278                         return true;
6279                 }
6280                 page_counter_charge(&memcg->tcpmem, nr_pages);
6281                 memcg->tcpmem_pressure = 1;
6282                 return false;
6283         }
6284 
6285         /* Don't block in the packet receive path */
6286         if (in_softirq())
6287                 gfp_mask = GFP_NOWAIT;
6288 
6289         mod_memcg_state(memcg, MEMCG_SOCK, nr_pages);
6290 
6291         if (try_charge(memcg, gfp_mask, nr_pages) == 0)
6292                 return true;
6293 
6294         try_charge(memcg, gfp_mask|__GFP_NOFAIL, nr_pages);
6295         return false;
6296 }
6297 
6298 /**
6299  * mem_cgroup_uncharge_skmem - uncharge socket memory
6300  * @memcg: memcg to uncharge
6301  * @nr_pages: number of pages to uncharge
6302  */
6303 void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
6304 {
6305         if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
6306                 page_counter_uncharge(&memcg->tcpmem, nr_pages);
6307                 return;
6308         }
6309 
6310         mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages);
6311 
6312         refill_stock(memcg, nr_pages);
6313 }
6314 
6315 static int __init cgroup_memory(char *s)
6316 {
6317         char *token;
6318 
6319         while ((token = strsep(&s, ",")) != NULL) {
6320                 if (!*token)
6321                         continue;
6322                 if (!strcmp(token, "nosocket"))
6323                         cgroup_memory_nosocket = true;
6324                 if (!strcmp(token, "nokmem"))
6325                         cgroup_memory_nokmem = true;
6326         }
6327         return 0;
6328 }
6329 __setup("cgroup.memory=", cgroup_memory);
6330 
6331 /*
6332  * subsys_initcall() for memory controller.
6333  *
6334  * Some parts like memcg_hotplug_cpu_dead() have to be initialized from this
6335  * context because of lock dependencies (cgroup_lock -> cpu hotplug) but
6336  * basically everything that doesn't depend on a specific mem_cgroup structure
6337  * should be initialized from here.
6338  */
6339 static int __init mem_cgroup_init(void)
6340 {
6341         int cpu, node;
6342 
6343 #ifdef CONFIG_MEMCG_KMEM
6344         /*
6345          * Kmem cache creation is mostly done with the slab_mutex held,
6346          * so use a workqueue with limited concurrency to avoid stalling
6347          * all worker threads in case lots of cgroups are created and
6348          * destroyed simultaneously.
6349          */
6350         memcg_kmem_cache_wq = alloc_workqueue("memcg_kmem_cache", 0, 1);
6351         BUG_ON(!memcg_kmem_cache_wq);
6352 #endif
6353 
6354         cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL,
6355                                   memcg_hotplug_cpu_dead);
6356 
6357         for_each_possible_cpu(cpu)
6358                 INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work,
6359                           drain_local_stock);
6360 
6361         for_each_node(node) {
6362                 struct mem_cgroup_tree_per_node *rtpn;
6363 
6364                 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL,
6365                                     node_online(node) ? node : NUMA_NO_NODE);
6366 
6367                 rtpn->rb_root = RB_ROOT;
6368                 rtpn->rb_rightmost = NULL;
6369                 spin_lock_init(&rtpn->lock);
6370                 soft_limit_tree.rb_tree_per_node[node] = rtpn;
6371         }
6372 
6373         return 0;
6374 }
6375 subsys_initcall(mem_cgroup_init);
6376 
6377 #ifdef CONFIG_MEMCG_SWAP
6378 static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg)
6379 {
6380         while (!atomic_inc_not_zero(&memcg->id.ref)) {
6381                 /*
6382                  * The root cgroup cannot be destroyed, so it's refcount must
6383                  * always be >= 1.
6384                  */
6385                 if (WARN_ON_ONCE(memcg == root_mem_cgroup)) {
6386                         VM_BUG_ON(1);
6387                         break;
6388                 }
6389                 memcg = parent_mem_cgroup(memcg);
6390                 if (!memcg)
6391                         memcg = root_mem_cgroup;
6392         }
6393         return memcg;
6394 }
6395 
6396 /**
6397  * mem_cgroup_swapout - transfer a memsw charge to swap
6398  * @page: page whose memsw charge to transfer
6399  * @entry: swap entry to move the charge to
6400  *
6401  * Transfer the memsw charge of @page to @entry.
6402  */
6403 void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
6404 {
6405         struct mem_cgroup *memcg, *swap_memcg;
6406         unsigned int nr_entries;
6407         unsigned short oldid;
6408 
6409         VM_BUG_ON_PAGE(PageLRU(page), page);
6410         VM_BUG_ON_PAGE(page_count(page), page);
6411 
6412         if (!do_memsw_account())
6413                 return;
6414 
6415         memcg = page->mem_cgroup;
6416 
6417         /* Readahead page, never charged */
6418         if (!memcg)
6419                 return;
6420 
6421         /*
6422          * In case the memcg owning these pages has been offlined and doesn't
6423          * have an ID allocated to it anymore, charge the closest online
6424          * ancestor for the swap instead and transfer the memory+swap charge.
6425          */
6426         swap_memcg = mem_cgroup_id_get_online(memcg);
6427         nr_entries = hpage_nr_pages(page);
6428         /* Get references for the tail pages, too */
6429         if (nr_entries > 1)
6430                 mem_cgroup_id_get_many(swap_memcg, nr_entries - 1);
6431         oldid = swap_cgroup_record(entry, mem_cgroup_id(swap_memcg),
6432                                    nr_entries);
6433         VM_BUG_ON_PAGE(oldid, page);
6434         mod_memcg_state(swap_memcg, MEMCG_SWAP, nr_entries);
6435 
6436         page->mem_cgroup = NULL;
6437 
6438         if (!mem_cgroup_is_root(memcg))
6439                 page_counter_uncharge(&memcg->memory, nr_entries);
6440 
6441         if (memcg != swap_memcg) {
6442                 if (!mem_cgroup_is_root(swap_memcg))
6443                         page_counter_charge(&swap_memcg->memsw, nr_entries);
6444                 page_counter_uncharge(&memcg->memsw, nr_entries);
6445         }
6446 
6447         /*
6448          * Interrupts should be disabled here because the caller holds the
6449          * i_pages lock which is taken with interrupts-off. It is
6450          * important here to have the interrupts disabled because it is the
6451          * only synchronisation we have for updating the per-CPU variables.
6452          */
6453         VM_BUG_ON(!irqs_disabled());
6454         mem_cgroup_charge_statistics(memcg, page, PageTransHuge(page),
6455                                      -nr_entries);
6456         memcg_check_events(memcg, page);
6457 
6458         if (!mem_cgroup_is_root(memcg))
6459                 css_put_many(&memcg->css, nr_entries);
6460 }
6461 
6462 /**
6463  * mem_cgroup_try_charge_swap - try charging swap space for a page
6464  * @page: page being added to swap
6465  * @entry: swap entry to charge
6466  *
6467  * Try to charge @page's memcg for the swap space at @entry.
6468  *
6469  * Returns 0 on success, -ENOMEM on failure.
6470  */
6471 int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
6472 {
6473         unsigned int nr_pages = hpage_nr_pages(page);
6474         struct page_counter *counter;
6475         struct mem_cgroup *memcg;
6476         unsigned short oldid;
6477 
6478         if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) || !do_swap_account)
6479                 return 0;
6480 
6481         memcg = page->mem_cgroup;
6482 
6483         /* Readahead page, never charged */
6484         if (!memcg)
6485                 return 0;
6486 
6487         if (!entry.val) {
6488                 memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
6489                 return 0;
6490         }
6491 
6492         memcg = mem_cgroup_id_get_online(memcg);
6493 
6494         if (!mem_cgroup_is_root(memcg) &&
6495             !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) {
6496                 memcg_memory_event(memcg, MEMCG_SWAP_MAX);
6497                 memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
6498                 mem_cgroup_id_put(memcg);
6499                 return -ENOMEM;
6500         }
6501 
6502         /* Get references for the tail pages, too */
6503         if (nr_pages > 1)
6504                 mem_cgroup_id_get_many(memcg, nr_pages - 1);
6505         oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg), nr_pages);
6506         VM_BUG_ON_PAGE(oldid, page);
6507         mod_memcg_state(memcg, MEMCG_SWAP, nr_pages);
6508 
6509         return 0;
6510 }
6511 
6512 /**
6513  * mem_cgroup_uncharge_swap - uncharge swap space
6514  * @entry: swap entry to uncharge
6515  * @nr_pages: the amount of swap space to uncharge
6516  */
6517 void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
6518 {
6519         struct mem_cgroup *memcg;
6520         unsigned short id;
6521 
6522         if (!do_swap_account)
6523                 return;
6524 
6525         id = swap_cgroup_record(entry, 0, nr_pages);
6526         rcu_read_lock();
6527         memcg = mem_cgroup_from_id(id);
6528         if (memcg) {
6529                 if (!mem_cgroup_is_root(memcg)) {
6530                         if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
6531                                 page_counter_uncharge(&memcg->swap, nr_pages);
6532                         else
6533                                 page_counter_uncharge(&memcg->memsw, nr_pages);
6534                 }
6535                 mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages);
6536                 mem_cgroup_id_put_many(memcg, nr_pages);
6537         }
6538         rcu_read_unlock();
6539 }
6540 
6541 long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
6542 {
6543         long nr_swap_pages = get_nr_swap_pages();
6544 
6545         if (!do_swap_account || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
6546                 return nr_swap_pages;
6547         for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
6548                 nr_swap_pages = min_t(long, nr_swap_pages,
6549                                       READ_ONCE(memcg->swap.max) -
6550                                       page_counter_read(&memcg->swap));
6551         return nr_swap_pages;
6552 }
6553 
6554 bool mem_cgroup_swap_full(struct page *page)
6555 {
6556         struct mem_cgroup *memcg;
6557 
6558         VM_BUG_ON_PAGE(!PageLocked(page), page);
6559 
6560         if (vm_swap_full())
6561                 return true;
6562         if (!do_swap_account || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
6563                 return false;
6564 
6565         memcg = page->mem_cgroup;
6566         if (!memcg)
6567                 return false;
6568 
6569         for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
6570                 if (page_counter_read(&memcg->swap) * 2 >= memcg->swap.max)
6571                         return true;
6572 
6573         return false;
6574 }
6575 
6576 /* for remember boot option*/
6577 #ifdef CONFIG_MEMCG_SWAP_ENABLED
6578 static int really_do_swap_account __initdata = 1;
6579 #else
6580 static int really_do_swap_account __initdata;
6581 #endif
6582 
6583 static int __init enable_swap_account(char *s)
6584 {
6585         if (!strcmp(s, "1"))
6586                 really_do_swap_account = 1;
6587         else if (!strcmp(s, ""))
6588                 really_do_swap_account = 0;
6589         return 1;
6590 }
6591 __setup("swapaccount=", enable_swap_account);
6592 
6593 static u64 swap_current_read(struct cgroup_subsys_state *css,
6594                              struct cftype *cft)
6595 {
6596         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
6597 
6598         return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE;
6599 }
6600 
6601 static int swap_max_show(struct seq_file *m, void *v)
6602 {
6603         struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
6604         unsigned long max = READ_ONCE(memcg->swap.max);
6605 
6606         if (max == PAGE_COUNTER_MAX)
6607                 seq_puts(m, "max\n");
6608         else
6609                 seq_printf(m, "%llu\n", (u64)max * PAGE_SIZE);
6610 
6611         return 0;
6612 }
6613 
6614 static ssize_t swap_max_write(struct kernfs_open_file *of,
6615                               char *buf, size_t nbytes, loff_t off)
6616 {
6617         struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6618         unsigned long max;
6619         int err;
6620 
6621         buf = strstrip(buf);
6622         err = page_counter_memparse(buf, "max", &max);
6623         if (err)
6624                 return err;
6625 
6626         xchg(&memcg->swap.max, max);
6627 
6628         return nbytes;
6629 }
6630 
6631 static int swap_events_show(struct seq_file *m, void *v)
6632 {
6633         struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
6634 
6635         seq_printf(m, "max %lu\n",
6636                    atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]));
6637         seq_printf(m, "fail %lu\n",
6638                    atomic_long_read(&memcg->memory_events[MEMCG_SWAP_FAIL]));
6639 
6640         return 0;
6641 }
6642 
6643 static struct cftype swap_files[] = {
6644         {
6645                 .name = "swap.current",
6646                 .flags = CFTYPE_NOT_ON_ROOT,
6647                 .read_u64 = swap_current_read,
6648         },
6649         {
6650                 .name = "swap.max",
6651                 .flags = CFTYPE_NOT_ON_ROOT,
6652                 .seq_show = swap_max_show,
6653                 .write = swap_max_write,
6654         },
6655         {
6656                 .name = "swap.events",
6657                 .flags = CFTYPE_NOT_ON_ROOT,
6658                 .file_offset = offsetof(struct mem_cgroup, swap_events_file),
6659                 .seq_show = swap_events_show,
6660         },
6661         { }     /* terminate */
6662 };
6663 
6664 static struct cftype memsw_cgroup_files[] = {
6665         {
6666                 .name = "memsw.usage_in_bytes",
6667                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
6668                 .read_u64 = mem_cgroup_read_u64,
6669         },
6670         {
6671                 .name = "memsw.max_usage_in_bytes",
6672                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
6673                 .write = mem_cgroup_reset,
6674                 .read_u64 = mem_cgroup_read_u64,
6675         },
6676         {
6677                 .name = "memsw.limit_in_bytes",
6678                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
6679                 .write = mem_cgroup_write,
6680                 .read_u64 = mem_cgroup_read_u64,
6681         },
6682         {
6683                 .name = "memsw.failcnt",
6684                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
6685                 .write = mem_cgroup_reset,
6686                 .read_u64 = mem_cgroup_read_u64,
6687         },
6688         { },    /* terminate */
6689 };
6690 
6691 static int __init mem_cgroup_swap_init(void)
6692 {
6693         if (!mem_cgroup_disabled() && really_do_swap_account) {
6694                 do_swap_account = 1;
6695                 WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys,
6696                                                swap_files));
6697                 WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys,
6698                                                   memsw_cgroup_files));
6699         }
6700         return 0;
6701 }
6702 subsys_initcall(mem_cgroup_swap_init);
6703 
6704 #endif /* CONFIG_MEMCG_SWAP */
6705 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp