~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/mm/memcontrol.c

Version: ~ [ linux-4.14 ] ~ [ linux-4.13.12 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.61 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.97 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.46 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.80 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.50 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.95 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.27.62 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /* memcontrol.c - Memory Controller
  2  *
  3  * Copyright IBM Corporation, 2007
  4  * Author Balbir Singh <balbir@linux.vnet.ibm.com>
  5  *
  6  * Copyright 2007 OpenVZ SWsoft Inc
  7  * Author: Pavel Emelianov <xemul@openvz.org>
  8  *
  9  * Memory thresholds
 10  * Copyright (C) 2009 Nokia Corporation
 11  * Author: Kirill A. Shutemov
 12  *
 13  * Kernel Memory Controller
 14  * Copyright (C) 2012 Parallels Inc. and Google Inc.
 15  * Authors: Glauber Costa and Suleiman Souhlal
 16  *
 17  * Native page reclaim
 18  * Charge lifetime sanitation
 19  * Lockless page tracking & accounting
 20  * Unified hierarchy configuration model
 21  * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner
 22  *
 23  * This program is free software; you can redistribute it and/or modify
 24  * it under the terms of the GNU General Public License as published by
 25  * the Free Software Foundation; either version 2 of the License, or
 26  * (at your option) any later version.
 27  *
 28  * This program is distributed in the hope that it will be useful,
 29  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 30  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 31  * GNU General Public License for more details.
 32  */
 33 
 34 #include <linux/page_counter.h>
 35 #include <linux/memcontrol.h>
 36 #include <linux/cgroup.h>
 37 #include <linux/mm.h>
 38 #include <linux/sched/mm.h>
 39 #include <linux/shmem_fs.h>
 40 #include <linux/hugetlb.h>
 41 #include <linux/pagemap.h>
 42 #include <linux/smp.h>
 43 #include <linux/page-flags.h>
 44 #include <linux/backing-dev.h>
 45 #include <linux/bit_spinlock.h>
 46 #include <linux/rcupdate.h>
 47 #include <linux/limits.h>
 48 #include <linux/export.h>
 49 #include <linux/mutex.h>
 50 #include <linux/rbtree.h>
 51 #include <linux/slab.h>
 52 #include <linux/swap.h>
 53 #include <linux/swapops.h>
 54 #include <linux/spinlock.h>
 55 #include <linux/eventfd.h>
 56 #include <linux/poll.h>
 57 #include <linux/sort.h>
 58 #include <linux/fs.h>
 59 #include <linux/seq_file.h>
 60 #include <linux/vmpressure.h>
 61 #include <linux/mm_inline.h>
 62 #include <linux/swap_cgroup.h>
 63 #include <linux/cpu.h>
 64 #include <linux/oom.h>
 65 #include <linux/lockdep.h>
 66 #include <linux/file.h>
 67 #include <linux/tracehook.h>
 68 #include "internal.h"
 69 #include <net/sock.h>
 70 #include <net/ip.h>
 71 #include "slab.h"
 72 
 73 #include <linux/uaccess.h>
 74 
 75 #include <trace/events/vmscan.h>
 76 
 77 struct cgroup_subsys memory_cgrp_subsys __read_mostly;
 78 EXPORT_SYMBOL(memory_cgrp_subsys);
 79 
 80 struct mem_cgroup *root_mem_cgroup __read_mostly;
 81 
 82 #define MEM_CGROUP_RECLAIM_RETRIES      5
 83 
 84 /* Socket memory accounting disabled? */
 85 static bool cgroup_memory_nosocket;
 86 
 87 /* Kernel memory accounting disabled? */
 88 static bool cgroup_memory_nokmem;
 89 
 90 /* Whether the swap controller is active */
 91 #ifdef CONFIG_MEMCG_SWAP
 92 int do_swap_account __read_mostly;
 93 #else
 94 #define do_swap_account         0
 95 #endif
 96 
 97 /* Whether legacy memory+swap accounting is active */
 98 static bool do_memsw_account(void)
 99 {
100         return !cgroup_subsys_on_dfl(memory_cgrp_subsys) && do_swap_account;
101 }
102 
103 static const char *const mem_cgroup_lru_names[] = {
104         "inactive_anon",
105         "active_anon",
106         "inactive_file",
107         "active_file",
108         "unevictable",
109 };
110 
111 #define THRESHOLDS_EVENTS_TARGET 128
112 #define SOFTLIMIT_EVENTS_TARGET 1024
113 #define NUMAINFO_EVENTS_TARGET  1024
114 
115 /*
116  * Cgroups above their limits are maintained in a RB-Tree, independent of
117  * their hierarchy representation
118  */
119 
120 struct mem_cgroup_tree_per_node {
121         struct rb_root rb_root;
122         struct rb_node *rb_rightmost;
123         spinlock_t lock;
124 };
125 
126 struct mem_cgroup_tree {
127         struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
128 };
129 
130 static struct mem_cgroup_tree soft_limit_tree __read_mostly;
131 
132 /* for OOM */
133 struct mem_cgroup_eventfd_list {
134         struct list_head list;
135         struct eventfd_ctx *eventfd;
136 };
137 
138 /*
139  * cgroup_event represents events which userspace want to receive.
140  */
141 struct mem_cgroup_event {
142         /*
143          * memcg which the event belongs to.
144          */
145         struct mem_cgroup *memcg;
146         /*
147          * eventfd to signal userspace about the event.
148          */
149         struct eventfd_ctx *eventfd;
150         /*
151          * Each of these stored in a list by the cgroup.
152          */
153         struct list_head list;
154         /*
155          * register_event() callback will be used to add new userspace
156          * waiter for changes related to this event.  Use eventfd_signal()
157          * on eventfd to send notification to userspace.
158          */
159         int (*register_event)(struct mem_cgroup *memcg,
160                               struct eventfd_ctx *eventfd, const char *args);
161         /*
162          * unregister_event() callback will be called when userspace closes
163          * the eventfd or on cgroup removing.  This callback must be set,
164          * if you want provide notification functionality.
165          */
166         void (*unregister_event)(struct mem_cgroup *memcg,
167                                  struct eventfd_ctx *eventfd);
168         /*
169          * All fields below needed to unregister event when
170          * userspace closes eventfd.
171          */
172         poll_table pt;
173         wait_queue_head_t *wqh;
174         wait_queue_entry_t wait;
175         struct work_struct remove;
176 };
177 
178 static void mem_cgroup_threshold(struct mem_cgroup *memcg);
179 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
180 
181 /* Stuffs for move charges at task migration. */
182 /*
183  * Types of charges to be moved.
184  */
185 #define MOVE_ANON       0x1U
186 #define MOVE_FILE       0x2U
187 #define MOVE_MASK       (MOVE_ANON | MOVE_FILE)
188 
189 /* "mc" and its members are protected by cgroup_mutex */
190 static struct move_charge_struct {
191         spinlock_t        lock; /* for from, to */
192         struct mm_struct  *mm;
193         struct mem_cgroup *from;
194         struct mem_cgroup *to;
195         unsigned long flags;
196         unsigned long precharge;
197         unsigned long moved_charge;
198         unsigned long moved_swap;
199         struct task_struct *moving_task;        /* a task moving charges */
200         wait_queue_head_t waitq;                /* a waitq for other context */
201 } mc = {
202         .lock = __SPIN_LOCK_UNLOCKED(mc.lock),
203         .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
204 };
205 
206 /*
207  * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
208  * limit reclaim to prevent infinite loops, if they ever occur.
209  */
210 #define MEM_CGROUP_MAX_RECLAIM_LOOPS            100
211 #define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2
212 
213 enum charge_type {
214         MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
215         MEM_CGROUP_CHARGE_TYPE_ANON,
216         MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */
217         MEM_CGROUP_CHARGE_TYPE_DROP,    /* a page was unused swap cache */
218         NR_CHARGE_TYPE,
219 };
220 
221 /* for encoding cft->private value on file */
222 enum res_type {
223         _MEM,
224         _MEMSWAP,
225         _OOM_TYPE,
226         _KMEM,
227         _TCP,
228 };
229 
230 #define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val))
231 #define MEMFILE_TYPE(val)       ((val) >> 16 & 0xffff)
232 #define MEMFILE_ATTR(val)       ((val) & 0xffff)
233 /* Used for OOM nofiier */
234 #define OOM_CONTROL             (0)
235 
236 /* Some nice accessors for the vmpressure. */
237 struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
238 {
239         if (!memcg)
240                 memcg = root_mem_cgroup;
241         return &memcg->vmpressure;
242 }
243 
244 struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
245 {
246         return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
247 }
248 
249 static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
250 {
251         return (memcg == root_mem_cgroup);
252 }
253 
254 #ifndef CONFIG_SLOB
255 /*
256  * This will be the memcg's index in each cache's ->memcg_params.memcg_caches.
257  * The main reason for not using cgroup id for this:
258  *  this works better in sparse environments, where we have a lot of memcgs,
259  *  but only a few kmem-limited. Or also, if we have, for instance, 200
260  *  memcgs, and none but the 200th is kmem-limited, we'd have to have a
261  *  200 entry array for that.
262  *
263  * The current size of the caches array is stored in memcg_nr_cache_ids. It
264  * will double each time we have to increase it.
265  */
266 static DEFINE_IDA(memcg_cache_ida);
267 int memcg_nr_cache_ids;
268 
269 /* Protects memcg_nr_cache_ids */
270 static DECLARE_RWSEM(memcg_cache_ids_sem);
271 
272 void memcg_get_cache_ids(void)
273 {
274         down_read(&memcg_cache_ids_sem);
275 }
276 
277 void memcg_put_cache_ids(void)
278 {
279         up_read(&memcg_cache_ids_sem);
280 }
281 
282 /*
283  * MIN_SIZE is different than 1, because we would like to avoid going through
284  * the alloc/free process all the time. In a small machine, 4 kmem-limited
285  * cgroups is a reasonable guess. In the future, it could be a parameter or
286  * tunable, but that is strictly not necessary.
287  *
288  * MAX_SIZE should be as large as the number of cgrp_ids. Ideally, we could get
289  * this constant directly from cgroup, but it is understandable that this is
290  * better kept as an internal representation in cgroup.c. In any case, the
291  * cgrp_id space is not getting any smaller, and we don't have to necessarily
292  * increase ours as well if it increases.
293  */
294 #define MEMCG_CACHES_MIN_SIZE 4
295 #define MEMCG_CACHES_MAX_SIZE MEM_CGROUP_ID_MAX
296 
297 /*
298  * A lot of the calls to the cache allocation functions are expected to be
299  * inlined by the compiler. Since the calls to memcg_kmem_get_cache are
300  * conditional to this static branch, we'll have to allow modules that does
301  * kmem_cache_alloc and the such to see this symbol as well
302  */
303 DEFINE_STATIC_KEY_FALSE(memcg_kmem_enabled_key);
304 EXPORT_SYMBOL(memcg_kmem_enabled_key);
305 
306 struct workqueue_struct *memcg_kmem_cache_wq;
307 
308 #endif /* !CONFIG_SLOB */
309 
310 /**
311  * mem_cgroup_css_from_page - css of the memcg associated with a page
312  * @page: page of interest
313  *
314  * If memcg is bound to the default hierarchy, css of the memcg associated
315  * with @page is returned.  The returned css remains associated with @page
316  * until it is released.
317  *
318  * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup
319  * is returned.
320  */
321 struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page)
322 {
323         struct mem_cgroup *memcg;
324 
325         memcg = page->mem_cgroup;
326 
327         if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
328                 memcg = root_mem_cgroup;
329 
330         return &memcg->css;
331 }
332 
333 /**
334  * page_cgroup_ino - return inode number of the memcg a page is charged to
335  * @page: the page
336  *
337  * Look up the closest online ancestor of the memory cgroup @page is charged to
338  * and return its inode number or 0 if @page is not charged to any cgroup. It
339  * is safe to call this function without holding a reference to @page.
340  *
341  * Note, this function is inherently racy, because there is nothing to prevent
342  * the cgroup inode from getting torn down and potentially reallocated a moment
343  * after page_cgroup_ino() returns, so it only should be used by callers that
344  * do not care (such as procfs interfaces).
345  */
346 ino_t page_cgroup_ino(struct page *page)
347 {
348         struct mem_cgroup *memcg;
349         unsigned long ino = 0;
350 
351         rcu_read_lock();
352         memcg = READ_ONCE(page->mem_cgroup);
353         while (memcg && !(memcg->css.flags & CSS_ONLINE))
354                 memcg = parent_mem_cgroup(memcg);
355         if (memcg)
356                 ino = cgroup_ino(memcg->css.cgroup);
357         rcu_read_unlock();
358         return ino;
359 }
360 
361 static struct mem_cgroup_per_node *
362 mem_cgroup_page_nodeinfo(struct mem_cgroup *memcg, struct page *page)
363 {
364         int nid = page_to_nid(page);
365 
366         return memcg->nodeinfo[nid];
367 }
368 
369 static struct mem_cgroup_tree_per_node *
370 soft_limit_tree_node(int nid)
371 {
372         return soft_limit_tree.rb_tree_per_node[nid];
373 }
374 
375 static struct mem_cgroup_tree_per_node *
376 soft_limit_tree_from_page(struct page *page)
377 {
378         int nid = page_to_nid(page);
379 
380         return soft_limit_tree.rb_tree_per_node[nid];
381 }
382 
383 static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz,
384                                          struct mem_cgroup_tree_per_node *mctz,
385                                          unsigned long new_usage_in_excess)
386 {
387         struct rb_node **p = &mctz->rb_root.rb_node;
388         struct rb_node *parent = NULL;
389         struct mem_cgroup_per_node *mz_node;
390         bool rightmost = true;
391 
392         if (mz->on_tree)
393                 return;
394 
395         mz->usage_in_excess = new_usage_in_excess;
396         if (!mz->usage_in_excess)
397                 return;
398         while (*p) {
399                 parent = *p;
400                 mz_node = rb_entry(parent, struct mem_cgroup_per_node,
401                                         tree_node);
402                 if (mz->usage_in_excess < mz_node->usage_in_excess) {
403                         p = &(*p)->rb_left;
404                         rightmost = false;
405                 }
406 
407                 /*
408                  * We can't avoid mem cgroups that are over their soft
409                  * limit by the same amount
410                  */
411                 else if (mz->usage_in_excess >= mz_node->usage_in_excess)
412                         p = &(*p)->rb_right;
413         }
414 
415         if (rightmost)
416                 mctz->rb_rightmost = &mz->tree_node;
417 
418         rb_link_node(&mz->tree_node, parent, p);
419         rb_insert_color(&mz->tree_node, &mctz->rb_root);
420         mz->on_tree = true;
421 }
422 
423 static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
424                                          struct mem_cgroup_tree_per_node *mctz)
425 {
426         if (!mz->on_tree)
427                 return;
428 
429         if (&mz->tree_node == mctz->rb_rightmost)
430                 mctz->rb_rightmost = rb_prev(&mz->tree_node);
431 
432         rb_erase(&mz->tree_node, &mctz->rb_root);
433         mz->on_tree = false;
434 }
435 
436 static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
437                                        struct mem_cgroup_tree_per_node *mctz)
438 {
439         unsigned long flags;
440 
441         spin_lock_irqsave(&mctz->lock, flags);
442         __mem_cgroup_remove_exceeded(mz, mctz);
443         spin_unlock_irqrestore(&mctz->lock, flags);
444 }
445 
446 static unsigned long soft_limit_excess(struct mem_cgroup *memcg)
447 {
448         unsigned long nr_pages = page_counter_read(&memcg->memory);
449         unsigned long soft_limit = READ_ONCE(memcg->soft_limit);
450         unsigned long excess = 0;
451 
452         if (nr_pages > soft_limit)
453                 excess = nr_pages - soft_limit;
454 
455         return excess;
456 }
457 
458 static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
459 {
460         unsigned long excess;
461         struct mem_cgroup_per_node *mz;
462         struct mem_cgroup_tree_per_node *mctz;
463 
464         mctz = soft_limit_tree_from_page(page);
465         if (!mctz)
466                 return;
467         /*
468          * Necessary to update all ancestors when hierarchy is used.
469          * because their event counter is not touched.
470          */
471         for (; memcg; memcg = parent_mem_cgroup(memcg)) {
472                 mz = mem_cgroup_page_nodeinfo(memcg, page);
473                 excess = soft_limit_excess(memcg);
474                 /*
475                  * We have to update the tree if mz is on RB-tree or
476                  * mem is over its softlimit.
477                  */
478                 if (excess || mz->on_tree) {
479                         unsigned long flags;
480 
481                         spin_lock_irqsave(&mctz->lock, flags);
482                         /* if on-tree, remove it */
483                         if (mz->on_tree)
484                                 __mem_cgroup_remove_exceeded(mz, mctz);
485                         /*
486                          * Insert again. mz->usage_in_excess will be updated.
487                          * If excess is 0, no tree ops.
488                          */
489                         __mem_cgroup_insert_exceeded(mz, mctz, excess);
490                         spin_unlock_irqrestore(&mctz->lock, flags);
491                 }
492         }
493 }
494 
495 static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
496 {
497         struct mem_cgroup_tree_per_node *mctz;
498         struct mem_cgroup_per_node *mz;
499         int nid;
500 
501         for_each_node(nid) {
502                 mz = mem_cgroup_nodeinfo(memcg, nid);
503                 mctz = soft_limit_tree_node(nid);
504                 if (mctz)
505                         mem_cgroup_remove_exceeded(mz, mctz);
506         }
507 }
508 
509 static struct mem_cgroup_per_node *
510 __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
511 {
512         struct mem_cgroup_per_node *mz;
513 
514 retry:
515         mz = NULL;
516         if (!mctz->rb_rightmost)
517                 goto done;              /* Nothing to reclaim from */
518 
519         mz = rb_entry(mctz->rb_rightmost,
520                       struct mem_cgroup_per_node, tree_node);
521         /*
522          * Remove the node now but someone else can add it back,
523          * we will to add it back at the end of reclaim to its correct
524          * position in the tree.
525          */
526         __mem_cgroup_remove_exceeded(mz, mctz);
527         if (!soft_limit_excess(mz->memcg) ||
528             !css_tryget_online(&mz->memcg->css))
529                 goto retry;
530 done:
531         return mz;
532 }
533 
534 static struct mem_cgroup_per_node *
535 mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
536 {
537         struct mem_cgroup_per_node *mz;
538 
539         spin_lock_irq(&mctz->lock);
540         mz = __mem_cgroup_largest_soft_limit_node(mctz);
541         spin_unlock_irq(&mctz->lock);
542         return mz;
543 }
544 
545 /*
546  * Return page count for single (non recursive) @memcg.
547  *
548  * Implementation Note: reading percpu statistics for memcg.
549  *
550  * Both of vmstat[] and percpu_counter has threshold and do periodic
551  * synchronization to implement "quick" read. There are trade-off between
552  * reading cost and precision of value. Then, we may have a chance to implement
553  * a periodic synchronization of counter in memcg's counter.
554  *
555  * But this _read() function is used for user interface now. The user accounts
556  * memory usage by memory cgroup and he _always_ requires exact value because
557  * he accounts memory. Even if we provide quick-and-fuzzy read, we always
558  * have to visit all online cpus and make sum. So, for now, unnecessary
559  * synchronization is not implemented. (just implemented for cpu hotplug)
560  *
561  * If there are kernel internal actions which can make use of some not-exact
562  * value, and reading all cpu value can be performance bottleneck in some
563  * common workload, threshold and synchronization as vmstat[] should be
564  * implemented.
565  *
566  * The parameter idx can be of type enum memcg_event_item or vm_event_item.
567  */
568 
569 static unsigned long memcg_sum_events(struct mem_cgroup *memcg,
570                                       int event)
571 {
572         unsigned long val = 0;
573         int cpu;
574 
575         for_each_possible_cpu(cpu)
576                 val += per_cpu(memcg->stat->events[event], cpu);
577         return val;
578 }
579 
580 static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
581                                          struct page *page,
582                                          bool compound, int nr_pages)
583 {
584         /*
585          * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is
586          * counted as CACHE even if it's on ANON LRU.
587          */
588         if (PageAnon(page))
589                 __this_cpu_add(memcg->stat->count[MEMCG_RSS], nr_pages);
590         else {
591                 __this_cpu_add(memcg->stat->count[MEMCG_CACHE], nr_pages);
592                 if (PageSwapBacked(page))
593                         __this_cpu_add(memcg->stat->count[NR_SHMEM], nr_pages);
594         }
595 
596         if (compound) {
597                 VM_BUG_ON_PAGE(!PageTransHuge(page), page);
598                 __this_cpu_add(memcg->stat->count[MEMCG_RSS_HUGE], nr_pages);
599         }
600 
601         /* pagein of a big page is an event. So, ignore page size */
602         if (nr_pages > 0)
603                 __this_cpu_inc(memcg->stat->events[PGPGIN]);
604         else {
605                 __this_cpu_inc(memcg->stat->events[PGPGOUT]);
606                 nr_pages = -nr_pages; /* for event */
607         }
608 
609         __this_cpu_add(memcg->stat->nr_page_events, nr_pages);
610 }
611 
612 unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
613                                            int nid, unsigned int lru_mask)
614 {
615         struct lruvec *lruvec = mem_cgroup_lruvec(NODE_DATA(nid), memcg);
616         unsigned long nr = 0;
617         enum lru_list lru;
618 
619         VM_BUG_ON((unsigned)nid >= nr_node_ids);
620 
621         for_each_lru(lru) {
622                 if (!(BIT(lru) & lru_mask))
623                         continue;
624                 nr += mem_cgroup_get_lru_size(lruvec, lru);
625         }
626         return nr;
627 }
628 
629 static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
630                         unsigned int lru_mask)
631 {
632         unsigned long nr = 0;
633         int nid;
634 
635         for_each_node_state(nid, N_MEMORY)
636                 nr += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask);
637         return nr;
638 }
639 
640 static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
641                                        enum mem_cgroup_events_target target)
642 {
643         unsigned long val, next;
644 
645         val = __this_cpu_read(memcg->stat->nr_page_events);
646         next = __this_cpu_read(memcg->stat->targets[target]);
647         /* from time_after() in jiffies.h */
648         if ((long)(next - val) < 0) {
649                 switch (target) {
650                 case MEM_CGROUP_TARGET_THRESH:
651                         next = val + THRESHOLDS_EVENTS_TARGET;
652                         break;
653                 case MEM_CGROUP_TARGET_SOFTLIMIT:
654                         next = val + SOFTLIMIT_EVENTS_TARGET;
655                         break;
656                 case MEM_CGROUP_TARGET_NUMAINFO:
657                         next = val + NUMAINFO_EVENTS_TARGET;
658                         break;
659                 default:
660                         break;
661                 }
662                 __this_cpu_write(memcg->stat->targets[target], next);
663                 return true;
664         }
665         return false;
666 }
667 
668 /*
669  * Check events in order.
670  *
671  */
672 static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
673 {
674         /* threshold event is triggered in finer grain than soft limit */
675         if (unlikely(mem_cgroup_event_ratelimit(memcg,
676                                                 MEM_CGROUP_TARGET_THRESH))) {
677                 bool do_softlimit;
678                 bool do_numainfo __maybe_unused;
679 
680                 do_softlimit = mem_cgroup_event_ratelimit(memcg,
681                                                 MEM_CGROUP_TARGET_SOFTLIMIT);
682 #if MAX_NUMNODES > 1
683                 do_numainfo = mem_cgroup_event_ratelimit(memcg,
684                                                 MEM_CGROUP_TARGET_NUMAINFO);
685 #endif
686                 mem_cgroup_threshold(memcg);
687                 if (unlikely(do_softlimit))
688                         mem_cgroup_update_tree(memcg, page);
689 #if MAX_NUMNODES > 1
690                 if (unlikely(do_numainfo))
691                         atomic_inc(&memcg->numainfo_events);
692 #endif
693         }
694 }
695 
696 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
697 {
698         /*
699          * mm_update_next_owner() may clear mm->owner to NULL
700          * if it races with swapoff, page migration, etc.
701          * So this can be called with p == NULL.
702          */
703         if (unlikely(!p))
704                 return NULL;
705 
706         return mem_cgroup_from_css(task_css(p, memory_cgrp_id));
707 }
708 EXPORT_SYMBOL(mem_cgroup_from_task);
709 
710 static struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
711 {
712         struct mem_cgroup *memcg = NULL;
713 
714         rcu_read_lock();
715         do {
716                 /*
717                  * Page cache insertions can happen withou an
718                  * actual mm context, e.g. during disk probing
719                  * on boot, loopback IO, acct() writes etc.
720                  */
721                 if (unlikely(!mm))
722                         memcg = root_mem_cgroup;
723                 else {
724                         memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
725                         if (unlikely(!memcg))
726                                 memcg = root_mem_cgroup;
727                 }
728         } while (!css_tryget_online(&memcg->css));
729         rcu_read_unlock();
730         return memcg;
731 }
732 
733 /**
734  * mem_cgroup_iter - iterate over memory cgroup hierarchy
735  * @root: hierarchy root
736  * @prev: previously returned memcg, NULL on first invocation
737  * @reclaim: cookie for shared reclaim walks, NULL for full walks
738  *
739  * Returns references to children of the hierarchy below @root, or
740  * @root itself, or %NULL after a full round-trip.
741  *
742  * Caller must pass the return value in @prev on subsequent
743  * invocations for reference counting, or use mem_cgroup_iter_break()
744  * to cancel a hierarchy walk before the round-trip is complete.
745  *
746  * Reclaimers can specify a zone and a priority level in @reclaim to
747  * divide up the memcgs in the hierarchy among all concurrent
748  * reclaimers operating on the same zone and priority.
749  */
750 struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
751                                    struct mem_cgroup *prev,
752                                    struct mem_cgroup_reclaim_cookie *reclaim)
753 {
754         struct mem_cgroup_reclaim_iter *uninitialized_var(iter);
755         struct cgroup_subsys_state *css = NULL;
756         struct mem_cgroup *memcg = NULL;
757         struct mem_cgroup *pos = NULL;
758 
759         if (mem_cgroup_disabled())
760                 return NULL;
761 
762         if (!root)
763                 root = root_mem_cgroup;
764 
765         if (prev && !reclaim)
766                 pos = prev;
767 
768         if (!root->use_hierarchy && root != root_mem_cgroup) {
769                 if (prev)
770                         goto out;
771                 return root;
772         }
773 
774         rcu_read_lock();
775 
776         if (reclaim) {
777                 struct mem_cgroup_per_node *mz;
778 
779                 mz = mem_cgroup_nodeinfo(root, reclaim->pgdat->node_id);
780                 iter = &mz->iter[reclaim->priority];
781 
782                 if (prev && reclaim->generation != iter->generation)
783                         goto out_unlock;
784 
785                 while (1) {
786                         pos = READ_ONCE(iter->position);
787                         if (!pos || css_tryget(&pos->css))
788                                 break;
789                         /*
790                          * css reference reached zero, so iter->position will
791                          * be cleared by ->css_released. However, we should not
792                          * rely on this happening soon, because ->css_released
793                          * is called from a work queue, and by busy-waiting we
794                          * might block it. So we clear iter->position right
795                          * away.
796                          */
797                         (void)cmpxchg(&iter->position, pos, NULL);
798                 }
799         }
800 
801         if (pos)
802                 css = &pos->css;
803 
804         for (;;) {
805                 css = css_next_descendant_pre(css, &root->css);
806                 if (!css) {
807                         /*
808                          * Reclaimers share the hierarchy walk, and a
809                          * new one might jump in right at the end of
810                          * the hierarchy - make sure they see at least
811                          * one group and restart from the beginning.
812                          */
813                         if (!prev)
814                                 continue;
815                         break;
816                 }
817 
818                 /*
819                  * Verify the css and acquire a reference.  The root
820                  * is provided by the caller, so we know it's alive
821                  * and kicking, and don't take an extra reference.
822                  */
823                 memcg = mem_cgroup_from_css(css);
824 
825                 if (css == &root->css)
826                         break;
827 
828                 if (css_tryget(css))
829                         break;
830 
831                 memcg = NULL;
832         }
833 
834         if (reclaim) {
835                 /*
836                  * The position could have already been updated by a competing
837                  * thread, so check that the value hasn't changed since we read
838                  * it to avoid reclaiming from the same cgroup twice.
839                  */
840                 (void)cmpxchg(&iter->position, pos, memcg);
841 
842                 if (pos)
843                         css_put(&pos->css);
844 
845                 if (!memcg)
846                         iter->generation++;
847                 else if (!prev)
848                         reclaim->generation = iter->generation;
849         }
850 
851 out_unlock:
852         rcu_read_unlock();
853 out:
854         if (prev && prev != root)
855                 css_put(&prev->css);
856 
857         return memcg;
858 }
859 
860 /**
861  * mem_cgroup_iter_break - abort a hierarchy walk prematurely
862  * @root: hierarchy root
863  * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
864  */
865 void mem_cgroup_iter_break(struct mem_cgroup *root,
866                            struct mem_cgroup *prev)
867 {
868         if (!root)
869                 root = root_mem_cgroup;
870         if (prev && prev != root)
871                 css_put(&prev->css);
872 }
873 
874 static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg)
875 {
876         struct mem_cgroup *memcg = dead_memcg;
877         struct mem_cgroup_reclaim_iter *iter;
878         struct mem_cgroup_per_node *mz;
879         int nid;
880         int i;
881 
882         while ((memcg = parent_mem_cgroup(memcg))) {
883                 for_each_node(nid) {
884                         mz = mem_cgroup_nodeinfo(memcg, nid);
885                         for (i = 0; i <= DEF_PRIORITY; i++) {
886                                 iter = &mz->iter[i];
887                                 cmpxchg(&iter->position,
888                                         dead_memcg, NULL);
889                         }
890                 }
891         }
892 }
893 
894 /*
895  * Iteration constructs for visiting all cgroups (under a tree).  If
896  * loops are exited prematurely (break), mem_cgroup_iter_break() must
897  * be used for reference counting.
898  */
899 #define for_each_mem_cgroup_tree(iter, root)            \
900         for (iter = mem_cgroup_iter(root, NULL, NULL);  \
901              iter != NULL;                              \
902              iter = mem_cgroup_iter(root, iter, NULL))
903 
904 #define for_each_mem_cgroup(iter)                       \
905         for (iter = mem_cgroup_iter(NULL, NULL, NULL);  \
906              iter != NULL;                              \
907              iter = mem_cgroup_iter(NULL, iter, NULL))
908 
909 /**
910  * mem_cgroup_scan_tasks - iterate over tasks of a memory cgroup hierarchy
911  * @memcg: hierarchy root
912  * @fn: function to call for each task
913  * @arg: argument passed to @fn
914  *
915  * This function iterates over tasks attached to @memcg or to any of its
916  * descendants and calls @fn for each task. If @fn returns a non-zero
917  * value, the function breaks the iteration loop and returns the value.
918  * Otherwise, it will iterate over all tasks and return 0.
919  *
920  * This function must not be called for the root memory cgroup.
921  */
922 int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
923                           int (*fn)(struct task_struct *, void *), void *arg)
924 {
925         struct mem_cgroup *iter;
926         int ret = 0;
927 
928         BUG_ON(memcg == root_mem_cgroup);
929 
930         for_each_mem_cgroup_tree(iter, memcg) {
931                 struct css_task_iter it;
932                 struct task_struct *task;
933 
934                 css_task_iter_start(&iter->css, 0, &it);
935                 while (!ret && (task = css_task_iter_next(&it)))
936                         ret = fn(task, arg);
937                 css_task_iter_end(&it);
938                 if (ret) {
939                         mem_cgroup_iter_break(memcg, iter);
940                         break;
941                 }
942         }
943         return ret;
944 }
945 
946 /**
947  * mem_cgroup_page_lruvec - return lruvec for isolating/putting an LRU page
948  * @page: the page
949  * @zone: zone of the page
950  *
951  * This function is only safe when following the LRU page isolation
952  * and putback protocol: the LRU lock must be held, and the page must
953  * either be PageLRU() or the caller must have isolated/allocated it.
954  */
955 struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct pglist_data *pgdat)
956 {
957         struct mem_cgroup_per_node *mz;
958         struct mem_cgroup *memcg;
959         struct lruvec *lruvec;
960 
961         if (mem_cgroup_disabled()) {
962                 lruvec = &pgdat->lruvec;
963                 goto out;
964         }
965 
966         memcg = page->mem_cgroup;
967         /*
968          * Swapcache readahead pages are added to the LRU - and
969          * possibly migrated - before they are charged.
970          */
971         if (!memcg)
972                 memcg = root_mem_cgroup;
973 
974         mz = mem_cgroup_page_nodeinfo(memcg, page);
975         lruvec = &mz->lruvec;
976 out:
977         /*
978          * Since a node can be onlined after the mem_cgroup was created,
979          * we have to be prepared to initialize lruvec->zone here;
980          * and if offlined then reonlined, we need to reinitialize it.
981          */
982         if (unlikely(lruvec->pgdat != pgdat))
983                 lruvec->pgdat = pgdat;
984         return lruvec;
985 }
986 
987 /**
988  * mem_cgroup_update_lru_size - account for adding or removing an lru page
989  * @lruvec: mem_cgroup per zone lru vector
990  * @lru: index of lru list the page is sitting on
991  * @zid: zone id of the accounted pages
992  * @nr_pages: positive when adding or negative when removing
993  *
994  * This function must be called under lru_lock, just before a page is added
995  * to or just after a page is removed from an lru list (that ordering being
996  * so as to allow it to check that lru_size 0 is consistent with list_empty).
997  */
998 void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
999                                 int zid, int nr_pages)
1000 {
1001         struct mem_cgroup_per_node *mz;
1002         unsigned long *lru_size;
1003         long size;
1004 
1005         if (mem_cgroup_disabled())
1006                 return;
1007 
1008         mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
1009         lru_size = &mz->lru_zone_size[zid][lru];
1010 
1011         if (nr_pages < 0)
1012                 *lru_size += nr_pages;
1013 
1014         size = *lru_size;
1015         if (WARN_ONCE(size < 0,
1016                 "%s(%p, %d, %d): lru_size %ld\n",
1017                 __func__, lruvec, lru, nr_pages, size)) {
1018                 VM_BUG_ON(1);
1019                 *lru_size = 0;
1020         }
1021 
1022         if (nr_pages > 0)
1023                 *lru_size += nr_pages;
1024 }
1025 
1026 bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg)
1027 {
1028         struct mem_cgroup *task_memcg;
1029         struct task_struct *p;
1030         bool ret;
1031 
1032         p = find_lock_task_mm(task);
1033         if (p) {
1034                 task_memcg = get_mem_cgroup_from_mm(p->mm);
1035                 task_unlock(p);
1036         } else {
1037                 /*
1038                  * All threads may have already detached their mm's, but the oom
1039                  * killer still needs to detect if they have already been oom
1040                  * killed to prevent needlessly killing additional tasks.
1041                  */
1042                 rcu_read_lock();
1043                 task_memcg = mem_cgroup_from_task(task);
1044                 css_get(&task_memcg->css);
1045                 rcu_read_unlock();
1046         }
1047         ret = mem_cgroup_is_descendant(task_memcg, memcg);
1048         css_put(&task_memcg->css);
1049         return ret;
1050 }
1051 
1052 /**
1053  * mem_cgroup_margin - calculate chargeable space of a memory cgroup
1054  * @memcg: the memory cgroup
1055  *
1056  * Returns the maximum amount of memory @mem can be charged with, in
1057  * pages.
1058  */
1059 static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
1060 {
1061         unsigned long margin = 0;
1062         unsigned long count;
1063         unsigned long limit;
1064 
1065         count = page_counter_read(&memcg->memory);
1066         limit = READ_ONCE(memcg->memory.limit);
1067         if (count < limit)
1068                 margin = limit - count;
1069 
1070         if (do_memsw_account()) {
1071                 count = page_counter_read(&memcg->memsw);
1072                 limit = READ_ONCE(memcg->memsw.limit);
1073                 if (count <= limit)
1074                         margin = min(margin, limit - count);
1075                 else
1076                         margin = 0;
1077         }
1078 
1079         return margin;
1080 }
1081 
1082 /*
1083  * A routine for checking "mem" is under move_account() or not.
1084  *
1085  * Checking a cgroup is mc.from or mc.to or under hierarchy of
1086  * moving cgroups. This is for waiting at high-memory pressure
1087  * caused by "move".
1088  */
1089 static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
1090 {
1091         struct mem_cgroup *from;
1092         struct mem_cgroup *to;
1093         bool ret = false;
1094         /*
1095          * Unlike task_move routines, we access mc.to, mc.from not under
1096          * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
1097          */
1098         spin_lock(&mc.lock);
1099         from = mc.from;
1100         to = mc.to;
1101         if (!from)
1102                 goto unlock;
1103 
1104         ret = mem_cgroup_is_descendant(from, memcg) ||
1105                 mem_cgroup_is_descendant(to, memcg);
1106 unlock:
1107         spin_unlock(&mc.lock);
1108         return ret;
1109 }
1110 
1111 static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
1112 {
1113         if (mc.moving_task && current != mc.moving_task) {
1114                 if (mem_cgroup_under_move(memcg)) {
1115                         DEFINE_WAIT(wait);
1116                         prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
1117                         /* moving charge context might have finished. */
1118                         if (mc.moving_task)
1119                                 schedule();
1120                         finish_wait(&mc.waitq, &wait);
1121                         return true;
1122                 }
1123         }
1124         return false;
1125 }
1126 
1127 unsigned int memcg1_stats[] = {
1128         MEMCG_CACHE,
1129         MEMCG_RSS,
1130         MEMCG_RSS_HUGE,
1131         NR_SHMEM,
1132         NR_FILE_MAPPED,
1133         NR_FILE_DIRTY,
1134         NR_WRITEBACK,
1135         MEMCG_SWAP,
1136 };
1137 
1138 static const char *const memcg1_stat_names[] = {
1139         "cache",
1140         "rss",
1141         "rss_huge",
1142         "shmem",
1143         "mapped_file",
1144         "dirty",
1145         "writeback",
1146         "swap",
1147 };
1148 
1149 #define K(x) ((x) << (PAGE_SHIFT-10))
1150 /**
1151  * mem_cgroup_print_oom_info: Print OOM information relevant to memory controller.
1152  * @memcg: The memory cgroup that went over limit
1153  * @p: Task that is going to be killed
1154  *
1155  * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1156  * enabled
1157  */
1158 void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
1159 {
1160         struct mem_cgroup *iter;
1161         unsigned int i;
1162 
1163         rcu_read_lock();
1164 
1165         if (p) {
1166                 pr_info("Task in ");
1167                 pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id));
1168                 pr_cont(" killed as a result of limit of ");
1169         } else {
1170                 pr_info("Memory limit reached of cgroup ");
1171         }
1172 
1173         pr_cont_cgroup_path(memcg->css.cgroup);
1174         pr_cont("\n");
1175 
1176         rcu_read_unlock();
1177 
1178         pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n",
1179                 K((u64)page_counter_read(&memcg->memory)),
1180                 K((u64)memcg->memory.limit), memcg->memory.failcnt);
1181         pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n",
1182                 K((u64)page_counter_read(&memcg->memsw)),
1183                 K((u64)memcg->memsw.limit), memcg->memsw.failcnt);
1184         pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n",
1185                 K((u64)page_counter_read(&memcg->kmem)),
1186                 K((u64)memcg->kmem.limit), memcg->kmem.failcnt);
1187 
1188         for_each_mem_cgroup_tree(iter, memcg) {
1189                 pr_info("Memory cgroup stats for ");
1190                 pr_cont_cgroup_path(iter->css.cgroup);
1191                 pr_cont(":");
1192 
1193                 for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
1194                         if (memcg1_stats[i] == MEMCG_SWAP && !do_swap_account)
1195                                 continue;
1196                         pr_cont(" %s:%luKB", memcg1_stat_names[i],
1197                                 K(memcg_page_state(iter, memcg1_stats[i])));
1198                 }
1199 
1200                 for (i = 0; i < NR_LRU_LISTS; i++)
1201                         pr_cont(" %s:%luKB", mem_cgroup_lru_names[i],
1202                                 K(mem_cgroup_nr_lru_pages(iter, BIT(i))));
1203 
1204                 pr_cont("\n");
1205         }
1206 }
1207 
1208 /*
1209  * This function returns the number of memcg under hierarchy tree. Returns
1210  * 1(self count) if no children.
1211  */
1212 static int mem_cgroup_count_children(struct mem_cgroup *memcg)
1213 {
1214         int num = 0;
1215         struct mem_cgroup *iter;
1216 
1217         for_each_mem_cgroup_tree(iter, memcg)
1218                 num++;
1219         return num;
1220 }
1221 
1222 /*
1223  * Return the memory (and swap, if configured) limit for a memcg.
1224  */
1225 unsigned long mem_cgroup_get_limit(struct mem_cgroup *memcg)
1226 {
1227         unsigned long limit;
1228 
1229         limit = memcg->memory.limit;
1230         if (mem_cgroup_swappiness(memcg)) {
1231                 unsigned long memsw_limit;
1232                 unsigned long swap_limit;
1233 
1234                 memsw_limit = memcg->memsw.limit;
1235                 swap_limit = memcg->swap.limit;
1236                 swap_limit = min(swap_limit, (unsigned long)total_swap_pages);
1237                 limit = min(limit + swap_limit, memsw_limit);
1238         }
1239         return limit;
1240 }
1241 
1242 static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
1243                                      int order)
1244 {
1245         struct oom_control oc = {
1246                 .zonelist = NULL,
1247                 .nodemask = NULL,
1248                 .memcg = memcg,
1249                 .gfp_mask = gfp_mask,
1250                 .order = order,
1251         };
1252         bool ret;
1253 
1254         mutex_lock(&oom_lock);
1255         ret = out_of_memory(&oc);
1256         mutex_unlock(&oom_lock);
1257         return ret;
1258 }
1259 
1260 #if MAX_NUMNODES > 1
1261 
1262 /**
1263  * test_mem_cgroup_node_reclaimable
1264  * @memcg: the target memcg
1265  * @nid: the node ID to be checked.
1266  * @noswap : specify true here if the user wants flle only information.
1267  *
1268  * This function returns whether the specified memcg contains any
1269  * reclaimable pages on a node. Returns true if there are any reclaimable
1270  * pages in the node.
1271  */
1272 static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
1273                 int nid, bool noswap)
1274 {
1275         if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE))
1276                 return true;
1277         if (noswap || !total_swap_pages)
1278                 return false;
1279         if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON))
1280                 return true;
1281         return false;
1282 
1283 }
1284 
1285 /*
1286  * Always updating the nodemask is not very good - even if we have an empty
1287  * list or the wrong list here, we can start from some node and traverse all
1288  * nodes based on the zonelist. So update the list loosely once per 10 secs.
1289  *
1290  */
1291 static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
1292 {
1293         int nid;
1294         /*
1295          * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
1296          * pagein/pageout changes since the last update.
1297          */
1298         if (!atomic_read(&memcg->numainfo_events))
1299                 return;
1300         if (atomic_inc_return(&memcg->numainfo_updating) > 1)
1301                 return;
1302 
1303         /* make a nodemask where this memcg uses memory from */
1304         memcg->scan_nodes = node_states[N_MEMORY];
1305 
1306         for_each_node_mask(nid, node_states[N_MEMORY]) {
1307 
1308                 if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
1309                         node_clear(nid, memcg->scan_nodes);
1310         }
1311 
1312         atomic_set(&memcg->numainfo_events, 0);
1313         atomic_set(&memcg->numainfo_updating, 0);
1314 }
1315 
1316 /*
1317  * Selecting a node where we start reclaim from. Because what we need is just
1318  * reducing usage counter, start from anywhere is O,K. Considering
1319  * memory reclaim from current node, there are pros. and cons.
1320  *
1321  * Freeing memory from current node means freeing memory from a node which
1322  * we'll use or we've used. So, it may make LRU bad. And if several threads
1323  * hit limits, it will see a contention on a node. But freeing from remote
1324  * node means more costs for memory reclaim because of memory latency.
1325  *
1326  * Now, we use round-robin. Better algorithm is welcomed.
1327  */
1328 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1329 {
1330         int node;
1331 
1332         mem_cgroup_may_update_nodemask(memcg);
1333         node = memcg->last_scanned_node;
1334 
1335         node = next_node_in(node, memcg->scan_nodes);
1336         /*
1337          * mem_cgroup_may_update_nodemask might have seen no reclaimmable pages
1338          * last time it really checked all the LRUs due to rate limiting.
1339          * Fallback to the current node in that case for simplicity.
1340          */
1341         if (unlikely(node == MAX_NUMNODES))
1342                 node = numa_node_id();
1343 
1344         memcg->last_scanned_node = node;
1345         return node;
1346 }
1347 #else
1348 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1349 {
1350         return 0;
1351 }
1352 #endif
1353 
1354 static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
1355                                    pg_data_t *pgdat,
1356                                    gfp_t gfp_mask,
1357                                    unsigned long *total_scanned)
1358 {
1359         struct mem_cgroup *victim = NULL;
1360         int total = 0;
1361         int loop = 0;
1362         unsigned long excess;
1363         unsigned long nr_scanned;
1364         struct mem_cgroup_reclaim_cookie reclaim = {
1365                 .pgdat = pgdat,
1366                 .priority = 0,
1367         };
1368 
1369         excess = soft_limit_excess(root_memcg);
1370 
1371         while (1) {
1372                 victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
1373                 if (!victim) {
1374                         loop++;
1375                         if (loop >= 2) {
1376                                 /*
1377                                  * If we have not been able to reclaim
1378                                  * anything, it might because there are
1379                                  * no reclaimable pages under this hierarchy
1380                                  */
1381                                 if (!total)
1382                                         break;
1383                                 /*
1384                                  * We want to do more targeted reclaim.
1385                                  * excess >> 2 is not to excessive so as to
1386                                  * reclaim too much, nor too less that we keep
1387                                  * coming back to reclaim from this cgroup
1388                                  */
1389                                 if (total >= (excess >> 2) ||
1390                                         (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
1391                                         break;
1392                         }
1393                         continue;
1394                 }
1395                 total += mem_cgroup_shrink_node(victim, gfp_mask, false,
1396                                         pgdat, &nr_scanned);
1397                 *total_scanned += nr_scanned;
1398                 if (!soft_limit_excess(root_memcg))
1399                         break;
1400         }
1401         mem_cgroup_iter_break(root_memcg, victim);
1402         return total;
1403 }
1404 
1405 #ifdef CONFIG_LOCKDEP
1406 static struct lockdep_map memcg_oom_lock_dep_map = {
1407         .name = "memcg_oom_lock",
1408 };
1409 #endif
1410 
1411 static DEFINE_SPINLOCK(memcg_oom_lock);
1412 
1413 /*
1414  * Check OOM-Killer is already running under our hierarchy.
1415  * If someone is running, return false.
1416  */
1417 static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
1418 {
1419         struct mem_cgroup *iter, *failed = NULL;
1420 
1421         spin_lock(&memcg_oom_lock);
1422 
1423         for_each_mem_cgroup_tree(iter, memcg) {
1424                 if (iter->oom_lock) {
1425                         /*
1426                          * this subtree of our hierarchy is already locked
1427                          * so we cannot give a lock.
1428                          */
1429                         failed = iter;
1430                         mem_cgroup_iter_break(memcg, iter);
1431                         break;
1432                 } else
1433                         iter->oom_lock = true;
1434         }
1435 
1436         if (failed) {
1437                 /*
1438                  * OK, we failed to lock the whole subtree so we have
1439                  * to clean up what we set up to the failing subtree
1440                  */
1441                 for_each_mem_cgroup_tree(iter, memcg) {
1442                         if (iter == failed) {
1443                                 mem_cgroup_iter_break(memcg, iter);
1444                                 break;
1445                         }
1446                         iter->oom_lock = false;
1447                 }
1448         } else
1449                 mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_);
1450 
1451         spin_unlock(&memcg_oom_lock);
1452 
1453         return !failed;
1454 }
1455 
1456 static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
1457 {
1458         struct mem_cgroup *iter;
1459 
1460         spin_lock(&memcg_oom_lock);
1461         mutex_release(&memcg_oom_lock_dep_map, 1, _RET_IP_);
1462         for_each_mem_cgroup_tree(iter, memcg)
1463                 iter->oom_lock = false;
1464         spin_unlock(&memcg_oom_lock);
1465 }
1466 
1467 static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
1468 {
1469         struct mem_cgroup *iter;
1470 
1471         spin_lock(&memcg_oom_lock);
1472         for_each_mem_cgroup_tree(iter, memcg)
1473                 iter->under_oom++;
1474         spin_unlock(&memcg_oom_lock);
1475 }
1476 
1477 static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
1478 {
1479         struct mem_cgroup *iter;
1480 
1481         /*
1482          * When a new child is created while the hierarchy is under oom,
1483          * mem_cgroup_oom_lock() may not be called. Watch for underflow.
1484          */
1485         spin_lock(&memcg_oom_lock);
1486         for_each_mem_cgroup_tree(iter, memcg)
1487                 if (iter->under_oom > 0)
1488                         iter->under_oom--;
1489         spin_unlock(&memcg_oom_lock);
1490 }
1491 
1492 static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
1493 
1494 struct oom_wait_info {
1495         struct mem_cgroup *memcg;
1496         wait_queue_entry_t      wait;
1497 };
1498 
1499 static int memcg_oom_wake_function(wait_queue_entry_t *wait,
1500         unsigned mode, int sync, void *arg)
1501 {
1502         struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
1503         struct mem_cgroup *oom_wait_memcg;
1504         struct oom_wait_info *oom_wait_info;
1505 
1506         oom_wait_info = container_of(wait, struct oom_wait_info, wait);
1507         oom_wait_memcg = oom_wait_info->memcg;
1508 
1509         if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) &&
1510             !mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg))
1511                 return 0;
1512         return autoremove_wake_function(wait, mode, sync, arg);
1513 }
1514 
1515 static void memcg_oom_recover(struct mem_cgroup *memcg)
1516 {
1517         /*
1518          * For the following lockless ->under_oom test, the only required
1519          * guarantee is that it must see the state asserted by an OOM when
1520          * this function is called as a result of userland actions
1521          * triggered by the notification of the OOM.  This is trivially
1522          * achieved by invoking mem_cgroup_mark_under_oom() before
1523          * triggering notification.
1524          */
1525         if (memcg && memcg->under_oom)
1526                 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
1527 }
1528 
1529 static void mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
1530 {
1531         if (!current->memcg_may_oom)
1532                 return;
1533         /*
1534          * We are in the middle of the charge context here, so we
1535          * don't want to block when potentially sitting on a callstack
1536          * that holds all kinds of filesystem and mm locks.
1537          *
1538          * Also, the caller may handle a failed allocation gracefully
1539          * (like optional page cache readahead) and so an OOM killer
1540          * invocation might not even be necessary.
1541          *
1542          * That's why we don't do anything here except remember the
1543          * OOM context and then deal with it at the end of the page
1544          * fault when the stack is unwound, the locks are released,
1545          * and when we know whether the fault was overall successful.
1546          */
1547         css_get(&memcg->css);
1548         current->memcg_in_oom = memcg;
1549         current->memcg_oom_gfp_mask = mask;
1550         current->memcg_oom_order = order;
1551 }
1552 
1553 /**
1554  * mem_cgroup_oom_synchronize - complete memcg OOM handling
1555  * @handle: actually kill/wait or just clean up the OOM state
1556  *
1557  * This has to be called at the end of a page fault if the memcg OOM
1558  * handler was enabled.
1559  *
1560  * Memcg supports userspace OOM handling where failed allocations must
1561  * sleep on a waitqueue until the userspace task resolves the
1562  * situation.  Sleeping directly in the charge context with all kinds
1563  * of locks held is not a good idea, instead we remember an OOM state
1564  * in the task and mem_cgroup_oom_synchronize() has to be called at
1565  * the end of the page fault to complete the OOM handling.
1566  *
1567  * Returns %true if an ongoing memcg OOM situation was detected and
1568  * completed, %false otherwise.
1569  */
1570 bool mem_cgroup_oom_synchronize(bool handle)
1571 {
1572         struct mem_cgroup *memcg = current->memcg_in_oom;
1573         struct oom_wait_info owait;
1574         bool locked;
1575 
1576         /* OOM is global, do not handle */
1577         if (!memcg)
1578                 return false;
1579 
1580         if (!handle)
1581                 goto cleanup;
1582 
1583         owait.memcg = memcg;
1584         owait.wait.flags = 0;
1585         owait.wait.func = memcg_oom_wake_function;
1586         owait.wait.private = current;
1587         INIT_LIST_HEAD(&owait.wait.entry);
1588 
1589         prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
1590         mem_cgroup_mark_under_oom(memcg);
1591 
1592         locked = mem_cgroup_oom_trylock(memcg);
1593 
1594         if (locked)
1595                 mem_cgroup_oom_notify(memcg);
1596 
1597         if (locked && !memcg->oom_kill_disable) {
1598                 mem_cgroup_unmark_under_oom(memcg);
1599                 finish_wait(&memcg_oom_waitq, &owait.wait);
1600                 mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask,
1601                                          current->memcg_oom_order);
1602         } else {
1603                 schedule();
1604                 mem_cgroup_unmark_under_oom(memcg);
1605                 finish_wait(&memcg_oom_waitq, &owait.wait);
1606         }
1607 
1608         if (locked) {
1609                 mem_cgroup_oom_unlock(memcg);
1610                 /*
1611                  * There is no guarantee that an OOM-lock contender
1612                  * sees the wakeups triggered by the OOM kill
1613                  * uncharges.  Wake any sleepers explicitely.
1614                  */
1615                 memcg_oom_recover(memcg);
1616         }
1617 cleanup:
1618         current->memcg_in_oom = NULL;
1619         css_put(&memcg->css);
1620         return true;
1621 }
1622 
1623 /**
1624  * lock_page_memcg - lock a page->mem_cgroup binding
1625  * @page: the page
1626  *
1627  * This function protects unlocked LRU pages from being moved to
1628  * another cgroup.
1629  *
1630  * It ensures lifetime of the returned memcg. Caller is responsible
1631  * for the lifetime of the page; __unlock_page_memcg() is available
1632  * when @page might get freed inside the locked section.
1633  */
1634 struct mem_cgroup *lock_page_memcg(struct page *page)
1635 {
1636         struct mem_cgroup *memcg;
1637         unsigned long flags;
1638 
1639         /*
1640          * The RCU lock is held throughout the transaction.  The fast
1641          * path can get away without acquiring the memcg->move_lock
1642          * because page moving starts with an RCU grace period.
1643          *
1644          * The RCU lock also protects the memcg from being freed when
1645          * the page state that is going to change is the only thing
1646          * preventing the page itself from being freed. E.g. writeback
1647          * doesn't hold a page reference and relies on PG_writeback to
1648          * keep off truncation, migration and so forth.
1649          */
1650         rcu_read_lock();
1651 
1652         if (mem_cgroup_disabled())
1653                 return NULL;
1654 again:
1655         memcg = page->mem_cgroup;
1656         if (unlikely(!memcg))
1657                 return NULL;
1658 
1659         if (atomic_read(&memcg->moving_account) <= 0)
1660                 return memcg;
1661 
1662         spin_lock_irqsave(&memcg->move_lock, flags);
1663         if (memcg != page->mem_cgroup) {
1664                 spin_unlock_irqrestore(&memcg->move_lock, flags);
1665                 goto again;
1666         }
1667 
1668         /*
1669          * When charge migration first begins, we can have locked and
1670          * unlocked page stat updates happening concurrently.  Track
1671          * the task who has the lock for unlock_page_memcg().
1672          */
1673         memcg->move_lock_task = current;
1674         memcg->move_lock_flags = flags;
1675 
1676         return memcg;
1677 }
1678 EXPORT_SYMBOL(lock_page_memcg);
1679 
1680 /**
1681  * __unlock_page_memcg - unlock and unpin a memcg
1682  * @memcg: the memcg
1683  *
1684  * Unlock and unpin a memcg returned by lock_page_memcg().
1685  */
1686 void __unlock_page_memcg(struct mem_cgroup *memcg)
1687 {
1688         if (memcg && memcg->move_lock_task == current) {
1689                 unsigned long flags = memcg->move_lock_flags;
1690 
1691                 memcg->move_lock_task = NULL;
1692                 memcg->move_lock_flags = 0;
1693 
1694                 spin_unlock_irqrestore(&memcg->move_lock, flags);
1695         }
1696 
1697         rcu_read_unlock();
1698 }
1699 
1700 /**
1701  * unlock_page_memcg - unlock a page->mem_cgroup binding
1702  * @page: the page
1703  */
1704 void unlock_page_memcg(struct page *page)
1705 {
1706         __unlock_page_memcg(page->mem_cgroup);
1707 }
1708 EXPORT_SYMBOL(unlock_page_memcg);
1709 
1710 /*
1711  * size of first charge trial. "32" comes from vmscan.c's magic value.
1712  * TODO: maybe necessary to use big numbers in big irons.
1713  */
1714 #define CHARGE_BATCH    32U
1715 struct memcg_stock_pcp {
1716         struct mem_cgroup *cached; /* this never be root cgroup */
1717         unsigned int nr_pages;
1718         struct work_struct work;
1719         unsigned long flags;
1720 #define FLUSHING_CACHED_CHARGE  0
1721 };
1722 static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
1723 static DEFINE_MUTEX(percpu_charge_mutex);
1724 
1725 /**
1726  * consume_stock: Try to consume stocked charge on this cpu.
1727  * @memcg: memcg to consume from.
1728  * @nr_pages: how many pages to charge.
1729  *
1730  * The charges will only happen if @memcg matches the current cpu's memcg
1731  * stock, and at least @nr_pages are available in that stock.  Failure to
1732  * service an allocation will refill the stock.
1733  *
1734  * returns true if successful, false otherwise.
1735  */
1736 static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
1737 {
1738         struct memcg_stock_pcp *stock;
1739         unsigned long flags;
1740         bool ret = false;
1741 
1742         if (nr_pages > CHARGE_BATCH)
1743                 return ret;
1744 
1745         local_irq_save(flags);
1746 
1747         stock = this_cpu_ptr(&memcg_stock);
1748         if (memcg == stock->cached && stock->nr_pages >= nr_pages) {
1749                 stock->nr_pages -= nr_pages;
1750                 ret = true;
1751         }
1752 
1753         local_irq_restore(flags);
1754 
1755         return ret;
1756 }
1757 
1758 /*
1759  * Returns stocks cached in percpu and reset cached information.
1760  */
1761 static void drain_stock(struct memcg_stock_pcp *stock)
1762 {
1763         struct mem_cgroup *old = stock->cached;
1764 
1765         if (stock->nr_pages) {
1766                 page_counter_uncharge(&old->memory, stock->nr_pages);
1767                 if (do_memsw_account())
1768                         page_counter_uncharge(&old->memsw, stock->nr_pages);
1769                 css_put_many(&old->css, stock->nr_pages);
1770                 stock->nr_pages = 0;
1771         }
1772         stock->cached = NULL;
1773 }
1774 
1775 static void drain_local_stock(struct work_struct *dummy)
1776 {
1777         struct memcg_stock_pcp *stock;
1778         unsigned long flags;
1779 
1780         /*
1781          * The only protection from memory hotplug vs. drain_stock races is
1782          * that we always operate on local CPU stock here with IRQ disabled
1783          */
1784         local_irq_save(flags);
1785 
1786         stock = this_cpu_ptr(&memcg_stock);
1787         drain_stock(stock);
1788         clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
1789 
1790         local_irq_restore(flags);
1791 }
1792 
1793 /*
1794  * Cache charges(val) to local per_cpu area.
1795  * This will be consumed by consume_stock() function, later.
1796  */
1797 static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
1798 {
1799         struct memcg_stock_pcp *stock;
1800         unsigned long flags;
1801 
1802         local_irq_save(flags);
1803 
1804         stock = this_cpu_ptr(&memcg_stock);
1805         if (stock->cached != memcg) { /* reset if necessary */
1806                 drain_stock(stock);
1807                 stock->cached = memcg;
1808         }
1809         stock->nr_pages += nr_pages;
1810 
1811         if (stock->nr_pages > CHARGE_BATCH)
1812                 drain_stock(stock);
1813 
1814         local_irq_restore(flags);
1815 }
1816 
1817 /*
1818  * Drains all per-CPU charge caches for given root_memcg resp. subtree
1819  * of the hierarchy under it.
1820  */
1821 static void drain_all_stock(struct mem_cgroup *root_memcg)
1822 {
1823         int cpu, curcpu;
1824 
1825         /* If someone's already draining, avoid adding running more workers. */
1826         if (!mutex_trylock(&percpu_charge_mutex))
1827                 return;
1828         /*
1829          * Notify other cpus that system-wide "drain" is running
1830          * We do not care about races with the cpu hotplug because cpu down
1831          * as well as workers from this path always operate on the local
1832          * per-cpu data. CPU up doesn't touch memcg_stock at all.
1833          */
1834         curcpu = get_cpu();
1835         for_each_online_cpu(cpu) {
1836                 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
1837                 struct mem_cgroup *memcg;
1838 
1839                 memcg = stock->cached;
1840                 if (!memcg || !stock->nr_pages || !css_tryget(&memcg->css))
1841                         continue;
1842                 if (!mem_cgroup_is_descendant(memcg, root_memcg)) {
1843                         css_put(&memcg->css);
1844                         continue;
1845                 }
1846                 if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
1847                         if (cpu == curcpu)
1848                                 drain_local_stock(&stock->work);
1849                         else
1850                                 schedule_work_on(cpu, &stock->work);
1851                 }
1852                 css_put(&memcg->css);
1853         }
1854         put_cpu();
1855         mutex_unlock(&percpu_charge_mutex);
1856 }
1857 
1858 static int memcg_hotplug_cpu_dead(unsigned int cpu)
1859 {
1860         struct memcg_stock_pcp *stock;
1861 
1862         stock = &per_cpu(memcg_stock, cpu);
1863         drain_stock(stock);
1864         return 0;
1865 }
1866 
1867 static void reclaim_high(struct mem_cgroup *memcg,
1868                          unsigned int nr_pages,
1869                          gfp_t gfp_mask)
1870 {
1871         do {
1872                 if (page_counter_read(&memcg->memory) <= memcg->high)
1873                         continue;
1874                 mem_cgroup_event(memcg, MEMCG_HIGH);
1875                 try_to_free_mem_cgroup_pages(memcg, nr_pages, gfp_mask, true);
1876         } while ((memcg = parent_mem_cgroup(memcg)));
1877 }
1878 
1879 static void high_work_func(struct work_struct *work)
1880 {
1881         struct mem_cgroup *memcg;
1882 
1883         memcg = container_of(work, struct mem_cgroup, high_work);
1884         reclaim_high(memcg, CHARGE_BATCH, GFP_KERNEL);
1885 }
1886 
1887 /*
1888  * Scheduled by try_charge() to be executed from the userland return path
1889  * and reclaims memory over the high limit.
1890  */
1891 void mem_cgroup_handle_over_high(void)
1892 {
1893         unsigned int nr_pages = current->memcg_nr_pages_over_high;
1894         struct mem_cgroup *memcg;
1895 
1896         if (likely(!nr_pages))
1897                 return;
1898 
1899         memcg = get_mem_cgroup_from_mm(current->mm);
1900         reclaim_high(memcg, nr_pages, GFP_KERNEL);
1901         css_put(&memcg->css);
1902         current->memcg_nr_pages_over_high = 0;
1903 }
1904 
1905 static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
1906                       unsigned int nr_pages)
1907 {
1908         unsigned int batch = max(CHARGE_BATCH, nr_pages);
1909         int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
1910         struct mem_cgroup *mem_over_limit;
1911         struct page_counter *counter;
1912         unsigned long nr_reclaimed;
1913         bool may_swap = true;
1914         bool drained = false;
1915 
1916         if (mem_cgroup_is_root(memcg))
1917                 return 0;
1918 retry:
1919         if (consume_stock(memcg, nr_pages))
1920                 return 0;
1921 
1922         if (!do_memsw_account() ||
1923             page_counter_try_charge(&memcg->memsw, batch, &counter)) {
1924                 if (page_counter_try_charge(&memcg->memory, batch, &counter))
1925                         goto done_restock;
1926                 if (do_memsw_account())
1927                         page_counter_uncharge(&memcg->memsw, batch);
1928                 mem_over_limit = mem_cgroup_from_counter(counter, memory);
1929         } else {
1930                 mem_over_limit = mem_cgroup_from_counter(counter, memsw);
1931                 may_swap = false;
1932         }
1933 
1934         if (batch > nr_pages) {
1935                 batch = nr_pages;
1936                 goto retry;
1937         }
1938 
1939         /*
1940          * Unlike in global OOM situations, memcg is not in a physical
1941          * memory shortage.  Allow dying and OOM-killed tasks to
1942          * bypass the last charges so that they can exit quickly and
1943          * free their memory.
1944          */
1945         if (unlikely(tsk_is_oom_victim(current) ||
1946                      fatal_signal_pending(current) ||
1947                      current->flags & PF_EXITING))
1948                 goto force;
1949 
1950         /*
1951          * Prevent unbounded recursion when reclaim operations need to
1952          * allocate memory. This might exceed the limits temporarily,
1953          * but we prefer facilitating memory reclaim and getting back
1954          * under the limit over triggering OOM kills in these cases.
1955          */
1956         if (unlikely(current->flags & PF_MEMALLOC))
1957                 goto force;
1958 
1959         if (unlikely(task_in_memcg_oom(current)))
1960                 goto nomem;
1961 
1962         if (!gfpflags_allow_blocking(gfp_mask))
1963                 goto nomem;
1964 
1965         mem_cgroup_event(mem_over_limit, MEMCG_MAX);
1966 
1967         nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages,
1968                                                     gfp_mask, may_swap);
1969 
1970         if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
1971                 goto retry;
1972 
1973         if (!drained) {
1974                 drain_all_stock(mem_over_limit);
1975                 drained = true;
1976                 goto retry;
1977         }
1978 
1979         if (gfp_mask & __GFP_NORETRY)
1980                 goto nomem;
1981         /*
1982          * Even though the limit is exceeded at this point, reclaim
1983          * may have been able to free some pages.  Retry the charge
1984          * before killing the task.
1985          *
1986          * Only for regular pages, though: huge pages are rather
1987          * unlikely to succeed so close to the limit, and we fall back
1988          * to regular pages anyway in case of failure.
1989          */
1990         if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER))
1991                 goto retry;
1992         /*
1993          * At task move, charge accounts can be doubly counted. So, it's
1994          * better to wait until the end of task_move if something is going on.
1995          */
1996         if (mem_cgroup_wait_acct_move(mem_over_limit))
1997                 goto retry;
1998 
1999         if (nr_retries--)
2000                 goto retry;
2001 
2002         if (gfp_mask & __GFP_NOFAIL)
2003                 goto force;
2004 
2005         if (fatal_signal_pending(current))
2006                 goto force;
2007 
2008         mem_cgroup_event(mem_over_limit, MEMCG_OOM);
2009 
2010         mem_cgroup_oom(mem_over_limit, gfp_mask,
2011                        get_order(nr_pages * PAGE_SIZE));
2012 nomem:
2013         if (!(gfp_mask & __GFP_NOFAIL))
2014                 return -ENOMEM;
2015 force:
2016         /*
2017          * The allocation either can't fail or will lead to more memory
2018          * being freed very soon.  Allow memory usage go over the limit
2019          * temporarily by force charging it.
2020          */
2021         page_counter_charge(&memcg->memory, nr_pages);
2022         if (do_memsw_account())
2023                 page_counter_charge(&memcg->memsw, nr_pages);
2024         css_get_many(&memcg->css, nr_pages);
2025 
2026         return 0;
2027 
2028 done_restock:
2029         css_get_many(&memcg->css, batch);
2030         if (batch > nr_pages)
2031                 refill_stock(memcg, batch - nr_pages);
2032 
2033         /*
2034          * If the hierarchy is above the normal consumption range, schedule
2035          * reclaim on returning to userland.  We can perform reclaim here
2036          * if __GFP_RECLAIM but let's always punt for simplicity and so that
2037          * GFP_KERNEL can consistently be used during reclaim.  @memcg is
2038          * not recorded as it most likely matches current's and won't
2039          * change in the meantime.  As high limit is checked again before
2040          * reclaim, the cost of mismatch is negligible.
2041          */
2042         do {
2043                 if (page_counter_read(&memcg->memory) > memcg->high) {
2044                         /* Don't bother a random interrupted task */
2045                         if (in_interrupt()) {
2046                                 schedule_work(&memcg->high_work);
2047                                 break;
2048                         }
2049                         current->memcg_nr_pages_over_high += batch;
2050                         set_notify_resume(current);
2051                         break;
2052                 }
2053         } while ((memcg = parent_mem_cgroup(memcg)));
2054 
2055         return 0;
2056 }
2057 
2058 static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
2059 {
2060         if (mem_cgroup_is_root(memcg))
2061                 return;
2062 
2063         page_counter_uncharge(&memcg->memory, nr_pages);
2064         if (do_memsw_account())
2065                 page_counter_uncharge(&memcg->memsw, nr_pages);
2066 
2067         css_put_many(&memcg->css, nr_pages);
2068 }
2069 
2070 static void lock_page_lru(struct page *page, int *isolated)
2071 {
2072         struct zone *zone = page_zone(page);
2073 
2074         spin_lock_irq(zone_lru_lock(zone));
2075         if (PageLRU(page)) {
2076                 struct lruvec *lruvec;
2077 
2078                 lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);
2079                 ClearPageLRU(page);
2080                 del_page_from_lru_list(page, lruvec, page_lru(page));
2081                 *isolated = 1;
2082         } else
2083                 *isolated = 0;
2084 }
2085 
2086 static void unlock_page_lru(struct page *page, int isolated)
2087 {
2088         struct zone *zone = page_zone(page);
2089 
2090         if (isolated) {
2091                 struct lruvec *lruvec;
2092 
2093                 lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);
2094                 VM_BUG_ON_PAGE(PageLRU(page), page);
2095                 SetPageLRU(page);
2096                 add_page_to_lru_list(page, lruvec, page_lru(page));
2097         }
2098         spin_unlock_irq(zone_lru_lock(zone));
2099 }
2100 
2101 static void commit_charge(struct page *page, struct mem_cgroup *memcg,
2102                           bool lrucare)
2103 {
2104         int isolated;
2105 
2106         VM_BUG_ON_PAGE(page->mem_cgroup, page);
2107 
2108         /*
2109          * In some cases, SwapCache and FUSE(splice_buf->radixtree), the page
2110          * may already be on some other mem_cgroup's LRU.  Take care of it.
2111          */
2112         if (lrucare)
2113                 lock_page_lru(page, &isolated);
2114 
2115         /*
2116          * Nobody should be changing or seriously looking at
2117          * page->mem_cgroup at this point:
2118          *
2119          * - the page is uncharged
2120          *
2121          * - the page is off-LRU
2122          *
2123          * - an anonymous fault has exclusive page access, except for
2124          *   a locked page table
2125          *
2126          * - a page cache insertion, a swapin fault, or a migration
2127          *   have the page locked
2128          */
2129         page->mem_cgroup = memcg;
2130 
2131         if (lrucare)
2132                 unlock_page_lru(page, isolated);
2133 }
2134 
2135 #ifndef CONFIG_SLOB
2136 static int memcg_alloc_cache_id(void)
2137 {
2138         int id, size;
2139         int err;
2140 
2141         id = ida_simple_get(&memcg_cache_ida,
2142                             0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
2143         if (id < 0)
2144                 return id;
2145 
2146         if (id < memcg_nr_cache_ids)
2147                 return id;
2148 
2149         /*
2150          * There's no space for the new id in memcg_caches arrays,
2151          * so we have to grow them.
2152          */
2153         down_write(&memcg_cache_ids_sem);
2154 
2155         size = 2 * (id + 1);
2156         if (size < MEMCG_CACHES_MIN_SIZE)
2157                 size = MEMCG_CACHES_MIN_SIZE;
2158         else if (size > MEMCG_CACHES_MAX_SIZE)
2159                 size = MEMCG_CACHES_MAX_SIZE;
2160 
2161         err = memcg_update_all_caches(size);
2162         if (!err)
2163                 err = memcg_update_all_list_lrus(size);
2164         if (!err)
2165                 memcg_nr_cache_ids = size;
2166 
2167         up_write(&memcg_cache_ids_sem);
2168 
2169         if (err) {
2170                 ida_simple_remove(&memcg_cache_ida, id);
2171                 return err;
2172         }
2173         return id;
2174 }
2175 
2176 static void memcg_free_cache_id(int id)
2177 {
2178         ida_simple_remove(&memcg_cache_ida, id);
2179 }
2180 
2181 struct memcg_kmem_cache_create_work {
2182         struct mem_cgroup *memcg;
2183         struct kmem_cache *cachep;
2184         struct work_struct work;
2185 };
2186 
2187 static void memcg_kmem_cache_create_func(struct work_struct *w)
2188 {
2189         struct memcg_kmem_cache_create_work *cw =
2190                 container_of(w, struct memcg_kmem_cache_create_work, work);
2191         struct mem_cgroup *memcg = cw->memcg;
2192         struct kmem_cache *cachep = cw->cachep;
2193 
2194         memcg_create_kmem_cache(memcg, cachep);
2195 
2196         css_put(&memcg->css);
2197         kfree(cw);
2198 }
2199 
2200 /*
2201  * Enqueue the creation of a per-memcg kmem_cache.
2202  */
2203 static void __memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,
2204                                                struct kmem_cache *cachep)
2205 {
2206         struct memcg_kmem_cache_create_work *cw;
2207 
2208         cw = kmalloc(sizeof(*cw), GFP_NOWAIT);
2209         if (!cw)
2210                 return;
2211 
2212         css_get(&memcg->css);
2213 
2214         cw->memcg = memcg;
2215         cw->cachep = cachep;
2216         INIT_WORK(&cw->work, memcg_kmem_cache_create_func);
2217 
2218         queue_work(memcg_kmem_cache_wq, &cw->work);
2219 }
2220 
2221 static void memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,
2222                                              struct kmem_cache *cachep)
2223 {
2224         /*
2225          * We need to stop accounting when we kmalloc, because if the
2226          * corresponding kmalloc cache is not yet created, the first allocation
2227          * in __memcg_schedule_kmem_cache_create will recurse.
2228          *
2229          * However, it is better to enclose the whole function. Depending on
2230          * the debugging options enabled, INIT_WORK(), for instance, can
2231          * trigger an allocation. This too, will make us recurse. Because at
2232          * this point we can't allow ourselves back into memcg_kmem_get_cache,
2233          * the safest choice is to do it like this, wrapping the whole function.
2234          */
2235         current->memcg_kmem_skip_account = 1;
2236         __memcg_schedule_kmem_cache_create(memcg, cachep);
2237         current->memcg_kmem_skip_account = 0;
2238 }
2239 
2240 static inline bool memcg_kmem_bypass(void)
2241 {
2242         if (in_interrupt() || !current->mm || (current->flags & PF_KTHREAD))
2243                 return true;
2244         return false;
2245 }
2246 
2247 /**
2248  * memcg_kmem_get_cache: select the correct per-memcg cache for allocation
2249  * @cachep: the original global kmem cache
2250  *
2251  * Return the kmem_cache we're supposed to use for a slab allocation.
2252  * We try to use the current memcg's version of the cache.
2253  *
2254  * If the cache does not exist yet, if we are the first user of it, we
2255  * create it asynchronously in a workqueue and let the current allocation
2256  * go through with the original cache.
2257  *
2258  * This function takes a reference to the cache it returns to assure it
2259  * won't get destroyed while we are working with it. Once the caller is
2260  * done with it, memcg_kmem_put_cache() must be called to release the
2261  * reference.
2262  */
2263 struct kmem_cache *memcg_kmem_get_cache(struct kmem_cache *cachep)
2264 {
2265         struct mem_cgroup *memcg;
2266         struct kmem_cache *memcg_cachep;
2267         int kmemcg_id;
2268 
2269         VM_BUG_ON(!is_root_cache(cachep));
2270 
2271         if (memcg_kmem_bypass())
2272                 return cachep;
2273 
2274         if (current->memcg_kmem_skip_account)
2275                 return cachep;
2276 
2277         memcg = get_mem_cgroup_from_mm(current->mm);
2278         kmemcg_id = READ_ONCE(memcg->kmemcg_id);
2279         if (kmemcg_id < 0)
2280                 goto out;
2281 
2282         memcg_cachep = cache_from_memcg_idx(cachep, kmemcg_id);
2283         if (likely(memcg_cachep))
2284                 return memcg_cachep;
2285 
2286         /*
2287          * If we are in a safe context (can wait, and not in interrupt
2288          * context), we could be be predictable and return right away.
2289          * This would guarantee that the allocation being performed
2290          * already belongs in the new cache.
2291          *
2292          * However, there are some clashes that can arrive from locking.
2293          * For instance, because we acquire the slab_mutex while doing
2294          * memcg_create_kmem_cache, this means no further allocation
2295          * could happen with the slab_mutex held. So it's better to
2296          * defer everything.
2297          */
2298         memcg_schedule_kmem_cache_create(memcg, cachep);
2299 out:
2300         css_put(&memcg->css);
2301         return cachep;
2302 }
2303 
2304 /**
2305  * memcg_kmem_put_cache: drop reference taken by memcg_kmem_get_cache
2306  * @cachep: the cache returned by memcg_kmem_get_cache
2307  */
2308 void memcg_kmem_put_cache(struct kmem_cache *cachep)
2309 {
2310         if (!is_root_cache(cachep))
2311                 css_put(&cachep->memcg_params.memcg->css);
2312 }
2313 
2314 /**
2315  * memcg_kmem_charge: charge a kmem page
2316  * @page: page to charge
2317  * @gfp: reclaim mode
2318  * @order: allocation order
2319  * @memcg: memory cgroup to charge
2320  *
2321  * Returns 0 on success, an error code on failure.
2322  */
2323 int memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order,
2324                             struct mem_cgroup *memcg)
2325 {
2326         unsigned int nr_pages = 1 << order;
2327         struct page_counter *counter;
2328         int ret;
2329 
2330         ret = try_charge(memcg, gfp, nr_pages);
2331         if (ret)
2332                 return ret;
2333 
2334         if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
2335             !page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) {
2336                 cancel_charge(memcg, nr_pages);
2337                 return -ENOMEM;
2338         }
2339 
2340         page->mem_cgroup = memcg;
2341 
2342         return 0;
2343 }
2344 
2345 /**
2346  * memcg_kmem_charge: charge a kmem page to the current memory cgroup
2347  * @page: page to charge
2348  * @gfp: reclaim mode
2349  * @order: allocation order
2350  *
2351  * Returns 0 on success, an error code on failure.
2352  */
2353 int memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
2354 {
2355         struct mem_cgroup *memcg;
2356         int ret = 0;
2357 
2358         if (memcg_kmem_bypass())
2359                 return 0;
2360 
2361         memcg = get_mem_cgroup_from_mm(current->mm);
2362         if (!mem_cgroup_is_root(memcg)) {
2363                 ret = memcg_kmem_charge_memcg(page, gfp, order, memcg);
2364                 if (!ret)
2365                         __SetPageKmemcg(page);
2366         }
2367         css_put(&memcg->css);
2368         return ret;
2369 }
2370 /**
2371  * memcg_kmem_uncharge: uncharge a kmem page
2372  * @page: page to uncharge
2373  * @order: allocation order
2374  */
2375 void memcg_kmem_uncharge(struct page *page, int order)
2376 {
2377         struct mem_cgroup *memcg = page->mem_cgroup;
2378         unsigned int nr_pages = 1 << order;
2379 
2380         if (!memcg)
2381                 return;
2382 
2383         VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page);
2384 
2385         if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
2386                 page_counter_uncharge(&memcg->kmem, nr_pages);
2387 
2388         page_counter_uncharge(&memcg->memory, nr_pages);
2389         if (do_memsw_account())
2390                 page_counter_uncharge(&memcg->memsw, nr_pages);
2391 
2392         page->mem_cgroup = NULL;
2393 
2394         /* slab pages do not have PageKmemcg flag set */
2395         if (PageKmemcg(page))
2396                 __ClearPageKmemcg(page);
2397 
2398         css_put_many(&memcg->css, nr_pages);
2399 }
2400 #endif /* !CONFIG_SLOB */
2401 
2402 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
2403 
2404 /*
2405  * Because tail pages are not marked as "used", set it. We're under
2406  * zone_lru_lock and migration entries setup in all page mappings.
2407  */
2408 void mem_cgroup_split_huge_fixup(struct page *head)
2409 {
2410         int i;
2411 
2412         if (mem_cgroup_disabled())
2413                 return;
2414 
2415         for (i = 1; i < HPAGE_PMD_NR; i++)
2416                 head[i].mem_cgroup = head->mem_cgroup;
2417 
2418         __this_cpu_sub(head->mem_cgroup->stat->count[MEMCG_RSS_HUGE],
2419                        HPAGE_PMD_NR);
2420 }
2421 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
2422 
2423 #ifdef CONFIG_MEMCG_SWAP
2424 static void mem_cgroup_swap_statistics(struct mem_cgroup *memcg,
2425                                        int nr_entries)
2426 {
2427         this_cpu_add(memcg->stat->count[MEMCG_SWAP], nr_entries);
2428 }
2429 
2430 /**
2431  * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
2432  * @entry: swap entry to be moved
2433  * @from:  mem_cgroup which the entry is moved from
2434  * @to:  mem_cgroup which the entry is moved to
2435  *
2436  * It succeeds only when the swap_cgroup's record for this entry is the same
2437  * as the mem_cgroup's id of @from.
2438  *
2439  * Returns 0 on success, -EINVAL on failure.
2440  *
2441  * The caller must have charged to @to, IOW, called page_counter_charge() about
2442  * both res and memsw, and called css_get().
2443  */
2444 static int mem_cgroup_move_swap_account(swp_entry_t entry,
2445                                 struct mem_cgroup *from, struct mem_cgroup *to)
2446 {
2447         unsigned short old_id, new_id;
2448 
2449         old_id = mem_cgroup_id(from);
2450         new_id = mem_cgroup_id(to);
2451 
2452         if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
2453                 mem_cgroup_swap_statistics(from, -1);
2454                 mem_cgroup_swap_statistics(to, 1);
2455                 return 0;
2456         }
2457         return -EINVAL;
2458 }
2459 #else
2460 static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
2461                                 struct mem_cgroup *from, struct mem_cgroup *to)
2462 {
2463         return -EINVAL;
2464 }
2465 #endif
2466 
2467 static DEFINE_MUTEX(memcg_limit_mutex);
2468 
2469 static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
2470                                    unsigned long limit)
2471 {
2472         unsigned long curusage;
2473         unsigned long oldusage;
2474         bool enlarge = false;
2475         int retry_count;
2476         int ret;
2477 
2478         /*
2479          * For keeping hierarchical_reclaim simple, how long we should retry
2480          * is depends on callers. We set our retry-count to be function
2481          * of # of children which we should visit in this loop.
2482          */
2483         retry_count = MEM_CGROUP_RECLAIM_RETRIES *
2484                       mem_cgroup_count_children(memcg);
2485 
2486         oldusage = page_counter_read(&memcg->memory);
2487 
2488         do {
2489                 if (signal_pending(current)) {
2490                         ret = -EINTR;
2491                         break;
2492                 }
2493 
2494                 mutex_lock(&memcg_limit_mutex);
2495                 if (limit > memcg->memsw.limit) {
2496                         mutex_unlock(&memcg_limit_mutex);
2497                         ret = -EINVAL;
2498                         break;
2499                 }
2500                 if (limit > memcg->memory.limit)
2501                         enlarge = true;
2502                 ret = page_counter_limit(&memcg->memory, limit);
2503                 mutex_unlock(&memcg_limit_mutex);
2504 
2505                 if (!ret)
2506                         break;
2507 
2508                 try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, true);
2509 
2510                 curusage = page_counter_read(&memcg->memory);
2511                 /* Usage is reduced ? */
2512                 if (curusage >= oldusage)
2513                         retry_count--;
2514                 else
2515                         oldusage = curusage;
2516         } while (retry_count);
2517 
2518         if (!ret && enlarge)
2519                 memcg_oom_recover(memcg);
2520 
2521         return ret;
2522 }
2523 
2524 static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
2525                                          unsigned long limit)
2526 {
2527         unsigned long curusage;
2528         unsigned long oldusage;
2529         bool enlarge = false;
2530         int retry_count;
2531         int ret;
2532 
2533         /* see mem_cgroup_resize_res_limit */
2534         retry_count = MEM_CGROUP_RECLAIM_RETRIES *
2535                       mem_cgroup_count_children(memcg);
2536 
2537         oldusage = page_counter_read(&memcg->memsw);
2538 
2539         do {
2540                 if (signal_pending(current)) {
2541                         ret = -EINTR;
2542                         break;
2543                 }
2544 
2545                 mutex_lock(&memcg_limit_mutex);
2546                 if (limit < memcg->memory.limit) {
2547                         mutex_unlock(&memcg_limit_mutex);
2548                         ret = -EINVAL;
2549                         break;
2550                 }
2551                 if (limit > memcg->memsw.limit)
2552                         enlarge = true;
2553                 ret = page_counter_limit(&memcg->memsw, limit);
2554                 mutex_unlock(&memcg_limit_mutex);
2555 
2556                 if (!ret)
2557                         break;
2558 
2559                 try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, false);
2560 
2561                 curusage = page_counter_read(&memcg->memsw);
2562                 /* Usage is reduced ? */
2563                 if (curusage >= oldusage)
2564                         retry_count--;
2565                 else
2566                         oldusage = curusage;
2567         } while (retry_count);
2568 
2569         if (!ret && enlarge)
2570                 memcg_oom_recover(memcg);
2571 
2572         return ret;
2573 }
2574 
2575 unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
2576                                             gfp_t gfp_mask,
2577                                             unsigned long *total_scanned)
2578 {
2579         unsigned long nr_reclaimed = 0;
2580         struct mem_cgroup_per_node *mz, *next_mz = NULL;
2581         unsigned long reclaimed;
2582         int loop = 0;
2583         struct mem_cgroup_tree_per_node *mctz;
2584         unsigned long excess;
2585         unsigned long nr_scanned;
2586 
2587         if (order > 0)
2588                 return 0;
2589 
2590         mctz = soft_limit_tree_node(pgdat->node_id);
2591 
2592         /*
2593          * Do not even bother to check the largest node if the root
2594          * is empty. Do it lockless to prevent lock bouncing. Races
2595          * are acceptable as soft limit is best effort anyway.
2596          */
2597         if (!mctz || RB_EMPTY_ROOT(&mctz->rb_root))
2598                 return 0;
2599 
2600         /*
2601          * This loop can run a while, specially if mem_cgroup's continuously
2602          * keep exceeding their soft limit and putting the system under
2603          * pressure
2604          */
2605         do {
2606                 if (next_mz)
2607                         mz = next_mz;
2608                 else
2609                         mz = mem_cgroup_largest_soft_limit_node(mctz);
2610                 if (!mz)
2611                         break;
2612 
2613                 nr_scanned = 0;
2614                 reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat,
2615                                                     gfp_mask, &nr_scanned);
2616                 nr_reclaimed += reclaimed;
2617                 *total_scanned += nr_scanned;
2618                 spin_lock_irq(&mctz->lock);
2619                 __mem_cgroup_remove_exceeded(mz, mctz);
2620 
2621                 /*
2622                  * If we failed to reclaim anything from this memory cgroup
2623                  * it is time to move on to the next cgroup
2624                  */
2625                 next_mz = NULL;
2626                 if (!reclaimed)
2627                         next_mz = __mem_cgroup_largest_soft_limit_node(mctz);
2628 
2629                 excess = soft_limit_excess(mz->memcg);
2630                 /*
2631                  * One school of thought says that we should not add
2632                  * back the node to the tree if reclaim returns 0.
2633                  * But our reclaim could return 0, simply because due
2634                  * to priority we are exposing a smaller subset of
2635                  * memory to reclaim from. Consider this as a longer
2636                  * term TODO.
2637                  */
2638                 /* If excess == 0, no tree ops */
2639                 __mem_cgroup_insert_exceeded(mz, mctz, excess);
2640                 spin_unlock_irq(&mctz->lock);
2641                 css_put(&mz->memcg->css);
2642                 loop++;
2643                 /*
2644                  * Could not reclaim anything and there are no more
2645                  * mem cgroups to try or we seem to be looping without
2646                  * reclaiming anything.
2647                  */
2648                 if (!nr_reclaimed &&
2649                         (next_mz == NULL ||
2650                         loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
2651                         break;
2652         } while (!nr_reclaimed);
2653         if (next_mz)
2654                 css_put(&next_mz->memcg->css);
2655         return nr_reclaimed;
2656 }
2657 
2658 /*
2659  * Test whether @memcg has children, dead or alive.  Note that this
2660  * function doesn't care whether @memcg has use_hierarchy enabled and
2661  * returns %true if there are child csses according to the cgroup
2662  * hierarchy.  Testing use_hierarchy is the caller's responsiblity.
2663  */
2664 static inline bool memcg_has_children(struct mem_cgroup *memcg)
2665 {
2666         bool ret;
2667 
2668         rcu_read_lock();
2669         ret = css_next_child(NULL, &memcg->css);
2670         rcu_read_unlock();
2671         return ret;
2672 }
2673 
2674 /*
2675  * Reclaims as many pages from the given memcg as possible.
2676  *
2677  * Caller is responsible for holding css reference for memcg.
2678  */
2679 static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
2680 {
2681         int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
2682 
2683         /* we call try-to-free pages for make this cgroup empty */
2684         lru_add_drain_all();
2685         /* try to free all pages in this cgroup */
2686         while (nr_retries && page_counter_read(&memcg->memory)) {
2687                 int progress;
2688 
2689                 if (signal_pending(current))
2690                         return -EINTR;
2691 
2692                 progress = try_to_free_mem_cgroup_pages(memcg, 1,
2693                                                         GFP_KERNEL, true);
2694                 if (!progress) {
2695                         nr_retries--;
2696                         /* maybe some writeback is necessary */
2697                         congestion_wait(BLK_RW_ASYNC, HZ/10);
2698                 }
2699 
2700         }
2701 
2702         return 0;
2703 }
2704 
2705 static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of,
2706                                             char *buf, size_t nbytes,
2707                                             loff_t off)
2708 {
2709         struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
2710 
2711         if (mem_cgroup_is_root(memcg))
2712                 return -EINVAL;
2713         return mem_cgroup_force_empty(memcg) ?: nbytes;
2714 }
2715 
2716 static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css,
2717                                      struct cftype *cft)
2718 {
2719         return mem_cgroup_from_css(css)->use_hierarchy;
2720 }
2721 
2722 static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
2723                                       struct cftype *cft, u64 val)
2724 {
2725         int retval = 0;
2726         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
2727         struct mem_cgroup *parent_memcg = mem_cgroup_from_css(memcg->css.parent);
2728 
2729         if (memcg->use_hierarchy == val)
2730                 return 0;
2731 
2732         /*
2733          * If parent's use_hierarchy is set, we can't make any modifications
2734          * in the child subtrees. If it is unset, then the change can
2735          * occur, provided the current cgroup has no children.
2736          *
2737          * For the root cgroup, parent_mem is NULL, we allow value to be
2738          * set if there are no children.
2739          */
2740         if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
2741                                 (val == 1 || val == 0)) {
2742                 if (!memcg_has_children(memcg))
2743                         memcg->use_hierarchy = val;
2744                 else
2745                         retval = -EBUSY;
2746         } else
2747                 retval = -EINVAL;
2748 
2749         return retval;
2750 }
2751 
2752 static void tree_stat(struct mem_cgroup *memcg, unsigned long *stat)
2753 {
2754         struct mem_cgroup *iter;
2755         int i;
2756 
2757         memset(stat, 0, sizeof(*stat) * MEMCG_NR_STAT);
2758 
2759         for_each_mem_cgroup_tree(iter, memcg) {
2760                 for (i = 0; i < MEMCG_NR_STAT; i++)
2761                         stat[i] += memcg_page_state(iter, i);
2762         }
2763 }
2764 
2765 static void tree_events(struct mem_cgroup *memcg, unsigned long *events)
2766 {
2767         struct mem_cgroup *iter;
2768         int i;
2769 
2770         memset(events, 0, sizeof(*events) * MEMCG_NR_EVENTS);
2771 
2772         for_each_mem_cgroup_tree(iter, memcg) {
2773                 for (i = 0; i < MEMCG_NR_EVENTS; i++)
2774                         events[i] += memcg_sum_events(iter, i);
2775         }
2776 }
2777 
2778 static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
2779 {
2780         unsigned long val = 0;
2781 
2782         if (mem_cgroup_is_root(memcg)) {
2783                 struct mem_cgroup *iter;
2784 
2785                 for_each_mem_cgroup_tree(iter, memcg) {
2786                         val += memcg_page_state(iter, MEMCG_CACHE);
2787                         val += memcg_page_state(iter, MEMCG_RSS);
2788                         if (swap)
2789                                 val += memcg_page_state(iter, MEMCG_SWAP);
2790                 }
2791         } else {
2792                 if (!swap)
2793                         val = page_counter_read(&memcg->memory);
2794                 else
2795                         val = page_counter_read(&memcg->memsw);
2796         }
2797         return val;
2798 }
2799 
2800 enum {
2801         RES_USAGE,
2802         RES_LIMIT,
2803         RES_MAX_USAGE,
2804         RES_FAILCNT,
2805         RES_SOFT_LIMIT,
2806 };
2807 
2808 static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
2809                                struct cftype *cft)
2810 {
2811         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
2812         struct page_counter *counter;
2813 
2814         switch (MEMFILE_TYPE(cft->private)) {
2815         case _MEM:
2816                 counter = &memcg->memory;
2817                 break;
2818         case _MEMSWAP:
2819                 counter = &memcg->memsw;
2820                 break;
2821         case _KMEM:
2822                 counter = &memcg->kmem;
2823                 break;
2824         case _TCP:
2825                 counter = &memcg->tcpmem;
2826                 break;
2827         default:
2828                 BUG();
2829         }
2830 
2831         switch (MEMFILE_ATTR(cft->private)) {
2832         case RES_USAGE:
2833                 if (counter == &memcg->memory)
2834                         return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE;
2835                 if (counter == &memcg->memsw)
2836                         return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE;
2837                 return (u64)page_counter_read(counter) * PAGE_SIZE;
2838         case RES_LIMIT:
2839                 return (u64)counter->limit * PAGE_SIZE;
2840         case RES_MAX_USAGE:
2841                 return (u64)counter->watermark * PAGE_SIZE;
2842         case RES_FAILCNT:
2843                 return counter->failcnt;
2844         case RES_SOFT_LIMIT:
2845                 return (u64)memcg->soft_limit * PAGE_SIZE;
2846         default:
2847                 BUG();
2848         }
2849 }
2850 
2851 #ifndef CONFIG_SLOB
2852 static int memcg_online_kmem(struct mem_cgroup *memcg)
2853 {
2854         int memcg_id;
2855 
2856         if (cgroup_memory_nokmem)
2857                 return 0;
2858 
2859         BUG_ON(memcg->kmemcg_id >= 0);
2860         BUG_ON(memcg->kmem_state);
2861 
2862         memcg_id = memcg_alloc_cache_id();
2863         if (memcg_id < 0)
2864                 return memcg_id;
2865 
2866         static_branch_inc(&memcg_kmem_enabled_key);
2867         /*
2868          * A memory cgroup is considered kmem-online as soon as it gets
2869          * kmemcg_id. Setting the id after enabling static branching will
2870          * guarantee no one starts accounting before all call sites are
2871          * patched.
2872          */
2873         memcg->kmemcg_id = memcg_id;
2874         memcg->kmem_state = KMEM_ONLINE;
2875         INIT_LIST_HEAD(&memcg->kmem_caches);
2876 
2877         return 0;
2878 }
2879 
2880 static void memcg_offline_kmem(struct mem_cgroup *memcg)
2881 {
2882         struct cgroup_subsys_state *css;
2883         struct mem_cgroup *parent, *child;
2884         int kmemcg_id;
2885 
2886         if (memcg->kmem_state != KMEM_ONLINE)
2887                 return;
2888         /*
2889          * Clear the online state before clearing memcg_caches array
2890          * entries. The slab_mutex in memcg_deactivate_kmem_caches()
2891          * guarantees that no cache will be created for this cgroup
2892          * after we are done (see memcg_create_kmem_cache()).
2893          */
2894         memcg->kmem_state = KMEM_ALLOCATED;
2895 
2896         memcg_deactivate_kmem_caches(memcg);
2897 
2898         kmemcg_id = memcg->kmemcg_id;
2899         BUG_ON(kmemcg_id < 0);
2900 
2901         parent = parent_mem_cgroup(memcg);
2902         if (!parent)
2903                 parent = root_mem_cgroup;
2904 
2905         /*
2906          * Change kmemcg_id of this cgroup and all its descendants to the
2907          * parent's id, and then move all entries from this cgroup's list_lrus
2908          * to ones of the parent. After we have finished, all list_lrus
2909          * corresponding to this cgroup are guaranteed to remain empty. The
2910          * ordering is imposed by list_lru_node->lock taken by
2911          * memcg_drain_all_list_lrus().
2912          */
2913         rcu_read_lock(); /* can be called from css_free w/o cgroup_mutex */
2914         css_for_each_descendant_pre(css, &memcg->css) {
2915                 child = mem_cgroup_from_css(css);
2916                 BUG_ON(child->kmemcg_id != kmemcg_id);
2917                 child->kmemcg_id = parent->kmemcg_id;
2918                 if (!memcg->use_hierarchy)
2919                         break;
2920         }
2921         rcu_read_unlock();
2922 
2923         memcg_drain_all_list_lrus(kmemcg_id, parent->kmemcg_id);
2924 
2925         memcg_free_cache_id(kmemcg_id);
2926 }
2927 
2928 static void memcg_free_kmem(struct mem_cgroup *memcg)
2929 {
2930         /* css_alloc() failed, offlining didn't happen */
2931         if (unlikely(memcg->kmem_state == KMEM_ONLINE))
2932                 memcg_offline_kmem(memcg);
2933 
2934         if (memcg->kmem_state == KMEM_ALLOCATED) {
2935                 memcg_destroy_kmem_caches(memcg);
2936                 static_branch_dec(&memcg_kmem_enabled_key);
2937                 WARN_ON(page_counter_read(&memcg->kmem));
2938         }
2939 }
2940 #else
2941 static int memcg_online_kmem(struct mem_cgroup *memcg)
2942 {
2943         return 0;
2944 }
2945 static void memcg_offline_kmem(struct mem_cgroup *memcg)
2946 {
2947 }
2948 static void memcg_free_kmem(struct mem_cgroup *memcg)
2949 {
2950 }
2951 #endif /* !CONFIG_SLOB */
2952 
2953 static int memcg_update_kmem_limit(struct mem_cgroup *memcg,
2954                                    unsigned long limit)
2955 {
2956         int ret;
2957 
2958         mutex_lock(&memcg_limit_mutex);
2959         ret = page_counter_limit(&memcg->kmem, limit);
2960         mutex_unlock(&memcg_limit_mutex);
2961         return ret;
2962 }
2963 
2964 static int memcg_update_tcp_limit(struct mem_cgroup *memcg, unsigned long limit)
2965 {
2966         int ret;
2967 
2968         mutex_lock(&memcg_limit_mutex);
2969 
2970         ret = page_counter_limit(&memcg->tcpmem, limit);
2971         if (ret)
2972                 goto out;
2973 
2974         if (!memcg->tcpmem_active) {
2975                 /*
2976                  * The active flag needs to be written after the static_key
2977                  * update. This is what guarantees that the socket activation
2978                  * function is the last one to run. See mem_cgroup_sk_alloc()
2979                  * for details, and note that we don't mark any socket as
2980                  * belonging to this memcg until that flag is up.
2981                  *
2982                  * We need to do this, because static_keys will span multiple
2983                  * sites, but we can't control their order. If we mark a socket
2984                  * as accounted, but the accounting functions are not patched in
2985                  * yet, we'll lose accounting.
2986                  *
2987                  * We never race with the readers in mem_cgroup_sk_alloc(),
2988                  * because when this value change, the code to process it is not
2989                  * patched in yet.
2990                  */
2991                 static_branch_inc(&memcg_sockets_enabled_key);
2992                 memcg->tcpmem_active = true;
2993         }
2994 out:
2995         mutex_unlock(&memcg_limit_mutex);
2996         return ret;
2997 }
2998 
2999 /*
3000  * The user of this function is...
3001  * RES_LIMIT.
3002  */
3003 static ssize_t mem_cgroup_write(struct kernfs_open_file *of,
3004                                 char *buf, size_t nbytes, loff_t off)
3005 {
3006         struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3007         unsigned long nr_pages;
3008         int ret;
3009 
3010         buf = strstrip(buf);
3011         ret = page_counter_memparse(buf, "-1", &nr_pages);
3012         if (ret)
3013                 return ret;
3014 
3015         switch (MEMFILE_ATTR(of_cft(of)->private)) {
3016         case RES_LIMIT:
3017                 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
3018                         ret = -EINVAL;
3019                         break;
3020                 }
3021                 switch (MEMFILE_TYPE(of_cft(of)->private)) {
3022                 case _MEM:
3023                         ret = mem_cgroup_resize_limit(memcg, nr_pages);
3024                         break;
3025                 case _MEMSWAP:
3026                         ret = mem_cgroup_resize_memsw_limit(memcg, nr_pages);
3027                         break;
3028                 case _KMEM:
3029                         ret = memcg_update_kmem_limit(memcg, nr_pages);
3030                         break;
3031                 case _TCP:
3032                         ret = memcg_update_tcp_limit(memcg, nr_pages);
3033                         break;
3034                 }
3035                 break;
3036         case RES_SOFT_LIMIT:
3037                 memcg->soft_limit = nr_pages;
3038                 ret = 0;
3039                 break;
3040         }
3041         return ret ?: nbytes;
3042 }
3043 
3044 static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf,
3045                                 size_t nbytes, loff_t off)
3046 {
3047         struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3048         struct page_counter *counter;
3049 
3050         switch (MEMFILE_TYPE(of_cft(of)->private)) {
3051         case _MEM:
3052                 counter = &memcg->memory;
3053                 break;
3054         case _MEMSWAP:
3055                 counter = &memcg->memsw;
3056                 break;
3057         case _KMEM:
3058                 counter = &memcg->kmem;
3059                 break;
3060         case _TCP:
3061                 counter = &memcg->tcpmem;
3062                 break;
3063         default:
3064                 BUG();
3065         }
3066 
3067         switch (MEMFILE_ATTR(of_cft(of)->private)) {
3068         case RES_MAX_USAGE:
3069                 page_counter_reset_watermark(counter);
3070                 break;
3071         case RES_FAILCNT:
3072                 counter->failcnt = 0;
3073                 break;
3074         default:
3075                 BUG();
3076         }
3077 
3078         return nbytes;
3079 }
3080 
3081 static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
3082                                         struct cftype *cft)
3083 {
3084         return mem_cgroup_from_css(css)->move_charge_at_immigrate;
3085 }
3086 
3087 #ifdef CONFIG_MMU
3088 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3089                                         struct cftype *cft, u64 val)
3090 {
3091         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3092 
3093         if (val & ~MOVE_MASK)
3094                 return -EINVAL;
3095 
3096         /*
3097          * No kind of locking is needed in here, because ->can_attach() will
3098          * check this value once in the beginning of the process, and then carry
3099          * on with stale data. This means that changes to this value will only
3100          * affect task migrations starting after the change.
3101          */
3102         memcg->move_charge_at_immigrate = val;
3103         return 0;
3104 }
3105 #else
3106 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3107                                         struct cftype *cft, u64 val)
3108 {
3109         return -ENOSYS;
3110 }
3111 #endif
3112 
3113 #ifdef CONFIG_NUMA
3114 static int memcg_numa_stat_show(struct seq_file *m, void *v)
3115 {
3116         struct numa_stat {
3117                 const char *name;
3118                 unsigned int lru_mask;
3119         };
3120 
3121         static const struct numa_stat stats[] = {
3122                 { "total", LRU_ALL },
3123                 { "file", LRU_ALL_FILE },
3124                 { "anon", LRU_ALL_ANON },
3125                 { "unevictable", BIT(LRU_UNEVICTABLE) },
3126         };
3127         const struct numa_stat *stat;
3128         int nid;
3129         unsigned long nr;
3130         struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
3131 
3132         for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
3133                 nr = mem_cgroup_nr_lru_pages(memcg, stat->lru_mask);
3134                 seq_printf(m, "%s=%lu", stat->name, nr);
3135                 for_each_node_state(nid, N_MEMORY) {
3136                         nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
3137                                                           stat->lru_mask);
3138                         seq_printf(m, " N%d=%lu", nid, nr);
3139                 }
3140                 seq_putc(m, '\n');
3141         }
3142 
3143         for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
3144                 struct mem_cgroup *iter;
3145 
3146                 nr = 0;
3147                 for_each_mem_cgroup_tree(iter, memcg)
3148                         nr += mem_cgroup_nr_lru_pages(iter, stat->lru_mask);
3149                 seq_printf(m, "hierarchical_%s=%lu", stat->name, nr);
3150                 for_each_node_state(nid, N_MEMORY) {
3151                         nr = 0;
3152                         for_each_mem_cgroup_tree(iter, memcg)
3153                                 nr += mem_cgroup_node_nr_lru_pages(
3154                                         iter, nid, stat->lru_mask);
3155                         seq_printf(m, " N%d=%lu", nid, nr);
3156                 }
3157                 seq_putc(m, '\n');
3158         }
3159 
3160         return 0;
3161 }
3162 #endif /* CONFIG_NUMA */
3163 
3164 /* Universal VM events cgroup1 shows, original sort order */
3165 unsigned int memcg1_events[] = {
3166         PGPGIN,
3167         PGPGOUT,
3168         PGFAULT,
3169         PGMAJFAULT,
3170 };
3171 
3172 static const char *const memcg1_event_names[] = {
3173         "pgpgin",
3174         "pgpgout",
3175         "pgfault",
3176         "pgmajfault",
3177 };
3178 
3179 static int memcg_stat_show(struct seq_file *m, void *v)
3180 {
3181         struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
3182         unsigned long memory, memsw;
3183         struct mem_cgroup *mi;
3184         unsigned int i;
3185 
3186         BUILD_BUG_ON(ARRAY_SIZE(memcg1_stat_names) != ARRAY_SIZE(memcg1_stats));
3187         BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_lru_names) != NR_LRU_LISTS);
3188 
3189         for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
3190                 if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
3191                         continue;
3192                 seq_printf(m, "%s %lu\n", memcg1_stat_names[i],
3193                            memcg_page_state(memcg, memcg1_stats[i]) *
3194                            PAGE_SIZE);
3195         }
3196 
3197         for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
3198                 seq_printf(m, "%s %lu\n", memcg1_event_names[i],
3199                            memcg_sum_events(memcg, memcg1_events[i]));
3200 
3201         for (i = 0; i < NR_LRU_LISTS; i++)
3202                 seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i],
3203                            mem_cgroup_nr_lru_pages(memcg, BIT(i)) * PAGE_SIZE);
3204 
3205         /* Hierarchical information */
3206         memory = memsw = PAGE_COUNTER_MAX;
3207         for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) {
3208                 memory = min(memory, mi->memory.limit);
3209                 memsw = min(memsw, mi->memsw.limit);
3210         }
3211         seq_printf(m, "hierarchical_memory_limit %llu\n",
3212                    (u64)memory * PAGE_SIZE);
3213         if (do_memsw_account())
3214                 seq_printf(m, "hierarchical_memsw_limit %llu\n",
3215                            (u64)memsw * PAGE_SIZE);
3216 
3217         for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
3218                 unsigned long long val = 0;
3219 
3220                 if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
3221                         continue;
3222                 for_each_mem_cgroup_tree(mi, memcg)
3223                         val += memcg_page_state(mi, memcg1_stats[i]) *
3224                         PAGE_SIZE;
3225                 seq_printf(m, "total_%s %llu\n", memcg1_stat_names[i], val);
3226         }
3227 
3228         for (i = 0; i < ARRAY_SIZE(memcg1_events); i++) {
3229                 unsigned long long val = 0;
3230 
3231                 for_each_mem_cgroup_tree(mi, memcg)
3232                         val += memcg_sum_events(mi, memcg1_events[i]);
3233                 seq_printf(m, "total_%s %llu\n", memcg1_event_names[i], val);
3234         }
3235 
3236         for (i = 0; i < NR_LRU_LISTS; i++) {
3237                 unsigned long long val = 0;
3238 
3239                 for_each_mem_cgroup_tree(mi, memcg)
3240                         val += mem_cgroup_nr_lru_pages(mi, BIT(i)) * PAGE_SIZE;
3241                 seq_printf(m, "total_%s %llu\n", mem_cgroup_lru_names[i], val);
3242         }
3243 
3244 #ifdef CONFIG_DEBUG_VM
3245         {
3246                 pg_data_t *pgdat;
3247                 struct mem_cgroup_per_node *mz;
3248                 struct zone_reclaim_stat *rstat;
3249                 unsigned long recent_rotated[2] = {0, 0};
3250                 unsigned long recent_scanned[2] = {0, 0};
3251 
3252                 for_each_online_pgdat(pgdat) {
3253                         mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
3254                         rstat = &mz->lruvec.reclaim_stat;
3255 
3256                         recent_rotated[0] += rstat->recent_rotated[0];
3257                         recent_rotated[1] += rstat->recent_rotated[1];
3258                         recent_scanned[0] += rstat->recent_scanned[0];
3259                         recent_scanned[1] += rstat->recent_scanned[1];
3260                 }
3261                 seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]);
3262                 seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]);
3263                 seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]);
3264                 seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]);
3265         }
3266 #endif
3267 
3268         return 0;
3269 }
3270 
3271 static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css,
3272                                       struct cftype *cft)
3273 {
3274         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3275 
3276         return mem_cgroup_swappiness(memcg);
3277 }
3278 
3279 static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
3280                                        struct cftype *cft, u64 val)
3281 {
3282         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3283 
3284         if (val > 100)
3285                 return -EINVAL;
3286 
3287         if (css->parent)
3288                 memcg->swappiness = val;
3289         else
3290                 vm_swappiness = val;
3291 
3292         return 0;
3293 }
3294 
3295 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
3296 {
3297         struct mem_cgroup_threshold_ary *t;
3298         unsigned long usage;
3299         int i;
3300 
3301         rcu_read_lock();
3302         if (!swap)
3303                 t = rcu_dereference(memcg->thresholds.primary);
3304         else
3305                 t = rcu_dereference(memcg->memsw_thresholds.primary);
3306 
3307         if (!t)
3308                 goto unlock;
3309 
3310         usage = mem_cgroup_usage(memcg, swap);
3311 
3312         /*
3313          * current_threshold points to threshold just below or equal to usage.
3314          * If it's not true, a threshold was crossed after last
3315          * call of __mem_cgroup_threshold().
3316          */
3317         i = t->current_threshold;
3318 
3319         /*
3320          * Iterate backward over array of thresholds starting from
3321          * current_threshold and check if a threshold is crossed.
3322          * If none of thresholds below usage is crossed, we read
3323          * only one element of the array here.
3324          */
3325         for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
3326                 eventfd_signal(t->entries[i].eventfd, 1);
3327 
3328         /* i = current_threshold + 1 */
3329         i++;
3330 
3331         /*
3332          * Iterate forward over array of thresholds starting from
3333          * current_threshold+1 and check if a threshold is crossed.
3334          * If none of thresholds above usage is crossed, we read
3335          * only one element of the array here.
3336          */
3337         for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
3338                 eventfd_signal(t->entries[i].eventfd, 1);
3339 
3340         /* Update current_threshold */
3341         t->current_threshold = i - 1;
3342 unlock:
3343         rcu_read_unlock();
3344 }
3345 
3346 static void mem_cgroup_threshold(struct mem_cgroup *memcg)
3347 {
3348         while (memcg) {
3349                 __mem_cgroup_threshold(memcg, false);
3350                 if (do_memsw_account())
3351                         __mem_cgroup_threshold(memcg, true);
3352 
3353                 memcg = parent_mem_cgroup(memcg);
3354         }
3355 }
3356 
3357 static int compare_thresholds(const void *a, const void *b)
3358 {
3359         const struct mem_cgroup_threshold *_a = a;
3360         const struct mem_cgroup_threshold *_b = b;
3361 
3362         if (_a->threshold > _b->threshold)
3363                 return 1;
3364 
3365         if (_a->threshold < _b->threshold)
3366                 return -1;
3367 
3368         return 0;
3369 }
3370 
3371 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
3372 {
3373         struct mem_cgroup_eventfd_list *ev;
3374 
3375         spin_lock(&memcg_oom_lock);
3376 
3377         list_for_each_entry(ev, &memcg->oom_notify, list)
3378                 eventfd_signal(ev->eventfd, 1);
3379 
3380         spin_unlock(&memcg_oom_lock);
3381         return 0;
3382 }
3383 
3384 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
3385 {
3386         struct mem_cgroup *iter;
3387 
3388         for_each_mem_cgroup_tree(iter, memcg)
3389                 mem_cgroup_oom_notify_cb(iter);
3390 }
3391 
3392 static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
3393         struct eventfd_ctx *eventfd, const char *args, enum res_type type)
3394 {
3395         struct mem_cgroup_thresholds *thresholds;
3396         struct mem_cgroup_threshold_ary *new;
3397         unsigned long threshold;
3398         unsigned long usage;
3399         int i, size, ret;
3400 
3401         ret = page_counter_memparse(args, "-1", &threshold);
3402         if (ret)
3403                 return ret;
3404 
3405         mutex_lock(&memcg->thresholds_lock);
3406 
3407         if (type == _MEM) {
3408                 thresholds = &memcg->thresholds;
3409                 usage = mem_cgroup_usage(memcg, false);
3410         } else if (type == _MEMSWAP) {
3411                 thresholds = &memcg->memsw_thresholds;
3412                 usage = mem_cgroup_usage(memcg, true);
3413         } else
3414                 BUG();
3415 
3416         /* Check if a threshold crossed before adding a new one */
3417         if (thresholds->primary)
3418                 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
3419 
3420         size = thresholds->primary ? thresholds->primary->size + 1 : 1;
3421 
3422         /* Allocate memory for new array of thresholds */
3423         new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
3424                         GFP_KERNEL);
3425         if (!new) {
3426                 ret = -ENOMEM;
3427                 goto unlock;
3428         }
3429         new->size = size;
3430 
3431         /* Copy thresholds (if any) to new array */
3432         if (thresholds->primary) {
3433                 memcpy(new->entries, thresholds->primary->entries, (size - 1) *
3434                                 sizeof(struct mem_cgroup_threshold));
3435         }
3436 
3437         /* Add new threshold */
3438         new->entries[size - 1].eventfd = eventfd;
3439         new->entries[size - 1].threshold = threshold;
3440 
3441         /* Sort thresholds. Registering of new threshold isn't time-critical */
3442         sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
3443                         compare_thresholds, NULL);
3444 
3445         /* Find current threshold */
3446         new->current_threshold = -1;
3447         for (i = 0; i < size; i++) {
3448                 if (new->entries[i].threshold <= usage) {
3449                         /*
3450                          * new->current_threshold will not be used until
3451                          * rcu_assign_pointer(), so it's safe to increment
3452                          * it here.
3453                          */
3454                         ++new->current_threshold;
3455                 } else
3456                         break;
3457         }
3458 
3459         /* Free old spare buffer and save old primary buffer as spare */
3460         kfree(thresholds->spare);
3461         thresholds->spare = thresholds->primary;
3462 
3463         rcu_assign_pointer(thresholds->primary, new);
3464 
3465         /* To be sure that nobody uses thresholds */
3466         synchronize_rcu();
3467 
3468 unlock:
3469         mutex_unlock(&memcg->thresholds_lock);
3470 
3471         return ret;
3472 }
3473 
3474 static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
3475         struct eventfd_ctx *eventfd, const char *args)
3476 {
3477         return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM);
3478 }
3479 
3480 static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg,
3481         struct eventfd_ctx *eventfd, const char *args)
3482 {
3483         return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP);
3484 }
3485 
3486 static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
3487         struct eventfd_ctx *eventfd, enum res_type type)
3488 {
3489         struct mem_cgroup_thresholds *thresholds;
3490         struct mem_cgroup_threshold_ary *new;
3491         unsigned long usage;
3492         int i, j, size;
3493 
3494         mutex_lock(&memcg->thresholds_lock);
3495 
3496         if (type == _MEM) {
3497                 thresholds = &memcg->thresholds;
3498                 usage = mem_cgroup_usage(memcg, false);
3499         } else if (type == _MEMSWAP) {
3500                 thresholds = &memcg->memsw_thresholds;
3501                 usage = mem_cgroup_usage(memcg, true);
3502         } else
3503                 BUG();
3504 
3505         if (!thresholds->primary)
3506                 goto unlock;
3507 
3508         /* Check if a threshold crossed before removing */
3509         __mem_cgroup_threshold(memcg, type == _MEMSWAP);
3510 
3511         /* Calculate new number of threshold */
3512         size = 0;
3513         for (i = 0; i < thresholds->primary->size; i++) {
3514                 if (thresholds->primary->entries[i].eventfd != eventfd)
3515                         size++;
3516         }
3517 
3518         new = thresholds->spare;
3519 
3520         /* Set thresholds array to NULL if we don't have thresholds */
3521         if (!size) {
3522                 kfree(new);
3523                 new = NULL;
3524                 goto swap_buffers;
3525         }
3526 
3527         new->size = size;
3528 
3529         /* Copy thresholds and find current threshold */
3530         new->current_threshold = -1;
3531         for (i = 0, j = 0; i < thresholds->primary->size; i++) {
3532                 if (thresholds->primary->entries[i].eventfd == eventfd)
3533                         continue;
3534 
3535                 new->entries[j] = thresholds->primary->entries[i];
3536                 if (new->entries[j].threshold <= usage) {
3537                         /*
3538                          * new->current_threshold will not be used
3539                          * until rcu_assign_pointer(), so it's safe to increment
3540                          * it here.
3541                          */
3542                         ++new->current_threshold;
3543                 }
3544                 j++;
3545         }
3546 
3547 swap_buffers:
3548         /* Swap primary and spare array */
3549         thresholds->spare = thresholds->primary;
3550 
3551         rcu_assign_pointer(thresholds->primary, new);
3552 
3553         /* To be sure that nobody uses thresholds */
3554         synchronize_rcu();
3555 
3556         /* If all events are unregistered, free the spare array */
3557         if (!new) {
3558                 kfree(thresholds->spare);
3559                 thresholds->spare = NULL;
3560         }
3561 unlock:
3562         mutex_unlock(&memcg->thresholds_lock);
3563 }
3564 
3565 static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
3566         struct eventfd_ctx *eventfd)
3567 {
3568         return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM);
3569 }
3570 
3571 static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
3572         struct eventfd_ctx *eventfd)
3573 {
3574         return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP);
3575 }
3576 
3577 static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg,
3578         struct eventfd_ctx *eventfd, const char *args)
3579 {
3580         struct mem_cgroup_eventfd_list *event;
3581 
3582         event = kmalloc(sizeof(*event), GFP_KERNEL);
3583         if (!event)
3584                 return -ENOMEM;
3585 
3586         spin_lock(&memcg_oom_lock);
3587 
3588         event->eventfd = eventfd;
3589         list_add(&event->list, &memcg->oom_notify);
3590 
3591         /* already in OOM ? */
3592         if (memcg->under_oom)
3593                 eventfd_signal(eventfd, 1);
3594         spin_unlock(&memcg_oom_lock);
3595 
3596         return 0;
3597 }
3598 
3599 static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg,
3600         struct eventfd_ctx *eventfd)
3601 {
3602         struct mem_cgroup_eventfd_list *ev, *tmp;
3603 
3604         spin_lock(&memcg_oom_lock);
3605 
3606         list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
3607                 if (ev->eventfd == eventfd) {
3608                         list_del(&ev->list);
3609                         kfree(ev);
3610                 }
3611         }
3612 
3613         spin_unlock(&memcg_oom_lock);
3614 }
3615 
3616 static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v)
3617 {
3618         struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(sf));
3619 
3620         seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable);
3621         seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom);
3622         seq_printf(sf, "oom_kill %lu\n", memcg_sum_events(memcg, OOM_KILL));
3623         return 0;
3624 }
3625 
3626 static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
3627         struct cftype *cft, u64 val)
3628 {
3629         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3630 
3631         /* cannot set to root cgroup and only 0 and 1 are allowed */
3632         if (!css->parent || !((val == 0) || (val == 1)))
3633                 return -EINVAL;
3634 
3635         memcg->oom_kill_disable = val;
3636         if (!val)
3637                 memcg_oom_recover(memcg);
3638 
3639         return 0;
3640 }
3641 
3642 #ifdef CONFIG_CGROUP_WRITEBACK
3643 
3644 struct list_head *mem_cgroup_cgwb_list(struct mem_cgroup *memcg)
3645 {
3646         return &memcg->cgwb_list;
3647 }
3648 
3649 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
3650 {
3651         return wb_domain_init(&memcg->cgwb_domain, gfp);
3652 }
3653 
3654 static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
3655 {
3656         wb_domain_exit(&memcg->cgwb_domain);
3657 }
3658 
3659 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
3660 {
3661         wb_domain_size_changed(&memcg->cgwb_domain);
3662 }
3663 
3664 struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
3665 {
3666         struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
3667 
3668         if (!memcg->css.parent)
3669                 return NULL;
3670 
3671         return &memcg->cgwb_domain;
3672 }
3673 
3674 /**
3675  * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg
3676  * @wb: bdi_writeback in question
3677  * @pfilepages: out parameter for number of file pages
3678  * @pheadroom: out parameter for number of allocatable pages according to memcg
3679  * @pdirty: out parameter for number of dirty pages
3680  * @pwriteback: out parameter for number of pages under writeback
3681  *
3682  * Determine the numbers of file, headroom, dirty, and writeback pages in
3683  * @wb's memcg.  File, dirty and writeback are self-explanatory.  Headroom
3684  * is a bit more involved.
3685  *
3686  * A memcg's headroom is "min(max, high) - used".  In the hierarchy, the
3687  * headroom is calculated as the lowest headroom of itself and the
3688  * ancestors.  Note that this doesn't consider the actual amount of
3689  * available memory in the system.  The caller should further cap
3690  * *@pheadroom accordingly.
3691  */
3692 void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
3693                          unsigned long *pheadroom, unsigned long *pdirty,
3694                          unsigned long *pwriteback)
3695 {
3696         struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
3697         struct mem_cgroup *parent;
3698 
3699         *pdirty = memcg_page_state(memcg, NR_FILE_DIRTY);
3700 
3701         /* this should eventually include NR_UNSTABLE_NFS */
3702         *pwriteback = memcg_page_state(memcg, NR_WRITEBACK);
3703         *pfilepages = mem_cgroup_nr_lru_pages(memcg, (1 << LRU_INACTIVE_FILE) |
3704                                                      (1 << LRU_ACTIVE_FILE));
3705         *pheadroom = PAGE_COUNTER_MAX;
3706 
3707         while ((parent = parent_mem_cgroup(memcg))) {
3708                 unsigned long ceiling = min(memcg->memory.limit, memcg->high);
3709                 unsigned long used = page_counter_read(&memcg->memory);
3710 
3711                 *pheadroom = min(*pheadroom, ceiling - min(ceiling, used));
3712                 memcg = parent;
3713         }
3714 }
3715 
3716 #else   /* CONFIG_CGROUP_WRITEBACK */
3717 
3718 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
3719 {
3720         return 0;
3721 }
3722 
3723 static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
3724 {
3725 }
3726 
3727 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
3728 {
3729 }
3730 
3731 #endif  /* CONFIG_CGROUP_WRITEBACK */
3732 
3733 /*
3734  * DO NOT USE IN NEW FILES.
3735  *
3736  * "cgroup.event_control" implementation.
3737  *
3738  * This is way over-engineered.  It tries to support fully configurable
3739  * events for each user.  Such level of flexibility is completely
3740  * unnecessary especially in the light of the planned unified hierarchy.
3741  *
3742  * Please deprecate this and replace with something simpler if at all
3743  * possible.
3744  */
3745 
3746 /*
3747  * Unregister event and free resources.
3748  *
3749  * Gets called from workqueue.
3750  */
3751 static void memcg_event_remove(struct work_struct *work)
3752 {
3753         struct mem_cgroup_event *event =
3754                 container_of(work, struct mem_cgroup_event, remove);
3755         struct mem_cgroup *memcg = event->memcg;
3756 
3757         remove_wait_queue(event->wqh, &event->wait);
3758 
3759         event->unregister_event(memcg, event->eventfd);
3760 
3761         /* Notify userspace the event is going away. */
3762         eventfd_signal(event->eventfd, 1);
3763 
3764         eventfd_ctx_put(event->eventfd);
3765         kfree(event);
3766         css_put(&memcg->css);
3767 }
3768 
3769 /*
3770  * Gets called on POLLHUP on eventfd when user closes it.
3771  *
3772  * Called with wqh->lock held and interrupts disabled.
3773  */
3774 static int memcg_event_wake(wait_queue_entry_t *wait, unsigned mode,
3775                             int sync, void *key)
3776 {
3777         struct mem_cgroup_event *event =
3778                 container_of(wait, struct mem_cgroup_event, wait);
3779         struct mem_cgroup *memcg = event->memcg;
3780         unsigned long flags = (unsigned long)key;
3781 
3782         if (flags & POLLHUP) {
3783                 /*
3784                  * If the event has been detached at cgroup removal, we
3785                  * can simply return knowing the other side will cleanup
3786                  * for us.
3787                  *
3788                  * We can't race against event freeing since the other
3789                  * side will require wqh->lock via remove_wait_queue(),
3790                  * which we hold.
3791                  */
3792                 spin_lock(&memcg->event_list_lock);
3793                 if (!list_empty(&event->list)) {
3794                         list_del_init(&event->list);
3795                         /*
3796                          * We are in atomic context, but cgroup_event_remove()
3797                          * may sleep, so we have to call it in workqueue.
3798                          */
3799                         schedule_work(&event->remove);
3800                 }
3801                 spin_unlock(&memcg->event_list_lock);
3802         }
3803 
3804         return 0;
3805 }
3806 
3807 static void memcg_event_ptable_queue_proc(struct file *file,
3808                 wait_queue_head_t *wqh, poll_table *pt)
3809 {
3810         struct mem_cgroup_event *event =
3811                 container_of(pt, struct mem_cgroup_event, pt);
3812 
3813         event->wqh = wqh;
3814         add_wait_queue(wqh, &event->wait);
3815 }
3816 
3817 /*
3818  * DO NOT USE IN NEW FILES.
3819  *
3820  * Parse input and register new cgroup event handler.
3821  *
3822  * Input must be in format '<event_fd> <control_fd> <args>'.
3823  * Interpretation of args is defined by control file implementation.
3824  */
3825 static ssize_t memcg_write_event_control(struct kernfs_open_file *of,
3826                                          char *buf, size_t nbytes, loff_t off)
3827 {
3828         struct cgroup_subsys_state *css = of_css(of);
3829         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3830         struct mem_cgroup_event *event;
3831         struct cgroup_subsys_state *cfile_css;
3832         unsigned int efd, cfd;
3833         struct fd efile;
3834         struct fd cfile;
3835         const char *name;
3836         char *endp;
3837         int ret;
3838 
3839         buf = strstrip(buf);
3840 
3841         efd = simple_strtoul(buf, &endp, 10);
3842         if (*endp != ' ')
3843                 return -EINVAL;
3844         buf = endp + 1;
3845 
3846         cfd = simple_strtoul(buf, &endp, 10);
3847         if ((*endp != ' ') && (*endp != '\0'))
3848                 return -EINVAL;
3849         buf = endp + 1;
3850 
3851         event = kzalloc(sizeof(*event), GFP_KERNEL);
3852         if (!event)
3853                 return -ENOMEM;
3854 
3855         event->memcg = memcg;
3856         INIT_LIST_HEAD(&event->list);
3857         init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc);
3858         init_waitqueue_func_entry(&event->wait, memcg_event_wake);
3859         INIT_WORK(&event->remove, memcg_event_remove);
3860 
3861         efile = fdget(efd);
3862         if (!efile.file) {
3863                 ret = -EBADF;
3864                 goto out_kfree;
3865         }
3866 
3867         event->eventfd = eventfd_ctx_fileget(efile.file);
3868         if (IS_ERR(event->eventfd)) {
3869                 ret = PTR_ERR(event->eventfd);
3870                 goto out_put_efile;
3871         }
3872 
3873         cfile = fdget(cfd);
3874         if (!cfile.file) {
3875                 ret = -EBADF;
3876                 goto out_put_eventfd;
3877         }
3878 
3879         /* the process need read permission on control file */
3880         /* AV: shouldn't we check that it's been opened for read instead? */
3881         ret = inode_permission(file_inode(cfile.file), MAY_READ);
3882         if (ret < 0)
3883                 goto out_put_cfile;
3884 
3885         /*
3886          * Determine the event callbacks and set them in @event.  This used
3887          * to be done via struct cftype but cgroup core no longer knows
3888          * about these events.  The following is crude but the whole thing
3889          * is for compatibility anyway.
3890          *
3891          * DO NOT ADD NEW FILES.
3892          */
3893         name = cfile.file->f_path.dentry->d_name.name;
3894 
3895         if (!strcmp(name, "memory.usage_in_bytes")) {
3896                 event->register_event = mem_cgroup_usage_register_event;
3897                 event->unregister_event = mem_cgroup_usage_unregister_event;
3898         } else if (!strcmp(name, "memory.oom_control")) {
3899                 event->register_event = mem_cgroup_oom_register_event;
3900                 event->unregister_event = mem_cgroup_oom_unregister_event;
3901         } else if (!strcmp(name, "memory.pressure_level")) {
3902                 event->register_event = vmpressure_register_event;
3903                 event->unregister_event = vmpressure_unregister_event;
3904         } else if (!strcmp(name, "memory.memsw.usage_in_bytes")) {
3905                 event->register_event = memsw_cgroup_usage_register_event;
3906                 event->unregister_event = memsw_cgroup_usage_unregister_event;
3907         } else {
3908                 ret = -EINVAL;
3909                 goto out_put_cfile;
3910         }
3911 
3912         /*
3913          * Verify @cfile should belong to @css.  Also, remaining events are
3914          * automatically removed on cgroup destruction but the removal is
3915          * asynchronous, so take an extra ref on @css.
3916          */
3917         cfile_css = css_tryget_online_from_dir(cfile.file->f_path.dentry->d_parent,
3918                                                &memory_cgrp_subsys);
3919         ret = -EINVAL;
3920         if (IS_ERR(cfile_css))
3921                 goto out_put_cfile;
3922         if (cfile_css != css) {
3923                 css_put(cfile_css);
3924                 goto out_put_cfile;
3925         }
3926 
3927         ret = event->register_event(memcg, event->eventfd, buf);
3928         if (ret)
3929                 goto out_put_css;
3930 
3931         efile.file->f_op->poll(efile.file, &event->pt);
3932 
3933         spin_lock(&memcg->event_list_lock);
3934         list_add(&event->list, &memcg->event_list);
3935         spin_unlock(&memcg->event_list_lock);
3936 
3937         fdput(cfile);
3938         fdput(efile);
3939 
3940         return nbytes;
3941 
3942 out_put_css:
3943         css_put(css);
3944 out_put_cfile:
3945         fdput(cfile);
3946 out_put_eventfd:
3947         eventfd_ctx_put(event->eventfd);
3948 out_put_efile:
3949         fdput(efile);
3950 out_kfree:
3951         kfree(event);
3952 
3953         return ret;
3954 }
3955 
3956 static struct cftype mem_cgroup_legacy_files[] = {
3957         {
3958                 .name = "usage_in_bytes",
3959                 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
3960                 .read_u64 = mem_cgroup_read_u64,
3961         },
3962         {
3963                 .name = "max_usage_in_bytes",
3964                 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
3965                 .write = mem_cgroup_reset,
3966                 .read_u64 = mem_cgroup_read_u64,
3967         },
3968         {
3969                 .name = "limit_in_bytes",
3970                 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
3971                 .write = mem_cgroup_write,
3972                 .read_u64 = mem_cgroup_read_u64,
3973         },
3974         {
3975                 .name = "soft_limit_in_bytes",
3976                 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
3977                 .write = mem_cgroup_write,
3978                 .read_u64 = mem_cgroup_read_u64,
3979         },
3980         {
3981                 .name = "failcnt",
3982                 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
3983                 .write = mem_cgroup_reset,
3984                 .read_u64 = mem_cgroup_read_u64,
3985         },
3986         {
3987                 .name = "stat",
3988                 .seq_show = memcg_stat_show,
3989         },
3990         {
3991                 .name = "force_empty",
3992                 .write = mem_cgroup_force_empty_write,
3993         },
3994         {
3995                 .name = "use_hierarchy",
3996                 .write_u64 = mem_cgroup_hierarchy_write,
3997                 .read_u64 = mem_cgroup_hierarchy_read,
3998         },
3999         {
4000                 .name = "cgroup.event_control",         /* XXX: for compat */
4001                 .write = memcg_write_event_control,
4002                 .flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE,
4003         },
4004         {
4005                 .name = "swappiness",
4006                 .read_u64 = mem_cgroup_swappiness_read,
4007                 .write_u64 = mem_cgroup_swappiness_write,
4008         },
4009         {
4010                 .name = "move_charge_at_immigrate",
4011                 .read_u64 = mem_cgroup_move_charge_read,
4012                 .write_u64 = mem_cgroup_move_charge_write,
4013         },
4014         {
4015                 .name = "oom_control",
4016                 .seq_show = mem_cgroup_oom_control_read,
4017                 .write_u64 = mem_cgroup_oom_control_write,
4018                 .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
4019         },
4020         {
4021                 .name = "pressure_level",
4022         },
4023 #ifdef CONFIG_NUMA
4024         {
4025                 .name = "numa_stat",
4026                 .seq_show = memcg_numa_stat_show,
4027         },
4028 #endif
4029         {
4030                 .name = "kmem.limit_in_bytes",
4031                 .private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
4032                 .write = mem_cgroup_write,
4033                 .read_u64 = mem_cgroup_read_u64,
4034         },
4035         {
4036                 .name = "kmem.usage_in_bytes",
4037                 .private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
4038                 .read_u64 = mem_cgroup_read_u64,
4039         },
4040         {
4041                 .name = "kmem.failcnt",
4042                 .private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
4043                 .write = mem_cgroup_reset,
4044                 .read_u64 = mem_cgroup_read_u64,
4045         },
4046         {
4047                 .name = "kmem.max_usage_in_bytes",
4048                 .private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
4049                 .write = mem_cgroup_reset,
4050                 .read_u64 = mem_cgroup_read_u64,
4051         },
4052 #ifdef CONFIG_SLABINFO
4053         {
4054                 .name = "kmem.slabinfo",
4055                 .seq_start = memcg_slab_start,
4056                 .seq_next = memcg_slab_next,
4057                 .seq_stop = memcg_slab_stop,
4058                 .seq_show = memcg_slab_show,
4059         },
4060 #endif
4061         {
4062                 .name = "kmem.tcp.limit_in_bytes",
4063                 .private = MEMFILE_PRIVATE(_TCP, RES_LIMIT),
4064                 .write = mem_cgroup_write,
4065                 .read_u64 = mem_cgroup_read_u64,
4066         },
4067         {
4068                 .name = "kmem.tcp.usage_in_bytes",
4069                 .private = MEMFILE_PRIVATE(_TCP, RES_USAGE),
4070                 .read_u64 = mem_cgroup_read_u64,
4071         },
4072         {
4073                 .name = "kmem.tcp.failcnt",
4074                 .private = MEMFILE_PRIVATE(_TCP, RES_FAILCNT),
4075                 .write = mem_cgroup_reset,
4076                 .read_u64 = mem_cgroup_read_u64,
4077         },
4078         {
4079                 .name = "kmem.tcp.max_usage_in_bytes",
4080                 .private = MEMFILE_PRIVATE(_TCP, RES_MAX_USAGE),
4081                 .write = mem_cgroup_reset,
4082                 .read_u64 = mem_cgroup_read_u64,
4083         },
4084         { },    /* terminate */
4085 };
4086 
4087 /*
4088  * Private memory cgroup IDR
4089  *
4090  * Swap-out records and page cache shadow entries need to store memcg
4091  * references in constrained space, so we maintain an ID space that is
4092  * limited to 16 bit (MEM_CGROUP_ID_MAX), limiting the total number of
4093  * memory-controlled cgroups to 64k.
4094  *
4095  * However, there usually are many references to the oflline CSS after
4096  * the cgroup has been destroyed, such as page cache or reclaimable
4097  * slab objects, that don't need to hang on to the ID. We want to keep
4098  * those dead CSS from occupying IDs, or we might quickly exhaust the
4099  * relatively small ID space and prevent the creation of new cgroups
4100  * even when there are much fewer than 64k cgroups - possibly none.
4101  *
4102  * Maintain a private 16-bit ID space for memcg, and allow the ID to
4103  * be freed and recycled when it's no longer needed, which is usually
4104  * when the CSS is offlined.
4105  *
4106  * The only exception to that are records of swapped out tmpfs/shmem
4107  * pages that need to be attributed to live ancestors on swapin. But
4108  * those references are manageable from userspace.
4109  */
4110 
4111 static DEFINE_IDR(mem_cgroup_idr);
4112 
4113 static void mem_cgroup_id_get_many(struct mem_cgroup *memcg, unsigned int n)
4114 {
4115         VM_BUG_ON(atomic_read(&memcg->id.ref) <= 0);
4116         atomic_add(n, &memcg->id.ref);
4117 }
4118 
4119 static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n)
4120 {
4121         VM_BUG_ON(atomic_read(&memcg->id.ref) < n);
4122         if (atomic_sub_and_test(n, &memcg->id.ref)) {
4123                 idr_remove(&mem_cgroup_idr, memcg->id.id);
4124                 memcg->id.id = 0;
4125 
4126                 /* Memcg ID pins CSS */
4127                 css_put(&memcg->css);
4128         }
4129 }
4130 
4131 static inline void mem_cgroup_id_get(struct mem_cgroup *memcg)
4132 {
4133         mem_cgroup_id_get_many(memcg, 1);
4134 }
4135 
4136 static inline void mem_cgroup_id_put(struct mem_cgroup *memcg)
4137 {
4138         mem_cgroup_id_put_many(memcg, 1);
4139 }
4140 
4141 /**
4142  * mem_cgroup_from_id - look up a memcg from a memcg id
4143  * @id: the memcg id to look up
4144  *
4145  * Caller must hold rcu_read_lock().
4146  */
4147 struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
4148 {
4149         WARN_ON_ONCE(!rcu_read_lock_held());
4150         return idr_find(&mem_cgroup_idr, id);
4151 }
4152 
4153 static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
4154 {
4155         struct mem_cgroup_per_node *pn;
4156         int tmp = node;
4157         /*
4158          * This routine is called against possible nodes.
4159          * But it's BUG to call kmalloc() against offline node.
4160          *
4161          * TODO: this routine can waste much memory for nodes which will
4162          *       never be onlined. It's better to use memory hotplug callback
4163          *       function.
4164          */
4165         if (!node_state(node, N_NORMAL_MEMORY))
4166                 tmp = -1;
4167         pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
4168         if (!pn)
4169                 return 1;
4170 
4171         pn->lruvec_stat = alloc_percpu(struct lruvec_stat);
4172         if (!pn->lruvec_stat) {
4173                 kfree(pn);
4174                 return 1;
4175         }
4176 
4177         lruvec_init(&pn->lruvec);
4178         pn->usage_in_excess = 0;
4179         pn->on_tree = false;
4180         pn->memcg = memcg;
4181 
4182         memcg->nodeinfo[node] = pn;
4183         return 0;
4184 }
4185 
4186 static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
4187 {
4188         struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
4189 
4190         free_percpu(pn->lruvec_stat);
4191         kfree(pn);
4192 }
4193 
4194 static void __mem_cgroup_free(struct mem_cgroup *memcg)
4195 {
4196         int node;
4197 
4198         for_each_node(node)
4199                 free_mem_cgroup_per_node_info(memcg, node);
4200         free_percpu(memcg->stat);
4201         kfree(memcg);
4202 }
4203 
4204 static void mem_cgroup_free(struct mem_cgroup *memcg)
4205 {
4206         memcg_wb_domain_exit(memcg);
4207         __mem_cgroup_free(memcg);
4208 }
4209 
4210 static struct mem_cgroup *mem_cgroup_alloc(void)
4211 {
4212         struct mem_cgroup *memcg;
4213         size_t size;
4214         int node;
4215 
4216         size = sizeof(struct mem_cgroup);
4217         size += nr_node_ids * sizeof(struct mem_cgroup_per_node *);
4218 
4219         memcg = kzalloc(size, GFP_KERNEL);
4220         if (!memcg)
4221                 return NULL;
4222 
4223         memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL,
4224                                  1, MEM_CGROUP_ID_MAX,
4225                                  GFP_KERNEL);
4226         if (memcg->id.id < 0)
4227                 goto fail;
4228 
4229         memcg->stat = alloc_percpu(struct mem_cgroup_stat_cpu);
4230         if (!memcg->stat)
4231                 goto fail;
4232 
4233         for_each_node(node)
4234                 if (alloc_mem_cgroup_per_node_info(memcg, node))
4235                         goto fail;
4236 
4237         if (memcg_wb_domain_init(memcg, GFP_KERNEL))
4238                 goto fail;
4239 
4240         INIT_WORK(&memcg->high_work, high_work_func);
4241         memcg->last_scanned_node = MAX_NUMNODES;
4242         INIT_LIST_HEAD(&memcg->oom_notify);
4243         mutex_init(&memcg->thresholds_lock);
4244         spin_lock_init(&memcg->move_lock);
4245         vmpressure_init(&memcg->vmpressure);
4246         INIT_LIST_HEAD(&memcg->event_list);
4247         spin_lock_init(&memcg->event_list_lock);
4248         memcg->socket_pressure = jiffies;
4249 #ifndef CONFIG_SLOB
4250         memcg->kmemcg_id = -1;
4251 #endif
4252 #ifdef CONFIG_CGROUP_WRITEBACK
4253         INIT_LIST_HEAD(&memcg->cgwb_list);
4254 #endif
4255         idr_replace(&mem_cgroup_idr, memcg, memcg->id.id);
4256         return memcg;
4257 fail:
4258         if (memcg->id.id > 0)
4259                 idr_remove(&mem_cgroup_idr, memcg->id.id);
4260         __mem_cgroup_free(memcg);
4261         return NULL;
4262 }
4263 
4264 static struct cgroup_subsys_state * __ref
4265 mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
4266 {
4267         struct mem_cgroup *parent = mem_cgroup_from_css(parent_css);
4268         struct mem_cgroup *memcg;
4269         long error = -ENOMEM;
4270 
4271         memcg = mem_cgroup_alloc();
4272         if (!memcg)
4273                 return ERR_PTR(error);
4274 
4275         memcg->high = PAGE_COUNTER_MAX;
4276         memcg->soft_limit = PAGE_COUNTER_MAX;
4277         if (parent) {
4278                 memcg->swappiness = mem_cgroup_swappiness(parent);
4279                 memcg->oom_kill_disable = parent->oom_kill_disable;
4280         }
4281         if (parent && parent->use_hierarchy) {
4282                 memcg->use_hierarchy = true;
4283                 page_counter_init(&memcg->memory, &parent->memory);
4284                 page_counter_init(&memcg->swap, &parent->swap);
4285                 page_counter_init(&memcg->memsw, &parent->memsw);
4286                 page_counter_init(&memcg->kmem, &parent->kmem);
4287                 page_counter_init(&memcg->tcpmem, &parent->tcpmem);
4288         } else {
4289                 page_counter_init(&memcg->memory, NULL);
4290                 page_counter_init(&memcg->swap, NULL);
4291                 page_counter_init(&memcg->memsw, NULL);
4292                 page_counter_init(&memcg->kmem, NULL);
4293                 page_counter_init(&memcg->tcpmem, NULL);
4294                 /*
4295                  * Deeper hierachy with use_hierarchy == false doesn't make
4296                  * much sense so let cgroup subsystem know about this
4297                  * unfortunate state in our controller.
4298                  */
4299                 if (parent != root_mem_cgroup)
4300                         memory_cgrp_subsys.broken_hierarchy = true;
4301         }
4302 
4303         /* The following stuff does not apply to the root */
4304         if (!parent) {
4305                 root_mem_cgroup = memcg;
4306                 return &memcg->css;
4307         }
4308 
4309         error = memcg_online_kmem(memcg);
4310         if (error)
4311                 goto fail;
4312 
4313         if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
4314                 static_branch_inc(&memcg_sockets_enabled_key);
4315 
4316         return &memcg->css;
4317 fail:
4318         mem_cgroup_free(memcg);
4319         return ERR_PTR(-ENOMEM);
4320 }
4321 
4322 static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
4323 {
4324         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4325 
4326         /* Online state pins memcg ID, memcg ID pins CSS */
4327         atomic_set(&memcg->id.ref, 1);
4328         css_get(css);
4329         return 0;
4330 }
4331 
4332 static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
4333 {
4334         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4335         struct mem_cgroup_event *event, *tmp;
4336 
4337         /*
4338          * Unregister events and notify userspace.
4339          * Notify userspace about cgroup removing only after rmdir of cgroup
4340          * directory to avoid race between userspace and kernelspace.
4341          */
4342         spin_lock(&memcg->event_list_lock);
4343         list_for_each_entry_safe(event, tmp, &memcg->event_list, list) {
4344                 list_del_init(&event->list);
4345                 schedule_work(&event->remove);
4346         }
4347         spin_unlock(&memcg->event_list_lock);
4348 
4349         memcg->low = 0;
4350 
4351         memcg_offline_kmem(memcg);
4352         wb_memcg_offline(memcg);
4353 
4354         mem_cgroup_id_put(memcg);
4355 }
4356 
4357 static void mem_cgroup_css_released(struct cgroup_subsys_state *css)
4358 {
4359         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4360 
4361         invalidate_reclaim_iterators(memcg);
4362 }
4363 
4364 static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
4365 {
4366         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4367 
4368         if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
4369                 static_branch_dec(&memcg_sockets_enabled_key);
4370 
4371         if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active)
4372                 static_branch_dec(&memcg_sockets_enabled_key);
4373 
4374         vmpressure_cleanup(&memcg->vmpressure);
4375         cancel_work_sync(&memcg->high_work);
4376         mem_cgroup_remove_from_trees(memcg);
4377         memcg_free_kmem(memcg);
4378         mem_cgroup_free(memcg);
4379 }
4380 
4381 /**
4382  * mem_cgroup_css_reset - reset the states of a mem_cgroup
4383  * @css: the target css
4384  *
4385  * Reset the states of the mem_cgroup associated with @css.  This is
4386  * invoked when the userland requests disabling on the default hierarchy
4387  * but the memcg is pinned through dependency.  The memcg should stop
4388  * applying policies and should revert to the vanilla state as it may be
4389  * made visible again.
4390  *
4391  * The current implementation only resets the essential configurations.
4392  * This needs to be expanded to cover all the visible parts.
4393  */
4394 static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
4395 {
4396         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4397 
4398         page_counter_limit(&memcg->memory, PAGE_COUNTER_MAX);
4399         page_counter_limit(&memcg->swap, PAGE_COUNTER_MAX);
4400         page_counter_limit(&memcg->memsw, PAGE_COUNTER_MAX);
4401         page_counter_limit(&memcg->kmem, PAGE_COUNTER_MAX);
4402         page_counter_limit(&memcg->tcpmem, PAGE_COUNTER_MAX);
4403         memcg->low = 0;
4404         memcg->high = PAGE_COUNTER_MAX;
4405         memcg->soft_limit = PAGE_COUNTER_MAX;
4406         memcg_wb_domain_size_changed(memcg);
4407 }
4408 
4409 #ifdef CONFIG_MMU
4410 /* Handlers for move charge at task migration. */
4411 static int mem_cgroup_do_precharge(unsigned long count)
4412 {
4413         int ret;
4414 
4415         /* Try a single bulk charge without reclaim first, kswapd may wake */
4416         ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_DIRECT_RECLAIM, count);
4417         if (!ret) {
4418                 mc.precharge += count;
4419                 return ret;
4420         }
4421 
4422         /* Try charges one by one with reclaim, but do not retry */
4423         while (count--) {
4424                 ret = try_charge(mc.to, GFP_KERNEL | __GFP_NORETRY, 1);
4425                 if (ret)
4426                         return ret;
4427                 mc.precharge++;
4428                 cond_resched();
4429         }
4430         return 0;
4431 }
4432 
4433 union mc_target {
4434         struct page     *page;
4435         swp_entry_t     ent;
4436 };
4437 
4438 enum mc_target_type {
4439         MC_TARGET_NONE = 0,
4440         MC_TARGET_PAGE,
4441         MC_TARGET_SWAP,
4442         MC_TARGET_DEVICE,
4443 };
4444 
4445 static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
4446                                                 unsigned long addr, pte_t ptent)
4447 {
4448         struct page *page = _vm_normal_page(vma, addr, ptent, true);
4449 
4450         if (!page || !page_mapped(page))
4451                 return NULL;
4452         if (PageAnon(page)) {
4453                 if (!(mc.flags & MOVE_ANON))
4454                         return NULL;
4455         } else {
4456                 if (!(mc.flags & MOVE_FILE))
4457                         return NULL;
4458         }
4459         if (!get_page_unless_zero(page))
4460                 return NULL;
4461 
4462         return page;
4463 }
4464 
4465 #if defined(CONFIG_SWAP) || defined(CONFIG_DEVICE_PRIVATE)
4466 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
4467                         pte_t ptent, swp_entry_t *entry)
4468 {
4469         struct page *page = NULL;
4470         swp_entry_t ent = pte_to_swp_entry(ptent);
4471 
4472         if (!(mc.flags & MOVE_ANON) || non_swap_entry(ent))
4473                 return NULL;
4474 
4475         /*
4476          * Handle MEMORY_DEVICE_PRIVATE which are ZONE_DEVICE page belonging to
4477          * a device and because they are not accessible by CPU they are store
4478          * as special swap entry in the CPU page table.
4479          */
4480         if (is_device_private_entry(ent)) {
4481                 page = device_private_entry_to_page(ent);
4482                 /*
4483                  * MEMORY_DEVICE_PRIVATE means ZONE_DEVICE page and which have
4484                  * a refcount of 1 when free (unlike normal page)
4485                  */
4486                 if (!page_ref_add_unless(page, 1, 1))
4487                         return NULL;
4488                 return page;
4489         }
4490 
4491         /*
4492          * Because lookup_swap_cache() updates some statistics counter,
4493          * we call find_get_page() with swapper_space directly.
4494          */
4495         page = find_get_page(swap_address_space(ent), swp_offset(ent));
4496         if (do_memsw_account())
4497                 entry->val = ent.val;
4498 
4499         return page;
4500 }
4501 #else
4502 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
4503                         pte_t ptent, swp_entry_t *entry)
4504 {
4505         return NULL;
4506 }
4507 #endif
4508 
4509 static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
4510                         unsigned long addr, pte_t ptent, swp_entry_t *entry)
4511 {
4512         struct page *page = NULL;
4513         struct address_space *mapping;
4514         pgoff_t pgoff;
4515 
4516         if (!vma->vm_file) /* anonymous vma */
4517                 return NULL;
4518         if (!(mc.flags & MOVE_FILE))
4519                 return NULL;
4520 
4521         mapping = vma->vm_file->f_mapping;
4522         pgoff = linear_page_index(vma, addr);
4523 
4524         /* page is moved even if it's not RSS of this task(page-faulted). */
4525 #ifdef CONFIG_SWAP
4526         /* shmem/tmpfs may report page out on swap: account for that too. */
4527         if (shmem_mapping(mapping)) {
4528                 page = find_get_entry(mapping, pgoff);
4529                 if (radix_tree_exceptional_entry(page)) {
4530                         swp_entry_t swp = radix_to_swp_entry(page);
4531                         if (do_memsw_account())
4532                                 *entry = swp;
4533                         page = find_get_page(swap_address_space(swp),
4534                                              swp_offset(swp));
4535                 }
4536         } else
4537                 page = find_get_page(mapping, pgoff);
4538 #else
4539         page = find_get_page(mapping, pgoff);
4540 #endif
4541         return page;
4542 }
4543 
4544 /**
4545  * mem_cgroup_move_account - move account of the page
4546  * @page: the page
4547  * @compound: charge the page as compound or small page
4548  * @from: mem_cgroup which the page is moved from.
4549  * @to: mem_cgroup which the page is moved to. @from != @to.
4550  *
4551  * The caller must make sure the page is not on LRU (isolate_page() is useful.)
4552  *
4553  * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
4554  * from old cgroup.
4555  */
4556 static int mem_cgroup_move_account(struct page *page,
4557                                    bool compound,
4558                                    struct mem_cgroup *from,
4559                                    struct mem_cgroup *to)
4560 {
4561         unsigned long flags;
4562         unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
4563         int ret;
4564         bool anon;
4565 
4566         VM_BUG_ON(from == to);
4567         VM_BUG_ON_PAGE(PageLRU(page), page);
4568         VM_BUG_ON(compound && !PageTransHuge(page));
4569 
4570         /*
4571          * Prevent mem_cgroup_migrate() from looking at
4572          * page->mem_cgroup of its source page while we change it.
4573          */
4574         ret = -EBUSY;
4575         if (!trylock_page(page))
4576                 goto out;
4577 
4578         ret = -EINVAL;
4579         if (page->mem_cgroup != from)
4580                 goto out_unlock;
4581 
4582         anon = PageAnon(page);
4583 
4584         spin_lock_irqsave(&from->move_lock, flags);
4585 
4586         if (!anon && page_mapped(page)) {
4587                 __this_cpu_sub(from->stat->count[NR_FILE_MAPPED], nr_pages);
4588                 __this_cpu_add(to->stat->count[NR_FILE_MAPPED], nr_pages);
4589         }
4590 
4591         /*
4592          * move_lock grabbed above and caller set from->moving_account, so
4593          * mod_memcg_page_state will serialize updates to PageDirty.
4594          * So mapping should be stable for dirty pages.
4595          */
4596         if (!anon && PageDirty(page)) {
4597                 struct address_space *mapping = page_mapping(page);
4598 
4599                 if (mapping_cap_account_dirty(mapping)) {
4600                         __this_cpu_sub(from->stat->count[NR_FILE_DIRTY],
4601                                        nr_pages);
4602                         __this_cpu_add(to->stat->count[NR_FILE_DIRTY],
4603                                        nr_pages);
4604                 }
4605         }
4606 
4607         if (PageWriteback(page)) {
4608                 __this_cpu_sub(from->stat->count[NR_WRITEBACK], nr_pages);
4609                 __this_cpu_add(to->stat->count[NR_WRITEBACK], nr_pages);
4610         }
4611 
4612         /*
4613          * It is safe to change page->mem_cgroup here because the page
4614          * is referenced, charged, and isolated - we can't race with
4615          * uncharging, charging, migration, or LRU putback.
4616          */
4617 
4618         /* caller should have done css_get */
4619         page->mem_cgroup = to;
4620         spin_unlock_irqrestore(&from->move_lock, flags);
4621 
4622         ret = 0;
4623 
4624         local_irq_disable();
4625         mem_cgroup_charge_statistics(to, page, compound, nr_pages);
4626         memcg_check_events(to, page);
4627         mem_cgroup_charge_statistics(from, page, compound, -nr_pages);
4628         memcg_check_events(from, page);
4629         local_irq_enable();
4630 out_unlock:
4631         unlock_page(page);
4632 out:
4633         return ret;
4634 }
4635 
4636 /**
4637  * get_mctgt_type - get target type of moving charge
4638  * @vma: the vma the pte to be checked belongs
4639  * @addr: the address corresponding to the pte to be checked
4640  * @ptent: the pte to be checked
4641  * @target: the pointer the target page or swap ent will be stored(can be NULL)
4642  *
4643  * Returns
4644  *   0(MC_TARGET_NONE): if the pte is not a target for move charge.
4645  *   1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
4646  *     move charge. if @target is not NULL, the page is stored in target->page
4647  *     with extra refcnt got(Callers should handle it).
4648  *   2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
4649  *     target for charge migration. if @target is not NULL, the entry is stored
4650  *     in target->ent.
4651  *   3(MC_TARGET_DEVICE): like MC_TARGET_PAGE  but page is MEMORY_DEVICE_PUBLIC
4652  *     or MEMORY_DEVICE_PRIVATE (so ZONE_DEVICE page and thus not on the lru).
4653  *     For now we such page is charge like a regular page would be as for all
4654  *     intent and purposes it is just special memory taking the place of a
4655  *     regular page.
4656  *
4657  *     See Documentations/vm/hmm.txt and include/linux/hmm.h
4658  *
4659  * Called with pte lock held.
4660  */
4661 
4662 static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
4663                 unsigned long addr, pte_t ptent, union mc_target *target)
4664 {
4665         struct page *page = NULL;
4666         enum mc_target_type ret = MC_TARGET_NONE;
4667         swp_entry_t ent = { .val = 0 };
4668 
4669         if (pte_present(ptent))
4670                 page = mc_handle_present_pte(vma, addr, ptent);
4671         else if (is_swap_pte(ptent))
4672                 page = mc_handle_swap_pte(vma, ptent, &ent);
4673         else if (pte_none(ptent))
4674                 page = mc_handle_file_pte(vma, addr, ptent, &ent);
4675 
4676         if (!page && !ent.val)
4677                 return ret;
4678         if (page) {
4679                 /*
4680                  * Do only loose check w/o serialization.
4681                  * mem_cgroup_move_account() checks the page is valid or
4682                  * not under LRU exclusion.
4683                  */
4684                 if (page->mem_cgroup == mc.from) {
4685                         ret = MC_TARGET_PAGE;
4686                         if (is_device_private_page(page) ||
4687                             is_device_public_page(page))
4688                                 ret = MC_TARGET_DEVICE;
4689                         if (target)
4690                                 target->page = page;
4691                 }
4692                 if (!ret || !target)
4693                         put_page(page);
4694         }
4695         /*
4696          * There is a swap entry and a page doesn't exist or isn't charged.
4697          * But we cannot move a tail-page in a THP.
4698          */
4699         if (ent.val && !ret && (!page || !PageTransCompound(page)) &&
4700             mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) {
4701                 ret = MC_TARGET_SWAP;
4702                 if (target)
4703                         target->ent = ent;
4704         }
4705         return ret;
4706 }
4707 
4708 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4709 /*
4710  * We don't consider PMD mapped swapping or file mapped pages because THP does
4711  * not support them for now.
4712  * Caller should make sure that pmd_trans_huge(pmd) is true.
4713  */
4714 static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
4715                 unsigned long addr, pmd_t pmd, union mc_target *target)
4716 {
4717         struct page *page = NULL;
4718         enum mc_target_type ret = MC_TARGET_NONE;
4719 
4720         if (unlikely(is_swap_pmd(pmd))) {
4721                 VM_BUG_ON(thp_migration_supported() &&
4722                                   !is_pmd_migration_entry(pmd));
4723                 return ret;
4724         }
4725         page = pmd_page(pmd);
4726         VM_BUG_ON_PAGE(!page || !PageHead(page), page);
4727         if (!(mc.flags & MOVE_ANON))
4728                 return ret;
4729         if (page->mem_cgroup == mc.from) {
4730                 ret = MC_TARGET_PAGE;
4731                 if (target) {
4732                         get_page(page);
4733                         target->page = page;
4734                 }
4735         }
4736         return ret;
4737 }
4738 #else
4739 static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
4740                 unsigned long addr, pmd_t pmd, union mc_target *target)
4741 {
4742         return MC_TARGET_NONE;
4743 }
4744 #endif
4745 
4746 static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
4747                                         unsigned long addr, unsigned long end,
4748                                         struct mm_walk *walk)
4749 {
4750         struct vm_area_struct *vma = walk->vma;
4751         pte_t *pte;
4752         spinlock_t *ptl;
4753 
4754         ptl = pmd_trans_huge_lock(pmd, vma);
4755         if (ptl) {
4756                 /*
4757                  * Note their can not be MC_TARGET_DEVICE for now as we do not
4758                  * support transparent huge page with MEMORY_DEVICE_PUBLIC or
4759                  * MEMORY_DEVICE_PRIVATE but this might change.
4760                  */
4761                 if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
4762                         mc.precharge += HPAGE_PMD_NR;
4763                 spin_unlock(ptl);
4764                 return 0;
4765         }
4766 
4767         if (pmd_trans_unstable(pmd))
4768                 return 0;
4769         pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
4770         for (; addr != end; pte++, addr += PAGE_SIZE)
4771                 if (get_mctgt_type(vma, addr, *pte, NULL))
4772                         mc.precharge++; /* increment precharge temporarily */
4773         pte_unmap_unlock(pte - 1, ptl);
4774         cond_resched();
4775 
4776         return 0;
4777 }
4778 
4779 static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
4780 {
4781         unsigned long precharge;
4782 
4783         struct mm_walk mem_cgroup_count_precharge_walk = {
4784                 .pmd_entry = mem_cgroup_count_precharge_pte_range,
4785                 .mm = mm,
4786         };
4787         down_read(&mm->mmap_sem);
4788         walk_page_range(0, mm->highest_vm_end,
4789                         &mem_cgroup_count_precharge_walk);
4790         up_read(&mm->mmap_sem);
4791 
4792         precharge = mc.precharge;
4793         mc.precharge = 0;
4794 
4795         return precharge;
4796 }
4797 
4798 static int mem_cgroup_precharge_mc(struct mm_struct *mm)
4799 {
4800         unsigned long precharge = mem_cgroup_count_precharge(mm);
4801 
4802         VM_BUG_ON(mc.moving_task);
4803         mc.moving_task = current;
4804         return mem_cgroup_do_precharge(precharge);
4805 }
4806 
4807 /* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
4808 static void __mem_cgroup_clear_mc(void)
4809 {
4810         struct mem_cgroup *from = mc.from;
4811         struct mem_cgroup *to = mc.to;
4812 
4813         /* we must uncharge all the leftover precharges from mc.to */
4814         if (mc.precharge) {
4815                 cancel_charge(mc.to, mc.precharge);
4816                 mc.precharge = 0;
4817         }
4818         /*
4819          * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
4820          * we must uncharge here.
4821          */
4822         if (mc.moved_charge) {
4823                 cancel_charge(mc.from, mc.moved_charge);
4824                 mc.moved_charge = 0;
4825         }
4826         /* we must fixup refcnts and charges */
4827         if (mc.moved_swap) {
4828                 /* uncharge swap account from the old cgroup */
4829                 if (!mem_cgroup_is_root(mc.from))
4830                         page_counter_uncharge(&mc.from->memsw, mc.moved_swap);
4831 
4832                 mem_cgroup_id_put_many(mc.from, mc.moved_swap);
4833 
4834                 /*
4835                  * we charged both to->memory and to->memsw, so we
4836                  * should uncharge to->memory.
4837                  */
4838                 if (!mem_cgroup_is_root(mc.to))
4839                         page_counter_uncharge(&mc.to->memory, mc.moved_swap);
4840 
4841                 mem_cgroup_id_get_many(mc.to, mc.moved_swap);
4842                 css_put_many(&mc.to->css, mc.moved_swap);
4843 
4844                 mc.moved_swap = 0;
4845         }
4846         memcg_oom_recover(from);
4847         memcg_oom_recover(to);
4848         wake_up_all(&mc.waitq);
4849 }
4850 
4851 static void mem_cgroup_clear_mc(void)
4852 {
4853         struct mm_struct *mm = mc.mm;
4854 
4855         /*
4856          * we must clear moving_task before waking up waiters at the end of
4857          * task migration.
4858          */
4859         mc.moving_task = NULL;
4860         __mem_cgroup_clear_mc();
4861         spin_lock(&mc.lock);
4862         mc.from = NULL;
4863         mc.to = NULL;
4864         mc.mm = NULL;
4865         spin_unlock(&mc.lock);
4866 
4867         mmput(mm);
4868 }
4869 
4870 static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
4871 {
4872         struct cgroup_subsys_state *css;
4873         struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */
4874         struct mem_cgroup *from;
4875         struct task_struct *leader, *p;
4876         struct mm_struct *mm;
4877         unsigned long move_flags;
4878         int ret = 0;
4879 
4880         /* charge immigration isn't supported on the default hierarchy */
4881         if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
4882                 return 0;
4883 
4884         /*
4885          * Multi-process migrations only happen on the default hierarchy
4886          * where charge immigration is not used.  Perform charge
4887          * immigration if @tset contains a leader and whine if there are
4888          * multiple.
4889          */
4890         p = NULL;
4891         cgroup_taskset_for_each_leader(leader, css, tset) {
4892                 WARN_ON_ONCE(p);
4893                 p = leader;
4894                 memcg = mem_cgroup_from_css(css);
4895         }
4896         if (!p)
4897                 return 0;
4898 
4899         /*
4900          * We are now commited to this value whatever it is. Changes in this
4901          * tunable will only affect upcoming migrations, not the current one.
4902          * So we need to save it, and keep it going.
4903          */
4904         move_flags = READ_ONCE(memcg->move_charge_at_immigrate);
4905         if (!move_flags)
4906                 return 0;
4907 
4908         from = mem_cgroup_from_task(p);
4909 
4910         VM_BUG_ON(from == memcg);
4911 
4912         mm = get_task_mm(p);
4913         if (!mm)
4914                 return 0;
4915         /* We move charges only when we move a owner of the mm */
4916         if (mm->owner == p) {
4917                 VM_BUG_ON(mc.from);
4918                 VM_BUG_ON(mc.to);
4919                 VM_BUG_ON(mc.precharge);
4920                 VM_BUG_ON(mc.moved_charge);
4921                 VM_BUG_ON(mc.moved_swap);
4922 
4923                 spin_lock(&mc.lock);
4924                 mc.mm = mm;
4925                 mc.from = from;
4926                 mc.to = memcg;
4927                 mc.flags = move_flags;
4928                 spin_unlock(&mc.lock);
4929                 /* We set mc.moving_task later */
4930 
4931                 ret = mem_cgroup_precharge_mc(mm);
4932                 if (ret)
4933                         mem_cgroup_clear_mc();
4934         } else {
4935                 mmput(mm);
4936         }
4937         return ret;
4938 }
4939 
4940 static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
4941 {
4942         if (mc.to)
4943                 mem_cgroup_clear_mc();
4944 }
4945 
4946 static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
4947                                 unsigned long addr, unsigned long end,
4948                                 struct mm_walk *walk)
4949 {
4950         int ret = 0;
4951         struct vm_area_struct *vma = walk->vma;
4952         pte_t *pte;
4953         spinlock_t *ptl;
4954         enum mc_target_type target_type;
4955         union mc_target target;
4956         struct page *page;
4957 
4958         ptl = pmd_trans_huge_lock(pmd, vma);
4959         if (ptl) {
4960                 if (mc.precharge < HPAGE_PMD_NR) {
4961                         spin_unlock(ptl);
4962                         return 0;
4963                 }
4964                 target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
4965                 if (target_type == MC_TARGET_PAGE) {
4966                         page = target.page;
4967                         if (!isolate_lru_page(page)) {
4968                                 if (!mem_cgroup_move_account(page, true,
4969                                                              mc.from, mc.to)) {
4970                                         mc.precharge -= HPAGE_PMD_NR;
4971                                         mc.moved_charge += HPAGE_PMD_NR;
4972                                 }
4973                                 putback_lru_page(page);
4974                         }
4975                         put_page(page);
4976                 } else if (target_type == MC_TARGET_DEVICE) {
4977                         page = target.page;
4978                         if (!mem_cgroup_move_account(page, true,
4979                                                      mc.from, mc.to)) {
4980                                 mc.precharge -= HPAGE_PMD_NR;
4981                                 mc.moved_charge += HPAGE_PMD_NR;
4982                         }
4983                         put_page(page);
4984                 }
4985                 spin_unlock(ptl);
4986                 return 0;
4987         }
4988 
4989         if (pmd_trans_unstable(pmd))
4990                 return 0;
4991 retry:
4992         pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
4993         for (; addr != end; addr += PAGE_SIZE) {
4994                 pte_t ptent = *(pte++);
4995                 bool device = false;
4996                 swp_entry_t ent;
4997 
4998                 if (!mc.precharge)
4999                         break;
5000 
5001                 switch (get_mctgt_type(vma, addr, ptent, &target)) {
5002                 case MC_TARGET_DEVICE:
5003                         device = true;
5004                         /* fall through */
5005                 case MC_TARGET_PAGE:
5006                         page = target.page;
5007                         /*
5008                          * We can have a part of the split pmd here. Moving it
5009                          * can be done but it would be too convoluted so simply
5010                          * ignore such a partial THP and keep it in original
5011                          * memcg. There should be somebody mapping the head.
5012                          */
5013                         if (PageTransCompound(page))
5014                                 goto put;
5015                         if (!device && isolate_lru_page(page))
5016                                 goto put;
5017                         if (!mem_cgroup_move_account(page, false,
5018                                                 mc.from, mc.to)) {
5019                                 mc.precharge--;
5020                                 /* we uncharge from mc.from later. */
5021                                 mc.moved_charge++;
5022                         }
5023                         if (!device)
5024                                 putback_lru_page(page);
5025 put:                    /* get_mctgt_type() gets the page */
5026                         put_page(page);
5027                         break;
5028                 case MC_TARGET_SWAP:
5029                         ent = target.ent;
5030                         if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
5031                                 mc.precharge--;
5032                                 /* we fixup refcnts and charges later. */
5033                                 mc.moved_swap++;
5034                         }
5035                         break;
5036                 default:
5037                         break;
5038                 }
5039         }
5040         pte_unmap_unlock(pte - 1, ptl);
5041         cond_resched();
5042 
5043         if (addr != end) {
5044                 /*
5045                  * We have consumed all precharges we got in can_attach().
5046                  * We try charge one by one, but don't do any additional
5047                  * charges to mc.to if we have failed in charge once in attach()
5048                  * phase.
5049                  */
5050                 ret = mem_cgroup_do_precharge(1);
5051                 if (!ret)
5052                         goto retry;
5053         }
5054 
5055         return ret;
5056 }
5057 
5058 static void mem_cgroup_move_charge(void)
5059 {
5060         struct mm_walk mem_cgroup_move_charge_walk = {
5061                 .pmd_entry = mem_cgroup_move_charge_pte_range,
5062                 .mm = mc.mm,
5063         };
5064 
5065         lru_add_drain_all();
5066         /*
5067          * Signal lock_page_memcg() to take the memcg's move_lock
5068          * while we're moving its pages to another memcg. Then wait
5069          * for already started RCU-only updates to finish.
5070          */
5071         atomic_inc(&mc.from->moving_account);
5072         synchronize_rcu();
5073 retry:
5074         if (unlikely(!down_read_trylock(&mc.mm->mmap_sem))) {
5075                 /*
5076                  * Someone who are holding the mmap_sem might be waiting in
5077                  * waitq. So we cancel all extra charges, wake up all waiters,
5078                  * and retry. Because we cancel precharges, we might not be able
5079                  * to move enough charges, but moving charge is a best-effort
5080                  * feature anyway, so it wouldn't be a big problem.
5081                  */
5082                 __mem_cgroup_clear_mc();
5083                 cond_resched();
5084                 goto retry;
5085         }
5086         /*
5087          * When we have consumed all precharges and failed in doing
5088          * additional charge, the page walk just aborts.
5089          */
5090         walk_page_range(0, mc.mm->highest_vm_end, &mem_cgroup_move_charge_walk);
5091 
5092         up_read(&mc.mm->mmap_sem);
5093         atomic_dec(&mc.from->moving_account);
5094 }
5095 
5096 static void mem_cgroup_move_task(void)
5097 {
5098         if (mc.to) {
5099                 mem_cgroup_move_charge();
5100                 mem_cgroup_clear_mc();
5101         }
5102 }
5103 #else   /* !CONFIG_MMU */
5104 static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
5105 {
5106         return 0;
5107 }
5108 static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
5109 {
5110 }
5111 static void mem_cgroup_move_task(void)
5112 {
5113 }
5114 #endif
5115 
5116 /*
5117  * Cgroup retains root cgroups across [un]mount cycles making it necessary
5118  * to verify whether we're attached to the default hierarchy on each mount
5119  * attempt.
5120  */
5121 static void mem_cgroup_bind(struct cgroup_subsys_state *root_css)
5122 {
5123         /*
5124          * use_hierarchy is forced on the default hierarchy.  cgroup core
5125          * guarantees that @root doesn't have any children, so turning it
5126          * on for the root memcg is enough.
5127          */
5128         if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
5129                 root_mem_cgroup->use_hierarchy = true;
5130         else
5131                 root_mem_cgroup->use_hierarchy = false;
5132 }
5133 
5134 static u64 memory_current_read(struct cgroup_subsys_state *css,
5135                                struct cftype *cft)
5136 {
5137         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5138 
5139         return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE;
5140 }
5141 
5142 static int memory_low_show(struct seq_file *m, void *v)
5143 {
5144         struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5145         unsigned long low = READ_ONCE(memcg->low);
5146 
5147         if (low == PAGE_COUNTER_MAX)
5148                 seq_puts(m, "max\n");
5149         else
5150                 seq_printf(m, "%llu\n", (u64)low * PAGE_SIZE);
5151 
5152         return 0;
5153 }
5154 
5155 static ssize_t memory_low_write(struct kernfs_open_file *of,
5156                                 char *buf, size_t nbytes, loff_t off)
5157 {
5158         struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5159         unsigned long low;
5160         int err;
5161 
5162         buf = strstrip(buf);
5163         err = page_counter_memparse(buf, "max", &low);
5164         if (err)
5165                 return err;
5166 
5167         memcg->low = low;
5168 
5169         return nbytes;
5170 }
5171 
5172 static int memory_high_show(struct seq_file *m, void *v)
5173 {
5174         struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5175         unsigned long high = READ_ONCE(memcg->high);
5176 
5177         if (high == PAGE_COUNTER_MAX)
5178                 seq_puts(m, "max\n");
5179         else
5180                 seq_printf(m, "%llu\n", (u64)high * PAGE_SIZE);
5181 
5182         return 0;
5183 }
5184 
5185 static ssize_t memory_high_write(struct kernfs_open_file *of,
5186                                  char *buf, size_t nbytes, loff_t off)
5187 {
5188         struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5189         unsigned long nr_pages;
5190         unsigned long high;
5191         int err;
5192 
5193         buf = strstrip(buf);
5194         err = page_counter_memparse(buf, "max", &high);
5195         if (err)
5196                 return err;
5197 
5198         memcg->high = high;
5199 
5200         nr_pages = page_counter_read(&memcg->memory);
5201         if (nr_pages > high)
5202                 try_to_free_mem_cgroup_pages(memcg, nr_pages - high,
5203                                              GFP_KERNEL, true);
5204 
5205         memcg_wb_domain_size_changed(memcg);
5206         return nbytes;
5207 }
5208 
5209 static int memory_max_show(struct seq_file *m, void *v)
5210 {
5211         struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5212         unsigned long max = READ_ONCE(memcg->memory.limit);
5213 
5214         if (max == PAGE_COUNTER_MAX)
5215                 seq_puts(m, "max\n");
5216         else
5217                 seq_printf(m, "%llu\n", (u64)max * PAGE_SIZE);
5218 
5219         return 0;
5220 }
5221 
5222 static ssize_t memory_max_write(struct kernfs_open_file *of,
5223                                 char *buf, size_t nbytes, loff_t off)
5224 {
5225         struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5226         unsigned int nr_reclaims = MEM_CGROUP_RECLAIM_RETRIES;
5227         bool drained = false;
5228         unsigned long max;
5229         int err;
5230 
5231         buf = strstrip(buf);
5232         err = page_counter_memparse(buf, "max", &max);
5233         if (err)
5234                 return err;
5235 
5236         xchg(&memcg->memory.limit, max);
5237 
5238         for (;;) {
5239                 unsigned long nr_pages = page_counter_read(&memcg->memory);
5240 
5241                 if (nr_pages <= max)
5242                         break;
5243 
5244                 if (signal_pending(current)) {
5245                         err = -EINTR;
5246                         break;
5247                 }
5248 
5249                 if (!drained) {
5250                         drain_all_stock(memcg);
5251                         drained = true;
5252                         continue;
5253                 }
5254 
5255                 if (nr_reclaims) {
5256                         if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max,
5257                                                           GFP_KERNEL, true))
5258                                 nr_reclaims--;
5259                         continue;
5260                 }
5261 
5262                 mem_cgroup_event(memcg, MEMCG_OOM);
5263                 if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0))
5264                         break;
5265         }
5266 
5267         memcg_wb_domain_size_changed(memcg);
5268         return nbytes;
5269 }
5270 
5271 static int memory_events_show(struct seq_file *m, void *v)
5272 {
5273         struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5274 
5275         seq_printf(m, "low %lu\n", memcg_sum_events(memcg, MEMCG_LOW));
5276         seq_printf(m, "high %lu\n", memcg_sum_events(memcg, MEMCG_HIGH));
5277         seq_printf(m, "max %lu\n", memcg_sum_events(memcg, MEMCG_MAX));
5278         seq_printf(m, "oom %lu\n", memcg_sum_events(memcg, MEMCG_OOM));
5279         seq_printf(m, "oom_kill %lu\n", memcg_sum_events(memcg, OOM_KILL));
5280 
5281         return 0;
5282 }
5283 
5284 static int memory_stat_show(struct seq_file *m, void *v)
5285 {
5286         struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5287         unsigned long stat[MEMCG_NR_STAT];
5288         unsigned long events[MEMCG_NR_EVENTS];
5289         int i;
5290 
5291         /*
5292          * Provide statistics on the state of the memory subsystem as
5293          * well as cumulative event counters that show past behavior.
5294          *
5295          * This list is ordered following a combination of these gradients:
5296          * 1) generic big picture -> specifics and details
5297          * 2) reflecting userspace activity -> reflecting kernel heuristics
5298          *
5299          * Current memory state:
5300          */
5301 
5302         tree_stat(memcg, stat);
5303         tree_events(memcg, events);
5304 
5305         seq_printf(m, "anon %llu\n",
5306                    (u64)stat[MEMCG_RSS] * PAGE_SIZE);
5307         seq_printf(m, "file %llu\n",
5308                    (u64)stat[MEMCG_CACHE] * PAGE_SIZE);
5309         seq_printf(m, "kernel_stack %llu\n",
5310                    (u64)stat[MEMCG_KERNEL_STACK_KB] * 1024);
5311         seq_printf(m, "slab %llu\n",
5312                    (u64)(stat[NR_SLAB_RECLAIMABLE] +
5313                          stat[NR_SLAB_UNRECLAIMABLE]) * PAGE_SIZE);
5314         seq_printf(m, "sock %llu\n",
5315                    (u64)stat[MEMCG_SOCK] * PAGE_SIZE);
5316 
5317         seq_printf(m, "shmem %llu\n",
5318                    (u64)stat[NR_SHMEM] * PAGE_SIZE);
5319         seq_printf(m, "file_mapped %llu\n",
5320                    (u64)stat[NR_FILE_MAPPED] * PAGE_SIZE);
5321         seq_printf(m, "file_dirty %llu\n",
5322                    (u64)stat[NR_FILE_DIRTY] * PAGE_SIZE);
5323         seq_printf(m, "file_writeback %llu\n",
5324                    (u64)stat[NR_WRITEBACK] * PAGE_SIZE);
5325 
5326         for (i = 0; i < NR_LRU_LISTS; i++) {
5327                 struct mem_cgroup *mi;
5328                 unsigned long val = 0;
5329 
5330                 for_each_mem_cgroup_tree(mi, memcg)
5331                         val += mem_cgroup_nr_lru_pages(mi, BIT(i));
5332                 seq_printf(m, "%s %llu\n",
5333                            mem_cgroup_lru_names[i], (u64)val * PAGE_SIZE);
5334         }
5335 
5336         seq_printf(m, "slab_reclaimable %llu\n",
5337                    (u64)stat[NR_SLAB_RECLAIMABLE] * PAGE_SIZE);
5338         seq_printf(m, "slab_unreclaimable %llu\n",
5339                    (u64)stat[NR_SLAB_UNRECLAIMABLE] * PAGE_SIZE);
5340 
5341         /* Accumulated memory events */
5342 
5343         seq_printf(m, "pgfault %lu\n", events[PGFAULT]);
5344         seq_printf(m, "pgmajfault %lu\n", events[PGMAJFAULT]);
5345 
5346         seq_printf(m, "pgrefill %lu\n", events[PGREFILL]);
5347         seq_printf(m, "pgscan %lu\n", events[PGSCAN_KSWAPD] +
5348                    events[PGSCAN_DIRECT]);
5349         seq_printf(m, "pgsteal %lu\n", events[PGSTEAL_KSWAPD] +
5350                    events[PGSTEAL_DIRECT]);
5351         seq_printf(m, "pgactivate %lu\n", events[PGACTIVATE]);
5352         seq_printf(m, "pgdeactivate %lu\n", events[PGDEACTIVATE]);
5353         seq_printf(m, "pglazyfree %lu\n", events[PGLAZYFREE]);
5354         seq_printf(m, "pglazyfreed %lu\n", events[PGLAZYFREED]);
5355 
5356         seq_printf(m, "workingset_refault %lu\n",
5357                    stat[WORKINGSET_REFAULT]);
5358         seq_printf(m, "workingset_activate %lu\n",
5359                    stat[WORKINGSET_ACTIVATE]);
5360         seq_printf(m, "workingset_nodereclaim %lu\n",
5361                    stat[WORKINGSET_NODERECLAIM]);
5362 
5363         return 0;
5364 }
5365 
5366 static struct cftype memory_files[] = {
5367         {
5368                 .name = "current",
5369                 .flags = CFTYPE_NOT_ON_ROOT,
5370                 .read_u64 = memory_current_read,
5371         },
5372         {
5373                 .name = "low",
5374                 .flags = CFTYPE_NOT_ON_ROOT,
5375                 .seq_show = memory_low_show,
5376                 .write = memory_low_write,
5377         },
5378         {
5379                 .name = "high",
5380                 .flags = CFTYPE_NOT_ON_ROOT,
5381                 .seq_show = memory_high_show,
5382                 .write = memory_high_write,
5383         },
5384         {
5385                 .name = "max",
5386                 .flags = CFTYPE_NOT_ON_ROOT,
5387                 .seq_show = memory_max_show,
5388                 .write = memory_max_write,
5389         },
5390         {
5391                 .name = "events",
5392                 .flags = CFTYPE_NOT_ON_ROOT,
5393                 .file_offset = offsetof(struct mem_cgroup, events_file),
5394                 .seq_show = memory_events_show,
5395         },
5396         {
5397                 .name = "stat",
5398                 .flags = CFTYPE_NOT_ON_ROOT,
5399                 .seq_show = memory_stat_show,
5400         },
5401         { }     /* terminate */
5402 };
5403 
5404 struct cgroup_subsys memory_cgrp_subsys = {
5405         .css_alloc = mem_cgroup_css_alloc,
5406         .css_online = mem_cgroup_css_online,
5407         .css_offline = mem_cgroup_css_offline,
5408         .css_released = mem_cgroup_css_released,
5409         .css_free = mem_cgroup_css_free,
5410         .css_reset = mem_cgroup_css_reset,
5411         .can_attach = mem_cgroup_can_attach,
5412         .cancel_attach = mem_cgroup_cancel_attach,
5413         .post_attach = mem_cgroup_move_task,
5414         .bind = mem_cgroup_bind,
5415         .dfl_cftypes = memory_files,
5416         .legacy_cftypes = mem_cgroup_legacy_files,
5417         .early_init = 0,
5418 };
5419 
5420 /**
5421  * mem_cgroup_low - check if memory consumption is below the normal range
5422  * @root: the top ancestor of the sub-tree being checked
5423  * @memcg: the memory cgroup to check
5424  *
5425  * Returns %true if memory consumption of @memcg, and that of all
5426  * ancestors up to (but not including) @root, is below the normal range.
5427  *
5428  * @root is exclusive; it is never low when looked at directly and isn't
5429  * checked when traversing the hierarchy.
5430  *
5431  * Excluding @root enables using memory.low to prioritize memory usage
5432  * between cgroups within a subtree of the hierarchy that is limited by
5433  * memory.high or memory.max.
5434  *
5435  * For example, given cgroup A with children B and C:
5436  *
5437  *    A
5438  *   / \
5439  *  B   C
5440  *
5441  * and
5442  *
5443  *  1. A/memory.current > A/memory.high
5444  *  2. A/B/memory.current < A/B/memory.low
5445  *  3. A/C/memory.current >= A/C/memory.low
5446  *
5447  * As 'A' is high, i.e. triggers reclaim from 'A', and 'B' is low, we
5448  * should reclaim from 'C' until 'A' is no longer high or until we can
5449  * no longer reclaim from 'C'.  If 'A', i.e. @root, isn't excluded by
5450  * mem_cgroup_low when reclaming from 'A', then 'B' won't be considered
5451  * low and we will reclaim indiscriminately from both 'B' and 'C'.
5452  */
5453 bool mem_cgroup_low(struct mem_cgroup *root, struct mem_cgroup *memcg)
5454 {
5455         if (mem_cgroup_disabled())
5456                 return false;
5457 
5458         if (!root)
5459                 root = root_mem_cgroup;
5460         if (memcg == root)
5461                 return false;
5462 
5463         for (; memcg != root; memcg = parent_mem_cgroup(memcg)) {
5464                 if (page_counter_read(&memcg->memory) >= memcg->low)
5465                         return false;
5466         }
5467 
5468         return true;
5469 }
5470 
5471 /**
5472  * mem_cgroup_try_charge - try charging a page
5473  * @page: page to charge
5474  * @mm: mm context of the victim
5475  * @gfp_mask: reclaim mode
5476  * @memcgp: charged memcg return
5477  * @compound: charge the page as compound or small page
5478  *
5479  * Try to charge @page to the memcg that @mm belongs to, reclaiming
5480  * pages according to @gfp_mask if necessary.
5481  *
5482  * Returns 0 on success, with *@memcgp pointing to the charged memcg.
5483  * Otherwise, an error code is returned.
5484  *
5485  * After page->mapping has been set up, the caller must finalize the
5486  * charge with mem_cgroup_commit_charge().  Or abort the transaction
5487  * with mem_cgroup_cancel_charge() in case page instantiation fails.
5488  */
5489 int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
5490                           gfp_t gfp_mask, struct mem_cgroup **memcgp,
5491                           bool compound)
5492 {
5493         struct mem_cgroup *memcg = NULL;
5494         unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
5495         int ret = 0;
5496 
5497         if (mem_cgroup_disabled())
5498                 goto out;
5499 
5500         if (PageSwapCache(page)) {
5501                 /*
5502                  * Every swap fault against a single page tries to charge the
5503                  * page, bail as early as possible.  shmem_unuse() encounters
5504                  * already charged pages, too.  The USED bit is protected by
5505                  * the page lock, which serializes swap cache removal, which
5506                  * in turn serializes uncharging.
5507                  */
5508                 VM_BUG_ON_PAGE(!PageLocked(page), page);
5509                 if (compound_head(page)->mem_cgroup)
5510                         goto out;
5511 
5512                 if (do_swap_account) {
5513                         swp_entry_t ent = { .val = page_private(page), };
5514                         unsigned short id = lookup_swap_cgroup_id(ent);
5515 
5516                         rcu_read_lock();
5517                         memcg = mem_cgroup_from_id(id);
5518                         if (memcg && !css_tryget_online(&memcg->css))
5519                                 memcg = NULL;
5520                         rcu_read_unlock();
5521                 }
5522         }
5523 
5524         if (!memcg)
5525                 memcg = get_mem_cgroup_from_mm(mm);
5526 
5527         ret = try_charge(memcg, gfp_mask, nr_pages);
5528 
5529         css_put(&memcg->css);
5530 out:
5531         *memcgp = memcg;
5532         return ret;
5533 }
5534 
5535 /**
5536  * mem_cgroup_commit_charge - commit a page charge
5537  * @page: page to charge
5538  * @memcg: memcg to charge the page to
5539  * @lrucare: page might be on LRU already
5540  * @compound: charge the page as compound or small page
5541  *
5542  * Finalize a charge transaction started by mem_cgroup_try_charge(),
5543  * after page->mapping has been set up.  This must happen atomically
5544  * as part of the page instantiation, i.e. under the page table lock
5545  * for anonymous pages, under the page lock for page and swap cache.
5546  *
5547  * In addition, the page must not be on the LRU during the commit, to
5548  * prevent racing with task migration.  If it might be, use @lrucare.
5549  *
5550  * Use mem_cgroup_cancel_charge() to cancel the transaction instead.
5551  */
5552 void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
5553                               bool lrucare, bool compound)
5554 {
5555         unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
5556 
5557         VM_BUG_ON_PAGE(!page->mapping, page);
5558         VM_BUG_ON_PAGE(PageLRU(page) && !lrucare, page);
5559 
5560         if (mem_cgroup_disabled())
5561                 return;
5562         /*
5563          * Swap faults will attempt to charge the same page multiple
5564          * times.  But reuse_swap_page() might have removed the page
5565          * from swapcache already, so we can't check PageSwapCache().
5566          */
5567         if (!memcg)
5568                 return;
5569 
5570         commit_charge(page, memcg, lrucare);
5571 
5572         local_irq_disable();
5573         mem_cgroup_charge_statistics(memcg, page, compound, nr_pages);
5574         memcg_check_events(memcg, page);
5575         local_irq_enable();
5576 
5577         if (do_memsw_account() && PageSwapCache(page)) {
5578                 swp_entry_t entry = { .val = page_private(page) };
5579                 /*
5580                  * The swap entry might not get freed for a long time,
5581                  * let's not wait for it.  The page already received a
5582                  * memory+swap charge, drop the swap entry duplicate.
5583                  */
5584                 mem_cgroup_uncharge_swap(entry, nr_pages);
5585         }
5586 }
5587 
5588 /**
5589  * mem_cgroup_cancel_charge - cancel a page charge
5590  * @page: page to charge
5591  * @memcg: memcg to charge the page to
5592  * @compound: charge the page as compound or small page
5593  *
5594  * Cancel a charge transaction started by mem_cgroup_try_charge().
5595  */
5596 void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg,
5597                 bool compound)
5598 {
5599         unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
5600 
5601         if (mem_cgroup_disabled())
5602                 return;
5603         /*
5604          * Swap faults will attempt to charge the same page multiple
5605          * times.  But reuse_swap_page() might have removed the page
5606          * from swapcache already, so we can't check PageSwapCache().
5607          */
5608         if (!memcg)
5609                 return;
5610 
5611         cancel_charge(memcg, nr_pages);
5612 }
5613 
5614 struct uncharge_gather {
5615         struct mem_cgroup *memcg;
5616         unsigned long pgpgout;
5617         unsigned long nr_anon;
5618         unsigned long nr_file;
5619         unsigned long nr_kmem;
5620         unsigned long nr_huge;
5621         unsigned long nr_shmem;
5622         struct page *dummy_page;
5623 };
5624 
5625 static inline void uncharge_gather_clear(struct uncharge_gather *ug)
5626 {
5627         memset(ug, 0, sizeof(*ug));
5628 }
5629 
5630 static void uncharge_batch(const struct uncharge_gather *ug)
5631 {
5632         unsigned long nr_pages = ug->nr_anon + ug->nr_file + ug->nr_kmem;
5633         unsigned long flags;
5634 
5635         if (!mem_cgroup_is_root(ug->memcg)) {
5636                 page_counter_uncharge(&ug->memcg->memory, nr_pages);
5637                 if (do_memsw_account())
5638                         page_counter_uncharge(&ug->memcg->memsw, nr_pages);
5639                 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && ug->nr_kmem)
5640                         page_counter_uncharge(&ug->memcg->kmem, ug->nr_kmem);
5641                 memcg_oom_recover(ug->memcg);
5642         }
5643 
5644         local_irq_save(flags);
5645         __this_cpu_sub(ug->memcg->stat->count[MEMCG_RSS], ug->nr_anon);
5646         __this_cpu_sub(ug->memcg->stat->count[MEMCG_CACHE], ug->nr_file);
5647         __this_cpu_sub(ug->memcg->stat->count[MEMCG_RSS_HUGE], ug->nr_huge);
5648         __this_cpu_sub(ug->memcg->stat->count[NR_SHMEM], ug->nr_shmem);
5649         __this_cpu_add(ug->memcg->stat->events[PGPGOUT], ug->pgpgout);
5650         __this_cpu_add(ug->memcg->stat->nr_page_events, nr_pages);
5651         memcg_check_events(ug->memcg, ug->dummy_page);
5652         local_irq_restore(flags);
5653 
5654         if (!mem_cgroup_is_root(ug->memcg))
5655                 css_put_many(&ug->memcg->css, nr_pages);
5656 }
5657 
5658 static void uncharge_page(struct page *page, struct uncharge_gather *ug)
5659 {
5660         VM_BUG_ON_PAGE(PageLRU(page), page);
5661         VM_BUG_ON_PAGE(page_count(page) && !is_zone_device_page(page) &&
5662                         !PageHWPoison(page) , page);
5663 
5664         if (!page->mem_cgroup)
5665                 return;
5666 
5667         /*
5668          * Nobody should be changing or seriously looking at
5669          * page->mem_cgroup at this point, we have fully
5670          * exclusive access to the page.
5671          */
5672 
5673         if (ug->memcg != page->mem_cgroup) {
5674                 if (ug->memcg) {
5675                         uncharge_batch(ug);
5676                         uncharge_gather_clear(ug);
5677                 }
5678                 ug->memcg = page->mem_cgroup;
5679         }
5680 
5681         if (!PageKmemcg(page)) {
5682                 unsigned int nr_pages = 1;
5683 
5684                 if (PageTransHuge(page)) {
5685                         nr_pages <<= compound_order(page);
5686                         ug->nr_huge += nr_pages;
5687                 }
5688                 if (PageAnon(page))
5689                         ug->nr_anon += nr_pages;
5690                 else {
5691                         ug->nr_file += nr_pages;
5692                         if (PageSwapBacked(page))
5693                                 ug->nr_shmem += nr_pages;
5694                 }
5695                 ug->pgpgout++;
5696         } else {
5697                 ug->nr_kmem += 1 << compound_order(page);
5698                 __ClearPageKmemcg(page);
5699         }
5700 
5701         ug->dummy_page = page;
5702         page->mem_cgroup = NULL;
5703 }
5704 
5705 static void uncharge_list(struct list_head *page_list)
5706 {
5707         struct uncharge_gather ug;
5708         struct list_head *next;
5709 
5710         uncharge_gather_clear(&ug);
5711 
5712         /*
5713          * Note that the list can be a single page->lru; hence the
5714          * do-while loop instead of a simple list_for_each_entry().
5715          */
5716         next = page_list->next;
5717         do {
5718                 struct page *page;
5719 
5720                 page = list_entry(next, struct page, lru);
5721                 next = page->lru.next;
5722 
5723                 uncharge_page(page, &ug);
5724         } while (next != page_list);
5725 
5726         if (ug.memcg)
5727                 uncharge_batch(&ug);
5728 }
5729 
5730 /**
5731  * mem_cgroup_uncharge - uncharge a page
5732  * @page: page to uncharge
5733  *
5734  * Uncharge a page previously charged with mem_cgroup_try_charge() and
5735  * mem_cgroup_commit_charge().
5736  */
5737 void mem_cgroup_uncharge(struct page *page)
5738 {
5739         struct uncharge_gather ug;
5740 
5741         if (mem_cgroup_disabled())
5742                 return;
5743 
5744         /* Don't touch page->lru of any random page, pre-check: */
5745         if (!page->mem_cgroup)
5746                 return;
5747 
5748         uncharge_gather_clear(&ug);
5749         uncharge_page(page, &ug);
5750         uncharge_batch(&ug);
5751 }
5752 
5753 /**
5754  * mem_cgroup_uncharge_list - uncharge a list of page
5755  * @page_list: list of pages to uncharge
5756  *
5757  * Uncharge a list of pages previously charged with
5758  * mem_cgroup_try_charge() and mem_cgroup_commit_charge().
5759  */
5760 void mem_cgroup_uncharge_list(struct list_head *page_list)
5761 {
5762         if (mem_cgroup_disabled())
5763                 return;
5764 
5765         if (!list_empty(page_list))
5766                 uncharge_list(page_list);
5767 }
5768 
5769 /**
5770  * mem_cgroup_migrate - charge a page's replacement
5771  * @oldpage: currently circulating page
5772  * @newpage: replacement page
5773  *
5774  * Charge @newpage as a replacement page for @oldpage. @oldpage will
5775  * be uncharged upon free.
5776  *
5777  * Both pages must be locked, @newpage->mapping must be set up.
5778  */
5779 void mem_cgroup_migrate(struct page *oldpage, struct page *newpage)
5780 {
5781         struct mem_cgroup *memcg;
5782         unsigned int nr_pages;
5783         bool compound;
5784         unsigned long flags;
5785 
5786         VM_BUG_ON_PAGE(!PageLocked(oldpage), oldpage);
5787         VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
5788         VM_BUG_ON_PAGE(PageAnon(oldpage) != PageAnon(newpage), newpage);
5789         VM_BUG_ON_PAGE(PageTransHuge(oldpage) != PageTransHuge(newpage),
5790                        newpage);
5791 
5792         if (mem_cgroup_disabled())
5793                 return;
5794 
5795         /* Page cache replacement: new page already charged? */
5796         if (newpage->mem_cgroup)
5797                 return;
5798 
5799         /* Swapcache readahead pages can get replaced before being charged */
5800         memcg = oldpage->mem_cgroup;
5801         if (!memcg)
5802                 return;
5803 
5804         /* Force-charge the new page. The old one will be freed soon */
5805         compound = PageTransHuge(newpage);
5806         nr_pages = compound ? hpage_nr_pages(newpage) : 1;
5807 
5808         page_counter_charge(&memcg->memory, nr_pages);
5809         if (do_memsw_account())
5810                 page_counter_charge(&memcg->memsw, nr_pages);
5811         css_get_many(&memcg->css, nr_pages);
5812 
5813         commit_charge(newpage, memcg, false);
5814 
5815         local_irq_save(flags);
5816         mem_cgroup_charge_statistics(memcg, newpage, compound, nr_pages);
5817         memcg_check_events(memcg, newpage);
5818         local_irq_restore(flags);
5819 }
5820 
5821 DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key);
5822 EXPORT_SYMBOL(memcg_sockets_enabled_key);
5823 
5824 void mem_cgroup_sk_alloc(struct sock *sk)
5825 {
5826         struct mem_cgroup *memcg;
5827 
5828         if (!mem_cgroup_sockets_enabled)
5829                 return;
5830 
5831         rcu_read_lock();
5832         memcg = mem_cgroup_from_task(current);
5833         if (memcg == root_mem_cgroup)
5834                 goto out;
5835         if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active)
5836                 goto out;
5837         if (css_tryget_online(&memcg->css))
5838                 sk->sk_memcg = memcg;
5839 out:
5840         rcu_read_unlock();
5841 }
5842 
5843 void mem_cgroup_sk_free(struct sock *sk)
5844 {
5845         if (sk->sk_memcg)
5846                 css_put(&sk->sk_memcg->css);
5847 }
5848 
5849 /**
5850  * mem_cgroup_charge_skmem - charge socket memory
5851  * @memcg: memcg to charge
5852  * @nr_pages: number of pages to charge
5853  *
5854  * Charges @nr_pages to @memcg. Returns %true if the charge fit within
5855  * @memcg's configured limit, %false if the charge had to be forced.
5856  */
5857 bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
5858 {
5859         gfp_t gfp_mask = GFP_KERNEL;
5860 
5861         if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
5862                 struct page_counter *fail;
5863 
5864                 if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) {
5865                         memcg->tcpmem_pressure = 0;
5866                         return true;
5867                 }
5868                 page_counter_charge(&memcg->tcpmem, nr_pages);
5869                 memcg->tcpmem_pressure = 1;
5870                 return false;
5871         }
5872 
5873         /* Don't block in the packet receive path */
5874         if (in_softirq())
5875                 gfp_mask = GFP_NOWAIT;
5876 
5877         this_cpu_add(memcg->stat->count[MEMCG_SOCK], nr_pages);
5878 
5879         if (try_charge(memcg, gfp_mask, nr_pages) == 0)
5880                 return true;
5881 
5882         try_charge(memcg, gfp_mask|__GFP_NOFAIL, nr_pages);
5883         return false;
5884 }
5885 
5886 /**
5887  * mem_cgroup_uncharge_skmem - uncharge socket memory
5888  * @memcg - memcg to uncharge
5889  * @nr_pages - number of pages to uncharge
5890  */
5891 void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
5892 {
5893         if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
5894                 page_counter_uncharge(&memcg->tcpmem, nr_pages);
5895                 return;
5896         }
5897 
5898         this_cpu_sub(memcg->stat->count[MEMCG_SOCK], nr_pages);
5899 
5900         refill_stock(memcg, nr_pages);
5901 }
5902 
5903 static int __init cgroup_memory(char *s)
5904 {
5905         char *token;
5906 
5907         while ((token = strsep(&s, ",")) != NULL) {
5908                 if (!*token)
5909                         continue;
5910                 if (!strcmp(token, "nosocket"))
5911                         cgroup_memory_nosocket = true;
5912                 if (!strcmp(token, "nokmem"))
5913                         cgroup_memory_nokmem = true;
5914         }
5915         return 0;
5916 }
5917 __setup("cgroup.memory=", cgroup_memory);
5918 
5919 /*
5920  * subsys_initcall() for memory controller.
5921  *
5922  * Some parts like memcg_hotplug_cpu_dead() have to be initialized from this
5923  * context because of lock dependencies (cgroup_lock -> cpu hotplug) but
5924  * basically everything that doesn't depend on a specific mem_cgroup structure
5925  * should be initialized from here.
5926  */
5927 static int __init mem_cgroup_init(void)
5928 {
5929         int cpu, node;
5930 
5931 #ifndef CONFIG_SLOB
5932         /*
5933          * Kmem cache creation is mostly done with the slab_mutex held,
5934          * so use a workqueue with limited concurrency to avoid stalling
5935          * all worker threads in case lots of cgroups are created and
5936          * destroyed simultaneously.
5937          */
5938         memcg_kmem_cache_wq = alloc_workqueue("memcg_kmem_cache", 0, 1);
5939         BUG_ON(!memcg_kmem_cache_wq);
5940 #endif
5941 
5942         cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL,
5943                                   memcg_hotplug_cpu_dead);
5944 
5945         for_each_possible_cpu(cpu)
5946                 INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work,
5947                           drain_local_stock);
5948 
5949         for_each_node(node) {
5950                 struct mem_cgroup_tree_per_node *rtpn;
5951 
5952                 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL,
5953                                     node_online(node) ? node : NUMA_NO_NODE);
5954 
5955                 rtpn->rb_root = RB_ROOT;
5956                 rtpn->rb_rightmost = NULL;
5957                 spin_lock_init(&rtpn->lock);
5958                 soft_limit_tree.rb_tree_per_node[node] = rtpn;
5959         }
5960 
5961         return 0;
5962 }
5963 subsys_initcall(mem_cgroup_init);
5964 
5965 #ifdef CONFIG_MEMCG_SWAP
5966 static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg)
5967 {
5968         while (!atomic_inc_not_zero(&memcg->id.ref)) {
5969                 /*
5970                  * The root cgroup cannot be destroyed, so it's refcount must
5971                  * always be >= 1.
5972                  */
5973                 if (WARN_ON_ONCE(memcg == root_mem_cgroup)) {
5974                         VM_BUG_ON(1);
5975                         break;
5976                 }
5977                 memcg = parent_mem_cgroup(memcg);
5978                 if (!memcg)
5979                         memcg = root_mem_cgroup;
5980         }
5981         return memcg;
5982 }
5983 
5984 /**
5985  * mem_cgroup_swapout - transfer a memsw charge to swap
5986  * @page: page whose memsw charge to transfer
5987  * @entry: swap entry to move the charge to
5988  *
5989  * Transfer the memsw charge of @page to @entry.
5990  */
5991 void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
5992 {
5993         struct mem_cgroup *memcg, *swap_memcg;
5994         unsigned int nr_entries;
5995         unsigned short oldid;
5996 
5997         VM_BUG_ON_PAGE(PageLRU(page), page);
5998         VM_BUG_ON_PAGE(page_count(page), page);
5999 
6000         if (!do_memsw_account())
6001                 return;
6002 
6003         memcg = page->mem_cgroup;
6004 
6005         /* Readahead page, never charged */
6006         if (!memcg)
6007                 return;
6008 
6009         /*
6010          * In case the memcg owning these pages has been offlined and doesn't
6011          * have an ID allocated to it anymore, charge the closest online
6012          * ancestor for the swap instead and transfer the memory+swap charge.
6013          */
6014         swap_memcg = mem_cgroup_id_get_online(memcg);
6015         nr_entries = hpage_nr_pages(page);
6016         /* Get references for the tail pages, too */
6017         if (nr_entries > 1)
6018                 mem_cgroup_id_get_many(swap_memcg, nr_entries - 1);
6019         oldid = swap_cgroup_record(entry, mem_cgroup_id(swap_memcg),
6020                                    nr_entries);
6021         VM_BUG_ON_PAGE(oldid, page);
6022         mem_cgroup_swap_statistics(swap_memcg, nr_entries);
6023 
6024         page->mem_cgroup = NULL;
6025 
6026         if (!mem_cgroup_is_root(memcg))
6027                 page_counter_uncharge(&memcg->memory, nr_entries);
6028 
6029         if (memcg != swap_memcg) {
6030                 if (!mem_cgroup_is_root(swap_memcg))
6031                         page_counter_charge(&swap_memcg->memsw, nr_entries);
6032                 page_counter_uncharge(&memcg->memsw, nr_entries);
6033         }
6034 
6035         /*
6036          * Interrupts should be disabled here because the caller holds the
6037          * mapping->tree_lock lock which is taken with interrupts-off. It is
6038          * important here to have the interrupts disabled because it is the
6039          * only synchronisation we have for udpating the per-CPU variables.
6040          */
6041         VM_BUG_ON(!irqs_disabled());
6042         mem_cgroup_charge_statistics(memcg, page, PageTransHuge(page),
6043                                      -nr_entries);
6044         memcg_check_events(memcg, page);
6045 
6046         if (!mem_cgroup_is_root(memcg))
6047                 css_put(&memcg->css);
6048 }
6049 
6050 /**
6051  * mem_cgroup_try_charge_swap - try charging swap space for a page
6052  * @page: page being added to swap
6053  * @entry: swap entry to charge
6054  *
6055  * Try to charge @page's memcg for the swap space at @entry.
6056  *
6057  * Returns 0 on success, -ENOMEM on failure.
6058  */
6059 int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
6060 {
6061         unsigned int nr_pages = hpage_nr_pages(page);
6062         struct page_counter *counter;
6063         struct mem_cgroup *memcg;
6064         unsigned short oldid;
6065 
6066         if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) || !do_swap_account)
6067                 return 0;
6068 
6069         memcg = page->mem_cgroup;
6070 
6071         /* Readahead page, never charged */
6072         if (!memcg)
6073                 return 0;
6074 
6075         memcg = mem_cgroup_id_get_online(memcg);
6076 
6077         if (!mem_cgroup_is_root(memcg) &&
6078             !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) {
6079                 mem_cgroup_id_put(memcg);
6080                 return -ENOMEM;
6081         }
6082 
6083         /* Get references for the tail pages, too */
6084         if (nr_pages > 1)
6085                 mem_cgroup_id_get_many(memcg, nr_pages - 1);
6086         oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg), nr_pages);
6087         VM_BUG_ON_PAGE(oldid, page);
6088         mem_cgroup_swap_statistics(memcg, nr_pages);
6089 
6090         return 0;
6091 }
6092 
6093 /**
6094  * mem_cgroup_uncharge_swap - uncharge swap space
6095  * @entry: swap entry to uncharge
6096  * @nr_pages: the amount of swap space to uncharge
6097  */
6098 void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
6099 {
6100         struct mem_cgroup *memcg;
6101         unsigned short id;
6102 
6103         if (!do_swap_account)
6104                 return;
6105 
6106         id = swap_cgroup_record(entry, 0, nr_pages);
6107         rcu_read_lock();
6108         memcg = mem_cgroup_from_id(id);
6109         if (memcg) {
6110                 if (!mem_cgroup_is_root(memcg)) {
6111                         if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
6112                                 page_counter_uncharge(&memcg->swap, nr_pages);
6113                         else
6114                                 page_counter_uncharge(&memcg->memsw, nr_pages);
6115                 }
6116                 mem_cgroup_swap_statistics(memcg, -nr_pages);
6117                 mem_cgroup_id_put_many(memcg, nr_pages);
6118         }
6119         rcu_read_unlock();
6120 }
6121 
6122 long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
6123 {
6124         long nr_swap_pages = get_nr_swap_pages();
6125 
6126         if (!do_swap_account || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
6127                 return nr_swap_pages;
6128         for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
6129                 nr_swap_pages = min_t(long, nr_swap_pages,
6130                                       READ_ONCE(memcg->swap.limit) -
6131                                       page_counter_read(&memcg->swap));
6132         return nr_swap_pages;
6133 }
6134 
6135 bool mem_cgroup_swap_full(struct page *page)
6136 {
6137         struct mem_cgroup *memcg;
6138 
6139         VM_BUG_ON_PAGE(!PageLocked(page), page);
6140 
6141         if (vm_swap_full())
6142                 return true;
6143         if (!do_swap_account || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
6144                 return false;
6145 
6146         memcg = page->mem_cgroup;
6147         if (!memcg)
6148                 return false;
6149 
6150         for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
6151                 if (page_counter_read(&memcg->swap) * 2 >= memcg->swap.limit)
6152                         return true;
6153 
6154         return false;
6155 }
6156 
6157 /* for remember boot option*/
6158 #ifdef CONFIG_MEMCG_SWAP_ENABLED
6159 static int really_do_swap_account __initdata = 1;
6160 #else
6161 static int really_do_swap_account __initdata;
6162 #endif
6163 
6164 static int __init enable_swap_account(char *s)
6165 {
6166         if (!strcmp(s, "1"))
6167                 really_do_swap_account = 1;
6168         else if (!strcmp(s, ""))
6169                 really_do_swap_account = 0;
6170         return 1;
6171 }
6172 __setup("swapaccount=", enable_swap_account);
6173 
6174 static u64 swap_current_read(struct cgroup_subsys_state *css,
6175                              struct cftype *cft)
6176 {
6177         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
6178 
6179         return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE;
6180 }
6181 
6182 static int swap_max_show(struct seq_file *m, void *v)
6183 {
6184         struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
6185         unsigned long max = READ_ONCE(memcg->swap.limit);
6186 
6187         if (max == PAGE_COUNTER_MAX)
6188                 seq_puts(m, "max\n");
6189         else
6190                 seq_printf(m, "%llu\n", (u64)max * PAGE_SIZE);
6191 
6192         return 0;
6193 }
6194 
6195 static ssize_t swap_max_write(struct kernfs_open_file *of,
6196                               char *buf, size_t nbytes, loff_t off)
6197 {
6198         struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6199         unsigned long max;
6200         int err;
6201 
6202         buf = strstrip(buf);
6203         err = page_counter_memparse(buf, "max", &max);
6204         if (err)
6205                 return err;
6206 
6207         mutex_lock(&memcg_limit_mutex);
6208         err = page_counter_limit(&memcg->swap, max);
6209         mutex_unlock(&memcg_limit_mutex);
6210         if (err)
6211                 return err;
6212 
6213         return nbytes;
6214 }
6215 
6216 static struct cftype swap_files[] = {
6217         {
6218                 .name = "swap.current",
6219                 .flags = CFTYPE_NOT_ON_ROOT,
6220                 .read_u64 = swap_current_read,
6221         },
6222         {
6223                 .name = "swap.max",
6224                 .flags = CFTYPE_NOT_ON_ROOT,
6225                 .seq_show = swap_max_show,
6226                 .write = swap_max_write,
6227         },
6228         { }     /* terminate */
6229 };
6230 
6231 static struct cftype memsw_cgroup_files[] = {
6232         {
6233                 .name = "memsw.usage_in_bytes",
6234                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
6235                 .read_u64 = mem_cgroup_read_u64,
6236         },
6237         {
6238                 .name = "memsw.max_usage_in_bytes",
6239                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
6240                 .write = mem_cgroup_reset,
6241                 .read_u64 = mem_cgroup_read_u64,
6242         },
6243         {
6244                 .name = "memsw.limit_in_bytes",
6245                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
6246                 .write = mem_cgroup_write,
6247                 .read_u64 = mem_cgroup_read_u64,
6248         },
6249         {
6250                 .name = "memsw.failcnt",
6251                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
6252                 .write = mem_cgroup_reset,
6253                 .read_u64 = mem_cgroup_read_u64,
6254         },
6255         { },    /* terminate */
6256 };
6257 
6258 static int __init mem_cgroup_swap_init(void)
6259 {
6260         if (!mem_cgroup_disabled() && really_do_swap_account) {
6261                 do_swap_account = 1;
6262                 WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys,
6263                                                swap_files));
6264                 WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys,
6265                                                   memsw_cgroup_files));
6266         }
6267         return 0;
6268 }
6269 subsys_initcall(mem_cgroup_swap_init);
6270 
6271 #endif /* CONFIG_MEMCG_SWAP */
6272 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp