~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/mm/mempolicy.c

Version: ~ [ linux-5.5-rc7 ] ~ [ linux-5.4.13 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.97 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.166 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.210 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.210 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.81 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Simple NUMA memory policy for the Linux kernel.
  3  *
  4  * Copyright 2003,2004 Andi Kleen, SuSE Labs.
  5  * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.
  6  * Subject to the GNU Public License, version 2.
  7  *
  8  * NUMA policy allows the user to give hints in which node(s) memory should
  9  * be allocated.
 10  *
 11  * Support four policies per VMA and per process:
 12  *
 13  * The VMA policy has priority over the process policy for a page fault.
 14  *
 15  * interleave     Allocate memory interleaved over a set of nodes,
 16  *                with normal fallback if it fails.
 17  *                For VMA based allocations this interleaves based on the
 18  *                offset into the backing object or offset into the mapping
 19  *                for anonymous memory. For process policy an process counter
 20  *                is used.
 21  *
 22  * bind           Only allocate memory on a specific set of nodes,
 23  *                no fallback.
 24  *                FIXME: memory is allocated starting with the first node
 25  *                to the last. It would be better if bind would truly restrict
 26  *                the allocation to memory nodes instead
 27  *
 28  * preferred       Try a specific node first before normal fallback.
 29  *                As a special case NUMA_NO_NODE here means do the allocation
 30  *                on the local CPU. This is normally identical to default,
 31  *                but useful to set in a VMA when you have a non default
 32  *                process policy.
 33  *
 34  * default        Allocate on the local node first, or when on a VMA
 35  *                use the process policy. This is what Linux always did
 36  *                in a NUMA aware kernel and still does by, ahem, default.
 37  *
 38  * The process policy is applied for most non interrupt memory allocations
 39  * in that process' context. Interrupts ignore the policies and always
 40  * try to allocate on the local CPU. The VMA policy is only applied for memory
 41  * allocations for a VMA in the VM.
 42  *
 43  * Currently there are a few corner cases in swapping where the policy
 44  * is not applied, but the majority should be handled. When process policy
 45  * is used it is not remembered over swap outs/swap ins.
 46  *
 47  * Only the highest zone in the zone hierarchy gets policied. Allocations
 48  * requesting a lower zone just use default policy. This implies that
 49  * on systems with highmem kernel lowmem allocation don't get policied.
 50  * Same with GFP_DMA allocations.
 51  *
 52  * For shmfs/tmpfs/hugetlbfs shared memory the policy is shared between
 53  * all users and remembered even when nobody has memory mapped.
 54  */
 55 
 56 /* Notebook:
 57    fix mmap readahead to honour policy and enable policy for any page cache
 58    object
 59    statistics for bigpages
 60    global policy for page cache? currently it uses process policy. Requires
 61    first item above.
 62    handle mremap for shared memory (currently ignored for the policy)
 63    grows down?
 64    make bind policy root only? It can trigger oom much faster and the
 65    kernel is not always grateful with that.
 66 */
 67 
 68 #include <linux/mempolicy.h>
 69 #include <linux/mm.h>
 70 #include <linux/highmem.h>
 71 #include <linux/hugetlb.h>
 72 #include <linux/kernel.h>
 73 #include <linux/sched.h>
 74 #include <linux/nodemask.h>
 75 #include <linux/cpuset.h>
 76 #include <linux/slab.h>
 77 #include <linux/string.h>
 78 #include <linux/export.h>
 79 #include <linux/nsproxy.h>
 80 #include <linux/interrupt.h>
 81 #include <linux/init.h>
 82 #include <linux/compat.h>
 83 #include <linux/swap.h>
 84 #include <linux/seq_file.h>
 85 #include <linux/proc_fs.h>
 86 #include <linux/migrate.h>
 87 #include <linux/ksm.h>
 88 #include <linux/rmap.h>
 89 #include <linux/security.h>
 90 #include <linux/syscalls.h>
 91 #include <linux/ctype.h>
 92 #include <linux/mm_inline.h>
 93 #include <linux/mmu_notifier.h>
 94 
 95 #include <asm/tlbflush.h>
 96 #include <asm/uaccess.h>
 97 #include <linux/random.h>
 98 
 99 #include "internal.h"
100 
101 /* Internal flags */
102 #define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0)    /* Skip checks for continuous vmas */
103 #define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1)          /* Invert check for nodemask */
104 
105 static struct kmem_cache *policy_cache;
106 static struct kmem_cache *sn_cache;
107 
108 /* Highest zone. An specific allocation for a zone below that is not
109    policied. */
110 enum zone_type policy_zone = 0;
111 
112 /*
113  * run-time system-wide default policy => local allocation
114  */
115 static struct mempolicy default_policy = {
116         .refcnt = ATOMIC_INIT(1), /* never free it */
117         .mode = MPOL_PREFERRED,
118         .flags = MPOL_F_LOCAL,
119 };
120 
121 static struct mempolicy preferred_node_policy[MAX_NUMNODES];
122 
123 static struct mempolicy *get_task_policy(struct task_struct *p)
124 {
125         struct mempolicy *pol = p->mempolicy;
126 
127         if (!pol) {
128                 int node = numa_node_id();
129 
130                 if (node != NUMA_NO_NODE) {
131                         pol = &preferred_node_policy[node];
132                         /*
133                          * preferred_node_policy is not initialised early in
134                          * boot
135                          */
136                         if (!pol->mode)
137                                 pol = NULL;
138                 }
139         }
140 
141         return pol;
142 }
143 
144 static const struct mempolicy_operations {
145         int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
146         /*
147          * If read-side task has no lock to protect task->mempolicy, write-side
148          * task will rebind the task->mempolicy by two step. The first step is
149          * setting all the newly nodes, and the second step is cleaning all the
150          * disallowed nodes. In this way, we can avoid finding no node to alloc
151          * page.
152          * If we have a lock to protect task->mempolicy in read-side, we do
153          * rebind directly.
154          *
155          * step:
156          *      MPOL_REBIND_ONCE - do rebind work at once
157          *      MPOL_REBIND_STEP1 - set all the newly nodes
158          *      MPOL_REBIND_STEP2 - clean all the disallowed nodes
159          */
160         void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes,
161                         enum mpol_rebind_step step);
162 } mpol_ops[MPOL_MAX];
163 
164 /* Check that the nodemask contains at least one populated zone */
165 static int is_valid_nodemask(const nodemask_t *nodemask)
166 {
167         return nodes_intersects(*nodemask, node_states[N_MEMORY]);
168 }
169 
170 static inline int mpol_store_user_nodemask(const struct mempolicy *pol)
171 {
172         return pol->flags & MPOL_MODE_FLAGS;
173 }
174 
175 static void mpol_relative_nodemask(nodemask_t *ret, const nodemask_t *orig,
176                                    const nodemask_t *rel)
177 {
178         nodemask_t tmp;
179         nodes_fold(tmp, *orig, nodes_weight(*rel));
180         nodes_onto(*ret, tmp, *rel);
181 }
182 
183 static int mpol_new_interleave(struct mempolicy *pol, const nodemask_t *nodes)
184 {
185         if (nodes_empty(*nodes))
186                 return -EINVAL;
187         pol->v.nodes = *nodes;
188         return 0;
189 }
190 
191 static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes)
192 {
193         if (!nodes)
194                 pol->flags |= MPOL_F_LOCAL;     /* local allocation */
195         else if (nodes_empty(*nodes))
196                 return -EINVAL;                 /*  no allowed nodes */
197         else
198                 pol->v.preferred_node = first_node(*nodes);
199         return 0;
200 }
201 
202 static int mpol_new_bind(struct mempolicy *pol, const nodemask_t *nodes)
203 {
204         if (!is_valid_nodemask(nodes))
205                 return -EINVAL;
206         pol->v.nodes = *nodes;
207         return 0;
208 }
209 
210 /*
211  * mpol_set_nodemask is called after mpol_new() to set up the nodemask, if
212  * any, for the new policy.  mpol_new() has already validated the nodes
213  * parameter with respect to the policy mode and flags.  But, we need to
214  * handle an empty nodemask with MPOL_PREFERRED here.
215  *
216  * Must be called holding task's alloc_lock to protect task's mems_allowed
217  * and mempolicy.  May also be called holding the mmap_semaphore for write.
218  */
219 static int mpol_set_nodemask(struct mempolicy *pol,
220                      const nodemask_t *nodes, struct nodemask_scratch *nsc)
221 {
222         int ret;
223 
224         /* if mode is MPOL_DEFAULT, pol is NULL. This is right. */
225         if (pol == NULL)
226                 return 0;
227         /* Check N_MEMORY */
228         nodes_and(nsc->mask1,
229                   cpuset_current_mems_allowed, node_states[N_MEMORY]);
230 
231         VM_BUG_ON(!nodes);
232         if (pol->mode == MPOL_PREFERRED && nodes_empty(*nodes))
233                 nodes = NULL;   /* explicit local allocation */
234         else {
235                 if (pol->flags & MPOL_F_RELATIVE_NODES)
236                         mpol_relative_nodemask(&nsc->mask2, nodes,&nsc->mask1);
237                 else
238                         nodes_and(nsc->mask2, *nodes, nsc->mask1);
239 
240                 if (mpol_store_user_nodemask(pol))
241                         pol->w.user_nodemask = *nodes;
242                 else
243                         pol->w.cpuset_mems_allowed =
244                                                 cpuset_current_mems_allowed;
245         }
246 
247         if (nodes)
248                 ret = mpol_ops[pol->mode].create(pol, &nsc->mask2);
249         else
250                 ret = mpol_ops[pol->mode].create(pol, NULL);
251         return ret;
252 }
253 
254 /*
255  * This function just creates a new policy, does some check and simple
256  * initialization. You must invoke mpol_set_nodemask() to set nodes.
257  */
258 static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags,
259                                   nodemask_t *nodes)
260 {
261         struct mempolicy *policy;
262 
263         pr_debug("setting mode %d flags %d nodes[0] %lx\n",
264                  mode, flags, nodes ? nodes_addr(*nodes)[0] : NUMA_NO_NODE);
265 
266         if (mode == MPOL_DEFAULT) {
267                 if (nodes && !nodes_empty(*nodes))
268                         return ERR_PTR(-EINVAL);
269                 return NULL;
270         }
271         VM_BUG_ON(!nodes);
272 
273         /*
274          * MPOL_PREFERRED cannot be used with MPOL_F_STATIC_NODES or
275          * MPOL_F_RELATIVE_NODES if the nodemask is empty (local allocation).
276          * All other modes require a valid pointer to a non-empty nodemask.
277          */
278         if (mode == MPOL_PREFERRED) {
279                 if (nodes_empty(*nodes)) {
280                         if (((flags & MPOL_F_STATIC_NODES) ||
281                              (flags & MPOL_F_RELATIVE_NODES)))
282                                 return ERR_PTR(-EINVAL);
283                 }
284         } else if (mode == MPOL_LOCAL) {
285                 if (!nodes_empty(*nodes))
286                         return ERR_PTR(-EINVAL);
287                 mode = MPOL_PREFERRED;
288         } else if (nodes_empty(*nodes))
289                 return ERR_PTR(-EINVAL);
290         policy = kmem_cache_alloc(policy_cache, GFP_KERNEL);
291         if (!policy)
292                 return ERR_PTR(-ENOMEM);
293         atomic_set(&policy->refcnt, 1);
294         policy->mode = mode;
295         policy->flags = flags;
296 
297         return policy;
298 }
299 
300 /* Slow path of a mpol destructor. */
301 void __mpol_put(struct mempolicy *p)
302 {
303         if (!atomic_dec_and_test(&p->refcnt))
304                 return;
305         kmem_cache_free(policy_cache, p);
306 }
307 
308 static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes,
309                                 enum mpol_rebind_step step)
310 {
311 }
312 
313 /*
314  * step:
315  *      MPOL_REBIND_ONCE  - do rebind work at once
316  *      MPOL_REBIND_STEP1 - set all the newly nodes
317  *      MPOL_REBIND_STEP2 - clean all the disallowed nodes
318  */
319 static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes,
320                                  enum mpol_rebind_step step)
321 {
322         nodemask_t tmp;
323 
324         if (pol->flags & MPOL_F_STATIC_NODES)
325                 nodes_and(tmp, pol->w.user_nodemask, *nodes);
326         else if (pol->flags & MPOL_F_RELATIVE_NODES)
327                 mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
328         else {
329                 /*
330                  * if step == 1, we use ->w.cpuset_mems_allowed to cache the
331                  * result
332                  */
333                 if (step == MPOL_REBIND_ONCE || step == MPOL_REBIND_STEP1) {
334                         nodes_remap(tmp, pol->v.nodes,
335                                         pol->w.cpuset_mems_allowed, *nodes);
336                         pol->w.cpuset_mems_allowed = step ? tmp : *nodes;
337                 } else if (step == MPOL_REBIND_STEP2) {
338                         tmp = pol->w.cpuset_mems_allowed;
339                         pol->w.cpuset_mems_allowed = *nodes;
340                 } else
341                         BUG();
342         }
343 
344         if (nodes_empty(tmp))
345                 tmp = *nodes;
346 
347         if (step == MPOL_REBIND_STEP1)
348                 nodes_or(pol->v.nodes, pol->v.nodes, tmp);
349         else if (step == MPOL_REBIND_ONCE || step == MPOL_REBIND_STEP2)
350                 pol->v.nodes = tmp;
351         else
352                 BUG();
353 
354         if (!node_isset(current->il_next, tmp)) {
355                 current->il_next = next_node(current->il_next, tmp);
356                 if (current->il_next >= MAX_NUMNODES)
357                         current->il_next = first_node(tmp);
358                 if (current->il_next >= MAX_NUMNODES)
359                         current->il_next = numa_node_id();
360         }
361 }
362 
363 static void mpol_rebind_preferred(struct mempolicy *pol,
364                                   const nodemask_t *nodes,
365                                   enum mpol_rebind_step step)
366 {
367         nodemask_t tmp;
368 
369         if (pol->flags & MPOL_F_STATIC_NODES) {
370                 int node = first_node(pol->w.user_nodemask);
371 
372                 if (node_isset(node, *nodes)) {
373                         pol->v.preferred_node = node;
374                         pol->flags &= ~MPOL_F_LOCAL;
375                 } else
376                         pol->flags |= MPOL_F_LOCAL;
377         } else if (pol->flags & MPOL_F_RELATIVE_NODES) {
378                 mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
379                 pol->v.preferred_node = first_node(tmp);
380         } else if (!(pol->flags & MPOL_F_LOCAL)) {
381                 pol->v.preferred_node = node_remap(pol->v.preferred_node,
382                                                    pol->w.cpuset_mems_allowed,
383                                                    *nodes);
384                 pol->w.cpuset_mems_allowed = *nodes;
385         }
386 }
387 
388 /*
389  * mpol_rebind_policy - Migrate a policy to a different set of nodes
390  *
391  * If read-side task has no lock to protect task->mempolicy, write-side
392  * task will rebind the task->mempolicy by two step. The first step is
393  * setting all the newly nodes, and the second step is cleaning all the
394  * disallowed nodes. In this way, we can avoid finding no node to alloc
395  * page.
396  * If we have a lock to protect task->mempolicy in read-side, we do
397  * rebind directly.
398  *
399  * step:
400  *      MPOL_REBIND_ONCE  - do rebind work at once
401  *      MPOL_REBIND_STEP1 - set all the newly nodes
402  *      MPOL_REBIND_STEP2 - clean all the disallowed nodes
403  */
404 static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask,
405                                 enum mpol_rebind_step step)
406 {
407         if (!pol)
408                 return;
409         if (!mpol_store_user_nodemask(pol) && step == MPOL_REBIND_ONCE &&
410             nodes_equal(pol->w.cpuset_mems_allowed, *newmask))
411                 return;
412 
413         if (step == MPOL_REBIND_STEP1 && (pol->flags & MPOL_F_REBINDING))
414                 return;
415 
416         if (step == MPOL_REBIND_STEP2 && !(pol->flags & MPOL_F_REBINDING))
417                 BUG();
418 
419         if (step == MPOL_REBIND_STEP1)
420                 pol->flags |= MPOL_F_REBINDING;
421         else if (step == MPOL_REBIND_STEP2)
422                 pol->flags &= ~MPOL_F_REBINDING;
423         else if (step >= MPOL_REBIND_NSTEP)
424                 BUG();
425 
426         mpol_ops[pol->mode].rebind(pol, newmask, step);
427 }
428 
429 /*
430  * Wrapper for mpol_rebind_policy() that just requires task
431  * pointer, and updates task mempolicy.
432  *
433  * Called with task's alloc_lock held.
434  */
435 
436 void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new,
437                         enum mpol_rebind_step step)
438 {
439         mpol_rebind_policy(tsk->mempolicy, new, step);
440 }
441 
442 /*
443  * Rebind each vma in mm to new nodemask.
444  *
445  * Call holding a reference to mm.  Takes mm->mmap_sem during call.
446  */
447 
448 void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
449 {
450         struct vm_area_struct *vma;
451 
452         down_write(&mm->mmap_sem);
453         for (vma = mm->mmap; vma; vma = vma->vm_next)
454                 mpol_rebind_policy(vma->vm_policy, new, MPOL_REBIND_ONCE);
455         up_write(&mm->mmap_sem);
456 }
457 
458 static const struct mempolicy_operations mpol_ops[MPOL_MAX] = {
459         [MPOL_DEFAULT] = {
460                 .rebind = mpol_rebind_default,
461         },
462         [MPOL_INTERLEAVE] = {
463                 .create = mpol_new_interleave,
464                 .rebind = mpol_rebind_nodemask,
465         },
466         [MPOL_PREFERRED] = {
467                 .create = mpol_new_preferred,
468                 .rebind = mpol_rebind_preferred,
469         },
470         [MPOL_BIND] = {
471                 .create = mpol_new_bind,
472                 .rebind = mpol_rebind_nodemask,
473         },
474 };
475 
476 static void migrate_page_add(struct page *page, struct list_head *pagelist,
477                                 unsigned long flags);
478 
479 /*
480  * Scan through pages checking if pages follow certain conditions,
481  * and move them to the pagelist if they do.
482  */
483 static int queue_pages_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
484                 unsigned long addr, unsigned long end,
485                 const nodemask_t *nodes, unsigned long flags,
486                 void *private)
487 {
488         pte_t *orig_pte;
489         pte_t *pte;
490         spinlock_t *ptl;
491 
492         orig_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
493         do {
494                 struct page *page;
495                 int nid;
496 
497                 if (!pte_present(*pte))
498                         continue;
499                 page = vm_normal_page(vma, addr, *pte);
500                 if (!page)
501                         continue;
502                 /*
503                  * vm_normal_page() filters out zero pages, but there might
504                  * still be PageReserved pages to skip, perhaps in a VDSO.
505                  */
506                 if (PageReserved(page))
507                         continue;
508                 nid = page_to_nid(page);
509                 if (node_isset(nid, *nodes) == !!(flags & MPOL_MF_INVERT))
510                         continue;
511 
512                 if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
513                         migrate_page_add(page, private, flags);
514                 else
515                         break;
516         } while (pte++, addr += PAGE_SIZE, addr != end);
517         pte_unmap_unlock(orig_pte, ptl);
518         return addr != end;
519 }
520 
521 static void queue_pages_hugetlb_pmd_range(struct vm_area_struct *vma,
522                 pmd_t *pmd, const nodemask_t *nodes, unsigned long flags,
523                                     void *private)
524 {
525 #ifdef CONFIG_HUGETLB_PAGE
526         int nid;
527         struct page *page;
528         pte_t entry;
529 
530         spin_lock(&vma->vm_mm->page_table_lock);
531         entry = huge_ptep_get((pte_t *)pmd);
532         if (!pte_present(entry))
533                 goto unlock;
534         page = pte_page(entry);
535         nid = page_to_nid(page);
536         if (node_isset(nid, *nodes) == !!(flags & MPOL_MF_INVERT))
537                 goto unlock;
538         /* With MPOL_MF_MOVE, we migrate only unshared hugepage. */
539         if (flags & (MPOL_MF_MOVE_ALL) ||
540             (flags & MPOL_MF_MOVE && page_mapcount(page) == 1))
541                 isolate_huge_page(page, private);
542 unlock:
543         spin_unlock(&vma->vm_mm->page_table_lock);
544 #else
545         BUG();
546 #endif
547 }
548 
549 static inline int queue_pages_pmd_range(struct vm_area_struct *vma, pud_t *pud,
550                 unsigned long addr, unsigned long end,
551                 const nodemask_t *nodes, unsigned long flags,
552                 void *private)
553 {
554         pmd_t *pmd;
555         unsigned long next;
556 
557         pmd = pmd_offset(pud, addr);
558         do {
559                 next = pmd_addr_end(addr, end);
560                 if (!pmd_present(*pmd))
561                         continue;
562                 if (pmd_huge(*pmd) && is_vm_hugetlb_page(vma)) {
563                         queue_pages_hugetlb_pmd_range(vma, pmd, nodes,
564                                                 flags, private);
565                         continue;
566                 }
567                 split_huge_page_pmd(vma, addr, pmd);
568                 if (pmd_none_or_trans_huge_or_clear_bad(pmd))
569                         continue;
570                 if (queue_pages_pte_range(vma, pmd, addr, next, nodes,
571                                     flags, private))
572                         return -EIO;
573         } while (pmd++, addr = next, addr != end);
574         return 0;
575 }
576 
577 static inline int queue_pages_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
578                 unsigned long addr, unsigned long end,
579                 const nodemask_t *nodes, unsigned long flags,
580                 void *private)
581 {
582         pud_t *pud;
583         unsigned long next;
584 
585         pud = pud_offset(pgd, addr);
586         do {
587                 next = pud_addr_end(addr, end);
588                 if (pud_huge(*pud) && is_vm_hugetlb_page(vma))
589                         continue;
590                 if (pud_none_or_clear_bad(pud))
591                         continue;
592                 if (queue_pages_pmd_range(vma, pud, addr, next, nodes,
593                                     flags, private))
594                         return -EIO;
595         } while (pud++, addr = next, addr != end);
596         return 0;
597 }
598 
599 static inline int queue_pages_pgd_range(struct vm_area_struct *vma,
600                 unsigned long addr, unsigned long end,
601                 const nodemask_t *nodes, unsigned long flags,
602                 void *private)
603 {
604         pgd_t *pgd;
605         unsigned long next;
606 
607         pgd = pgd_offset(vma->vm_mm, addr);
608         do {
609                 next = pgd_addr_end(addr, end);
610                 if (pgd_none_or_clear_bad(pgd))
611                         continue;
612                 if (queue_pages_pud_range(vma, pgd, addr, next, nodes,
613                                     flags, private))
614                         return -EIO;
615         } while (pgd++, addr = next, addr != end);
616         return 0;
617 }
618 
619 #ifdef CONFIG_ARCH_USES_NUMA_PROT_NONE
620 /*
621  * This is used to mark a range of virtual addresses to be inaccessible.
622  * These are later cleared by a NUMA hinting fault. Depending on these
623  * faults, pages may be migrated for better NUMA placement.
624  *
625  * This is assuming that NUMA faults are handled using PROT_NONE. If
626  * an architecture makes a different choice, it will need further
627  * changes to the core.
628  */
629 unsigned long change_prot_numa(struct vm_area_struct *vma,
630                         unsigned long addr, unsigned long end)
631 {
632         int nr_updated;
633         BUILD_BUG_ON(_PAGE_NUMA != _PAGE_PROTNONE);
634 
635         nr_updated = change_protection(vma, addr, end, vma->vm_page_prot, 0, 1);
636         if (nr_updated)
637                 count_vm_numa_events(NUMA_PTE_UPDATES, nr_updated);
638 
639         return nr_updated;
640 }
641 #else
642 static unsigned long change_prot_numa(struct vm_area_struct *vma,
643                         unsigned long addr, unsigned long end)
644 {
645         return 0;
646 }
647 #endif /* CONFIG_ARCH_USES_NUMA_PROT_NONE */
648 
649 /*
650  * Walk through page tables and collect pages to be migrated.
651  *
652  * If pages found in a given range are on a set of nodes (determined by
653  * @nodes and @flags,) it's isolated and queued to the pagelist which is
654  * passed via @private.)
655  */
656 static int
657 queue_pages_range(struct mm_struct *mm, unsigned long start, unsigned long end,
658                 const nodemask_t *nodes, unsigned long flags, void *private)
659 {
660         int err = 0;
661         struct vm_area_struct *vma, *prev;
662 
663         vma = find_vma(mm, start);
664         if (!vma)
665                 return -EFAULT;
666         prev = NULL;
667         for (; vma && vma->vm_start < end; vma = vma->vm_next) {
668                 unsigned long endvma = vma->vm_end;
669 
670                 if (endvma > end)
671                         endvma = end;
672                 if (vma->vm_start > start)
673                         start = vma->vm_start;
674 
675                 if (!(flags & MPOL_MF_DISCONTIG_OK)) {
676                         if (!vma->vm_next && vma->vm_end < end)
677                                 return -EFAULT;
678                         if (prev && prev->vm_end < vma->vm_start)
679                                 return -EFAULT;
680                 }
681 
682                 if (flags & MPOL_MF_LAZY) {
683                         change_prot_numa(vma, start, endvma);
684                         goto next;
685                 }
686 
687                 if ((flags & MPOL_MF_STRICT) ||
688                      ((flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) &&
689                       vma_migratable(vma))) {
690 
691                         err = queue_pages_pgd_range(vma, start, endvma, nodes,
692                                                 flags, private);
693                         if (err)
694                                 break;
695                 }
696 next:
697                 prev = vma;
698         }
699         return err;
700 }
701 
702 /*
703  * Apply policy to a single VMA
704  * This must be called with the mmap_sem held for writing.
705  */
706 static int vma_replace_policy(struct vm_area_struct *vma,
707                                                 struct mempolicy *pol)
708 {
709         int err;
710         struct mempolicy *old;
711         struct mempolicy *new;
712 
713         pr_debug("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n",
714                  vma->vm_start, vma->vm_end, vma->vm_pgoff,
715                  vma->vm_ops, vma->vm_file,
716                  vma->vm_ops ? vma->vm_ops->set_policy : NULL);
717 
718         new = mpol_dup(pol);
719         if (IS_ERR(new))
720                 return PTR_ERR(new);
721 
722         if (vma->vm_ops && vma->vm_ops->set_policy) {
723                 err = vma->vm_ops->set_policy(vma, new);
724                 if (err)
725                         goto err_out;
726         }
727 
728         old = vma->vm_policy;
729         vma->vm_policy = new; /* protected by mmap_sem */
730         mpol_put(old);
731 
732         return 0;
733  err_out:
734         mpol_put(new);
735         return err;
736 }
737 
738 /* Step 2: apply policy to a range and do splits. */
739 static int mbind_range(struct mm_struct *mm, unsigned long start,
740                        unsigned long end, struct mempolicy *new_pol)
741 {
742         struct vm_area_struct *next;
743         struct vm_area_struct *prev;
744         struct vm_area_struct *vma;
745         int err = 0;
746         pgoff_t pgoff;
747         unsigned long vmstart;
748         unsigned long vmend;
749 
750         vma = find_vma(mm, start);
751         if (!vma || vma->vm_start > start)
752                 return -EFAULT;
753 
754         prev = vma->vm_prev;
755         if (start > vma->vm_start)
756                 prev = vma;
757 
758         for (; vma && vma->vm_start < end; prev = vma, vma = next) {
759                 next = vma->vm_next;
760                 vmstart = max(start, vma->vm_start);
761                 vmend   = min(end, vma->vm_end);
762 
763                 if (mpol_equal(vma_policy(vma), new_pol))
764                         continue;
765 
766                 pgoff = vma->vm_pgoff +
767                         ((vmstart - vma->vm_start) >> PAGE_SHIFT);
768                 prev = vma_merge(mm, prev, vmstart, vmend, vma->vm_flags,
769                                   vma->anon_vma, vma->vm_file, pgoff,
770                                   new_pol);
771                 if (prev) {
772                         vma = prev;
773                         next = vma->vm_next;
774                         if (mpol_equal(vma_policy(vma), new_pol))
775                                 continue;
776                         /* vma_merge() joined vma && vma->next, case 8 */
777                         goto replace;
778                 }
779                 if (vma->vm_start != vmstart) {
780                         err = split_vma(vma->vm_mm, vma, vmstart, 1);
781                         if (err)
782                                 goto out;
783                 }
784                 if (vma->vm_end != vmend) {
785                         err = split_vma(vma->vm_mm, vma, vmend, 0);
786                         if (err)
787                                 goto out;
788                 }
789  replace:
790                 err = vma_replace_policy(vma, new_pol);
791                 if (err)
792                         goto out;
793         }
794 
795  out:
796         return err;
797 }
798 
799 /*
800  * Update task->flags PF_MEMPOLICY bit: set iff non-default
801  * mempolicy.  Allows more rapid checking of this (combined perhaps
802  * with other PF_* flag bits) on memory allocation hot code paths.
803  *
804  * If called from outside this file, the task 'p' should -only- be
805  * a newly forked child not yet visible on the task list, because
806  * manipulating the task flags of a visible task is not safe.
807  *
808  * The above limitation is why this routine has the funny name
809  * mpol_fix_fork_child_flag().
810  *
811  * It is also safe to call this with a task pointer of current,
812  * which the static wrapper mpol_set_task_struct_flag() does,
813  * for use within this file.
814  */
815 
816 void mpol_fix_fork_child_flag(struct task_struct *p)
817 {
818         if (p->mempolicy)
819                 p->flags |= PF_MEMPOLICY;
820         else
821                 p->flags &= ~PF_MEMPOLICY;
822 }
823 
824 static void mpol_set_task_struct_flag(void)
825 {
826         mpol_fix_fork_child_flag(current);
827 }
828 
829 /* Set the process memory policy */
830 static long do_set_mempolicy(unsigned short mode, unsigned short flags,
831                              nodemask_t *nodes)
832 {
833         struct mempolicy *new, *old;
834         struct mm_struct *mm = current->mm;
835         NODEMASK_SCRATCH(scratch);
836         int ret;
837 
838         if (!scratch)
839                 return -ENOMEM;
840 
841         new = mpol_new(mode, flags, nodes);
842         if (IS_ERR(new)) {
843                 ret = PTR_ERR(new);
844                 goto out;
845         }
846         /*
847          * prevent changing our mempolicy while show_numa_maps()
848          * is using it.
849          * Note:  do_set_mempolicy() can be called at init time
850          * with no 'mm'.
851          */
852         if (mm)
853                 down_write(&mm->mmap_sem);
854         task_lock(current);
855         ret = mpol_set_nodemask(new, nodes, scratch);
856         if (ret) {
857                 task_unlock(current);
858                 if (mm)
859                         up_write(&mm->mmap_sem);
860                 mpol_put(new);
861                 goto out;
862         }
863         old = current->mempolicy;
864         current->mempolicy = new;
865         mpol_set_task_struct_flag();
866         if (new && new->mode == MPOL_INTERLEAVE &&
867             nodes_weight(new->v.nodes))
868                 current->il_next = first_node(new->v.nodes);
869         task_unlock(current);
870         if (mm)
871                 up_write(&mm->mmap_sem);
872 
873         mpol_put(old);
874         ret = 0;
875 out:
876         NODEMASK_SCRATCH_FREE(scratch);
877         return ret;
878 }
879 
880 /*
881  * Return nodemask for policy for get_mempolicy() query
882  *
883  * Called with task's alloc_lock held
884  */
885 static void get_policy_nodemask(struct mempolicy *p, nodemask_t *nodes)
886 {
887         nodes_clear(*nodes);
888         if (p == &default_policy)
889                 return;
890 
891         switch (p->mode) {
892         case MPOL_BIND:
893                 /* Fall through */
894         case MPOL_INTERLEAVE:
895                 *nodes = p->v.nodes;
896                 break;
897         case MPOL_PREFERRED:
898                 if (!(p->flags & MPOL_F_LOCAL))
899                         node_set(p->v.preferred_node, *nodes);
900                 /* else return empty node mask for local allocation */
901                 break;
902         default:
903                 BUG();
904         }
905 }
906 
907 static int lookup_node(struct mm_struct *mm, unsigned long addr)
908 {
909         struct page *p;
910         int err;
911 
912         err = get_user_pages(current, mm, addr & PAGE_MASK, 1, 0, 0, &p, NULL);
913         if (err >= 0) {
914                 err = page_to_nid(p);
915                 put_page(p);
916         }
917         return err;
918 }
919 
920 /* Retrieve NUMA policy */
921 static long do_get_mempolicy(int *policy, nodemask_t *nmask,
922                              unsigned long addr, unsigned long flags)
923 {
924         int err;
925         struct mm_struct *mm = current->mm;
926         struct vm_area_struct *vma = NULL;
927         struct mempolicy *pol = current->mempolicy;
928 
929         if (flags &
930                 ~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR|MPOL_F_MEMS_ALLOWED))
931                 return -EINVAL;
932 
933         if (flags & MPOL_F_MEMS_ALLOWED) {
934                 if (flags & (MPOL_F_NODE|MPOL_F_ADDR))
935                         return -EINVAL;
936                 *policy = 0;    /* just so it's initialized */
937                 task_lock(current);
938                 *nmask  = cpuset_current_mems_allowed;
939                 task_unlock(current);
940                 return 0;
941         }
942 
943         if (flags & MPOL_F_ADDR) {
944                 /*
945                  * Do NOT fall back to task policy if the
946                  * vma/shared policy at addr is NULL.  We
947                  * want to return MPOL_DEFAULT in this case.
948                  */
949                 down_read(&mm->mmap_sem);
950                 vma = find_vma_intersection(mm, addr, addr+1);
951                 if (!vma) {
952                         up_read(&mm->mmap_sem);
953                         return -EFAULT;
954                 }
955                 if (vma->vm_ops && vma->vm_ops->get_policy)
956                         pol = vma->vm_ops->get_policy(vma, addr);
957                 else
958                         pol = vma->vm_policy;
959         } else if (addr)
960                 return -EINVAL;
961 
962         if (!pol)
963                 pol = &default_policy;  /* indicates default behavior */
964 
965         if (flags & MPOL_F_NODE) {
966                 if (flags & MPOL_F_ADDR) {
967                         err = lookup_node(mm, addr);
968                         if (err < 0)
969                                 goto out;
970                         *policy = err;
971                 } else if (pol == current->mempolicy &&
972                                 pol->mode == MPOL_INTERLEAVE) {
973                         *policy = current->il_next;
974                 } else {
975                         err = -EINVAL;
976                         goto out;
977                 }
978         } else {
979                 *policy = pol == &default_policy ? MPOL_DEFAULT :
980                                                 pol->mode;
981                 /*
982                  * Internal mempolicy flags must be masked off before exposing
983                  * the policy to userspace.
984                  */
985                 *policy |= (pol->flags & MPOL_MODE_FLAGS);
986         }
987 
988         if (vma) {
989                 up_read(&current->mm->mmap_sem);
990                 vma = NULL;
991         }
992 
993         err = 0;
994         if (nmask) {
995                 if (mpol_store_user_nodemask(pol)) {
996                         *nmask = pol->w.user_nodemask;
997                 } else {
998                         task_lock(current);
999                         get_policy_nodemask(pol, nmask);
1000                         task_unlock(current);
1001                 }
1002         }
1003 
1004  out:
1005         mpol_cond_put(pol);
1006         if (vma)
1007                 up_read(&current->mm->mmap_sem);
1008         return err;
1009 }
1010 
1011 #ifdef CONFIG_MIGRATION
1012 /*
1013  * page migration
1014  */
1015 static void migrate_page_add(struct page *page, struct list_head *pagelist,
1016                                 unsigned long flags)
1017 {
1018         /*
1019          * Avoid migrating a page that is shared with others.
1020          */
1021         if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(page) == 1) {
1022                 if (!isolate_lru_page(page)) {
1023                         list_add_tail(&page->lru, pagelist);
1024                         inc_zone_page_state(page, NR_ISOLATED_ANON +
1025                                             page_is_file_cache(page));
1026                 }
1027         }
1028 }
1029 
1030 static struct page *new_node_page(struct page *page, unsigned long node, int **x)
1031 {
1032         if (PageHuge(page))
1033                 return alloc_huge_page_node(page_hstate(compound_head(page)),
1034                                         node);
1035         else
1036                 return alloc_pages_exact_node(node, GFP_HIGHUSER_MOVABLE, 0);
1037 }
1038 
1039 /*
1040  * Migrate pages from one node to a target node.
1041  * Returns error or the number of pages not migrated.
1042  */
1043 static int migrate_to_node(struct mm_struct *mm, int source, int dest,
1044                            int flags)
1045 {
1046         nodemask_t nmask;
1047         LIST_HEAD(pagelist);
1048         int err = 0;
1049 
1050         nodes_clear(nmask);
1051         node_set(source, nmask);
1052 
1053         /*
1054          * This does not "check" the range but isolates all pages that
1055          * need migration.  Between passing in the full user address
1056          * space range and MPOL_MF_DISCONTIG_OK, this call can not fail.
1057          */
1058         VM_BUG_ON(!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)));
1059         queue_pages_range(mm, mm->mmap->vm_start, mm->task_size, &nmask,
1060                         flags | MPOL_MF_DISCONTIG_OK, &pagelist);
1061 
1062         if (!list_empty(&pagelist)) {
1063                 err = migrate_pages(&pagelist, new_node_page, NULL, dest,
1064                                         MIGRATE_SYNC, MR_SYSCALL);
1065                 if (err)
1066                         putback_movable_pages(&pagelist);
1067         }
1068 
1069         return err;
1070 }
1071 
1072 /*
1073  * Move pages between the two nodesets so as to preserve the physical
1074  * layout as much as possible.
1075  *
1076  * Returns the number of page that could not be moved.
1077  */
1078 int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from,
1079                      const nodemask_t *to, int flags)
1080 {
1081         int busy = 0;
1082         int err;
1083         nodemask_t tmp;
1084 
1085         err = migrate_prep();
1086         if (err)
1087                 return err;
1088 
1089         down_read(&mm->mmap_sem);
1090 
1091         err = migrate_vmas(mm, from, to, flags);
1092         if (err)
1093                 goto out;
1094 
1095         /*
1096          * Find a 'source' bit set in 'tmp' whose corresponding 'dest'
1097          * bit in 'to' is not also set in 'tmp'.  Clear the found 'source'
1098          * bit in 'tmp', and return that <source, dest> pair for migration.
1099          * The pair of nodemasks 'to' and 'from' define the map.
1100          *
1101          * If no pair of bits is found that way, fallback to picking some
1102          * pair of 'source' and 'dest' bits that are not the same.  If the
1103          * 'source' and 'dest' bits are the same, this represents a node
1104          * that will be migrating to itself, so no pages need move.
1105          *
1106          * If no bits are left in 'tmp', or if all remaining bits left
1107          * in 'tmp' correspond to the same bit in 'to', return false
1108          * (nothing left to migrate).
1109          *
1110          * This lets us pick a pair of nodes to migrate between, such that
1111          * if possible the dest node is not already occupied by some other
1112          * source node, minimizing the risk of overloading the memory on a
1113          * node that would happen if we migrated incoming memory to a node
1114          * before migrating outgoing memory source that same node.
1115          *
1116          * A single scan of tmp is sufficient.  As we go, we remember the
1117          * most recent <s, d> pair that moved (s != d).  If we find a pair
1118          * that not only moved, but what's better, moved to an empty slot
1119          * (d is not set in tmp), then we break out then, with that pair.
1120          * Otherwise when we finish scanning from_tmp, we at least have the
1121          * most recent <s, d> pair that moved.  If we get all the way through
1122          * the scan of tmp without finding any node that moved, much less
1123          * moved to an empty node, then there is nothing left worth migrating.
1124          */
1125 
1126         tmp = *from;
1127         while (!nodes_empty(tmp)) {
1128                 int s,d;
1129                 int source = -1;
1130                 int dest = 0;
1131 
1132                 for_each_node_mask(s, tmp) {
1133 
1134                         /*
1135                          * do_migrate_pages() tries to maintain the relative
1136                          * node relationship of the pages established between
1137                          * threads and memory areas.
1138                          *
1139                          * However if the number of source nodes is not equal to
1140                          * the number of destination nodes we can not preserve
1141                          * this node relative relationship.  In that case, skip
1142                          * copying memory from a node that is in the destination
1143                          * mask.
1144                          *
1145                          * Example: [2,3,4] -> [3,4,5] moves everything.
1146                          *          [0-7] - > [3,4,5] moves only 0,1,2,6,7.
1147                          */
1148 
1149                         if ((nodes_weight(*from) != nodes_weight(*to)) &&
1150                                                 (node_isset(s, *to)))
1151                                 continue;
1152 
1153                         d = node_remap(s, *from, *to);
1154                         if (s == d)
1155                                 continue;
1156 
1157                         source = s;     /* Node moved. Memorize */
1158                         dest = d;
1159 
1160                         /* dest not in remaining from nodes? */
1161                         if (!node_isset(dest, tmp))
1162                                 break;
1163                 }
1164                 if (source == -1)
1165                         break;
1166 
1167                 node_clear(source, tmp);
1168                 err = migrate_to_node(mm, source, dest, flags);
1169                 if (err > 0)
1170                         busy += err;
1171                 if (err < 0)
1172                         break;
1173         }
1174 out:
1175         up_read(&mm->mmap_sem);
1176         if (err < 0)
1177                 return err;
1178         return busy;
1179 
1180 }
1181 
1182 /*
1183  * Allocate a new page for page migration based on vma policy.
1184  * Start by assuming the page is mapped by the same vma as contains @start.
1185  * Search forward from there, if not.  N.B., this assumes that the
1186  * list of pages handed to migrate_pages()--which is how we get here--
1187  * is in virtual address order.
1188  */
1189 static struct page *new_page(struct page *page, unsigned long start, int **x)
1190 {
1191         struct vm_area_struct *vma;
1192         unsigned long uninitialized_var(address);
1193 
1194         vma = find_vma(current->mm, start);
1195         while (vma) {
1196                 address = page_address_in_vma(page, vma);
1197                 if (address != -EFAULT)
1198                         break;
1199                 vma = vma->vm_next;
1200         }
1201 
1202         if (PageHuge(page)) {
1203                 BUG_ON(!vma);
1204                 return alloc_huge_page_noerr(vma, address, 1);
1205         }
1206         /*
1207          * if !vma, alloc_page_vma() will use task or system default policy
1208          */
1209         return alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
1210 }
1211 #else
1212 
1213 static void migrate_page_add(struct page *page, struct list_head *pagelist,
1214                                 unsigned long flags)
1215 {
1216 }
1217 
1218 int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from,
1219                      const nodemask_t *to, int flags)
1220 {
1221         return -ENOSYS;
1222 }
1223 
1224 static struct page *new_page(struct page *page, unsigned long start, int **x)
1225 {
1226         return NULL;
1227 }
1228 #endif
1229 
1230 static long do_mbind(unsigned long start, unsigned long len,
1231                      unsigned short mode, unsigned short mode_flags,
1232                      nodemask_t *nmask, unsigned long flags)
1233 {
1234         struct mm_struct *mm = current->mm;
1235         struct mempolicy *new;
1236         unsigned long end;
1237         int err;
1238         LIST_HEAD(pagelist);
1239 
1240         if (flags & ~(unsigned long)MPOL_MF_VALID)
1241                 return -EINVAL;
1242         if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
1243                 return -EPERM;
1244 
1245         if (start & ~PAGE_MASK)
1246                 return -EINVAL;
1247 
1248         if (mode == MPOL_DEFAULT)
1249                 flags &= ~MPOL_MF_STRICT;
1250 
1251         len = (len + PAGE_SIZE - 1) & PAGE_MASK;
1252         end = start + len;
1253 
1254         if (end < start)
1255                 return -EINVAL;
1256         if (end == start)
1257                 return 0;
1258 
1259         new = mpol_new(mode, mode_flags, nmask);
1260         if (IS_ERR(new))
1261                 return PTR_ERR(new);
1262 
1263         if (flags & MPOL_MF_LAZY)
1264                 new->flags |= MPOL_F_MOF;
1265 
1266         /*
1267          * If we are using the default policy then operation
1268          * on discontinuous address spaces is okay after all
1269          */
1270         if (!new)
1271                 flags |= MPOL_MF_DISCONTIG_OK;
1272 
1273         pr_debug("mbind %lx-%lx mode:%d flags:%d nodes:%lx\n",
1274                  start, start + len, mode, mode_flags,
1275                  nmask ? nodes_addr(*nmask)[0] : NUMA_NO_NODE);
1276 
1277         if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
1278 
1279                 err = migrate_prep();
1280                 if (err)
1281                         goto mpol_out;
1282         }
1283         {
1284                 NODEMASK_SCRATCH(scratch);
1285                 if (scratch) {
1286                         down_write(&mm->mmap_sem);
1287                         task_lock(current);
1288                         err = mpol_set_nodemask(new, nmask, scratch);
1289                         task_unlock(current);
1290                         if (err)
1291                                 up_write(&mm->mmap_sem);
1292                 } else
1293                         err = -ENOMEM;
1294                 NODEMASK_SCRATCH_FREE(scratch);
1295         }
1296         if (err)
1297                 goto mpol_out;
1298 
1299         err = queue_pages_range(mm, start, end, nmask,
1300                           flags | MPOL_MF_INVERT, &pagelist);
1301         if (!err)
1302                 err = mbind_range(mm, start, end, new);
1303 
1304         if (!err) {
1305                 int nr_failed = 0;
1306 
1307                 if (!list_empty(&pagelist)) {
1308                         WARN_ON_ONCE(flags & MPOL_MF_LAZY);
1309                         nr_failed = migrate_pages(&pagelist, new_page, NULL,
1310                                 start, MIGRATE_SYNC, MR_MEMPOLICY_MBIND);
1311                         if (nr_failed)
1312                                 putback_movable_pages(&pagelist);
1313                 }
1314 
1315                 if (nr_failed && (flags & MPOL_MF_STRICT))
1316                         err = -EIO;
1317         } else
1318                 putback_movable_pages(&pagelist);
1319 
1320         up_write(&mm->mmap_sem);
1321  mpol_out:
1322         mpol_put(new);
1323         return err;
1324 }
1325 
1326 /*
1327  * User space interface with variable sized bitmaps for nodelists.
1328  */
1329 
1330 /* Copy a node mask from user space. */
1331 static int get_nodes(nodemask_t *nodes, const unsigned long __user *nmask,
1332                      unsigned long maxnode)
1333 {
1334         unsigned long k;
1335         unsigned long nlongs;
1336         unsigned long endmask;
1337 
1338         --maxnode;
1339         nodes_clear(*nodes);
1340         if (maxnode == 0 || !nmask)
1341                 return 0;
1342         if (maxnode > PAGE_SIZE*BITS_PER_BYTE)
1343                 return -EINVAL;
1344 
1345         nlongs = BITS_TO_LONGS(maxnode);
1346         if ((maxnode % BITS_PER_LONG) == 0)
1347                 endmask = ~0UL;
1348         else
1349                 endmask = (1UL << (maxnode % BITS_PER_LONG)) - 1;
1350 
1351         /* When the user specified more nodes than supported just check
1352            if the non supported part is all zero. */
1353         if (nlongs > BITS_TO_LONGS(MAX_NUMNODES)) {
1354                 if (nlongs > PAGE_SIZE/sizeof(long))
1355                         return -EINVAL;
1356                 for (k = BITS_TO_LONGS(MAX_NUMNODES); k < nlongs; k++) {
1357                         unsigned long t;
1358                         if (get_user(t, nmask + k))
1359                                 return -EFAULT;
1360                         if (k == nlongs - 1) {
1361                                 if (t & endmask)
1362                                         return -EINVAL;
1363                         } else if (t)
1364                                 return -EINVAL;
1365                 }
1366                 nlongs = BITS_TO_LONGS(MAX_NUMNODES);
1367                 endmask = ~0UL;
1368         }
1369 
1370         if (copy_from_user(nodes_addr(*nodes), nmask, nlongs*sizeof(unsigned long)))
1371                 return -EFAULT;
1372         nodes_addr(*nodes)[nlongs-1] &= endmask;
1373         return 0;
1374 }
1375 
1376 /* Copy a kernel node mask to user space */
1377 static int copy_nodes_to_user(unsigned long __user *mask, unsigned long maxnode,
1378                               nodemask_t *nodes)
1379 {
1380         unsigned long copy = ALIGN(maxnode-1, 64) / 8;
1381         const int nbytes = BITS_TO_LONGS(MAX_NUMNODES) * sizeof(long);
1382 
1383         if (copy > nbytes) {
1384                 if (copy > PAGE_SIZE)
1385                         return -EINVAL;
1386                 if (clear_user((char __user *)mask + nbytes, copy - nbytes))
1387                         return -EFAULT;
1388                 copy = nbytes;
1389         }
1390         return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0;
1391 }
1392 
1393 SYSCALL_DEFINE6(mbind, unsigned long, start, unsigned long, len,
1394                 unsigned long, mode, unsigned long __user *, nmask,
1395                 unsigned long, maxnode, unsigned, flags)
1396 {
1397         nodemask_t nodes;
1398         int err;
1399         unsigned short mode_flags;
1400 
1401         mode_flags = mode & MPOL_MODE_FLAGS;
1402         mode &= ~MPOL_MODE_FLAGS;
1403         if (mode >= MPOL_MAX)
1404                 return -EINVAL;
1405         if ((mode_flags & MPOL_F_STATIC_NODES) &&
1406             (mode_flags & MPOL_F_RELATIVE_NODES))
1407                 return -EINVAL;
1408         err = get_nodes(&nodes, nmask, maxnode);
1409         if (err)
1410                 return err;
1411         return do_mbind(start, len, mode, mode_flags, &nodes, flags);
1412 }
1413 
1414 /* Set the process memory policy */
1415 SYSCALL_DEFINE3(set_mempolicy, int, mode, unsigned long __user *, nmask,
1416                 unsigned long, maxnode)
1417 {
1418         int err;
1419         nodemask_t nodes;
1420         unsigned short flags;
1421 
1422         flags = mode & MPOL_MODE_FLAGS;
1423         mode &= ~MPOL_MODE_FLAGS;
1424         if ((unsigned int)mode >= MPOL_MAX)
1425                 return -EINVAL;
1426         if ((flags & MPOL_F_STATIC_NODES) && (flags & MPOL_F_RELATIVE_NODES))
1427                 return -EINVAL;
1428         err = get_nodes(&nodes, nmask, maxnode);
1429         if (err)
1430                 return err;
1431         return do_set_mempolicy(mode, flags, &nodes);
1432 }
1433 
1434 SYSCALL_DEFINE4(migrate_pages, pid_t, pid, unsigned long, maxnode,
1435                 const unsigned long __user *, old_nodes,
1436                 const unsigned long __user *, new_nodes)
1437 {
1438         const struct cred *cred = current_cred(), *tcred;
1439         struct mm_struct *mm = NULL;
1440         struct task_struct *task;
1441         nodemask_t task_nodes;
1442         int err;
1443         nodemask_t *old;
1444         nodemask_t *new;
1445         NODEMASK_SCRATCH(scratch);
1446 
1447         if (!scratch)
1448                 return -ENOMEM;
1449 
1450         old = &scratch->mask1;
1451         new = &scratch->mask2;
1452 
1453         err = get_nodes(old, old_nodes, maxnode);
1454         if (err)
1455                 goto out;
1456 
1457         err = get_nodes(new, new_nodes, maxnode);
1458         if (err)
1459                 goto out;
1460 
1461         /* Find the mm_struct */
1462         rcu_read_lock();
1463         task = pid ? find_task_by_vpid(pid) : current;
1464         if (!task) {
1465                 rcu_read_unlock();
1466                 err = -ESRCH;
1467                 goto out;
1468         }
1469         get_task_struct(task);
1470 
1471         err = -EINVAL;
1472 
1473         /*
1474          * Check if this process has the right to modify the specified
1475          * process. The right exists if the process has administrative
1476          * capabilities, superuser privileges or the same
1477          * userid as the target process.
1478          */
1479         tcred = __task_cred(task);
1480         if (!uid_eq(cred->euid, tcred->suid) && !uid_eq(cred->euid, tcred->uid) &&
1481             !uid_eq(cred->uid,  tcred->suid) && !uid_eq(cred->uid,  tcred->uid) &&
1482             !capable(CAP_SYS_NICE)) {
1483                 rcu_read_unlock();
1484                 err = -EPERM;
1485                 goto out_put;
1486         }
1487         rcu_read_unlock();
1488 
1489         task_nodes = cpuset_mems_allowed(task);
1490         /* Is the user allowed to access the target nodes? */
1491         if (!nodes_subset(*new, task_nodes) && !capable(CAP_SYS_NICE)) {
1492                 err = -EPERM;
1493                 goto out_put;
1494         }
1495 
1496         if (!nodes_subset(*new, node_states[N_MEMORY])) {
1497                 err = -EINVAL;
1498                 goto out_put;
1499         }
1500 
1501         err = security_task_movememory(task);
1502         if (err)
1503                 goto out_put;
1504 
1505         mm = get_task_mm(task);
1506         put_task_struct(task);
1507 
1508         if (!mm) {
1509                 err = -EINVAL;
1510                 goto out;
1511         }
1512 
1513         err = do_migrate_pages(mm, old, new,
1514                 capable(CAP_SYS_NICE) ? MPOL_MF_MOVE_ALL : MPOL_MF_MOVE);
1515 
1516         mmput(mm);
1517 out:
1518         NODEMASK_SCRATCH_FREE(scratch);
1519 
1520         return err;
1521 
1522 out_put:
1523         put_task_struct(task);
1524         goto out;
1525 
1526 }
1527 
1528 
1529 /* Retrieve NUMA policy */
1530 SYSCALL_DEFINE5(get_mempolicy, int __user *, policy,
1531                 unsigned long __user *, nmask, unsigned long, maxnode,
1532                 unsigned long, addr, unsigned long, flags)
1533 {
1534         int err;
1535         int uninitialized_var(pval);
1536         nodemask_t nodes;
1537 
1538         if (nmask != NULL && maxnode < MAX_NUMNODES)
1539                 return -EINVAL;
1540 
1541         err = do_get_mempolicy(&pval, &nodes, addr, flags);
1542 
1543         if (err)
1544                 return err;
1545 
1546         if (policy && put_user(pval, policy))
1547                 return -EFAULT;
1548 
1549         if (nmask)
1550                 err = copy_nodes_to_user(nmask, maxnode, &nodes);
1551 
1552         return err;
1553 }
1554 
1555 #ifdef CONFIG_COMPAT
1556 
1557 asmlinkage long compat_sys_get_mempolicy(int __user *policy,
1558                                      compat_ulong_t __user *nmask,
1559                                      compat_ulong_t maxnode,
1560                                      compat_ulong_t addr, compat_ulong_t flags)
1561 {
1562         long err;
1563         unsigned long __user *nm = NULL;
1564         unsigned long nr_bits, alloc_size;
1565         DECLARE_BITMAP(bm, MAX_NUMNODES);
1566 
1567         nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1568         alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1569 
1570         if (nmask)
1571                 nm = compat_alloc_user_space(alloc_size);
1572 
1573         err = sys_get_mempolicy(policy, nm, nr_bits+1, addr, flags);
1574 
1575         if (!err && nmask) {
1576                 unsigned long copy_size;
1577                 copy_size = min_t(unsigned long, sizeof(bm), alloc_size);
1578                 err = copy_from_user(bm, nm, copy_size);
1579                 /* ensure entire bitmap is zeroed */
1580                 err |= clear_user(nmask, ALIGN(maxnode-1, 8) / 8);
1581                 err |= compat_put_bitmap(nmask, bm, nr_bits);
1582         }
1583 
1584         return err;
1585 }
1586 
1587 asmlinkage long compat_sys_set_mempolicy(int mode, compat_ulong_t __user *nmask,
1588                                      compat_ulong_t maxnode)
1589 {
1590         unsigned long __user *nm = NULL;
1591         unsigned long nr_bits, alloc_size;
1592         DECLARE_BITMAP(bm, MAX_NUMNODES);
1593 
1594         nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1595         alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1596 
1597         if (nmask) {
1598                 if (compat_get_bitmap(bm, nmask, nr_bits))
1599                         return -EFAULT;
1600                 nm = compat_alloc_user_space(alloc_size);
1601                 if (copy_to_user(nm, bm, alloc_size))
1602                         return -EFAULT;
1603         }
1604 
1605         return sys_set_mempolicy(mode, nm, nr_bits+1);
1606 }
1607 
1608 asmlinkage long compat_sys_mbind(compat_ulong_t start, compat_ulong_t len,
1609                              compat_ulong_t mode, compat_ulong_t __user *nmask,
1610                              compat_ulong_t maxnode, compat_ulong_t flags)
1611 {
1612         unsigned long __user *nm = NULL;
1613         unsigned long nr_bits, alloc_size;
1614         nodemask_t bm;
1615 
1616         nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1617         alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1618 
1619         if (nmask) {
1620                 if (compat_get_bitmap(nodes_addr(bm), nmask, nr_bits))
1621                         return -EFAULT;
1622                 nm = compat_alloc_user_space(alloc_size);
1623                 if (copy_to_user(nm, nodes_addr(bm), alloc_size))
1624                         return -EFAULT;
1625         }
1626 
1627         return sys_mbind(start, len, mode, nm, nr_bits+1, flags);
1628 }
1629 
1630 #endif
1631 
1632 /*
1633  * get_vma_policy(@task, @vma, @addr)
1634  * @task - task for fallback if vma policy == default
1635  * @vma   - virtual memory area whose policy is sought
1636  * @addr  - address in @vma for shared policy lookup
1637  *
1638  * Returns effective policy for a VMA at specified address.
1639  * Falls back to @task or system default policy, as necessary.
1640  * Current or other task's task mempolicy and non-shared vma policies must be
1641  * protected by task_lock(task) by the caller.
1642  * Shared policies [those marked as MPOL_F_SHARED] require an extra reference
1643  * count--added by the get_policy() vm_op, as appropriate--to protect against
1644  * freeing by another task.  It is the caller's responsibility to free the
1645  * extra reference for shared policies.
1646  */
1647 struct mempolicy *get_vma_policy(struct task_struct *task,
1648                 struct vm_area_struct *vma, unsigned long addr)
1649 {
1650         struct mempolicy *pol = get_task_policy(task);
1651 
1652         if (vma) {
1653                 if (vma->vm_ops && vma->vm_ops->get_policy) {
1654                         struct mempolicy *vpol = vma->vm_ops->get_policy(vma,
1655                                                                         addr);
1656                         if (vpol)
1657                                 pol = vpol;
1658                 } else if (vma->vm_policy) {
1659                         pol = vma->vm_policy;
1660 
1661                         /*
1662                          * shmem_alloc_page() passes MPOL_F_SHARED policy with
1663                          * a pseudo vma whose vma->vm_ops=NULL. Take a reference
1664                          * count on these policies which will be dropped by
1665                          * mpol_cond_put() later
1666                          */
1667                         if (mpol_needs_cond_ref(pol))
1668                                 mpol_get(pol);
1669                 }
1670         }
1671         if (!pol)
1672                 pol = &default_policy;
1673         return pol;
1674 }
1675 
1676 static int apply_policy_zone(struct mempolicy *policy, enum zone_type zone)
1677 {
1678         enum zone_type dynamic_policy_zone = policy_zone;
1679 
1680         BUG_ON(dynamic_policy_zone == ZONE_MOVABLE);
1681 
1682         /*
1683          * if policy->v.nodes has movable memory only,
1684          * we apply policy when gfp_zone(gfp) = ZONE_MOVABLE only.
1685          *
1686          * policy->v.nodes is intersect with node_states[N_MEMORY].
1687          * so if the following test faile, it implies
1688          * policy->v.nodes has movable memory only.
1689          */
1690         if (!nodes_intersects(policy->v.nodes, node_states[N_HIGH_MEMORY]))
1691                 dynamic_policy_zone = ZONE_MOVABLE;
1692 
1693         return zone >= dynamic_policy_zone;
1694 }
1695 
1696 /*
1697  * Return a nodemask representing a mempolicy for filtering nodes for
1698  * page allocation
1699  */
1700 static nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *policy)
1701 {
1702         /* Lower zones don't get a nodemask applied for MPOL_BIND */
1703         if (unlikely(policy->mode == MPOL_BIND) &&
1704                         apply_policy_zone(policy, gfp_zone(gfp)) &&
1705                         cpuset_nodemask_valid_mems_allowed(&policy->v.nodes))
1706                 return &policy->v.nodes;
1707 
1708         return NULL;
1709 }
1710 
1711 /* Return a zonelist indicated by gfp for node representing a mempolicy */
1712 static struct zonelist *policy_zonelist(gfp_t gfp, struct mempolicy *policy,
1713         int nd)
1714 {
1715         switch (policy->mode) {
1716         case MPOL_PREFERRED:
1717                 if (!(policy->flags & MPOL_F_LOCAL))
1718                         nd = policy->v.preferred_node;
1719                 break;
1720         case MPOL_BIND:
1721                 /*
1722                  * Normally, MPOL_BIND allocations are node-local within the
1723                  * allowed nodemask.  However, if __GFP_THISNODE is set and the
1724                  * current node isn't part of the mask, we use the zonelist for
1725                  * the first node in the mask instead.
1726                  */
1727                 if (unlikely(gfp & __GFP_THISNODE) &&
1728                                 unlikely(!node_isset(nd, policy->v.nodes)))
1729                         nd = first_node(policy->v.nodes);
1730                 break;
1731         default:
1732                 BUG();
1733         }
1734         return node_zonelist(nd, gfp);
1735 }
1736 
1737 /* Do dynamic interleaving for a process */
1738 static unsigned interleave_nodes(struct mempolicy *policy)
1739 {
1740         unsigned nid, next;
1741         struct task_struct *me = current;
1742 
1743         nid = me->il_next;
1744         next = next_node(nid, policy->v.nodes);
1745         if (next >= MAX_NUMNODES)
1746                 next = first_node(policy->v.nodes);
1747         if (next < MAX_NUMNODES)
1748                 me->il_next = next;
1749         return nid;
1750 }
1751 
1752 /*
1753  * Depending on the memory policy provide a node from which to allocate the
1754  * next slab entry.
1755  * @policy must be protected by freeing by the caller.  If @policy is
1756  * the current task's mempolicy, this protection is implicit, as only the
1757  * task can change it's policy.  The system default policy requires no
1758  * such protection.
1759  */
1760 unsigned slab_node(void)
1761 {
1762         struct mempolicy *policy;
1763 
1764         if (in_interrupt())
1765                 return numa_node_id();
1766 
1767         policy = current->mempolicy;
1768         if (!policy || policy->flags & MPOL_F_LOCAL)
1769                 return numa_node_id();
1770 
1771         switch (policy->mode) {
1772         case MPOL_PREFERRED:
1773                 /*
1774                  * handled MPOL_F_LOCAL above
1775                  */
1776                 return policy->v.preferred_node;
1777 
1778         case MPOL_INTERLEAVE:
1779                 return interleave_nodes(policy);
1780 
1781         case MPOL_BIND: {
1782                 /*
1783                  * Follow bind policy behavior and start allocation at the
1784                  * first node.
1785                  */
1786                 struct zonelist *zonelist;
1787                 struct zone *zone;
1788                 enum zone_type highest_zoneidx = gfp_zone(GFP_KERNEL);
1789                 zonelist = &NODE_DATA(numa_node_id())->node_zonelists[0];
1790                 (void)first_zones_zonelist(zonelist, highest_zoneidx,
1791                                                         &policy->v.nodes,
1792                                                         &zone);
1793                 return zone ? zone->node : numa_node_id();
1794         }
1795 
1796         default:
1797                 BUG();
1798         }
1799 }
1800 
1801 /* Do static interleaving for a VMA with known offset. */
1802 static unsigned offset_il_node(struct mempolicy *pol,
1803                 struct vm_area_struct *vma, unsigned long off)
1804 {
1805         unsigned nnodes = nodes_weight(pol->v.nodes);
1806         unsigned target;
1807         int c;
1808         int nid = -1;
1809 
1810         if (!nnodes)
1811                 return numa_node_id();
1812         target = (unsigned int)off % nnodes;
1813         c = 0;
1814         do {
1815                 nid = next_node(nid, pol->v.nodes);
1816                 c++;
1817         } while (c <= target);
1818         return nid;
1819 }
1820 
1821 /* Determine a node number for interleave */
1822 static inline unsigned interleave_nid(struct mempolicy *pol,
1823                  struct vm_area_struct *vma, unsigned long addr, int shift)
1824 {
1825         if (vma) {
1826                 unsigned long off;
1827 
1828                 /*
1829                  * for small pages, there is no difference between
1830                  * shift and PAGE_SHIFT, so the bit-shift is safe.
1831                  * for huge pages, since vm_pgoff is in units of small
1832                  * pages, we need to shift off the always 0 bits to get
1833                  * a useful offset.
1834                  */
1835                 BUG_ON(shift < PAGE_SHIFT);
1836                 off = vma->vm_pgoff >> (shift - PAGE_SHIFT);
1837                 off += (addr - vma->vm_start) >> shift;
1838                 return offset_il_node(pol, vma, off);
1839         } else
1840                 return interleave_nodes(pol);
1841 }
1842 
1843 /*
1844  * Return the bit number of a random bit set in the nodemask.
1845  * (returns -1 if nodemask is empty)
1846  */
1847 int node_random(const nodemask_t *maskp)
1848 {
1849         int w, bit = -1;
1850 
1851         w = nodes_weight(*maskp);
1852         if (w)
1853                 bit = bitmap_ord_to_pos(maskp->bits,
1854                         get_random_int() % w, MAX_NUMNODES);
1855         return bit;
1856 }
1857 
1858 #ifdef CONFIG_HUGETLBFS
1859 /*
1860  * huge_zonelist(@vma, @addr, @gfp_flags, @mpol)
1861  * @vma = virtual memory area whose policy is sought
1862  * @addr = address in @vma for shared policy lookup and interleave policy
1863  * @gfp_flags = for requested zone
1864  * @mpol = pointer to mempolicy pointer for reference counted mempolicy
1865  * @nodemask = pointer to nodemask pointer for MPOL_BIND nodemask
1866  *
1867  * Returns a zonelist suitable for a huge page allocation and a pointer
1868  * to the struct mempolicy for conditional unref after allocation.
1869  * If the effective policy is 'BIND, returns a pointer to the mempolicy's
1870  * @nodemask for filtering the zonelist.
1871  *
1872  * Must be protected by read_mems_allowed_begin()
1873  */
1874 struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr,
1875                                 gfp_t gfp_flags, struct mempolicy **mpol,
1876                                 nodemask_t **nodemask)
1877 {
1878         struct zonelist *zl;
1879 
1880         *mpol = get_vma_policy(current, vma, addr);
1881         *nodemask = NULL;       /* assume !MPOL_BIND */
1882 
1883         if (unlikely((*mpol)->mode == MPOL_INTERLEAVE)) {
1884                 zl = node_zonelist(interleave_nid(*mpol, vma, addr,
1885                                 huge_page_shift(hstate_vma(vma))), gfp_flags);
1886         } else {
1887                 zl = policy_zonelist(gfp_flags, *mpol, numa_node_id());
1888                 if ((*mpol)->mode == MPOL_BIND)
1889                         *nodemask = &(*mpol)->v.nodes;
1890         }
1891         return zl;
1892 }
1893 
1894 /*
1895  * init_nodemask_of_mempolicy
1896  *
1897  * If the current task's mempolicy is "default" [NULL], return 'false'
1898  * to indicate default policy.  Otherwise, extract the policy nodemask
1899  * for 'bind' or 'interleave' policy into the argument nodemask, or
1900  * initialize the argument nodemask to contain the single node for
1901  * 'preferred' or 'local' policy and return 'true' to indicate presence
1902  * of non-default mempolicy.
1903  *
1904  * We don't bother with reference counting the mempolicy [mpol_get/put]
1905  * because the current task is examining it's own mempolicy and a task's
1906  * mempolicy is only ever changed by the task itself.
1907  *
1908  * N.B., it is the caller's responsibility to free a returned nodemask.
1909  */
1910 bool init_nodemask_of_mempolicy(nodemask_t *mask)
1911 {
1912         struct mempolicy *mempolicy;
1913         int nid;
1914 
1915         if (!(mask && current->mempolicy))
1916                 return false;
1917 
1918         task_lock(current);
1919         mempolicy = current->mempolicy;
1920         switch (mempolicy->mode) {
1921         case MPOL_PREFERRED:
1922                 if (mempolicy->flags & MPOL_F_LOCAL)
1923                         nid = numa_node_id();
1924                 else
1925                         nid = mempolicy->v.preferred_node;
1926                 init_nodemask_of_node(mask, nid);
1927                 break;
1928 
1929         case MPOL_BIND:
1930                 /* Fall through */
1931         case MPOL_INTERLEAVE:
1932                 *mask =  mempolicy->v.nodes;
1933                 break;
1934 
1935         default:
1936                 BUG();
1937         }
1938         task_unlock(current);
1939 
1940         return true;
1941 }
1942 #endif
1943 
1944 /*
1945  * mempolicy_nodemask_intersects
1946  *
1947  * If tsk's mempolicy is "default" [NULL], return 'true' to indicate default
1948  * policy.  Otherwise, check for intersection between mask and the policy
1949  * nodemask for 'bind' or 'interleave' policy.  For 'perferred' or 'local'
1950  * policy, always return true since it may allocate elsewhere on fallback.
1951  *
1952  * Takes task_lock(tsk) to prevent freeing of its mempolicy.
1953  */
1954 bool mempolicy_nodemask_intersects(struct task_struct *tsk,
1955                                         const nodemask_t *mask)
1956 {
1957         struct mempolicy *mempolicy;
1958         bool ret = true;
1959 
1960         if (!mask)
1961                 return ret;
1962         task_lock(tsk);
1963         mempolicy = tsk->mempolicy;
1964         if (!mempolicy)
1965                 goto out;
1966 
1967         switch (mempolicy->mode) {
1968         case MPOL_PREFERRED:
1969                 /*
1970                  * MPOL_PREFERRED and MPOL_F_LOCAL are only preferred nodes to
1971                  * allocate from, they may fallback to other nodes when oom.
1972                  * Thus, it's possible for tsk to have allocated memory from
1973                  * nodes in mask.
1974                  */
1975                 break;
1976         case MPOL_BIND:
1977         case MPOL_INTERLEAVE:
1978                 ret = nodes_intersects(mempolicy->v.nodes, *mask);
1979                 break;
1980         default:
1981                 BUG();
1982         }
1983 out:
1984         task_unlock(tsk);
1985         return ret;
1986 }
1987 
1988 /* Allocate a page in interleaved policy.
1989    Own path because it needs to do special accounting. */
1990 static struct page *alloc_page_interleave(gfp_t gfp, unsigned order,
1991                                         unsigned nid)
1992 {
1993         struct zonelist *zl;
1994         struct page *page;
1995 
1996         zl = node_zonelist(nid, gfp);
1997         page = __alloc_pages(gfp, order, zl);
1998         if (page && page_zone(page) == zonelist_zone(&zl->_zonerefs[0]))
1999                 inc_zone_page_state(page, NUMA_INTERLEAVE_HIT);
2000         return page;
2001 }
2002 
2003 /**
2004  *      alloc_pages_vma - Allocate a page for a VMA.
2005  *
2006  *      @gfp:
2007  *      %GFP_USER    user allocation.
2008  *      %GFP_KERNEL  kernel allocations,
2009  *      %GFP_HIGHMEM highmem/user allocations,
2010  *      %GFP_FS      allocation should not call back into a file system.
2011  *      %GFP_ATOMIC  don't sleep.
2012  *
2013  *      @order:Order of the GFP allocation.
2014  *      @vma:  Pointer to VMA or NULL if not available.
2015  *      @addr: Virtual Address of the allocation. Must be inside the VMA.
2016  *
2017  *      This function allocates a page from the kernel page pool and applies
2018  *      a NUMA policy associated with the VMA or the current process.
2019  *      When VMA is not NULL caller must hold down_read on the mmap_sem of the
2020  *      mm_struct of the VMA to prevent it from going away. Should be used for
2021  *      all allocations for pages that will be mapped into
2022  *      user space. Returns NULL when no page can be allocated.
2023  *
2024  *      Should be called with the mm_sem of the vma hold.
2025  */
2026 struct page *
2027 alloc_pages_vma(gfp_t gfp, int order, struct vm_area_struct *vma,
2028                 unsigned long addr, int node)
2029 {
2030         struct mempolicy *pol;
2031         struct page *page;
2032         unsigned int cpuset_mems_cookie;
2033 
2034 retry_cpuset:
2035         pol = get_vma_policy(current, vma, addr);
2036         cpuset_mems_cookie = read_mems_allowed_begin();
2037 
2038         if (unlikely(pol->mode == MPOL_INTERLEAVE)) {
2039                 unsigned nid;
2040 
2041                 nid = interleave_nid(pol, vma, addr, PAGE_SHIFT + order);
2042                 mpol_cond_put(pol);
2043                 page = alloc_page_interleave(gfp, order, nid);
2044                 if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
2045                         goto retry_cpuset;
2046 
2047                 return page;
2048         }
2049         page = __alloc_pages_nodemask(gfp, order,
2050                                       policy_zonelist(gfp, pol, node),
2051                                       policy_nodemask(gfp, pol));
2052         if (unlikely(mpol_needs_cond_ref(pol)))
2053                 __mpol_put(pol);
2054         if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
2055                 goto retry_cpuset;
2056         return page;
2057 }
2058 
2059 /**
2060  *      alloc_pages_current - Allocate pages.
2061  *
2062  *      @gfp:
2063  *              %GFP_USER   user allocation,
2064  *              %GFP_KERNEL kernel allocation,
2065  *              %GFP_HIGHMEM highmem allocation,
2066  *              %GFP_FS     don't call back into a file system.
2067  *              %GFP_ATOMIC don't sleep.
2068  *      @order: Power of two of allocation size in pages. 0 is a single page.
2069  *
2070  *      Allocate a page from the kernel page pool.  When not in
2071  *      interrupt context and apply the current process NUMA policy.
2072  *      Returns NULL when no page can be allocated.
2073  *
2074  *      Don't call cpuset_update_task_memory_state() unless
2075  *      1) it's ok to take cpuset_sem (can WAIT), and
2076  *      2) allocating for current task (not interrupt).
2077  */
2078 struct page *alloc_pages_current(gfp_t gfp, unsigned order)
2079 {
2080         struct mempolicy *pol = get_task_policy(current);
2081         struct page *page;
2082         unsigned int cpuset_mems_cookie;
2083 
2084         if (!pol || in_interrupt() || (gfp & __GFP_THISNODE))
2085                 pol = &default_policy;
2086 
2087 retry_cpuset:
2088         cpuset_mems_cookie = read_mems_allowed_begin();
2089 
2090         /*
2091          * No reference counting needed for current->mempolicy
2092          * nor system default_policy
2093          */
2094         if (pol->mode == MPOL_INTERLEAVE)
2095                 page = alloc_page_interleave(gfp, order, interleave_nodes(pol));
2096         else
2097                 page = __alloc_pages_nodemask(gfp, order,
2098                                 policy_zonelist(gfp, pol, numa_node_id()),
2099                                 policy_nodemask(gfp, pol));
2100 
2101         if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
2102                 goto retry_cpuset;
2103 
2104         return page;
2105 }
2106 EXPORT_SYMBOL(alloc_pages_current);
2107 
2108 int vma_dup_policy(struct vm_area_struct *src, struct vm_area_struct *dst)
2109 {
2110         struct mempolicy *pol = mpol_dup(vma_policy(src));
2111 
2112         if (IS_ERR(pol))
2113                 return PTR_ERR(pol);
2114         dst->vm_policy = pol;
2115         return 0;
2116 }
2117 
2118 /*
2119  * If mpol_dup() sees current->cpuset == cpuset_being_rebound, then it
2120  * rebinds the mempolicy its copying by calling mpol_rebind_policy()
2121  * with the mems_allowed returned by cpuset_mems_allowed().  This
2122  * keeps mempolicies cpuset relative after its cpuset moves.  See
2123  * further kernel/cpuset.c update_nodemask().
2124  *
2125  * current's mempolicy may be rebinded by the other task(the task that changes
2126  * cpuset's mems), so we needn't do rebind work for current task.
2127  */
2128 
2129 /* Slow path of a mempolicy duplicate */
2130 struct mempolicy *__mpol_dup(struct mempolicy *old)
2131 {
2132         struct mempolicy *new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
2133 
2134         if (!new)
2135                 return ERR_PTR(-ENOMEM);
2136 
2137         /* task's mempolicy is protected by alloc_lock */
2138         if (old == current->mempolicy) {
2139                 task_lock(current);
2140                 *new = *old;
2141                 task_unlock(current);
2142         } else
2143                 *new = *old;
2144 
2145         if (current_cpuset_is_being_rebound()) {
2146                 nodemask_t mems = cpuset_mems_allowed(current);
2147                 if (new->flags & MPOL_F_REBINDING)
2148                         mpol_rebind_policy(new, &mems, MPOL_REBIND_STEP2);
2149                 else
2150                         mpol_rebind_policy(new, &mems, MPOL_REBIND_ONCE);
2151         }
2152         atomic_set(&new->refcnt, 1);
2153         return new;
2154 }
2155 
2156 /* Slow path of a mempolicy comparison */
2157 bool __mpol_equal(struct mempolicy *a, struct mempolicy *b)
2158 {
2159         if (!a || !b)
2160                 return false;
2161         if (a->mode != b->mode)
2162                 return false;
2163         if (a->flags != b->flags)
2164                 return false;
2165         if (mpol_store_user_nodemask(a))
2166                 if (!nodes_equal(a->w.user_nodemask, b->w.user_nodemask))
2167                         return false;
2168 
2169         switch (a->mode) {
2170         case MPOL_BIND:
2171                 /* Fall through */
2172         case MPOL_INTERLEAVE:
2173                 return !!nodes_equal(a->v.nodes, b->v.nodes);
2174         case MPOL_PREFERRED:
2175                 return a->v.preferred_node == b->v.preferred_node;
2176         default:
2177                 BUG();
2178                 return false;
2179         }
2180 }
2181 
2182 /*
2183  * Shared memory backing store policy support.
2184  *
2185  * Remember policies even when nobody has shared memory mapped.
2186  * The policies are kept in Red-Black tree linked from the inode.
2187  * They are protected by the sp->lock spinlock, which should be held
2188  * for any accesses to the tree.
2189  */
2190 
2191 /* lookup first element intersecting start-end */
2192 /* Caller holds sp->lock */
2193 static struct sp_node *
2194 sp_lookup(struct shared_policy *sp, unsigned long start, unsigned long end)
2195 {
2196         struct rb_node *n = sp->root.rb_node;
2197 
2198         while (n) {
2199                 struct sp_node *p = rb_entry(n, struct sp_node, nd);
2200 
2201                 if (start >= p->end)
2202                         n = n->rb_right;
2203                 else if (end <= p->start)
2204                         n = n->rb_left;
2205                 else
2206                         break;
2207         }
2208         if (!n)
2209                 return NULL;
2210         for (;;) {
2211                 struct sp_node *w = NULL;
2212                 struct rb_node *prev = rb_prev(n);
2213                 if (!prev)
2214                         break;
2215                 w = rb_entry(prev, struct sp_node, nd);
2216                 if (w->end <= start)
2217                         break;
2218                 n = prev;
2219         }
2220         return rb_entry(n, struct sp_node, nd);
2221 }
2222 
2223 /* Insert a new shared policy into the list. */
2224 /* Caller holds sp->lock */
2225 static void sp_insert(struct shared_policy *sp, struct sp_node *new)
2226 {
2227         struct rb_node **p = &sp->root.rb_node;
2228         struct rb_node *parent = NULL;
2229         struct sp_node *nd;
2230 
2231         while (*p) {
2232                 parent = *p;
2233                 nd = rb_entry(parent, struct sp_node, nd);
2234                 if (new->start < nd->start)
2235                         p = &(*p)->rb_left;
2236                 else if (new->end > nd->end)
2237                         p = &(*p)->rb_right;
2238                 else
2239                         BUG();
2240         }
2241         rb_link_node(&new->nd, parent, p);
2242         rb_insert_color(&new->nd, &sp->root);
2243         pr_debug("inserting %lx-%lx: %d\n", new->start, new->end,
2244                  new->policy ? new->policy->mode : 0);
2245 }
2246 
2247 /* Find shared policy intersecting idx */
2248 struct mempolicy *
2249 mpol_shared_policy_lookup(struct shared_policy *sp, unsigned long idx)
2250 {
2251         struct mempolicy *pol = NULL;
2252         struct sp_node *sn;
2253 
2254         if (!sp->root.rb_node)
2255                 return NULL;
2256         spin_lock(&sp->lock);
2257         sn = sp_lookup(sp, idx, idx+1);
2258         if (sn) {
2259                 mpol_get(sn->policy);
2260                 pol = sn->policy;
2261         }
2262         spin_unlock(&sp->lock);
2263         return pol;
2264 }
2265 
2266 static void sp_free(struct sp_node *n)
2267 {
2268         mpol_put(n->policy);
2269         kmem_cache_free(sn_cache, n);
2270 }
2271 
2272 /**
2273  * mpol_misplaced - check whether current page node is valid in policy
2274  *
2275  * @page   - page to be checked
2276  * @vma    - vm area where page mapped
2277  * @addr   - virtual address where page mapped
2278  *
2279  * Lookup current policy node id for vma,addr and "compare to" page's
2280  * node id.
2281  *
2282  * Returns:
2283  *      -1      - not misplaced, page is in the right node
2284  *      node    - node id where the page should be
2285  *
2286  * Policy determination "mimics" alloc_page_vma().
2287  * Called from fault path where we know the vma and faulting address.
2288  */
2289 int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long addr)
2290 {
2291         struct mempolicy *pol;
2292         struct zone *zone;
2293         int curnid = page_to_nid(page);
2294         unsigned long pgoff;
2295         int polnid = -1;
2296         int ret = -1;
2297 
2298         BUG_ON(!vma);
2299 
2300         pol = get_vma_policy(current, vma, addr);
2301         if (!(pol->flags & MPOL_F_MOF))
2302                 goto out;
2303 
2304         switch (pol->mode) {
2305         case MPOL_INTERLEAVE:
2306                 BUG_ON(addr >= vma->vm_end);
2307                 BUG_ON(addr < vma->vm_start);
2308 
2309                 pgoff = vma->vm_pgoff;
2310                 pgoff += (addr - vma->vm_start) >> PAGE_SHIFT;
2311                 polnid = offset_il_node(pol, vma, pgoff);
2312                 break;
2313 
2314         case MPOL_PREFERRED:
2315                 if (pol->flags & MPOL_F_LOCAL)
2316                         polnid = numa_node_id();
2317                 else
2318                         polnid = pol->v.preferred_node;
2319                 break;
2320 
2321         case MPOL_BIND:
2322                 /*
2323                  * allows binding to multiple nodes.
2324                  * use current page if in policy nodemask,
2325                  * else select nearest allowed node, if any.
2326                  * If no allowed nodes, use current [!misplaced].
2327                  */
2328                 if (node_isset(curnid, pol->v.nodes))
2329                         goto out;
2330                 (void)first_zones_zonelist(
2331                                 node_zonelist(numa_node_id(), GFP_HIGHUSER),
2332                                 gfp_zone(GFP_HIGHUSER),
2333                                 &pol->v.nodes, &zone);
2334                 polnid = zone->node;
2335                 break;
2336 
2337         default:
2338                 BUG();
2339         }
2340 
2341         /* Migrate the page towards the node whose CPU is referencing it */
2342         if (pol->flags & MPOL_F_MORON) {
2343                 int last_nid;
2344 
2345                 polnid = numa_node_id();
2346 
2347                 /*
2348                  * Multi-stage node selection is used in conjunction
2349                  * with a periodic migration fault to build a temporal
2350                  * task<->page relation. By using a two-stage filter we
2351                  * remove short/unlikely relations.
2352                  *
2353                  * Using P(p) ~ n_p / n_t as per frequentist
2354                  * probability, we can equate a task's usage of a
2355                  * particular page (n_p) per total usage of this
2356                  * page (n_t) (in a given time-span) to a probability.
2357                  *
2358                  * Our periodic faults will sample this probability and
2359                  * getting the same result twice in a row, given these
2360                  * samples are fully independent, is then given by
2361                  * P(n)^2, provided our sample period is sufficiently
2362                  * short compared to the usage pattern.
2363                  *
2364                  * This quadric squishes small probabilities, making
2365                  * it less likely we act on an unlikely task<->page
2366                  * relation.
2367                  */
2368                 last_nid = page_nid_xchg_last(page, polnid);
2369                 if (last_nid != polnid)
2370                         goto out;
2371         }
2372 
2373         if (curnid != polnid)
2374                 ret = polnid;
2375 out:
2376         mpol_cond_put(pol);
2377 
2378         return ret;
2379 }
2380 
2381 static void sp_delete(struct shared_policy *sp, struct sp_node *n)
2382 {
2383         pr_debug("deleting %lx-l%lx\n", n->start, n->end);
2384         rb_erase(&n->nd, &sp->root);
2385         sp_free(n);
2386 }
2387 
2388 static void sp_node_init(struct sp_node *node, unsigned long start,
2389                         unsigned long end, struct mempolicy *pol)
2390 {
2391         node->start = start;
2392         node->end = end;
2393         node->policy = pol;
2394 }
2395 
2396 static struct sp_node *sp_alloc(unsigned long start, unsigned long end,
2397                                 struct mempolicy *pol)
2398 {
2399         struct sp_node *n;
2400         struct mempolicy *newpol;
2401 
2402         n = kmem_cache_alloc(sn_cache, GFP_KERNEL);
2403         if (!n)
2404                 return NULL;
2405 
2406         newpol = mpol_dup(pol);
2407         if (IS_ERR(newpol)) {
2408                 kmem_cache_free(sn_cache, n);
2409                 return NULL;
2410         }
2411         newpol->flags |= MPOL_F_SHARED;
2412         sp_node_init(n, start, end, newpol);
2413 
2414         return n;
2415 }
2416 
2417 /* Replace a policy range. */
2418 static int shared_policy_replace(struct shared_policy *sp, unsigned long start,
2419                                  unsigned long end, struct sp_node *new)
2420 {
2421         struct sp_node *n;
2422         struct sp_node *n_new = NULL;
2423         struct mempolicy *mpol_new = NULL;
2424         int ret = 0;
2425 
2426 restart:
2427         spin_lock(&sp->lock);
2428         n = sp_lookup(sp, start, end);
2429         /* Take care of old policies in the same range. */
2430         while (n && n->start < end) {
2431                 struct rb_node *next = rb_next(&n->nd);
2432                 if (n->start >= start) {
2433                         if (n->end <= end)
2434                                 sp_delete(sp, n);
2435                         else
2436                                 n->start = end;
2437                 } else {
2438                         /* Old policy spanning whole new range. */
2439                         if (n->end > end) {
2440                                 if (!n_new)
2441                                         goto alloc_new;
2442 
2443                                 *mpol_new = *n->policy;
2444                                 atomic_set(&mpol_new->refcnt, 1);
2445                                 sp_node_init(n_new, end, n->end, mpol_new);
2446                                 n->end = start;
2447                                 sp_insert(sp, n_new);
2448                                 n_new = NULL;
2449                                 mpol_new = NULL;
2450                                 break;
2451                         } else
2452                                 n->end = start;
2453                 }
2454                 if (!next)
2455                         break;
2456                 n = rb_entry(next, struct sp_node, nd);
2457         }
2458         if (new)
2459                 sp_insert(sp, new);
2460         spin_unlock(&sp->lock);
2461         ret = 0;
2462 
2463 err_out:
2464         if (mpol_new)
2465                 mpol_put(mpol_new);
2466         if (n_new)
2467                 kmem_cache_free(sn_cache, n_new);
2468 
2469         return ret;
2470 
2471 alloc_new:
2472         spin_unlock(&sp->lock);
2473         ret = -ENOMEM;
2474         n_new = kmem_cache_alloc(sn_cache, GFP_KERNEL);
2475         if (!n_new)
2476                 goto err_out;
2477         mpol_new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
2478         if (!mpol_new)
2479                 goto err_out;
2480         goto restart;
2481 }
2482 
2483 /**
2484  * mpol_shared_policy_init - initialize shared policy for inode
2485  * @sp: pointer to inode shared policy
2486  * @mpol:  struct mempolicy to install
2487  *
2488  * Install non-NULL @mpol in inode's shared policy rb-tree.
2489  * On entry, the current task has a reference on a non-NULL @mpol.
2490  * This must be released on exit.
2491  * This is called at get_inode() calls and we can use GFP_KERNEL.
2492  */
2493 void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol)
2494 {
2495         int ret;
2496 
2497         sp->root = RB_ROOT;             /* empty tree == default mempolicy */
2498         spin_lock_init(&sp->lock);
2499 
2500         if (mpol) {
2501                 struct vm_area_struct pvma;
2502                 struct mempolicy *new;
2503                 NODEMASK_SCRATCH(scratch);
2504 
2505                 if (!scratch)
2506                         goto put_mpol;
2507                 /* contextualize the tmpfs mount point mempolicy */
2508                 new = mpol_new(mpol->mode, mpol->flags, &mpol->w.user_nodemask);
2509                 if (IS_ERR(new))
2510                         goto free_scratch; /* no valid nodemask intersection */
2511 
2512                 task_lock(current);
2513                 ret = mpol_set_nodemask(new, &mpol->w.user_nodemask, scratch);
2514                 task_unlock(current);
2515                 if (ret)
2516                         goto put_new;
2517 
2518                 /* Create pseudo-vma that contains just the policy */
2519                 memset(&pvma, 0, sizeof(struct vm_area_struct));
2520                 pvma.vm_end = TASK_SIZE;        /* policy covers entire file */
2521                 mpol_set_shared_policy(sp, &pvma, new); /* adds ref */
2522 
2523 put_new:
2524                 mpol_put(new);                  /* drop initial ref */
2525 free_scratch:
2526                 NODEMASK_SCRATCH_FREE(scratch);
2527 put_mpol:
2528                 mpol_put(mpol); /* drop our incoming ref on sb mpol */
2529         }
2530 }
2531 
2532 int mpol_set_shared_policy(struct shared_policy *info,
2533                         struct vm_area_struct *vma, struct mempolicy *npol)
2534 {
2535         int err;
2536         struct sp_node *new = NULL;
2537         unsigned long sz = vma_pages(vma);
2538 
2539         pr_debug("set_shared_policy %lx sz %lu %d %d %lx\n",
2540                  vma->vm_pgoff,
2541                  sz, npol ? npol->mode : -1,
2542                  npol ? npol->flags : -1,
2543                  npol ? nodes_addr(npol->v.nodes)[0] : NUMA_NO_NODE);
2544 
2545         if (npol) {
2546                 new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, npol);
2547                 if (!new)
2548                         return -ENOMEM;
2549         }
2550         err = shared_policy_replace(info, vma->vm_pgoff, vma->vm_pgoff+sz, new);
2551         if (err && new)
2552                 sp_free(new);
2553         return err;
2554 }
2555 
2556 /* Free a backing policy store on inode delete. */
2557 void mpol_free_shared_policy(struct shared_policy *p)
2558 {
2559         struct sp_node *n;
2560         struct rb_node *next;
2561 
2562         if (!p->root.rb_node)
2563                 return;
2564         spin_lock(&p->lock);
2565         next = rb_first(&p->root);
2566         while (next) {
2567                 n = rb_entry(next, struct sp_node, nd);
2568                 next = rb_next(&n->nd);
2569                 sp_delete(p, n);
2570         }
2571         spin_unlock(&p->lock);
2572 }
2573 
2574 #ifdef CONFIG_NUMA_BALANCING
2575 static int __initdata numabalancing_override;
2576 
2577 static void __init check_numabalancing_enable(void)
2578 {
2579         bool numabalancing_default = false;
2580 
2581         if (IS_ENABLED(CONFIG_NUMA_BALANCING_DEFAULT_ENABLED))
2582                 numabalancing_default = true;
2583 
2584         /* Parsed by setup_numabalancing. override == 1 enables, -1 disables */
2585         if (numabalancing_override)
2586                 set_numabalancing_state(numabalancing_override == 1);
2587 
2588         if (num_online_nodes() > 1 && !numabalancing_override) {
2589                 printk(KERN_INFO "%s automatic NUMA balancing. "
2590                         "Configure with numa_balancing= or sysctl",
2591                         numabalancing_default ? "Enabling" : "Disabling");
2592                 set_numabalancing_state(numabalancing_default);
2593         }
2594 }
2595 
2596 static int __init setup_numabalancing(char *str)
2597 {
2598         int ret = 0;
2599         if (!str)
2600                 goto out;
2601 
2602         if (!strcmp(str, "enable")) {
2603                 numabalancing_override = 1;
2604                 ret = 1;
2605         } else if (!strcmp(str, "disable")) {
2606                 numabalancing_override = -1;
2607                 ret = 1;
2608         }
2609 out:
2610         if (!ret)
2611                 printk(KERN_WARNING "Unable to parse numa_balancing=\n");
2612 
2613         return ret;
2614 }
2615 __setup("numa_balancing=", setup_numabalancing);
2616 #else
2617 static inline void __init check_numabalancing_enable(void)
2618 {
2619 }
2620 #endif /* CONFIG_NUMA_BALANCING */
2621 
2622 /* assumes fs == KERNEL_DS */
2623 void __init numa_policy_init(void)
2624 {
2625         nodemask_t interleave_nodes;
2626         unsigned long largest = 0;
2627         int nid, prefer = 0;
2628 
2629         policy_cache = kmem_cache_create("numa_policy",
2630                                          sizeof(struct mempolicy),
2631                                          0, SLAB_PANIC, NULL);
2632 
2633         sn_cache = kmem_cache_create("shared_policy_node",
2634                                      sizeof(struct sp_node),
2635                                      0, SLAB_PANIC, NULL);
2636 
2637         for_each_node(nid) {
2638                 preferred_node_policy[nid] = (struct mempolicy) {
2639                         .refcnt = ATOMIC_INIT(1),
2640                         .mode = MPOL_PREFERRED,
2641                         .flags = MPOL_F_MOF | MPOL_F_MORON,
2642                         .v = { .preferred_node = nid, },
2643                 };
2644         }
2645 
2646         /*
2647          * Set interleaving policy for system init. Interleaving is only
2648          * enabled across suitably sized nodes (default is >= 16MB), or
2649          * fall back to the largest node if they're all smaller.
2650          */
2651         nodes_clear(interleave_nodes);
2652         for_each_node_state(nid, N_MEMORY) {
2653                 unsigned long total_pages = node_present_pages(nid);
2654 
2655                 /* Preserve the largest node */
2656                 if (largest < total_pages) {
2657                         largest = total_pages;
2658                         prefer = nid;
2659                 }
2660 
2661                 /* Interleave this node? */
2662                 if ((total_pages << PAGE_SHIFT) >= (16 << 20))
2663                         node_set(nid, interleave_nodes);
2664         }
2665 
2666         /* All too small, use the largest */
2667         if (unlikely(nodes_empty(interleave_nodes)))
2668                 node_set(prefer, interleave_nodes);
2669 
2670         if (do_set_mempolicy(MPOL_INTERLEAVE, 0, &interleave_nodes))
2671                 printk("numa_policy_init: interleaving failed\n");
2672 
2673         check_numabalancing_enable();
2674 }
2675 
2676 /* Reset policy of current process to default */
2677 void numa_default_policy(void)
2678 {
2679         do_set_mempolicy(MPOL_DEFAULT, 0, NULL);
2680 }
2681 
2682 /*
2683  * Parse and format mempolicy from/to strings
2684  */
2685 
2686 /*
2687  * "local" is implemented internally by MPOL_PREFERRED with MPOL_F_LOCAL flag.
2688  */
2689 static const char * const policy_modes[] =
2690 {
2691         [MPOL_DEFAULT]    = "default",
2692         [MPOL_PREFERRED]  = "prefer",
2693         [MPOL_BIND]       = "bind",
2694         [MPOL_INTERLEAVE] = "interleave",
2695         [MPOL_LOCAL]      = "local",
2696 };
2697 
2698 
2699 #ifdef CONFIG_TMPFS
2700 /**
2701  * mpol_parse_str - parse string to mempolicy, for tmpfs mpol mount option.
2702  * @str:  string containing mempolicy to parse
2703  * @mpol:  pointer to struct mempolicy pointer, returned on success.
2704  *
2705  * Format of input:
2706  *      <mode>[=<flags>][:<nodelist>]
2707  *
2708  * On success, returns 0, else 1
2709  */
2710 int mpol_parse_str(char *str, struct mempolicy **mpol)
2711 {
2712         struct mempolicy *new = NULL;
2713         unsigned short mode;
2714         unsigned short mode_flags;
2715         nodemask_t nodes;
2716         char *nodelist = strchr(str, ':');
2717         char *flags = strchr(str, '=');
2718         int err = 1;
2719 
2720         if (nodelist) {
2721                 /* NUL-terminate mode or flags string */
2722                 *nodelist++ = '\0';
2723                 if (nodelist_parse(nodelist, nodes))
2724                         goto out;
2725                 if (!nodes_subset(nodes, node_states[N_MEMORY]))
2726                         goto out;
2727         } else
2728                 nodes_clear(nodes);
2729 
2730         if (flags)
2731                 *flags++ = '\0';        /* terminate mode string */
2732 
2733         for (mode = 0; mode < MPOL_MAX; mode++) {
2734                 if (!strcmp(str, policy_modes[mode])) {
2735                         break;
2736                 }
2737         }
2738         if (mode >= MPOL_MAX)
2739                 goto out;
2740 
2741         switch (mode) {
2742         case MPOL_PREFERRED:
2743                 /*
2744                  * Insist on a nodelist of one node only
2745                  */
2746                 if (nodelist) {
2747                         char *rest = nodelist;
2748                         while (isdigit(*rest))
2749                                 rest++;
2750                         if (*rest)
2751                                 goto out;
2752                 }
2753                 break;
2754         case MPOL_INTERLEAVE:
2755                 /*
2756                  * Default to online nodes with memory if no nodelist
2757                  */
2758                 if (!nodelist)
2759                         nodes = node_states[N_MEMORY];
2760                 break;
2761         case MPOL_LOCAL:
2762                 /*
2763                  * Don't allow a nodelist;  mpol_new() checks flags
2764                  */
2765                 if (nodelist)
2766                         goto out;
2767                 mode = MPOL_PREFERRED;
2768                 break;
2769         case MPOL_DEFAULT:
2770                 /*
2771                  * Insist on a empty nodelist
2772                  */
2773                 if (!nodelist)
2774                         err = 0;
2775                 goto out;
2776         case MPOL_BIND:
2777                 /*
2778                  * Insist on a nodelist
2779                  */
2780                 if (!nodelist)
2781                         goto out;
2782         }
2783 
2784         mode_flags = 0;
2785         if (flags) {
2786                 /*
2787                  * Currently, we only support two mutually exclusive
2788                  * mode flags.
2789                  */
2790                 if (!strcmp(flags, "static"))
2791                         mode_flags |= MPOL_F_STATIC_NODES;
2792                 else if (!strcmp(flags, "relative"))
2793                         mode_flags |= MPOL_F_RELATIVE_NODES;
2794                 else
2795                         goto out;
2796         }
2797 
2798         new = mpol_new(mode, mode_flags, &nodes);
2799         if (IS_ERR(new))
2800                 goto out;
2801 
2802         /*
2803          * Save nodes for mpol_to_str() to show the tmpfs mount options
2804          * for /proc/mounts, /proc/pid/mounts and /proc/pid/mountinfo.
2805          */
2806         if (mode != MPOL_PREFERRED)
2807                 new->v.nodes = nodes;
2808         else if (nodelist)
2809                 new->v.preferred_node = first_node(nodes);
2810         else
2811                 new->flags |= MPOL_F_LOCAL;
2812 
2813         /*
2814          * Save nodes for contextualization: this will be used to "clone"
2815          * the mempolicy in a specific context [cpuset] at a later time.
2816          */
2817         new->w.user_nodemask = nodes;
2818 
2819         err = 0;
2820 
2821 out:
2822         /* Restore string for error message */
2823         if (nodelist)
2824                 *--nodelist = ':';
2825         if (flags)
2826                 *--flags = '=';
2827         if (!err)
2828                 *mpol = new;
2829         return err;
2830 }
2831 #endif /* CONFIG_TMPFS */
2832 
2833 /**
2834  * mpol_to_str - format a mempolicy structure for printing
2835  * @buffer:  to contain formatted mempolicy string
2836  * @maxlen:  length of @buffer
2837  * @pol:  pointer to mempolicy to be formatted
2838  *
2839  * Convert a mempolicy into a string.
2840  * Returns the number of characters in buffer (if positive)
2841  * or an error (negative)
2842  */
2843 int mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol)
2844 {
2845         char *p = buffer;
2846         int l;
2847         nodemask_t nodes;
2848         unsigned short mode;
2849         unsigned short flags = pol ? pol->flags : 0;
2850 
2851         /*
2852          * Sanity check:  room for longest mode, flag and some nodes
2853          */
2854         VM_BUG_ON(maxlen < strlen("interleave") + strlen("relative") + 16);
2855 
2856         if (!pol || pol == &default_policy || (pol->flags & MPOL_F_MORON))
2857                 mode = MPOL_DEFAULT;
2858         else
2859                 mode = pol->mode;
2860 
2861         switch (mode) {
2862         case MPOL_DEFAULT:
2863                 nodes_clear(nodes);
2864                 break;
2865 
2866         case MPOL_PREFERRED:
2867                 nodes_clear(nodes);
2868                 if (flags & MPOL_F_LOCAL)
2869                         mode = MPOL_LOCAL;
2870                 else
2871                         node_set(pol->v.preferred_node, nodes);
2872                 break;
2873 
2874         case MPOL_BIND:
2875                 /* Fall through */
2876         case MPOL_INTERLEAVE:
2877                 nodes = pol->v.nodes;
2878                 break;
2879 
2880         default:
2881                 return -EINVAL;
2882         }
2883 
2884         l = strlen(policy_modes[mode]);
2885         if (buffer + maxlen < p + l + 1)
2886                 return -ENOSPC;
2887 
2888         strcpy(p, policy_modes[mode]);
2889         p += l;
2890 
2891         if (flags & MPOL_MODE_FLAGS) {
2892                 if (buffer + maxlen < p + 2)
2893                         return -ENOSPC;
2894                 *p++ = '=';
2895 
2896                 /*
2897                  * Currently, the only defined flags are mutually exclusive
2898                  */
2899                 if (flags & MPOL_F_STATIC_NODES)
2900                         p += snprintf(p, buffer + maxlen - p, "static");
2901                 else if (flags & MPOL_F_RELATIVE_NODES)
2902                         p += snprintf(p, buffer + maxlen - p, "relative");
2903         }
2904 
2905         if (!nodes_empty(nodes)) {
2906                 if (buffer + maxlen < p + 2)
2907                         return -ENOSPC;
2908                 *p++ = ':';
2909                 p += nodelist_scnprintf(p, buffer + maxlen - p, nodes);
2910         }
2911         return p - buffer;
2912 }
2913 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp