~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/arm64/mm/numa.c

Version: ~ [ linux-5.6-rc1 ] ~ [ linux-5.5.2 ] ~ [ linux-5.4.17 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.102 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.170 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.213 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.213 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.81 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * NUMA support, based on the x86 implementation.
  3  *
  4  * Copyright (C) 2015 Cavium Inc.
  5  * Author: Ganapatrao Kulkarni <gkulkarni@cavium.com>
  6  *
  7  * This program is free software; you can redistribute it and/or modify
  8  * it under the terms of the GNU General Public License version 2 as
  9  * published by the Free Software Foundation.
 10  *
 11  * This program is distributed in the hope that it will be useful,
 12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 14  * GNU General Public License for more details.
 15  *
 16  * You should have received a copy of the GNU General Public License
 17  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 18  */
 19 
 20 #define pr_fmt(fmt) "NUMA: " fmt
 21 
 22 #include <linux/acpi.h>
 23 #include <linux/bootmem.h>
 24 #include <linux/memblock.h>
 25 #include <linux/module.h>
 26 #include <linux/of.h>
 27 
 28 #include <asm/acpi.h>
 29 #include <asm/sections.h>
 30 
 31 struct pglist_data *node_data[MAX_NUMNODES] __read_mostly;
 32 EXPORT_SYMBOL(node_data);
 33 nodemask_t numa_nodes_parsed __initdata;
 34 static int cpu_to_node_map[NR_CPUS] = { [0 ... NR_CPUS-1] = NUMA_NO_NODE };
 35 
 36 static int numa_distance_cnt;
 37 static u8 *numa_distance;
 38 bool numa_off;
 39 
 40 static __init int numa_parse_early_param(char *opt)
 41 {
 42         if (!opt)
 43                 return -EINVAL;
 44         if (!strncmp(opt, "off", 3))
 45                 numa_off = true;
 46 
 47         return 0;
 48 }
 49 early_param("numa", numa_parse_early_param);
 50 
 51 cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
 52 EXPORT_SYMBOL(node_to_cpumask_map);
 53 
 54 #ifdef CONFIG_DEBUG_PER_CPU_MAPS
 55 
 56 /*
 57  * Returns a pointer to the bitmask of CPUs on Node 'node'.
 58  */
 59 const struct cpumask *cpumask_of_node(int node)
 60 {
 61         if (WARN_ON(node >= nr_node_ids))
 62                 return cpu_none_mask;
 63 
 64         if (WARN_ON(node_to_cpumask_map[node] == NULL))
 65                 return cpu_online_mask;
 66 
 67         return node_to_cpumask_map[node];
 68 }
 69 EXPORT_SYMBOL(cpumask_of_node);
 70 
 71 #endif
 72 
 73 static void map_cpu_to_node(unsigned int cpu, int nid)
 74 {
 75         set_cpu_numa_node(cpu, nid);
 76         if (nid >= 0)
 77                 cpumask_set_cpu(cpu, node_to_cpumask_map[nid]);
 78 }
 79 
 80 void numa_clear_node(unsigned int cpu)
 81 {
 82         int nid = cpu_to_node(cpu);
 83 
 84         if (nid >= 0)
 85                 cpumask_clear_cpu(cpu, node_to_cpumask_map[nid]);
 86         set_cpu_numa_node(cpu, NUMA_NO_NODE);
 87 }
 88 
 89 /*
 90  * Allocate node_to_cpumask_map based on number of available nodes
 91  * Requires node_possible_map to be valid.
 92  *
 93  * Note: cpumask_of_node() is not valid until after this is done.
 94  * (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.)
 95  */
 96 static void __init setup_node_to_cpumask_map(void)
 97 {
 98         int node;
 99 
100         /* setup nr_node_ids if not done yet */
101         if (nr_node_ids == MAX_NUMNODES)
102                 setup_nr_node_ids();
103 
104         /* allocate and clear the mapping */
105         for (node = 0; node < nr_node_ids; node++) {
106                 alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]);
107                 cpumask_clear(node_to_cpumask_map[node]);
108         }
109 
110         /* cpumask_of_node() will now work */
111         pr_debug("Node to cpumask map for %d nodes\n", nr_node_ids);
112 }
113 
114 /*
115  *  Set the cpu to node and mem mapping
116  */
117 void numa_store_cpu_info(unsigned int cpu)
118 {
119         map_cpu_to_node(cpu, cpu_to_node_map[cpu]);
120 }
121 
122 void __init early_map_cpu_to_node(unsigned int cpu, int nid)
123 {
124         /* fallback to node 0 */
125         if (nid < 0 || nid >= MAX_NUMNODES || numa_off)
126                 nid = 0;
127 
128         cpu_to_node_map[cpu] = nid;
129 
130         /*
131          * We should set the numa node of cpu0 as soon as possible, because it
132          * has already been set up online before. cpu_to_node(0) will soon be
133          * called.
134          */
135         if (!cpu)
136                 set_cpu_numa_node(cpu, nid);
137 }
138 
139 #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
140 unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
141 EXPORT_SYMBOL(__per_cpu_offset);
142 
143 static int __init early_cpu_to_node(int cpu)
144 {
145         return cpu_to_node_map[cpu];
146 }
147 
148 static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)
149 {
150         return node_distance(early_cpu_to_node(from), early_cpu_to_node(to));
151 }
152 
153 static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size,
154                                        size_t align)
155 {
156         int nid = early_cpu_to_node(cpu);
157 
158         return  memblock_virt_alloc_try_nid(size, align,
159                         __pa(MAX_DMA_ADDRESS), MEMBLOCK_ALLOC_ACCESSIBLE, nid);
160 }
161 
162 static void __init pcpu_fc_free(void *ptr, size_t size)
163 {
164         memblock_free_early(__pa(ptr), size);
165 }
166 
167 void __init setup_per_cpu_areas(void)
168 {
169         unsigned long delta;
170         unsigned int cpu;
171         int rc;
172 
173         /*
174          * Always reserve area for module percpu variables.  That's
175          * what the legacy allocator did.
176          */
177         rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE,
178                                     PERCPU_DYNAMIC_RESERVE, PAGE_SIZE,
179                                     pcpu_cpu_distance,
180                                     pcpu_fc_alloc, pcpu_fc_free);
181         if (rc < 0)
182                 panic("Failed to initialize percpu areas.");
183 
184         delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
185         for_each_possible_cpu(cpu)
186                 __per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];
187 }
188 #endif
189 
190 /**
191  * numa_add_memblk - Set node id to memblk
192  * @nid: NUMA node ID of the new memblk
193  * @start: Start address of the new memblk
194  * @end:  End address of the new memblk
195  *
196  * RETURNS:
197  * 0 on success, -errno on failure.
198  */
199 int __init numa_add_memblk(int nid, u64 start, u64 end)
200 {
201         int ret;
202 
203         ret = memblock_set_node(start, (end - start), &memblock.memory, nid);
204         if (ret < 0) {
205                 pr_err("memblock [0x%llx - 0x%llx] failed to add on node %d\n",
206                         start, (end - 1), nid);
207                 return ret;
208         }
209 
210         node_set(nid, numa_nodes_parsed);
211         pr_info("Adding memblock [0x%llx - 0x%llx] on node %d\n",
212                         start, (end - 1), nid);
213         return ret;
214 }
215 
216 /**
217  * Initialize NODE_DATA for a node on the local memory
218  */
219 static void __init setup_node_data(int nid, u64 start_pfn, u64 end_pfn)
220 {
221         const size_t nd_size = roundup(sizeof(pg_data_t), SMP_CACHE_BYTES);
222         u64 nd_pa;
223         void *nd;
224         int tnid;
225 
226         if (start_pfn < end_pfn)
227                 pr_info("Initmem setup node %d [mem %#010Lx-%#010Lx]\n", nid,
228                         start_pfn << PAGE_SHIFT, (end_pfn << PAGE_SHIFT) - 1);
229         else
230                 pr_info("Initmem setup node %d [<memory-less node>]\n", nid);
231 
232         nd_pa = memblock_alloc_try_nid(nd_size, SMP_CACHE_BYTES, nid);
233         nd = __va(nd_pa);
234 
235         /* report and initialize */
236         pr_info("NODE_DATA [mem %#010Lx-%#010Lx]\n",
237                 nd_pa, nd_pa + nd_size - 1);
238         tnid = early_pfn_to_nid(nd_pa >> PAGE_SHIFT);
239         if (tnid != nid)
240                 pr_info("NODE_DATA(%d) on node %d\n", nid, tnid);
241 
242         node_data[nid] = nd;
243         memset(NODE_DATA(nid), 0, sizeof(pg_data_t));
244         NODE_DATA(nid)->node_id = nid;
245         NODE_DATA(nid)->node_start_pfn = start_pfn;
246         NODE_DATA(nid)->node_spanned_pages = end_pfn - start_pfn;
247 }
248 
249 /**
250  * numa_free_distance
251  *
252  * The current table is freed.
253  */
254 void __init numa_free_distance(void)
255 {
256         size_t size;
257 
258         if (!numa_distance)
259                 return;
260 
261         size = numa_distance_cnt * numa_distance_cnt *
262                 sizeof(numa_distance[0]);
263 
264         memblock_free(__pa(numa_distance), size);
265         numa_distance_cnt = 0;
266         numa_distance = NULL;
267 }
268 
269 /**
270  *
271  * Create a new NUMA distance table.
272  *
273  */
274 static int __init numa_alloc_distance(void)
275 {
276         size_t size;
277         u64 phys;
278         int i, j;
279 
280         size = nr_node_ids * nr_node_ids * sizeof(numa_distance[0]);
281         phys = memblock_find_in_range(0, PFN_PHYS(max_pfn),
282                                       size, PAGE_SIZE);
283         if (WARN_ON(!phys))
284                 return -ENOMEM;
285 
286         memblock_reserve(phys, size);
287 
288         numa_distance = __va(phys);
289         numa_distance_cnt = nr_node_ids;
290 
291         /* fill with the default distances */
292         for (i = 0; i < numa_distance_cnt; i++)
293                 for (j = 0; j < numa_distance_cnt; j++)
294                         numa_distance[i * numa_distance_cnt + j] = i == j ?
295                                 LOCAL_DISTANCE : REMOTE_DISTANCE;
296 
297         pr_debug("Initialized distance table, cnt=%d\n", numa_distance_cnt);
298 
299         return 0;
300 }
301 
302 /**
303  * numa_set_distance - Set inter node NUMA distance from node to node.
304  * @from: the 'from' node to set distance
305  * @to: the 'to'  node to set distance
306  * @distance: NUMA distance
307  *
308  * Set the distance from node @from to @to to @distance.
309  * If distance table doesn't exist, a warning is printed.
310  *
311  * If @from or @to is higher than the highest known node or lower than zero
312  * or @distance doesn't make sense, the call is ignored.
313  *
314  */
315 void __init numa_set_distance(int from, int to, int distance)
316 {
317         if (!numa_distance) {
318                 pr_warn_once("Warning: distance table not allocated yet\n");
319                 return;
320         }
321 
322         if (from >= numa_distance_cnt || to >= numa_distance_cnt ||
323                         from < 0 || to < 0) {
324                 pr_warn_once("Warning: node ids are out of bound, from=%d to=%d distance=%d\n",
325                             from, to, distance);
326                 return;
327         }
328 
329         if ((u8)distance != distance ||
330             (from == to && distance != LOCAL_DISTANCE)) {
331                 pr_warn_once("Warning: invalid distance parameter, from=%d to=%d distance=%d\n",
332                              from, to, distance);
333                 return;
334         }
335 
336         numa_distance[from * numa_distance_cnt + to] = distance;
337 }
338 
339 /**
340  * Return NUMA distance @from to @to
341  */
342 int __node_distance(int from, int to)
343 {
344         if (from >= numa_distance_cnt || to >= numa_distance_cnt)
345                 return from == to ? LOCAL_DISTANCE : REMOTE_DISTANCE;
346         return numa_distance[from * numa_distance_cnt + to];
347 }
348 EXPORT_SYMBOL(__node_distance);
349 
350 static int __init numa_register_nodes(void)
351 {
352         int nid;
353         struct memblock_region *mblk;
354 
355         /* Check that valid nid is set to memblks */
356         for_each_memblock(memory, mblk)
357                 if (mblk->nid == NUMA_NO_NODE || mblk->nid >= MAX_NUMNODES) {
358                         pr_warn("Warning: invalid memblk node %d [mem %#010Lx-%#010Lx]\n",
359                                 mblk->nid, mblk->base,
360                                 mblk->base + mblk->size - 1);
361                         return -EINVAL;
362                 }
363 
364         /* Finally register nodes. */
365         for_each_node_mask(nid, numa_nodes_parsed) {
366                 unsigned long start_pfn, end_pfn;
367 
368                 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
369                 setup_node_data(nid, start_pfn, end_pfn);
370                 node_set_online(nid);
371         }
372 
373         /* Setup online nodes to actual nodes*/
374         node_possible_map = numa_nodes_parsed;
375 
376         return 0;
377 }
378 
379 static int __init numa_init(int (*init_func)(void))
380 {
381         int ret;
382 
383         nodes_clear(numa_nodes_parsed);
384         nodes_clear(node_possible_map);
385         nodes_clear(node_online_map);
386         numa_free_distance();
387 
388         ret = numa_alloc_distance();
389         if (ret < 0)
390                 return ret;
391 
392         ret = init_func();
393         if (ret < 0)
394                 return ret;
395 
396         if (nodes_empty(numa_nodes_parsed)) {
397                 pr_info("No NUMA configuration found\n");
398                 return -EINVAL;
399         }
400 
401         ret = numa_register_nodes();
402         if (ret < 0)
403                 return ret;
404 
405         setup_node_to_cpumask_map();
406 
407         return 0;
408 }
409 
410 /**
411  * dummy_numa_init - Fallback dummy NUMA init
412  *
413  * Used if there's no underlying NUMA architecture, NUMA initialization
414  * fails, or NUMA is disabled on the command line.
415  *
416  * Must online at least one node (node 0) and add memory blocks that cover all
417  * allowed memory. It is unlikely that this function fails.
418  */
419 static int __init dummy_numa_init(void)
420 {
421         int ret;
422         struct memblock_region *mblk;
423 
424         if (numa_off)
425                 pr_info("NUMA disabled\n"); /* Forced off on command line. */
426         pr_info("Faking a node at [mem %#018Lx-%#018Lx]\n",
427                 0LLU, PFN_PHYS(max_pfn) - 1);
428 
429         for_each_memblock(memory, mblk) {
430                 ret = numa_add_memblk(0, mblk->base, mblk->base + mblk->size);
431                 if (!ret)
432                         continue;
433 
434                 pr_err("NUMA init failed\n");
435                 return ret;
436         }
437 
438         numa_off = true;
439         return 0;
440 }
441 
442 /**
443  * arm64_numa_init - Initialize NUMA
444  *
445  * Try each configured NUMA initialization method until one succeeds.  The
446  * last fallback is dummy single node config encomapssing whole memory.
447  */
448 void __init arm64_numa_init(void)
449 {
450         if (!numa_off) {
451                 if (!acpi_disabled && !numa_init(arm64_acpi_numa_init))
452                         return;
453                 if (acpi_disabled && !numa_init(of_numa_init))
454                         return;
455         }
456 
457         numa_init(dummy_numa_init);
458 }
459 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp