~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/arm64/kernel/smp.c

Version: ~ [ linux-5.6 ] ~ [ linux-5.5.13 ] ~ [ linux-5.4.28 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.113 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.174 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.217 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.217 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.82 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * SMP initialisation and IPI support
  3  * Based on arch/arm/kernel/smp.c
  4  *
  5  * Copyright (C) 2012 ARM Ltd.
  6  *
  7  * This program is free software; you can redistribute it and/or modify
  8  * it under the terms of the GNU General Public License version 2 as
  9  * published by the Free Software Foundation.
 10  *
 11  * This program is distributed in the hope that it will be useful,
 12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 14  * GNU General Public License for more details.
 15  *
 16  * You should have received a copy of the GNU General Public License
 17  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 18  */
 19 
 20 #include <linux/delay.h>
 21 #include <linux/init.h>
 22 #include <linux/spinlock.h>
 23 #include <linux/sched.h>
 24 #include <linux/interrupt.h>
 25 #include <linux/cache.h>
 26 #include <linux/profile.h>
 27 #include <linux/errno.h>
 28 #include <linux/mm.h>
 29 #include <linux/err.h>
 30 #include <linux/cpu.h>
 31 #include <linux/smp.h>
 32 #include <linux/seq_file.h>
 33 #include <linux/irq.h>
 34 #include <linux/percpu.h>
 35 #include <linux/clockchips.h>
 36 #include <linux/completion.h>
 37 #include <linux/of.h>
 38 
 39 #include <asm/atomic.h>
 40 #include <asm/cacheflush.h>
 41 #include <asm/cputype.h>
 42 #include <asm/mmu_context.h>
 43 #include <asm/pgtable.h>
 44 #include <asm/pgalloc.h>
 45 #include <asm/processor.h>
 46 #include <asm/smp_plat.h>
 47 #include <asm/sections.h>
 48 #include <asm/tlbflush.h>
 49 #include <asm/ptrace.h>
 50 
 51 /*
 52  * as from 2.5, kernels no longer have an init_tasks structure
 53  * so we need some other way of telling a new secondary core
 54  * where to place its SVC stack
 55  */
 56 struct secondary_data secondary_data;
 57 volatile unsigned long secondary_holding_pen_release = INVALID_HWID;
 58 
 59 enum ipi_msg_type {
 60         IPI_RESCHEDULE,
 61         IPI_CALL_FUNC,
 62         IPI_CALL_FUNC_SINGLE,
 63         IPI_CPU_STOP,
 64 };
 65 
 66 static DEFINE_RAW_SPINLOCK(boot_lock);
 67 
 68 /*
 69  * Write secondary_holding_pen_release in a way that is guaranteed to be
 70  * visible to all observers, irrespective of whether they're taking part
 71  * in coherency or not.  This is necessary for the hotplug code to work
 72  * reliably.
 73  */
 74 static void write_pen_release(u64 val)
 75 {
 76         void *start = (void *)&secondary_holding_pen_release;
 77         unsigned long size = sizeof(secondary_holding_pen_release);
 78 
 79         secondary_holding_pen_release = val;
 80         __flush_dcache_area(start, size);
 81 }
 82 
 83 /*
 84  * Boot a secondary CPU, and assign it the specified idle task.
 85  * This also gives us the initial stack to use for this CPU.
 86  */
 87 static int boot_secondary(unsigned int cpu, struct task_struct *idle)
 88 {
 89         unsigned long timeout;
 90 
 91         /*
 92          * Set synchronisation state between this boot processor
 93          * and the secondary one
 94          */
 95         raw_spin_lock(&boot_lock);
 96 
 97         /*
 98          * Update the pen release flag.
 99          */
100         write_pen_release(cpu_logical_map(cpu));
101 
102         /*
103          * Send an event, causing the secondaries to read pen_release.
104          */
105         sev();
106 
107         timeout = jiffies + (1 * HZ);
108         while (time_before(jiffies, timeout)) {
109                 if (secondary_holding_pen_release == INVALID_HWID)
110                         break;
111                 udelay(10);
112         }
113 
114         /*
115          * Now the secondary core is starting up let it run its
116          * calibrations, then wait for it to finish
117          */
118         raw_spin_unlock(&boot_lock);
119 
120         return secondary_holding_pen_release != INVALID_HWID ? -ENOSYS : 0;
121 }
122 
123 static DECLARE_COMPLETION(cpu_running);
124 
125 int __cpu_up(unsigned int cpu, struct task_struct *idle)
126 {
127         int ret;
128 
129         /*
130          * We need to tell the secondary core where to find its stack and the
131          * page tables.
132          */
133         secondary_data.stack = task_stack_page(idle) + THREAD_START_SP;
134         __flush_dcache_area(&secondary_data, sizeof(secondary_data));
135 
136         /*
137          * Now bring the CPU into our world.
138          */
139         ret = boot_secondary(cpu, idle);
140         if (ret == 0) {
141                 /*
142                  * CPU was successfully started, wait for it to come online or
143                  * time out.
144                  */
145                 wait_for_completion_timeout(&cpu_running,
146                                             msecs_to_jiffies(1000));
147 
148                 if (!cpu_online(cpu)) {
149                         pr_crit("CPU%u: failed to come online\n", cpu);
150                         ret = -EIO;
151                 }
152         } else {
153                 pr_err("CPU%u: failed to boot: %d\n", cpu, ret);
154         }
155 
156         secondary_data.stack = NULL;
157 
158         return ret;
159 }
160 
161 /*
162  * This is the secondary CPU boot entry.  We're using this CPUs
163  * idle thread stack, but a set of temporary page tables.
164  */
165 asmlinkage void secondary_start_kernel(void)
166 {
167         struct mm_struct *mm = &init_mm;
168         unsigned int cpu = smp_processor_id();
169 
170         printk("CPU%u: Booted secondary processor\n", cpu);
171 
172         /*
173          * All kernel threads share the same mm context; grab a
174          * reference and switch to it.
175          */
176         atomic_inc(&mm->mm_count);
177         current->active_mm = mm;
178         cpumask_set_cpu(cpu, mm_cpumask(mm));
179 
180         /*
181          * TTBR0 is only used for the identity mapping at this stage. Make it
182          * point to zero page to avoid speculatively fetching new entries.
183          */
184         cpu_set_reserved_ttbr0();
185         flush_tlb_all();
186 
187         preempt_disable();
188         trace_hardirqs_off();
189 
190         /*
191          * Let the primary processor know we're out of the
192          * pen, then head off into the C entry point
193          */
194         write_pen_release(INVALID_HWID);
195 
196         /*
197          * Synchronise with the boot thread.
198          */
199         raw_spin_lock(&boot_lock);
200         raw_spin_unlock(&boot_lock);
201 
202         /*
203          * Log the CPU info before it is marked online and might get read.
204          */
205         cpuinfo_store_cpu();
206 
207         /*
208          * OK, now it's safe to let the boot CPU continue.  Wait for
209          * the CPU migration code to notice that the CPU is online
210          * before we continue.
211          */
212         set_cpu_online(cpu, true);
213         complete(&cpu_running);
214 
215         /*
216          * Enable GIC and timers.
217          */
218         notify_cpu_starting(cpu);
219 
220         local_irq_enable();
221         local_fiq_enable();
222 
223         /*
224          * OK, it's off to the idle thread for us
225          */
226         cpu_startup_entry(CPUHP_ONLINE);
227 }
228 
229 void __init smp_cpus_done(unsigned int max_cpus)
230 {
231         pr_info("SMP: Total of %d processors activated.\n", num_online_cpus());
232 }
233 
234 void __init smp_prepare_boot_cpu(void)
235 {
236 }
237 
238 static void (*smp_cross_call)(const struct cpumask *, unsigned int);
239 
240 static const struct smp_enable_ops *enable_ops[] __initconst = {
241         &smp_spin_table_ops,
242         &smp_psci_ops,
243         NULL,
244 };
245 
246 static const struct smp_enable_ops *smp_enable_ops[NR_CPUS];
247 
248 static const struct smp_enable_ops * __init smp_get_enable_ops(const char *name)
249 {
250         const struct smp_enable_ops **ops = enable_ops;
251 
252         while (*ops) {
253                 if (!strcmp(name, (*ops)->name))
254                         return *ops;
255 
256                 ops++;
257         }
258 
259         return NULL;
260 }
261 
262 /*
263  * Enumerate the possible CPU set from the device tree and build the
264  * cpu logical map array containing MPIDR values related to logical
265  * cpus. Assumes that cpu_logical_map(0) has already been initialized.
266  */
267 void __init smp_init_cpus(void)
268 {
269         const char *enable_method;
270         struct device_node *dn = NULL;
271         int i, cpu = 1;
272         bool bootcpu_valid = false;
273 
274         while ((dn = of_find_node_by_type(dn, "cpu"))) {
275                 const u32 *cell;
276                 u64 hwid;
277 
278                 /*
279                  * A cpu node with missing "reg" property is
280                  * considered invalid to build a cpu_logical_map
281                  * entry.
282                  */
283                 cell = of_get_property(dn, "reg", NULL);
284                 if (!cell) {
285                         pr_err("%s: missing reg property\n", dn->full_name);
286                         goto next;
287                 }
288                 hwid = of_read_number(cell, of_n_addr_cells(dn));
289 
290                 /*
291                  * Non affinity bits must be set to 0 in the DT
292                  */
293                 if (hwid & ~MPIDR_HWID_BITMASK) {
294                         pr_err("%s: invalid reg property\n", dn->full_name);
295                         goto next;
296                 }
297 
298                 /*
299                  * Duplicate MPIDRs are a recipe for disaster. Scan
300                  * all initialized entries and check for
301                  * duplicates. If any is found just ignore the cpu.
302                  * cpu_logical_map was initialized to INVALID_HWID to
303                  * avoid matching valid MPIDR values.
304                  */
305                 for (i = 1; (i < cpu) && (i < NR_CPUS); i++) {
306                         if (cpu_logical_map(i) == hwid) {
307                                 pr_err("%s: duplicate cpu reg properties in the DT\n",
308                                         dn->full_name);
309                                 goto next;
310                         }
311                 }
312 
313                 /*
314                  * The numbering scheme requires that the boot CPU
315                  * must be assigned logical id 0. Record it so that
316                  * the logical map built from DT is validated and can
317                  * be used.
318                  */
319                 if (hwid == cpu_logical_map(0)) {
320                         if (bootcpu_valid) {
321                                 pr_err("%s: duplicate boot cpu reg property in DT\n",
322                                         dn->full_name);
323                                 goto next;
324                         }
325 
326                         bootcpu_valid = true;
327 
328                         /*
329                          * cpu_logical_map has already been
330                          * initialized and the boot cpu doesn't need
331                          * the enable-method so continue without
332                          * incrementing cpu.
333                          */
334                         continue;
335                 }
336 
337                 if (cpu >= NR_CPUS)
338                         goto next;
339 
340                 /*
341                  * We currently support only the "spin-table" enable-method.
342                  */
343                 enable_method = of_get_property(dn, "enable-method", NULL);
344                 if (!enable_method) {
345                         pr_err("%s: missing enable-method property\n",
346                                 dn->full_name);
347                         goto next;
348                 }
349 
350                 smp_enable_ops[cpu] = smp_get_enable_ops(enable_method);
351 
352                 if (!smp_enable_ops[cpu]) {
353                         pr_err("%s: invalid enable-method property: %s\n",
354                                dn->full_name, enable_method);
355                         goto next;
356                 }
357 
358                 if (smp_enable_ops[cpu]->init_cpu(dn, cpu))
359                         goto next;
360 
361                 pr_debug("cpu logical map 0x%llx\n", hwid);
362                 cpu_logical_map(cpu) = hwid;
363 next:
364                 cpu++;
365         }
366 
367         /* sanity check */
368         if (cpu > NR_CPUS)
369                 pr_warning("no. of cores (%d) greater than configured maximum of %d - clipping\n",
370                            cpu, NR_CPUS);
371 
372         if (!bootcpu_valid) {
373                 pr_err("DT missing boot CPU MPIDR, not enabling secondaries\n");
374                 return;
375         }
376 
377         /*
378          * All the cpus that made it to the cpu_logical_map have been
379          * validated so set them as possible cpus.
380          */
381         for (i = 0; i < NR_CPUS; i++)
382                 if (cpu_logical_map(i) != INVALID_HWID)
383                         set_cpu_possible(i, true);
384 }
385 
386 void __init smp_prepare_cpus(unsigned int max_cpus)
387 {
388         int cpu, err;
389         unsigned int ncores = num_possible_cpus();
390 
391         /*
392          * are we trying to boot more cores than exist?
393          */
394         if (max_cpus > ncores)
395                 max_cpus = ncores;
396 
397         /* Don't bother if we're effectively UP */
398         if (max_cpus <= 1)
399                 return;
400 
401         /*
402          * Initialise the present map (which describes the set of CPUs
403          * actually populated at the present time) and release the
404          * secondaries from the bootloader.
405          *
406          * Make sure we online at most (max_cpus - 1) additional CPUs.
407          */
408         max_cpus--;
409         for_each_possible_cpu(cpu) {
410                 if (max_cpus == 0)
411                         break;
412 
413                 if (cpu == smp_processor_id())
414                         continue;
415 
416                 if (!smp_enable_ops[cpu])
417                         continue;
418 
419                 err = smp_enable_ops[cpu]->prepare_cpu(cpu);
420                 if (err)
421                         continue;
422 
423                 set_cpu_present(cpu, true);
424                 max_cpus--;
425         }
426 }
427 
428 
429 void __init set_smp_cross_call(void (*fn)(const struct cpumask *, unsigned int))
430 {
431         smp_cross_call = fn;
432 }
433 
434 void arch_send_call_function_ipi_mask(const struct cpumask *mask)
435 {
436         smp_cross_call(mask, IPI_CALL_FUNC);
437 }
438 
439 void arch_send_call_function_single_ipi(int cpu)
440 {
441         smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC_SINGLE);
442 }
443 
444 static const char *ipi_types[NR_IPI] = {
445 #define S(x,s)  [x - IPI_RESCHEDULE] = s
446         S(IPI_RESCHEDULE, "Rescheduling interrupts"),
447         S(IPI_CALL_FUNC, "Function call interrupts"),
448         S(IPI_CALL_FUNC_SINGLE, "Single function call interrupts"),
449         S(IPI_CPU_STOP, "CPU stop interrupts"),
450 };
451 
452 void show_ipi_list(struct seq_file *p, int prec)
453 {
454         unsigned int cpu, i;
455 
456         for (i = 0; i < NR_IPI; i++) {
457                 seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i + IPI_RESCHEDULE,
458                            prec >= 4 ? " " : "");
459                 for_each_present_cpu(cpu)
460                         seq_printf(p, "%10u ",
461                                    __get_irq_stat(cpu, ipi_irqs[i]));
462                 seq_printf(p, "      %s\n", ipi_types[i]);
463         }
464 }
465 
466 u64 smp_irq_stat_cpu(unsigned int cpu)
467 {
468         u64 sum = 0;
469         int i;
470 
471         for (i = 0; i < NR_IPI; i++)
472                 sum += __get_irq_stat(cpu, ipi_irqs[i]);
473 
474         return sum;
475 }
476 
477 static DEFINE_RAW_SPINLOCK(stop_lock);
478 
479 /*
480  * ipi_cpu_stop - handle IPI from smp_send_stop()
481  */
482 static void ipi_cpu_stop(unsigned int cpu)
483 {
484         if (system_state == SYSTEM_BOOTING ||
485             system_state == SYSTEM_RUNNING) {
486                 raw_spin_lock(&stop_lock);
487                 pr_crit("CPU%u: stopping\n", cpu);
488                 dump_stack();
489                 raw_spin_unlock(&stop_lock);
490         }
491 
492         set_cpu_online(cpu, false);
493 
494         local_fiq_disable();
495         local_irq_disable();
496 
497         while (1)
498                 cpu_relax();
499 }
500 
501 /*
502  * Main handler for inter-processor interrupts
503  */
504 void handle_IPI(int ipinr, struct pt_regs *regs)
505 {
506         unsigned int cpu = smp_processor_id();
507         struct pt_regs *old_regs = set_irq_regs(regs);
508 
509         if (ipinr >= IPI_RESCHEDULE && ipinr < IPI_RESCHEDULE + NR_IPI)
510                 __inc_irq_stat(cpu, ipi_irqs[ipinr - IPI_RESCHEDULE]);
511 
512         switch (ipinr) {
513         case IPI_RESCHEDULE:
514                 scheduler_ipi();
515                 break;
516 
517         case IPI_CALL_FUNC:
518                 irq_enter();
519                 generic_smp_call_function_interrupt();
520                 irq_exit();
521                 break;
522 
523         case IPI_CALL_FUNC_SINGLE:
524                 irq_enter();
525                 generic_smp_call_function_single_interrupt();
526                 irq_exit();
527                 break;
528 
529         case IPI_CPU_STOP:
530                 irq_enter();
531                 ipi_cpu_stop(cpu);
532                 irq_exit();
533                 break;
534 
535         default:
536                 pr_crit("CPU%u: Unknown IPI message 0x%x\n", cpu, ipinr);
537                 break;
538         }
539         set_irq_regs(old_regs);
540 }
541 
542 void smp_send_reschedule(int cpu)
543 {
544         smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE);
545 }
546 
547 void smp_send_stop(void)
548 {
549         unsigned long timeout;
550 
551         if (num_online_cpus() > 1) {
552                 cpumask_t mask;
553 
554                 cpumask_copy(&mask, cpu_online_mask);
555                 cpu_clear(smp_processor_id(), mask);
556 
557                 smp_cross_call(&mask, IPI_CPU_STOP);
558         }
559 
560         /* Wait up to one second for other CPUs to stop */
561         timeout = USEC_PER_SEC;
562         while (num_online_cpus() > 1 && timeout--)
563                 udelay(1);
564 
565         if (num_online_cpus() > 1)
566                 pr_warning("SMP: failed to stop secondary CPUs\n");
567 }
568 
569 /*
570  * not supported here
571  */
572 int setup_profiling_timer(unsigned int multiplier)
573 {
574         return -EINVAL;
575 }
576 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp