~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/xtensa/kernel/smp.c

Version: ~ [ linux-5.4-rc3 ] ~ [ linux-5.3.6 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.79 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.149 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.196 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.196 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.75 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Xtensa SMP support functions.
  3  *
  4  * This file is subject to the terms and conditions of the GNU General Public
  5  * License.  See the file "COPYING" in the main directory of this archive
  6  * for more details.
  7  *
  8  * Copyright (C) 2008 - 2013 Tensilica Inc.
  9  *
 10  * Chris Zankel <chris@zankel.net>
 11  * Joe Taylor <joe@tensilica.com>
 12  * Pete Delaney <piet@tensilica.com
 13  */
 14 
 15 #include <linux/cpu.h>
 16 #include <linux/cpumask.h>
 17 #include <linux/delay.h>
 18 #include <linux/init.h>
 19 #include <linux/interrupt.h>
 20 #include <linux/irqdomain.h>
 21 #include <linux/irq.h>
 22 #include <linux/kdebug.h>
 23 #include <linux/module.h>
 24 #include <linux/sched/mm.h>
 25 #include <linux/sched/hotplug.h>
 26 #include <linux/sched/task_stack.h>
 27 #include <linux/reboot.h>
 28 #include <linux/seq_file.h>
 29 #include <linux/smp.h>
 30 #include <linux/thread_info.h>
 31 
 32 #include <asm/cacheflush.h>
 33 #include <asm/kdebug.h>
 34 #include <asm/mmu_context.h>
 35 #include <asm/mxregs.h>
 36 #include <asm/platform.h>
 37 #include <asm/tlbflush.h>
 38 #include <asm/traps.h>
 39 
 40 #ifdef CONFIG_SMP
 41 # if XCHAL_HAVE_S32C1I == 0
 42 #  error "The S32C1I option is required for SMP."
 43 # endif
 44 #endif
 45 
 46 static void system_invalidate_dcache_range(unsigned long start,
 47                 unsigned long size);
 48 static void system_flush_invalidate_dcache_range(unsigned long start,
 49                 unsigned long size);
 50 
 51 /* IPI (Inter Process Interrupt) */
 52 
 53 #define IPI_IRQ 0
 54 
 55 static irqreturn_t ipi_interrupt(int irq, void *dev_id);
 56 static struct irqaction ipi_irqaction = {
 57         .handler =      ipi_interrupt,
 58         .flags =        IRQF_PERCPU,
 59         .name =         "ipi",
 60 };
 61 
 62 void ipi_init(void)
 63 {
 64         unsigned irq = irq_create_mapping(NULL, IPI_IRQ);
 65         setup_irq(irq, &ipi_irqaction);
 66 }
 67 
 68 static inline unsigned int get_core_count(void)
 69 {
 70         /* Bits 18..21 of SYSCFGID contain the core count minus 1. */
 71         unsigned int syscfgid = get_er(SYSCFGID);
 72         return ((syscfgid >> 18) & 0xf) + 1;
 73 }
 74 
 75 static inline int get_core_id(void)
 76 {
 77         /* Bits 0...18 of SYSCFGID contain the core id  */
 78         unsigned int core_id = get_er(SYSCFGID);
 79         return core_id & 0x3fff;
 80 }
 81 
 82 void __init smp_prepare_cpus(unsigned int max_cpus)
 83 {
 84         unsigned i;
 85 
 86         for (i = 0; i < max_cpus; ++i)
 87                 set_cpu_present(i, true);
 88 }
 89 
 90 void __init smp_init_cpus(void)
 91 {
 92         unsigned i;
 93         unsigned int ncpus = get_core_count();
 94         unsigned int core_id = get_core_id();
 95 
 96         pr_info("%s: Core Count = %d\n", __func__, ncpus);
 97         pr_info("%s: Core Id = %d\n", __func__, core_id);
 98 
 99         for (i = 0; i < ncpus; ++i)
100                 set_cpu_possible(i, true);
101 }
102 
103 void __init smp_prepare_boot_cpu(void)
104 {
105         unsigned int cpu = smp_processor_id();
106         BUG_ON(cpu != 0);
107         cpu_asid_cache(cpu) = ASID_USER_FIRST;
108 }
109 
110 void __init smp_cpus_done(unsigned int max_cpus)
111 {
112 }
113 
114 static int boot_secondary_processors = 1; /* Set with xt-gdb via .xt-gdb */
115 static DECLARE_COMPLETION(cpu_running);
116 
117 void secondary_start_kernel(void)
118 {
119         struct mm_struct *mm = &init_mm;
120         unsigned int cpu = smp_processor_id();
121 
122         init_mmu();
123 
124 #ifdef CONFIG_DEBUG_KERNEL
125         if (boot_secondary_processors == 0) {
126                 pr_debug("%s: boot_secondary_processors:%d; Hanging cpu:%d\n",
127                         __func__, boot_secondary_processors, cpu);
128                 for (;;)
129                         __asm__ __volatile__ ("waiti " __stringify(LOCKLEVEL));
130         }
131 
132         pr_debug("%s: boot_secondary_processors:%d; Booting cpu:%d\n",
133                 __func__, boot_secondary_processors, cpu);
134 #endif
135         /* Init EXCSAVE1 */
136 
137         secondary_trap_init();
138 
139         /* All kernel threads share the same mm context. */
140 
141         mmget(mm);
142         mmgrab(mm);
143         current->active_mm = mm;
144         cpumask_set_cpu(cpu, mm_cpumask(mm));
145         enter_lazy_tlb(mm, current);
146 
147         preempt_disable();
148         trace_hardirqs_off();
149 
150         calibrate_delay();
151 
152         notify_cpu_starting(cpu);
153 
154         secondary_init_irq();
155         local_timer_setup(cpu);
156 
157         set_cpu_online(cpu, true);
158 
159         local_irq_enable();
160 
161         complete(&cpu_running);
162 
163         cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
164 }
165 
166 static void mx_cpu_start(void *p)
167 {
168         unsigned cpu = (unsigned)p;
169         unsigned long run_stall_mask = get_er(MPSCORE);
170 
171         set_er(run_stall_mask & ~(1u << cpu), MPSCORE);
172         pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n",
173                         __func__, cpu, run_stall_mask, get_er(MPSCORE));
174 }
175 
176 static void mx_cpu_stop(void *p)
177 {
178         unsigned cpu = (unsigned)p;
179         unsigned long run_stall_mask = get_er(MPSCORE);
180 
181         set_er(run_stall_mask | (1u << cpu), MPSCORE);
182         pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n",
183                         __func__, cpu, run_stall_mask, get_er(MPSCORE));
184 }
185 
186 #ifdef CONFIG_HOTPLUG_CPU
187 unsigned long cpu_start_id __cacheline_aligned;
188 #endif
189 unsigned long cpu_start_ccount;
190 
191 static int boot_secondary(unsigned int cpu, struct task_struct *ts)
192 {
193         unsigned long timeout = jiffies + msecs_to_jiffies(1000);
194         unsigned long ccount;
195         int i;
196 
197 #ifdef CONFIG_HOTPLUG_CPU
198         cpu_start_id = cpu;
199         system_flush_invalidate_dcache_range(
200                         (unsigned long)&cpu_start_id, sizeof(cpu_start_id));
201 #endif
202         smp_call_function_single(0, mx_cpu_start, (void *)cpu, 1);
203 
204         for (i = 0; i < 2; ++i) {
205                 do
206                         ccount = get_ccount();
207                 while (!ccount);
208 
209                 cpu_start_ccount = ccount;
210 
211                 while (time_before(jiffies, timeout)) {
212                         mb();
213                         if (!cpu_start_ccount)
214                                 break;
215                 }
216 
217                 if (cpu_start_ccount) {
218                         smp_call_function_single(0, mx_cpu_stop,
219                                         (void *)cpu, 1);
220                         cpu_start_ccount = 0;
221                         return -EIO;
222                 }
223         }
224         return 0;
225 }
226 
227 int __cpu_up(unsigned int cpu, struct task_struct *idle)
228 {
229         int ret = 0;
230 
231         if (cpu_asid_cache(cpu) == 0)
232                 cpu_asid_cache(cpu) = ASID_USER_FIRST;
233 
234         start_info.stack = (unsigned long)task_pt_regs(idle);
235         wmb();
236 
237         pr_debug("%s: Calling wakeup_secondary(cpu:%d, idle:%p, sp: %08lx)\n",
238                         __func__, cpu, idle, start_info.stack);
239 
240         ret = boot_secondary(cpu, idle);
241         if (ret == 0) {
242                 wait_for_completion_timeout(&cpu_running,
243                                 msecs_to_jiffies(1000));
244                 if (!cpu_online(cpu))
245                         ret = -EIO;
246         }
247 
248         if (ret)
249                 pr_err("CPU %u failed to boot\n", cpu);
250 
251         return ret;
252 }
253 
254 #ifdef CONFIG_HOTPLUG_CPU
255 
256 /*
257  * __cpu_disable runs on the processor to be shutdown.
258  */
259 int __cpu_disable(void)
260 {
261         unsigned int cpu = smp_processor_id();
262 
263         /*
264          * Take this CPU offline.  Once we clear this, we can't return,
265          * and we must not schedule until we're ready to give up the cpu.
266          */
267         set_cpu_online(cpu, false);
268 
269         /*
270          * OK - migrate IRQs away from this CPU
271          */
272         migrate_irqs();
273 
274         /*
275          * Flush user cache and TLB mappings, and then remove this CPU
276          * from the vm mask set of all processes.
277          */
278         local_flush_cache_all();
279         local_flush_tlb_all();
280         invalidate_page_directory();
281 
282         clear_tasks_mm_cpumask(cpu);
283 
284         return 0;
285 }
286 
287 static void platform_cpu_kill(unsigned int cpu)
288 {
289         smp_call_function_single(0, mx_cpu_stop, (void *)cpu, true);
290 }
291 
292 /*
293  * called on the thread which is asking for a CPU to be shutdown -
294  * waits until shutdown has completed, or it is timed out.
295  */
296 void __cpu_die(unsigned int cpu)
297 {
298         unsigned long timeout = jiffies + msecs_to_jiffies(1000);
299         while (time_before(jiffies, timeout)) {
300                 system_invalidate_dcache_range((unsigned long)&cpu_start_id,
301                                 sizeof(cpu_start_id));
302                 if (cpu_start_id == -cpu) {
303                         platform_cpu_kill(cpu);
304                         return;
305                 }
306         }
307         pr_err("CPU%u: unable to kill\n", cpu);
308 }
309 
310 void arch_cpu_idle_dead(void)
311 {
312         cpu_die();
313 }
314 /*
315  * Called from the idle thread for the CPU which has been shutdown.
316  *
317  * Note that we disable IRQs here, but do not re-enable them
318  * before returning to the caller. This is also the behaviour
319  * of the other hotplug-cpu capable cores, so presumably coming
320  * out of idle fixes this.
321  */
322 void __ref cpu_die(void)
323 {
324         idle_task_exit();
325         local_irq_disable();
326         __asm__ __volatile__(
327                         "       movi    a2, cpu_restart\n"
328                         "       jx      a2\n");
329 }
330 
331 #endif /* CONFIG_HOTPLUG_CPU */
332 
333 enum ipi_msg_type {
334         IPI_RESCHEDULE = 0,
335         IPI_CALL_FUNC,
336         IPI_CPU_STOP,
337         IPI_MAX
338 };
339 
340 static const struct {
341         const char *short_text;
342         const char *long_text;
343 } ipi_text[] = {
344         { .short_text = "RES", .long_text = "Rescheduling interrupts" },
345         { .short_text = "CAL", .long_text = "Function call interrupts" },
346         { .short_text = "DIE", .long_text = "CPU shutdown interrupts" },
347 };
348 
349 struct ipi_data {
350         unsigned long ipi_count[IPI_MAX];
351 };
352 
353 static DEFINE_PER_CPU(struct ipi_data, ipi_data);
354 
355 static void send_ipi_message(const struct cpumask *callmask,
356                 enum ipi_msg_type msg_id)
357 {
358         int index;
359         unsigned long mask = 0;
360 
361         for_each_cpu(index, callmask)
362                 if (index != smp_processor_id())
363                         mask |= 1 << index;
364 
365         set_er(mask, MIPISET(msg_id));
366 }
367 
368 void arch_send_call_function_ipi_mask(const struct cpumask *mask)
369 {
370         send_ipi_message(mask, IPI_CALL_FUNC);
371 }
372 
373 void arch_send_call_function_single_ipi(int cpu)
374 {
375         send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC);
376 }
377 
378 void smp_send_reschedule(int cpu)
379 {
380         send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE);
381 }
382 
383 void smp_send_stop(void)
384 {
385         struct cpumask targets;
386 
387         cpumask_copy(&targets, cpu_online_mask);
388         cpumask_clear_cpu(smp_processor_id(), &targets);
389         send_ipi_message(&targets, IPI_CPU_STOP);
390 }
391 
392 static void ipi_cpu_stop(unsigned int cpu)
393 {
394         set_cpu_online(cpu, false);
395         machine_halt();
396 }
397 
398 irqreturn_t ipi_interrupt(int irq, void *dev_id)
399 {
400         unsigned int cpu = smp_processor_id();
401         struct ipi_data *ipi = &per_cpu(ipi_data, cpu);
402         unsigned int msg;
403         unsigned i;
404 
405         msg = get_er(MIPICAUSE(cpu));
406         for (i = 0; i < IPI_MAX; i++)
407                 if (msg & (1 << i)) {
408                         set_er(1 << i, MIPICAUSE(cpu));
409                         ++ipi->ipi_count[i];
410                 }
411 
412         if (msg & (1 << IPI_RESCHEDULE))
413                 scheduler_ipi();
414         if (msg & (1 << IPI_CALL_FUNC))
415                 generic_smp_call_function_interrupt();
416         if (msg & (1 << IPI_CPU_STOP))
417                 ipi_cpu_stop(cpu);
418 
419         return IRQ_HANDLED;
420 }
421 
422 void show_ipi_list(struct seq_file *p, int prec)
423 {
424         unsigned int cpu;
425         unsigned i;
426 
427         for (i = 0; i < IPI_MAX; ++i) {
428                 seq_printf(p, "%*s:", prec, ipi_text[i].short_text);
429                 for_each_online_cpu(cpu)
430                         seq_printf(p, " %10lu",
431                                         per_cpu(ipi_data, cpu).ipi_count[i]);
432                 seq_printf(p, "   %s\n", ipi_text[i].long_text);
433         }
434 }
435 
436 int setup_profiling_timer(unsigned int multiplier)
437 {
438         pr_debug("setup_profiling_timer %d\n", multiplier);
439         return 0;
440 }
441 
442 /* TLB flush functions */
443 
444 struct flush_data {
445         struct vm_area_struct *vma;
446         unsigned long addr1;
447         unsigned long addr2;
448 };
449 
450 static void ipi_flush_tlb_all(void *arg)
451 {
452         local_flush_tlb_all();
453 }
454 
455 void flush_tlb_all(void)
456 {
457         on_each_cpu(ipi_flush_tlb_all, NULL, 1);
458 }
459 
460 static void ipi_flush_tlb_mm(void *arg)
461 {
462         local_flush_tlb_mm(arg);
463 }
464 
465 void flush_tlb_mm(struct mm_struct *mm)
466 {
467         on_each_cpu(ipi_flush_tlb_mm, mm, 1);
468 }
469 
470 static void ipi_flush_tlb_page(void *arg)
471 {
472         struct flush_data *fd = arg;
473         local_flush_tlb_page(fd->vma, fd->addr1);
474 }
475 
476 void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
477 {
478         struct flush_data fd = {
479                 .vma = vma,
480                 .addr1 = addr,
481         };
482         on_each_cpu(ipi_flush_tlb_page, &fd, 1);
483 }
484 
485 static void ipi_flush_tlb_range(void *arg)
486 {
487         struct flush_data *fd = arg;
488         local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2);
489 }
490 
491 void flush_tlb_range(struct vm_area_struct *vma,
492                      unsigned long start, unsigned long end)
493 {
494         struct flush_data fd = {
495                 .vma = vma,
496                 .addr1 = start,
497                 .addr2 = end,
498         };
499         on_each_cpu(ipi_flush_tlb_range, &fd, 1);
500 }
501 
502 static void ipi_flush_tlb_kernel_range(void *arg)
503 {
504         struct flush_data *fd = arg;
505         local_flush_tlb_kernel_range(fd->addr1, fd->addr2);
506 }
507 
508 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
509 {
510         struct flush_data fd = {
511                 .addr1 = start,
512                 .addr2 = end,
513         };
514         on_each_cpu(ipi_flush_tlb_kernel_range, &fd, 1);
515 }
516 
517 /* Cache flush functions */
518 
519 static void ipi_flush_cache_all(void *arg)
520 {
521         local_flush_cache_all();
522 }
523 
524 void flush_cache_all(void)
525 {
526         on_each_cpu(ipi_flush_cache_all, NULL, 1);
527 }
528 
529 static void ipi_flush_cache_page(void *arg)
530 {
531         struct flush_data *fd = arg;
532         local_flush_cache_page(fd->vma, fd->addr1, fd->addr2);
533 }
534 
535 void flush_cache_page(struct vm_area_struct *vma,
536                      unsigned long address, unsigned long pfn)
537 {
538         struct flush_data fd = {
539                 .vma = vma,
540                 .addr1 = address,
541                 .addr2 = pfn,
542         };
543         on_each_cpu(ipi_flush_cache_page, &fd, 1);
544 }
545 
546 static void ipi_flush_cache_range(void *arg)
547 {
548         struct flush_data *fd = arg;
549         local_flush_cache_range(fd->vma, fd->addr1, fd->addr2);
550 }
551 
552 void flush_cache_range(struct vm_area_struct *vma,
553                      unsigned long start, unsigned long end)
554 {
555         struct flush_data fd = {
556                 .vma = vma,
557                 .addr1 = start,
558                 .addr2 = end,
559         };
560         on_each_cpu(ipi_flush_cache_range, &fd, 1);
561 }
562 
563 static void ipi_flush_icache_range(void *arg)
564 {
565         struct flush_data *fd = arg;
566         local_flush_icache_range(fd->addr1, fd->addr2);
567 }
568 
569 void flush_icache_range(unsigned long start, unsigned long end)
570 {
571         struct flush_data fd = {
572                 .addr1 = start,
573                 .addr2 = end,
574         };
575         on_each_cpu(ipi_flush_icache_range, &fd, 1);
576 }
577 EXPORT_SYMBOL(flush_icache_range);
578 
579 /* ------------------------------------------------------------------------- */
580 
581 static void ipi_invalidate_dcache_range(void *arg)
582 {
583         struct flush_data *fd = arg;
584         __invalidate_dcache_range(fd->addr1, fd->addr2);
585 }
586 
587 static void system_invalidate_dcache_range(unsigned long start,
588                 unsigned long size)
589 {
590         struct flush_data fd = {
591                 .addr1 = start,
592                 .addr2 = size,
593         };
594         on_each_cpu(ipi_invalidate_dcache_range, &fd, 1);
595 }
596 
597 static void ipi_flush_invalidate_dcache_range(void *arg)
598 {
599         struct flush_data *fd = arg;
600         __flush_invalidate_dcache_range(fd->addr1, fd->addr2);
601 }
602 
603 static void system_flush_invalidate_dcache_range(unsigned long start,
604                 unsigned long size)
605 {
606         struct flush_data fd = {
607                 .addr1 = start,
608                 .addr2 = size,
609         };
610         on_each_cpu(ipi_flush_invalidate_dcache_range, &fd, 1);
611 }
612 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp