~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/xtensa/kernel/smp.c

Version: ~ [ linux-5.3-rc5 ] ~ [ linux-5.2.9 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.67 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.139 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.189 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.189 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.72 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Xtensa SMP support functions.
  3  *
  4  * This file is subject to the terms and conditions of the GNU General Public
  5  * License.  See the file "COPYING" in the main directory of this archive
  6  * for more details.
  7  *
  8  * Copyright (C) 2008 - 2013 Tensilica Inc.
  9  *
 10  * Chris Zankel <chris@zankel.net>
 11  * Joe Taylor <joe@tensilica.com>
 12  * Pete Delaney <piet@tensilica.com
 13  */
 14 
 15 #include <linux/cpu.h>
 16 #include <linux/cpumask.h>
 17 #include <linux/delay.h>
 18 #include <linux/init.h>
 19 #include <linux/interrupt.h>
 20 #include <linux/irqdomain.h>
 21 #include <linux/irq.h>
 22 #include <linux/kdebug.h>
 23 #include <linux/module.h>
 24 #include <linux/reboot.h>
 25 #include <linux/seq_file.h>
 26 #include <linux/smp.h>
 27 #include <linux/thread_info.h>
 28 
 29 #include <asm/cacheflush.h>
 30 #include <asm/kdebug.h>
 31 #include <asm/mmu_context.h>
 32 #include <asm/mxregs.h>
 33 #include <asm/platform.h>
 34 #include <asm/tlbflush.h>
 35 #include <asm/traps.h>
 36 
 37 #ifdef CONFIG_SMP
 38 # if XCHAL_HAVE_S32C1I == 0
 39 #  error "The S32C1I option is required for SMP."
 40 # endif
 41 #endif
 42 
 43 static void system_invalidate_dcache_range(unsigned long start,
 44                 unsigned long size);
 45 static void system_flush_invalidate_dcache_range(unsigned long start,
 46                 unsigned long size);
 47 
 48 /* IPI (Inter Process Interrupt) */
 49 
 50 #define IPI_IRQ 0
 51 
 52 static irqreturn_t ipi_interrupt(int irq, void *dev_id);
 53 static struct irqaction ipi_irqaction = {
 54         .handler =      ipi_interrupt,
 55         .flags =        IRQF_PERCPU,
 56         .name =         "ipi",
 57 };
 58 
 59 void ipi_init(void)
 60 {
 61         unsigned irq = irq_create_mapping(NULL, IPI_IRQ);
 62         setup_irq(irq, &ipi_irqaction);
 63 }
 64 
 65 static inline unsigned int get_core_count(void)
 66 {
 67         /* Bits 18..21 of SYSCFGID contain the core count minus 1. */
 68         unsigned int syscfgid = get_er(SYSCFGID);
 69         return ((syscfgid >> 18) & 0xf) + 1;
 70 }
 71 
 72 static inline int get_core_id(void)
 73 {
 74         /* Bits 0...18 of SYSCFGID contain the core id  */
 75         unsigned int core_id = get_er(SYSCFGID);
 76         return core_id & 0x3fff;
 77 }
 78 
 79 void __init smp_prepare_cpus(unsigned int max_cpus)
 80 {
 81         unsigned i;
 82 
 83         for (i = 0; i < max_cpus; ++i)
 84                 set_cpu_present(i, true);
 85 }
 86 
 87 void __init smp_init_cpus(void)
 88 {
 89         unsigned i;
 90         unsigned int ncpus = get_core_count();
 91         unsigned int core_id = get_core_id();
 92 
 93         pr_info("%s: Core Count = %d\n", __func__, ncpus);
 94         pr_info("%s: Core Id = %d\n", __func__, core_id);
 95 
 96         for (i = 0; i < ncpus; ++i)
 97                 set_cpu_possible(i, true);
 98 }
 99 
100 void __init smp_prepare_boot_cpu(void)
101 {
102         unsigned int cpu = smp_processor_id();
103         BUG_ON(cpu != 0);
104         cpu_asid_cache(cpu) = ASID_USER_FIRST;
105 }
106 
107 void __init smp_cpus_done(unsigned int max_cpus)
108 {
109 }
110 
111 static int boot_secondary_processors = 1; /* Set with xt-gdb via .xt-gdb */
112 static DECLARE_COMPLETION(cpu_running);
113 
114 void secondary_start_kernel(void)
115 {
116         struct mm_struct *mm = &init_mm;
117         unsigned int cpu = smp_processor_id();
118 
119         init_mmu();
120 
121 #ifdef CONFIG_DEBUG_KERNEL
122         if (boot_secondary_processors == 0) {
123                 pr_debug("%s: boot_secondary_processors:%d; Hanging cpu:%d\n",
124                         __func__, boot_secondary_processors, cpu);
125                 for (;;)
126                         __asm__ __volatile__ ("waiti " __stringify(LOCKLEVEL));
127         }
128 
129         pr_debug("%s: boot_secondary_processors:%d; Booting cpu:%d\n",
130                 __func__, boot_secondary_processors, cpu);
131 #endif
132         /* Init EXCSAVE1 */
133 
134         secondary_trap_init();
135 
136         /* All kernel threads share the same mm context. */
137 
138         atomic_inc(&mm->mm_users);
139         atomic_inc(&mm->mm_count);
140         current->active_mm = mm;
141         cpumask_set_cpu(cpu, mm_cpumask(mm));
142         enter_lazy_tlb(mm, current);
143 
144         preempt_disable();
145         trace_hardirqs_off();
146 
147         calibrate_delay();
148 
149         notify_cpu_starting(cpu);
150 
151         secondary_init_irq();
152         local_timer_setup(cpu);
153 
154         set_cpu_online(cpu, true);
155 
156         local_irq_enable();
157 
158         complete(&cpu_running);
159 
160         cpu_startup_entry(CPUHP_ONLINE);
161 }
162 
163 static void mx_cpu_start(void *p)
164 {
165         unsigned cpu = (unsigned)p;
166         unsigned long run_stall_mask = get_er(MPSCORE);
167 
168         set_er(run_stall_mask & ~(1u << cpu), MPSCORE);
169         pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n",
170                         __func__, cpu, run_stall_mask, get_er(MPSCORE));
171 }
172 
173 static void mx_cpu_stop(void *p)
174 {
175         unsigned cpu = (unsigned)p;
176         unsigned long run_stall_mask = get_er(MPSCORE);
177 
178         set_er(run_stall_mask | (1u << cpu), MPSCORE);
179         pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n",
180                         __func__, cpu, run_stall_mask, get_er(MPSCORE));
181 }
182 
183 #ifdef CONFIG_HOTPLUG_CPU
184 unsigned long cpu_start_id __cacheline_aligned;
185 #endif
186 unsigned long cpu_start_ccount;
187 
188 static int boot_secondary(unsigned int cpu, struct task_struct *ts)
189 {
190         unsigned long timeout = jiffies + msecs_to_jiffies(1000);
191         unsigned long ccount;
192         int i;
193 
194 #ifdef CONFIG_HOTPLUG_CPU
195         cpu_start_id = cpu;
196         system_flush_invalidate_dcache_range(
197                         (unsigned long)&cpu_start_id, sizeof(cpu_start_id));
198 #endif
199         smp_call_function_single(0, mx_cpu_start, (void *)cpu, 1);
200 
201         for (i = 0; i < 2; ++i) {
202                 do
203                         ccount = get_ccount();
204                 while (!ccount);
205 
206                 cpu_start_ccount = ccount;
207 
208                 while (time_before(jiffies, timeout)) {
209                         mb();
210                         if (!cpu_start_ccount)
211                                 break;
212                 }
213 
214                 if (cpu_start_ccount) {
215                         smp_call_function_single(0, mx_cpu_stop,
216                                         (void *)cpu, 1);
217                         cpu_start_ccount = 0;
218                         return -EIO;
219                 }
220         }
221         return 0;
222 }
223 
224 int __cpu_up(unsigned int cpu, struct task_struct *idle)
225 {
226         int ret = 0;
227 
228         if (cpu_asid_cache(cpu) == 0)
229                 cpu_asid_cache(cpu) = ASID_USER_FIRST;
230 
231         start_info.stack = (unsigned long)task_pt_regs(idle);
232         wmb();
233 
234         pr_debug("%s: Calling wakeup_secondary(cpu:%d, idle:%p, sp: %08lx)\n",
235                         __func__, cpu, idle, start_info.stack);
236 
237         ret = boot_secondary(cpu, idle);
238         if (ret == 0) {
239                 wait_for_completion_timeout(&cpu_running,
240                                 msecs_to_jiffies(1000));
241                 if (!cpu_online(cpu))
242                         ret = -EIO;
243         }
244 
245         if (ret)
246                 pr_err("CPU %u failed to boot\n", cpu);
247 
248         return ret;
249 }
250 
251 #ifdef CONFIG_HOTPLUG_CPU
252 
253 /*
254  * __cpu_disable runs on the processor to be shutdown.
255  */
256 int __cpu_disable(void)
257 {
258         unsigned int cpu = smp_processor_id();
259 
260         /*
261          * Take this CPU offline.  Once we clear this, we can't return,
262          * and we must not schedule until we're ready to give up the cpu.
263          */
264         set_cpu_online(cpu, false);
265 
266         /*
267          * OK - migrate IRQs away from this CPU
268          */
269         migrate_irqs();
270 
271         /*
272          * Flush user cache and TLB mappings, and then remove this CPU
273          * from the vm mask set of all processes.
274          */
275         local_flush_cache_all();
276         local_flush_tlb_all();
277         invalidate_page_directory();
278 
279         clear_tasks_mm_cpumask(cpu);
280 
281         return 0;
282 }
283 
284 static void platform_cpu_kill(unsigned int cpu)
285 {
286         smp_call_function_single(0, mx_cpu_stop, (void *)cpu, true);
287 }
288 
289 /*
290  * called on the thread which is asking for a CPU to be shutdown -
291  * waits until shutdown has completed, or it is timed out.
292  */
293 void __cpu_die(unsigned int cpu)
294 {
295         unsigned long timeout = jiffies + msecs_to_jiffies(1000);
296         while (time_before(jiffies, timeout)) {
297                 system_invalidate_dcache_range((unsigned long)&cpu_start_id,
298                                 sizeof(cpu_start_id));
299                 if (cpu_start_id == -cpu) {
300                         platform_cpu_kill(cpu);
301                         return;
302                 }
303         }
304         pr_err("CPU%u: unable to kill\n", cpu);
305 }
306 
307 void arch_cpu_idle_dead(void)
308 {
309         cpu_die();
310 }
311 /*
312  * Called from the idle thread for the CPU which has been shutdown.
313  *
314  * Note that we disable IRQs here, but do not re-enable them
315  * before returning to the caller. This is also the behaviour
316  * of the other hotplug-cpu capable cores, so presumably coming
317  * out of idle fixes this.
318  */
319 void __ref cpu_die(void)
320 {
321         idle_task_exit();
322         local_irq_disable();
323         __asm__ __volatile__(
324                         "       movi    a2, cpu_restart\n"
325                         "       jx      a2\n");
326 }
327 
328 #endif /* CONFIG_HOTPLUG_CPU */
329 
330 enum ipi_msg_type {
331         IPI_RESCHEDULE = 0,
332         IPI_CALL_FUNC,
333         IPI_CPU_STOP,
334         IPI_MAX
335 };
336 
337 static const struct {
338         const char *short_text;
339         const char *long_text;
340 } ipi_text[] = {
341         { .short_text = "RES", .long_text = "Rescheduling interrupts" },
342         { .short_text = "CAL", .long_text = "Function call interrupts" },
343         { .short_text = "DIE", .long_text = "CPU shutdown interrupts" },
344 };
345 
346 struct ipi_data {
347         unsigned long ipi_count[IPI_MAX];
348 };
349 
350 static DEFINE_PER_CPU(struct ipi_data, ipi_data);
351 
352 static void send_ipi_message(const struct cpumask *callmask,
353                 enum ipi_msg_type msg_id)
354 {
355         int index;
356         unsigned long mask = 0;
357 
358         for_each_cpu(index, callmask)
359                 if (index != smp_processor_id())
360                         mask |= 1 << index;
361 
362         set_er(mask, MIPISET(msg_id));
363 }
364 
365 void arch_send_call_function_ipi_mask(const struct cpumask *mask)
366 {
367         send_ipi_message(mask, IPI_CALL_FUNC);
368 }
369 
370 void arch_send_call_function_single_ipi(int cpu)
371 {
372         send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC);
373 }
374 
375 void smp_send_reschedule(int cpu)
376 {
377         send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE);
378 }
379 
380 void smp_send_stop(void)
381 {
382         struct cpumask targets;
383 
384         cpumask_copy(&targets, cpu_online_mask);
385         cpumask_clear_cpu(smp_processor_id(), &targets);
386         send_ipi_message(&targets, IPI_CPU_STOP);
387 }
388 
389 static void ipi_cpu_stop(unsigned int cpu)
390 {
391         set_cpu_online(cpu, false);
392         machine_halt();
393 }
394 
395 irqreturn_t ipi_interrupt(int irq, void *dev_id)
396 {
397         unsigned int cpu = smp_processor_id();
398         struct ipi_data *ipi = &per_cpu(ipi_data, cpu);
399         unsigned int msg;
400         unsigned i;
401 
402         msg = get_er(MIPICAUSE(cpu));
403         for (i = 0; i < IPI_MAX; i++)
404                 if (msg & (1 << i)) {
405                         set_er(1 << i, MIPICAUSE(cpu));
406                         ++ipi->ipi_count[i];
407                 }
408 
409         if (msg & (1 << IPI_RESCHEDULE))
410                 scheduler_ipi();
411         if (msg & (1 << IPI_CALL_FUNC))
412                 generic_smp_call_function_interrupt();
413         if (msg & (1 << IPI_CPU_STOP))
414                 ipi_cpu_stop(cpu);
415 
416         return IRQ_HANDLED;
417 }
418 
419 void show_ipi_list(struct seq_file *p, int prec)
420 {
421         unsigned int cpu;
422         unsigned i;
423 
424         for (i = 0; i < IPI_MAX; ++i) {
425                 seq_printf(p, "%*s:", prec, ipi_text[i].short_text);
426                 for_each_online_cpu(cpu)
427                         seq_printf(p, " %10lu",
428                                         per_cpu(ipi_data, cpu).ipi_count[i]);
429                 seq_printf(p, "   %s\n", ipi_text[i].long_text);
430         }
431 }
432 
433 int setup_profiling_timer(unsigned int multiplier)
434 {
435         pr_debug("setup_profiling_timer %d\n", multiplier);
436         return 0;
437 }
438 
439 /* TLB flush functions */
440 
441 struct flush_data {
442         struct vm_area_struct *vma;
443         unsigned long addr1;
444         unsigned long addr2;
445 };
446 
447 static void ipi_flush_tlb_all(void *arg)
448 {
449         local_flush_tlb_all();
450 }
451 
452 void flush_tlb_all(void)
453 {
454         on_each_cpu(ipi_flush_tlb_all, NULL, 1);
455 }
456 
457 static void ipi_flush_tlb_mm(void *arg)
458 {
459         local_flush_tlb_mm(arg);
460 }
461 
462 void flush_tlb_mm(struct mm_struct *mm)
463 {
464         on_each_cpu(ipi_flush_tlb_mm, mm, 1);
465 }
466 
467 static void ipi_flush_tlb_page(void *arg)
468 {
469         struct flush_data *fd = arg;
470         local_flush_tlb_page(fd->vma, fd->addr1);
471 }
472 
473 void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
474 {
475         struct flush_data fd = {
476                 .vma = vma,
477                 .addr1 = addr,
478         };
479         on_each_cpu(ipi_flush_tlb_page, &fd, 1);
480 }
481 
482 static void ipi_flush_tlb_range(void *arg)
483 {
484         struct flush_data *fd = arg;
485         local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2);
486 }
487 
488 void flush_tlb_range(struct vm_area_struct *vma,
489                      unsigned long start, unsigned long end)
490 {
491         struct flush_data fd = {
492                 .vma = vma,
493                 .addr1 = start,
494                 .addr2 = end,
495         };
496         on_each_cpu(ipi_flush_tlb_range, &fd, 1);
497 }
498 
499 static void ipi_flush_tlb_kernel_range(void *arg)
500 {
501         struct flush_data *fd = arg;
502         local_flush_tlb_kernel_range(fd->addr1, fd->addr2);
503 }
504 
505 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
506 {
507         struct flush_data fd = {
508                 .addr1 = start,
509                 .addr2 = end,
510         };
511         on_each_cpu(ipi_flush_tlb_kernel_range, &fd, 1);
512 }
513 
514 /* Cache flush functions */
515 
516 static void ipi_flush_cache_all(void *arg)
517 {
518         local_flush_cache_all();
519 }
520 
521 void flush_cache_all(void)
522 {
523         on_each_cpu(ipi_flush_cache_all, NULL, 1);
524 }
525 
526 static void ipi_flush_cache_page(void *arg)
527 {
528         struct flush_data *fd = arg;
529         local_flush_cache_page(fd->vma, fd->addr1, fd->addr2);
530 }
531 
532 void flush_cache_page(struct vm_area_struct *vma,
533                      unsigned long address, unsigned long pfn)
534 {
535         struct flush_data fd = {
536                 .vma = vma,
537                 .addr1 = address,
538                 .addr2 = pfn,
539         };
540         on_each_cpu(ipi_flush_cache_page, &fd, 1);
541 }
542 
543 static void ipi_flush_cache_range(void *arg)
544 {
545         struct flush_data *fd = arg;
546         local_flush_cache_range(fd->vma, fd->addr1, fd->addr2);
547 }
548 
549 void flush_cache_range(struct vm_area_struct *vma,
550                      unsigned long start, unsigned long end)
551 {
552         struct flush_data fd = {
553                 .vma = vma,
554                 .addr1 = start,
555                 .addr2 = end,
556         };
557         on_each_cpu(ipi_flush_cache_range, &fd, 1);
558 }
559 
560 static void ipi_flush_icache_range(void *arg)
561 {
562         struct flush_data *fd = arg;
563         local_flush_icache_range(fd->addr1, fd->addr2);
564 }
565 
566 void flush_icache_range(unsigned long start, unsigned long end)
567 {
568         struct flush_data fd = {
569                 .addr1 = start,
570                 .addr2 = end,
571         };
572         on_each_cpu(ipi_flush_icache_range, &fd, 1);
573 }
574 EXPORT_SYMBOL(flush_icache_range);
575 
576 /* ------------------------------------------------------------------------- */
577 
578 static void ipi_invalidate_dcache_range(void *arg)
579 {
580         struct flush_data *fd = arg;
581         __invalidate_dcache_range(fd->addr1, fd->addr2);
582 }
583 
584 static void system_invalidate_dcache_range(unsigned long start,
585                 unsigned long size)
586 {
587         struct flush_data fd = {
588                 .addr1 = start,
589                 .addr2 = size,
590         };
591         on_each_cpu(ipi_invalidate_dcache_range, &fd, 1);
592 }
593 
594 static void ipi_flush_invalidate_dcache_range(void *arg)
595 {
596         struct flush_data *fd = arg;
597         __flush_invalidate_dcache_range(fd->addr1, fd->addr2);
598 }
599 
600 static void system_flush_invalidate_dcache_range(unsigned long start,
601                 unsigned long size)
602 {
603         struct flush_data fd = {
604                 .addr1 = start,
605                 .addr2 = size,
606         };
607         on_each_cpu(ipi_flush_invalidate_dcache_range, &fd, 1);
608 }
609 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp