~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/kernel/process.c

Version: ~ [ linux-5.14-rc3 ] ~ [ linux-5.13.5 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.53 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.135 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.198 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.240 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.276 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.276 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  2 
  3 #include <linux/errno.h>
  4 #include <linux/kernel.h>
  5 #include <linux/mm.h>
  6 #include <linux/smp.h>
  7 #include <linux/prctl.h>
  8 #include <linux/slab.h>
  9 #include <linux/sched.h>
 10 #include <linux/module.h>
 11 #include <linux/pm.h>
 12 #include <linux/clockchips.h>
 13 #include <linux/random.h>
 14 #include <linux/user-return-notifier.h>
 15 #include <linux/dmi.h>
 16 #include <linux/utsname.h>
 17 #include <linux/stackprotector.h>
 18 #include <linux/tick.h>
 19 #include <linux/cpuidle.h>
 20 #include <trace/events/power.h>
 21 #include <linux/hw_breakpoint.h>
 22 #include <asm/cpu.h>
 23 #include <asm/apic.h>
 24 #include <asm/syscalls.h>
 25 #include <asm/idle.h>
 26 #include <asm/uaccess.h>
 27 #include <asm/mwait.h>
 28 #include <asm/i387.h>
 29 #include <asm/fpu-internal.h>
 30 #include <asm/debugreg.h>
 31 #include <asm/nmi.h>
 32 #include <asm/tlbflush.h>
 33 
 34 /*
 35  * per-CPU TSS segments. Threads are completely 'soft' on Linux,
 36  * no more per-task TSS's. The TSS size is kept cacheline-aligned
 37  * so they are allowed to end up in the .data..cacheline_aligned
 38  * section. Since TSS's are completely CPU-local, we want them
 39  * on exact cacheline boundaries, to eliminate cacheline ping-pong.
 40  */
 41 __visible DEFINE_PER_CPU_SHARED_ALIGNED(struct tss_struct, init_tss) = INIT_TSS;
 42 
 43 #ifdef CONFIG_X86_64
 44 static DEFINE_PER_CPU(unsigned char, is_idle);
 45 static ATOMIC_NOTIFIER_HEAD(idle_notifier);
 46 
 47 void idle_notifier_register(struct notifier_block *n)
 48 {
 49         atomic_notifier_chain_register(&idle_notifier, n);
 50 }
 51 EXPORT_SYMBOL_GPL(idle_notifier_register);
 52 
 53 void idle_notifier_unregister(struct notifier_block *n)
 54 {
 55         atomic_notifier_chain_unregister(&idle_notifier, n);
 56 }
 57 EXPORT_SYMBOL_GPL(idle_notifier_unregister);
 58 #endif
 59 
 60 struct kmem_cache *task_xstate_cachep;
 61 EXPORT_SYMBOL_GPL(task_xstate_cachep);
 62 
 63 /*
 64  * this gets called so that we can store lazy state into memory and copy the
 65  * current task into the new thread.
 66  */
 67 int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
 68 {
 69         *dst = *src;
 70 
 71         dst->thread.fpu_counter = 0;
 72         dst->thread.fpu.has_fpu = 0;
 73         dst->thread.fpu.last_cpu = ~0;
 74         dst->thread.fpu.state = NULL;
 75         if (tsk_used_math(src)) {
 76                 int err = fpu_alloc(&dst->thread.fpu);
 77                 if (err)
 78                         return err;
 79                 fpu_copy(dst, src);
 80         }
 81         return 0;
 82 }
 83 
 84 void free_thread_xstate(struct task_struct *tsk)
 85 {
 86         fpu_free(&tsk->thread.fpu);
 87 }
 88 
 89 void arch_release_task_struct(struct task_struct *tsk)
 90 {
 91         free_thread_xstate(tsk);
 92 }
 93 
 94 void arch_task_cache_init(void)
 95 {
 96         task_xstate_cachep =
 97                 kmem_cache_create("task_xstate", xstate_size,
 98                                   __alignof__(union thread_xstate),
 99                                   SLAB_PANIC | SLAB_NOTRACK, NULL);
100         setup_xstate_comp();
101 }
102 
103 /*
104  * Free current thread data structures etc..
105  */
106 void exit_thread(void)
107 {
108         struct task_struct *me = current;
109         struct thread_struct *t = &me->thread;
110         unsigned long *bp = t->io_bitmap_ptr;
111 
112         if (bp) {
113                 struct tss_struct *tss = &per_cpu(init_tss, get_cpu());
114 
115                 t->io_bitmap_ptr = NULL;
116                 clear_thread_flag(TIF_IO_BITMAP);
117                 /*
118                  * Careful, clear this in the TSS too:
119                  */
120                 memset(tss->io_bitmap, 0xff, t->io_bitmap_max);
121                 t->io_bitmap_max = 0;
122                 put_cpu();
123                 kfree(bp);
124         }
125 
126         drop_fpu(me);
127 }
128 
129 void flush_thread(void)
130 {
131         struct task_struct *tsk = current;
132 
133         flush_ptrace_hw_breakpoint(tsk);
134         memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
135         drop_init_fpu(tsk);
136         /*
137          * Free the FPU state for non xsave platforms. They get reallocated
138          * lazily at the first use.
139          */
140         if (!use_eager_fpu())
141                 free_thread_xstate(tsk);
142 }
143 
144 static void hard_disable_TSC(void)
145 {
146         cr4_set_bits(X86_CR4_TSD);
147 }
148 
149 void disable_TSC(void)
150 {
151         preempt_disable();
152         if (!test_and_set_thread_flag(TIF_NOTSC))
153                 /*
154                  * Must flip the CPU state synchronously with
155                  * TIF_NOTSC in the current running context.
156                  */
157                 hard_disable_TSC();
158         preempt_enable();
159 }
160 
161 static void hard_enable_TSC(void)
162 {
163         cr4_clear_bits(X86_CR4_TSD);
164 }
165 
166 static void enable_TSC(void)
167 {
168         preempt_disable();
169         if (test_and_clear_thread_flag(TIF_NOTSC))
170                 /*
171                  * Must flip the CPU state synchronously with
172                  * TIF_NOTSC in the current running context.
173                  */
174                 hard_enable_TSC();
175         preempt_enable();
176 }
177 
178 int get_tsc_mode(unsigned long adr)
179 {
180         unsigned int val;
181 
182         if (test_thread_flag(TIF_NOTSC))
183                 val = PR_TSC_SIGSEGV;
184         else
185                 val = PR_TSC_ENABLE;
186 
187         return put_user(val, (unsigned int __user *)adr);
188 }
189 
190 int set_tsc_mode(unsigned int val)
191 {
192         if (val == PR_TSC_SIGSEGV)
193                 disable_TSC();
194         else if (val == PR_TSC_ENABLE)
195                 enable_TSC();
196         else
197                 return -EINVAL;
198 
199         return 0;
200 }
201 
202 void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
203                       struct tss_struct *tss)
204 {
205         struct thread_struct *prev, *next;
206 
207         prev = &prev_p->thread;
208         next = &next_p->thread;
209 
210         if (test_tsk_thread_flag(prev_p, TIF_BLOCKSTEP) ^
211             test_tsk_thread_flag(next_p, TIF_BLOCKSTEP)) {
212                 unsigned long debugctl = get_debugctlmsr();
213 
214                 debugctl &= ~DEBUGCTLMSR_BTF;
215                 if (test_tsk_thread_flag(next_p, TIF_BLOCKSTEP))
216                         debugctl |= DEBUGCTLMSR_BTF;
217 
218                 update_debugctlmsr(debugctl);
219         }
220 
221         if (test_tsk_thread_flag(prev_p, TIF_NOTSC) ^
222             test_tsk_thread_flag(next_p, TIF_NOTSC)) {
223                 /* prev and next are different */
224                 if (test_tsk_thread_flag(next_p, TIF_NOTSC))
225                         hard_disable_TSC();
226                 else
227                         hard_enable_TSC();
228         }
229 
230         if (test_tsk_thread_flag(next_p, TIF_IO_BITMAP)) {
231                 /*
232                  * Copy the relevant range of the IO bitmap.
233                  * Normally this is 128 bytes or less:
234                  */
235                 memcpy(tss->io_bitmap, next->io_bitmap_ptr,
236                        max(prev->io_bitmap_max, next->io_bitmap_max));
237         } else if (test_tsk_thread_flag(prev_p, TIF_IO_BITMAP)) {
238                 /*
239                  * Clear any possible leftover bits:
240                  */
241                 memset(tss->io_bitmap, 0xff, prev->io_bitmap_max);
242         }
243         propagate_user_return_notify(prev_p, next_p);
244 }
245 
246 /*
247  * Idle related variables and functions
248  */
249 unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE;
250 EXPORT_SYMBOL(boot_option_idle_override);
251 
252 static void (*x86_idle)(void);
253 
254 #ifndef CONFIG_SMP
255 static inline void play_dead(void)
256 {
257         BUG();
258 }
259 #endif
260 
261 #ifdef CONFIG_X86_64
262 void enter_idle(void)
263 {
264         this_cpu_write(is_idle, 1);
265         atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
266 }
267 
268 static void __exit_idle(void)
269 {
270         if (x86_test_and_clear_bit_percpu(0, is_idle) == 0)
271                 return;
272         atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL);
273 }
274 
275 /* Called from interrupts to signify idle end */
276 void exit_idle(void)
277 {
278         /* idle loop has pid 0 */
279         if (current->pid)
280                 return;
281         __exit_idle();
282 }
283 #endif
284 
285 void arch_cpu_idle_enter(void)
286 {
287         local_touch_nmi();
288         enter_idle();
289 }
290 
291 void arch_cpu_idle_exit(void)
292 {
293         __exit_idle();
294 }
295 
296 void arch_cpu_idle_dead(void)
297 {
298         play_dead();
299 }
300 
301 /*
302  * Called from the generic idle code.
303  */
304 void arch_cpu_idle(void)
305 {
306         x86_idle();
307 }
308 
309 /*
310  * We use this if we don't have any better idle routine..
311  */
312 void default_idle(void)
313 {
314         trace_cpu_idle_rcuidle(1, smp_processor_id());
315         safe_halt();
316         trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
317 }
318 #ifdef CONFIG_APM_MODULE
319 EXPORT_SYMBOL(default_idle);
320 #endif
321 
322 #ifdef CONFIG_XEN
323 bool xen_set_default_idle(void)
324 {
325         bool ret = !!x86_idle;
326 
327         x86_idle = default_idle;
328 
329         return ret;
330 }
331 #endif
332 void stop_this_cpu(void *dummy)
333 {
334         local_irq_disable();
335         /*
336          * Remove this CPU:
337          */
338         set_cpu_online(smp_processor_id(), false);
339         disable_local_APIC();
340 
341         for (;;)
342                 halt();
343 }
344 
345 bool amd_e400_c1e_detected;
346 EXPORT_SYMBOL(amd_e400_c1e_detected);
347 
348 static cpumask_var_t amd_e400_c1e_mask;
349 
350 void amd_e400_remove_cpu(int cpu)
351 {
352         if (amd_e400_c1e_mask != NULL)
353                 cpumask_clear_cpu(cpu, amd_e400_c1e_mask);
354 }
355 
356 /*
357  * AMD Erratum 400 aware idle routine. We check for C1E active in the interrupt
358  * pending message MSR. If we detect C1E, then we handle it the same
359  * way as C3 power states (local apic timer and TSC stop)
360  */
361 static void amd_e400_idle(void)
362 {
363         if (!amd_e400_c1e_detected) {
364                 u32 lo, hi;
365 
366                 rdmsr(MSR_K8_INT_PENDING_MSG, lo, hi);
367 
368                 if (lo & K8_INTP_C1E_ACTIVE_MASK) {
369                         amd_e400_c1e_detected = true;
370                         if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
371                                 mark_tsc_unstable("TSC halt in AMD C1E");
372                         pr_info("System has AMD C1E enabled\n");
373                 }
374         }
375 
376         if (amd_e400_c1e_detected) {
377                 int cpu = smp_processor_id();
378 
379                 if (!cpumask_test_cpu(cpu, amd_e400_c1e_mask)) {
380                         cpumask_set_cpu(cpu, amd_e400_c1e_mask);
381                         /*
382                          * Force broadcast so ACPI can not interfere.
383                          */
384                         clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_FORCE,
385                                            &cpu);
386                         pr_info("Switch to broadcast mode on CPU%d\n", cpu);
387                 }
388                 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &cpu);
389 
390                 default_idle();
391 
392                 /*
393                  * The switch back from broadcast mode needs to be
394                  * called with interrupts disabled.
395                  */
396                 local_irq_disable();
397                 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &cpu);
398                 local_irq_enable();
399         } else
400                 default_idle();
401 }
402 
403 /*
404  * Intel Core2 and older machines prefer MWAIT over HALT for C1.
405  * We can't rely on cpuidle installing MWAIT, because it will not load
406  * on systems that support only C1 -- so the boot default must be MWAIT.
407  *
408  * Some AMD machines are the opposite, they depend on using HALT.
409  *
410  * So for default C1, which is used during boot until cpuidle loads,
411  * use MWAIT-C1 on Intel HW that has it, else use HALT.
412  */
413 static int prefer_mwait_c1_over_halt(const struct cpuinfo_x86 *c)
414 {
415         if (c->x86_vendor != X86_VENDOR_INTEL)
416                 return 0;
417 
418         if (!cpu_has(c, X86_FEATURE_MWAIT))
419                 return 0;
420 
421         return 1;
422 }
423 
424 /*
425  * MONITOR/MWAIT with no hints, used for default default C1 state.
426  * This invokes MWAIT with interrutps enabled and no flags,
427  * which is backwards compatible with the original MWAIT implementation.
428  */
429 
430 static void mwait_idle(void)
431 {
432         if (!current_set_polling_and_test()) {
433                 trace_cpu_idle_rcuidle(1, smp_processor_id());
434                 if (this_cpu_has(X86_BUG_CLFLUSH_MONITOR)) {
435                         smp_mb(); /* quirk */
436                         clflush((void *)&current_thread_info()->flags);
437                         smp_mb(); /* quirk */
438                 }
439 
440                 __monitor((void *)&current_thread_info()->flags, 0, 0);
441                 if (!need_resched())
442                         __sti_mwait(0, 0);
443                 else
444                         local_irq_enable();
445                 trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
446         } else {
447                 local_irq_enable();
448         }
449         __current_clr_polling();
450 }
451 
452 void select_idle_routine(const struct cpuinfo_x86 *c)
453 {
454 #ifdef CONFIG_SMP
455         if (boot_option_idle_override == IDLE_POLL && smp_num_siblings > 1)
456                 pr_warn_once("WARNING: polling idle and HT enabled, performance may degrade\n");
457 #endif
458         if (x86_idle || boot_option_idle_override == IDLE_POLL)
459                 return;
460 
461         if (cpu_has_bug(c, X86_BUG_AMD_APIC_C1E)) {
462                 /* E400: APIC timer interrupt does not wake up CPU from C1e */
463                 pr_info("using AMD E400 aware idle routine\n");
464                 x86_idle = amd_e400_idle;
465         } else if (prefer_mwait_c1_over_halt(c)) {
466                 pr_info("using mwait in idle threads\n");
467                 x86_idle = mwait_idle;
468         } else
469                 x86_idle = default_idle;
470 }
471 
472 void __init init_amd_e400_c1e_mask(void)
473 {
474         /* If we're using amd_e400_idle, we need to allocate amd_e400_c1e_mask. */
475         if (x86_idle == amd_e400_idle)
476                 zalloc_cpumask_var(&amd_e400_c1e_mask, GFP_KERNEL);
477 }
478 
479 static int __init idle_setup(char *str)
480 {
481         if (!str)
482                 return -EINVAL;
483 
484         if (!strcmp(str, "poll")) {
485                 pr_info("using polling idle threads\n");
486                 boot_option_idle_override = IDLE_POLL;
487                 cpu_idle_poll_ctrl(true);
488         } else if (!strcmp(str, "halt")) {
489                 /*
490                  * When the boot option of idle=halt is added, halt is
491                  * forced to be used for CPU idle. In such case CPU C2/C3
492                  * won't be used again.
493                  * To continue to load the CPU idle driver, don't touch
494                  * the boot_option_idle_override.
495                  */
496                 x86_idle = default_idle;
497                 boot_option_idle_override = IDLE_HALT;
498         } else if (!strcmp(str, "nomwait")) {
499                 /*
500                  * If the boot option of "idle=nomwait" is added,
501                  * it means that mwait will be disabled for CPU C2/C3
502                  * states. In such case it won't touch the variable
503                  * of boot_option_idle_override.
504                  */
505                 boot_option_idle_override = IDLE_NOMWAIT;
506         } else
507                 return -1;
508 
509         return 0;
510 }
511 early_param("idle", idle_setup);
512 
513 unsigned long arch_align_stack(unsigned long sp)
514 {
515         if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
516                 sp -= get_random_int() % 8192;
517         return sp & ~0xf;
518 }
519 
520 unsigned long arch_randomize_brk(struct mm_struct *mm)
521 {
522         unsigned long range_end = mm->brk + 0x02000000;
523         return randomize_range(mm->brk, range_end, 0) ? : mm->brk;
524 }
525 
526 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp