~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/kernel/process.c

Version: ~ [ linux-5.15-rc5 ] ~ [ linux-5.14.11 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.72 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.152 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.210 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.250 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.286 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.288 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  2 
  3 #include <linux/errno.h>
  4 #include <linux/kernel.h>
  5 #include <linux/mm.h>
  6 #include <linux/smp.h>
  7 #include <linux/prctl.h>
  8 #include <linux/slab.h>
  9 #include <linux/sched.h>
 10 #include <linux/module.h>
 11 #include <linux/pm.h>
 12 #include <linux/clockchips.h>
 13 #include <linux/random.h>
 14 #include <linux/user-return-notifier.h>
 15 #include <linux/dmi.h>
 16 #include <linux/utsname.h>
 17 #include <linux/stackprotector.h>
 18 #include <linux/tick.h>
 19 #include <linux/cpuidle.h>
 20 #include <trace/events/power.h>
 21 #include <linux/hw_breakpoint.h>
 22 #include <asm/cpu.h>
 23 #include <asm/apic.h>
 24 #include <asm/syscalls.h>
 25 #include <asm/idle.h>
 26 #include <asm/uaccess.h>
 27 #include <asm/mwait.h>
 28 #include <asm/i387.h>
 29 #include <asm/fpu-internal.h>
 30 #include <asm/debugreg.h>
 31 #include <asm/nmi.h>
 32 #include <asm/tlbflush.h>
 33 #include <asm/spec-ctrl.h>
 34 
 35 #include "process.h"
 36 
 37 /*
 38  * per-CPU TSS segments. Threads are completely 'soft' on Linux,
 39  * no more per-task TSS's. The TSS size is kept cacheline-aligned
 40  * so they are allowed to end up in the .data..cacheline_aligned
 41  * section. Since TSS's are completely CPU-local, we want them
 42  * on exact cacheline boundaries, to eliminate cacheline ping-pong.
 43  */
 44 __visible DEFINE_PER_CPU_SHARED_ALIGNED_USER_MAPPED(struct tss_struct, init_tss) = INIT_TSS;
 45 
 46 #ifdef CONFIG_X86_64
 47 static DEFINE_PER_CPU(unsigned char, is_idle);
 48 static ATOMIC_NOTIFIER_HEAD(idle_notifier);
 49 
 50 void idle_notifier_register(struct notifier_block *n)
 51 {
 52         atomic_notifier_chain_register(&idle_notifier, n);
 53 }
 54 EXPORT_SYMBOL_GPL(idle_notifier_register);
 55 
 56 void idle_notifier_unregister(struct notifier_block *n)
 57 {
 58         atomic_notifier_chain_unregister(&idle_notifier, n);
 59 }
 60 EXPORT_SYMBOL_GPL(idle_notifier_unregister);
 61 #endif
 62 
 63 struct kmem_cache *task_xstate_cachep;
 64 EXPORT_SYMBOL_GPL(task_xstate_cachep);
 65 
 66 /*
 67  * this gets called so that we can store lazy state into memory and copy the
 68  * current task into the new thread.
 69  */
 70 int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
 71 {
 72         int ret;
 73 
 74         *dst = *src;
 75         if (fpu_allocated(&src->thread.fpu)) {
 76                 memset(&dst->thread.fpu, 0, sizeof(dst->thread.fpu));
 77                 ret = fpu_alloc(&dst->thread.fpu);
 78                 if (ret)
 79                         return ret;
 80                 fpu_copy(dst, src);
 81         }
 82         return 0;
 83 }
 84 
 85 void free_thread_xstate(struct task_struct *tsk)
 86 {
 87         fpu_free(&tsk->thread.fpu);
 88 }
 89 
 90 void arch_release_task_struct(struct task_struct *tsk)
 91 {
 92         free_thread_xstate(tsk);
 93 }
 94 
 95 void arch_task_cache_init(void)
 96 {
 97         task_xstate_cachep =
 98                 kmem_cache_create("task_xstate", xstate_size,
 99                                   __alignof__(union thread_xstate),
100                                   SLAB_PANIC | SLAB_NOTRACK, NULL);
101 }
102 
103 /*
104  * Free current thread data structures etc..
105  */
106 void exit_thread(void)
107 {
108         struct task_struct *me = current;
109         struct thread_struct *t = &me->thread;
110         unsigned long *bp = t->io_bitmap_ptr;
111 
112         if (bp) {
113                 struct tss_struct *tss = &per_cpu(init_tss, get_cpu());
114 
115                 t->io_bitmap_ptr = NULL;
116                 clear_thread_flag(TIF_IO_BITMAP);
117                 /*
118                  * Careful, clear this in the TSS too:
119                  */
120                 memset(tss->io_bitmap, 0xff, t->io_bitmap_max);
121                 t->io_bitmap_max = 0;
122                 put_cpu();
123                 kfree(bp);
124         }
125 
126         drop_fpu(me);
127 }
128 
129 void flush_thread(void)
130 {
131         struct task_struct *tsk = current;
132 
133         flush_ptrace_hw_breakpoint(tsk);
134         memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
135         drop_init_fpu(tsk);
136         /*
137          * Free the FPU state for non xsave platforms. They get reallocated
138          * lazily at the first use.
139          */
140         if (!use_eager_fpu())
141                 free_thread_xstate(tsk);
142 }
143 
144 static void hard_disable_TSC(void)
145 {
146         cr4_set_bits(X86_CR4_TSD);
147 }
148 
149 void disable_TSC(void)
150 {
151         preempt_disable();
152         if (!test_and_set_thread_flag(TIF_NOTSC))
153                 /*
154                  * Must flip the CPU state synchronously with
155                  * TIF_NOTSC in the current running context.
156                  */
157                 hard_disable_TSC();
158         preempt_enable();
159 }
160 
161 static void hard_enable_TSC(void)
162 {
163         cr4_clear_bits(X86_CR4_TSD);
164 }
165 
166 static void enable_TSC(void)
167 {
168         preempt_disable();
169         if (test_and_clear_thread_flag(TIF_NOTSC))
170                 /*
171                  * Must flip the CPU state synchronously with
172                  * TIF_NOTSC in the current running context.
173                  */
174                 hard_enable_TSC();
175         preempt_enable();
176 }
177 
178 int get_tsc_mode(unsigned long adr)
179 {
180         unsigned int val;
181 
182         if (test_thread_flag(TIF_NOTSC))
183                 val = PR_TSC_SIGSEGV;
184         else
185                 val = PR_TSC_ENABLE;
186 
187         return put_user(val, (unsigned int __user *)adr);
188 }
189 
190 int set_tsc_mode(unsigned int val)
191 {
192         if (val == PR_TSC_SIGSEGV)
193                 disable_TSC();
194         else if (val == PR_TSC_ENABLE)
195                 enable_TSC();
196         else
197                 return -EINVAL;
198 
199         return 0;
200 }
201 
202 static inline void switch_to_bitmap(struct thread_struct *prev,
203                                     struct thread_struct *next,
204                                     unsigned long tifp, unsigned long tifn)
205 {
206         struct tss_struct *tss = this_cpu_ptr(&init_tss);
207 
208         if (tifn & _TIF_IO_BITMAP) {
209                 /*
210                  * Copy the relevant range of the IO bitmap.
211                  * Normally this is 128 bytes or less:
212                  */
213                 memcpy(tss->io_bitmap, next->io_bitmap_ptr,
214                        max(prev->io_bitmap_max, next->io_bitmap_max));
215         } else if (tifp & _TIF_IO_BITMAP) {
216                 /*
217                  * Clear any possible leftover bits:
218                  */
219                 memset(tss->io_bitmap, 0xff, prev->io_bitmap_max);
220         }
221 }
222 
223 #ifdef CONFIG_SMP
224 
225 struct ssb_state {
226         struct ssb_state        *shared_state;
227         raw_spinlock_t          lock;
228         unsigned int            disable_state;
229         unsigned long           local_state;
230 };
231 
232 #define LSTATE_SSB      0
233 
234 static DEFINE_PER_CPU(struct ssb_state, ssb_state);
235 
236 void speculative_store_bypass_ht_init(void)
237 {
238         struct ssb_state *st = this_cpu_ptr(&ssb_state);
239         unsigned int this_cpu = smp_processor_id();
240         unsigned int cpu;
241 
242         st->local_state = 0;
243 
244         /*
245          * Shared state setup happens once on the first bringup
246          * of the CPU. It's not destroyed on CPU hotunplug.
247          */
248         if (st->shared_state)
249                 return;
250 
251         raw_spin_lock_init(&st->lock);
252 
253         /*
254          * Go over HT siblings and check whether one of them has set up the
255          * shared state pointer already.
256          */
257         for_each_cpu(cpu, topology_thread_cpumask(this_cpu)) {
258                 if (cpu == this_cpu)
259                         continue;
260 
261                 if (!per_cpu(ssb_state, cpu).shared_state)
262                         continue;
263 
264                 /* Link it to the state of the sibling: */
265                 st->shared_state = per_cpu(ssb_state, cpu).shared_state;
266                 return;
267         }
268 
269         /*
270          * First HT sibling to come up on the core.  Link shared state of
271          * the first HT sibling to itself. The siblings on the same core
272          * which come up later will see the shared state pointer and link
273          * themself to the state of this CPU.
274          */
275         st->shared_state = st;
276 }
277 
278 /*
279  * Logic is: First HT sibling enables SSBD for both siblings in the core
280  * and last sibling to disable it, disables it for the whole core. This how
281  * MSR_SPEC_CTRL works in "hardware":
282  *
283  *  CORE_SPEC_CTRL = THREAD0_SPEC_CTRL | THREAD1_SPEC_CTRL
284  */
285 static __always_inline void amd_set_core_ssb_state(unsigned long tifn)
286 {
287         struct ssb_state *st = this_cpu_ptr(&ssb_state);
288         u64 msr = x86_amd_ls_cfg_base;
289 
290         if (!static_cpu_has(X86_FEATURE_ZEN)) {
291                 msr |= ssbd_tif_to_amd_ls_cfg(tifn);
292                 wrmsrl(MSR_AMD64_LS_CFG, msr);
293                 return;
294         }
295 
296         if (tifn & _TIF_SSBD) {
297                 /*
298                  * Since this can race with prctl(), block reentry on the
299                  * same CPU.
300                  */
301                 if (__test_and_set_bit(LSTATE_SSB, &st->local_state))
302                         return;
303 
304                 msr |= x86_amd_ls_cfg_ssbd_mask;
305 
306                 raw_spin_lock(&st->shared_state->lock);
307                 /* First sibling enables SSBD: */
308                 if (!st->shared_state->disable_state)
309                         wrmsrl(MSR_AMD64_LS_CFG, msr);
310                 st->shared_state->disable_state++;
311                 raw_spin_unlock(&st->shared_state->lock);
312         } else {
313                 if (!__test_and_clear_bit(LSTATE_SSB, &st->local_state))
314                         return;
315 
316                 raw_spin_lock(&st->shared_state->lock);
317                 st->shared_state->disable_state--;
318                 if (!st->shared_state->disable_state)
319                         wrmsrl(MSR_AMD64_LS_CFG, msr);
320                 raw_spin_unlock(&st->shared_state->lock);
321         }
322 }
323 #else
324 static __always_inline void amd_set_core_ssb_state(unsigned long tifn)
325 {
326         u64 msr = x86_amd_ls_cfg_base | ssbd_tif_to_amd_ls_cfg(tifn);
327 
328         wrmsrl(MSR_AMD64_LS_CFG, msr);
329 }
330 #endif
331 
332 static __always_inline void amd_set_ssb_virt_state(unsigned long tifn)
333 {
334         /*
335          * SSBD has the same definition in SPEC_CTRL and VIRT_SPEC_CTRL,
336          * so ssbd_tif_to_spec_ctrl() just works.
337          */
338         wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, ssbd_tif_to_spec_ctrl(tifn));
339 }
340 
341 /*
342  * Update the MSRs managing speculation control, during context switch.
343  *
344  * tifp: Previous task's thread flags
345  * tifn: Next task's thread flags
346  */
347 static __always_inline void __speculation_ctrl_update(unsigned long tifp,
348                                                       unsigned long tifn)
349 {
350         unsigned long tif_diff = tifp ^ tifn;
351         u64 msr = x86_spec_ctrl_base;
352         bool updmsr = false;
353 
354         lockdep_assert_irqs_disabled();
355 
356         /*
357          * If TIF_SSBD is different, select the proper mitigation
358          * method. Note that if SSBD mitigation is disabled or permanentely
359          * enabled this branch can't be taken because nothing can set
360          * TIF_SSBD.
361          */
362         if (tif_diff & _TIF_SSBD) {
363                 if (static_cpu_has(X86_FEATURE_VIRT_SSBD)) {
364                         amd_set_ssb_virt_state(tifn);
365                 } else if (static_cpu_has(X86_FEATURE_LS_CFG_SSBD)) {
366                         amd_set_core_ssb_state(tifn);
367                 } else if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) ||
368                            static_cpu_has(X86_FEATURE_AMD_SSBD)) {
369                         msr |= ssbd_tif_to_spec_ctrl(tifn);
370                         updmsr  = true;
371                 }
372         }
373 
374         /*
375          * Only evaluate TIF_SPEC_IB if conditional STIBP is enabled,
376          * otherwise avoid the MSR write.
377          */
378         if (IS_ENABLED(CONFIG_SMP) &&
379             static_branch_unlikely(&switch_to_cond_stibp)) {
380                 updmsr |= !!(tif_diff & _TIF_SPEC_IB);
381                 msr |= stibp_tif_to_spec_ctrl(tifn);
382         }
383 
384         if (updmsr)
385                 wrmsrl(MSR_IA32_SPEC_CTRL, msr);
386 }
387 
388 static unsigned long speculation_ctrl_update_tif(struct task_struct *tsk)
389 {
390         if (test_and_clear_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE)) {
391                 if (task_spec_ssb_disable(tsk))
392                         set_tsk_thread_flag(tsk, TIF_SSBD);
393                 else
394                         clear_tsk_thread_flag(tsk, TIF_SSBD);
395 
396                 if (task_spec_ib_disable(tsk))
397                         set_tsk_thread_flag(tsk, TIF_SPEC_IB);
398                 else
399                         clear_tsk_thread_flag(tsk, TIF_SPEC_IB);
400         }
401         /* Return the updated threadinfo flags*/
402         return task_thread_info(tsk)->flags;
403 }
404 
405 void speculation_ctrl_update(unsigned long tif)
406 {
407         unsigned long flags;
408 
409         /* Forced update. Make sure all relevant TIF flags are different */
410         local_irq_save(flags);
411         __speculation_ctrl_update(~tif, tif);
412         local_irq_restore(flags);
413 }
414 
415 /* Called from seccomp/prctl update */
416 void speculation_ctrl_update_current(void)
417 {
418         preempt_disable();
419         speculation_ctrl_update(speculation_ctrl_update_tif(current));
420         preempt_enable();
421 }
422 
423 void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p)
424 {
425         struct thread_struct *prev, *next;
426         unsigned long tifp, tifn;
427 
428         prev = &prev_p->thread;
429         next = &next_p->thread;
430 
431         tifn = ACCESS_ONCE(task_thread_info(next_p)->flags);
432         tifp = ACCESS_ONCE(task_thread_info(prev_p)->flags);
433         switch_to_bitmap(prev, next, tifp, tifn);
434 
435         propagate_user_return_notify(prev_p, next_p);
436 
437         if ((tifp & _TIF_BLOCKSTEP || tifn & _TIF_BLOCKSTEP) &&
438             arch_has_block_step()) {
439                 unsigned long debugctl, msk;
440 
441                 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
442                 debugctl &= ~DEBUGCTLMSR_BTF;
443                 msk = tifn & _TIF_BLOCKSTEP;
444                 debugctl |= (msk >> TIF_BLOCKSTEP) << DEBUGCTLMSR_BTF_SHIFT;
445                 wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
446         }
447 
448         if ((tifp ^ tifn) & _TIF_NOTSC) {
449                 if (tifn & _TIF_NOTSC)
450                         hard_disable_TSC();
451                 else
452                         hard_enable_TSC();
453         }
454 
455         if (likely(!((tifp | tifn) & _TIF_SPEC_FORCE_UPDATE))) {
456                 __speculation_ctrl_update(tifp, tifn);
457         } else {
458                 speculation_ctrl_update_tif(prev_p);
459                 tifn = speculation_ctrl_update_tif(next_p);
460 
461                 /* Enforce MSR update to ensure consistent state */
462                 __speculation_ctrl_update(~tifn, tifn);
463         }
464 }
465 
466 /*
467  * Idle related variables and functions
468  */
469 unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE;
470 EXPORT_SYMBOL(boot_option_idle_override);
471 
472 static void (*x86_idle)(void);
473 
474 #ifndef CONFIG_SMP
475 static inline void play_dead(void)
476 {
477         BUG();
478 }
479 #endif
480 
481 #ifdef CONFIG_X86_64
482 void enter_idle(void)
483 {
484         this_cpu_write(is_idle, 1);
485         atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
486 }
487 
488 static void __exit_idle(void)
489 {
490         if (x86_test_and_clear_bit_percpu(0, is_idle) == 0)
491                 return;
492         atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL);
493 }
494 
495 /* Called from interrupts to signify idle end */
496 void exit_idle(void)
497 {
498         /* idle loop has pid 0 */
499         if (current->pid)
500                 return;
501         __exit_idle();
502 }
503 #endif
504 
505 void arch_cpu_idle_enter(void)
506 {
507         local_touch_nmi();
508         enter_idle();
509 }
510 
511 void arch_cpu_idle_exit(void)
512 {
513         __exit_idle();
514 }
515 
516 void arch_cpu_idle_dead(void)
517 {
518         play_dead();
519 }
520 
521 /*
522  * Called from the generic idle code.
523  */
524 void arch_cpu_idle(void)
525 {
526         x86_idle();
527 }
528 
529 /*
530  * We use this if we don't have any better idle routine..
531  */
532 void default_idle(void)
533 {
534         trace_cpu_idle_rcuidle(1, smp_processor_id());
535         safe_halt();
536         trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
537 }
538 #ifdef CONFIG_APM_MODULE
539 EXPORT_SYMBOL(default_idle);
540 #endif
541 
542 #ifdef CONFIG_XEN
543 bool xen_set_default_idle(void)
544 {
545         bool ret = !!x86_idle;
546 
547         x86_idle = default_idle;
548 
549         return ret;
550 }
551 #endif
552 void stop_this_cpu(void *dummy)
553 {
554         local_irq_disable();
555         /*
556          * Remove this CPU:
557          */
558         set_cpu_online(smp_processor_id(), false);
559         disable_local_APIC();
560 
561         for (;;)
562                 halt();
563 }
564 
565 bool amd_e400_c1e_detected;
566 EXPORT_SYMBOL(amd_e400_c1e_detected);
567 
568 static cpumask_var_t amd_e400_c1e_mask;
569 
570 void amd_e400_remove_cpu(int cpu)
571 {
572         if (amd_e400_c1e_mask != NULL)
573                 cpumask_clear_cpu(cpu, amd_e400_c1e_mask);
574 }
575 
576 /*
577  * AMD Erratum 400 aware idle routine. We check for C1E active in the interrupt
578  * pending message MSR. If we detect C1E, then we handle it the same
579  * way as C3 power states (local apic timer and TSC stop)
580  */
581 static void amd_e400_idle(void)
582 {
583         if (!amd_e400_c1e_detected) {
584                 u32 lo, hi;
585 
586                 rdmsr(MSR_K8_INT_PENDING_MSG, lo, hi);
587 
588                 if (lo & K8_INTP_C1E_ACTIVE_MASK) {
589                         amd_e400_c1e_detected = true;
590                         if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
591                                 mark_tsc_unstable("TSC halt in AMD C1E");
592                         pr_info("System has AMD C1E enabled\n");
593                 }
594         }
595 
596         if (amd_e400_c1e_detected) {
597                 int cpu = smp_processor_id();
598 
599                 if (!cpumask_test_cpu(cpu, amd_e400_c1e_mask)) {
600                         cpumask_set_cpu(cpu, amd_e400_c1e_mask);
601                         /*
602                          * Force broadcast so ACPI can not interfere.
603                          */
604                         clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_FORCE,
605                                            &cpu);
606                         pr_info("Switch to broadcast mode on CPU%d\n", cpu);
607                 }
608                 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &cpu);
609 
610                 default_idle();
611 
612                 /*
613                  * The switch back from broadcast mode needs to be
614                  * called with interrupts disabled.
615                  */
616                 local_irq_disable();
617                 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &cpu);
618                 local_irq_enable();
619         } else
620                 default_idle();
621 }
622 
623 /*
624  * Intel Core2 and older machines prefer MWAIT over HALT for C1.
625  * We can't rely on cpuidle installing MWAIT, because it will not load
626  * on systems that support only C1 -- so the boot default must be MWAIT.
627  *
628  * Some AMD machines are the opposite, they depend on using HALT.
629  *
630  * So for default C1, which is used during boot until cpuidle loads,
631  * use MWAIT-C1 on Intel HW that has it, else use HALT.
632  */
633 static int prefer_mwait_c1_over_halt(const struct cpuinfo_x86 *c)
634 {
635         if (c->x86_vendor != X86_VENDOR_INTEL)
636                 return 0;
637 
638         if (!cpu_has(c, X86_FEATURE_MWAIT))
639                 return 0;
640 
641         return 1;
642 }
643 
644 /*
645  * MONITOR/MWAIT with no hints, used for default default C1 state.
646  * This invokes MWAIT with interrutps enabled and no flags,
647  * which is backwards compatible with the original MWAIT implementation.
648  */
649 
650 static void mwait_idle(void)
651 {
652         if (!current_set_polling_and_test()) {
653                 if (static_cpu_has(X86_FEATURE_CLFLUSH_MONITOR)) {
654                         mb();
655                         clflush((void *)&current_thread_info()->flags);
656                         mb();
657                 }
658 
659                 __monitor((void *)&current_thread_info()->flags, 0, 0);
660                 if (!need_resched())
661                         __sti_mwait(0, 0);
662                 else
663                         local_irq_enable();
664         } else
665                 local_irq_enable();
666         current_clr_polling();
667 }
668 
669 void select_idle_routine(const struct cpuinfo_x86 *c)
670 {
671 #ifdef CONFIG_SMP
672         if (boot_option_idle_override == IDLE_POLL && smp_num_siblings > 1)
673                 pr_warn_once("WARNING: polling idle and HT enabled, performance may degrade\n");
674 #endif
675         if (x86_idle || boot_option_idle_override == IDLE_POLL)
676                 return;
677 
678         if (cpu_has_bug(c, X86_BUG_AMD_APIC_C1E)) {
679                 /* E400: APIC timer interrupt does not wake up CPU from C1e */
680                 pr_info("using AMD E400 aware idle routine\n");
681                 x86_idle = amd_e400_idle;
682         } else if (prefer_mwait_c1_over_halt(c)) {
683                 pr_info("using mwait in idle threads\n");
684                 x86_idle = mwait_idle;
685         } else
686                 x86_idle = default_idle;
687 }
688 
689 void __init init_amd_e400_c1e_mask(void)
690 {
691         /* If we're using amd_e400_idle, we need to allocate amd_e400_c1e_mask. */
692         if (x86_idle == amd_e400_idle)
693                 zalloc_cpumask_var(&amd_e400_c1e_mask, GFP_KERNEL);
694 }
695 
696 static int __init idle_setup(char *str)
697 {
698         if (!str)
699                 return -EINVAL;
700 
701         if (!strcmp(str, "poll")) {
702                 pr_info("using polling idle threads\n");
703                 boot_option_idle_override = IDLE_POLL;
704                 cpu_idle_poll_ctrl(true);
705         } else if (!strcmp(str, "halt")) {
706                 /*
707                  * When the boot option of idle=halt is added, halt is
708                  * forced to be used for CPU idle. In such case CPU C2/C3
709                  * won't be used again.
710                  * To continue to load the CPU idle driver, don't touch
711                  * the boot_option_idle_override.
712                  */
713                 x86_idle = default_idle;
714                 boot_option_idle_override = IDLE_HALT;
715         } else if (!strcmp(str, "nomwait")) {
716                 /*
717                  * If the boot option of "idle=nomwait" is added,
718                  * it means that mwait will be disabled for CPU C2/C3
719                  * states. In such case it won't touch the variable
720                  * of boot_option_idle_override.
721                  */
722                 boot_option_idle_override = IDLE_NOMWAIT;
723         } else
724                 return -1;
725 
726         return 0;
727 }
728 early_param("idle", idle_setup);
729 
730 unsigned long arch_align_stack(unsigned long sp)
731 {
732         if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
733                 sp -= get_random_int() % 8192;
734         return sp & ~0xf;
735 }
736 
737 unsigned long arch_randomize_brk(struct mm_struct *mm)
738 {
739         unsigned long range_end = mm->brk + 0x02000000;
740         return randomize_range(mm->brk, range_end, 0) ? : mm->brk;
741 }
742 
743 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp