~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/kernel/fpu/init.c

Version: ~ [ linux-5.4-rc7 ] ~ [ linux-5.3.11 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.84 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.154 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.201 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.201 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.77 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * x86 FPU boot time init code:
  3  */
  4 #include <asm/fpu/internal.h>
  5 #include <asm/tlbflush.h>
  6 #include <asm/setup.h>
  7 #include <asm/cmdline.h>
  8 
  9 #include <linux/sched.h>
 10 #include <linux/init.h>
 11 
 12 /*
 13  * Initialize the registers found in all CPUs, CR0 and CR4:
 14  */
 15 static void fpu__init_cpu_generic(void)
 16 {
 17         unsigned long cr0;
 18         unsigned long cr4_mask = 0;
 19 
 20         if (boot_cpu_has(X86_FEATURE_FXSR))
 21                 cr4_mask |= X86_CR4_OSFXSR;
 22         if (boot_cpu_has(X86_FEATURE_XMM))
 23                 cr4_mask |= X86_CR4_OSXMMEXCPT;
 24         if (cr4_mask)
 25                 cr4_set_bits(cr4_mask);
 26 
 27         cr0 = read_cr0();
 28         cr0 &= ~(X86_CR0_TS|X86_CR0_EM); /* clear TS and EM */
 29         if (!boot_cpu_has(X86_FEATURE_FPU))
 30                 cr0 |= X86_CR0_EM;
 31         write_cr0(cr0);
 32 
 33         /* Flush out any pending x87 state: */
 34 #ifdef CONFIG_MATH_EMULATION
 35         if (!boot_cpu_has(X86_FEATURE_FPU))
 36                 fpstate_init_soft(&current->thread.fpu.state.soft);
 37         else
 38 #endif
 39                 asm volatile ("fninit");
 40 }
 41 
 42 /*
 43  * Enable all supported FPU features. Called when a CPU is brought online:
 44  */
 45 void fpu__init_cpu(void)
 46 {
 47         fpu__init_cpu_generic();
 48         fpu__init_cpu_xstate();
 49 }
 50 
 51 /*
 52  * The earliest FPU detection code.
 53  *
 54  * Set the X86_FEATURE_FPU CPU-capability bit based on
 55  * trying to execute an actual sequence of FPU instructions:
 56  */
 57 static void fpu__init_system_early_generic(struct cpuinfo_x86 *c)
 58 {
 59         unsigned long cr0;
 60         u16 fsw, fcw;
 61 
 62         fsw = fcw = 0xffff;
 63 
 64         cr0 = read_cr0();
 65         cr0 &= ~(X86_CR0_TS | X86_CR0_EM);
 66         write_cr0(cr0);
 67 
 68         if (!test_bit(X86_FEATURE_FPU, (unsigned long *)cpu_caps_cleared)) {
 69                 asm volatile("fninit ; fnstsw %0 ; fnstcw %1"
 70                              : "+m" (fsw), "+m" (fcw));
 71 
 72                 if (fsw == 0 && (fcw & 0x103f) == 0x003f)
 73                         set_cpu_cap(c, X86_FEATURE_FPU);
 74                 else
 75                         clear_cpu_cap(c, X86_FEATURE_FPU);
 76         }
 77 
 78 #ifndef CONFIG_MATH_EMULATION
 79         if (!boot_cpu_has(X86_FEATURE_FPU)) {
 80                 pr_emerg("x86/fpu: Giving up, no FPU found and no math emulation present\n");
 81                 for (;;)
 82                         asm volatile("hlt");
 83         }
 84 #endif
 85 }
 86 
 87 /*
 88  * Boot time FPU feature detection code:
 89  */
 90 unsigned int mxcsr_feature_mask __read_mostly = 0xffffffffu;
 91 
 92 static void __init fpu__init_system_mxcsr(void)
 93 {
 94         unsigned int mask = 0;
 95 
 96         if (boot_cpu_has(X86_FEATURE_FXSR)) {
 97                 /* Static because GCC does not get 16-byte stack alignment right: */
 98                 static struct fxregs_state fxregs __initdata;
 99 
100                 asm volatile("fxsave %0" : "+m" (fxregs));
101 
102                 mask = fxregs.mxcsr_mask;
103 
104                 /*
105                  * If zero then use the default features mask,
106                  * which has all features set, except the
107                  * denormals-are-zero feature bit:
108                  */
109                 if (mask == 0)
110                         mask = 0x0000ffbf;
111         }
112         mxcsr_feature_mask &= mask;
113 }
114 
115 /*
116  * Once per bootup FPU initialization sequences that will run on most x86 CPUs:
117  */
118 static void __init fpu__init_system_generic(void)
119 {
120         /*
121          * Set up the legacy init FPU context. (xstate init might overwrite this
122          * with a more modern format, if the CPU supports it.)
123          */
124         fpstate_init(&init_fpstate);
125 
126         fpu__init_system_mxcsr();
127 }
128 
129 /*
130  * Size of the FPU context state. All tasks in the system use the
131  * same context size, regardless of what portion they use.
132  * This is inherent to the XSAVE architecture which puts all state
133  * components into a single, continuous memory block:
134  */
135 unsigned int fpu_kernel_xstate_size;
136 EXPORT_SYMBOL_GPL(fpu_kernel_xstate_size);
137 
138 /* Get alignment of the TYPE. */
139 #define TYPE_ALIGN(TYPE) offsetof(struct { char x; TYPE test; }, test)
140 
141 /*
142  * Enforce that 'MEMBER' is the last field of 'TYPE'.
143  *
144  * Align the computed size with alignment of the TYPE,
145  * because that's how C aligns structs.
146  */
147 #define CHECK_MEMBER_AT_END_OF(TYPE, MEMBER) \
148         BUILD_BUG_ON(sizeof(TYPE) != ALIGN(offsetofend(TYPE, MEMBER), \
149                                            TYPE_ALIGN(TYPE)))
150 
151 /*
152  * We append the 'struct fpu' to the task_struct:
153  */
154 static void __init fpu__init_task_struct_size(void)
155 {
156         int task_size = sizeof(struct task_struct);
157 
158         /*
159          * Subtract off the static size of the register state.
160          * It potentially has a bunch of padding.
161          */
162         task_size -= sizeof(((struct task_struct *)0)->thread.fpu.state);
163 
164         /*
165          * Add back the dynamically-calculated register state
166          * size.
167          */
168         task_size += fpu_kernel_xstate_size;
169 
170         /*
171          * We dynamically size 'struct fpu', so we require that
172          * it be at the end of 'thread_struct' and that
173          * 'thread_struct' be at the end of 'task_struct'.  If
174          * you hit a compile error here, check the structure to
175          * see if something got added to the end.
176          */
177         CHECK_MEMBER_AT_END_OF(struct fpu, state);
178         CHECK_MEMBER_AT_END_OF(struct thread_struct, fpu);
179         CHECK_MEMBER_AT_END_OF(struct task_struct, thread);
180 
181         arch_task_struct_size = task_size;
182 }
183 
184 /*
185  * Set up the user and kernel xstate sizes based on the legacy FPU context size.
186  *
187  * We set this up first, and later it will be overwritten by
188  * fpu__init_system_xstate() if the CPU knows about xstates.
189  */
190 static void __init fpu__init_system_xstate_size_legacy(void)
191 {
192         static int on_boot_cpu __initdata = 1;
193 
194         WARN_ON_FPU(!on_boot_cpu);
195         on_boot_cpu = 0;
196 
197         /*
198          * Note that xstate sizes might be overwritten later during
199          * fpu__init_system_xstate().
200          */
201 
202         if (!boot_cpu_has(X86_FEATURE_FPU)) {
203                 /*
204                  * Disable xsave as we do not support it if i387
205                  * emulation is enabled.
206                  */
207                 setup_clear_cpu_cap(X86_FEATURE_XSAVE);
208                 setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
209                 fpu_kernel_xstate_size = sizeof(struct swregs_state);
210         } else {
211                 if (boot_cpu_has(X86_FEATURE_FXSR))
212                         fpu_kernel_xstate_size =
213                                 sizeof(struct fxregs_state);
214                 else
215                         fpu_kernel_xstate_size =
216                                 sizeof(struct fregs_state);
217         }
218 
219         fpu_user_xstate_size = fpu_kernel_xstate_size;
220 }
221 
222 /*
223  * Find supported xfeatures based on cpu features and command-line input.
224  * This must be called after fpu__init_parse_early_param() is called and
225  * xfeatures_mask is enumerated.
226  */
227 u64 __init fpu__get_supported_xfeatures_mask(void)
228 {
229         return XCNTXT_MASK;
230 }
231 
232 /* Legacy code to initialize eager fpu mode. */
233 static void __init fpu__init_system_ctx_switch(void)
234 {
235         static bool on_boot_cpu __initdata = 1;
236 
237         WARN_ON_FPU(!on_boot_cpu);
238         on_boot_cpu = 0;
239 
240         WARN_ON_FPU(current->thread.fpu.fpstate_active);
241 }
242 
243 /*
244  * We parse fpu parameters early because fpu__init_system() is executed
245  * before parse_early_param().
246  */
247 static void __init fpu__init_parse_early_param(void)
248 {
249         if (cmdline_find_option_bool(boot_command_line, "no387"))
250                 setup_clear_cpu_cap(X86_FEATURE_FPU);
251 
252         if (cmdline_find_option_bool(boot_command_line, "nofxsr")) {
253                 setup_clear_cpu_cap(X86_FEATURE_FXSR);
254                 setup_clear_cpu_cap(X86_FEATURE_FXSR_OPT);
255                 setup_clear_cpu_cap(X86_FEATURE_XMM);
256         }
257 
258         if (cmdline_find_option_bool(boot_command_line, "noxsave"))
259                 fpu__xstate_clear_all_cpu_caps();
260 
261         if (cmdline_find_option_bool(boot_command_line, "noxsaveopt"))
262                 setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
263 
264         if (cmdline_find_option_bool(boot_command_line, "noxsaves"))
265                 setup_clear_cpu_cap(X86_FEATURE_XSAVES);
266 }
267 
268 /*
269  * Called on the boot CPU once per system bootup, to set up the initial
270  * FPU state that is later cloned into all processes:
271  */
272 void __init fpu__init_system(struct cpuinfo_x86 *c)
273 {
274         fpu__init_parse_early_param();
275         fpu__init_system_early_generic(c);
276 
277         /*
278          * The FPU has to be operational for some of the
279          * later FPU init activities:
280          */
281         fpu__init_cpu();
282 
283         fpu__init_system_generic();
284         fpu__init_system_xstate_size_legacy();
285         fpu__init_system_xstate();
286         fpu__init_task_struct_size();
287 
288         fpu__init_system_ctx_switch();
289 }
290 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp