~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/kernel/fpu/core.c

Version: ~ [ linux-5.12 ] ~ [ linux-5.11.16 ] ~ [ linux-5.10.32 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.114 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.188 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.231 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.267 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.267 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  *  Copyright (C) 1994 Linus Torvalds
  3  *
  4  *  Pentium III FXSR, SSE support
  5  *  General FPU state handling cleanups
  6  *      Gareth Hughes <gareth@valinux.com>, May 2000
  7  */
  8 #include <asm/fpu/internal.h>
  9 #include <asm/fpu/regset.h>
 10 #include <asm/fpu/signal.h>
 11 #include <asm/fpu/types.h>
 12 #include <asm/traps.h>
 13 #include <asm/irq_regs.h>
 14 
 15 #include <linux/hardirq.h>
 16 #include <linux/pkeys.h>
 17 
 18 #define CREATE_TRACE_POINTS
 19 #include <asm/trace/fpu.h>
 20 
 21 /*
 22  * Represents the initial FPU state. It's mostly (but not completely) zeroes,
 23  * depending on the FPU hardware format:
 24  */
 25 union fpregs_state init_fpstate __read_mostly;
 26 
 27 /*
 28  * Track whether the kernel is using the FPU state
 29  * currently.
 30  *
 31  * This flag is used:
 32  *
 33  *   - by IRQ context code to potentially use the FPU
 34  *     if it's unused.
 35  *
 36  *   - to debug kernel_fpu_begin()/end() correctness
 37  */
 38 static DEFINE_PER_CPU(bool, in_kernel_fpu);
 39 
 40 /*
 41  * Track which context is using the FPU on the CPU:
 42  */
 43 DEFINE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx);
 44 
 45 static void kernel_fpu_disable(void)
 46 {
 47         WARN_ON_FPU(this_cpu_read(in_kernel_fpu));
 48         this_cpu_write(in_kernel_fpu, true);
 49 }
 50 
 51 static void kernel_fpu_enable(void)
 52 {
 53         WARN_ON_FPU(!this_cpu_read(in_kernel_fpu));
 54         this_cpu_write(in_kernel_fpu, false);
 55 }
 56 
 57 static bool kernel_fpu_disabled(void)
 58 {
 59         return this_cpu_read(in_kernel_fpu);
 60 }
 61 
 62 static bool interrupted_kernel_fpu_idle(void)
 63 {
 64         return !kernel_fpu_disabled();
 65 }
 66 
 67 /*
 68  * Were we in user mode (or vm86 mode) when we were
 69  * interrupted?
 70  *
 71  * Doing kernel_fpu_begin/end() is ok if we are running
 72  * in an interrupt context from user mode - we'll just
 73  * save the FPU state as required.
 74  */
 75 static bool interrupted_user_mode(void)
 76 {
 77         struct pt_regs *regs = get_irq_regs();
 78         return regs && user_mode(regs);
 79 }
 80 
 81 /*
 82  * Can we use the FPU in kernel mode with the
 83  * whole "kernel_fpu_begin/end()" sequence?
 84  *
 85  * It's always ok in process context (ie "not interrupt")
 86  * but it is sometimes ok even from an irq.
 87  */
 88 bool irq_fpu_usable(void)
 89 {
 90         return !in_interrupt() ||
 91                 interrupted_user_mode() ||
 92                 interrupted_kernel_fpu_idle();
 93 }
 94 EXPORT_SYMBOL(irq_fpu_usable);
 95 
 96 void __kernel_fpu_begin(void)
 97 {
 98         struct fpu *fpu = &current->thread.fpu;
 99 
100         WARN_ON_FPU(!irq_fpu_usable());
101 
102         kernel_fpu_disable();
103 
104         if (fpu->initialized) {
105                 /*
106                  * Ignore return value -- we don't care if reg state
107                  * is clobbered.
108                  */
109                 copy_fpregs_to_fpstate(fpu);
110         } else {
111                 __cpu_invalidate_fpregs_state();
112         }
113 }
114 EXPORT_SYMBOL(__kernel_fpu_begin);
115 
116 void __kernel_fpu_end(void)
117 {
118         struct fpu *fpu = &current->thread.fpu;
119 
120         if (fpu->initialized)
121                 copy_kernel_to_fpregs(&fpu->state);
122 
123         kernel_fpu_enable();
124 }
125 EXPORT_SYMBOL(__kernel_fpu_end);
126 
127 void kernel_fpu_begin(void)
128 {
129         preempt_disable();
130         __kernel_fpu_begin();
131 }
132 EXPORT_SYMBOL_GPL(kernel_fpu_begin);
133 
134 void kernel_fpu_end(void)
135 {
136         __kernel_fpu_end();
137         preempt_enable();
138 }
139 EXPORT_SYMBOL_GPL(kernel_fpu_end);
140 
141 /*
142  * Save the FPU state (mark it for reload if necessary):
143  *
144  * This only ever gets called for the current task.
145  */
146 void fpu__save(struct fpu *fpu)
147 {
148         WARN_ON_FPU(fpu != &current->thread.fpu);
149 
150         preempt_disable();
151         trace_x86_fpu_before_save(fpu);
152         if (fpu->initialized) {
153                 if (!copy_fpregs_to_fpstate(fpu)) {
154                         copy_kernel_to_fpregs(&fpu->state);
155                 }
156         }
157         trace_x86_fpu_after_save(fpu);
158         preempt_enable();
159 }
160 EXPORT_SYMBOL_GPL(fpu__save);
161 
162 /*
163  * Legacy x87 fpstate state init:
164  */
165 static inline void fpstate_init_fstate(struct fregs_state *fp)
166 {
167         fp->cwd = 0xffff037fu;
168         fp->swd = 0xffff0000u;
169         fp->twd = 0xffffffffu;
170         fp->fos = 0xffff0000u;
171 }
172 
173 void fpstate_init(union fpregs_state *state)
174 {
175         if (!static_cpu_has(X86_FEATURE_FPU)) {
176                 fpstate_init_soft(&state->soft);
177                 return;
178         }
179 
180         memset(state, 0, fpu_kernel_xstate_size);
181 
182         if (static_cpu_has(X86_FEATURE_XSAVES))
183                 fpstate_init_xstate(&state->xsave);
184         if (static_cpu_has(X86_FEATURE_FXSR))
185                 fpstate_init_fxstate(&state->fxsave);
186         else
187                 fpstate_init_fstate(&state->fsave);
188 }
189 EXPORT_SYMBOL_GPL(fpstate_init);
190 
191 int fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu)
192 {
193         dst_fpu->last_cpu = -1;
194 
195         if (!src_fpu->initialized || !static_cpu_has(X86_FEATURE_FPU))
196                 return 0;
197 
198         WARN_ON_FPU(src_fpu != &current->thread.fpu);
199 
200         /*
201          * Don't let 'init optimized' areas of the XSAVE area
202          * leak into the child task:
203          */
204         memset(&dst_fpu->state.xsave, 0, fpu_kernel_xstate_size);
205 
206         /*
207          * Save current FPU registers directly into the child
208          * FPU context, without any memory-to-memory copying.
209          *
210          * ( The function 'fails' in the FNSAVE case, which destroys
211          *   register contents so we have to copy them back. )
212          */
213         if (!copy_fpregs_to_fpstate(dst_fpu)) {
214                 memcpy(&src_fpu->state, &dst_fpu->state, fpu_kernel_xstate_size);
215                 copy_kernel_to_fpregs(&src_fpu->state);
216         }
217 
218         trace_x86_fpu_copy_src(src_fpu);
219         trace_x86_fpu_copy_dst(dst_fpu);
220 
221         return 0;
222 }
223 
224 /*
225  * Activate the current task's in-memory FPU context,
226  * if it has not been used before:
227  */
228 void fpu__initialize(struct fpu *fpu)
229 {
230         WARN_ON_FPU(fpu != &current->thread.fpu);
231 
232         if (!fpu->initialized) {
233                 fpstate_init(&fpu->state);
234                 trace_x86_fpu_init_state(fpu);
235 
236                 trace_x86_fpu_activate_state(fpu);
237                 /* Safe to do for the current task: */
238                 fpu->initialized = 1;
239         }
240 }
241 EXPORT_SYMBOL_GPL(fpu__initialize);
242 
243 /*
244  * This function must be called before we read a task's fpstate.
245  *
246  * There's two cases where this gets called:
247  *
248  * - for the current task (when coredumping), in which case we have
249  *   to save the latest FPU registers into the fpstate,
250  *
251  * - or it's called for stopped tasks (ptrace), in which case the
252  *   registers were already saved by the context-switch code when
253  *   the task scheduled out - we only have to initialize the registers
254  *   if they've never been initialized.
255  *
256  * If the task has used the FPU before then save it.
257  */
258 void fpu__prepare_read(struct fpu *fpu)
259 {
260         if (fpu == &current->thread.fpu) {
261                 fpu__save(fpu);
262         } else {
263                 if (!fpu->initialized) {
264                         fpstate_init(&fpu->state);
265                         trace_x86_fpu_init_state(fpu);
266 
267                         trace_x86_fpu_activate_state(fpu);
268                         /* Safe to do for current and for stopped child tasks: */
269                         fpu->initialized = 1;
270                 }
271         }
272 }
273 
274 /*
275  * This function must be called before we write a task's fpstate.
276  *
277  * If the task has used the FPU before then invalidate any cached FPU registers.
278  * If the task has not used the FPU before then initialize its fpstate.
279  *
280  * After this function call, after registers in the fpstate are
281  * modified and the child task has woken up, the child task will
282  * restore the modified FPU state from the modified context. If we
283  * didn't clear its cached status here then the cached in-registers
284  * state pending on its former CPU could be restored, corrupting
285  * the modifications.
286  */
287 void fpu__prepare_write(struct fpu *fpu)
288 {
289         /*
290          * Only stopped child tasks can be used to modify the FPU
291          * state in the fpstate buffer:
292          */
293         WARN_ON_FPU(fpu == &current->thread.fpu);
294 
295         if (fpu->initialized) {
296                 /* Invalidate any cached state: */
297                 __fpu_invalidate_fpregs_state(fpu);
298         } else {
299                 fpstate_init(&fpu->state);
300                 trace_x86_fpu_init_state(fpu);
301 
302                 trace_x86_fpu_activate_state(fpu);
303                 /* Safe to do for stopped child tasks: */
304                 fpu->initialized = 1;
305         }
306 }
307 
308 /*
309  * 'fpu__restore()' is called to copy FPU registers from
310  * the FPU fpstate to the live hw registers and to activate
311  * access to the hardware registers, so that FPU instructions
312  * can be used afterwards.
313  *
314  * Must be called with kernel preemption disabled (for example
315  * with local interrupts disabled, as it is in the case of
316  * do_device_not_available()).
317  */
318 void fpu__restore(struct fpu *fpu)
319 {
320         fpu__initialize(fpu);
321 
322         /* Avoid __kernel_fpu_begin() right after fpregs_activate() */
323         kernel_fpu_disable();
324         trace_x86_fpu_before_restore(fpu);
325         fpregs_activate(fpu);
326         copy_kernel_to_fpregs(&fpu->state);
327         trace_x86_fpu_after_restore(fpu);
328         kernel_fpu_enable();
329 }
330 EXPORT_SYMBOL_GPL(fpu__restore);
331 
332 /*
333  * Drops current FPU state: deactivates the fpregs and
334  * the fpstate. NOTE: it still leaves previous contents
335  * in the fpregs in the eager-FPU case.
336  *
337  * This function can be used in cases where we know that
338  * a state-restore is coming: either an explicit one,
339  * or a reschedule.
340  */
341 void fpu__drop(struct fpu *fpu)
342 {
343         preempt_disable();
344 
345         if (fpu == &current->thread.fpu) {
346                 if (fpu->initialized) {
347                         /* Ignore delayed exceptions from user space */
348                         asm volatile("1: fwait\n"
349                                      "2:\n"
350                                      _ASM_EXTABLE(1b, 2b));
351                         fpregs_deactivate(fpu);
352                 }
353         }
354 
355         fpu->initialized = 0;
356 
357         trace_x86_fpu_dropped(fpu);
358 
359         preempt_enable();
360 }
361 
362 /*
363  * Clear FPU registers by setting them up from
364  * the init fpstate:
365  */
366 static inline void copy_init_fpstate_to_fpregs(void)
367 {
368         if (use_xsave())
369                 copy_kernel_to_xregs(&init_fpstate.xsave, -1);
370         else if (static_cpu_has(X86_FEATURE_FXSR))
371                 copy_kernel_to_fxregs(&init_fpstate.fxsave);
372         else
373                 copy_kernel_to_fregs(&init_fpstate.fsave);
374 
375         if (boot_cpu_has(X86_FEATURE_OSPKE))
376                 copy_init_pkru_to_fpregs();
377 }
378 
379 /*
380  * Clear the FPU state back to init state.
381  *
382  * Called by sys_execve(), by the signal handler code and by various
383  * error paths.
384  */
385 void fpu__clear(struct fpu *fpu)
386 {
387         WARN_ON_FPU(fpu != &current->thread.fpu); /* Almost certainly an anomaly */
388 
389         fpu__drop(fpu);
390 
391         /*
392          * Make sure fpstate is cleared and initialized.
393          */
394         if (static_cpu_has(X86_FEATURE_FPU)) {
395                 preempt_disable();
396                 fpu__initialize(fpu);
397                 user_fpu_begin();
398                 copy_init_fpstate_to_fpregs();
399                 preempt_enable();
400         }
401 }
402 
403 /*
404  * x87 math exception handling:
405  */
406 
407 int fpu__exception_code(struct fpu *fpu, int trap_nr)
408 {
409         int err;
410 
411         if (trap_nr == X86_TRAP_MF) {
412                 unsigned short cwd, swd;
413                 /*
414                  * (~cwd & swd) will mask out exceptions that are not set to unmasked
415                  * status.  0x3f is the exception bits in these regs, 0x200 is the
416                  * C1 reg you need in case of a stack fault, 0x040 is the stack
417                  * fault bit.  We should only be taking one exception at a time,
418                  * so if this combination doesn't produce any single exception,
419                  * then we have a bad program that isn't synchronizing its FPU usage
420                  * and it will suffer the consequences since we won't be able to
421                  * fully reproduce the context of the exception.
422                  */
423                 if (boot_cpu_has(X86_FEATURE_FXSR)) {
424                         cwd = fpu->state.fxsave.cwd;
425                         swd = fpu->state.fxsave.swd;
426                 } else {
427                         cwd = (unsigned short)fpu->state.fsave.cwd;
428                         swd = (unsigned short)fpu->state.fsave.swd;
429                 }
430 
431                 err = swd & ~cwd;
432         } else {
433                 /*
434                  * The SIMD FPU exceptions are handled a little differently, as there
435                  * is only a single status/control register.  Thus, to determine which
436                  * unmasked exception was caught we must mask the exception mask bits
437                  * at 0x1f80, and then use these to mask the exception bits at 0x3f.
438                  */
439                 unsigned short mxcsr = MXCSR_DEFAULT;
440 
441                 if (boot_cpu_has(X86_FEATURE_XMM))
442                         mxcsr = fpu->state.fxsave.mxcsr;
443 
444                 err = ~(mxcsr >> 7) & mxcsr;
445         }
446 
447         if (err & 0x001) {      /* Invalid op */
448                 /*
449                  * swd & 0x240 == 0x040: Stack Underflow
450                  * swd & 0x240 == 0x240: Stack Overflow
451                  * User must clear the SF bit (0x40) if set
452                  */
453                 return FPE_FLTINV;
454         } else if (err & 0x004) { /* Divide by Zero */
455                 return FPE_FLTDIV;
456         } else if (err & 0x008) { /* Overflow */
457                 return FPE_FLTOVF;
458         } else if (err & 0x012) { /* Denormal, Underflow */
459                 return FPE_FLTUND;
460         } else if (err & 0x020) { /* Precision */
461                 return FPE_FLTRES;
462         }
463 
464         /*
465          * If we're using IRQ 13, or supposedly even some trap
466          * X86_TRAP_MF implementations, it's possible
467          * we get a spurious trap, which is not an error.
468          */
469         return 0;
470 }
471 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp