~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/kernel/fpu/signal.c

Version: ~ [ linux-6.2-rc3 ] ~ [ linux-6.1.5 ] ~ [ linux-6.0.19 ] ~ [ linux-5.19.17 ] ~ [ linux-5.18.19 ] ~ [ linux-5.17.15 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.87 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.162 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.228 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.269 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.302 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.337 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.302 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 /*
  3  * FPU signal frame handling routines.
  4  */
  5 
  6 #include <linux/compat.h>
  7 #include <linux/cpu.h>
  8 #include <linux/pagemap.h>
  9 
 10 #include <asm/fpu/internal.h>
 11 #include <asm/fpu/signal.h>
 12 #include <asm/fpu/regset.h>
 13 #include <asm/fpu/xstate.h>
 14 
 15 #include <asm/sigframe.h>
 16 #include <asm/trace/fpu.h>
 17 
 18 static struct _fpx_sw_bytes fx_sw_reserved __ro_after_init;
 19 static struct _fpx_sw_bytes fx_sw_reserved_ia32 __ro_after_init;
 20 
 21 /*
 22  * Check for the presence of extended state information in the
 23  * user fpstate pointer in the sigcontext.
 24  */
 25 static inline int check_xstate_in_sigframe(struct fxregs_state __user *fxbuf,
 26                                            struct _fpx_sw_bytes *fx_sw)
 27 {
 28         int min_xstate_size = sizeof(struct fxregs_state) +
 29                               sizeof(struct xstate_header);
 30         void __user *fpstate = fxbuf;
 31         unsigned int magic2;
 32 
 33         if (__copy_from_user(fx_sw, &fxbuf->sw_reserved[0], sizeof(*fx_sw)))
 34                 return -EFAULT;
 35 
 36         /* Check for the first magic field and other error scenarios. */
 37         if (fx_sw->magic1 != FP_XSTATE_MAGIC1 ||
 38             fx_sw->xstate_size < min_xstate_size ||
 39             fx_sw->xstate_size > fpu_user_xstate_size ||
 40             fx_sw->xstate_size > fx_sw->extended_size)
 41                 goto setfx;
 42 
 43         /*
 44          * Check for the presence of second magic word at the end of memory
 45          * layout. This detects the case where the user just copied the legacy
 46          * fpstate layout with out copying the extended state information
 47          * in the memory layout.
 48          */
 49         if (__get_user(magic2, (__u32 __user *)(fpstate + fx_sw->xstate_size)))
 50                 return -EFAULT;
 51 
 52         if (likely(magic2 == FP_XSTATE_MAGIC2))
 53                 return 0;
 54 setfx:
 55         trace_x86_fpu_xstate_check_failed(&current->thread.fpu);
 56 
 57         /* Set the parameters for fx only state */
 58         fx_sw->magic1 = 0;
 59         fx_sw->xstate_size = sizeof(struct fxregs_state);
 60         fx_sw->xfeatures = XFEATURE_MASK_FPSSE;
 61         return 0;
 62 }
 63 
 64 /*
 65  * Signal frame handlers.
 66  */
 67 static inline int save_fsave_header(struct task_struct *tsk, void __user *buf)
 68 {
 69         if (use_fxsr()) {
 70                 struct xregs_state *xsave = &tsk->thread.fpu.state.xsave;
 71                 struct user_i387_ia32_struct env;
 72                 struct _fpstate_32 __user *fp = buf;
 73 
 74                 fpregs_lock();
 75                 if (!test_thread_flag(TIF_NEED_FPU_LOAD))
 76                         fxsave(&tsk->thread.fpu.state.fxsave);
 77                 fpregs_unlock();
 78 
 79                 convert_from_fxsr(&env, tsk);
 80 
 81                 if (__copy_to_user(buf, &env, sizeof(env)) ||
 82                     __put_user(xsave->i387.swd, &fp->status) ||
 83                     __put_user(X86_FXSR_MAGIC, &fp->magic))
 84                         return -1;
 85         } else {
 86                 struct fregs_state __user *fp = buf;
 87                 u32 swd;
 88                 if (__get_user(swd, &fp->swd) || __put_user(swd, &fp->status))
 89                         return -1;
 90         }
 91 
 92         return 0;
 93 }
 94 
 95 static inline int save_xstate_epilog(void __user *buf, int ia32_frame)
 96 {
 97         struct xregs_state __user *x = buf;
 98         struct _fpx_sw_bytes *sw_bytes;
 99         u32 xfeatures;
100         int err;
101 
102         /* Setup the bytes not touched by the [f]xsave and reserved for SW. */
103         sw_bytes = ia32_frame ? &fx_sw_reserved_ia32 : &fx_sw_reserved;
104         err = __copy_to_user(&x->i387.sw_reserved, sw_bytes, sizeof(*sw_bytes));
105 
106         if (!use_xsave())
107                 return err;
108 
109         err |= __put_user(FP_XSTATE_MAGIC2,
110                           (__u32 __user *)(buf + fpu_user_xstate_size));
111 
112         /*
113          * Read the xfeatures which we copied (directly from the cpu or
114          * from the state in task struct) to the user buffers.
115          */
116         err |= __get_user(xfeatures, (__u32 __user *)&x->header.xfeatures);
117 
118         /*
119          * For legacy compatible, we always set FP/SSE bits in the bit
120          * vector while saving the state to the user context. This will
121          * enable us capturing any changes(during sigreturn) to
122          * the FP/SSE bits by the legacy applications which don't touch
123          * xfeatures in the xsave header.
124          *
125          * xsave aware apps can change the xfeatures in the xsave
126          * header as well as change any contents in the memory layout.
127          * xrestore as part of sigreturn will capture all the changes.
128          */
129         xfeatures |= XFEATURE_MASK_FPSSE;
130 
131         err |= __put_user(xfeatures, (__u32 __user *)&x->header.xfeatures);
132 
133         return err;
134 }
135 
136 static inline int copy_fpregs_to_sigframe(struct xregs_state __user *buf)
137 {
138         int err;
139 
140         if (use_xsave())
141                 err = xsave_to_user_sigframe(buf);
142         else if (use_fxsr())
143                 err = fxsave_to_user_sigframe((struct fxregs_state __user *) buf);
144         else
145                 err = fnsave_to_user_sigframe((struct fregs_state __user *) buf);
146 
147         if (unlikely(err) && __clear_user(buf, fpu_user_xstate_size))
148                 err = -EFAULT;
149         return err;
150 }
151 
152 /*
153  * Save the fpu, extended register state to the user signal frame.
154  *
155  * 'buf_fx' is the 64-byte aligned pointer at which the [f|fx|x]save
156  *  state is copied.
157  *  'buf' points to the 'buf_fx' or to the fsave header followed by 'buf_fx'.
158  *
159  *      buf == buf_fx for 64-bit frames and 32-bit fsave frame.
160  *      buf != buf_fx for 32-bit frames with fxstate.
161  *
162  * Try to save it directly to the user frame with disabled page fault handler.
163  * If this fails then do the slow path where the FPU state is first saved to
164  * task's fpu->state and then copy it to the user frame pointed to by the
165  * aligned pointer 'buf_fx'.
166  *
167  * If this is a 32-bit frame with fxstate, put a fsave header before
168  * the aligned state at 'buf_fx'.
169  *
170  * For [f]xsave state, update the SW reserved fields in the [f]xsave frame
171  * indicating the absence/presence of the extended state to the user.
172  */
173 int copy_fpstate_to_sigframe(void __user *buf, void __user *buf_fx, int size)
174 {
175         struct task_struct *tsk = current;
176         int ia32_fxstate = (buf != buf_fx);
177         int ret;
178 
179         ia32_fxstate &= (IS_ENABLED(CONFIG_X86_32) ||
180                          IS_ENABLED(CONFIG_IA32_EMULATION));
181 
182         if (!static_cpu_has(X86_FEATURE_FPU)) {
183                 struct user_i387_ia32_struct fp;
184                 fpregs_soft_get(current, NULL, (struct membuf){.p = &fp,
185                                                 .left = sizeof(fp)});
186                 return copy_to_user(buf, &fp, sizeof(fp)) ? -EFAULT : 0;
187         }
188 
189         if (!access_ok(buf, size))
190                 return -EACCES;
191 retry:
192         /*
193          * Load the FPU registers if they are not valid for the current task.
194          * With a valid FPU state we can attempt to save the state directly to
195          * userland's stack frame which will likely succeed. If it does not,
196          * resolve the fault in the user memory and try again.
197          */
198         fpregs_lock();
199         if (test_thread_flag(TIF_NEED_FPU_LOAD))
200                 fpregs_restore_userregs();
201 
202         pagefault_disable();
203         ret = copy_fpregs_to_sigframe(buf_fx);
204         pagefault_enable();
205         fpregs_unlock();
206 
207         if (ret) {
208                 if (!fault_in_pages_writeable(buf_fx, fpu_user_xstate_size))
209                         goto retry;
210                 return -EFAULT;
211         }
212 
213         /* Save the fsave header for the 32-bit frames. */
214         if ((ia32_fxstate || !use_fxsr()) && save_fsave_header(tsk, buf))
215                 return -1;
216 
217         if (use_fxsr() && save_xstate_epilog(buf_fx, ia32_fxstate))
218                 return -1;
219 
220         return 0;
221 }
222 
223 static int __restore_fpregs_from_user(void __user *buf, u64 xrestore,
224                                       bool fx_only)
225 {
226         if (use_xsave()) {
227                 u64 init_bv = xfeatures_mask_uabi() & ~xrestore;
228                 int ret;
229 
230                 if (likely(!fx_only))
231                         ret = xrstor_from_user_sigframe(buf, xrestore);
232                 else
233                         ret = fxrstor_from_user_sigframe(buf);
234 
235                 if (!ret && unlikely(init_bv))
236                         os_xrstor(&init_fpstate.xsave, init_bv);
237                 return ret;
238         } else if (use_fxsr()) {
239                 return fxrstor_from_user_sigframe(buf);
240         } else {
241                 return frstor_from_user_sigframe(buf);
242         }
243 }
244 
245 /*
246  * Attempt to restore the FPU registers directly from user memory.
247  * Pagefaults are handled and any errors returned are fatal.
248  */
249 static int restore_fpregs_from_user(void __user *buf, u64 xrestore,
250                                     bool fx_only, unsigned int size)
251 {
252         struct fpu *fpu = &current->thread.fpu;
253         int ret;
254 
255 retry:
256         fpregs_lock();
257         pagefault_disable();
258         ret = __restore_fpregs_from_user(buf, xrestore, fx_only);
259         pagefault_enable();
260 
261         if (unlikely(ret)) {
262                 /*
263                  * The above did an FPU restore operation, restricted to
264                  * the user portion of the registers, and failed, but the
265                  * microcode might have modified the FPU registers
266                  * nevertheless.
267                  *
268                  * If the FPU registers do not belong to current, then
269                  * invalidate the FPU register state otherwise the task
270                  * might preempt current and return to user space with
271                  * corrupted FPU registers.
272                  */
273                 if (test_thread_flag(TIF_NEED_FPU_LOAD))
274                         __cpu_invalidate_fpregs_state();
275                 fpregs_unlock();
276 
277                 /* Try to handle #PF, but anything else is fatal. */
278                 if (ret != -EFAULT)
279                         return -EINVAL;
280 
281                 ret = fault_in_pages_readable(buf, size);
282                 if (!ret)
283                         goto retry;
284                 return ret;
285         }
286 
287         /*
288          * Restore supervisor states: previous context switch etc has done
289          * XSAVES and saved the supervisor states in the kernel buffer from
290          * which they can be restored now.
291          *
292          * It would be optimal to handle this with a single XRSTORS, but
293          * this does not work because the rest of the FPU registers have
294          * been restored from a user buffer directly.
295          */
296         if (test_thread_flag(TIF_NEED_FPU_LOAD) && xfeatures_mask_supervisor())
297                 os_xrstor(&fpu->state.xsave, xfeatures_mask_supervisor());
298 
299         fpregs_mark_activate();
300         fpregs_unlock();
301         return 0;
302 }
303 
304 static int __fpu_restore_sig(void __user *buf, void __user *buf_fx,
305                              bool ia32_fxstate)
306 {
307         int state_size = fpu_kernel_xstate_size;
308         struct task_struct *tsk = current;
309         struct fpu *fpu = &tsk->thread.fpu;
310         struct user_i387_ia32_struct env;
311         u64 user_xfeatures = 0;
312         bool fx_only = false;
313         int ret;
314 
315         if (use_xsave()) {
316                 struct _fpx_sw_bytes fx_sw_user;
317 
318                 ret = check_xstate_in_sigframe(buf_fx, &fx_sw_user);
319                 if (unlikely(ret))
320                         return ret;
321 
322                 fx_only = !fx_sw_user.magic1;
323                 state_size = fx_sw_user.xstate_size;
324                 user_xfeatures = fx_sw_user.xfeatures;
325         } else {
326                 user_xfeatures = XFEATURE_MASK_FPSSE;
327         }
328 
329         if (likely(!ia32_fxstate)) {
330                 /*
331                  * Attempt to restore the FPU registers directly from user
332                  * memory. For that to succeed, the user access cannot cause page
333                  * faults. If it does, fall back to the slow path below, going
334                  * through the kernel buffer with the enabled pagefault handler.
335                  */
336                 return restore_fpregs_from_user(buf_fx, user_xfeatures, fx_only,
337                                                 state_size);
338         }
339 
340         /*
341          * Copy the legacy state because the FP portion of the FX frame has
342          * to be ignored for histerical raisins. The legacy state is folded
343          * in once the larger state has been copied.
344          */
345         ret = __copy_from_user(&env, buf, sizeof(env));
346         if (ret)
347                 return ret;
348 
349         /*
350          * By setting TIF_NEED_FPU_LOAD it is ensured that our xstate is
351          * not modified on context switch and that the xstate is considered
352          * to be loaded again on return to userland (overriding last_cpu avoids
353          * the optimisation).
354          */
355         fpregs_lock();
356         if (!test_thread_flag(TIF_NEED_FPU_LOAD)) {
357                 /*
358                  * If supervisor states are available then save the
359                  * hardware state in current's fpstate so that the
360                  * supervisor state is preserved. Save the full state for
361                  * simplicity. There is no point in optimizing this by only
362                  * saving the supervisor states and then shuffle them to
363                  * the right place in memory. It's ia32 mode. Shrug.
364                  */
365                 if (xfeatures_mask_supervisor())
366                         os_xsave(&fpu->state.xsave);
367                 set_thread_flag(TIF_NEED_FPU_LOAD);
368         }
369         __fpu_invalidate_fpregs_state(fpu);
370         __cpu_invalidate_fpregs_state();
371         fpregs_unlock();
372 
373         if (use_xsave() && !fx_only) {
374                 ret = copy_sigframe_from_user_to_xstate(&fpu->state.xsave, buf_fx);
375                 if (ret)
376                         return ret;
377         } else {
378                 if (__copy_from_user(&fpu->state.fxsave, buf_fx,
379                                      sizeof(fpu->state.fxsave)))
380                         return -EFAULT;
381 
382                 if (IS_ENABLED(CONFIG_X86_64)) {
383                         /* Reject invalid MXCSR values. */
384                         if (fpu->state.fxsave.mxcsr & ~mxcsr_feature_mask)
385                                 return -EINVAL;
386                 } else {
387                         /* Mask invalid bits out for historical reasons (broken hardware). */
388                         fpu->state.fxsave.mxcsr &= mxcsr_feature_mask;
389                 }
390 
391                 /* Enforce XFEATURE_MASK_FPSSE when XSAVE is enabled */
392                 if (use_xsave())
393                         fpu->state.xsave.header.xfeatures |= XFEATURE_MASK_FPSSE;
394         }
395 
396         /* Fold the legacy FP storage */
397         convert_to_fxsr(&fpu->state.fxsave, &env);
398 
399         fpregs_lock();
400         if (use_xsave()) {
401                 /*
402                  * Remove all UABI feature bits not set in user_xfeatures
403                  * from the memory xstate header which makes the full
404                  * restore below bring them into init state. This works for
405                  * fx_only mode as well because that has only FP and SSE
406                  * set in user_xfeatures.
407                  *
408                  * Preserve supervisor states!
409                  */
410                 u64 mask = user_xfeatures | xfeatures_mask_supervisor();
411 
412                 fpu->state.xsave.header.xfeatures &= mask;
413                 ret = os_xrstor_safe(&fpu->state.xsave, xfeatures_mask_all);
414         } else {
415                 ret = fxrstor_safe(&fpu->state.fxsave);
416         }
417 
418         if (likely(!ret))
419                 fpregs_mark_activate();
420 
421         fpregs_unlock();
422         return ret;
423 }
424 static inline int xstate_sigframe_size(void)
425 {
426         return use_xsave() ? fpu_user_xstate_size + FP_XSTATE_MAGIC2_SIZE :
427                         fpu_user_xstate_size;
428 }
429 
430 /*
431  * Restore FPU state from a sigframe:
432  */
433 int fpu__restore_sig(void __user *buf, int ia32_frame)
434 {
435         unsigned int size = xstate_sigframe_size();
436         struct fpu *fpu = &current->thread.fpu;
437         void __user *buf_fx = buf;
438         bool ia32_fxstate = false;
439         int ret;
440 
441         if (unlikely(!buf)) {
442                 fpu__clear_user_states(fpu);
443                 return 0;
444         }
445 
446         ia32_frame &= (IS_ENABLED(CONFIG_X86_32) ||
447                        IS_ENABLED(CONFIG_IA32_EMULATION));
448 
449         /*
450          * Only FXSR enabled systems need the FX state quirk.
451          * FRSTOR does not need it and can use the fast path.
452          */
453         if (ia32_frame && use_fxsr()) {
454                 buf_fx = buf + sizeof(struct fregs_state);
455                 size += sizeof(struct fregs_state);
456                 ia32_fxstate = true;
457         }
458 
459         if (!access_ok(buf, size)) {
460                 ret = -EACCES;
461                 goto out;
462         }
463 
464         if (!IS_ENABLED(CONFIG_X86_64) && !cpu_feature_enabled(X86_FEATURE_FPU)) {
465                 ret = fpregs_soft_set(current, NULL, 0,
466                                       sizeof(struct user_i387_ia32_struct),
467                                       NULL, buf);
468         } else {
469                 ret = __fpu_restore_sig(buf, buf_fx, ia32_fxstate);
470         }
471 
472 out:
473         if (unlikely(ret))
474                 fpu__clear_user_states(fpu);
475         return ret;
476 }
477 
478 unsigned long
479 fpu__alloc_mathframe(unsigned long sp, int ia32_frame,
480                      unsigned long *buf_fx, unsigned long *size)
481 {
482         unsigned long frame_size = xstate_sigframe_size();
483 
484         *buf_fx = sp = round_down(sp - frame_size, 64);
485         if (ia32_frame && use_fxsr()) {
486                 frame_size += sizeof(struct fregs_state);
487                 sp -= sizeof(struct fregs_state);
488         }
489 
490         *size = frame_size;
491 
492         return sp;
493 }
494 
495 unsigned long fpu__get_fpstate_size(void)
496 {
497         unsigned long ret = xstate_sigframe_size();
498 
499         /*
500          * This space is needed on (most) 32-bit kernels, or when a 32-bit
501          * app is running on a 64-bit kernel. To keep things simple, just
502          * assume the worst case and always include space for 'freg_state',
503          * even for 64-bit apps on 64-bit kernels. This wastes a bit of
504          * space, but keeps the code simple.
505          */
506         if ((IS_ENABLED(CONFIG_IA32_EMULATION) ||
507              IS_ENABLED(CONFIG_X86_32)) && use_fxsr())
508                 ret += sizeof(struct fregs_state);
509 
510         return ret;
511 }
512 
513 /*
514  * Prepare the SW reserved portion of the fxsave memory layout, indicating
515  * the presence of the extended state information in the memory layout
516  * pointed by the fpstate pointer in the sigcontext.
517  * This will be saved when ever the FP and extended state context is
518  * saved on the user stack during the signal handler delivery to the user.
519  */
520 void fpu__init_prepare_fx_sw_frame(void)
521 {
522         int size = fpu_user_xstate_size + FP_XSTATE_MAGIC2_SIZE;
523 
524         fx_sw_reserved.magic1 = FP_XSTATE_MAGIC1;
525         fx_sw_reserved.extended_size = size;
526         fx_sw_reserved.xfeatures = xfeatures_mask_uabi();
527         fx_sw_reserved.xstate_size = fpu_user_xstate_size;
528 
529         if (IS_ENABLED(CONFIG_IA32_EMULATION) ||
530             IS_ENABLED(CONFIG_X86_32)) {
531                 int fsave_header_size = sizeof(struct fregs_state);
532 
533                 fx_sw_reserved_ia32 = fx_sw_reserved;
534                 fx_sw_reserved_ia32.extended_size = size + fsave_header_size;
535         }
536 }
537 
538 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp