~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/entry/common.c

Version: ~ [ linux-5.1-rc5 ] ~ [ linux-5.0.7 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.34 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.111 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.168 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.178 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.138 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.65 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * common.c - C code for kernel entry and exit
  3  * Copyright (c) 2015 Andrew Lutomirski
  4  * GPL v2
  5  *
  6  * Based on asm and ptrace code by many authors.  The code here originated
  7  * in ptrace.c and signal.c.
  8  */
  9 
 10 #include <linux/kernel.h>
 11 #include <linux/sched.h>
 12 #include <linux/sched/task_stack.h>
 13 #include <linux/mm.h>
 14 #include <linux/smp.h>
 15 #include <linux/errno.h>
 16 #include <linux/ptrace.h>
 17 #include <linux/tracehook.h>
 18 #include <linux/audit.h>
 19 #include <linux/seccomp.h>
 20 #include <linux/signal.h>
 21 #include <linux/export.h>
 22 #include <linux/context_tracking.h>
 23 #include <linux/user-return-notifier.h>
 24 #include <linux/nospec.h>
 25 #include <linux/uprobes.h>
 26 #include <linux/livepatch.h>
 27 #include <linux/syscalls.h>
 28 
 29 #include <asm/desc.h>
 30 #include <asm/traps.h>
 31 #include <asm/vdso.h>
 32 #include <linux/uaccess.h>
 33 #include <asm/cpufeature.h>
 34 
 35 #define CREATE_TRACE_POINTS
 36 #include <trace/events/syscalls.h>
 37 
 38 #ifdef CONFIG_CONTEXT_TRACKING
 39 /* Called on entry from user mode with IRQs off. */
 40 __visible inline void enter_from_user_mode(void)
 41 {
 42         CT_WARN_ON(ct_state() != CONTEXT_USER);
 43         user_exit_irqoff();
 44 }
 45 #else
 46 static inline void enter_from_user_mode(void) {}
 47 #endif
 48 
 49 static void do_audit_syscall_entry(struct pt_regs *regs, u32 arch)
 50 {
 51 #ifdef CONFIG_X86_64
 52         if (arch == AUDIT_ARCH_X86_64) {
 53                 audit_syscall_entry(regs->orig_ax, regs->di,
 54                                     regs->si, regs->dx, regs->r10);
 55         } else
 56 #endif
 57         {
 58                 audit_syscall_entry(regs->orig_ax, regs->bx,
 59                                     regs->cx, regs->dx, regs->si);
 60         }
 61 }
 62 
 63 /*
 64  * Returns the syscall nr to run (which should match regs->orig_ax) or -1
 65  * to skip the syscall.
 66  */
 67 static long syscall_trace_enter(struct pt_regs *regs)
 68 {
 69         u32 arch = in_ia32_syscall() ? AUDIT_ARCH_I386 : AUDIT_ARCH_X86_64;
 70 
 71         struct thread_info *ti = current_thread_info();
 72         unsigned long ret = 0;
 73         bool emulated = false;
 74         u32 work;
 75 
 76         if (IS_ENABLED(CONFIG_DEBUG_ENTRY))
 77                 BUG_ON(regs != task_pt_regs(current));
 78 
 79         work = READ_ONCE(ti->flags) & _TIF_WORK_SYSCALL_ENTRY;
 80 
 81         if (unlikely(work & _TIF_SYSCALL_EMU))
 82                 emulated = true;
 83 
 84         if ((emulated || (work & _TIF_SYSCALL_TRACE)) &&
 85             tracehook_report_syscall_entry(regs))
 86                 return -1L;
 87 
 88         if (emulated)
 89                 return -1L;
 90 
 91 #ifdef CONFIG_SECCOMP
 92         /*
 93          * Do seccomp after ptrace, to catch any tracer changes.
 94          */
 95         if (work & _TIF_SECCOMP) {
 96                 struct seccomp_data sd;
 97 
 98                 sd.arch = arch;
 99                 sd.nr = regs->orig_ax;
100                 sd.instruction_pointer = regs->ip;
101 #ifdef CONFIG_X86_64
102                 if (arch == AUDIT_ARCH_X86_64) {
103                         sd.args[0] = regs->di;
104                         sd.args[1] = regs->si;
105                         sd.args[2] = regs->dx;
106                         sd.args[3] = regs->r10;
107                         sd.args[4] = regs->r8;
108                         sd.args[5] = regs->r9;
109                 } else
110 #endif
111                 {
112                         sd.args[0] = regs->bx;
113                         sd.args[1] = regs->cx;
114                         sd.args[2] = regs->dx;
115                         sd.args[3] = regs->si;
116                         sd.args[4] = regs->di;
117                         sd.args[5] = regs->bp;
118                 }
119 
120                 ret = __secure_computing(&sd);
121                 if (ret == -1)
122                         return ret;
123         }
124 #endif
125 
126         if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT)))
127                 trace_sys_enter(regs, regs->orig_ax);
128 
129         do_audit_syscall_entry(regs, arch);
130 
131         return ret ?: regs->orig_ax;
132 }
133 
134 #define EXIT_TO_USERMODE_LOOP_FLAGS                             \
135         (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_UPROBE |   \
136          _TIF_NEED_RESCHED | _TIF_USER_RETURN_NOTIFY | _TIF_PATCH_PENDING)
137 
138 static void exit_to_usermode_loop(struct pt_regs *regs, u32 cached_flags)
139 {
140         /*
141          * In order to return to user mode, we need to have IRQs off with
142          * none of EXIT_TO_USERMODE_LOOP_FLAGS set.  Several of these flags
143          * can be set at any time on preemptible kernels if we have IRQs on,
144          * so we need to loop.  Disabling preemption wouldn't help: doing the
145          * work to clear some of the flags can sleep.
146          */
147         while (true) {
148                 /* We have work to do. */
149                 local_irq_enable();
150 
151                 if (cached_flags & _TIF_NEED_RESCHED)
152                         schedule();
153 
154                 if (cached_flags & _TIF_UPROBE)
155                         uprobe_notify_resume(regs);
156 
157                 if (cached_flags & _TIF_PATCH_PENDING)
158                         klp_update_patch_state(current);
159 
160                 /* deal with pending signal delivery */
161                 if (cached_flags & _TIF_SIGPENDING)
162                         do_signal(regs);
163 
164                 if (cached_flags & _TIF_NOTIFY_RESUME) {
165                         clear_thread_flag(TIF_NOTIFY_RESUME);
166                         tracehook_notify_resume(regs);
167                         rseq_handle_notify_resume(NULL, regs);
168                 }
169 
170                 if (cached_flags & _TIF_USER_RETURN_NOTIFY)
171                         fire_user_return_notifiers();
172 
173                 /* Disable IRQs and retry */
174                 local_irq_disable();
175 
176                 cached_flags = READ_ONCE(current_thread_info()->flags);
177 
178                 if (!(cached_flags & EXIT_TO_USERMODE_LOOP_FLAGS))
179                         break;
180         }
181 }
182 
183 /* Called with IRQs disabled. */
184 __visible inline void prepare_exit_to_usermode(struct pt_regs *regs)
185 {
186         struct thread_info *ti = current_thread_info();
187         u32 cached_flags;
188 
189         addr_limit_user_check();
190 
191         lockdep_assert_irqs_disabled();
192         lockdep_sys_exit();
193 
194         cached_flags = READ_ONCE(ti->flags);
195 
196         if (unlikely(cached_flags & EXIT_TO_USERMODE_LOOP_FLAGS))
197                 exit_to_usermode_loop(regs, cached_flags);
198 
199 #ifdef CONFIG_COMPAT
200         /*
201          * Compat syscalls set TS_COMPAT.  Make sure we clear it before
202          * returning to user mode.  We need to clear it *after* signal
203          * handling, because syscall restart has a fixup for compat
204          * syscalls.  The fixup is exercised by the ptrace_syscall_32
205          * selftest.
206          *
207          * We also need to clear TS_REGS_POKED_I386: the 32-bit tracer
208          * special case only applies after poking regs and before the
209          * very next return to user mode.
210          */
211         ti->status &= ~(TS_COMPAT|TS_I386_REGS_POKED);
212 #endif
213 
214         user_enter_irqoff();
215 }
216 
217 #define SYSCALL_EXIT_WORK_FLAGS                         \
218         (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT |      \
219          _TIF_SINGLESTEP | _TIF_SYSCALL_TRACEPOINT)
220 
221 static void syscall_slow_exit_work(struct pt_regs *regs, u32 cached_flags)
222 {
223         bool step;
224 
225         audit_syscall_exit(regs);
226 
227         if (cached_flags & _TIF_SYSCALL_TRACEPOINT)
228                 trace_sys_exit(regs, regs->ax);
229 
230         /*
231          * If TIF_SYSCALL_EMU is set, we only get here because of
232          * TIF_SINGLESTEP (i.e. this is PTRACE_SYSEMU_SINGLESTEP).
233          * We already reported this syscall instruction in
234          * syscall_trace_enter().
235          */
236         step = unlikely(
237                 (cached_flags & (_TIF_SINGLESTEP | _TIF_SYSCALL_EMU))
238                 == _TIF_SINGLESTEP);
239         if (step || cached_flags & _TIF_SYSCALL_TRACE)
240                 tracehook_report_syscall_exit(regs, step);
241 }
242 
243 /*
244  * Called with IRQs on and fully valid regs.  Returns with IRQs off in a
245  * state such that we can immediately switch to user mode.
246  */
247 __visible inline void syscall_return_slowpath(struct pt_regs *regs)
248 {
249         struct thread_info *ti = current_thread_info();
250         u32 cached_flags = READ_ONCE(ti->flags);
251 
252         CT_WARN_ON(ct_state() != CONTEXT_KERNEL);
253 
254         if (IS_ENABLED(CONFIG_PROVE_LOCKING) &&
255             WARN(irqs_disabled(), "syscall %ld left IRQs disabled", regs->orig_ax))
256                 local_irq_enable();
257 
258         rseq_syscall(regs);
259 
260         /*
261          * First do one-time work.  If these work items are enabled, we
262          * want to run them exactly once per syscall exit with IRQs on.
263          */
264         if (unlikely(cached_flags & SYSCALL_EXIT_WORK_FLAGS))
265                 syscall_slow_exit_work(regs, cached_flags);
266 
267         local_irq_disable();
268         prepare_exit_to_usermode(regs);
269 }
270 
271 #ifdef CONFIG_X86_64
272 __visible void do_syscall_64(unsigned long nr, struct pt_regs *regs)
273 {
274         struct thread_info *ti;
275 
276         enter_from_user_mode();
277         local_irq_enable();
278         ti = current_thread_info();
279         if (READ_ONCE(ti->flags) & _TIF_WORK_SYSCALL_ENTRY)
280                 nr = syscall_trace_enter(regs);
281 
282         /*
283          * NB: Native and x32 syscalls are dispatched from the same
284          * table.  The only functional difference is the x32 bit in
285          * regs->orig_ax, which changes the behavior of some syscalls.
286          */
287         nr &= __SYSCALL_MASK;
288         if (likely(nr < NR_syscalls)) {
289                 nr = array_index_nospec(nr, NR_syscalls);
290                 regs->ax = sys_call_table[nr](regs);
291         }
292 
293         syscall_return_slowpath(regs);
294 }
295 #endif
296 
297 #if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION)
298 /*
299  * Does a 32-bit syscall.  Called with IRQs on in CONTEXT_KERNEL.  Does
300  * all entry and exit work and returns with IRQs off.  This function is
301  * extremely hot in workloads that use it, and it's usually called from
302  * do_fast_syscall_32, so forcibly inline it to improve performance.
303  */
304 static __always_inline void do_syscall_32_irqs_on(struct pt_regs *regs)
305 {
306         struct thread_info *ti = current_thread_info();
307         unsigned int nr = (unsigned int)regs->orig_ax;
308 
309 #ifdef CONFIG_IA32_EMULATION
310         ti->status |= TS_COMPAT;
311 #endif
312 
313         if (READ_ONCE(ti->flags) & _TIF_WORK_SYSCALL_ENTRY) {
314                 /*
315                  * Subtlety here: if ptrace pokes something larger than
316                  * 2^32-1 into orig_ax, this truncates it.  This may or
317                  * may not be necessary, but it matches the old asm
318                  * behavior.
319                  */
320                 nr = syscall_trace_enter(regs);
321         }
322 
323         if (likely(nr < IA32_NR_syscalls)) {
324                 nr = array_index_nospec(nr, IA32_NR_syscalls);
325 #ifdef CONFIG_IA32_EMULATION
326                 regs->ax = ia32_sys_call_table[nr](regs);
327 #else
328                 /*
329                  * It's possible that a 32-bit syscall implementation
330                  * takes a 64-bit parameter but nonetheless assumes that
331                  * the high bits are zero.  Make sure we zero-extend all
332                  * of the args.
333                  */
334                 regs->ax = ia32_sys_call_table[nr](
335                         (unsigned int)regs->bx, (unsigned int)regs->cx,
336                         (unsigned int)regs->dx, (unsigned int)regs->si,
337                         (unsigned int)regs->di, (unsigned int)regs->bp);
338 #endif /* CONFIG_IA32_EMULATION */
339         }
340 
341         syscall_return_slowpath(regs);
342 }
343 
344 /* Handles int $0x80 */
345 __visible void do_int80_syscall_32(struct pt_regs *regs)
346 {
347         enter_from_user_mode();
348         local_irq_enable();
349         do_syscall_32_irqs_on(regs);
350 }
351 
352 /* Returns 0 to return using IRET or 1 to return using SYSEXIT/SYSRETL. */
353 __visible long do_fast_syscall_32(struct pt_regs *regs)
354 {
355         /*
356          * Called using the internal vDSO SYSENTER/SYSCALL32 calling
357          * convention.  Adjust regs so it looks like we entered using int80.
358          */
359 
360         unsigned long landing_pad = (unsigned long)current->mm->context.vdso +
361                 vdso_image_32.sym_int80_landing_pad;
362 
363         /*
364          * SYSENTER loses EIP, and even SYSCALL32 needs us to skip forward
365          * so that 'regs->ip -= 2' lands back on an int $0x80 instruction.
366          * Fix it up.
367          */
368         regs->ip = landing_pad;
369 
370         enter_from_user_mode();
371 
372         local_irq_enable();
373 
374         /* Fetch EBP from where the vDSO stashed it. */
375         if (
376 #ifdef CONFIG_X86_64
377                 /*
378                  * Micro-optimization: the pointer we're following is explicitly
379                  * 32 bits, so it can't be out of range.
380                  */
381                 __get_user(*(u32 *)&regs->bp,
382                             (u32 __user __force *)(unsigned long)(u32)regs->sp)
383 #else
384                 get_user(*(u32 *)&regs->bp,
385                          (u32 __user __force *)(unsigned long)(u32)regs->sp)
386 #endif
387                 ) {
388 
389                 /* User code screwed up. */
390                 local_irq_disable();
391                 regs->ax = -EFAULT;
392                 prepare_exit_to_usermode(regs);
393                 return 0;       /* Keep it simple: use IRET. */
394         }
395 
396         /* Now this is just like a normal syscall. */
397         do_syscall_32_irqs_on(regs);
398 
399 #ifdef CONFIG_X86_64
400         /*
401          * Opportunistic SYSRETL: if possible, try to return using SYSRETL.
402          * SYSRETL is available on all 64-bit CPUs, so we don't need to
403          * bother with SYSEXIT.
404          *
405          * Unlike 64-bit opportunistic SYSRET, we can't check that CX == IP,
406          * because the ECX fixup above will ensure that this is essentially
407          * never the case.
408          */
409         return regs->cs == __USER32_CS && regs->ss == __USER_DS &&
410                 regs->ip == landing_pad &&
411                 (regs->flags & (X86_EFLAGS_RF | X86_EFLAGS_TF)) == 0;
412 #else
413         /*
414          * Opportunistic SYSEXIT: if possible, try to return using SYSEXIT.
415          *
416          * Unlike 64-bit opportunistic SYSRET, we can't check that CX == IP,
417          * because the ECX fixup above will ensure that this is essentially
418          * never the case.
419          *
420          * We don't allow syscalls at all from VM86 mode, but we still
421          * need to check VM, because we might be returning from sys_vm86.
422          */
423         return static_cpu_has(X86_FEATURE_SEP) &&
424                 regs->cs == __USER_CS && regs->ss == __USER_DS &&
425                 regs->ip == landing_pad &&
426                 (regs->flags & (X86_EFLAGS_RF | X86_EFLAGS_TF | X86_EFLAGS_VM)) == 0;
427 #endif
428 }
429 #endif
430 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp