~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/mm/mpx.c

Version: ~ [ linux-5.12-rc1 ] ~ [ linux-5.11.2 ] ~ [ linux-5.10.19 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.101 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.177 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.222 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.258 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.258 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * mpx.c - Memory Protection eXtensions
  3  *
  4  * Copyright (c) 2014, Intel Corporation.
  5  * Qiaowei Ren <qiaowei.ren@intel.com>
  6  * Dave Hansen <dave.hansen@intel.com>
  7  */
  8 #include <linux/kernel.h>
  9 #include <linux/slab.h>
 10 #include <linux/syscalls.h>
 11 #include <linux/sched/sysctl.h>
 12 
 13 #include <asm/insn.h>
 14 #include <asm/mman.h>
 15 #include <asm/mmu_context.h>
 16 #include <asm/mpx.h>
 17 #include <asm/processor.h>
 18 #include <asm/fpu/internal.h>
 19 
 20 #define CREATE_TRACE_POINTS
 21 #include <asm/trace/mpx.h>
 22 
 23 static inline unsigned long mpx_bd_size_bytes(struct mm_struct *mm)
 24 {
 25         if (is_64bit_mm(mm))
 26                 return MPX_BD_SIZE_BYTES_64;
 27         else
 28                 return MPX_BD_SIZE_BYTES_32;
 29 }
 30 
 31 static inline unsigned long mpx_bt_size_bytes(struct mm_struct *mm)
 32 {
 33         if (is_64bit_mm(mm))
 34                 return MPX_BT_SIZE_BYTES_64;
 35         else
 36                 return MPX_BT_SIZE_BYTES_32;
 37 }
 38 
 39 /*
 40  * This is really a simplified "vm_mmap". it only handles MPX
 41  * bounds tables (the bounds directory is user-allocated).
 42  */
 43 static unsigned long mpx_mmap(unsigned long len)
 44 {
 45         unsigned long ret;
 46         unsigned long addr, pgoff;
 47         struct mm_struct *mm = current->mm;
 48         vm_flags_t vm_flags;
 49         struct vm_area_struct *vma;
 50 
 51         /* Only bounds table can be allocated here */
 52         if (len != mpx_bt_size_bytes(mm))
 53                 return -EINVAL;
 54 
 55         down_write(&mm->mmap_sem);
 56 
 57         /* Too many mappings? */
 58         if (mm->map_count > sysctl_max_map_count) {
 59                 ret = -ENOMEM;
 60                 goto out;
 61         }
 62 
 63         /* Obtain the address to map to. we verify (or select) it and ensure
 64          * that it represents a valid section of the address space.
 65          */
 66         addr = get_unmapped_area(NULL, 0, len, 0, MAP_ANONYMOUS | MAP_PRIVATE);
 67         if (addr & ~PAGE_MASK) {
 68                 ret = addr;
 69                 goto out;
 70         }
 71 
 72         vm_flags = VM_READ | VM_WRITE | VM_MPX |
 73                         mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
 74 
 75         /* Set pgoff according to addr for anon_vma */
 76         pgoff = addr >> PAGE_SHIFT;
 77 
 78         ret = mmap_region(NULL, addr, len, vm_flags, pgoff);
 79         if (IS_ERR_VALUE(ret))
 80                 goto out;
 81 
 82         vma = find_vma(mm, ret);
 83         if (!vma) {
 84                 ret = -ENOMEM;
 85                 goto out;
 86         }
 87 
 88         if (vm_flags & VM_LOCKED) {
 89                 up_write(&mm->mmap_sem);
 90                 mm_populate(ret, len);
 91                 return ret;
 92         }
 93 
 94 out:
 95         up_write(&mm->mmap_sem);
 96         return ret;
 97 }
 98 
 99 enum reg_type {
100         REG_TYPE_RM = 0,
101         REG_TYPE_INDEX,
102         REG_TYPE_BASE,
103 };
104 
105 static int get_reg_offset(struct insn *insn, struct pt_regs *regs,
106                           enum reg_type type)
107 {
108         int regno = 0;
109 
110         static const int regoff[] = {
111                 offsetof(struct pt_regs, ax),
112                 offsetof(struct pt_regs, cx),
113                 offsetof(struct pt_regs, dx),
114                 offsetof(struct pt_regs, bx),
115                 offsetof(struct pt_regs, sp),
116                 offsetof(struct pt_regs, bp),
117                 offsetof(struct pt_regs, si),
118                 offsetof(struct pt_regs, di),
119 #ifdef CONFIG_X86_64
120                 offsetof(struct pt_regs, r8),
121                 offsetof(struct pt_regs, r9),
122                 offsetof(struct pt_regs, r10),
123                 offsetof(struct pt_regs, r11),
124                 offsetof(struct pt_regs, r12),
125                 offsetof(struct pt_regs, r13),
126                 offsetof(struct pt_regs, r14),
127                 offsetof(struct pt_regs, r15),
128 #endif
129         };
130         int nr_registers = ARRAY_SIZE(regoff);
131         /*
132          * Don't possibly decode a 32-bit instructions as
133          * reading a 64-bit-only register.
134          */
135         if (IS_ENABLED(CONFIG_X86_64) && !insn->x86_64)
136                 nr_registers -= 8;
137 
138         switch (type) {
139         case REG_TYPE_RM:
140                 regno = X86_MODRM_RM(insn->modrm.value);
141                 if (X86_REX_B(insn->rex_prefix.value) == 1)
142                         regno += 8;
143                 break;
144 
145         case REG_TYPE_INDEX:
146                 regno = X86_SIB_INDEX(insn->sib.value);
147                 if (X86_REX_X(insn->rex_prefix.value) == 1)
148                         regno += 8;
149                 break;
150 
151         case REG_TYPE_BASE:
152                 regno = X86_SIB_BASE(insn->sib.value);
153                 if (X86_REX_B(insn->rex_prefix.value) == 1)
154                         regno += 8;
155                 break;
156 
157         default:
158                 pr_err("invalid register type");
159                 BUG();
160                 break;
161         }
162 
163         if (regno > nr_registers) {
164                 WARN_ONCE(1, "decoded an instruction with an invalid register");
165                 return -EINVAL;
166         }
167         return regoff[regno];
168 }
169 
170 /*
171  * return the address being referenced be instruction
172  * for rm=3 returning the content of the rm reg
173  * for rm!=3 calculates the address using SIB and Disp
174  */
175 static void __user *mpx_get_addr_ref(struct insn *insn, struct pt_regs *regs)
176 {
177         unsigned long addr, base, indx;
178         int addr_offset, base_offset, indx_offset;
179         insn_byte_t sib;
180 
181         insn_get_modrm(insn);
182         insn_get_sib(insn);
183         sib = insn->sib.value;
184 
185         if (X86_MODRM_MOD(insn->modrm.value) == 3) {
186                 addr_offset = get_reg_offset(insn, regs, REG_TYPE_RM);
187                 if (addr_offset < 0)
188                         goto out_err;
189                 addr = regs_get_register(regs, addr_offset);
190         } else {
191                 if (insn->sib.nbytes) {
192                         base_offset = get_reg_offset(insn, regs, REG_TYPE_BASE);
193                         if (base_offset < 0)
194                                 goto out_err;
195 
196                         indx_offset = get_reg_offset(insn, regs, REG_TYPE_INDEX);
197                         if (indx_offset < 0)
198                                 goto out_err;
199 
200                         base = regs_get_register(regs, base_offset);
201                         indx = regs_get_register(regs, indx_offset);
202                         addr = base + indx * (1 << X86_SIB_SCALE(sib));
203                 } else {
204                         addr_offset = get_reg_offset(insn, regs, REG_TYPE_RM);
205                         if (addr_offset < 0)
206                                 goto out_err;
207                         addr = regs_get_register(regs, addr_offset);
208                 }
209                 addr += insn->displacement.value;
210         }
211         return (void __user *)addr;
212 out_err:
213         return (void __user *)-1;
214 }
215 
216 static int mpx_insn_decode(struct insn *insn,
217                            struct pt_regs *regs)
218 {
219         unsigned char buf[MAX_INSN_SIZE];
220         int x86_64 = !test_thread_flag(TIF_IA32);
221         int not_copied;
222         int nr_copied;
223 
224         not_copied = copy_from_user(buf, (void __user *)regs->ip, sizeof(buf));
225         nr_copied = sizeof(buf) - not_copied;
226         /*
227          * The decoder _should_ fail nicely if we pass it a short buffer.
228          * But, let's not depend on that implementation detail.  If we
229          * did not get anything, just error out now.
230          */
231         if (!nr_copied)
232                 return -EFAULT;
233         insn_init(insn, buf, nr_copied, x86_64);
234         insn_get_length(insn);
235         /*
236          * copy_from_user() tries to get as many bytes as we could see in
237          * the largest possible instruction.  If the instruction we are
238          * after is shorter than that _and_ we attempt to copy from
239          * something unreadable, we might get a short read.  This is OK
240          * as long as the read did not stop in the middle of the
241          * instruction.  Check to see if we got a partial instruction.
242          */
243         if (nr_copied < insn->length)
244                 return -EFAULT;
245 
246         insn_get_opcode(insn);
247         /*
248          * We only _really_ need to decode bndcl/bndcn/bndcu
249          * Error out on anything else.
250          */
251         if (insn->opcode.bytes[0] != 0x0f)
252                 goto bad_opcode;
253         if ((insn->opcode.bytes[1] != 0x1a) &&
254             (insn->opcode.bytes[1] != 0x1b))
255                 goto bad_opcode;
256 
257         return 0;
258 bad_opcode:
259         return -EINVAL;
260 }
261 
262 /*
263  * If a bounds overflow occurs then a #BR is generated. This
264  * function decodes MPX instructions to get violation address
265  * and set this address into extended struct siginfo.
266  *
267  * Note that this is not a super precise way of doing this.
268  * Userspace could have, by the time we get here, written
269  * anything it wants in to the instructions.  We can not
270  * trust anything about it.  They might not be valid
271  * instructions or might encode invalid registers, etc...
272  *
273  * The caller is expected to kfree() the returned siginfo_t.
274  */
275 siginfo_t *mpx_generate_siginfo(struct pt_regs *regs)
276 {
277         const struct bndreg *bndregs, *bndreg;
278         siginfo_t *info = NULL;
279         struct insn insn;
280         uint8_t bndregno;
281         int err;
282 
283         err = mpx_insn_decode(&insn, regs);
284         if (err)
285                 goto err_out;
286 
287         /*
288          * We know at this point that we are only dealing with
289          * MPX instructions.
290          */
291         insn_get_modrm(&insn);
292         bndregno = X86_MODRM_REG(insn.modrm.value);
293         if (bndregno > 3) {
294                 err = -EINVAL;
295                 goto err_out;
296         }
297         /* get bndregs field from current task's xsave area */
298         bndregs = get_xsave_field_ptr(XSTATE_BNDREGS);
299         if (!bndregs) {
300                 err = -EINVAL;
301                 goto err_out;
302         }
303         /* now go select the individual register in the set of 4 */
304         bndreg = &bndregs[bndregno];
305 
306         info = kzalloc(sizeof(*info), GFP_KERNEL);
307         if (!info) {
308                 err = -ENOMEM;
309                 goto err_out;
310         }
311         /*
312          * The registers are always 64-bit, but the upper 32
313          * bits are ignored in 32-bit mode.  Also, note that the
314          * upper bounds are architecturally represented in 1's
315          * complement form.
316          *
317          * The 'unsigned long' cast is because the compiler
318          * complains when casting from integers to different-size
319          * pointers.
320          */
321         info->si_lower = (void __user *)(unsigned long)bndreg->lower_bound;
322         info->si_upper = (void __user *)(unsigned long)~bndreg->upper_bound;
323         info->si_addr_lsb = 0;
324         info->si_signo = SIGSEGV;
325         info->si_errno = 0;
326         info->si_code = SEGV_BNDERR;
327         info->si_addr = mpx_get_addr_ref(&insn, regs);
328         /*
329          * We were not able to extract an address from the instruction,
330          * probably because there was something invalid in it.
331          */
332         if (info->si_addr == (void *)-1) {
333                 err = -EINVAL;
334                 goto err_out;
335         }
336         trace_mpx_bounds_register_exception(info->si_addr, bndreg);
337         return info;
338 err_out:
339         /* info might be NULL, but kfree() handles that */
340         kfree(info);
341         return ERR_PTR(err);
342 }
343 
344 static __user void *mpx_get_bounds_dir(void)
345 {
346         const struct bndcsr *bndcsr;
347 
348         if (!cpu_feature_enabled(X86_FEATURE_MPX))
349                 return MPX_INVALID_BOUNDS_DIR;
350 
351         /*
352          * The bounds directory pointer is stored in a register
353          * only accessible if we first do an xsave.
354          */
355         bndcsr = get_xsave_field_ptr(XSTATE_BNDCSR);
356         if (!bndcsr)
357                 return MPX_INVALID_BOUNDS_DIR;
358 
359         /*
360          * Make sure the register looks valid by checking the
361          * enable bit.
362          */
363         if (!(bndcsr->bndcfgu & MPX_BNDCFG_ENABLE_FLAG))
364                 return MPX_INVALID_BOUNDS_DIR;
365 
366         /*
367          * Lastly, mask off the low bits used for configuration
368          * flags, and return the address of the bounds table.
369          */
370         return (void __user *)(unsigned long)
371                 (bndcsr->bndcfgu & MPX_BNDCFG_ADDR_MASK);
372 }
373 
374 int mpx_enable_management(void)
375 {
376         void __user *bd_base = MPX_INVALID_BOUNDS_DIR;
377         struct mm_struct *mm = current->mm;
378         int ret = 0;
379 
380         /*
381          * runtime in the userspace will be responsible for allocation of
382          * the bounds directory. Then, it will save the base of the bounds
383          * directory into XSAVE/XRSTOR Save Area and enable MPX through
384          * XRSTOR instruction.
385          *
386          * The copy_xregs_to_kernel() beneath get_xsave_field_ptr() is
387          * expected to be relatively expensive. Storing the bounds
388          * directory here means that we do not have to do xsave in the
389          * unmap path; we can just use mm->bd_addr instead.
390          */
391         bd_base = mpx_get_bounds_dir();
392         down_write(&mm->mmap_sem);
393         mm->bd_addr = bd_base;
394         if (mm->bd_addr == MPX_INVALID_BOUNDS_DIR)
395                 ret = -ENXIO;
396 
397         up_write(&mm->mmap_sem);
398         return ret;
399 }
400 
401 int mpx_disable_management(void)
402 {
403         struct mm_struct *mm = current->mm;
404 
405         if (!cpu_feature_enabled(X86_FEATURE_MPX))
406                 return -ENXIO;
407 
408         down_write(&mm->mmap_sem);
409         mm->bd_addr = MPX_INVALID_BOUNDS_DIR;
410         up_write(&mm->mmap_sem);
411         return 0;
412 }
413 
414 static int mpx_cmpxchg_bd_entry(struct mm_struct *mm,
415                 unsigned long *curval,
416                 unsigned long __user *addr,
417                 unsigned long old_val, unsigned long new_val)
418 {
419         int ret;
420         /*
421          * user_atomic_cmpxchg_inatomic() actually uses sizeof()
422          * the pointer that we pass to it to figure out how much
423          * data to cmpxchg.  We have to be careful here not to
424          * pass a pointer to a 64-bit data type when we only want
425          * a 32-bit copy.
426          */
427         if (is_64bit_mm(mm)) {
428                 ret = user_atomic_cmpxchg_inatomic(curval,
429                                 addr, old_val, new_val);
430         } else {
431                 u32 uninitialized_var(curval_32);
432                 u32 old_val_32 = old_val;
433                 u32 new_val_32 = new_val;
434                 u32 __user *addr_32 = (u32 __user *)addr;
435 
436                 ret = user_atomic_cmpxchg_inatomic(&curval_32,
437                                 addr_32, old_val_32, new_val_32);
438                 *curval = curval_32;
439         }
440         return ret;
441 }
442 
443 /*
444  * With 32-bit mode, a bounds directory is 4MB, and the size of each
445  * bounds table is 16KB. With 64-bit mode, a bounds directory is 2GB,
446  * and the size of each bounds table is 4MB.
447  */
448 static int allocate_bt(struct mm_struct *mm, long __user *bd_entry)
449 {
450         unsigned long expected_old_val = 0;
451         unsigned long actual_old_val = 0;
452         unsigned long bt_addr;
453         unsigned long bd_new_entry;
454         int ret = 0;
455 
456         /*
457          * Carve the virtual space out of userspace for the new
458          * bounds table:
459          */
460         bt_addr = mpx_mmap(mpx_bt_size_bytes(mm));
461         if (IS_ERR((void *)bt_addr))
462                 return PTR_ERR((void *)bt_addr);
463         /*
464          * Set the valid flag (kinda like _PAGE_PRESENT in a pte)
465          */
466         bd_new_entry = bt_addr | MPX_BD_ENTRY_VALID_FLAG;
467 
468         /*
469          * Go poke the address of the new bounds table in to the
470          * bounds directory entry out in userspace memory.  Note:
471          * we may race with another CPU instantiating the same table.
472          * In that case the cmpxchg will see an unexpected
473          * 'actual_old_val'.
474          *
475          * This can fault, but that's OK because we do not hold
476          * mmap_sem at this point, unlike some of the other part
477          * of the MPX code that have to pagefault_disable().
478          */
479         ret = mpx_cmpxchg_bd_entry(mm, &actual_old_val, bd_entry,
480                                    expected_old_val, bd_new_entry);
481         if (ret)
482                 goto out_unmap;
483 
484         /*
485          * The user_atomic_cmpxchg_inatomic() will only return nonzero
486          * for faults, *not* if the cmpxchg itself fails.  Now we must
487          * verify that the cmpxchg itself completed successfully.
488          */
489         /*
490          * We expected an empty 'expected_old_val', but instead found
491          * an apparently valid entry.  Assume we raced with another
492          * thread to instantiate this table and desclare succecss.
493          */
494         if (actual_old_val & MPX_BD_ENTRY_VALID_FLAG) {
495                 ret = 0;
496                 goto out_unmap;
497         }
498         /*
499          * We found a non-empty bd_entry but it did not have the
500          * VALID_FLAG set.  Return an error which will result in
501          * a SEGV since this probably means that somebody scribbled
502          * some invalid data in to a bounds table.
503          */
504         if (expected_old_val != actual_old_val) {
505                 ret = -EINVAL;
506                 goto out_unmap;
507         }
508         trace_mpx_new_bounds_table(bt_addr);
509         return 0;
510 out_unmap:
511         vm_munmap(bt_addr, mpx_bt_size_bytes(mm));
512         return ret;
513 }
514 
515 /*
516  * When a BNDSTX instruction attempts to save bounds to a bounds
517  * table, it will first attempt to look up the table in the
518  * first-level bounds directory.  If it does not find a table in
519  * the directory, a #BR is generated and we get here in order to
520  * allocate a new table.
521  *
522  * With 32-bit mode, the size of BD is 4MB, and the size of each
523  * bound table is 16KB. With 64-bit mode, the size of BD is 2GB,
524  * and the size of each bound table is 4MB.
525  */
526 static int do_mpx_bt_fault(void)
527 {
528         unsigned long bd_entry, bd_base;
529         const struct bndcsr *bndcsr;
530         struct mm_struct *mm = current->mm;
531 
532         bndcsr = get_xsave_field_ptr(XSTATE_BNDCSR);
533         if (!bndcsr)
534                 return -EINVAL;
535         /*
536          * Mask off the preserve and enable bits
537          */
538         bd_base = bndcsr->bndcfgu & MPX_BNDCFG_ADDR_MASK;
539         /*
540          * The hardware provides the address of the missing or invalid
541          * entry via BNDSTATUS, so we don't have to go look it up.
542          */
543         bd_entry = bndcsr->bndstatus & MPX_BNDSTA_ADDR_MASK;
544         /*
545          * Make sure the directory entry is within where we think
546          * the directory is.
547          */
548         if ((bd_entry < bd_base) ||
549             (bd_entry >= bd_base + mpx_bd_size_bytes(mm)))
550                 return -EINVAL;
551 
552         return allocate_bt(mm, (long __user *)bd_entry);
553 }
554 
555 int mpx_handle_bd_fault(void)
556 {
557         /*
558          * Userspace never asked us to manage the bounds tables,
559          * so refuse to help.
560          */
561         if (!kernel_managing_mpx_tables(current->mm))
562                 return -EINVAL;
563 
564         if (do_mpx_bt_fault()) {
565                 force_sig(SIGSEGV, current);
566                 /*
567                  * The force_sig() is essentially "handling" this
568                  * exception, so we do not pass up the error
569                  * from do_mpx_bt_fault().
570                  */
571         }
572         return 0;
573 }
574 
575 /*
576  * A thin wrapper around get_user_pages().  Returns 0 if the
577  * fault was resolved or -errno if not.
578  */
579 static int mpx_resolve_fault(long __user *addr, int write)
580 {
581         long gup_ret;
582         int nr_pages = 1;
583         int force = 0;
584 
585         gup_ret = get_user_pages(current, current->mm, (unsigned long)addr,
586                                  nr_pages, write, force, NULL, NULL);
587         /*
588          * get_user_pages() returns number of pages gotten.
589          * 0 means we failed to fault in and get anything,
590          * probably because 'addr' is bad.
591          */
592         if (!gup_ret)
593                 return -EFAULT;
594         /* Other error, return it */
595         if (gup_ret < 0)
596                 return gup_ret;
597         /* must have gup'd a page and gup_ret>0, success */
598         return 0;
599 }
600 
601 static unsigned long mpx_bd_entry_to_bt_addr(struct mm_struct *mm,
602                                              unsigned long bd_entry)
603 {
604         unsigned long bt_addr = bd_entry;
605         int align_to_bytes;
606         /*
607          * Bit 0 in a bt_entry is always the valid bit.
608          */
609         bt_addr &= ~MPX_BD_ENTRY_VALID_FLAG;
610         /*
611          * Tables are naturally aligned at 8-byte boundaries
612          * on 64-bit and 4-byte boundaries on 32-bit.  The
613          * documentation makes it appear that the low bits
614          * are ignored by the hardware, so we do the same.
615          */
616         if (is_64bit_mm(mm))
617                 align_to_bytes = 8;
618         else
619                 align_to_bytes = 4;
620         bt_addr &= ~(align_to_bytes-1);
621         return bt_addr;
622 }
623 
624 /*
625  * We only want to do a 4-byte get_user() on 32-bit.  Otherwise,
626  * we might run off the end of the bounds table if we are on
627  * a 64-bit kernel and try to get 8 bytes.
628  */
629 int get_user_bd_entry(struct mm_struct *mm, unsigned long *bd_entry_ret,
630                 long __user *bd_entry_ptr)
631 {
632         u32 bd_entry_32;
633         int ret;
634 
635         if (is_64bit_mm(mm))
636                 return get_user(*bd_entry_ret, bd_entry_ptr);
637 
638         /*
639          * Note that get_user() uses the type of the *pointer* to
640          * establish the size of the get, not the destination.
641          */
642         ret = get_user(bd_entry_32, (u32 __user *)bd_entry_ptr);
643         *bd_entry_ret = bd_entry_32;
644         return ret;
645 }
646 
647 /*
648  * Get the base of bounds tables pointed by specific bounds
649  * directory entry.
650  */
651 static int get_bt_addr(struct mm_struct *mm,
652                         long __user *bd_entry_ptr,
653                         unsigned long *bt_addr_result)
654 {
655         int ret;
656         int valid_bit;
657         unsigned long bd_entry;
658         unsigned long bt_addr;
659 
660         if (!access_ok(VERIFY_READ, (bd_entry_ptr), sizeof(*bd_entry_ptr)))
661                 return -EFAULT;
662 
663         while (1) {
664                 int need_write = 0;
665 
666                 pagefault_disable();
667                 ret = get_user_bd_entry(mm, &bd_entry, bd_entry_ptr);
668                 pagefault_enable();
669                 if (!ret)
670                         break;
671                 if (ret == -EFAULT)
672                         ret = mpx_resolve_fault(bd_entry_ptr, need_write);
673                 /*
674                  * If we could not resolve the fault, consider it
675                  * userspace's fault and error out.
676                  */
677                 if (ret)
678                         return ret;
679         }
680 
681         valid_bit = bd_entry & MPX_BD_ENTRY_VALID_FLAG;
682         bt_addr = mpx_bd_entry_to_bt_addr(mm, bd_entry);
683 
684         /*
685          * When the kernel is managing bounds tables, a bounds directory
686          * entry will either have a valid address (plus the valid bit)
687          * *OR* be completely empty. If we see a !valid entry *and* some
688          * data in the address field, we know something is wrong. This
689          * -EINVAL return will cause a SIGSEGV.
690          */
691         if (!valid_bit && bt_addr)
692                 return -EINVAL;
693         /*
694          * Do we have an completely zeroed bt entry?  That is OK.  It
695          * just means there was no bounds table for this memory.  Make
696          * sure to distinguish this from -EINVAL, which will cause
697          * a SEGV.
698          */
699         if (!valid_bit)
700                 return -ENOENT;
701 
702         *bt_addr_result = bt_addr;
703         return 0;
704 }
705 
706 static inline int bt_entry_size_bytes(struct mm_struct *mm)
707 {
708         if (is_64bit_mm(mm))
709                 return MPX_BT_ENTRY_BYTES_64;
710         else
711                 return MPX_BT_ENTRY_BYTES_32;
712 }
713 
714 /*
715  * Take a virtual address and turns it in to the offset in bytes
716  * inside of the bounds table where the bounds table entry
717  * controlling 'addr' can be found.
718  */
719 static unsigned long mpx_get_bt_entry_offset_bytes(struct mm_struct *mm,
720                 unsigned long addr)
721 {
722         unsigned long bt_table_nr_entries;
723         unsigned long offset = addr;
724 
725         if (is_64bit_mm(mm)) {
726                 /* Bottom 3 bits are ignored on 64-bit */
727                 offset >>= 3;
728                 bt_table_nr_entries = MPX_BT_NR_ENTRIES_64;
729         } else {
730                 /* Bottom 2 bits are ignored on 32-bit */
731                 offset >>= 2;
732                 bt_table_nr_entries = MPX_BT_NR_ENTRIES_32;
733         }
734         /*
735          * We know the size of the table in to which we are
736          * indexing, and we have eliminated all the low bits
737          * which are ignored for indexing.
738          *
739          * Mask out all the high bits which we do not need
740          * to index in to the table.  Note that the tables
741          * are always powers of two so this gives us a proper
742          * mask.
743          */
744         offset &= (bt_table_nr_entries-1);
745         /*
746          * We now have an entry offset in terms of *entries* in
747          * the table.  We need to scale it back up to bytes.
748          */
749         offset *= bt_entry_size_bytes(mm);
750         return offset;
751 }
752 
753 /*
754  * How much virtual address space does a single bounds
755  * directory entry cover?
756  *
757  * Note, we need a long long because 4GB doesn't fit in
758  * to a long on 32-bit.
759  */
760 static inline unsigned long bd_entry_virt_space(struct mm_struct *mm)
761 {
762         unsigned long long virt_space;
763         unsigned long long GB = (1ULL << 30);
764 
765         /*
766          * This covers 32-bit emulation as well as 32-bit kernels
767          * running on 64-bit harware.
768          */
769         if (!is_64bit_mm(mm))
770                 return (4ULL * GB) / MPX_BD_NR_ENTRIES_32;
771 
772         /*
773          * 'x86_virt_bits' returns what the hardware is capable
774          * of, and returns the full >32-bit adddress space when
775          * running 32-bit kernels on 64-bit hardware.
776          */
777         virt_space = (1ULL << boot_cpu_data.x86_virt_bits);
778         return virt_space / MPX_BD_NR_ENTRIES_64;
779 }
780 
781 /*
782  * Free the backing physical pages of bounds table 'bt_addr'.
783  * Assume start...end is within that bounds table.
784  */
785 static noinline int zap_bt_entries_mapping(struct mm_struct *mm,
786                 unsigned long bt_addr,
787                 unsigned long start_mapping, unsigned long end_mapping)
788 {
789         struct vm_area_struct *vma;
790         unsigned long addr, len;
791         unsigned long start;
792         unsigned long end;
793 
794         /*
795          * if we 'end' on a boundary, the offset will be 0 which
796          * is not what we want.  Back it up a byte to get the
797          * last bt entry.  Then once we have the entry itself,
798          * move 'end' back up by the table entry size.
799          */
800         start = bt_addr + mpx_get_bt_entry_offset_bytes(mm, start_mapping);
801         end   = bt_addr + mpx_get_bt_entry_offset_bytes(mm, end_mapping - 1);
802         /*
803          * Move end back up by one entry.  Among other things
804          * this ensures that it remains page-aligned and does
805          * not screw up zap_page_range()
806          */
807         end += bt_entry_size_bytes(mm);
808 
809         /*
810          * Find the first overlapping vma. If vma->vm_start > start, there
811          * will be a hole in the bounds table. This -EINVAL return will
812          * cause a SIGSEGV.
813          */
814         vma = find_vma(mm, start);
815         if (!vma || vma->vm_start > start)
816                 return -EINVAL;
817 
818         /*
819          * A NUMA policy on a VM_MPX VMA could cause this bounds table to
820          * be split. So we need to look across the entire 'start -> end'
821          * range of this bounds table, find all of the VM_MPX VMAs, and
822          * zap only those.
823          */
824         addr = start;
825         while (vma && vma->vm_start < end) {
826                 /*
827                  * We followed a bounds directory entry down
828                  * here.  If we find a non-MPX VMA, that's bad,
829                  * so stop immediately and return an error.  This
830                  * probably results in a SIGSEGV.
831                  */
832                 if (!(vma->vm_flags & VM_MPX))
833                         return -EINVAL;
834 
835                 len = min(vma->vm_end, end) - addr;
836                 zap_page_range(vma, addr, len, NULL);
837                 trace_mpx_unmap_zap(addr, addr+len);
838 
839                 vma = vma->vm_next;
840                 addr = vma->vm_start;
841         }
842         return 0;
843 }
844 
845 static unsigned long mpx_get_bd_entry_offset(struct mm_struct *mm,
846                 unsigned long addr)
847 {
848         /*
849          * There are several ways to derive the bd offsets.  We
850          * use the following approach here:
851          * 1. We know the size of the virtual address space
852          * 2. We know the number of entries in a bounds table
853          * 3. We know that each entry covers a fixed amount of
854          *    virtual address space.
855          * So, we can just divide the virtual address by the
856          * virtual space used by one entry to determine which
857          * entry "controls" the given virtual address.
858          */
859         if (is_64bit_mm(mm)) {
860                 int bd_entry_size = 8; /* 64-bit pointer */
861                 /*
862                  * Take the 64-bit addressing hole in to account.
863                  */
864                 addr &= ((1UL << boot_cpu_data.x86_virt_bits) - 1);
865                 return (addr / bd_entry_virt_space(mm)) * bd_entry_size;
866         } else {
867                 int bd_entry_size = 4; /* 32-bit pointer */
868                 /*
869                  * 32-bit has no hole so this case needs no mask
870                  */
871                 return (addr / bd_entry_virt_space(mm)) * bd_entry_size;
872         }
873         /*
874          * The two return calls above are exact copies.  If we
875          * pull out a single copy and put it in here, gcc won't
876          * realize that we're doing a power-of-2 divide and use
877          * shifts.  It uses a real divide.  If we put them up
878          * there, it manages to figure it out (gcc 4.8.3).
879          */
880 }
881 
882 static int unmap_entire_bt(struct mm_struct *mm,
883                 long __user *bd_entry, unsigned long bt_addr)
884 {
885         unsigned long expected_old_val = bt_addr | MPX_BD_ENTRY_VALID_FLAG;
886         unsigned long uninitialized_var(actual_old_val);
887         int ret;
888 
889         while (1) {
890                 int need_write = 1;
891                 unsigned long cleared_bd_entry = 0;
892 
893                 pagefault_disable();
894                 ret = mpx_cmpxchg_bd_entry(mm, &actual_old_val,
895                                 bd_entry, expected_old_val, cleared_bd_entry);
896                 pagefault_enable();
897                 if (!ret)
898                         break;
899                 if (ret == -EFAULT)
900                         ret = mpx_resolve_fault(bd_entry, need_write);
901                 /*
902                  * If we could not resolve the fault, consider it
903                  * userspace's fault and error out.
904                  */
905                 if (ret)
906                         return ret;
907         }
908         /*
909          * The cmpxchg was performed, check the results.
910          */
911         if (actual_old_val != expected_old_val) {
912                 /*
913                  * Someone else raced with us to unmap the table.
914                  * That is OK, since we were both trying to do
915                  * the same thing.  Declare success.
916                  */
917                 if (!actual_old_val)
918                         return 0;
919                 /*
920                  * Something messed with the bounds directory
921                  * entry.  We hold mmap_sem for read or write
922                  * here, so it could not be a _new_ bounds table
923                  * that someone just allocated.  Something is
924                  * wrong, so pass up the error and SIGSEGV.
925                  */
926                 return -EINVAL;
927         }
928         /*
929          * Note, we are likely being called under do_munmap() already. To
930          * avoid recursion, do_munmap() will check whether it comes
931          * from one bounds table through VM_MPX flag.
932          */
933         return do_munmap(mm, bt_addr, mpx_bt_size_bytes(mm));
934 }
935 
936 static int try_unmap_single_bt(struct mm_struct *mm,
937                unsigned long start, unsigned long end)
938 {
939         struct vm_area_struct *next;
940         struct vm_area_struct *prev;
941         /*
942          * "bta" == Bounds Table Area: the area controlled by the
943          * bounds table that we are unmapping.
944          */
945         unsigned long bta_start_vaddr = start & ~(bd_entry_virt_space(mm)-1);
946         unsigned long bta_end_vaddr = bta_start_vaddr + bd_entry_virt_space(mm);
947         unsigned long uninitialized_var(bt_addr);
948         void __user *bde_vaddr;
949         int ret;
950         /*
951          * We already unlinked the VMAs from the mm's rbtree so 'start'
952          * is guaranteed to be in a hole. This gets us the first VMA
953          * before the hole in to 'prev' and the next VMA after the hole
954          * in to 'next'.
955          */
956         next = find_vma_prev(mm, start, &prev);
957         /*
958          * Do not count other MPX bounds table VMAs as neighbors.
959          * Although theoretically possible, we do not allow bounds
960          * tables for bounds tables so our heads do not explode.
961          * If we count them as neighbors here, we may end up with
962          * lots of tables even though we have no actual table
963          * entries in use.
964          */
965         while (next && (next->vm_flags & VM_MPX))
966                 next = next->vm_next;
967         while (prev && (prev->vm_flags & VM_MPX))
968                 prev = prev->vm_prev;
969         /*
970          * We know 'start' and 'end' lie within an area controlled
971          * by a single bounds table.  See if there are any other
972          * VMAs controlled by that bounds table.  If there are not
973          * then we can "expand" the are we are unmapping to possibly
974          * cover the entire table.
975          */
976         next = find_vma_prev(mm, start, &prev);
977         if ((!prev || prev->vm_end <= bta_start_vaddr) &&
978             (!next || next->vm_start >= bta_end_vaddr)) {
979                 /*
980                  * No neighbor VMAs controlled by same bounds
981                  * table.  Try to unmap the whole thing
982                  */
983                 start = bta_start_vaddr;
984                 end = bta_end_vaddr;
985         }
986 
987         bde_vaddr = mm->bd_addr + mpx_get_bd_entry_offset(mm, start);
988         ret = get_bt_addr(mm, bde_vaddr, &bt_addr);
989         /*
990          * No bounds table there, so nothing to unmap.
991          */
992         if (ret == -ENOENT) {
993                 ret = 0;
994                 return 0;
995         }
996         if (ret)
997                 return ret;
998         /*
999          * We are unmapping an entire table.  Either because the
1000          * unmap that started this whole process was large enough
1001          * to cover an entire table, or that the unmap was small
1002          * but was the area covered by a bounds table.
1003          */
1004         if ((start == bta_start_vaddr) &&
1005             (end == bta_end_vaddr))
1006                 return unmap_entire_bt(mm, bde_vaddr, bt_addr);
1007         return zap_bt_entries_mapping(mm, bt_addr, start, end);
1008 }
1009 
1010 static int mpx_unmap_tables(struct mm_struct *mm,
1011                 unsigned long start, unsigned long end)
1012 {
1013         unsigned long one_unmap_start;
1014         trace_mpx_unmap_search(start, end);
1015 
1016         one_unmap_start = start;
1017         while (one_unmap_start < end) {
1018                 int ret;
1019                 unsigned long next_unmap_start = ALIGN(one_unmap_start+1,
1020                                                        bd_entry_virt_space(mm));
1021                 unsigned long one_unmap_end = end;
1022                 /*
1023                  * if the end is beyond the current bounds table,
1024                  * move it back so we only deal with a single one
1025                  * at a time
1026                  */
1027                 if (one_unmap_end > next_unmap_start)
1028                         one_unmap_end = next_unmap_start;
1029                 ret = try_unmap_single_bt(mm, one_unmap_start, one_unmap_end);
1030                 if (ret)
1031                         return ret;
1032 
1033                 one_unmap_start = next_unmap_start;
1034         }
1035         return 0;
1036 }
1037 
1038 /*
1039  * Free unused bounds tables covered in a virtual address region being
1040  * munmap()ed. Assume end > start.
1041  *
1042  * This function will be called by do_munmap(), and the VMAs covering
1043  * the virtual address region start...end have already been split if
1044  * necessary, and the 'vma' is the first vma in this range (start -> end).
1045  */
1046 void mpx_notify_unmap(struct mm_struct *mm, struct vm_area_struct *vma,
1047                 unsigned long start, unsigned long end)
1048 {
1049         int ret;
1050 
1051         /*
1052          * Refuse to do anything unless userspace has asked
1053          * the kernel to help manage the bounds tables,
1054          */
1055         if (!kernel_managing_mpx_tables(current->mm))
1056                 return;
1057         /*
1058          * This will look across the entire 'start -> end' range,
1059          * and find all of the non-VM_MPX VMAs.
1060          *
1061          * To avoid recursion, if a VM_MPX vma is found in the range
1062          * (start->end), we will not continue follow-up work. This
1063          * recursion represents having bounds tables for bounds tables,
1064          * which should not occur normally. Being strict about it here
1065          * helps ensure that we do not have an exploitable stack overflow.
1066          */
1067         do {
1068                 if (vma->vm_flags & VM_MPX)
1069                         return;
1070                 vma = vma->vm_next;
1071         } while (vma && vma->vm_start < end);
1072 
1073         ret = mpx_unmap_tables(mm, start, end);
1074         if (ret)
1075                 force_sig(SIGSEGV, current);
1076 }
1077 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp