1 /*
2 * mpx.c - Memory Protection eXtensions
3 *
4 * Copyright (c) 2014, Intel Corporation.
5 * Qiaowei Ren <qiaowei.ren@intel.com>
6 * Dave Hansen <dave.hansen@intel.com>
7 */
8 #include <linux/kernel.h>
9 #include <linux/slab.h>
10 #include <linux/syscalls.h>
11 #include <linux/sched/sysctl.h>
12
13 #include <asm/insn.h>
14 #include <asm/mman.h>
15 #include <asm/mmu_context.h>
16 #include <asm/mpx.h>
17 #include <asm/processor.h>
18 #include <asm/fpu/internal.h>
19
20 #define CREATE_TRACE_POINTS
21 #include <asm/trace/mpx.h>
22
23 static inline unsigned long mpx_bd_size_bytes(struct mm_struct *mm)
24 {
25 if (is_64bit_mm(mm))
26 return MPX_BD_SIZE_BYTES_64;
27 else
28 return MPX_BD_SIZE_BYTES_32;
29 }
30
31 static inline unsigned long mpx_bt_size_bytes(struct mm_struct *mm)
32 {
33 if (is_64bit_mm(mm))
34 return MPX_BT_SIZE_BYTES_64;
35 else
36 return MPX_BT_SIZE_BYTES_32;
37 }
38
39 /*
40 * This is really a simplified "vm_mmap". it only handles MPX
41 * bounds tables (the bounds directory is user-allocated).
42 */
43 static unsigned long mpx_mmap(unsigned long len)
44 {
45 unsigned long ret;
46 unsigned long addr, pgoff;
47 struct mm_struct *mm = current->mm;
48 vm_flags_t vm_flags;
49 struct vm_area_struct *vma;
50
51 /* Only bounds table can be allocated here */
52 if (len != mpx_bt_size_bytes(mm))
53 return -EINVAL;
54
55 down_write(&mm->mmap_sem);
56
57 /* Too many mappings? */
58 if (mm->map_count > sysctl_max_map_count) {
59 ret = -ENOMEM;
60 goto out;
61 }
62
63 /* Obtain the address to map to. we verify (or select) it and ensure
64 * that it represents a valid section of the address space.
65 */
66 addr = get_unmapped_area(NULL, 0, len, 0, MAP_ANONYMOUS | MAP_PRIVATE);
67 if (addr & ~PAGE_MASK) {
68 ret = addr;
69 goto out;
70 }
71
72 vm_flags = VM_READ | VM_WRITE | VM_MPX |
73 mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
74
75 /* Set pgoff according to addr for anon_vma */
76 pgoff = addr >> PAGE_SHIFT;
77
78 ret = mmap_region(NULL, addr, len, vm_flags, pgoff);
79 if (IS_ERR_VALUE(ret))
80 goto out;
81
82 vma = find_vma(mm, ret);
83 if (!vma) {
84 ret = -ENOMEM;
85 goto out;
86 }
87
88 if (vm_flags & VM_LOCKED) {
89 up_write(&mm->mmap_sem);
90 mm_populate(ret, len);
91 return ret;
92 }
93
94 out:
95 up_write(&mm->mmap_sem);
96 return ret;
97 }
98
99 enum reg_type {
100 REG_TYPE_RM = 0,
101 REG_TYPE_INDEX,
102 REG_TYPE_BASE,
103 };
104
105 static int get_reg_offset(struct insn *insn, struct pt_regs *regs,
106 enum reg_type type)
107 {
108 int regno = 0;
109
110 static const int regoff[] = {
111 offsetof(struct pt_regs, ax),
112 offsetof(struct pt_regs, cx),
113 offsetof(struct pt_regs, dx),
114 offsetof(struct pt_regs, bx),
115 offsetof(struct pt_regs, sp),
116 offsetof(struct pt_regs, bp),
117 offsetof(struct pt_regs, si),
118 offsetof(struct pt_regs, di),
119 #ifdef CONFIG_X86_64
120 offsetof(struct pt_regs, r8),
121 offsetof(struct pt_regs, r9),
122 offsetof(struct pt_regs, r10),
123 offsetof(struct pt_regs, r11),
124 offsetof(struct pt_regs, r12),
125 offsetof(struct pt_regs, r13),
126 offsetof(struct pt_regs, r14),
127 offsetof(struct pt_regs, r15),
128 #endif
129 };
130 int nr_registers = ARRAY_SIZE(regoff);
131 /*
132 * Don't possibly decode a 32-bit instructions as
133 * reading a 64-bit-only register.
134 */
135 if (IS_ENABLED(CONFIG_X86_64) && !insn->x86_64)
136 nr_registers -= 8;
137
138 switch (type) {
139 case REG_TYPE_RM:
140 regno = X86_MODRM_RM(insn->modrm.value);
141 if (X86_REX_B(insn->rex_prefix.value) == 1)
142 regno += 8;
143 break;
144
145 case REG_TYPE_INDEX:
146 regno = X86_SIB_INDEX(insn->sib.value);
147 if (X86_REX_X(insn->rex_prefix.value) == 1)
148 regno += 8;
149 break;
150
151 case REG_TYPE_BASE:
152 regno = X86_SIB_BASE(insn->sib.value);
153 if (X86_REX_B(insn->rex_prefix.value) == 1)
154 regno += 8;
155 break;
156
157 default:
158 pr_err("invalid register type");
159 BUG();
160 break;
161 }
162
163 if (regno > nr_registers) {
164 WARN_ONCE(1, "decoded an instruction with an invalid register");
165 return -EINVAL;
166 }
167 return regoff[regno];
168 }
169
170 /*
171 * return the address being referenced be instruction
172 * for rm=3 returning the content of the rm reg
173 * for rm!=3 calculates the address using SIB and Disp
174 */
175 static void __user *mpx_get_addr_ref(struct insn *insn, struct pt_regs *regs)
176 {
177 unsigned long addr, base, indx;
178 int addr_offset, base_offset, indx_offset;
179 insn_byte_t sib;
180
181 insn_get_modrm(insn);
182 insn_get_sib(insn);
183 sib = insn->sib.value;
184
185 if (X86_MODRM_MOD(insn->modrm.value) == 3) {
186 addr_offset = get_reg_offset(insn, regs, REG_TYPE_RM);
187 if (addr_offset < 0)
188 goto out_err;
189 addr = regs_get_register(regs, addr_offset);
190 } else {
191 if (insn->sib.nbytes) {
192 base_offset = get_reg_offset(insn, regs, REG_TYPE_BASE);
193 if (base_offset < 0)
194 goto out_err;
195
196 indx_offset = get_reg_offset(insn, regs, REG_TYPE_INDEX);
197 if (indx_offset < 0)
198 goto out_err;
199
200 base = regs_get_register(regs, base_offset);
201 indx = regs_get_register(regs, indx_offset);
202 addr = base + indx * (1 << X86_SIB_SCALE(sib));
203 } else {
204 addr_offset = get_reg_offset(insn, regs, REG_TYPE_RM);
205 if (addr_offset < 0)
206 goto out_err;
207 addr = regs_get_register(regs, addr_offset);
208 }
209 addr += insn->displacement.value;
210 }
211 return (void __user *)addr;
212 out_err:
213 return (void __user *)-1;
214 }
215
216 static int mpx_insn_decode(struct insn *insn,
217 struct pt_regs *regs)
218 {
219 unsigned char buf[MAX_INSN_SIZE];
220 int x86_64 = !test_thread_flag(TIF_IA32);
221 int not_copied;
222 int nr_copied;
223
224 not_copied = copy_from_user(buf, (void __user *)regs->ip, sizeof(buf));
225 nr_copied = sizeof(buf) - not_copied;
226 /*
227 * The decoder _should_ fail nicely if we pass it a short buffer.
228 * But, let's not depend on that implementation detail. If we
229 * did not get anything, just error out now.
230 */
231 if (!nr_copied)
232 return -EFAULT;
233 insn_init(insn, buf, nr_copied, x86_64);
234 insn_get_length(insn);
235 /*
236 * copy_from_user() tries to get as many bytes as we could see in
237 * the largest possible instruction. If the instruction we are
238 * after is shorter than that _and_ we attempt to copy from
239 * something unreadable, we might get a short read. This is OK
240 * as long as the read did not stop in the middle of the
241 * instruction. Check to see if we got a partial instruction.
242 */
243 if (nr_copied < insn->length)
244 return -EFAULT;
245
246 insn_get_opcode(insn);
247 /*
248 * We only _really_ need to decode bndcl/bndcn/bndcu
249 * Error out on anything else.
250 */
251 if (insn->opcode.bytes[0] != 0x0f)
252 goto bad_opcode;
253 if ((insn->opcode.bytes[1] != 0x1a) &&
254 (insn->opcode.bytes[1] != 0x1b))
255 goto bad_opcode;
256
257 return 0;
258 bad_opcode:
259 return -EINVAL;
260 }
261
262 /*
263 * If a bounds overflow occurs then a #BR is generated. This
264 * function decodes MPX instructions to get violation address
265 * and set this address into extended struct siginfo.
266 *
267 * Note that this is not a super precise way of doing this.
268 * Userspace could have, by the time we get here, written
269 * anything it wants in to the instructions. We can not
270 * trust anything about it. They might not be valid
271 * instructions or might encode invalid registers, etc...
272 *
273 * The caller is expected to kfree() the returned siginfo_t.
274 */
275 siginfo_t *mpx_generate_siginfo(struct pt_regs *regs)
276 {
277 const struct bndreg *bndregs, *bndreg;
278 siginfo_t *info = NULL;
279 struct insn insn;
280 uint8_t bndregno;
281 int err;
282
283 err = mpx_insn_decode(&insn, regs);
284 if (err)
285 goto err_out;
286
287 /*
288 * We know at this point that we are only dealing with
289 * MPX instructions.
290 */
291 insn_get_modrm(&insn);
292 bndregno = X86_MODRM_REG(insn.modrm.value);
293 if (bndregno > 3) {
294 err = -EINVAL;
295 goto err_out;
296 }
297 /* get bndregs field from current task's xsave area */
298 bndregs = get_xsave_field_ptr(XSTATE_BNDREGS);
299 if (!bndregs) {
300 err = -EINVAL;
301 goto err_out;
302 }
303 /* now go select the individual register in the set of 4 */
304 bndreg = &bndregs[bndregno];
305
306 info = kzalloc(sizeof(*info), GFP_KERNEL);
307 if (!info) {
308 err = -ENOMEM;
309 goto err_out;
310 }
311 /*
312 * The registers are always 64-bit, but the upper 32
313 * bits are ignored in 32-bit mode. Also, note that the
314 * upper bounds are architecturally represented in 1's
315 * complement form.
316 *
317 * The 'unsigned long' cast is because the compiler
318 * complains when casting from integers to different-size
319 * pointers.
320 */
321 info->si_lower = (void __user *)(unsigned long)bndreg->lower_bound;
322 info->si_upper = (void __user *)(unsigned long)~bndreg->upper_bound;
323 info->si_addr_lsb = 0;
324 info->si_signo = SIGSEGV;
325 info->si_errno = 0;
326 info->si_code = SEGV_BNDERR;
327 info->si_addr = mpx_get_addr_ref(&insn, regs);
328 /*
329 * We were not able to extract an address from the instruction,
330 * probably because there was something invalid in it.
331 */
332 if (info->si_addr == (void *)-1) {
333 err = -EINVAL;
334 goto err_out;
335 }
336 trace_mpx_bounds_register_exception(info->si_addr, bndreg);
337 return info;
338 err_out:
339 /* info might be NULL, but kfree() handles that */
340 kfree(info);
341 return ERR_PTR(err);
342 }
343
344 static __user void *mpx_get_bounds_dir(void)
345 {
346 const struct bndcsr *bndcsr;
347
348 if (!cpu_feature_enabled(X86_FEATURE_MPX))
349 return MPX_INVALID_BOUNDS_DIR;
350
351 /*
352 * The bounds directory pointer is stored in a register
353 * only accessible if we first do an xsave.
354 */
355 bndcsr = get_xsave_field_ptr(XSTATE_BNDCSR);
356 if (!bndcsr)
357 return MPX_INVALID_BOUNDS_DIR;
358
359 /*
360 * Make sure the register looks valid by checking the
361 * enable bit.
362 */
363 if (!(bndcsr->bndcfgu & MPX_BNDCFG_ENABLE_FLAG))
364 return MPX_INVALID_BOUNDS_DIR;
365
366 /*
367 * Lastly, mask off the low bits used for configuration
368 * flags, and return the address of the bounds table.
369 */
370 return (void __user *)(unsigned long)
371 (bndcsr->bndcfgu & MPX_BNDCFG_ADDR_MASK);
372 }
373
374 int mpx_enable_management(void)
375 {
376 void __user *bd_base = MPX_INVALID_BOUNDS_DIR;
377 struct mm_struct *mm = current->mm;
378 int ret = 0;
379
380 /*
381 * runtime in the userspace will be responsible for allocation of
382 * the bounds directory. Then, it will save the base of the bounds
383 * directory into XSAVE/XRSTOR Save Area and enable MPX through
384 * XRSTOR instruction.
385 *
386 * The copy_xregs_to_kernel() beneath get_xsave_field_ptr() is
387 * expected to be relatively expensive. Storing the bounds
388 * directory here means that we do not have to do xsave in the
389 * unmap path; we can just use mm->bd_addr instead.
390 */
391 bd_base = mpx_get_bounds_dir();
392 down_write(&mm->mmap_sem);
393 mm->bd_addr = bd_base;
394 if (mm->bd_addr == MPX_INVALID_BOUNDS_DIR)
395 ret = -ENXIO;
396
397 up_write(&mm->mmap_sem);
398 return ret;
399 }
400
401 int mpx_disable_management(void)
402 {
403 struct mm_struct *mm = current->mm;
404
405 if (!cpu_feature_enabled(X86_FEATURE_MPX))
406 return -ENXIO;
407
408 down_write(&mm->mmap_sem);
409 mm->bd_addr = MPX_INVALID_BOUNDS_DIR;
410 up_write(&mm->mmap_sem);
411 return 0;
412 }
413
414 static int mpx_cmpxchg_bd_entry(struct mm_struct *mm,
415 unsigned long *curval,
416 unsigned long __user *addr,
417 unsigned long old_val, unsigned long new_val)
418 {
419 int ret;
420 /*
421 * user_atomic_cmpxchg_inatomic() actually uses sizeof()
422 * the pointer that we pass to it to figure out how much
423 * data to cmpxchg. We have to be careful here not to
424 * pass a pointer to a 64-bit data type when we only want
425 * a 32-bit copy.
426 */
427 if (is_64bit_mm(mm)) {
428 ret = user_atomic_cmpxchg_inatomic(curval,
429 addr, old_val, new_val);
430 } else {
431 u32 uninitialized_var(curval_32);
432 u32 old_val_32 = old_val;
433 u32 new_val_32 = new_val;
434 u32 __user *addr_32 = (u32 __user *)addr;
435
436 ret = user_atomic_cmpxchg_inatomic(&curval_32,
437 addr_32, old_val_32, new_val_32);
438 *curval = curval_32;
439 }
440 return ret;
441 }
442
443 /*
444 * With 32-bit mode, a bounds directory is 4MB, and the size of each
445 * bounds table is 16KB. With 64-bit mode, a bounds directory is 2GB,
446 * and the size of each bounds table is 4MB.
447 */
448 static int allocate_bt(struct mm_struct *mm, long __user *bd_entry)
449 {
450 unsigned long expected_old_val = 0;
451 unsigned long actual_old_val = 0;
452 unsigned long bt_addr;
453 unsigned long bd_new_entry;
454 int ret = 0;
455
456 /*
457 * Carve the virtual space out of userspace for the new
458 * bounds table:
459 */
460 bt_addr = mpx_mmap(mpx_bt_size_bytes(mm));
461 if (IS_ERR((void *)bt_addr))
462 return PTR_ERR((void *)bt_addr);
463 /*
464 * Set the valid flag (kinda like _PAGE_PRESENT in a pte)
465 */
466 bd_new_entry = bt_addr | MPX_BD_ENTRY_VALID_FLAG;
467
468 /*
469 * Go poke the address of the new bounds table in to the
470 * bounds directory entry out in userspace memory. Note:
471 * we may race with another CPU instantiating the same table.
472 * In that case the cmpxchg will see an unexpected
473 * 'actual_old_val'.
474 *
475 * This can fault, but that's OK because we do not hold
476 * mmap_sem at this point, unlike some of the other part
477 * of the MPX code that have to pagefault_disable().
478 */
479 ret = mpx_cmpxchg_bd_entry(mm, &actual_old_val, bd_entry,
480 expected_old_val, bd_new_entry);
481 if (ret)
482 goto out_unmap;
483
484 /*
485 * The user_atomic_cmpxchg_inatomic() will only return nonzero
486 * for faults, *not* if the cmpxchg itself fails. Now we must
487 * verify that the cmpxchg itself completed successfully.
488 */
489 /*
490 * We expected an empty 'expected_old_val', but instead found
491 * an apparently valid entry. Assume we raced with another
492 * thread to instantiate this table and desclare succecss.
493 */
494 if (actual_old_val & MPX_BD_ENTRY_VALID_FLAG) {
495 ret = 0;
496 goto out_unmap;
497 }
498 /*
499 * We found a non-empty bd_entry but it did not have the
500 * VALID_FLAG set. Return an error which will result in
501 * a SEGV since this probably means that somebody scribbled
502 * some invalid data in to a bounds table.
503 */
504 if (expected_old_val != actual_old_val) {
505 ret = -EINVAL;
506 goto out_unmap;
507 }
508 trace_mpx_new_bounds_table(bt_addr);
509 return 0;
510 out_unmap:
511 vm_munmap(bt_addr, mpx_bt_size_bytes(mm));
512 return ret;
513 }
514
515 /*
516 * When a BNDSTX instruction attempts to save bounds to a bounds
517 * table, it will first attempt to look up the table in the
518 * first-level bounds directory. If it does not find a table in
519 * the directory, a #BR is generated and we get here in order to
520 * allocate a new table.
521 *
522 * With 32-bit mode, the size of BD is 4MB, and the size of each
523 * bound table is 16KB. With 64-bit mode, the size of BD is 2GB,
524 * and the size of each bound table is 4MB.
525 */
526 static int do_mpx_bt_fault(void)
527 {
528 unsigned long bd_entry, bd_base;
529 const struct bndcsr *bndcsr;
530 struct mm_struct *mm = current->mm;
531
532 bndcsr = get_xsave_field_ptr(XSTATE_BNDCSR);
533 if (!bndcsr)
534 return -EINVAL;
535 /*
536 * Mask off the preserve and enable bits
537 */
538 bd_base = bndcsr->bndcfgu & MPX_BNDCFG_ADDR_MASK;
539 /*
540 * The hardware provides the address of the missing or invalid
541 * entry via BNDSTATUS, so we don't have to go look it up.
542 */
543 bd_entry = bndcsr->bndstatus & MPX_BNDSTA_ADDR_MASK;
544 /*
545 * Make sure the directory entry is within where we think
546 * the directory is.
547 */
548 if ((bd_entry < bd_base) ||
549 (bd_entry >= bd_base + mpx_bd_size_bytes(mm)))
550 return -EINVAL;
551
552 return allocate_bt(mm, (long __user *)bd_entry);
553 }
554
555 int mpx_handle_bd_fault(void)
556 {
557 /*
558 * Userspace never asked us to manage the bounds tables,
559 * so refuse to help.
560 */
561 if (!kernel_managing_mpx_tables(current->mm))
562 return -EINVAL;
563
564 if (do_mpx_bt_fault()) {
565 force_sig(SIGSEGV, current);
566 /*
567 * The force_sig() is essentially "handling" this
568 * exception, so we do not pass up the error
569 * from do_mpx_bt_fault().
570 */
571 }
572 return 0;
573 }
574
575 /*
576 * A thin wrapper around get_user_pages(). Returns 0 if the
577 * fault was resolved or -errno if not.
578 */
579 static int mpx_resolve_fault(long __user *addr, int write)
580 {
581 long gup_ret;
582 int nr_pages = 1;
583 int force = 0;
584
585 gup_ret = get_user_pages(current, current->mm, (unsigned long)addr,
586 nr_pages, write, force, NULL, NULL);
587 /*
588 * get_user_pages() returns number of pages gotten.
589 * 0 means we failed to fault in and get anything,
590 * probably because 'addr' is bad.
591 */
592 if (!gup_ret)
593 return -EFAULT;
594 /* Other error, return it */
595 if (gup_ret < 0)
596 return gup_ret;
597 /* must have gup'd a page and gup_ret>0, success */
598 return 0;
599 }
600
601 static unsigned long mpx_bd_entry_to_bt_addr(struct mm_struct *mm,
602 unsigned long bd_entry)
603 {
604 unsigned long bt_addr = bd_entry;
605 int align_to_bytes;
606 /*
607 * Bit 0 in a bt_entry is always the valid bit.
608 */
609 bt_addr &= ~MPX_BD_ENTRY_VALID_FLAG;
610 /*
611 * Tables are naturally aligned at 8-byte boundaries
612 * on 64-bit and 4-byte boundaries on 32-bit. The
613 * documentation makes it appear that the low bits
614 * are ignored by the hardware, so we do the same.
615 */
616 if (is_64bit_mm(mm))
617 align_to_bytes = 8;
618 else
619 align_to_bytes = 4;
620 bt_addr &= ~(align_to_bytes-1);
621 return bt_addr;
622 }
623
624 /*
625 * We only want to do a 4-byte get_user() on 32-bit. Otherwise,
626 * we might run off the end of the bounds table if we are on
627 * a 64-bit kernel and try to get 8 bytes.
628 */
629 int get_user_bd_entry(struct mm_struct *mm, unsigned long *bd_entry_ret,
630 long __user *bd_entry_ptr)
631 {
632 u32 bd_entry_32;
633 int ret;
634
635 if (is_64bit_mm(mm))
636 return get_user(*bd_entry_ret, bd_entry_ptr);
637
638 /*
639 * Note that get_user() uses the type of the *pointer* to
640 * establish the size of the get, not the destination.
641 */
642 ret = get_user(bd_entry_32, (u32 __user *)bd_entry_ptr);
643 *bd_entry_ret = bd_entry_32;
644 return ret;
645 }
646
647 /*
648 * Get the base of bounds tables pointed by specific bounds
649 * directory entry.
650 */
651 static int get_bt_addr(struct mm_struct *mm,
652 long __user *bd_entry_ptr,
653 unsigned long *bt_addr_result)
654 {
655 int ret;
656 int valid_bit;
657 unsigned long bd_entry;
658 unsigned long bt_addr;
659
660 if (!access_ok(VERIFY_READ, (bd_entry_ptr), sizeof(*bd_entry_ptr)))
661 return -EFAULT;
662
663 while (1) {
664 int need_write = 0;
665
666 pagefault_disable();
667 ret = get_user_bd_entry(mm, &bd_entry, bd_entry_ptr);
668 pagefault_enable();
669 if (!ret)
670 break;
671 if (ret == -EFAULT)
672 ret = mpx_resolve_fault(bd_entry_ptr, need_write);
673 /*
674 * If we could not resolve the fault, consider it
675 * userspace's fault and error out.
676 */
677 if (ret)
678 return ret;
679 }
680
681 valid_bit = bd_entry & MPX_BD_ENTRY_VALID_FLAG;
682 bt_addr = mpx_bd_entry_to_bt_addr(mm, bd_entry);
683
684 /*
685 * When the kernel is managing bounds tables, a bounds directory
686 * entry will either have a valid address (plus the valid bit)
687 * *OR* be completely empty. If we see a !valid entry *and* some
688 * data in the address field, we know something is wrong. This
689 * -EINVAL return will cause a SIGSEGV.
690 */
691 if (!valid_bit && bt_addr)
692 return -EINVAL;
693 /*
694 * Do we have an completely zeroed bt entry? That is OK. It
695 * just means there was no bounds table for this memory. Make
696 * sure to distinguish this from -EINVAL, which will cause
697 * a SEGV.
698 */
699 if (!valid_bit)
700 return -ENOENT;
701
702 *bt_addr_result = bt_addr;
703 return 0;
704 }
705
706 static inline int bt_entry_size_bytes(struct mm_struct *mm)
707 {
708 if (is_64bit_mm(mm))
709 return MPX_BT_ENTRY_BYTES_64;
710 else
711 return MPX_BT_ENTRY_BYTES_32;
712 }
713
714 /*
715 * Take a virtual address and turns it in to the offset in bytes
716 * inside of the bounds table where the bounds table entry
717 * controlling 'addr' can be found.
718 */
719 static unsigned long mpx_get_bt_entry_offset_bytes(struct mm_struct *mm,
720 unsigned long addr)
721 {
722 unsigned long bt_table_nr_entries;
723 unsigned long offset = addr;
724
725 if (is_64bit_mm(mm)) {
726 /* Bottom 3 bits are ignored on 64-bit */
727 offset >>= 3;
728 bt_table_nr_entries = MPX_BT_NR_ENTRIES_64;
729 } else {
730 /* Bottom 2 bits are ignored on 32-bit */
731 offset >>= 2;
732 bt_table_nr_entries = MPX_BT_NR_ENTRIES_32;
733 }
734 /*
735 * We know the size of the table in to which we are
736 * indexing, and we have eliminated all the low bits
737 * which are ignored for indexing.
738 *
739 * Mask out all the high bits which we do not need
740 * to index in to the table. Note that the tables
741 * are always powers of two so this gives us a proper
742 * mask.
743 */
744 offset &= (bt_table_nr_entries-1);
745 /*
746 * We now have an entry offset in terms of *entries* in
747 * the table. We need to scale it back up to bytes.
748 */
749 offset *= bt_entry_size_bytes(mm);
750 return offset;
751 }
752
753 /*
754 * How much virtual address space does a single bounds
755 * directory entry cover?
756 *
757 * Note, we need a long long because 4GB doesn't fit in
758 * to a long on 32-bit.
759 */
760 static inline unsigned long bd_entry_virt_space(struct mm_struct *mm)
761 {
762 unsigned long long virt_space;
763 unsigned long long GB = (1ULL << 30);
764
765 /*
766 * This covers 32-bit emulation as well as 32-bit kernels
767 * running on 64-bit harware.
768 */
769 if (!is_64bit_mm(mm))
770 return (4ULL * GB) / MPX_BD_NR_ENTRIES_32;
771
772 /*
773 * 'x86_virt_bits' returns what the hardware is capable
774 * of, and returns the full >32-bit adddress space when
775 * running 32-bit kernels on 64-bit hardware.
776 */
777 virt_space = (1ULL << boot_cpu_data.x86_virt_bits);
778 return virt_space / MPX_BD_NR_ENTRIES_64;
779 }
780
781 /*
782 * Free the backing physical pages of bounds table 'bt_addr'.
783 * Assume start...end is within that bounds table.
784 */
785 static noinline int zap_bt_entries_mapping(struct mm_struct *mm,
786 unsigned long bt_addr,
787 unsigned long start_mapping, unsigned long end_mapping)
788 {
789 struct vm_area_struct *vma;
790 unsigned long addr, len;
791 unsigned long start;
792 unsigned long end;
793
794 /*
795 * if we 'end' on a boundary, the offset will be 0 which
796 * is not what we want. Back it up a byte to get the
797 * last bt entry. Then once we have the entry itself,
798 * move 'end' back up by the table entry size.
799 */
800 start = bt_addr + mpx_get_bt_entry_offset_bytes(mm, start_mapping);
801 end = bt_addr + mpx_get_bt_entry_offset_bytes(mm, end_mapping - 1);
802 /*
803 * Move end back up by one entry. Among other things
804 * this ensures that it remains page-aligned and does
805 * not screw up zap_page_range()
806 */
807 end += bt_entry_size_bytes(mm);
808
809 /*
810 * Find the first overlapping vma. If vma->vm_start > start, there
811 * will be a hole in the bounds table. This -EINVAL return will
812 * cause a SIGSEGV.
813 */
814 vma = find_vma(mm, start);
815 if (!vma || vma->vm_start > start)
816 return -EINVAL;
817
818 /*
819 * A NUMA policy on a VM_MPX VMA could cause this bounds table to
820 * be split. So we need to look across the entire 'start -> end'
821 * range of this bounds table, find all of the VM_MPX VMAs, and
822 * zap only those.
823 */
824 addr = start;
825 while (vma && vma->vm_start < end) {
826 /*
827 * We followed a bounds directory entry down
828 * here. If we find a non-MPX VMA, that's bad,
829 * so stop immediately and return an error. This
830 * probably results in a SIGSEGV.
831 */
832 if (!(vma->vm_flags & VM_MPX))
833 return -EINVAL;
834
835 len = min(vma->vm_end, end) - addr;
836 zap_page_range(vma, addr, len, NULL);
837 trace_mpx_unmap_zap(addr, addr+len);
838
839 vma = vma->vm_next;
840 addr = vma->vm_start;
841 }
842 return 0;
843 }
844
845 static unsigned long mpx_get_bd_entry_offset(struct mm_struct *mm,
846 unsigned long addr)
847 {
848 /*
849 * There are several ways to derive the bd offsets. We
850 * use the following approach here:
851 * 1. We know the size of the virtual address space
852 * 2. We know the number of entries in a bounds table
853 * 3. We know that each entry covers a fixed amount of
854 * virtual address space.
855 * So, we can just divide the virtual address by the
856 * virtual space used by one entry to determine which
857 * entry "controls" the given virtual address.
858 */
859 if (is_64bit_mm(mm)) {
860 int bd_entry_size = 8; /* 64-bit pointer */
861 /*
862 * Take the 64-bit addressing hole in to account.
863 */
864 addr &= ((1UL << boot_cpu_data.x86_virt_bits) - 1);
865 return (addr / bd_entry_virt_space(mm)) * bd_entry_size;
866 } else {
867 int bd_entry_size = 4; /* 32-bit pointer */
868 /*
869 * 32-bit has no hole so this case needs no mask
870 */
871 return (addr / bd_entry_virt_space(mm)) * bd_entry_size;
872 }
873 /*
874 * The two return calls above are exact copies. If we
875 * pull out a single copy and put it in here, gcc won't
876 * realize that we're doing a power-of-2 divide and use
877 * shifts. It uses a real divide. If we put them up
878 * there, it manages to figure it out (gcc 4.8.3).
879 */
880 }
881
882 static int unmap_entire_bt(struct mm_struct *mm,
883 long __user *bd_entry, unsigned long bt_addr)
884 {
885 unsigned long expected_old_val = bt_addr | MPX_BD_ENTRY_VALID_FLAG;
886 unsigned long uninitialized_var(actual_old_val);
887 int ret;
888
889 while (1) {
890 int need_write = 1;
891 unsigned long cleared_bd_entry = 0;
892
893 pagefault_disable();
894 ret = mpx_cmpxchg_bd_entry(mm, &actual_old_val,
895 bd_entry, expected_old_val, cleared_bd_entry);
896 pagefault_enable();
897 if (!ret)
898 break;
899 if (ret == -EFAULT)
900 ret = mpx_resolve_fault(bd_entry, need_write);
901 /*
902 * If we could not resolve the fault, consider it
903 * userspace's fault and error out.
904 */
905 if (ret)
906 return ret;
907 }
908 /*
909 * The cmpxchg was performed, check the results.
910 */
911 if (actual_old_val != expected_old_val) {
912 /*
913 * Someone else raced with us to unmap the table.
914 * That is OK, since we were both trying to do
915 * the same thing. Declare success.
916 */
917 if (!actual_old_val)
918 return 0;
919 /*
920 * Something messed with the bounds directory
921 * entry. We hold mmap_sem for read or write
922 * here, so it could not be a _new_ bounds table
923 * that someone just allocated. Something is
924 * wrong, so pass up the error and SIGSEGV.
925 */
926 return -EINVAL;
927 }
928 /*
929 * Note, we are likely being called under do_munmap() already. To
930 * avoid recursion, do_munmap() will check whether it comes
931 * from one bounds table through VM_MPX flag.
932 */
933 return do_munmap(mm, bt_addr, mpx_bt_size_bytes(mm));
934 }
935
936 static int try_unmap_single_bt(struct mm_struct *mm,
937 unsigned long start, unsigned long end)
938 {
939 struct vm_area_struct *next;
940 struct vm_area_struct *prev;
941 /*
942 * "bta" == Bounds Table Area: the area controlled by the
943 * bounds table that we are unmapping.
944 */
945 unsigned long bta_start_vaddr = start & ~(bd_entry_virt_space(mm)-1);
946 unsigned long bta_end_vaddr = bta_start_vaddr + bd_entry_virt_space(mm);
947 unsigned long uninitialized_var(bt_addr);
948 void __user *bde_vaddr;
949 int ret;
950 /*
951 * We already unlinked the VMAs from the mm's rbtree so 'start'
952 * is guaranteed to be in a hole. This gets us the first VMA
953 * before the hole in to 'prev' and the next VMA after the hole
954 * in to 'next'.
955 */
956 next = find_vma_prev(mm, start, &prev);
957 /*
958 * Do not count other MPX bounds table VMAs as neighbors.
959 * Although theoretically possible, we do not allow bounds
960 * tables for bounds tables so our heads do not explode.
961 * If we count them as neighbors here, we may end up with
962 * lots of tables even though we have no actual table
963 * entries in use.
964 */
965 while (next && (next->vm_flags & VM_MPX))
966 next = next->vm_next;
967 while (prev && (prev->vm_flags & VM_MPX))
968 prev = prev->vm_prev;
969 /*
970 * We know 'start' and 'end' lie within an area controlled
971 * by a single bounds table. See if there are any other
972 * VMAs controlled by that bounds table. If there are not
973 * then we can "expand" the are we are unmapping to possibly
974 * cover the entire table.
975 */
976 next = find_vma_prev(mm, start, &prev);
977 if ((!prev || prev->vm_end <= bta_start_vaddr) &&
978 (!next || next->vm_start >= bta_end_vaddr)) {
979 /*
980 * No neighbor VMAs controlled by same bounds
981 * table. Try to unmap the whole thing
982 */
983 start = bta_start_vaddr;
984 end = bta_end_vaddr;
985 }
986
987 bde_vaddr = mm->bd_addr + mpx_get_bd_entry_offset(mm, start);
988 ret = get_bt_addr(mm, bde_vaddr, &bt_addr);
989 /*
990 * No bounds table there, so nothing to unmap.
991 */
992 if (ret == -ENOENT) {
993 ret = 0;
994 return 0;
995 }
996 if (ret)
997 return ret;
998 /*
999 * We are unmapping an entire table. Either because the
1000 * unmap that started this whole process was large enough
1001 * to cover an entire table, or that the unmap was small
1002 * but was the area covered by a bounds table.
1003 */
1004 if ((start == bta_start_vaddr) &&
1005 (end == bta_end_vaddr))
1006 return unmap_entire_bt(mm, bde_vaddr, bt_addr);
1007 return zap_bt_entries_mapping(mm, bt_addr, start, end);
1008 }
1009
1010 static int mpx_unmap_tables(struct mm_struct *mm,
1011 unsigned long start, unsigned long end)
1012 {
1013 unsigned long one_unmap_start;
1014 trace_mpx_unmap_search(start, end);
1015
1016 one_unmap_start = start;
1017 while (one_unmap_start < end) {
1018 int ret;
1019 unsigned long next_unmap_start = ALIGN(one_unmap_start+1,
1020 bd_entry_virt_space(mm));
1021 unsigned long one_unmap_end = end;
1022 /*
1023 * if the end is beyond the current bounds table,
1024 * move it back so we only deal with a single one
1025 * at a time
1026 */
1027 if (one_unmap_end > next_unmap_start)
1028 one_unmap_end = next_unmap_start;
1029 ret = try_unmap_single_bt(mm, one_unmap_start, one_unmap_end);
1030 if (ret)
1031 return ret;
1032
1033 one_unmap_start = next_unmap_start;
1034 }
1035 return 0;
1036 }
1037
1038 /*
1039 * Free unused bounds tables covered in a virtual address region being
1040 * munmap()ed. Assume end > start.
1041 *
1042 * This function will be called by do_munmap(), and the VMAs covering
1043 * the virtual address region start...end have already been split if
1044 * necessary, and the 'vma' is the first vma in this range (start -> end).
1045 */
1046 void mpx_notify_unmap(struct mm_struct *mm, struct vm_area_struct *vma,
1047 unsigned long start, unsigned long end)
1048 {
1049 int ret;
1050
1051 /*
1052 * Refuse to do anything unless userspace has asked
1053 * the kernel to help manage the bounds tables,
1054 */
1055 if (!kernel_managing_mpx_tables(current->mm))
1056 return;
1057 /*
1058 * This will look across the entire 'start -> end' range,
1059 * and find all of the non-VM_MPX VMAs.
1060 *
1061 * To avoid recursion, if a VM_MPX vma is found in the range
1062 * (start->end), we will not continue follow-up work. This
1063 * recursion represents having bounds tables for bounds tables,
1064 * which should not occur normally. Being strict about it here
1065 * helps ensure that we do not have an exploitable stack overflow.
1066 */
1067 do {
1068 if (vma->vm_flags & VM_MPX)
1069 return;
1070 vma = vma->vm_next;
1071 } while (vma && vma->vm_start < end);
1072
1073 ret = mpx_unmap_tables(mm, start, end);
1074 if (ret)
1075 force_sig(SIGSEGV, current);
1076 }
1077
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.