~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/include/asm/uaccess_64.h

Version: ~ [ linux-5.16 ] ~ [ linux-5.15.13 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.90 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.170 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.224 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.261 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.296 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.298 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 #ifndef _ASM_X86_UACCESS_64_H
  2 #define _ASM_X86_UACCESS_64_H
  3 
  4 /*
  5  * User space memory access functions
  6  */
  7 #include <linux/compiler.h>
  8 #include <linux/lockdep.h>
  9 #include <linux/kasan-checks.h>
 10 #include <asm/alternative.h>
 11 #include <asm/cpufeatures.h>
 12 #include <asm/page.h>
 13 
 14 /*
 15  * Copy To/From Userspace
 16  */
 17 
 18 /* Handles exceptions in both to and from, but doesn't do access_ok */
 19 __must_check unsigned long
 20 copy_user_enhanced_fast_string(void *to, const void *from, unsigned len);
 21 __must_check unsigned long
 22 copy_user_generic_string(void *to, const void *from, unsigned len);
 23 __must_check unsigned long
 24 copy_user_generic_unrolled(void *to, const void *from, unsigned len);
 25 
 26 static __always_inline __must_check unsigned long
 27 copy_user_generic(void *to, const void *from, unsigned len)
 28 {
 29         unsigned ret;
 30 
 31         /*
 32          * If CPU has ERMS feature, use copy_user_enhanced_fast_string.
 33          * Otherwise, if CPU has rep_good feature, use copy_user_generic_string.
 34          * Otherwise, use copy_user_generic_unrolled.
 35          */
 36         alternative_call_2(copy_user_generic_unrolled,
 37                          copy_user_generic_string,
 38                          X86_FEATURE_REP_GOOD,
 39                          copy_user_enhanced_fast_string,
 40                          X86_FEATURE_ERMS,
 41                          ASM_OUTPUT2("=a" (ret), "=D" (to), "=S" (from),
 42                                      "=d" (len)),
 43                          "1" (to), "2" (from), "3" (len)
 44                          : "memory", "rcx", "r8", "r9", "r10", "r11");
 45         return ret;
 46 }
 47 
 48 static __always_inline __must_check unsigned long
 49 raw_copy_from_user(void *dst, const void __user *src, unsigned long size)
 50 {
 51         int ret = 0;
 52 
 53         if (!__builtin_constant_p(size))
 54                 return copy_user_generic(dst, (__force void *)src, size);
 55         switch (size) {
 56         case 1:
 57                 __uaccess_begin();
 58                 __get_user_asm_nozero(*(u8 *)dst, (u8 __user *)src,
 59                               ret, "b", "b", "=q", 1);
 60                 __uaccess_end();
 61                 return ret;
 62         case 2:
 63                 __uaccess_begin();
 64                 __get_user_asm_nozero(*(u16 *)dst, (u16 __user *)src,
 65                               ret, "w", "w", "=r", 2);
 66                 __uaccess_end();
 67                 return ret;
 68         case 4:
 69                 __uaccess_begin();
 70                 __get_user_asm_nozero(*(u32 *)dst, (u32 __user *)src,
 71                               ret, "l", "k", "=r", 4);
 72                 __uaccess_end();
 73                 return ret;
 74         case 8:
 75                 __uaccess_begin();
 76                 __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src,
 77                               ret, "q", "", "=r", 8);
 78                 __uaccess_end();
 79                 return ret;
 80         case 10:
 81                 __uaccess_begin();
 82                 __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src,
 83                                ret, "q", "", "=r", 10);
 84                 if (likely(!ret))
 85                         __get_user_asm_nozero(*(u16 *)(8 + (char *)dst),
 86                                        (u16 __user *)(8 + (char __user *)src),
 87                                        ret, "w", "w", "=r", 2);
 88                 __uaccess_end();
 89                 return ret;
 90         case 16:
 91                 __uaccess_begin();
 92                 __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src,
 93                                ret, "q", "", "=r", 16);
 94                 if (likely(!ret))
 95                         __get_user_asm_nozero(*(u64 *)(8 + (char *)dst),
 96                                        (u64 __user *)(8 + (char __user *)src),
 97                                        ret, "q", "", "=r", 8);
 98                 __uaccess_end();
 99                 return ret;
100         default:
101                 return copy_user_generic(dst, (__force void *)src, size);
102         }
103 }
104 
105 static __always_inline __must_check unsigned long
106 raw_copy_to_user(void __user *dst, const void *src, unsigned long size)
107 {
108         int ret = 0;
109 
110         if (!__builtin_constant_p(size))
111                 return copy_user_generic((__force void *)dst, src, size);
112         switch (size) {
113         case 1:
114                 __uaccess_begin();
115                 __put_user_asm(*(u8 *)src, (u8 __user *)dst,
116                               ret, "b", "b", "iq", 1);
117                 __uaccess_end();
118                 return ret;
119         case 2:
120                 __uaccess_begin();
121                 __put_user_asm(*(u16 *)src, (u16 __user *)dst,
122                               ret, "w", "w", "ir", 2);
123                 __uaccess_end();
124                 return ret;
125         case 4:
126                 __uaccess_begin();
127                 __put_user_asm(*(u32 *)src, (u32 __user *)dst,
128                               ret, "l", "k", "ir", 4);
129                 __uaccess_end();
130                 return ret;
131         case 8:
132                 __uaccess_begin();
133                 __put_user_asm(*(u64 *)src, (u64 __user *)dst,
134                               ret, "q", "", "er", 8);
135                 __uaccess_end();
136                 return ret;
137         case 10:
138                 __uaccess_begin();
139                 __put_user_asm(*(u64 *)src, (u64 __user *)dst,
140                                ret, "q", "", "er", 10);
141                 if (likely(!ret)) {
142                         asm("":::"memory");
143                         __put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst,
144                                        ret, "w", "w", "ir", 2);
145                 }
146                 __uaccess_end();
147                 return ret;
148         case 16:
149                 __uaccess_begin();
150                 __put_user_asm(*(u64 *)src, (u64 __user *)dst,
151                                ret, "q", "", "er", 16);
152                 if (likely(!ret)) {
153                         asm("":::"memory");
154                         __put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst,
155                                        ret, "q", "", "er", 8);
156                 }
157                 __uaccess_end();
158                 return ret;
159         default:
160                 return copy_user_generic((__force void *)dst, src, size);
161         }
162 }
163 
164 static __always_inline __must_check
165 unsigned long raw_copy_in_user(void __user *dst, const void __user *src, unsigned long size)
166 {
167         return copy_user_generic((__force void *)dst,
168                                  (__force void *)src, size);
169 }
170 
171 extern long __copy_user_nocache(void *dst, const void __user *src,
172                                 unsigned size, int zerorest);
173 
174 static inline int
175 __copy_from_user_inatomic_nocache(void *dst, const void __user *src,
176                                   unsigned size)
177 {
178         kasan_check_write(dst, size);
179         return __copy_user_nocache(dst, src, size, 0);
180 }
181 
182 unsigned long
183 copy_user_handle_tail(char *to, char *from, unsigned len);
184 
185 #endif /* _ASM_X86_UACCESS_64_H */
186 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp