~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/lib/memcpy_32.c

Version: ~ [ linux-5.3-rc5 ] ~ [ linux-5.2.9 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.67 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.139 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.189 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.189 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.72 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 #include <linux/string.h>
  2 #include <linux/module.h>
  3 
  4 #undef memcpy
  5 #undef memset
  6 
  7 void *memcpy(void *to, const void *from, size_t n)
  8 {
  9 #ifdef CONFIG_X86_USE_3DNOW
 10         return __memcpy3d(to, from, n);
 11 #else
 12         return __memcpy(to, from, n);
 13 #endif
 14 }
 15 EXPORT_SYMBOL(memcpy);
 16 
 17 void *memset(void *s, int c, size_t count)
 18 {
 19         return __memset(s, c, count);
 20 }
 21 EXPORT_SYMBOL(memset);
 22 
 23 void *memmove(void *dest, const void *src, size_t n)
 24 {
 25         int d0,d1,d2,d3,d4,d5;
 26         char *ret = dest;
 27 
 28         __asm__ __volatile__(
 29                 /* Handle more 16bytes in loop */
 30                 "cmp $0x10, %0\n\t"
 31                 "jb     1f\n\t"
 32 
 33                 /* Decide forward/backward copy mode */
 34                 "cmp %2, %1\n\t"
 35                 "jb     2f\n\t"
 36 
 37                 /*
 38                  * movs instruction have many startup latency
 39                  * so we handle small size by general register.
 40                  */
 41                 "cmp  $680, %0\n\t"
 42                 "jb 3f\n\t"
 43                 /*
 44                  * movs instruction is only good for aligned case.
 45                  */
 46                 "mov %1, %3\n\t"
 47                 "xor %2, %3\n\t"
 48                 "and $0xff, %3\n\t"
 49                 "jz 4f\n\t"
 50                 "3:\n\t"
 51                 "sub $0x10, %0\n\t"
 52 
 53                 /*
 54                  * We gobble 16byts forward in each loop.
 55                  */
 56                 "3:\n\t"
 57                 "sub $0x10, %0\n\t"
 58                 "mov 0*4(%1), %3\n\t"
 59                 "mov 1*4(%1), %4\n\t"
 60                 "mov  %3, 0*4(%2)\n\t"
 61                 "mov  %4, 1*4(%2)\n\t"
 62                 "mov 2*4(%1), %3\n\t"
 63                 "mov 3*4(%1), %4\n\t"
 64                 "mov  %3, 2*4(%2)\n\t"
 65                 "mov  %4, 3*4(%2)\n\t"
 66                 "lea  0x10(%1), %1\n\t"
 67                 "lea  0x10(%2), %2\n\t"
 68                 "jae 3b\n\t"
 69                 "add $0x10, %0\n\t"
 70                 "jmp 1f\n\t"
 71 
 72                 /*
 73                  * Handle data forward by movs.
 74                  */
 75                 ".p2align 4\n\t"
 76                 "4:\n\t"
 77                 "mov -4(%1, %0), %3\n\t"
 78                 "lea -4(%2, %0), %4\n\t"
 79                 "shr $2, %0\n\t"
 80                 "rep movsl\n\t"
 81                 "mov %3, (%4)\n\t"
 82                 "jmp 11f\n\t"
 83                 /*
 84                  * Handle data backward by movs.
 85                  */
 86                 ".p2align 4\n\t"
 87                 "6:\n\t"
 88                 "mov (%1), %3\n\t"
 89                 "mov %2, %4\n\t"
 90                 "lea -4(%1, %0), %1\n\t"
 91                 "lea -4(%2, %0), %2\n\t"
 92                 "shr $2, %0\n\t"
 93                 "std\n\t"
 94                 "rep movsl\n\t"
 95                 "mov %3,(%4)\n\t"
 96                 "cld\n\t"
 97                 "jmp 11f\n\t"
 98 
 99                 /*
100                  * Start to prepare for backward copy.
101                  */
102                 ".p2align 4\n\t"
103                 "2:\n\t"
104                 "cmp  $680, %0\n\t"
105                 "jb 5f\n\t"
106                 "mov %1, %3\n\t"
107                 "xor %2, %3\n\t"
108                 "and $0xff, %3\n\t"
109                 "jz 6b\n\t"
110 
111                 /*
112                  * Calculate copy position to tail.
113                  */
114                 "5:\n\t"
115                 "add %0, %1\n\t"
116                 "add %0, %2\n\t"
117                 "sub $0x10, %0\n\t"
118 
119                 /*
120                  * We gobble 16byts backward in each loop.
121                  */
122                 "7:\n\t"
123                 "sub $0x10, %0\n\t"
124 
125                 "mov -1*4(%1), %3\n\t"
126                 "mov -2*4(%1), %4\n\t"
127                 "mov  %3, -1*4(%2)\n\t"
128                 "mov  %4, -2*4(%2)\n\t"
129                 "mov -3*4(%1), %3\n\t"
130                 "mov -4*4(%1), %4\n\t"
131                 "mov  %3, -3*4(%2)\n\t"
132                 "mov  %4, -4*4(%2)\n\t"
133                 "lea  -0x10(%1), %1\n\t"
134                 "lea  -0x10(%2), %2\n\t"
135                 "jae 7b\n\t"
136                 /*
137                  * Calculate copy position to head.
138                  */
139                 "add $0x10, %0\n\t"
140                 "sub %0, %1\n\t"
141                 "sub %0, %2\n\t"
142 
143                 /*
144                  * Move data from 8 bytes to 15 bytes.
145                  */
146                 ".p2align 4\n\t"
147                 "1:\n\t"
148                 "cmp $8, %0\n\t"
149                 "jb 8f\n\t"
150                 "mov 0*4(%1), %3\n\t"
151                 "mov 1*4(%1), %4\n\t"
152                 "mov -2*4(%1, %0), %5\n\t"
153                 "mov -1*4(%1, %0), %1\n\t"
154 
155                 "mov  %3, 0*4(%2)\n\t"
156                 "mov  %4, 1*4(%2)\n\t"
157                 "mov  %5, -2*4(%2, %0)\n\t"
158                 "mov  %1, -1*4(%2, %0)\n\t"
159                 "jmp 11f\n\t"
160 
161                 /*
162                  * Move data from 4 bytes to 7 bytes.
163                  */
164                 ".p2align 4\n\t"
165                 "8:\n\t"
166                 "cmp $4, %0\n\t"
167                 "jb 9f\n\t"
168                 "mov 0*4(%1), %3\n\t"
169                 "mov -1*4(%1, %0), %4\n\t"
170                 "mov  %3, 0*4(%2)\n\t"
171                 "mov  %4, -1*4(%2, %0)\n\t"
172                 "jmp 11f\n\t"
173 
174                 /*
175                  * Move data from 2 bytes to 3 bytes.
176                  */
177                 ".p2align 4\n\t"
178                 "9:\n\t"
179                 "cmp $2, %0\n\t"
180                 "jb 10f\n\t"
181                 "movw 0*2(%1), %%dx\n\t"
182                 "movw -1*2(%1, %0), %%bx\n\t"
183                 "movw %%dx, 0*2(%2)\n\t"
184                 "movw %%bx, -1*2(%2, %0)\n\t"
185                 "jmp 11f\n\t"
186 
187                 /*
188                  * Move data for 1 byte.
189                  */
190                 ".p2align 4\n\t"
191                 "10:\n\t"
192                 "cmp $1, %0\n\t"
193                 "jb 11f\n\t"
194                 "movb (%1), %%cl\n\t"
195                 "movb %%cl, (%2)\n\t"
196                 ".p2align 4\n\t"
197                 "11:"
198                 : "=&c" (d0), "=&S" (d1), "=&D" (d2),
199                   "=r" (d3),"=r" (d4), "=r"(d5)
200                 :"" (n),
201                  "1" (src),
202                  "2" (dest)
203                 :"memory");
204 
205         return ret;
206 
207 }
208 EXPORT_SYMBOL(memmove);
209 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp