~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/alpha/lib/memcpy.c

Version: ~ [ linux-5.13-rc1 ] ~ [ linux-5.12.2 ] ~ [ linux-5.11.19 ] ~ [ linux-5.10.35 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.117 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.190 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.232 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.268 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.268 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 /*
  3  *  linux/arch/alpha/lib/memcpy.c
  4  *
  5  *  Copyright (C) 1995  Linus Torvalds
  6  */
  7 
  8 /*
  9  * This is a reasonably optimized memcpy() routine.
 10  */
 11 
 12 /*
 13  * Note that the C code is written to be optimized into good assembly. However,
 14  * at this point gcc is unable to sanely compile "if (n >= 0)", resulting in a
 15  * explicit compare against 0 (instead of just using the proper "blt reg, xx" or
 16  * "bge reg, xx"). I hope alpha-gcc will be fixed to notice this eventually..
 17  */
 18 
 19 #include <linux/types.h>
 20 #include <linux/export.h>
 21 
 22 /*
 23  * This should be done in one go with ldq_u*2/mask/stq_u. Do it
 24  * with a macro so that we can fix it up later..
 25  */
 26 #define ALIGN_DEST_TO8_UP(d,s,n) \
 27         while (d & 7) { \
 28                 if (n <= 0) return; \
 29                 n--; \
 30                 *(char *) d = *(char *) s; \
 31                 d++; s++; \
 32         }
 33 #define ALIGN_DEST_TO8_DN(d,s,n) \
 34         while (d & 7) { \
 35                 if (n <= 0) return; \
 36                 n--; \
 37                 d--; s--; \
 38                 *(char *) d = *(char *) s; \
 39         }
 40 
 41 /*
 42  * This should similarly be done with ldq_u*2/mask/stq. The destination
 43  * is aligned, but we don't fill in a full quad-word
 44  */
 45 #define DO_REST_UP(d,s,n) \
 46         while (n > 0) { \
 47                 n--; \
 48                 *(char *) d = *(char *) s; \
 49                 d++; s++; \
 50         }
 51 #define DO_REST_DN(d,s,n) \
 52         while (n > 0) { \
 53                 n--; \
 54                 d--; s--; \
 55                 *(char *) d = *(char *) s; \
 56         }
 57 
 58 /*
 59  * This should be done with ldq/mask/stq. The source and destination are
 60  * aligned, but we don't fill in a full quad-word
 61  */
 62 #define DO_REST_ALIGNED_UP(d,s,n) DO_REST_UP(d,s,n)
 63 #define DO_REST_ALIGNED_DN(d,s,n) DO_REST_DN(d,s,n)
 64 
 65 /*
 66  * This does unaligned memory copies. We want to avoid storing to
 67  * an unaligned address, as that would do a read-modify-write cycle.
 68  * We also want to avoid double-reading the unaligned reads.
 69  *
 70  * Note the ordering to try to avoid load (and address generation) latencies.
 71  */
 72 static inline void __memcpy_unaligned_up (unsigned long d, unsigned long s,
 73                                           long n)
 74 {
 75         ALIGN_DEST_TO8_UP(d,s,n);
 76         n -= 8;                 /* to avoid compare against 8 in the loop */
 77         if (n >= 0) {
 78                 unsigned long low_word, high_word;
 79                 __asm__("ldq_u %0,%1":"=r" (low_word):"m" (*(unsigned long *) s));
 80                 do {
 81                         unsigned long tmp;
 82                         __asm__("ldq_u %0,%1":"=r" (high_word):"m" (*(unsigned long *)(s+8)));
 83                         n -= 8;
 84                         __asm__("extql %1,%2,%0"
 85                                 :"=r" (low_word)
 86                                 :"r" (low_word), "r" (s));
 87                         __asm__("extqh %1,%2,%0"
 88                                 :"=r" (tmp)
 89                                 :"r" (high_word), "r" (s));
 90                         s += 8;
 91                         *(unsigned long *) d = low_word | tmp;
 92                         d += 8;
 93                         low_word = high_word;
 94                 } while (n >= 0);
 95         }
 96         n += 8;
 97         DO_REST_UP(d,s,n);
 98 }
 99 
100 static inline void __memcpy_unaligned_dn (unsigned long d, unsigned long s,
101                                           long n)
102 {
103         /* I don't understand AXP assembler well enough for this. -Tim */
104         s += n;
105         d += n;
106         while (n--)
107                 * (char *) --d = * (char *) --s;
108 }
109 
110 /*
111  * Hmm.. Strange. The __asm__ here is there to make gcc use an integer register
112  * for the load-store. I don't know why, but it would seem that using a floating
113  * point register for the move seems to slow things down (very small difference,
114  * though).
115  *
116  * Note the ordering to try to avoid load (and address generation) latencies.
117  */
118 static inline void __memcpy_aligned_up (unsigned long d, unsigned long s,
119                                         long n)
120 {
121         ALIGN_DEST_TO8_UP(d,s,n);
122         n -= 8;
123         while (n >= 0) {
124                 unsigned long tmp;
125                 __asm__("ldq %0,%1":"=r" (tmp):"m" (*(unsigned long *) s));
126                 n -= 8;
127                 s += 8;
128                 *(unsigned long *) d = tmp;
129                 d += 8;
130         }
131         n += 8;
132         DO_REST_ALIGNED_UP(d,s,n);
133 }
134 static inline void __memcpy_aligned_dn (unsigned long d, unsigned long s,
135                                         long n)
136 {
137         s += n;
138         d += n;
139         ALIGN_DEST_TO8_DN(d,s,n);
140         n -= 8;
141         while (n >= 0) {
142                 unsigned long tmp;
143                 s -= 8;
144                 __asm__("ldq %0,%1":"=r" (tmp):"m" (*(unsigned long *) s));
145                 n -= 8;
146                 d -= 8;
147                 *(unsigned long *) d = tmp;
148         }
149         n += 8;
150         DO_REST_ALIGNED_DN(d,s,n);
151 }
152 
153 void * memcpy(void * dest, const void *src, size_t n)
154 {
155         if (!(((unsigned long) dest ^ (unsigned long) src) & 7)) {
156                 __memcpy_aligned_up ((unsigned long) dest, (unsigned long) src,
157                                      n);
158                 return dest;
159         }
160         __memcpy_unaligned_up ((unsigned long) dest, (unsigned long) src, n);
161         return dest;
162 }
163 EXPORT_SYMBOL(memcpy);
164 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp