PageRenderTime 22ms CodeModel.GetById 16ms app.highlight 4ms RepoModel.GetById 0ms app.codeStats 0ms

/arch/x86/lib/memcpy_32.c

https://bitbucket.org/ndreys/linux-sunxi
C | 208 lines | 146 code | 20 blank | 42 comment | 0 complexity | dd2bb4b37348d16c772565205144d035 MD5 | raw file
Possible License(s): GPL-2.0, LGPL-2.0, AGPL-1.0
  1#include <linux/string.h>
  2#include <linux/module.h>
  3
  4#undef memcpy
  5#undef memset
  6
  7void *memcpy(void *to, const void *from, size_t n)
  8{
  9#ifdef CONFIG_X86_USE_3DNOW
 10	return __memcpy3d(to, from, n);
 11#else
 12	return __memcpy(to, from, n);
 13#endif
 14}
 15EXPORT_SYMBOL(memcpy);
 16
 17void *memset(void *s, int c, size_t count)
 18{
 19	return __memset(s, c, count);
 20}
 21EXPORT_SYMBOL(memset);
 22
 23void *memmove(void *dest, const void *src, size_t n)
 24{
 25	int d0,d1,d2,d3,d4,d5;
 26	char *ret = dest;
 27
 28	__asm__ __volatile__(
 29		/* Handle more 16bytes in loop */
 30		"cmp $0x10, %0\n\t"
 31		"jb	1f\n\t"
 32
 33		/* Decide forward/backward copy mode */
 34		"cmp %2, %1\n\t"
 35		"jb	2f\n\t"
 36
 37		/*
 38		 * movs instruction have many startup latency
 39		 * so we handle small size by general register.
 40		 */
 41		"cmp  $680, %0\n\t"
 42		"jb 3f\n\t"
 43		/*
 44		 * movs instruction is only good for aligned case.
 45		 */
 46		"mov %1, %3\n\t"
 47		"xor %2, %3\n\t"
 48		"and $0xff, %3\n\t"
 49		"jz 4f\n\t"
 50		"3:\n\t"
 51		"sub $0x10, %0\n\t"
 52
 53		/*
 54		 * We gobble 16byts forward in each loop.
 55		 */
 56		"3:\n\t"
 57		"sub $0x10, %0\n\t"
 58		"mov 0*4(%1), %3\n\t"
 59		"mov 1*4(%1), %4\n\t"
 60		"mov  %3, 0*4(%2)\n\t"
 61		"mov  %4, 1*4(%2)\n\t"
 62		"mov 2*4(%1), %3\n\t"
 63		"mov 3*4(%1), %4\n\t"
 64		"mov  %3, 2*4(%2)\n\t"
 65		"mov  %4, 3*4(%2)\n\t"
 66		"lea  0x10(%1), %1\n\t"
 67		"lea  0x10(%2), %2\n\t"
 68		"jae 3b\n\t"
 69		"add $0x10, %0\n\t"
 70		"jmp 1f\n\t"
 71
 72		/*
 73		 * Handle data forward by movs.
 74		 */
 75		".p2align 4\n\t"
 76		"4:\n\t"
 77		"mov -4(%1, %0), %3\n\t"
 78		"lea -4(%2, %0), %4\n\t"
 79		"shr $2, %0\n\t"
 80		"rep movsl\n\t"
 81		"mov %3, (%4)\n\t"
 82		"jmp 11f\n\t"
 83		/*
 84		 * Handle data backward by movs.
 85		 */
 86		".p2align 4\n\t"
 87		"6:\n\t"
 88		"mov (%1), %3\n\t"
 89		"mov %2, %4\n\t"
 90		"lea -4(%1, %0), %1\n\t"
 91		"lea -4(%2, %0), %2\n\t"
 92		"shr $2, %0\n\t"
 93		"std\n\t"
 94		"rep movsl\n\t"
 95		"mov %3,(%4)\n\t"
 96		"cld\n\t"
 97		"jmp 11f\n\t"
 98
 99		/*
100		 * Start to prepare for backward copy.
101		 */
102		".p2align 4\n\t"
103		"2:\n\t"
104		"cmp  $680, %0\n\t"
105		"jb 5f\n\t"
106		"mov %1, %3\n\t"
107		"xor %2, %3\n\t"
108		"and $0xff, %3\n\t"
109		"jz 6b\n\t"
110
111		/*
112		 * Calculate copy position to tail.
113		 */
114		"5:\n\t"
115		"add %0, %1\n\t"
116		"add %0, %2\n\t"
117		"sub $0x10, %0\n\t"
118
119		/*
120		 * We gobble 16byts backward in each loop.
121		 */
122		"7:\n\t"
123		"sub $0x10, %0\n\t"
124
125		"mov -1*4(%1), %3\n\t"
126		"mov -2*4(%1), %4\n\t"
127		"mov  %3, -1*4(%2)\n\t"
128		"mov  %4, -2*4(%2)\n\t"
129		"mov -3*4(%1), %3\n\t"
130		"mov -4*4(%1), %4\n\t"
131		"mov  %3, -3*4(%2)\n\t"
132		"mov  %4, -4*4(%2)\n\t"
133		"lea  -0x10(%1), %1\n\t"
134		"lea  -0x10(%2), %2\n\t"
135		"jae 7b\n\t"
136		/*
137		 * Calculate copy position to head.
138		 */
139		"add $0x10, %0\n\t"
140		"sub %0, %1\n\t"
141		"sub %0, %2\n\t"
142
143		/*
144		 * Move data from 8 bytes to 15 bytes.
145		 */
146		".p2align 4\n\t"
147		"1:\n\t"
148		"cmp $8, %0\n\t"
149		"jb 8f\n\t"
150		"mov 0*4(%1), %3\n\t"
151		"mov 1*4(%1), %4\n\t"
152		"mov -2*4(%1, %0), %5\n\t"
153		"mov -1*4(%1, %0), %1\n\t"
154
155		"mov  %3, 0*4(%2)\n\t"
156		"mov  %4, 1*4(%2)\n\t"
157		"mov  %5, -2*4(%2, %0)\n\t"
158		"mov  %1, -1*4(%2, %0)\n\t"
159		"jmp 11f\n\t"
160
161		/*
162		 * Move data from 4 bytes to 7 bytes.
163		 */
164		".p2align 4\n\t"
165		"8:\n\t"
166		"cmp $4, %0\n\t"
167		"jb 9f\n\t"
168		"mov 0*4(%1), %3\n\t"
169		"mov -1*4(%1, %0), %4\n\t"
170		"mov  %3, 0*4(%2)\n\t"
171		"mov  %4, -1*4(%2, %0)\n\t"
172		"jmp 11f\n\t"
173
174		/*
175		 * Move data from 2 bytes to 3 bytes.
176		 */
177		".p2align 4\n\t"
178		"9:\n\t"
179		"cmp $2, %0\n\t"
180		"jb 10f\n\t"
181		"movw 0*2(%1), %%dx\n\t"
182		"movw -1*2(%1, %0), %%bx\n\t"
183		"movw %%dx, 0*2(%2)\n\t"
184		"movw %%bx, -1*2(%2, %0)\n\t"
185		"jmp 11f\n\t"
186
187		/*
188		 * Move data for 1 byte.
189		 */
190		".p2align 4\n\t"
191		"10:\n\t"
192		"cmp $1, %0\n\t"
193		"jb 11f\n\t"
194		"movb (%1), %%cl\n\t"
195		"movb %%cl, (%2)\n\t"
196		".p2align 4\n\t"
197		"11:"
198		: "=&c" (d0), "=&S" (d1), "=&D" (d2),
199		  "=r" (d3),"=r" (d4), "=r"(d5)
200		:"0" (n),
201		 "1" (src),
202		 "2" (dest)
203		:"memory");
204
205	return ret;
206
207}
208EXPORT_SYMBOL(memmove);