PageRenderTime 31ms CodeModel.GetById 9ms app.highlight 13ms RepoModel.GetById 0ms app.codeStats 0ms

/arch/xtensa/lib/usercopy.S

https://bitbucket.org/thekraven/iscream_thunderc-2.6.35
Assembly | 321 lines | 298 code | 23 blank | 0 comment | 25 complexity | 824dc57f0279834e297cd70893801fbe MD5 | raw file
Possible License(s): GPL-2.0, LGPL-2.0, AGPL-1.0
  1/*
  2 *  arch/xtensa/lib/usercopy.S
  3 *
  4 *  Copy to/from user space (derived from arch/xtensa/lib/hal/memcopy.S)
  5 *
  6 *  DO NOT COMBINE this function with <arch/xtensa/lib/hal/memcopy.S>.
  7 *  It needs to remain separate and distinct.  The hal files are part
  8 *  of the Xtensa link-time HAL, and those files may differ per
  9 *  processor configuration.  Patching the kernel for another
 10 *  processor configuration includes replacing the hal files, and we
 11 *  could lose the special functionality for accessing user-space
 12 *  memory during such a patch.  We sacrifice a little code space here
 13 *  in favor to simplify code maintenance.
 14 *
 15 *  This file is subject to the terms and conditions of the GNU General
 16 *  Public License.  See the file "COPYING" in the main directory of
 17 *  this archive for more details.
 18 *
 19 *  Copyright (C) 2002 Tensilica Inc.
 20 */
 21
 22
 23/*
 24 * size_t __xtensa_copy_user (void *dst, const void *src, size_t len);
 25 *
 26 * The returned value is the number of bytes not copied.  Implies zero
 27 * is success.
 28 *
 29 * The general case algorithm is as follows:
 30 *   If the destination and source are both aligned,
 31 *     do 16B chunks with a loop, and then finish up with
 32 *     8B, 4B, 2B, and 1B copies conditional on the length.
 33 *   If destination is aligned and source unaligned,
 34 *     do the same, but use SRC to align the source data.
 35 *   If destination is unaligned, align it by conditionally
 36 *     copying 1B and 2B and then retest.
 37 *   This code tries to use fall-through braches for the common
 38 *     case of aligned destinations (except for the branches to
 39 *     the alignment label).
 40 *
 41 * Register use:
 42 *	a0/ return address
 43 *	a1/ stack pointer
 44 *	a2/ return value
 45 *	a3/ src
 46 *	a4/ length
 47 *	a5/ dst
 48 *	a6/ tmp
 49 *	a7/ tmp
 50 *	a8/ tmp
 51 *	a9/ tmp
 52 *	a10/ tmp
 53 *	a11/ original length
 54 */
 55
 56#include <variant/core.h>
 57
 58#ifdef __XTENSA_EB__
 59#define ALIGN(R, W0, W1) src	R, W0, W1
 60#define SSA8(R)	ssa8b R
 61#else
 62#define ALIGN(R, W0, W1) src	R, W1, W0
 63#define SSA8(R)	ssa8l R
 64#endif
 65
 66/* Load or store instructions that may cause exceptions use the EX macro. */
 67
 68#define EX(insn,reg1,reg2,offset,handler)	\
 699:	insn	reg1, reg2, offset;		\
 70	.section __ex_table, "a";		\
 71	.word	9b, handler;			\
 72	.previous
 73
 74
 75	.text
 76	.align	4
 77	.global	__xtensa_copy_user
 78	.type	__xtensa_copy_user,@function
 79__xtensa_copy_user:
 80	entry	sp, 16		# minimal stack frame
 81	# a2/ dst, a3/ src, a4/ len
 82	mov	a5, a2		# copy dst so that a2 is return value
 83	mov	a11, a4		# preserve original len for error case
 84.Lcommon:
 85	bbsi.l	a2, 0, .Ldst1mod2 # if dst is 1 mod 2
 86	bbsi.l	a2, 1, .Ldst2mod4 # if dst is 2 mod 4
 87.Ldstaligned:	# return here from .Ldstunaligned when dst is aligned
 88	srli	a7, a4, 4	# number of loop iterations with 16B
 89				# per iteration
 90	movi	a8, 3		  # if source is also aligned,
 91	bnone	a3, a8, .Laligned # then use word copy
 92	SSA8(	a3)		# set shift amount from byte offset
 93	bnez	a4, .Lsrcunaligned
 94	movi	a2, 0		# return success for len==0
 95	retw
 96
 97/*
 98 * Destination is unaligned
 99 */
100
101.Ldst1mod2:	# dst is only byte aligned
102	bltui	a4, 7, .Lbytecopy	# do short copies byte by byte
103
104	# copy 1 byte
105	EX(l8ui, a6, a3, 0, l_fixup)
106	addi	a3, a3,  1
107	EX(s8i, a6, a5,  0, s_fixup)
108	addi	a5, a5,  1
109	addi	a4, a4, -1
110	bbci.l	a5, 1, .Ldstaligned	# if dst is now aligned, then
111					# return to main algorithm
112.Ldst2mod4:	# dst 16-bit aligned
113	# copy 2 bytes
114	bltui	a4, 6, .Lbytecopy	# do short copies byte by byte
115	EX(l8ui, a6, a3, 0, l_fixup)
116	EX(l8ui, a7, a3, 1, l_fixup)
117	addi	a3, a3,  2
118	EX(s8i, a6, a5,  0, s_fixup)
119	EX(s8i, a7, a5,  1, s_fixup)
120	addi	a5, a5,  2
121	addi	a4, a4, -2
122	j	.Ldstaligned	# dst is now aligned, return to main algorithm
123
124/*
125 * Byte by byte copy
126 */
127	.align	4
128	.byte	0		# 1 mod 4 alignment for LOOPNEZ
129				# (0 mod 4 alignment for LBEG)
130.Lbytecopy:
131#if XCHAL_HAVE_LOOPS
132	loopnez	a4, .Lbytecopydone
133#else /* !XCHAL_HAVE_LOOPS */
134	beqz	a4, .Lbytecopydone
135	add	a7, a3, a4	# a7 = end address for source
136#endif /* !XCHAL_HAVE_LOOPS */
137.Lnextbyte:
138	EX(l8ui, a6, a3, 0, l_fixup)
139	addi	a3, a3, 1
140	EX(s8i, a6, a5, 0, s_fixup)
141	addi	a5, a5, 1
142#if !XCHAL_HAVE_LOOPS
143	blt	a3, a7, .Lnextbyte
144#endif /* !XCHAL_HAVE_LOOPS */
145.Lbytecopydone:
146	movi	a2, 0		# return success for len bytes copied
147	retw
148
149/*
150 * Destination and source are word-aligned.
151 */
152	# copy 16 bytes per iteration for word-aligned dst and word-aligned src
153	.align	4		# 1 mod 4 alignment for LOOPNEZ
154	.byte	0		# (0 mod 4 alignment for LBEG)
155.Laligned:
156#if XCHAL_HAVE_LOOPS
157	loopnez	a7, .Loop1done
158#else /* !XCHAL_HAVE_LOOPS */
159	beqz	a7, .Loop1done
160	slli	a8, a7, 4
161	add	a8, a8, a3	# a8 = end of last 16B source chunk
162#endif /* !XCHAL_HAVE_LOOPS */
163.Loop1:
164	EX(l32i, a6, a3,  0, l_fixup)
165	EX(l32i, a7, a3,  4, l_fixup)
166	EX(s32i, a6, a5,  0, s_fixup)
167	EX(l32i, a6, a3,  8, l_fixup)
168	EX(s32i, a7, a5,  4, s_fixup)
169	EX(l32i, a7, a3, 12, l_fixup)
170	EX(s32i, a6, a5,  8, s_fixup)
171	addi	a3, a3, 16
172	EX(s32i, a7, a5, 12, s_fixup)
173	addi	a5, a5, 16
174#if !XCHAL_HAVE_LOOPS
175	blt	a3, a8, .Loop1
176#endif /* !XCHAL_HAVE_LOOPS */
177.Loop1done:
178	bbci.l	a4, 3, .L2
179	# copy 8 bytes
180	EX(l32i, a6, a3,  0, l_fixup)
181	EX(l32i, a7, a3,  4, l_fixup)
182	addi	a3, a3,  8
183	EX(s32i, a6, a5,  0, s_fixup)
184	EX(s32i, a7, a5,  4, s_fixup)
185	addi	a5, a5,  8
186.L2:
187	bbci.l	a4, 2, .L3
188	# copy 4 bytes
189	EX(l32i, a6, a3,  0, l_fixup)
190	addi	a3, a3,  4
191	EX(s32i, a6, a5,  0, s_fixup)
192	addi	a5, a5,  4
193.L3:
194	bbci.l	a4, 1, .L4
195	# copy 2 bytes
196	EX(l16ui, a6, a3,  0, l_fixup)
197	addi	a3, a3,  2
198	EX(s16i,  a6, a5,  0, s_fixup)
199	addi	a5, a5,  2
200.L4:
201	bbci.l	a4, 0, .L5
202	# copy 1 byte
203	EX(l8ui, a6, a3,  0, l_fixup)
204	EX(s8i,  a6, a5,  0, s_fixup)
205.L5:
206	movi	a2, 0		# return success for len bytes copied
207	retw
208
209/*
210 * Destination is aligned, Source is unaligned
211 */
212
213	.align	4
214	.byte	0		# 1 mod 4 alignement for LOOPNEZ
215				# (0 mod 4 alignment for LBEG)
216.Lsrcunaligned:
217	# copy 16 bytes per iteration for word-aligned dst and unaligned src
218	and	a10, a3, a8	# save unalignment offset for below
219	sub	a3, a3, a10	# align a3 (to avoid sim warnings only; not needed for hardware)
220	EX(l32i, a6, a3, 0, l_fixup)	# load first word
221#if XCHAL_HAVE_LOOPS
222	loopnez	a7, .Loop2done
223#else /* !XCHAL_HAVE_LOOPS */
224	beqz	a7, .Loop2done
225	slli	a10, a7, 4
226	add	a10, a10, a3	# a10 = end of last 16B source chunk
227#endif /* !XCHAL_HAVE_LOOPS */
228.Loop2:
229	EX(l32i, a7, a3,  4, l_fixup)
230	EX(l32i, a8, a3,  8, l_fixup)
231	ALIGN(	a6, a6, a7)
232	EX(s32i, a6, a5,  0, s_fixup)
233	EX(l32i, a9, a3, 12, l_fixup)
234	ALIGN(	a7, a7, a8)
235	EX(s32i, a7, a5,  4, s_fixup)
236	EX(l32i, a6, a3, 16, l_fixup)
237	ALIGN(	a8, a8, a9)
238	EX(s32i, a8, a5,  8, s_fixup)
239	addi	a3, a3, 16
240	ALIGN(	a9, a9, a6)
241	EX(s32i, a9, a5, 12, s_fixup)
242	addi	a5, a5, 16
243#if !XCHAL_HAVE_LOOPS
244	blt	a3, a10, .Loop2
245#endif /* !XCHAL_HAVE_LOOPS */
246.Loop2done:
247	bbci.l	a4, 3, .L12
248	# copy 8 bytes
249	EX(l32i, a7, a3,  4, l_fixup)
250	EX(l32i, a8, a3,  8, l_fixup)
251	ALIGN(	a6, a6, a7)
252	EX(s32i, a6, a5,  0, s_fixup)
253	addi	a3, a3,  8
254	ALIGN(	a7, a7, a8)
255	EX(s32i, a7, a5,  4, s_fixup)
256	addi	a5, a5,  8
257	mov	a6, a8
258.L12:
259	bbci.l	a4, 2, .L13
260	# copy 4 bytes
261	EX(l32i, a7, a3,  4, l_fixup)
262	addi	a3, a3,  4
263	ALIGN(	a6, a6, a7)
264	EX(s32i, a6, a5,  0, s_fixup)
265	addi	a5, a5,  4
266	mov	a6, a7
267.L13:
268	add	a3, a3, a10	# readjust a3 with correct misalignment
269	bbci.l	a4, 1, .L14
270	# copy 2 bytes
271	EX(l8ui, a6, a3,  0, l_fixup)
272	EX(l8ui, a7, a3,  1, l_fixup)
273	addi	a3, a3,  2
274	EX(s8i, a6, a5,  0, s_fixup)
275	EX(s8i, a7, a5,  1, s_fixup)
276	addi	a5, a5,  2
277.L14:
278	bbci.l	a4, 0, .L15
279	# copy 1 byte
280	EX(l8ui, a6, a3,  0, l_fixup)
281	EX(s8i,  a6, a5,  0, s_fixup)
282.L15:
283	movi	a2, 0		# return success for len bytes copied
284	retw
285
286
287	.section .fixup, "ax"
288	.align	4
289
290/* a2 = original dst; a5 = current dst; a11= original len
291 * bytes_copied = a5 - a2
292 * retval = bytes_not_copied = original len - bytes_copied
293 * retval = a11 - (a5 - a2)
294 *
295 * Clearing the remaining pieces of kernel memory plugs security
296 * holes.  This functionality is the equivalent of the *_zeroing
297 * functions that some architectures provide.
298 */
299
300.Lmemset:
301	.word	memset
302
303s_fixup:
304	sub	a2, a5, a2	/* a2 <-- bytes copied */
305	sub	a2, a11, a2	/* a2 <-- bytes not copied */
306	retw
307
308l_fixup:
309	sub	a2, a5, a2	/* a2 <-- bytes copied */
310	sub	a2, a11, a2	/* a2 <-- bytes not copied == return value */
311
312	/* void *memset(void *s, int c, size_t n); */
313	mov	a6, a5		/* s */
314	movi	a7, 0		/* c */
315	mov	a8, a2		/* n */
316	l32r	a4, .Lmemset
317	callx4	a4
318	/* Ignore memset return value in a6. */
319	/* a2 still contains bytes not copied. */
320	retw
321