PageRenderTime 22ms CodeModel.GetById 15ms app.highlight 5ms RepoModel.GetById 0ms app.codeStats 0ms

/arch/alpha/lib/ev6-copy_user.S

https://bitbucket.org/thekraven/iscream_thunderc-2.6.35
Assembly | 259 lines | 236 code | 23 blank | 0 comment | 9 complexity | e2c1ce0be835378acc3d812d36427abe MD5 | raw file
Possible License(s): GPL-2.0, LGPL-2.0, AGPL-1.0
  1/*
  2 * arch/alpha/lib/ev6-copy_user.S
  3 *
  4 * 21264 version contributed by Rick Gorton <rick.gorton@alpha-processor.com>
  5 *
  6 * Copy to/from user space, handling exceptions as we go..  This
  7 * isn't exactly pretty.
  8 *
  9 * This is essentially the same as "memcpy()", but with a few twists.
 10 * Notably, we have to make sure that $0 is always up-to-date and
 11 * contains the right "bytes left to copy" value (and that it is updated
 12 * only _after_ a successful copy). There is also some rather minor
 13 * exception setup stuff..
 14 *
 15 * NOTE! This is not directly C-callable, because the calling semantics are
 16 * different:
 17 *
 18 * Inputs:
 19 *	length in $0
 20 *	destination address in $6
 21 *	source address in $7
 22 *	return address in $28
 23 *
 24 * Outputs:
 25 *	bytes left to copy in $0
 26 *
 27 * Clobbers:
 28 *	$1,$2,$3,$4,$5,$6,$7
 29 *
 30 * Much of the information about 21264 scheduling/coding comes from:
 31 *	Compiler Writer's Guide for the Alpha 21264
 32 *	abbreviated as 'CWG' in other comments here
 33 *	ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html
 34 * Scheduling notation:
 35 *	E	- either cluster
 36 *	U	- upper subcluster; U0 - subcluster U0; U1 - subcluster U1
 37 *	L	- lower subcluster; L0 - subcluster L0; L1 - subcluster L1
 38 */
 39
 40/* Allow an exception for an insn; exit if we get one.  */
 41#define EXI(x,y...)			\
 42	99: x,##y;			\
 43	.section __ex_table,"a";	\
 44	.long 99b - .;			\
 45	lda $31, $exitin-99b($31);	\
 46	.previous
 47
 48#define EXO(x,y...)			\
 49	99: x,##y;			\
 50	.section __ex_table,"a";	\
 51	.long 99b - .;			\
 52	lda $31, $exitout-99b($31);	\
 53	.previous
 54
 55	.set noat
 56	.align 4
 57	.globl __copy_user
 58	.ent __copy_user
 59				# Pipeline info: Slotting & Comments
 60__copy_user:
 61	.prologue 0
 62	subq $0, 32, $1		# .. E  .. ..	: Is this going to be a small copy?
 63	beq $0, $zerolength	# U  .. .. ..	: U L U L
 64
 65	and $6,7,$3		# .. .. .. E	: is leading dest misalignment
 66	ble $1, $onebyteloop	# .. .. U  ..	: 1st branch : small amount of data
 67	beq $3, $destaligned	# .. U  .. ..	: 2nd (one cycle fetcher stall)
 68	subq $3, 8, $3		# E  .. .. ..	: L U U L : trip counter
 69/*
 70 * The fetcher stall also hides the 1 cycle cross-cluster stall for $3 (L --> U)
 71 * This loop aligns the destination a byte at a time
 72 * We know we have at least one trip through this loop
 73 */
 74$aligndest:
 75	EXI( ldbu $1,0($7) )	# .. .. .. L	: Keep loads separate from stores
 76	addq $6,1,$6		# .. .. E  ..	: Section 3.8 in the CWG
 77	addq $3,1,$3		# .. E  .. ..	:
 78	nop			# E  .. .. ..	: U L U L
 79
 80/*
 81 * the -1 is to compensate for the inc($6) done in a previous quadpack
 82 * which allows us zero dependencies within either quadpack in the loop
 83 */
 84	EXO( stb $1,-1($6) )	# .. .. .. L	:
 85	addq $7,1,$7		# .. .. E  ..	: Section 3.8 in the CWG
 86	subq $0,1,$0		# .. E  .. ..	:
 87	bne $3, $aligndest	# U  .. .. ..	: U L U L
 88
 89/*
 90 * If we fell through into here, we have a minimum of 33 - 7 bytes
 91 * If we arrived via branch, we have a minimum of 32 bytes
 92 */
 93$destaligned:
 94	and $7,7,$1		# .. .. .. E	: Check _current_ source alignment
 95	bic $0,7,$4		# .. .. E  ..	: number bytes as a quadword loop
 96	EXI( ldq_u $3,0($7) )	# .. L  .. ..	: Forward fetch for fallthrough code
 97	beq $1,$quadaligned	# U  .. .. ..	: U L U L
 98
 99/*
100 * In the worst case, we've just executed an ldq_u here from 0($7)
101 * and we'll repeat it once if we take the branch
102 */
103
104/* Misaligned quadword loop - not unrolled.  Leave it that way. */
105$misquad:
106	EXI( ldq_u $2,8($7) )	# .. .. .. L	:
107	subq $4,8,$4		# .. .. E  ..	:
108	extql $3,$7,$3		# .. U  .. ..	:
109	extqh $2,$7,$1		# U  .. .. ..	: U U L L
110
111	bis $3,$1,$1		# .. .. .. E	:
112	EXO( stq $1,0($6) )	# .. .. L  ..	:
113	addq $7,8,$7		# .. E  .. ..	:
114	subq $0,8,$0		# E  .. .. ..	: U L L U
115
116	addq $6,8,$6		# .. .. .. E	:
117	bis $2,$2,$3		# .. .. E  ..	:
118	nop			# .. E  .. ..	:
119	bne $4,$misquad		# U  .. .. ..	: U L U L
120
121	nop			# .. .. .. E
122	nop			# .. .. E  ..
123	nop			# .. E  .. ..
124	beq $0,$zerolength	# U  .. .. ..	: U L U L
125
126/* We know we have at least one trip through the byte loop */
127	EXI ( ldbu $2,0($7) )	# .. .. .. L	: No loads in the same quad
128	addq $6,1,$6		# .. .. E  ..	: as the store (Section 3.8 in CWG)
129	nop			# .. E  .. ..	:
130	br $31, $dirtyentry	# L0 .. .. ..	: L U U L
131/* Do the trailing byte loop load, then hop into the store part of the loop */
132
133/*
134 * A minimum of (33 - 7) bytes to do a quad at a time.
135 * Based upon the usage context, it's worth the effort to unroll this loop
136 * $0 - number of bytes to be moved
137 * $4 - number of bytes to move as quadwords
138 * $6 is current destination address
139 * $7 is current source address
140 */
141$quadaligned:
142	subq	$4, 32, $2	# .. .. .. E	: do not unroll for small stuff
143	nop			# .. .. E  ..
144	nop			# .. E  .. ..
145	blt	$2, $onequad	# U  .. .. ..	: U L U L
146
147/*
148 * There is a significant assumption here that the source and destination
149 * addresses differ by more than 32 bytes.  In this particular case, a
150 * sparsity of registers further bounds this to be a minimum of 8 bytes.
151 * But if this isn't met, then the output result will be incorrect.
152 * Furthermore, due to a lack of available registers, we really can't
153 * unroll this to be an 8x loop (which would enable us to use the wh64
154 * instruction memory hint instruction).
155 */
156$unroll4:
157	EXI( ldq $1,0($7) )	# .. .. .. L
158	EXI( ldq $2,8($7) )	# .. .. L  ..
159	subq	$4,32,$4	# .. E  .. ..
160	nop			# E  .. .. ..	: U U L L
161
162	addq	$7,16,$7	# .. .. .. E
163	EXO( stq $1,0($6) )	# .. .. L  ..
164	EXO( stq $2,8($6) )	# .. L  .. ..
165	subq	$0,16,$0	# E  .. .. ..	: U L L U
166
167	addq	$6,16,$6	# .. .. .. E
168	EXI( ldq $1,0($7) )	# .. .. L  ..
169	EXI( ldq $2,8($7) )	# .. L  .. ..
170	subq	$4, 32, $3	# E  .. .. ..	: U U L L : is there enough for another trip?
171
172	EXO( stq $1,0($6) )	# .. .. .. L
173	EXO( stq $2,8($6) )	# .. .. L  ..
174	subq	$0,16,$0	# .. E  .. ..
175	addq	$7,16,$7	# E  .. .. ..	: U L L U
176
177	nop			# .. .. .. E
178	nop			# .. .. E  ..
179	addq	$6,16,$6	# .. E  .. ..
180	bgt	$3,$unroll4	# U  .. .. ..	: U L U L
181
182	nop
183	nop
184	nop
185	beq	$4, $noquads
186
187$onequad:
188	EXI( ldq $1,0($7) )
189	subq	$4,8,$4
190	addq	$7,8,$7
191	nop
192
193	EXO( stq $1,0($6) )
194	subq	$0,8,$0
195	addq	$6,8,$6
196	bne	$4,$onequad
197
198$noquads:
199	nop
200	nop
201	nop
202	beq $0,$zerolength
203
204/*
205 * For small copies (or the tail of a larger copy), do a very simple byte loop.
206 * There's no point in doing a lot of complex alignment calculations to try to
207 * to quadword stuff for a small amount of data.
208 *	$0 - remaining number of bytes left to copy
209 *	$6 - current dest addr
210 *	$7 - current source addr
211 */
212
213$onebyteloop:
214	EXI ( ldbu $2,0($7) )	# .. .. .. L	: No loads in the same quad
215	addq $6,1,$6		# .. .. E  ..	: as the store (Section 3.8 in CWG)
216	nop			# .. E  .. ..	:
217	nop			# E  .. .. ..	: U L U L
218
219$dirtyentry:
220/*
221 * the -1 is to compensate for the inc($6) done in a previous quadpack
222 * which allows us zero dependencies within either quadpack in the loop
223 */
224	EXO ( stb $2,-1($6) )	# .. .. .. L	:
225	addq $7,1,$7		# .. .. E  ..	: quadpack as the load
226	subq $0,1,$0		# .. E  .. ..	: change count _after_ copy
227	bgt $0,$onebyteloop	# U  .. .. ..	: U L U L
228
229$zerolength:
230$exitout:			# Destination for exception recovery(?)
231	nop			# .. .. .. E
232	nop			# .. .. E  ..
233	nop			# .. E  .. ..
234	ret $31,($28),1		# L0 .. .. ..	: L U L U
235
236$exitin:
237
238	/* A stupid byte-by-byte zeroing of the rest of the output
239	   buffer.  This cures security holes by never leaving 
240	   random kernel data around to be copied elsewhere.  */
241
242	nop
243	nop
244	nop
245	mov	$0,$1
246
247$101:
248	EXO ( stb $31,0($6) )	# L
249	subq $1,1,$1		# E
250	addq $6,1,$6		# E
251	bgt $1,$101		# U
252
253	nop
254	nop
255	nop
256	ret $31,($28),1		# L0
257
258	.end __copy_user
259