PageRenderTime 50ms CodeModel.GetById 17ms app.highlight 28ms RepoModel.GetById 1ms app.codeStats 0ms

/arch/mips/mm/pg-r4k.c

https://bitbucket.org/evzijst/gittest
C | 489 lines | 363 code | 91 blank | 35 comment | 46 complexity | df68c32a2c30149f882f9b026d6a90c3 MD5 | raw file
  1/*
  2 * This file is subject to the terms and conditions of the GNU General Public
  3 * License.  See the file "COPYING" in the main directory of this archive
  4 * for more details.
  5 *
  6 * Copyright (C) 2003, 04, 05 Ralf Baechle (ralf@linux-mips.org)
  7 */
  8#include <linux/init.h>
  9#include <linux/kernel.h>
 10#include <linux/sched.h>
 11#include <linux/mm.h>
 12#include <linux/module.h>
 13#include <linux/proc_fs.h>
 14
 15#include <asm/cacheops.h>
 16#include <asm/inst.h>
 17#include <asm/io.h>
 18#include <asm/page.h>
 19#include <asm/pgtable.h>
 20#include <asm/prefetch.h>
 21#include <asm/system.h>
 22#include <asm/bootinfo.h>
 23#include <asm/mipsregs.h>
 24#include <asm/mmu_context.h>
 25#include <asm/cpu.h>
 26#include <asm/war.h>
 27
 28#define half_scache_line_size()		(cpu_scache_line_size() >> 1)
 29
 30/*
 31 * Maximum sizes:
 32 *
 33 * R4000 128 bytes S-cache:		0x58 bytes
 34 * R4600 v1.7:				0x5c bytes
 35 * R4600 v2.0:				0x60 bytes
 36 * With prefetching, 16 byte strides	0xa0 bytes
 37 */
 38
 39static unsigned int clear_page_array[0x130 / 4];
 40
 41void clear_page(void * page) __attribute__((alias("clear_page_array")));
 42
 43EXPORT_SYMBOL(clear_page);
 44
 45/*
 46 * Maximum sizes:
 47 *
 48 * R4000 128 bytes S-cache:		0x11c bytes
 49 * R4600 v1.7:				0x080 bytes
 50 * R4600 v2.0:				0x07c bytes
 51 * With prefetching, 16 byte strides	0x0b8 bytes
 52 */
 53static unsigned int copy_page_array[0x148 / 4];
 54
 55void copy_page(void *to, void *from) __attribute__((alias("copy_page_array")));
 56
 57EXPORT_SYMBOL(copy_page);
 58
 59/*
 60 * This is suboptimal for 32-bit kernels; we assume that R10000 is only used
 61 * with 64-bit kernels.  The prefetch offsets have been experimentally tuned
 62 * an Origin 200.
 63 */
 64static int pref_offset_clear __initdata = 512;
 65static int pref_offset_copy  __initdata = 256;
 66
 67static unsigned int pref_src_mode __initdata;
 68static unsigned int pref_dst_mode __initdata;
 69
 70static int load_offset __initdata;
 71static int store_offset __initdata;
 72
 73static unsigned int __initdata *dest, *epc;
 74
 75static unsigned int instruction_pending;
 76static union mips_instruction delayed_mi;
 77
 78static void __init emit_instruction(union mips_instruction mi)
 79{
 80	if (instruction_pending)
 81		*epc++ = delayed_mi.word;
 82
 83	instruction_pending = 1;
 84	delayed_mi = mi;
 85}
 86
 87static inline void flush_delay_slot_or_nop(void)
 88{
 89	if (instruction_pending) {
 90		*epc++ = delayed_mi.word;
 91		instruction_pending = 0;
 92		return;
 93	}
 94
 95	*epc++ = 0;
 96}
 97
 98static inline unsigned int *label(void)
 99{
100	if (instruction_pending) {
101		*epc++ = delayed_mi.word;
102		instruction_pending = 0;
103	}
104
105	return epc;
106}
107
108static inline void build_insn_word(unsigned int word)
109{
110	union mips_instruction mi;
111
112	mi.word		 = word;
113
114	emit_instruction(mi);
115}
116
117static inline void build_nop(void)
118{
119	build_insn_word(0);			/* nop */
120}
121
122static inline void build_src_pref(int advance)
123{
124	if (!(load_offset & (cpu_dcache_line_size() - 1))) {
125		union mips_instruction mi;
126
127		mi.i_format.opcode     = pref_op;
128		mi.i_format.rs         = 5;		/* $a1 */
129		mi.i_format.rt         = pref_src_mode;
130		mi.i_format.simmediate = load_offset + advance;
131
132		emit_instruction(mi);
133	}
134}
135
136static inline void __build_load_reg(int reg)
137{
138	union mips_instruction mi;
139	unsigned int width;
140
141	if (cpu_has_64bit_gp_regs) {
142		mi.i_format.opcode     = ld_op;
143		width = 8;
144	} else {
145		mi.i_format.opcode     = lw_op;
146		width = 4;
147	}
148	mi.i_format.rs         = 5;		/* $a1 */
149	mi.i_format.rt         = reg;		/* $reg */
150	mi.i_format.simmediate = load_offset;
151
152	load_offset += width;
153	emit_instruction(mi);
154}
155
156static inline void build_load_reg(int reg)
157{
158	if (cpu_has_prefetch)
159		build_src_pref(pref_offset_copy);
160
161	__build_load_reg(reg);
162}
163
164static inline void build_dst_pref(int advance)
165{
166	if (!(store_offset & (cpu_dcache_line_size() - 1))) {
167		union mips_instruction mi;
168
169		mi.i_format.opcode     = pref_op;
170		mi.i_format.rs         = 4;		/* $a0 */
171		mi.i_format.rt         = pref_dst_mode;
172		mi.i_format.simmediate = store_offset + advance;
173
174		emit_instruction(mi);
175	}
176}
177
178static inline void build_cdex_s(void)
179{
180	union mips_instruction mi;
181
182	if ((store_offset & (cpu_scache_line_size() - 1)))
183		return;
184
185	mi.c_format.opcode     = cache_op;
186	mi.c_format.rs         = 4;		/* $a0 */
187	mi.c_format.c_op       = 3;		/* Create Dirty Exclusive */
188	mi.c_format.cache      = 3;		/* Secondary Data Cache */
189	mi.c_format.simmediate = store_offset;
190
191	emit_instruction(mi);
192}
193
194static inline void build_cdex_p(void)
195{
196	union mips_instruction mi;
197
198	if (store_offset & (cpu_dcache_line_size() - 1))
199		return;
200
201	if (R4600_V1_HIT_CACHEOP_WAR && ((read_c0_prid() & 0xfff0) == 0x2010)) {
202		build_nop();
203		build_nop();
204		build_nop();
205		build_nop();
206	}
207
208	if (R4600_V2_HIT_CACHEOP_WAR && ((read_c0_prid() & 0xfff0) == 0x2020))
209		build_insn_word(0x8c200000);	/* lw      $zero, ($at) */
210
211	mi.c_format.opcode     = cache_op;
212	mi.c_format.rs         = 4;		/* $a0 */
213	mi.c_format.c_op       = 3;		/* Create Dirty Exclusive */
214	mi.c_format.cache      = 1;		/* Data Cache */
215	mi.c_format.simmediate = store_offset;
216
217	emit_instruction(mi);
218}
219
220static void __init __build_store_reg(int reg)
221{
222	union mips_instruction mi;
223	unsigned int width;
224
225	if (cpu_has_64bit_gp_regs ||
226	    (cpu_has_64bit_zero_reg && reg == 0)) {
227		mi.i_format.opcode     = sd_op;
228		width = 8;
229	} else {
230		mi.i_format.opcode     = sw_op;
231		width = 4;
232	}
233	mi.i_format.rs         = 4;		/* $a0 */
234	mi.i_format.rt         = reg;		/* $reg */
235	mi.i_format.simmediate = store_offset;
236
237	store_offset += width;
238	emit_instruction(mi);
239}
240
241static inline void build_store_reg(int reg)
242{
243	if (cpu_has_prefetch)
244		if (reg)
245			build_dst_pref(pref_offset_copy);
246		else
247			build_dst_pref(pref_offset_clear);
248	else if (cpu_has_cache_cdex_s)
249		build_cdex_s();
250	else if (cpu_has_cache_cdex_p)
251		build_cdex_p();
252
253	__build_store_reg(reg);
254}
255
256static inline void build_addiu_a2_a0(unsigned long offset)
257{
258	union mips_instruction mi;
259
260	BUG_ON(offset > 0x7fff);
261
262	mi.i_format.opcode     = cpu_has_64bit_gp_regs ? daddiu_op : addiu_op;
263	mi.i_format.rs         = 4;		/* $a0 */
264	mi.i_format.rt         = 6;		/* $a2 */
265	mi.i_format.simmediate = offset;
266
267	emit_instruction(mi);
268}
269
270static inline void build_addiu_a1(unsigned long offset)
271{
272	union mips_instruction mi;
273
274	BUG_ON(offset > 0x7fff);
275
276	mi.i_format.opcode     = cpu_has_64bit_gp_regs ? daddiu_op : addiu_op;
277	mi.i_format.rs         = 5;		/* $a1 */
278	mi.i_format.rt         = 5;		/* $a1 */
279	mi.i_format.simmediate = offset;
280
281	load_offset -= offset;
282
283	emit_instruction(mi);
284}
285
286static inline void build_addiu_a0(unsigned long offset)
287{
288	union mips_instruction mi;
289
290	BUG_ON(offset > 0x7fff);
291
292	mi.i_format.opcode     = cpu_has_64bit_gp_regs ? daddiu_op : addiu_op;
293	mi.i_format.rs         = 4;		/* $a0 */
294	mi.i_format.rt         = 4;		/* $a0 */
295	mi.i_format.simmediate = offset;
296
297	store_offset -= offset;
298
299	emit_instruction(mi);
300}
301
302static inline void build_bne(unsigned int *dest)
303{
304	union mips_instruction mi;
305
306	mi.i_format.opcode = bne_op;
307	mi.i_format.rs     = 6;			/* $a2 */
308	mi.i_format.rt     = 4;			/* $a0 */
309	mi.i_format.simmediate = dest - epc - 1;
310
311	*epc++ = mi.word;
312	flush_delay_slot_or_nop();
313}
314
315static inline void build_jr_ra(void)
316{
317	union mips_instruction mi;
318
319	mi.r_format.opcode = spec_op;
320	mi.r_format.rs     = 31;
321	mi.r_format.rt     = 0;
322	mi.r_format.rd     = 0;
323	mi.r_format.re     = 0;
324	mi.r_format.func   = jr_op;
325
326	*epc++ = mi.word;
327	flush_delay_slot_or_nop();
328}
329
330void __init build_clear_page(void)
331{
332	unsigned int loop_start;
333
334	epc = (unsigned int *) &clear_page_array;
335	instruction_pending = 0;
336	store_offset = 0;
337
338	if (cpu_has_prefetch) {
339		switch (current_cpu_data.cputype) {
340		case CPU_RM9000:
341			/*
342			 * As a workaround for erratum G105 which make the
343			 * PrepareForStore hint unusable we fall back to
344			 * StoreRetained on the RM9000.  Once it is known which
345			 * versions of the RM9000 we'll be able to condition-
346			 * alize this.
347			 */
348
349		case CPU_R10000:
350		case CPU_R12000:
351			pref_src_mode = Pref_LoadStreamed;
352			pref_dst_mode = Pref_StoreStreamed;
353			break;
354
355		default:
356			pref_src_mode = Pref_LoadStreamed;
357			pref_dst_mode = Pref_PrepareForStore;
358			break;
359		}
360	}
361
362	build_addiu_a2_a0(PAGE_SIZE - (cpu_has_prefetch ? pref_offset_clear : 0));
363
364	if (R4600_V2_HIT_CACHEOP_WAR && ((read_c0_prid() & 0xfff0) == 0x2020))
365		build_insn_word(0x3c01a000);	/* lui     $at, 0xa000  */
366
367dest = label();
368	do {
369		build_store_reg(0);
370		build_store_reg(0);
371		build_store_reg(0);
372		build_store_reg(0);
373	} while (store_offset < half_scache_line_size());
374	build_addiu_a0(2 * store_offset);
375	loop_start = store_offset;
376	do {
377		build_store_reg(0);
378		build_store_reg(0);
379		build_store_reg(0);
380		build_store_reg(0);
381	} while ((store_offset - loop_start) < half_scache_line_size());
382	build_bne(dest);
383
384	if (cpu_has_prefetch && pref_offset_clear) {
385		build_addiu_a2_a0(pref_offset_clear);
386	dest = label();
387		loop_start = store_offset;
388		do {
389			__build_store_reg(0);
390			__build_store_reg(0);
391			__build_store_reg(0);
392			__build_store_reg(0);
393		} while ((store_offset - loop_start) < half_scache_line_size());
394		build_addiu_a0(2 * store_offset);
395		loop_start = store_offset;
396		do {
397			__build_store_reg(0);
398			__build_store_reg(0);
399			__build_store_reg(0);
400			__build_store_reg(0);
401		} while ((store_offset - loop_start) < half_scache_line_size());
402		build_bne(dest);
403	}
404
405	build_jr_ra();
406
407	flush_icache_range((unsigned long)&clear_page_array,
408	                   (unsigned long) epc);
409
410	BUG_ON(epc > clear_page_array + ARRAY_SIZE(clear_page_array));
411}
412
413void __init build_copy_page(void)
414{
415	unsigned int loop_start;
416
417	epc = (unsigned int *) &copy_page_array;
418	store_offset = load_offset = 0;
419	instruction_pending = 0;
420
421	build_addiu_a2_a0(PAGE_SIZE - (cpu_has_prefetch ? pref_offset_copy : 0));
422
423	if (R4600_V2_HIT_CACHEOP_WAR && ((read_c0_prid() & 0xfff0) == 0x2020))
424		build_insn_word(0x3c01a000);	/* lui     $at, 0xa000  */
425
426dest = label();
427	loop_start = store_offset;
428	do {
429		build_load_reg( 8);
430		build_load_reg( 9);
431		build_load_reg(10);
432		build_load_reg(11);
433		build_store_reg( 8);
434		build_store_reg( 9);
435		build_store_reg(10);
436		build_store_reg(11);
437	} while ((store_offset - loop_start) < half_scache_line_size());
438	build_addiu_a0(2 * store_offset);
439	build_addiu_a1(2 * load_offset);
440	loop_start = store_offset;
441	do {
442		build_load_reg( 8);
443		build_load_reg( 9);
444		build_load_reg(10);
445		build_load_reg(11);
446		build_store_reg( 8);
447		build_store_reg( 9);
448		build_store_reg(10);
449		build_store_reg(11);
450	} while ((store_offset - loop_start) < half_scache_line_size());
451	build_bne(dest);
452
453	if (cpu_has_prefetch && pref_offset_copy) {
454		build_addiu_a2_a0(pref_offset_copy);
455	dest = label();
456		loop_start = store_offset;
457		do {
458			__build_load_reg( 8);
459			__build_load_reg( 9);
460			__build_load_reg(10);
461			__build_load_reg(11);
462			__build_store_reg( 8);
463			__build_store_reg( 9);
464			__build_store_reg(10);
465			__build_store_reg(11);
466		} while ((store_offset - loop_start) < half_scache_line_size());
467		build_addiu_a0(2 * store_offset);
468		build_addiu_a1(2 * load_offset);
469		loop_start = store_offset;
470		do {
471			__build_load_reg( 8);
472			__build_load_reg( 9);
473			__build_load_reg(10);
474			__build_load_reg(11);
475			__build_store_reg( 8);
476			__build_store_reg( 9);
477			__build_store_reg(10);
478			__build_store_reg(11);
479		} while ((store_offset - loop_start) < half_scache_line_size());
480		build_bne(dest);
481	}
482
483	build_jr_ra();
484
485	flush_icache_range((unsigned long)&copy_page_array,
486	                   (unsigned long) epc);
487
488	BUG_ON(epc > copy_page_array + ARRAY_SIZE(copy_page_array));
489}