PageRenderTime 73ms CodeModel.GetById 14ms app.highlight 48ms RepoModel.GetById 0ms app.codeStats 0ms

/arch/sparc/mm/sun4c.c

https://bitbucket.org/ndreys/linux-sunxi
C | 2166 lines | 1678 code | 324 blank | 164 comment | 207 complexity | 85bbe7265e0a735c557fdf1813a2cfbf MD5 | raw file
Possible License(s): GPL-2.0, LGPL-2.0, AGPL-1.0

Large files files are truncated, but you can click here to view the full file

   1/* sun4c.c: Doing in software what should be done in hardware.
   2 *
   3 * Copyright (C) 1996 David S. Miller (davem@davemloft.net)
   4 * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be)
   5 * Copyright (C) 1996 Andrew Tridgell (Andrew.Tridgell@anu.edu.au)
   6 * Copyright (C) 1997-2000 Anton Blanchard (anton@samba.org)
   7 * Copyright (C) 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
   8 */
   9
  10#define NR_TASK_BUCKETS 512
  11
  12#include <linux/kernel.h>
  13#include <linux/mm.h>
  14#include <linux/init.h>
  15#include <linux/slab.h>
  16#include <linux/bootmem.h>
  17#include <linux/highmem.h>
  18#include <linux/fs.h>
  19#include <linux/seq_file.h>
  20#include <linux/scatterlist.h>
  21#include <linux/bitmap.h>
  22
  23#include <asm/sections.h>
  24#include <asm/page.h>
  25#include <asm/pgalloc.h>
  26#include <asm/pgtable.h>
  27#include <asm/vaddrs.h>
  28#include <asm/idprom.h>
  29#include <asm/machines.h>
  30#include <asm/memreg.h>
  31#include <asm/processor.h>
  32#include <asm/auxio.h>
  33#include <asm/io.h>
  34#include <asm/oplib.h>
  35#include <asm/openprom.h>
  36#include <asm/mmu_context.h>
  37#include <asm/highmem.h>
  38#include <asm/btfixup.h>
  39#include <asm/cacheflush.h>
  40#include <asm/tlbflush.h>
  41
  42/* Because of our dynamic kernel TLB miss strategy, and how
  43 * our DVMA mapping allocation works, you _MUST_:
  44 *
  45 * 1) Disable interrupts _and_ not touch any dynamic kernel
  46 *    memory while messing with kernel MMU state.  By
  47 *    dynamic memory I mean any object which is not in
  48 *    the kernel image itself or a thread_union (both of
  49 *    which are locked into the MMU).
  50 * 2) Disable interrupts while messing with user MMU state.
  51 */
  52
  53extern int num_segmaps, num_contexts;
  54
  55extern unsigned long page_kernel;
  56
  57/* That's it, we prom_halt() on sun4c if the cache size is something other than 65536.
  58 * So let's save some cycles and just use that everywhere except for that bootup
  59 * sanity check.
  60 */
  61#define SUN4C_VAC_SIZE 65536
  62
  63#define SUN4C_KERNEL_BUCKETS 32
  64
  65/* Flushing the cache. */
  66struct sun4c_vac_props sun4c_vacinfo;
  67unsigned long sun4c_kernel_faults;
  68
  69/* Invalidate every sun4c cache line tag. */
  70static void __init sun4c_flush_all(void)
  71{
  72	unsigned long begin, end;
  73
  74	if (sun4c_vacinfo.on)
  75		panic("SUN4C: AIEEE, trying to invalidate vac while it is on.");
  76
  77	/* Clear 'valid' bit in all cache line tags */
  78	begin = AC_CACHETAGS;
  79	end = (AC_CACHETAGS + SUN4C_VAC_SIZE);
  80	while (begin < end) {
  81		__asm__ __volatile__("sta %%g0, [%0] %1\n\t" : :
  82				     "r" (begin), "i" (ASI_CONTROL));
  83		begin += sun4c_vacinfo.linesize;
  84	}
  85}
  86
  87static void sun4c_flush_context_hw(void)
  88{
  89	unsigned long end = SUN4C_VAC_SIZE;
  90
  91	__asm__ __volatile__(
  92		"1:	addcc	%0, -4096, %0\n\t"
  93		"	bne	1b\n\t"
  94		"	 sta	%%g0, [%0] %2"
  95	: "=&r" (end)
  96	: "0" (end), "i" (ASI_HWFLUSHCONTEXT)
  97	: "cc");
  98}
  99
 100/* Must be called minimally with IRQs disabled. */
 101static void sun4c_flush_segment_hw(unsigned long addr)
 102{
 103	if (sun4c_get_segmap(addr) != invalid_segment) {
 104		unsigned long vac_size = SUN4C_VAC_SIZE;
 105
 106		__asm__ __volatile__(
 107			"1:	addcc	%0, -4096, %0\n\t"
 108			"	bne	1b\n\t"
 109			"	 sta	%%g0, [%2 + %0] %3"
 110			: "=&r" (vac_size)
 111			: "0" (vac_size), "r" (addr), "i" (ASI_HWFLUSHSEG)
 112			: "cc");
 113	}
 114}
 115
 116/* File local boot time fixups. */
 117BTFIXUPDEF_CALL(void, sun4c_flush_page, unsigned long)
 118BTFIXUPDEF_CALL(void, sun4c_flush_segment, unsigned long)
 119BTFIXUPDEF_CALL(void, sun4c_flush_context, void)
 120
 121#define sun4c_flush_page(addr) BTFIXUP_CALL(sun4c_flush_page)(addr)
 122#define sun4c_flush_segment(addr) BTFIXUP_CALL(sun4c_flush_segment)(addr)
 123#define sun4c_flush_context() BTFIXUP_CALL(sun4c_flush_context)()
 124
 125/* Must be called minimally with interrupts disabled. */
 126static void sun4c_flush_page_hw(unsigned long addr)
 127{
 128	addr &= PAGE_MASK;
 129	if ((int)sun4c_get_pte(addr) < 0)
 130		__asm__ __volatile__("sta %%g0, [%0] %1"
 131				     : : "r" (addr), "i" (ASI_HWFLUSHPAGE));
 132}
 133
 134/* Don't inline the software version as it eats too many cache lines if expanded. */
 135static void sun4c_flush_context_sw(void)
 136{
 137	unsigned long nbytes = SUN4C_VAC_SIZE;
 138	unsigned long lsize = sun4c_vacinfo.linesize;
 139
 140	__asm__ __volatile__(
 141	"add	%2, %2, %%g1\n\t"
 142	"add	%2, %%g1, %%g2\n\t"
 143	"add	%2, %%g2, %%g3\n\t"
 144	"add	%2, %%g3, %%g4\n\t"
 145	"add	%2, %%g4, %%g5\n\t"
 146	"add	%2, %%g5, %%o4\n\t"
 147	"add	%2, %%o4, %%o5\n"
 148	"1:\n\t"
 149	"subcc	%0, %%o5, %0\n\t"
 150	"sta	%%g0, [%0] %3\n\t"
 151	"sta	%%g0, [%0 + %2] %3\n\t"
 152	"sta	%%g0, [%0 + %%g1] %3\n\t"
 153	"sta	%%g0, [%0 + %%g2] %3\n\t"
 154	"sta	%%g0, [%0 + %%g3] %3\n\t"
 155	"sta	%%g0, [%0 + %%g4] %3\n\t"
 156	"sta	%%g0, [%0 + %%g5] %3\n\t"
 157	"bg	1b\n\t"
 158	" sta	%%g0, [%1 + %%o4] %3\n"
 159	: "=&r" (nbytes)
 160	: "0" (nbytes), "r" (lsize), "i" (ASI_FLUSHCTX)
 161	: "g1", "g2", "g3", "g4", "g5", "o4", "o5", "cc");
 162}
 163
 164/* Don't inline the software version as it eats too many cache lines if expanded. */
 165static void sun4c_flush_segment_sw(unsigned long addr)
 166{
 167	if (sun4c_get_segmap(addr) != invalid_segment) {
 168		unsigned long nbytes = SUN4C_VAC_SIZE;
 169		unsigned long lsize = sun4c_vacinfo.linesize;
 170
 171		__asm__ __volatile__(
 172		"add	%2, %2, %%g1\n\t"
 173		"add	%2, %%g1, %%g2\n\t"
 174		"add	%2, %%g2, %%g3\n\t"
 175		"add	%2, %%g3, %%g4\n\t"
 176		"add	%2, %%g4, %%g5\n\t"
 177		"add	%2, %%g5, %%o4\n\t"
 178		"add	%2, %%o4, %%o5\n"
 179		"1:\n\t"
 180		"subcc	%1, %%o5, %1\n\t"
 181		"sta	%%g0, [%0] %6\n\t"
 182		"sta	%%g0, [%0 + %2] %6\n\t"
 183		"sta	%%g0, [%0 + %%g1] %6\n\t"
 184		"sta	%%g0, [%0 + %%g2] %6\n\t"
 185		"sta	%%g0, [%0 + %%g3] %6\n\t"
 186		"sta	%%g0, [%0 + %%g4] %6\n\t"
 187		"sta	%%g0, [%0 + %%g5] %6\n\t"
 188		"sta	%%g0, [%0 + %%o4] %6\n\t"
 189		"bg	1b\n\t"
 190		" add	%0, %%o5, %0\n"
 191		: "=&r" (addr), "=&r" (nbytes), "=&r" (lsize)
 192		: "0" (addr), "1" (nbytes), "2" (lsize),
 193		  "i" (ASI_FLUSHSEG)
 194		: "g1", "g2", "g3", "g4", "g5", "o4", "o5", "cc");
 195	}
 196}
 197
 198/* Don't inline the software version as it eats too many cache lines if expanded. */
 199static void sun4c_flush_page_sw(unsigned long addr)
 200{
 201	addr &= PAGE_MASK;
 202	if ((sun4c_get_pte(addr) & (_SUN4C_PAGE_NOCACHE | _SUN4C_PAGE_VALID)) ==
 203	    _SUN4C_PAGE_VALID) {
 204		unsigned long left = PAGE_SIZE;
 205		unsigned long lsize = sun4c_vacinfo.linesize;
 206
 207		__asm__ __volatile__(
 208		"add	%2, %2, %%g1\n\t"
 209		"add	%2, %%g1, %%g2\n\t"
 210		"add	%2, %%g2, %%g3\n\t"
 211		"add	%2, %%g3, %%g4\n\t"
 212		"add	%2, %%g4, %%g5\n\t"
 213		"add	%2, %%g5, %%o4\n\t"
 214		"add	%2, %%o4, %%o5\n"
 215		"1:\n\t"
 216		"subcc	%1, %%o5, %1\n\t"
 217		"sta	%%g0, [%0] %6\n\t"
 218		"sta	%%g0, [%0 + %2] %6\n\t"
 219		"sta	%%g0, [%0 + %%g1] %6\n\t"
 220		"sta	%%g0, [%0 + %%g2] %6\n\t"
 221		"sta	%%g0, [%0 + %%g3] %6\n\t"
 222		"sta	%%g0, [%0 + %%g4] %6\n\t"
 223		"sta	%%g0, [%0 + %%g5] %6\n\t"
 224		"sta	%%g0, [%0 + %%o4] %6\n\t"
 225		"bg	1b\n\t"
 226		" add	%0, %%o5, %0\n"
 227		: "=&r" (addr), "=&r" (left), "=&r" (lsize)
 228		: "0" (addr), "1" (left), "2" (lsize),
 229		  "i" (ASI_FLUSHPG)
 230		: "g1", "g2", "g3", "g4", "g5", "o4", "o5", "cc");
 231	}
 232}
 233
 234/* The sun4c's do have an on chip store buffer.  And the way you
 235 * clear them out isn't so obvious.  The only way I can think of
 236 * to accomplish this is to read the current context register,
 237 * store the same value there, then read an external hardware
 238 * register.
 239 */
 240void sun4c_complete_all_stores(void)
 241{
 242	volatile int _unused;
 243
 244	_unused = sun4c_get_context();
 245	sun4c_set_context(_unused);
 246	_unused = get_auxio();
 247}
 248
 249/* Bootup utility functions. */
 250static inline void sun4c_init_clean_segmap(unsigned char pseg)
 251{
 252	unsigned long vaddr;
 253
 254	sun4c_put_segmap(0, pseg);
 255	for (vaddr = 0; vaddr < SUN4C_REAL_PGDIR_SIZE; vaddr += PAGE_SIZE)
 256		sun4c_put_pte(vaddr, 0);
 257	sun4c_put_segmap(0, invalid_segment);
 258}
 259
 260static inline void sun4c_init_clean_mmu(unsigned long kernel_end)
 261{
 262	unsigned long vaddr;
 263	unsigned char savectx, ctx;
 264
 265	savectx = sun4c_get_context();
 266	for (ctx = 0; ctx < num_contexts; ctx++) {
 267		sun4c_set_context(ctx);
 268		for (vaddr = 0; vaddr < 0x20000000; vaddr += SUN4C_REAL_PGDIR_SIZE)
 269			sun4c_put_segmap(vaddr, invalid_segment);
 270		for (vaddr = 0xe0000000; vaddr < KERNBASE; vaddr += SUN4C_REAL_PGDIR_SIZE)
 271			sun4c_put_segmap(vaddr, invalid_segment);
 272		for (vaddr = kernel_end; vaddr < KADB_DEBUGGER_BEGVM; vaddr += SUN4C_REAL_PGDIR_SIZE)
 273			sun4c_put_segmap(vaddr, invalid_segment);
 274		for (vaddr = LINUX_OPPROM_ENDVM; vaddr; vaddr += SUN4C_REAL_PGDIR_SIZE)
 275			sun4c_put_segmap(vaddr, invalid_segment);
 276	}
 277	sun4c_set_context(savectx);
 278}
 279
 280void __init sun4c_probe_vac(void)
 281{
 282	sun4c_disable_vac();
 283
 284	if ((idprom->id_machtype == (SM_SUN4C | SM_4C_SS1)) ||
 285	    (idprom->id_machtype == (SM_SUN4C | SM_4C_SS1PLUS))) {
 286		/* PROM on SS1 lacks this info, to be super safe we
 287		 * hard code it here since this arch is cast in stone.
 288		 */
 289		sun4c_vacinfo.num_bytes = 65536;
 290		sun4c_vacinfo.linesize = 16;
 291	} else {
 292		sun4c_vacinfo.num_bytes =
 293		 prom_getintdefault(prom_root_node, "vac-size", 65536);
 294		sun4c_vacinfo.linesize =
 295		 prom_getintdefault(prom_root_node, "vac-linesize", 16);
 296	}
 297	sun4c_vacinfo.do_hwflushes =
 298	 prom_getintdefault(prom_root_node, "vac-hwflush", 0);
 299
 300	if (sun4c_vacinfo.do_hwflushes == 0)
 301		sun4c_vacinfo.do_hwflushes =
 302		 prom_getintdefault(prom_root_node, "vac_hwflush", 0);
 303
 304	if (sun4c_vacinfo.num_bytes != 65536) {
 305		prom_printf("WEIRD Sun4C VAC cache size, "
 306			    "tell sparclinux@vger.kernel.org");
 307		prom_halt();
 308	}
 309
 310	switch (sun4c_vacinfo.linesize) {
 311	case 16:
 312		sun4c_vacinfo.log2lsize = 4;
 313		break;
 314	case 32:
 315		sun4c_vacinfo.log2lsize = 5;
 316		break;
 317	default:
 318		prom_printf("probe_vac: Didn't expect vac-linesize of %d, halting\n",
 319			    sun4c_vacinfo.linesize);
 320		prom_halt();
 321	}
 322
 323	sun4c_flush_all();
 324	sun4c_enable_vac();
 325}
 326
 327/* Patch instructions for the low level kernel fault handler. */
 328extern unsigned long invalid_segment_patch1, invalid_segment_patch1_ff;
 329extern unsigned long invalid_segment_patch2, invalid_segment_patch2_ff;
 330extern unsigned long invalid_segment_patch1_1ff, invalid_segment_patch2_1ff;
 331extern unsigned long num_context_patch1, num_context_patch1_16;
 332extern unsigned long num_context_patch2_16;
 333extern unsigned long vac_linesize_patch, vac_linesize_patch_32;
 334extern unsigned long vac_hwflush_patch1, vac_hwflush_patch1_on;
 335extern unsigned long vac_hwflush_patch2, vac_hwflush_patch2_on;
 336
 337#define PATCH_INSN(src, dst) do {	\
 338		daddr = &(dst);		\
 339		iaddr = &(src);		\
 340		*daddr = *iaddr;	\
 341	} while (0)
 342
 343static void __init patch_kernel_fault_handler(void)
 344{
 345	unsigned long *iaddr, *daddr;
 346
 347	switch (num_segmaps) {
 348		case 128:
 349			/* Default, nothing to do. */
 350			break;
 351		case 256:
 352			PATCH_INSN(invalid_segment_patch1_ff,
 353				   invalid_segment_patch1);
 354			PATCH_INSN(invalid_segment_patch2_ff,
 355				   invalid_segment_patch2);
 356			break;
 357		case 512:
 358			PATCH_INSN(invalid_segment_patch1_1ff,
 359				   invalid_segment_patch1);
 360			PATCH_INSN(invalid_segment_patch2_1ff,
 361				   invalid_segment_patch2);
 362			break;
 363		default:
 364			prom_printf("Unhandled number of segmaps: %d\n",
 365				    num_segmaps);
 366			prom_halt();
 367	}
 368	switch (num_contexts) {
 369		case 8:
 370			/* Default, nothing to do. */
 371			break;
 372		case 16:
 373			PATCH_INSN(num_context_patch1_16,
 374				   num_context_patch1);
 375			break;
 376		default:
 377			prom_printf("Unhandled number of contexts: %d\n",
 378				    num_contexts);
 379			prom_halt();
 380	}
 381
 382	if (sun4c_vacinfo.do_hwflushes != 0) {
 383		PATCH_INSN(vac_hwflush_patch1_on, vac_hwflush_patch1);
 384		PATCH_INSN(vac_hwflush_patch2_on, vac_hwflush_patch2);
 385	} else {
 386		switch (sun4c_vacinfo.linesize) {
 387		case 16:
 388			/* Default, nothing to do. */
 389			break;
 390		case 32:
 391			PATCH_INSN(vac_linesize_patch_32, vac_linesize_patch);
 392			break;
 393		default:
 394			prom_printf("Impossible VAC linesize %d, halting...\n",
 395				    sun4c_vacinfo.linesize);
 396			prom_halt();
 397		}
 398	}
 399}
 400
 401static void __init sun4c_probe_mmu(void)
 402{
 403	if ((idprom->id_machtype == (SM_SUN4C | SM_4C_SS1)) ||
 404	    (idprom->id_machtype == (SM_SUN4C | SM_4C_SS1PLUS))) {
 405		/* Hardcode these just to be safe, PROM on SS1 does
 406		* not have this info available in the root node.
 407		*/
 408		num_segmaps = 128;
 409		num_contexts = 8;
 410	} else {
 411		num_segmaps =
 412		    prom_getintdefault(prom_root_node, "mmu-npmg", 128);
 413		num_contexts =
 414		    prom_getintdefault(prom_root_node, "mmu-nctx", 0x8);
 415	}
 416	patch_kernel_fault_handler();
 417}
 418
 419volatile unsigned long __iomem *sun4c_memerr_reg = NULL;
 420
 421void __init sun4c_probe_memerr_reg(void)
 422{
 423	phandle node;
 424	struct linux_prom_registers regs[1];
 425
 426	node = prom_getchild(prom_root_node);
 427	node = prom_searchsiblings(prom_root_node, "memory-error");
 428	if (!node)
 429		return;
 430	if (prom_getproperty(node, "reg", (char *)regs, sizeof(regs)) <= 0)
 431		return;
 432	/* hmm I think regs[0].which_io is zero here anyways */
 433	sun4c_memerr_reg = ioremap(regs[0].phys_addr, regs[0].reg_size);
 434}
 435
 436static inline void sun4c_init_ss2_cache_bug(void)
 437{
 438	if ((idprom->id_machtype == (SM_SUN4C | SM_4C_SS2)) ||
 439	    (idprom->id_machtype == (SM_SUN4C | SM_4C_IPX)) ||
 440	    (idprom->id_machtype == (SM_SUN4C | SM_4C_ELC))) {
 441		/* Whee.. */
 442		printk("SS2 cache bug detected, uncaching trap table page\n");
 443		sun4c_flush_page((unsigned int) &_start);
 444		sun4c_put_pte(((unsigned long) &_start),
 445			(sun4c_get_pte((unsigned long) &_start) | _SUN4C_PAGE_NOCACHE));
 446	}
 447}
 448
 449/* Addr is always aligned on a page boundary for us already. */
 450static int sun4c_map_dma_area(struct device *dev, dma_addr_t *pba, unsigned long va,
 451			      unsigned long addr, int len)
 452{
 453	unsigned long page, end;
 454
 455	*pba = addr;
 456
 457	end = PAGE_ALIGN((addr + len));
 458	while (addr < end) {
 459		page = va;
 460		sun4c_flush_page(page);
 461		page -= PAGE_OFFSET;
 462		page >>= PAGE_SHIFT;
 463		page |= (_SUN4C_PAGE_VALID | _SUN4C_PAGE_DIRTY |
 464			 _SUN4C_PAGE_NOCACHE | _SUN4C_PAGE_PRIV);
 465		sun4c_put_pte(addr, page);
 466		addr += PAGE_SIZE;
 467		va += PAGE_SIZE;
 468	}
 469
 470	return 0;
 471}
 472
 473static void sun4c_unmap_dma_area(struct device *dev, unsigned long busa, int len)
 474{
 475	/* Fortunately for us, bus_addr == uncached_virt in sun4c. */
 476	/* XXX Implement this */
 477}
 478
 479/* TLB management. */
 480
 481/* Don't change this struct without changing entry.S. This is used
 482 * in the in-window kernel fault handler, and you don't want to mess
 483 * with that. (See sun4c_fault in entry.S).
 484 */
 485struct sun4c_mmu_entry {
 486	struct sun4c_mmu_entry *next;
 487	struct sun4c_mmu_entry *prev;
 488	unsigned long vaddr;
 489	unsigned char pseg;
 490	unsigned char locked;
 491
 492	/* For user mappings only, and completely hidden from kernel
 493	 * TLB miss code.
 494	 */
 495	unsigned char ctx;
 496	struct sun4c_mmu_entry *lru_next;
 497	struct sun4c_mmu_entry *lru_prev;
 498};
 499
 500static struct sun4c_mmu_entry mmu_entry_pool[SUN4C_MAX_SEGMAPS];
 501
 502static void __init sun4c_init_mmu_entry_pool(void)
 503{
 504	int i;
 505
 506	for (i=0; i < SUN4C_MAX_SEGMAPS; i++) {
 507		mmu_entry_pool[i].pseg = i;
 508		mmu_entry_pool[i].next = NULL;
 509		mmu_entry_pool[i].prev = NULL;
 510		mmu_entry_pool[i].vaddr = 0;
 511		mmu_entry_pool[i].locked = 0;
 512		mmu_entry_pool[i].ctx = 0;
 513		mmu_entry_pool[i].lru_next = NULL;
 514		mmu_entry_pool[i].lru_prev = NULL;
 515	}
 516	mmu_entry_pool[invalid_segment].locked = 1;
 517}
 518
 519static inline void fix_permissions(unsigned long vaddr, unsigned long bits_on,
 520				   unsigned long bits_off)
 521{
 522	unsigned long start, end;
 523
 524	end = vaddr + SUN4C_REAL_PGDIR_SIZE;
 525	for (start = vaddr; start < end; start += PAGE_SIZE)
 526		if (sun4c_get_pte(start) & _SUN4C_PAGE_VALID)
 527			sun4c_put_pte(start, (sun4c_get_pte(start) | bits_on) &
 528				      ~bits_off);
 529}
 530
 531static inline void sun4c_init_map_kernelprom(unsigned long kernel_end)
 532{
 533	unsigned long vaddr;
 534	unsigned char pseg, ctx;
 535
 536	for (vaddr = KADB_DEBUGGER_BEGVM;
 537	     vaddr < LINUX_OPPROM_ENDVM;
 538	     vaddr += SUN4C_REAL_PGDIR_SIZE) {
 539		pseg = sun4c_get_segmap(vaddr);
 540		if (pseg != invalid_segment) {
 541			mmu_entry_pool[pseg].locked = 1;
 542			for (ctx = 0; ctx < num_contexts; ctx++)
 543				prom_putsegment(ctx, vaddr, pseg);
 544			fix_permissions(vaddr, _SUN4C_PAGE_PRIV, 0);
 545		}
 546	}
 547
 548	for (vaddr = KERNBASE; vaddr < kernel_end; vaddr += SUN4C_REAL_PGDIR_SIZE) {
 549		pseg = sun4c_get_segmap(vaddr);
 550		mmu_entry_pool[pseg].locked = 1;
 551		for (ctx = 0; ctx < num_contexts; ctx++)
 552			prom_putsegment(ctx, vaddr, pseg);
 553		fix_permissions(vaddr, _SUN4C_PAGE_PRIV, _SUN4C_PAGE_NOCACHE);
 554	}
 555}
 556
 557static void __init sun4c_init_lock_area(unsigned long start, unsigned long end)
 558{
 559	int i, ctx;
 560
 561	while (start < end) {
 562		for (i = 0; i < invalid_segment; i++)
 563			if (!mmu_entry_pool[i].locked)
 564				break;
 565		mmu_entry_pool[i].locked = 1;
 566		sun4c_init_clean_segmap(i);
 567		for (ctx = 0; ctx < num_contexts; ctx++)
 568			prom_putsegment(ctx, start, mmu_entry_pool[i].pseg);
 569		start += SUN4C_REAL_PGDIR_SIZE;
 570	}
 571}
 572
 573/* Don't change this struct without changing entry.S. This is used
 574 * in the in-window kernel fault handler, and you don't want to mess
 575 * with that. (See sun4c_fault in entry.S).
 576 */
 577struct sun4c_mmu_ring {
 578	struct sun4c_mmu_entry ringhd;
 579	int num_entries;
 580};
 581
 582static struct sun4c_mmu_ring sun4c_context_ring[SUN4C_MAX_CONTEXTS]; /* used user entries */
 583static struct sun4c_mmu_ring sun4c_ufree_ring;       /* free user entries */
 584static struct sun4c_mmu_ring sun4c_ulru_ring;	     /* LRU user entries */
 585struct sun4c_mmu_ring sun4c_kernel_ring;      /* used kernel entries */
 586struct sun4c_mmu_ring sun4c_kfree_ring;       /* free kernel entries */
 587
 588static inline void sun4c_init_rings(void)
 589{
 590	int i;
 591
 592	for (i = 0; i < SUN4C_MAX_CONTEXTS; i++) {
 593		sun4c_context_ring[i].ringhd.next =
 594			sun4c_context_ring[i].ringhd.prev =
 595			&sun4c_context_ring[i].ringhd;
 596		sun4c_context_ring[i].num_entries = 0;
 597	}
 598	sun4c_ufree_ring.ringhd.next = sun4c_ufree_ring.ringhd.prev =
 599		&sun4c_ufree_ring.ringhd;
 600	sun4c_ufree_ring.num_entries = 0;
 601	sun4c_ulru_ring.ringhd.lru_next = sun4c_ulru_ring.ringhd.lru_prev =
 602		&sun4c_ulru_ring.ringhd;
 603	sun4c_ulru_ring.num_entries = 0;
 604	sun4c_kernel_ring.ringhd.next = sun4c_kernel_ring.ringhd.prev =
 605		&sun4c_kernel_ring.ringhd;
 606	sun4c_kernel_ring.num_entries = 0;
 607	sun4c_kfree_ring.ringhd.next = sun4c_kfree_ring.ringhd.prev =
 608		&sun4c_kfree_ring.ringhd;
 609	sun4c_kfree_ring.num_entries = 0;
 610}
 611
 612static void add_ring(struct sun4c_mmu_ring *ring,
 613		     struct sun4c_mmu_entry *entry)
 614{
 615	struct sun4c_mmu_entry *head = &ring->ringhd;
 616
 617	entry->prev = head;
 618	(entry->next = head->next)->prev = entry;
 619	head->next = entry;
 620	ring->num_entries++;
 621}
 622
 623static inline void add_lru(struct sun4c_mmu_entry *entry)
 624{
 625	struct sun4c_mmu_ring *ring = &sun4c_ulru_ring;
 626	struct sun4c_mmu_entry *head = &ring->ringhd;
 627
 628	entry->lru_next = head;
 629	(entry->lru_prev = head->lru_prev)->lru_next = entry;
 630	head->lru_prev = entry;
 631}
 632
 633static void add_ring_ordered(struct sun4c_mmu_ring *ring,
 634			     struct sun4c_mmu_entry *entry)
 635{
 636	struct sun4c_mmu_entry *head = &ring->ringhd;
 637	unsigned long addr = entry->vaddr;
 638
 639	while ((head->next != &ring->ringhd) && (head->next->vaddr < addr))
 640		head = head->next;
 641
 642	entry->prev = head;
 643	(entry->next = head->next)->prev = entry;
 644	head->next = entry;
 645	ring->num_entries++;
 646
 647	add_lru(entry);
 648}
 649
 650static inline void remove_ring(struct sun4c_mmu_ring *ring,
 651				   struct sun4c_mmu_entry *entry)
 652{
 653	struct sun4c_mmu_entry *next = entry->next;
 654
 655	(next->prev = entry->prev)->next = next;
 656	ring->num_entries--;
 657}
 658
 659static void remove_lru(struct sun4c_mmu_entry *entry)
 660{
 661	struct sun4c_mmu_entry *next = entry->lru_next;
 662
 663	(next->lru_prev = entry->lru_prev)->lru_next = next;
 664}
 665
 666static void free_user_entry(int ctx, struct sun4c_mmu_entry *entry)
 667{
 668        remove_ring(sun4c_context_ring+ctx, entry);
 669	remove_lru(entry);
 670        add_ring(&sun4c_ufree_ring, entry);
 671}
 672
 673static void free_kernel_entry(struct sun4c_mmu_entry *entry,
 674			      struct sun4c_mmu_ring *ring)
 675{
 676        remove_ring(ring, entry);
 677        add_ring(&sun4c_kfree_ring, entry);
 678}
 679
 680static void __init sun4c_init_fill_kernel_ring(int howmany)
 681{
 682	int i;
 683
 684	while (howmany) {
 685		for (i = 0; i < invalid_segment; i++)
 686			if (!mmu_entry_pool[i].locked)
 687				break;
 688		mmu_entry_pool[i].locked = 1;
 689		sun4c_init_clean_segmap(i);
 690		add_ring(&sun4c_kfree_ring, &mmu_entry_pool[i]);
 691		howmany--;
 692	}
 693}
 694
 695static void __init sun4c_init_fill_user_ring(void)
 696{
 697	int i;
 698
 699	for (i = 0; i < invalid_segment; i++) {
 700		if (mmu_entry_pool[i].locked)
 701			continue;
 702		sun4c_init_clean_segmap(i);
 703		add_ring(&sun4c_ufree_ring, &mmu_entry_pool[i]);
 704	}
 705}
 706
 707static void sun4c_kernel_unmap(struct sun4c_mmu_entry *kentry)
 708{
 709	int savectx, ctx;
 710
 711	savectx = sun4c_get_context();
 712	for (ctx = 0; ctx < num_contexts; ctx++) {
 713		sun4c_set_context(ctx);
 714		sun4c_put_segmap(kentry->vaddr, invalid_segment);
 715	}
 716	sun4c_set_context(savectx);
 717}
 718
 719static void sun4c_kernel_map(struct sun4c_mmu_entry *kentry)
 720{
 721	int savectx, ctx;
 722
 723	savectx = sun4c_get_context();
 724	for (ctx = 0; ctx < num_contexts; ctx++) {
 725		sun4c_set_context(ctx);
 726		sun4c_put_segmap(kentry->vaddr, kentry->pseg);
 727	}
 728	sun4c_set_context(savectx);
 729}
 730
 731#define sun4c_user_unmap(__entry) \
 732	sun4c_put_segmap((__entry)->vaddr, invalid_segment)
 733
 734static void sun4c_demap_context(struct sun4c_mmu_ring *crp, unsigned char ctx)
 735{
 736	struct sun4c_mmu_entry *head = &crp->ringhd;
 737	unsigned long flags;
 738
 739	local_irq_save(flags);
 740	if (head->next != head) {
 741		struct sun4c_mmu_entry *entry = head->next;
 742		int savectx = sun4c_get_context();
 743
 744		flush_user_windows();
 745		sun4c_set_context(ctx);
 746		sun4c_flush_context();
 747		do {
 748			struct sun4c_mmu_entry *next = entry->next;
 749
 750			sun4c_user_unmap(entry);
 751			free_user_entry(ctx, entry);
 752
 753			entry = next;
 754		} while (entry != head);
 755		sun4c_set_context(savectx);
 756	}
 757	local_irq_restore(flags);
 758}
 759
 760static int sun4c_user_taken_entries;  /* This is how much we have.             */
 761static int max_user_taken_entries;    /* This limits us and prevents deadlock. */
 762
 763static struct sun4c_mmu_entry *sun4c_kernel_strategy(void)
 764{
 765	struct sun4c_mmu_entry *this_entry;
 766
 767	/* If some are free, return first one. */
 768	if (sun4c_kfree_ring.num_entries) {
 769		this_entry = sun4c_kfree_ring.ringhd.next;
 770		return this_entry;
 771	}
 772
 773	/* Else free one up. */
 774	this_entry = sun4c_kernel_ring.ringhd.prev;
 775	sun4c_flush_segment(this_entry->vaddr);
 776	sun4c_kernel_unmap(this_entry);
 777	free_kernel_entry(this_entry, &sun4c_kernel_ring);
 778	this_entry = sun4c_kfree_ring.ringhd.next;
 779
 780	return this_entry;
 781}
 782
 783/* Using this method to free up mmu entries eliminates a lot of
 784 * potential races since we have a kernel that incurs tlb
 785 * replacement faults.  There may be performance penalties.
 786 *
 787 * NOTE: Must be called with interrupts disabled.
 788 */
 789static struct sun4c_mmu_entry *sun4c_user_strategy(void)
 790{
 791	struct sun4c_mmu_entry *entry;
 792	unsigned char ctx;
 793	int savectx;
 794
 795	/* If some are free, return first one. */
 796	if (sun4c_ufree_ring.num_entries) {
 797		entry = sun4c_ufree_ring.ringhd.next;
 798		goto unlink_out;
 799	}
 800
 801	if (sun4c_user_taken_entries) {
 802		entry = sun4c_kernel_strategy();
 803		sun4c_user_taken_entries--;
 804		goto kunlink_out;
 805	}
 806
 807	/* Grab from the beginning of the LRU list. */
 808	entry = sun4c_ulru_ring.ringhd.lru_next;
 809	ctx = entry->ctx;
 810
 811	savectx = sun4c_get_context();
 812	flush_user_windows();
 813	sun4c_set_context(ctx);
 814	sun4c_flush_segment(entry->vaddr);
 815	sun4c_user_unmap(entry);
 816	remove_ring(sun4c_context_ring + ctx, entry);
 817	remove_lru(entry);
 818	sun4c_set_context(savectx);
 819
 820	return entry;
 821
 822unlink_out:
 823	remove_ring(&sun4c_ufree_ring, entry);
 824	return entry;
 825kunlink_out:
 826	remove_ring(&sun4c_kfree_ring, entry);
 827	return entry;
 828}
 829
 830/* NOTE: Must be called with interrupts disabled. */
 831void sun4c_grow_kernel_ring(void)
 832{
 833	struct sun4c_mmu_entry *entry;
 834
 835	/* Prevent deadlock condition. */
 836	if (sun4c_user_taken_entries >= max_user_taken_entries)
 837		return;
 838
 839	if (sun4c_ufree_ring.num_entries) {
 840		entry = sun4c_ufree_ring.ringhd.next;
 841        	remove_ring(&sun4c_ufree_ring, entry);
 842		add_ring(&sun4c_kfree_ring, entry);
 843		sun4c_user_taken_entries++;
 844	}
 845}
 846
 847/* 2 page buckets for task struct and kernel stack allocation.
 848 *
 849 * TASK_STACK_BEGIN
 850 * bucket[0]
 851 * bucket[1]
 852 *   [ ... ]
 853 * bucket[NR_TASK_BUCKETS-1]
 854 * TASK_STACK_BEGIN + (sizeof(struct task_bucket) * NR_TASK_BUCKETS)
 855 *
 856 * Each slot looks like:
 857 *
 858 *  page 1 --  task struct + beginning of kernel stack
 859 *  page 2 --  rest of kernel stack
 860 */
 861
 862union task_union *sun4c_bucket[NR_TASK_BUCKETS];
 863
 864static int sun4c_lowbucket_avail;
 865
 866#define BUCKET_EMPTY     ((union task_union *) 0)
 867#define BUCKET_SHIFT     (PAGE_SHIFT + 1)        /* log2(sizeof(struct task_bucket)) */
 868#define BUCKET_SIZE      (1 << BUCKET_SHIFT)
 869#define BUCKET_NUM(addr) ((((addr) - SUN4C_LOCK_VADDR) >> BUCKET_SHIFT))
 870#define BUCKET_ADDR(num) (((num) << BUCKET_SHIFT) + SUN4C_LOCK_VADDR)
 871#define BUCKET_PTE(page)       \
 872        ((((page) - PAGE_OFFSET) >> PAGE_SHIFT) | pgprot_val(SUN4C_PAGE_KERNEL))
 873#define BUCKET_PTE_PAGE(pte)   \
 874        (PAGE_OFFSET + (((pte) & SUN4C_PFN_MASK) << PAGE_SHIFT))
 875
 876static void get_locked_segment(unsigned long addr)
 877{
 878	struct sun4c_mmu_entry *stolen;
 879	unsigned long flags;
 880
 881	local_irq_save(flags);
 882	addr &= SUN4C_REAL_PGDIR_MASK;
 883	stolen = sun4c_user_strategy();
 884	max_user_taken_entries--;
 885	stolen->vaddr = addr;
 886	flush_user_windows();
 887	sun4c_kernel_map(stolen);
 888	local_irq_restore(flags);
 889}
 890
 891static void free_locked_segment(unsigned long addr)
 892{
 893	struct sun4c_mmu_entry *entry;
 894	unsigned long flags;
 895	unsigned char pseg;
 896
 897	local_irq_save(flags);
 898	addr &= SUN4C_REAL_PGDIR_MASK;
 899	pseg = sun4c_get_segmap(addr);
 900	entry = &mmu_entry_pool[pseg];
 901
 902	flush_user_windows();
 903	sun4c_flush_segment(addr);
 904	sun4c_kernel_unmap(entry);
 905	add_ring(&sun4c_ufree_ring, entry);
 906	max_user_taken_entries++;
 907	local_irq_restore(flags);
 908}
 909
 910static inline void garbage_collect(int entry)
 911{
 912	int start, end;
 913
 914	/* 32 buckets per segment... */
 915	entry &= ~31;
 916	start = entry;
 917	for (end = (start + 32); start < end; start++)
 918		if (sun4c_bucket[start] != BUCKET_EMPTY)
 919			return;
 920
 921	/* Entire segment empty, release it. */
 922	free_locked_segment(BUCKET_ADDR(entry));
 923}
 924
 925static struct thread_info *sun4c_alloc_thread_info_node(int node)
 926{
 927	unsigned long addr, pages;
 928	int entry;
 929
 930	pages = __get_free_pages(GFP_KERNEL, THREAD_INFO_ORDER);
 931	if (!pages)
 932		return NULL;
 933
 934	for (entry = sun4c_lowbucket_avail; entry < NR_TASK_BUCKETS; entry++)
 935		if (sun4c_bucket[entry] == BUCKET_EMPTY)
 936			break;
 937	if (entry == NR_TASK_BUCKETS) {
 938		free_pages(pages, THREAD_INFO_ORDER);
 939		return NULL;
 940	}
 941	if (entry >= sun4c_lowbucket_avail)
 942		sun4c_lowbucket_avail = entry + 1;
 943
 944	addr = BUCKET_ADDR(entry);
 945	sun4c_bucket[entry] = (union task_union *) addr;
 946	if(sun4c_get_segmap(addr) == invalid_segment)
 947		get_locked_segment(addr);
 948
 949	/* We are changing the virtual color of the page(s)
 950	 * so we must flush the cache to guarantee consistency.
 951	 */
 952	sun4c_flush_page(pages);
 953	sun4c_flush_page(pages + PAGE_SIZE);
 954
 955	sun4c_put_pte(addr, BUCKET_PTE(pages));
 956	sun4c_put_pte(addr + PAGE_SIZE, BUCKET_PTE(pages + PAGE_SIZE));
 957
 958#ifdef CONFIG_DEBUG_STACK_USAGE
 959	memset((void *)addr, 0, PAGE_SIZE << THREAD_INFO_ORDER);
 960#endif /* DEBUG_STACK_USAGE */
 961
 962	return (struct thread_info *) addr;
 963}
 964
 965static void sun4c_free_thread_info(struct thread_info *ti)
 966{
 967	unsigned long tiaddr = (unsigned long) ti;
 968	unsigned long pages = BUCKET_PTE_PAGE(sun4c_get_pte(tiaddr));
 969	int entry = BUCKET_NUM(tiaddr);
 970
 971	/* We are deleting a mapping, so the flush here is mandatory. */
 972	sun4c_flush_page(tiaddr);
 973	sun4c_flush_page(tiaddr + PAGE_SIZE);
 974
 975	sun4c_put_pte(tiaddr, 0);
 976	sun4c_put_pte(tiaddr + PAGE_SIZE, 0);
 977
 978	sun4c_bucket[entry] = BUCKET_EMPTY;
 979	if (entry < sun4c_lowbucket_avail)
 980		sun4c_lowbucket_avail = entry;
 981
 982	free_pages(pages, THREAD_INFO_ORDER);
 983	garbage_collect(entry);
 984}
 985
 986static void __init sun4c_init_buckets(void)
 987{
 988	int entry;
 989
 990	if (sizeof(union thread_union) != (PAGE_SIZE << THREAD_INFO_ORDER)) {
 991		extern void thread_info_size_is_bolixed_pete(void);
 992		thread_info_size_is_bolixed_pete();
 993	}
 994
 995	for (entry = 0; entry < NR_TASK_BUCKETS; entry++)
 996		sun4c_bucket[entry] = BUCKET_EMPTY;
 997	sun4c_lowbucket_avail = 0;
 998}
 999
1000static unsigned long sun4c_iobuffer_start;
1001static unsigned long sun4c_iobuffer_end;
1002static unsigned long sun4c_iobuffer_high;
1003static unsigned long *sun4c_iobuffer_map;
1004static int iobuffer_map_size;
1005
1006/*
1007 * Alias our pages so they do not cause a trap.
1008 * Also one page may be aliased into several I/O areas and we may
1009 * finish these I/O separately.
1010 */
1011static char *sun4c_lockarea(char *vaddr, unsigned long size)
1012{
1013	unsigned long base, scan;
1014	unsigned long npages;
1015	unsigned long vpage;
1016	unsigned long pte;
1017	unsigned long apage;
1018	unsigned long high;
1019	unsigned long flags;
1020
1021	npages = (((unsigned long)vaddr & ~PAGE_MASK) +
1022		  size + (PAGE_SIZE-1)) >> PAGE_SHIFT;
1023
1024	local_irq_save(flags);
1025	base = bitmap_find_next_zero_area(sun4c_iobuffer_map, iobuffer_map_size,
1026						0, npages, 0);
1027	if (base >= iobuffer_map_size)
1028		goto abend;
1029
1030	high = ((base + npages) << PAGE_SHIFT) + sun4c_iobuffer_start;
1031	high = SUN4C_REAL_PGDIR_ALIGN(high);
1032	while (high > sun4c_iobuffer_high) {
1033		get_locked_segment(sun4c_iobuffer_high);
1034		sun4c_iobuffer_high += SUN4C_REAL_PGDIR_SIZE;
1035	}
1036
1037	vpage = ((unsigned long) vaddr) & PAGE_MASK;
1038	for (scan = base; scan < base+npages; scan++) {
1039		pte = ((vpage-PAGE_OFFSET) >> PAGE_SHIFT);
1040 		pte |= pgprot_val(SUN4C_PAGE_KERNEL);
1041		pte |= _SUN4C_PAGE_NOCACHE;
1042		set_bit(scan, sun4c_iobuffer_map);
1043		apage = (scan << PAGE_SHIFT) + sun4c_iobuffer_start;
1044
1045		/* Flush original mapping so we see the right things later. */
1046		sun4c_flush_page(vpage);
1047
1048		sun4c_put_pte(apage, pte);
1049		vpage += PAGE_SIZE;
1050	}
1051	local_irq_restore(flags);
1052	return (char *) ((base << PAGE_SHIFT) + sun4c_iobuffer_start +
1053			 (((unsigned long) vaddr) & ~PAGE_MASK));
1054
1055abend:
1056	local_irq_restore(flags);
1057	printk("DMA vaddr=0x%p size=%08lx\n", vaddr, size);
1058	panic("Out of iobuffer table");
1059	return NULL;
1060}
1061
1062static void sun4c_unlockarea(char *vaddr, unsigned long size)
1063{
1064	unsigned long vpage, npages;
1065	unsigned long flags;
1066	int scan, high;
1067
1068	vpage = (unsigned long)vaddr & PAGE_MASK;
1069	npages = (((unsigned long)vaddr & ~PAGE_MASK) +
1070		  size + (PAGE_SIZE-1)) >> PAGE_SHIFT;
1071
1072	local_irq_save(flags);
1073	while (npages != 0) {
1074		--npages;
1075
1076		/* This mapping is marked non-cachable, no flush necessary. */
1077		sun4c_put_pte(vpage, 0);
1078		clear_bit((vpage - sun4c_iobuffer_start) >> PAGE_SHIFT,
1079			  sun4c_iobuffer_map);
1080		vpage += PAGE_SIZE;
1081	}
1082
1083	/* garbage collect */
1084	scan = (sun4c_iobuffer_high - sun4c_iobuffer_start) >> PAGE_SHIFT;
1085	while (scan >= 0 && !sun4c_iobuffer_map[scan >> 5])
1086		scan -= 32;
1087	scan += 32;
1088	high = sun4c_iobuffer_start + (scan << PAGE_SHIFT);
1089	high = SUN4C_REAL_PGDIR_ALIGN(high) + SUN4C_REAL_PGDIR_SIZE;
1090	while (high < sun4c_iobuffer_high) {
1091		sun4c_iobuffer_high -= SUN4C_REAL_PGDIR_SIZE;
1092		free_locked_segment(sun4c_iobuffer_high);
1093	}
1094	local_irq_restore(flags);
1095}
1096
1097/* Note the scsi code at init time passes to here buffers
1098 * which sit on the kernel stack, those are already locked
1099 * by implication and fool the page locking code above
1100 * if passed to by mistake.
1101 */
1102static __u32 sun4c_get_scsi_one(struct device *dev, char *bufptr, unsigned long len)
1103{
1104	unsigned long page;
1105
1106	page = ((unsigned long)bufptr) & PAGE_MASK;
1107	if (!virt_addr_valid(page)) {
1108		sun4c_flush_page(page);
1109		return (__u32)bufptr; /* already locked */
1110	}
1111	return (__u32)sun4c_lockarea(bufptr, len);
1112}
1113
1114static void sun4c_get_scsi_sgl(struct device *dev, struct scatterlist *sg, int sz)
1115{
1116	while (sz != 0) {
1117		--sz;
1118		sg->dma_address = (__u32)sun4c_lockarea(sg_virt(sg), sg->length);
1119		sg->dma_length = sg->length;
1120		sg = sg_next(sg);
1121	}
1122}
1123
1124static void sun4c_release_scsi_one(struct device *dev, __u32 bufptr, unsigned long len)
1125{
1126	if (bufptr < sun4c_iobuffer_start)
1127		return; /* On kernel stack or similar, see above */
1128	sun4c_unlockarea((char *)bufptr, len);
1129}
1130
1131static void sun4c_release_scsi_sgl(struct device *dev, struct scatterlist *sg, int sz)
1132{
1133	while (sz != 0) {
1134		--sz;
1135		sun4c_unlockarea((char *)sg->dma_address, sg->length);
1136		sg = sg_next(sg);
1137	}
1138}
1139
1140#define TASK_ENTRY_SIZE    BUCKET_SIZE /* see above */
1141#define LONG_ALIGN(x) (((x)+(sizeof(long))-1)&~((sizeof(long))-1))
1142
1143struct vm_area_struct sun4c_kstack_vma;
1144
1145static void __init sun4c_init_lock_areas(void)
1146{
1147	unsigned long sun4c_taskstack_start;
1148	unsigned long sun4c_taskstack_end;
1149	int bitmap_size;
1150
1151	sun4c_init_buckets();
1152	sun4c_taskstack_start = SUN4C_LOCK_VADDR;
1153	sun4c_taskstack_end = (sun4c_taskstack_start +
1154			       (TASK_ENTRY_SIZE * NR_TASK_BUCKETS));
1155	if (sun4c_taskstack_end >= SUN4C_LOCK_END) {
1156		prom_printf("Too many tasks, decrease NR_TASK_BUCKETS please.\n");
1157		prom_halt();
1158	}
1159
1160	sun4c_iobuffer_start = sun4c_iobuffer_high =
1161				SUN4C_REAL_PGDIR_ALIGN(sun4c_taskstack_end);
1162	sun4c_iobuffer_end = SUN4C_LOCK_END;
1163	bitmap_size = (sun4c_iobuffer_end - sun4c_iobuffer_start) >> PAGE_SHIFT;
1164	bitmap_size = (bitmap_size + 7) >> 3;
1165	bitmap_size = LONG_ALIGN(bitmap_size);
1166	iobuffer_map_size = bitmap_size << 3;
1167	sun4c_iobuffer_map = __alloc_bootmem(bitmap_size, SMP_CACHE_BYTES, 0UL);
1168	memset((void *) sun4c_iobuffer_map, 0, bitmap_size);
1169
1170	sun4c_kstack_vma.vm_mm = &init_mm;
1171	sun4c_kstack_vma.vm_start = sun4c_taskstack_start;
1172	sun4c_kstack_vma.vm_end = sun4c_taskstack_end;
1173	sun4c_kstack_vma.vm_page_prot = PAGE_SHARED;
1174	sun4c_kstack_vma.vm_flags = VM_READ | VM_WRITE | VM_EXEC;
1175	insert_vm_struct(&init_mm, &sun4c_kstack_vma);
1176}
1177
1178/* Cache flushing on the sun4c. */
1179static void sun4c_flush_cache_all(void)
1180{
1181	unsigned long begin, end;
1182
1183	flush_user_windows();
1184	begin = (KERNBASE + SUN4C_REAL_PGDIR_SIZE);
1185	end = (begin + SUN4C_VAC_SIZE);
1186
1187	if (sun4c_vacinfo.linesize == 32) {
1188		while (begin < end) {
1189			__asm__ __volatile__(
1190			"ld	[%0 + 0x00], %%g0\n\t"
1191			"ld	[%0 + 0x20], %%g0\n\t"
1192			"ld	[%0 + 0x40], %%g0\n\t"
1193			"ld	[%0 + 0x60], %%g0\n\t"
1194			"ld	[%0 + 0x80], %%g0\n\t"
1195			"ld	[%0 + 0xa0], %%g0\n\t"
1196			"ld	[%0 + 0xc0], %%g0\n\t"
1197			"ld	[%0 + 0xe0], %%g0\n\t"
1198			"ld	[%0 + 0x100], %%g0\n\t"
1199			"ld	[%0 + 0x120], %%g0\n\t"
1200			"ld	[%0 + 0x140], %%g0\n\t"
1201			"ld	[%0 + 0x160], %%g0\n\t"
1202			"ld	[%0 + 0x180], %%g0\n\t"
1203			"ld	[%0 + 0x1a0], %%g0\n\t"
1204			"ld	[%0 + 0x1c0], %%g0\n\t"
1205			"ld	[%0 + 0x1e0], %%g0\n"
1206			: : "r" (begin));
1207			begin += 512;
1208		}
1209	} else {
1210		while (begin < end) {
1211			__asm__ __volatile__(
1212			"ld	[%0 + 0x00], %%g0\n\t"
1213			"ld	[%0 + 0x10], %%g0\n\t"
1214			"ld	[%0 + 0x20], %%g0\n\t"
1215			"ld	[%0 + 0x30], %%g0\n\t"
1216			"ld	[%0 + 0x40], %%g0\n\t"
1217			"ld	[%0 + 0x50], %%g0\n\t"
1218			"ld	[%0 + 0x60], %%g0\n\t"
1219			"ld	[%0 + 0x70], %%g0\n\t"
1220			"ld	[%0 + 0x80], %%g0\n\t"
1221			"ld	[%0 + 0x90], %%g0\n\t"
1222			"ld	[%0 + 0xa0], %%g0\n\t"
1223			"ld	[%0 + 0xb0], %%g0\n\t"
1224			"ld	[%0 + 0xc0], %%g0\n\t"
1225			"ld	[%0 + 0xd0], %%g0\n\t"
1226			"ld	[%0 + 0xe0], %%g0\n\t"
1227			"ld	[%0 + 0xf0], %%g0\n"
1228			: : "r" (begin));
1229			begin += 256;
1230		}
1231	}
1232}
1233
1234static void sun4c_flush_cache_mm(struct mm_struct *mm)
1235{
1236	int new_ctx = mm->context;
1237
1238	if (new_ctx != NO_CONTEXT) {
1239		flush_user_windows();
1240
1241		if (sun4c_context_ring[new_ctx].num_entries) {
1242			struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd;
1243			unsigned long flags;
1244
1245			local_irq_save(flags);
1246			if (head->next != head) {
1247				struct sun4c_mmu_entry *entry = head->next;
1248				int savectx = sun4c_get_context();
1249
1250				sun4c_set_context(new_ctx);
1251				sun4c_flush_context();
1252				do {
1253					struct sun4c_mmu_entry *next = entry->next;
1254
1255					sun4c_user_unmap(entry);
1256					free_user_entry(new_ctx, entry);
1257
1258					entry = next;
1259				} while (entry != head);
1260				sun4c_set_context(savectx);
1261			}
1262			local_irq_restore(flags);
1263		}
1264	}
1265}
1266
1267static void sun4c_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
1268{
1269	struct mm_struct *mm = vma->vm_mm;
1270	int new_ctx = mm->context;
1271
1272	if (new_ctx != NO_CONTEXT) {
1273		struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd;
1274		struct sun4c_mmu_entry *entry;
1275		unsigned long flags;
1276
1277		flush_user_windows();
1278
1279		local_irq_save(flags);
1280		/* All user segmap chains are ordered on entry->vaddr. */
1281		for (entry = head->next;
1282		     (entry != head) && ((entry->vaddr+SUN4C_REAL_PGDIR_SIZE) < start);
1283		     entry = entry->next)
1284			;
1285
1286		/* Tracing various job mixtures showed that this conditional
1287		 * only passes ~35% of the time for most worse case situations,
1288		 * therefore we avoid all of this gross overhead ~65% of the time.
1289		 */
1290		if ((entry != head) && (entry->vaddr < end)) {
1291			int octx = sun4c_get_context();
1292			sun4c_set_context(new_ctx);
1293
1294			/* At this point, always, (start >= entry->vaddr) and
1295			 * (entry->vaddr < end), once the latter condition
1296			 * ceases to hold, or we hit the end of the list, we
1297			 * exit the loop.  The ordering of all user allocated
1298			 * segmaps makes this all work out so beautifully.
1299			 */
1300			do {
1301				struct sun4c_mmu_entry *next = entry->next;
1302				unsigned long realend;
1303
1304				/* "realstart" is always >= entry->vaddr */
1305				realend = entry->vaddr + SUN4C_REAL_PGDIR_SIZE;
1306				if (end < realend)
1307					realend = end;
1308				if ((realend - entry->vaddr) <= (PAGE_SIZE << 3)) {
1309					unsigned long page = entry->vaddr;
1310					while (page < realend) {
1311						sun4c_flush_page(page);
1312						page += PAGE_SIZE;
1313					}
1314				} else {
1315					sun4c_flush_segment(entry->vaddr);
1316					sun4c_user_unmap(entry);
1317					free_user_entry(new_ctx, entry);
1318				}
1319				entry = next;
1320			} while ((entry != head) && (entry->vaddr < end));
1321			sun4c_set_context(octx);
1322		}
1323		local_irq_restore(flags);
1324	}
1325}
1326
1327static void sun4c_flush_cache_page(struct vm_area_struct *vma, unsigned long page)
1328{
1329	struct mm_struct *mm = vma->vm_mm;
1330	int new_ctx = mm->context;
1331
1332	/* Sun4c has no separate I/D caches so cannot optimize for non
1333	 * text page flushes.
1334	 */
1335	if (new_ctx != NO_CONTEXT) {
1336		int octx = sun4c_get_context();
1337		unsigned long flags;
1338
1339		flush_user_windows();
1340		local_irq_save(flags);
1341		sun4c_set_context(new_ctx);
1342		sun4c_flush_page(page);
1343		sun4c_set_context(octx);
1344		local_irq_restore(flags);
1345	}
1346}
1347
1348static void sun4c_flush_page_to_ram(unsigned long page)
1349{
1350	unsigned long flags;
1351
1352	local_irq_save(flags);
1353	sun4c_flush_page(page);
1354	local_irq_restore(flags);
1355}
1356
1357/* Sun4c cache is unified, both instructions and data live there, so
1358 * no need to flush the on-stack instructions for new signal handlers.
1359 */
1360static void sun4c_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr)
1361{
1362}
1363
1364/* TLB flushing on the sun4c.  These routines count on the cache
1365 * flushing code to flush the user register windows so that we need
1366 * not do so when we get here.
1367 */
1368
1369static void sun4c_flush_tlb_all(void)
1370{
1371	struct sun4c_mmu_entry *this_entry, *next_entry;
1372	unsigned long flags;
1373	int savectx, ctx;
1374
1375	local_irq_save(flags);
1376	this_entry = sun4c_kernel_ring.ringhd.next;
1377	savectx = sun4c_get_context();
1378	flush_user_windows();
1379	while (sun4c_kernel_ring.num_entries) {
1380		next_entry = this_entry->next;
1381		sun4c_flush_segment(this_entry->vaddr);
1382		for (ctx = 0; ctx < num_contexts; ctx++) {
1383			sun4c_set_context(ctx);
1384			sun4c_put_segmap(this_entry->vaddr, invalid_segment);
1385		}
1386		free_kernel_entry(this_entry, &sun4c_kernel_ring);
1387		this_entry = next_entry;
1388	}
1389	sun4c_set_context(savectx);
1390	local_irq_restore(flags);
1391}
1392
1393static void sun4c_flush_tlb_mm(struct mm_struct *mm)
1394{
1395	int new_ctx = mm->context;
1396
1397	if (new_ctx != NO_CONTEXT) {
1398		struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd;
1399		unsigned long flags;
1400
1401		local_irq_save(flags);
1402		if (head->next != head) {
1403			struct sun4c_mmu_entry *entry = head->next;
1404			int savectx = sun4c_get_context();
1405
1406			sun4c_set_context(new_ctx);
1407			sun4c_flush_context();
1408			do {
1409				struct sun4c_mmu_entry *next = entry->next;
1410
1411				sun4c_user_unmap(entry);
1412				free_user_entry(new_ctx, entry);
1413
1414				entry = next;
1415			} while (entry != head);
1416			sun4c_set_context(savectx);
1417		}
1418		local_irq_restore(flags);
1419	}
1420}
1421
1422static void sun4c_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
1423{
1424	struct mm_struct *mm = vma->vm_mm;
1425	int new_ctx = mm->context;
1426
1427	if (new_ctx != NO_CONTEXT) {
1428		struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd;
1429		struct sun4c_mmu_entry *entry;
1430		unsigned long flags;
1431
1432		local_irq_save(flags);
1433		/* See commentary in sun4c_flush_cache_range(). */
1434		for (entry = head->next;
1435		     (entry != head) && ((entry->vaddr+SUN4C_REAL_PGDIR_SIZE) < start);
1436		     entry = entry->next)
1437			;
1438
1439		if ((entry != head) && (entry->vaddr < end)) {
1440			int octx = sun4c_get_context();
1441
1442			sun4c_set_context(new_ctx);
1443			do {
1444				struct sun4c_mmu_entry *next = entry->next;
1445
1446				sun4c_flush_segment(entry->vaddr);
1447				sun4c_user_unmap(entry);
1448				free_user_entry(new_ctx, entry);
1449
1450				entry = next;
1451			} while ((entry != head) && (entry->vaddr < end));
1452			sun4c_set_context(octx);
1453		}
1454		local_irq_restore(flags);
1455	}
1456}
1457
1458static void sun4c_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
1459{
1460	struct mm_struct *mm = vma->vm_mm;
1461	int new_ctx = mm->context;
1462
1463	if (new_ctx != NO_CONTEXT) {
1464		int savectx = sun4c_get_context();
1465		unsigned long flags;
1466
1467		local_irq_save(flags);
1468		sun4c_set_context(new_ctx);
1469		page &= PAGE_MASK;
1470		sun4c_flush_page(page);
1471		sun4c_put_pte(page, 0);
1472		sun4c_set_context(savectx);
1473		local_irq_restore(flags);
1474	}
1475}
1476
1477static inline void sun4c_mapioaddr(unsigned long physaddr, unsigned long virt_addr)
1478{
1479	unsigned long page_entry, pg_iobits;
1480
1481	pg_iobits = _SUN4C_PAGE_PRESENT | _SUN4C_READABLE | _SUN4C_WRITEABLE |
1482		    _SUN4C_PAGE_IO | _SUN4C_PAGE_NOCACHE;
1483
1484	page_entry = ((physaddr >> PAGE_SHIFT) & SUN4C_PFN_MASK);
1485	page_entry |= ((pg_iobits | _SUN4C_PAGE_PRIV) & ~(_SUN4C_PAGE_PRESENT));
1486	sun4c_put_pte(virt_addr, page_entry);
1487}
1488
1489static void sun4c_mapiorange(unsigned int bus, unsigned long xpa,
1490    unsigned long xva, unsigned int len)
1491{
1492	while (len != 0) {
1493		len -= PAGE_SIZE;
1494		sun4c_mapioaddr(xpa, xva);
1495		xva += PAGE_SIZE;
1496		xpa += PAGE_SIZE;
1497	}
1498}
1499
1500static void sun4c_unmapiorange(unsigned long virt_addr, unsigned int len)
1501{
1502	while (len != 0) {
1503		len -= PAGE_SIZE;
1504		sun4c_put_pte(virt_addr, 0);
1505		virt_addr += PAGE_SIZE;
1506	}
1507}
1508
1509static void sun4c_alloc_context(struct mm_struct *old_mm, struct mm_struct *mm)
1510{
1511	struct ctx_list *ctxp;
1512
1513	ctxp = ctx_free.next;
1514	if (ctxp != &ctx_free) {
1515		remove_from_ctx_list(ctxp);
1516		add_to_used_ctxlist(ctxp);
1517		mm->context = ctxp->ctx_number;
1518		ctxp->ctx_mm = mm;
1519		return;
1520	}
1521	ctxp = ctx_used.next;
1522	if (ctxp->ctx_mm == old_mm)
1523		ctxp = ctxp->next;
1524	remove_from_ctx_list(ctxp);
1525	add_to_used_ctxlist(ctxp);
1526	ctxp->ctx_mm->context = NO_CONTEXT;
1527	ctxp->ctx_mm = mm;
1528	mm->context = ctxp->ctx_number;
1529	sun4c_demap_context(&sun4c_context_ring[ctxp->ctx_number],
1530			       ctxp->ctx_number);
1531}
1532
1533/* Switch the current MM context. */
1534static void sun4c_switch_mm(struct mm_struct *old_mm, struct mm_struct *mm, struct task_struct *tsk, int cpu)
1535{
1536	struct ctx_list *ctx;
1537	int dirty = 0;
1538
1539	if (mm->context == NO_CONTEXT) {
1540		dirty = 1;
1541		sun4c_alloc_context(old_mm, mm);
1542	} else {
1543		/* Update the LRU ring of contexts. */
1544		ctx = ctx_list_pool + mm->context;
1545		remove_from_ctx_list(ctx);
1546		add_to_used_ctxlist(ctx);
1547	}
1548	if (dirty || old_mm != mm)
1549		sun4c_set_context(mm->context);
1550}
1551
1552static void sun4c_destroy_context(struct mm_struct *mm)
1553{
1554	struct ctx_list *ctx_old;
1555
1556	if (mm->context != NO_CONTEXT) {
1557		sun4c_demap_context(&sun4c_context_ring[mm->context], mm->context);
1558		ctx_old = ctx_list_pool + mm->context;
1559		remove_from_ctx_list(ctx_old);
1560		add_to_free_ctxlist(ctx_old);
1561		mm->context = NO_CONTEXT;
1562	}
1563}
1564
1565static void sun4c_mmu_info(struct seq_file *m)
1566{
1567	int used_user_entries, i;
1568
1569	used_user_entries = 0;
1570	for (i = 0; i < num_contexts; i++)
1571		used_user_entries += sun4c_context_ring[i].num_entries;
1572
1573	seq_printf(m, 
1574		   "vacsize\t\t: %d bytes\n"
1575		   "vachwflush\t: %s\n"
1576		   "vaclinesize\t: %d bytes\n"
1577		   "mmuctxs\t\t: %d\n"
1578		   "mmupsegs\t: %d\n"
1579		   "kernelpsegs\t: %d\n"
1580		   "kfreepsegs\t: %d\n"
1581		   "usedpsegs\t: %d\n"
1582		   "ufreepsegs\t: %d\n"
1583		   "user_taken\t: %d\n"
1584		   "max_taken\t: %d\n",
1585		   sun4c_vacinfo.num_bytes,
1586		   (sun4c_vacinfo.do_hwflushes ? "yes" : "no"),
1587		   sun4c_vacinfo.linesize,
1588		   num_contexts,
1589		   (invalid_segment + 1),
1590		   sun4c_kernel_ring.num_entries,
1591		   sun4c_kfree_ring.num_entries,
1592		   used_user_entries,
1593		   sun4c_ufree_ring.num_entries,
1594		   sun4c_user_taken_entries,
1595		   max_user_taken_entries);
1596}
1597
1598/* Nothing below here should touch the mmu hardware nor the mmu_entry
1599 * data structures.
1600 */
1601
1602/* First the functions which the mid-level code uses to directly
1603 * manipulate the software page tables.  Some defines since we are
1604 * emulating the i386 page directory layout.
1605 */
1606#define PGD_PRESENT  0x001
1607#define PGD_RW       0x002
1608#define PGD_USER     0x004
1609#define PGD_ACCESSED 0x020
1610#define PGD_DIRTY    0x040
1611#define PGD_TABLE    (PGD_PRESENT | PGD_RW | PGD_USER | PGD_ACCESSED | PGD_DIRTY)
1612
1613static void sun4c_set_pte(pte_t *ptep, pte_t pte)
1614{
1615	*ptep = pte;
1616}
1617
1618static void sun4c_pgd_set(pgd_t * pgdp, pmd_t * pmdp)
1619{
1620}
1621
1622static void sun4c_pmd_set(pmd_t * pmdp, pte_t * ptep)
1623{
1624	pmdp->pmdv[0] = PGD_TABLE | (unsigned long) ptep;
1625}
1626
1627static void sun4c_pmd_populate(pmd_t * pmdp, struct page * ptep)
1628{
1629	if (page_address(ptep) == NULL) BUG();	/* No highmem on sun4c */
1630	pmdp->pmdv[0] = PGD_TABLE | (unsigned long) page_address(ptep);
1631}
1632
1633static int sun4c_pte_present(pte_t pte)
1634{
1635	return ((pte_val(pte) & (_SUN4C_PAGE_PRESENT | _SUN4C_PAGE_PRIV)) != 0);
1636}
1637static void sun4c_pte_clear(pte_t *ptep)	{ *ptep = __pte(0); }
1638
1639static int sun4c_pmd_bad(pmd_t pmd)
1640{
1641	return (((pmd_val(pmd) & ~PAGE_MASK) != PGD_TABLE) ||
1642		(!virt_addr_valid(pmd_val(pmd))));
1643}
1644
1645static int sun4c_pmd_present(pmd_t pmd)
1646{
1647	return ((pmd_val(pmd) & PGD_PRESENT) != 0);
1648}
1649
1650#if 0 /* if PMD takes one word */
1651static void sun4c_pmd_clear(pmd_t *pmdp)	{ *pmdp = __pmd(0); }
1652#else /* if pmd_t is a longish aggregate */
1653static void sun4c_pmd_clear(pmd_t *pmdp) {
1654	memset((void *)pmdp, 0, sizeof(pmd_t));
1655}
1656#endif
1657
1658static int sun4c_pgd_none(pgd_t pgd)		{ return 0; }
1659static int sun4c_pgd_bad(pgd_t pgd)		{ return 0; }
1660static int sun4c_pgd_present(pgd_t pgd)	        { return 1; }
1661static void sun4c_pgd_clear(pgd_t * pgdp)	{ }
1662
1663/*
1664 * The following only work if pte_present() is true.
1665 * Undefined behaviour if not..
1666 */
1667static pte_t sun4c_pte_mkwrite(pte_t pte)
1668{
1669	pte = __pte(pte_val(pte) | _SUN4C_PAGE_WRITE);
1670	if (pte_val(pte) & _SUN4C_PAGE_MODIFIED)
1671		pte = __pte(pte_val(pte) | _SUN4C_PAGE_SILENT_WRITE);
1672	return pte;
1673}
1674
1675static pte_t sun4c_pte_mkdirty(pte_t pte)
1676{
1677	pte = __pte(pte_val(pte) | _SUN4C_PAGE_MODIFIED);
1678	if (pte_val(pte) & _SUN4C_PAGE_WRITE)
1679		pte = __pte(pte_val(pte) | _SUN4C_PAGE_SILENT_WRITE);
1680	return pte;
1681}
1682
1683static pte_t sun4c_pte_mkyoung(pte_t pte)
1684{
1685	pte = __pte(pte_val(pte) | _SUN4C_PAGE_ACCESSED);
1686	if (pte_val(pte) & _SUN4C_PAGE_READ)
1687		pte = __pte(pte_val(pte) | _SUN4C_PAGE_SILENT_READ);
1688	return pte;
1689}
1690
1691/*
1692 * Conversion functions: convert a page and protection to a page entry,
1693 * and a page entry and page directory to the page they refer to.
1694 */
1695static pte_t sun4c_mk_pte(struct page *page, pgprot_t pgprot)
1696{
1697	return __pte(page_to_pfn(page) | pgprot_val(pgprot));
1698}
1699
1700static pte_t sun4c_mk_pte_phys(unsigned long phys_page, pgprot_t pgprot)
1701{
1702	return __pte((phys_page >> PAGE_SHIFT) | pgprot_val(pgprot));
1703}
1704
1705static pte_t sun4c_mk_pte_io(unsigned long page, pgprot_t pgprot, int space)
1706{
1707	return __pte(((page - PAGE_OFFSET) >> PAGE_SHIFT) | pgprot_val(pgprot));
1708}
1709
1710static unsigned long sun4c_pte_pfn(pte_t pte)
1711{
1712	return pte_val(pte) & SUN4C_PFN_MASK;
1713}
1714
1715static pte_t sun4c_pgoff_to_pte(unsigned long pgoff)
1716{
1717	return __pte(pgoff | _SUN4C_PAGE_FILE);
1718}
1719
1720static unsigned long sun4c_pte_to_pgoff(pte_t pte)
1721{
1722	return pte_val(pte) & ((1UL << PTE_FILE_MAX_BITS) - 1);
1723}
1724
1725
1726static inline unsigned long sun4c_pmd_page_v(pmd_t pmd)
1727{
1728	return (pmd_val(pmd) & PAGE_MASK);
1729}
1730
1731static struct page *sun4c_pmd_page(pmd_t pmd)
1732{
1733	return virt_to_page(sun4c_pmd_page_v(pmd));
1734}
1735
1736static unsigned long sun4c_pgd_page(pgd_t pgd) { return 0; }
1737
1738/* to find an entry in a page-table-directory */
1739static inline pgd_t *sun4c_pgd_offset(struct mm_struct * mm, unsigned long address)
1740{
1741	return mm->pgd + (address >> SUN4C_PGDIR_SHIFT);
1742}
1743
1744/* Find an entry in the second-level page table.. */
1745static pmd_t *sun4c_pmd_offset(pgd_t * dir, unsigned long address)
1746{
1747	return (pmd_t *) dir;
1748}
1749
1750/* Find an entry in the third-level page table.. */ 
1751pte_t *sun4c_pte_offset_kernel(pmd_t * dir, unsigned long address)
1752{
1753	return (pte_t *) sun4c_pmd_page_v(*dir) +
1754			((address >> PAGE_SHIFT) & (SUN4C_PTRS_PER_PTE - 1));
1755}
1756
1757static unsigned long sun4c_swp_type(swp_entry_t entry)
1758{
1759	return (entry.val & SUN4C_SWP_TYPE_MASK);
1760}
1761
1762static unsigned long sun4c_swp_offset(swp_entry_t entry)
1763{
1764	return (entry.val >> SUN4C_SWP_OFF_SHIFT) & SUN4C_SWP_OFF_MASK;
1765}
1766
1767static swp_entry_t sun4c_swp_entry(unsigned long type, unsigned long offset)
1768{
1769	return (swp_entry_t) {
1770		  (offset & SUN4C_SWP_OFF_MASK) << SUN4C_SWP_OFF_SHIFT
1771		| (type & SUN4C_SWP_TYPE_MASK) };
1772}
1773
1774static void sun4c_free_pte_slow(pte_t *pte)
1775{
1776	free_page((unsigned long)pte);
1777}
1778
1779static void sun4c_free_pgd_slow(pgd_t *pgd)
1780{
1781	free_page((unsigned long)pgd);
1782}
1783
1784static pgd_t *sun4c_get_pgd_fast(void)
1785{
1786	unsigned long *ret;
1787
1788	if ((ret = pgd_quicklist) != NULL) {
1789		pgd_quicklist = (unsigned long *)(*ret);
1790		ret[0] = ret[1];
1791		pgtable_cache_size--;
1792	} else {
1793		pgd_t *init;
1794		
1795		ret = (unsigned long *)__get_free_page(GFP_KERNEL);
1796		memset (ret, 0, (KERNBASE / SUN4C_PGDIR_SIZE) * sizeof(pgd_t));
1797		init = sun4c_pgd_offset(&init_mm, 0);
1798		memcpy (((pgd_t *)ret) + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD,
1799			(PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
1800	}
1801	return (pgd_t *)ret;
1802}
1803
1804static void sun4c_free_pgd_fast(pgd_t *pgd)
1805{
1806	*(unsigned long *)pgd = (unsigned long) pgd_quicklist;
1807	pgd_quicklist = (unsigned long *) pgd;
1808	pgtable_cache_size++;
1809}
1810
1811
1812static inline pte_t *
1813sun4c_pte_alloc_one_fast(struct mm_struct *mm, unsigned long address)
1814{
1815	unsigned long *ret;
1816
1817	if ((ret = (unsigned long *)pte_quicklist) != NULL) {
1818		pte_quicklist = (unsigned long *)(*ret);
1819		ret[0] = ret[1];
1820		pgtable_cache_size--;
1821	}
1822	return (pte_t *)ret;
1823}
1824
1825static pte_t *sun4c_pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
1826{
1827	pte_t *pte;
1828
1829	if ((pte = sun4c_pte_alloc_one_fast(mm, address)) != NULL)
1830		return pte;
1831
1832	pte = (pte_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT);
1833	return pte;
1834}
1835
1836static pgtable_t sun4c_pte_alloc_one(struct mm_struct *mm, unsigned long address)
1837{
1838	pte_t *pte;
1839	struct page *page;
1840
1841	pte = sun4c_pte_alloc_one_kernel(mm, address);
1842	if (pte == NULL)
1843		return NULL;
1844	page = virt_to_page(pte);
1845	pgtable_page_ctor(page);
1846	return page;
1847}
1848
1849static inline void sun4c_free_pte_fast(pte_t *pte)
1850{
1851	*(unsigned long *)pte = (unsigned long) pte_quicklist;
1852	pte_quicklist = (unsigned long *) pte;
1853	pgtable_cache_size++;
1854}
1855
1856static void sun4c_pte_free(pgtable_t pte)
1857{
1858	pgtable_page_dtor(pte);
1859	sun4c_free_pte_fast(page_address(pte));
1860}
1861
1862/*
1863 * allocating and freeing a pmd is trivial: the 1-entry pmd is
1864 * inside the pgd, so has no extra memory …

Large files files are truncated, but you can click here to view the full file