/arch/powerpc/kernel/head_64.S

   1/* SPDX-License-Identifier: GPL-2.0-or-later */
   2/*
   3 *  PowerPC version
   4 *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
   5 *
   6 *  Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
   7 *    Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
   8 *  Adapted for Power Macintosh by Paul Mackerras.
   9 *  Low-level exception handlers and MMU support
  10 *  rewritten by Paul Mackerras.
  11 *    Copyright (C) 1996 Paul Mackerras.
  12 *
  13 *  Adapted for 64bit PowerPC by Dave Engebretsen, Peter Bergner, and
  14 *    Mike Corrigan {engebret|bergner|mikejc}@us.ibm.com
  15 *
  16 *  This file contains the entry point for the 64-bit kernel along
  17 *  with some early initialization code common to all 64-bit powerpc
  18 *  variants.
  19 */
  20
  21#include <linux/threads.h>
  22#include <linux/init.h>
  23#include <asm/reg.h>
  24#include <asm/page.h>
  25#include <asm/mmu.h>
  26#include <asm/ppc_asm.h>
  27#include <asm/head-64.h>
  28#include <asm/asm-offsets.h>
  29#include <asm/bug.h>
  30#include <asm/cputable.h>
  31#include <asm/setup.h>
  32#include <asm/hvcall.h>
  33#include <asm/thread_info.h>
  34#include <asm/firmware.h>
  35#include <asm/page_64.h>
  36#include <asm/irqflags.h>
  37#include <asm/kvm_book3s_asm.h>
  38#include <asm/ptrace.h>
  39#include <asm/hw_irq.h>
  40#include <asm/cputhreads.h>
  41#include <asm/ppc-opcode.h>
  42#include <asm/export.h>
  43#include <asm/feature-fixups.h>
  44
  45/* The physical memory is laid out such that the secondary processor
  46 * spin code sits at 0x0000...0x00ff. On server, the vectors follow
  47 * using the layout described in exceptions-64s.S
  48 */
  49
  50/*
  51 * Entering into this code we make the following assumptions:
  52 *
  53 *  For pSeries or server processors:
  54 *   1. The MMU is off & open firmware is running in real mode.
  55 *   2. The primary CPU enters at __start.
  56 *   3. If the RTAS supports "query-cpu-stopped-state", then secondary
  57 *      CPUs will enter as directed by "start-cpu" RTAS call, which is
  58 *      generic_secondary_smp_init, with PIR in r3.
  59 *   4. Else the secondary CPUs will enter at secondary_hold (0x60) as
  60 *      directed by the "start-cpu" RTS call, with PIR in r3.
  61 * -or- For OPAL entry:
  62 *   1. The MMU is off, processor in HV mode.
  63 *   2. The primary CPU enters at 0 with device-tree in r3, OPAL base
  64 *      in r8, and entry in r9 for debugging purposes.
  65 *   3. Secondary CPUs enter as directed by OPAL_START_CPU call, which
  66 *      is at generic_secondary_smp_init, with PIR in r3.
  67 *
  68 *  For Book3E processors:
  69 *   1. The MMU is on running in AS0 in a state defined in ePAPR
  70 *   2. The kernel is entered at __start
  71 */
  72
  73OPEN_FIXED_SECTION(first_256B, 0x0, 0x100)
  74USE_FIXED_SECTION(first_256B)
  75	/*
  76	 * Offsets are relative from the start of fixed section, and
  77	 * first_256B starts at 0. Offsets are a bit easier to use here
  78	 * than the fixed section entry macros.
  79	 */
  80	. = 0x0
  81_GLOBAL(__start)
  82	/* NOP this out unconditionally */
  83BEGIN_FTR_SECTION
  84	FIXUP_ENDIAN
  85	b	__start_initialization_multiplatform
  86END_FTR_SECTION(0, 1)
  87
  88	/* Catch branch to 0 in real mode */
  89	trap
  90
  91	/* Secondary processors spin on this value until it becomes non-zero.
  92	 * When non-zero, it contains the real address of the function the cpu
  93	 * should jump to.
  94	 */
  95	.balign 8
  96	.globl  __secondary_hold_spinloop
  97__secondary_hold_spinloop:
  98	.8byte	0x0
  99
 100	/* Secondary processors write this value with their cpu # */
 101	/* after they enter the spin loop immediately below.	  */
 102	.globl	__secondary_hold_acknowledge
 103__secondary_hold_acknowledge:
 104	.8byte	0x0
 105
 106#ifdef CONFIG_RELOCATABLE
 107	/* This flag is set to 1 by a loader if the kernel should run
 108	 * at the loaded address instead of the linked address.  This
 109	 * is used by kexec-tools to keep the the kdump kernel in the
 110	 * crash_kernel region.  The loader is responsible for
 111	 * observing the alignment requirement.
 112	 */
 113
 114#ifdef CONFIG_RELOCATABLE_TEST
 115#define RUN_AT_LOAD_DEFAULT 1		/* Test relocation, do not copy to 0 */
 116#else
 117#define RUN_AT_LOAD_DEFAULT 0x72756e30  /* "run0" -- relocate to 0 by default */
 118#endif
 119
 120	/* Do not move this variable as kexec-tools knows about it. */
 121	. = 0x5c
 122	.globl	__run_at_load
 123__run_at_load:
 124DEFINE_FIXED_SYMBOL(__run_at_load)
 125	.long	RUN_AT_LOAD_DEFAULT
 126#endif
 127
 128	. = 0x60
 129/*
 130 * The following code is used to hold secondary processors
 131 * in a spin loop after they have entered the kernel, but
 132 * before the bulk of the kernel has been relocated.  This code
 133 * is relocated to physical address 0x60 before prom_init is run.
 134 * All of it must fit below the first exception vector at 0x100.
 135 * Use .globl here not _GLOBAL because we want __secondary_hold
 136 * to be the actual text address, not a descriptor.
 137 */
 138	.globl	__secondary_hold
 139__secondary_hold:
 140	FIXUP_ENDIAN
 141#ifndef CONFIG_PPC_BOOK3E
 142	mfmsr	r24
 143	ori	r24,r24,MSR_RI
 144	mtmsrd	r24			/* RI on */
 145#endif
 146	/* Grab our physical cpu number */
 147	mr	r24,r3
 148	/* stash r4 for book3e */
 149	mr	r25,r4
 150
 151	/* Tell the master cpu we're here */
 152	/* Relocation is off & we are located at an address less */
 153	/* than 0x100, so only need to grab low order offset.    */
 154	std	r24,(ABS_ADDR(__secondary_hold_acknowledge))(0)
 155	sync
 156
 157	li	r26,0
 158#ifdef CONFIG_PPC_BOOK3E
 159	tovirt(r26,r26)
 160#endif
 161	/* All secondary cpus wait here until told to start. */
 162100:	ld	r12,(ABS_ADDR(__secondary_hold_spinloop))(r26)
 163	cmpdi	0,r12,0
 164	beq	100b
 165
 166#if defined(CONFIG_SMP) || defined(CONFIG_KEXEC_CORE)
 167#ifdef CONFIG_PPC_BOOK3E
 168	tovirt(r12,r12)
 169#endif
 170	mtctr	r12
 171	mr	r3,r24
 172	/*
 173	 * it may be the case that other platforms have r4 right to
 174	 * begin with, this gives us some safety in case it is not
 175	 */
 176#ifdef CONFIG_PPC_BOOK3E
 177	mr	r4,r25
 178#else
 179	li	r4,0
 180#endif
 181	/* Make sure that patched code is visible */
 182	isync
 183	bctr
 184#else
 1850:	trap
 186	EMIT_BUG_ENTRY 0b, __FILE__, __LINE__, 0
 187#endif
 188CLOSE_FIXED_SECTION(first_256B)
 189
 190/* This value is used to mark exception frames on the stack. */
 191	.section ".toc","aw"
 192exception_marker:
 193	.tc	ID_72656773_68657265[TC],0x7265677368657265
 194	.previous
 195
 196/*
 197 * On server, we include the exception vectors code here as it
 198 * relies on absolute addressing which is only possible within
 199 * this compilation unit
 200 */
 201#ifdef CONFIG_PPC_BOOK3S
 202#include "exceptions-64s.S"
 203#else
 204OPEN_TEXT_SECTION(0x100)
 205#endif
 206
 207USE_TEXT_SECTION()
 208
 209#ifdef CONFIG_PPC_BOOK3E
 210/*
 211 * The booting_thread_hwid holds the thread id we want to boot in cpu
 212 * hotplug case. It is set by cpu hotplug code, and is invalid by default.
 213 * The thread id is the same as the initial value of SPRN_PIR[THREAD_ID]
 214 * bit field.
 215 */
 216	.globl	booting_thread_hwid
 217booting_thread_hwid:
 218	.long  INVALID_THREAD_HWID
 219	.align 3
 220/*
 221 * start a thread in the same core
 222 * input parameters:
 223 * r3 = the thread physical id
 224 * r4 = the entry point where thread starts
 225 */
 226_GLOBAL(book3e_start_thread)
 227	LOAD_REG_IMMEDIATE(r5, MSR_KERNEL)
 228	cmpwi	r3, 0
 229	beq	10f
 230	cmpwi	r3, 1
 231	beq	11f
 232	/* If the thread id is invalid, just exit. */
 233	b	13f
 23410:
 235	MTTMR(TMRN_IMSR0, 5)
 236	MTTMR(TMRN_INIA0, 4)
 237	b	12f
 23811:
 239	MTTMR(TMRN_IMSR1, 5)
 240	MTTMR(TMRN_INIA1, 4)
 24112:
 242	isync
 243	li	r6, 1
 244	sld	r6, r6, r3
 245	mtspr	SPRN_TENS, r6
 24613:
 247	blr
 248
 249/*
 250 * stop a thread in the same core
 251 * input parameter:
 252 * r3 = the thread physical id
 253 */
 254_GLOBAL(book3e_stop_thread)
 255	cmpwi	r3, 0
 256	beq	10f
 257	cmpwi	r3, 1
 258	beq	10f
 259	/* If the thread id is invalid, just exit. */
 260	b	13f
 26110:
 262	li	r4, 1
 263	sld	r4, r4, r3
 264	mtspr	SPRN_TENC, r4
 26513:
 266	blr
 267
 268_GLOBAL(fsl_secondary_thread_init)
 269	mfspr	r4,SPRN_BUCSR
 270
 271	/* Enable branch prediction */
 272	lis     r3,BUCSR_INIT@h
 273	ori     r3,r3,BUCSR_INIT@l
 274	mtspr   SPRN_BUCSR,r3
 275	isync
 276
 277	/*
 278	 * Fix PIR to match the linear numbering in the device tree.
 279	 *
 280	 * On e6500, the reset value of PIR uses the low three bits for
 281	 * the thread within a core, and the upper bits for the core
 282	 * number.  There are two threads per core, so shift everything
 283	 * but the low bit right by two bits so that the cpu numbering is
 284	 * continuous.
 285	 *
 286	 * If the old value of BUCSR is non-zero, this thread has run
 287	 * before.  Thus, we assume we are coming from kexec or a similar
 288	 * scenario, and PIR is already set to the correct value.  This
 289	 * is a bit of a hack, but there are limited opportunities for
 290	 * getting information into the thread and the alternatives
 291	 * seemed like they'd be overkill.  We can't tell just by looking
 292	 * at the old PIR value which state it's in, since the same value
 293	 * could be valid for one thread out of reset and for a different
 294	 * thread in Linux.
 295	 */
 296
 297	mfspr	r3, SPRN_PIR
 298	cmpwi	r4,0
 299	bne	1f
 300	rlwimi	r3, r3, 30, 2, 30
 301	mtspr	SPRN_PIR, r3
 3021:
 303#endif
 304
 305_GLOBAL(generic_secondary_thread_init)
 306	mr	r24,r3
 307
 308	/* turn on 64-bit mode */
 309	bl	enable_64b_mode
 310
 311	/* get a valid TOC pointer, wherever we're mapped at */
 312	bl	relative_toc
 313	tovirt(r2,r2)
 314
 315#ifdef CONFIG_PPC_BOOK3E
 316	/* Book3E initialization */
 317	mr	r3,r24
 318	bl	book3e_secondary_thread_init
 319#endif
 320	b	generic_secondary_common_init
 321
 322/*
 323 * On pSeries and most other platforms, secondary processors spin
 324 * in the following code.
 325 * At entry, r3 = this processor's number (physical cpu id)
 326 *
 327 * On Book3E, r4 = 1 to indicate that the initial TLB entry for
 328 * this core already exists (setup via some other mechanism such
 329 * as SCOM before entry).
 330 */
 331_GLOBAL(generic_secondary_smp_init)
 332	FIXUP_ENDIAN
 333	mr	r24,r3
 334	mr	r25,r4
 335
 336	/* turn on 64-bit mode */
 337	bl	enable_64b_mode
 338
 339	/* get a valid TOC pointer, wherever we're mapped at */
 340	bl	relative_toc
 341	tovirt(r2,r2)
 342
 343#ifdef CONFIG_PPC_BOOK3E
 344	/* Book3E initialization */
 345	mr	r3,r24
 346	mr	r4,r25
 347	bl	book3e_secondary_core_init
 348
 349/*
 350 * After common core init has finished, check if the current thread is the
 351 * one we wanted to boot. If not, start the specified thread and stop the
 352 * current thread.
 353 */
 354	LOAD_REG_ADDR(r4, booting_thread_hwid)
 355	lwz     r3, 0(r4)
 356	li	r5, INVALID_THREAD_HWID
 357	cmpw	r3, r5
 358	beq	20f
 359
 360	/*
 361	 * The value of booting_thread_hwid has been stored in r3,
 362	 * so make it invalid.
 363	 */
 364	stw	r5, 0(r4)
 365
 366	/*
 367	 * Get the current thread id and check if it is the one we wanted.
 368	 * If not, start the one specified in booting_thread_hwid and stop
 369	 * the current thread.
 370	 */
 371	mfspr	r8, SPRN_TIR
 372	cmpw	r3, r8
 373	beq	20f
 374
 375	/* start the specified thread */
 376	LOAD_REG_ADDR(r5, fsl_secondary_thread_init)
 377	ld	r4, 0(r5)
 378	bl	book3e_start_thread
 379
 380	/* stop the current thread */
 381	mr	r3, r8
 382	bl	book3e_stop_thread
 38310:
 384	b	10b
 38520:
 386#endif
 387
 388generic_secondary_common_init:
 389	/* Set up a paca value for this processor. Since we have the
 390	 * physical cpu id in r24, we need to search the pacas to find
 391	 * which logical id maps to our physical one.
 392	 */
 393#ifndef CONFIG_SMP
 394	b	kexec_wait		/* wait for next kernel if !SMP	 */
 395#else
 396	LOAD_REG_ADDR(r8, paca_ptrs)	/* Load paca_ptrs pointe	 */
 397	ld	r8,0(r8)		/* Get base vaddr of array	 */
 398	LOAD_REG_ADDR(r7, nr_cpu_ids)	/* Load nr_cpu_ids address       */
 399	lwz	r7,0(r7)		/* also the max paca allocated 	 */
 400	li	r5,0			/* logical cpu id                */
 4011:
 402	sldi	r9,r5,3			/* get paca_ptrs[] index from cpu id */
 403	ldx	r13,r9,r8		/* r13 = paca_ptrs[cpu id]       */
 404	lhz	r6,PACAHWCPUID(r13)	/* Load HW procid from paca      */
 405	cmpw	r6,r24			/* Compare to our id             */
 406	beq	2f
 407	addi	r5,r5,1
 408	cmpw	r5,r7			/* Check if more pacas exist     */
 409	blt	1b
 410
 411	mr	r3,r24			/* not found, copy phys to r3	 */
 412	b	kexec_wait		/* next kernel might do better	 */
 413
 4142:	SET_PACA(r13)
 415#ifdef CONFIG_PPC_BOOK3E
 416	addi	r12,r13,PACA_EXTLB	/* and TLB exc frame in another  */
 417	mtspr	SPRN_SPRG_TLB_EXFRAME,r12
 418#endif
 419
 420	/* From now on, r24 is expected to be logical cpuid */
 421	mr	r24,r5
 422
 423	/* See if we need to call a cpu state restore handler */
 424	LOAD_REG_ADDR(r23, cur_cpu_spec)
 425	ld	r23,0(r23)
 426	ld	r12,CPU_SPEC_RESTORE(r23)
 427	cmpdi	0,r12,0
 428	beq	3f
 429#ifdef PPC64_ELF_ABI_v1
 430	ld	r12,0(r12)
 431#endif
 432	mtctr	r12
 433	bctrl
 434
 4353:	LOAD_REG_ADDR(r3, spinning_secondaries) /* Decrement spinning_secondaries */
 436	lwarx	r4,0,r3
 437	subi	r4,r4,1
 438	stwcx.	r4,0,r3
 439	bne	3b
 440	isync
 441
 4424:	HMT_LOW
 443	lbz	r23,PACAPROCSTART(r13)	/* Test if this processor should */
 444					/* start.			 */
 445	cmpwi	0,r23,0
 446	beq	4b			/* Loop until told to go	 */
 447
 448	sync				/* order paca.run and cur_cpu_spec */
 449	isync				/* In case code patching happened */
 450
 451	/* Create a temp kernel stack for use before relocation is on.	*/
 452	ld	r1,PACAEMERGSP(r13)
 453	subi	r1,r1,STACK_FRAME_OVERHEAD
 454
 455	b	__secondary_start
 456#endif /* SMP */
 457
 458/*
 459 * Turn the MMU off.
 460 * Assumes we're mapped EA == RA if the MMU is on.
 461 */
 462#ifdef CONFIG_PPC_BOOK3S
 463__mmu_off:
 464	mfmsr	r3
 465	andi.	r0,r3,MSR_IR|MSR_DR
 466	beqlr
 467	mflr	r4
 468	andc	r3,r3,r0
 469	mtspr	SPRN_SRR0,r4
 470	mtspr	SPRN_SRR1,r3
 471	sync
 472	rfid
 473	b	.	/* prevent speculative execution */
 474#endif
 475
 476
 477/*
 478 * Here is our main kernel entry point. We support currently 2 kind of entries
 479 * depending on the value of r5.
 480 *
 481 *   r5 != NULL -> OF entry, we go to prom_init, "legacy" parameter content
 482 *                 in r3...r7
 483 *   
 484 *   r5 == NULL -> kexec style entry. r3 is a physical pointer to the
 485 *                 DT block, r4 is a physical pointer to the kernel itself
 486 *
 487 */
 488__start_initialization_multiplatform:
 489	/* Make sure we are running in 64 bits mode */
 490	bl	enable_64b_mode
 491
 492	/* Get TOC pointer (current runtime address) */
 493	bl	relative_toc
 494
 495	/* find out where we are now */
 496	bcl	20,31,$+4
 4970:	mflr	r26			/* r26 = runtime addr here */
 498	addis	r26,r26,(_stext - 0b)@ha
 499	addi	r26,r26,(_stext - 0b)@l	/* current runtime base addr */
 500
 501	/*
 502	 * Are we booted from a PROM Of-type client-interface ?
 503	 */
 504	cmpldi	cr0,r5,0
 505	beq	1f
 506	b	__boot_from_prom		/* yes -> prom */
 5071:
 508	/* Save parameters */
 509	mr	r31,r3
 510	mr	r30,r4
 511#ifdef CONFIG_PPC_EARLY_DEBUG_OPAL
 512	/* Save OPAL entry */
 513	mr	r28,r8
 514	mr	r29,r9
 515#endif
 516
 517#ifdef CONFIG_PPC_BOOK3E
 518	bl	start_initialization_book3e
 519	b	__after_prom_start
 520#else
 521	/* Setup some critical 970 SPRs before switching MMU off */
 522	mfspr	r0,SPRN_PVR
 523	srwi	r0,r0,16
 524	cmpwi	r0,0x39		/* 970 */
 525	beq	1f
 526	cmpwi	r0,0x3c		/* 970FX */
 527	beq	1f
 528	cmpwi	r0,0x44		/* 970MP */
 529	beq	1f
 530	cmpwi	r0,0x45		/* 970GX */
 531	bne	2f
 5321:	bl	__cpu_preinit_ppc970
 5332:
 534
 535	/* Switch off MMU if not already off */
 536	bl	__mmu_off
 537	b	__after_prom_start
 538#endif /* CONFIG_PPC_BOOK3E */
 539
 540__REF
 541__boot_from_prom:
 542#ifdef CONFIG_PPC_OF_BOOT_TRAMPOLINE
 543	/* Save parameters */
 544	mr	r31,r3
 545	mr	r30,r4
 546	mr	r29,r5
 547	mr	r28,r6
 548	mr	r27,r7
 549
 550	/*
 551	 * Align the stack to 16-byte boundary
 552	 * Depending on the size and layout of the ELF sections in the initial
 553	 * boot binary, the stack pointer may be unaligned on PowerMac
 554	 */
 555	rldicr	r1,r1,0,59
 556
 557#ifdef CONFIG_RELOCATABLE
 558	/* Relocate code for where we are now */
 559	mr	r3,r26
 560	bl	relocate
 561#endif
 562
 563	/* Restore parameters */
 564	mr	r3,r31
 565	mr	r4,r30
 566	mr	r5,r29
 567	mr	r6,r28
 568	mr	r7,r27
 569
 570	/* Do all of the interaction with OF client interface */
 571	mr	r8,r26
 572	bl	prom_init
 573#endif /* #CONFIG_PPC_OF_BOOT_TRAMPOLINE */
 574
 575	/* We never return. We also hit that trap if trying to boot
 576	 * from OF while CONFIG_PPC_OF_BOOT_TRAMPOLINE isn't selected */
 577	trap
 578	.previous
 579
 580__after_prom_start:
 581#ifdef CONFIG_RELOCATABLE
 582	/* process relocations for the final address of the kernel */
 583	lis	r25,PAGE_OFFSET@highest	/* compute virtual base of kernel */
 584	sldi	r25,r25,32
 585#if defined(CONFIG_PPC_BOOK3E)
 586	tovirt(r26,r26)		/* on booke, we already run at PAGE_OFFSET */
 587#endif
 588	lwz	r7,(FIXED_SYMBOL_ABS_ADDR(__run_at_load))(r26)
 589#if defined(CONFIG_PPC_BOOK3E)
 590	tophys(r26,r26)
 591#endif
 592	cmplwi	cr0,r7,1	/* flagged to stay where we are ? */
 593	bne	1f
 594	add	r25,r25,r26
 5951:	mr	r3,r25
 596	bl	relocate
 597#if defined(CONFIG_PPC_BOOK3E)
 598	/* IVPR needs to be set after relocation. */
 599	bl	init_core_book3e
 600#endif
 601#endif
 602
 603/*
 604 * We need to run with _stext at physical address PHYSICAL_START.
 605 * This will leave some code in the first 256B of
 606 * real memory, which are reserved for software use.
 607 *
 608 * Note: This process overwrites the OF exception vectors.
 609 */
 610	li	r3,0			/* target addr */
 611#ifdef CONFIG_PPC_BOOK3E
 612	tovirt(r3,r3)		/* on booke, we already run at PAGE_OFFSET */
 613#endif
 614	mr.	r4,r26			/* In some cases the loader may  */
 615#if defined(CONFIG_PPC_BOOK3E)
 616	tovirt(r4,r4)
 617#endif
 618	beq	9f			/* have already put us at zero */
 619	li	r6,0x100		/* Start offset, the first 0x100 */
 620					/* bytes were copied earlier.	 */
 621
 622#ifdef CONFIG_RELOCATABLE
 623/*
 624 * Check if the kernel has to be running as relocatable kernel based on the
 625 * variable __run_at_load, if it is set the kernel is treated as relocatable
 626 * kernel, otherwise it will be moved to PHYSICAL_START
 627 */
 628#if defined(CONFIG_PPC_BOOK3E)
 629	tovirt(r26,r26)		/* on booke, we already run at PAGE_OFFSET */
 630#endif
 631	lwz	r7,(FIXED_SYMBOL_ABS_ADDR(__run_at_load))(r26)
 632	cmplwi	cr0,r7,1
 633	bne	3f
 634
 635#ifdef CONFIG_PPC_BOOK3E
 636	LOAD_REG_ADDR(r5, __end_interrupts)
 637	LOAD_REG_ADDR(r11, _stext)
 638	sub	r5,r5,r11
 639#else
 640	/* just copy interrupts */
 641	LOAD_REG_IMMEDIATE_SYM(r5, r11, FIXED_SYMBOL_ABS_ADDR(__end_interrupts))
 642#endif
 643	b	5f
 6443:
 645#endif
 646	/* # bytes of memory to copy */
 647	lis	r5,(ABS_ADDR(copy_to_here))@ha
 648	addi	r5,r5,(ABS_ADDR(copy_to_here))@l
 649
 650	bl	copy_and_flush		/* copy the first n bytes	 */
 651					/* this includes the code being	 */
 652					/* executed here.		 */
 653	/* Jump to the copy of this code that we just made */
 654	addis	r8,r3,(ABS_ADDR(4f))@ha
 655	addi	r12,r8,(ABS_ADDR(4f))@l
 656	mtctr	r12
 657	bctr
 658
 659.balign 8
 660p_end: .8byte _end - copy_to_here
 661
 6624:
 663	/*
 664	 * Now copy the rest of the kernel up to _end, add
 665	 * _end - copy_to_here to the copy limit and run again.
 666	 */
 667	addis   r8,r26,(ABS_ADDR(p_end))@ha
 668	ld      r8,(ABS_ADDR(p_end))@l(r8)
 669	add	r5,r5,r8
 6705:	bl	copy_and_flush		/* copy the rest */
 671
 6729:	b	start_here_multiplatform
 673
 674/*
 675 * Copy routine used to copy the kernel to start at physical address 0
 676 * and flush and invalidate the caches as needed.
 677 * r3 = dest addr, r4 = source addr, r5 = copy limit, r6 = start offset
 678 * on exit, r3, r4, r5 are unchanged, r6 is updated to be >= r5.
 679 *
 680 * Note: this routine *only* clobbers r0, r6 and lr
 681 */
 682_GLOBAL(copy_and_flush)
 683	addi	r5,r5,-8
 684	addi	r6,r6,-8
 6854:	li	r0,8			/* Use the smallest common	*/
 686					/* denominator cache line	*/
 687					/* size.  This results in	*/
 688					/* extra cache line flushes	*/
 689					/* but operation is correct.	*/
 690					/* Can't get cache line size	*/
 691					/* from NACA as it is being	*/
 692					/* moved too.			*/
 693
 694	mtctr	r0			/* put # words/line in ctr	*/
 6953:	addi	r6,r6,8			/* copy a cache line		*/
 696	ldx	r0,r6,r4
 697	stdx	r0,r6,r3
 698	bdnz	3b
 699	dcbst	r6,r3			/* write it to memory		*/
 700	sync
 701	icbi	r6,r3			/* flush the icache line	*/
 702	cmpld	0,r6,r5
 703	blt	4b
 704	sync
 705	addi	r5,r5,8
 706	addi	r6,r6,8
 707	isync
 708	blr
 709
 710.align 8
 711copy_to_here:
 712
 713#ifdef CONFIG_SMP
 714#ifdef CONFIG_PPC_PMAC
 715/*
 716 * On PowerMac, secondary processors starts from the reset vector, which
 717 * is temporarily turned into a call to one of the functions below.
 718 */
 719	.section ".text";
 720	.align 2 ;
 721
 722	.globl	__secondary_start_pmac_0
 723__secondary_start_pmac_0:
 724	/* NB the entries for cpus 0, 1, 2 must each occupy 8 bytes. */
 725	li	r24,0
 726	b	1f
 727	li	r24,1
 728	b	1f
 729	li	r24,2
 730	b	1f
 731	li	r24,3
 7321:
 733	
 734_GLOBAL(pmac_secondary_start)
 735	/* turn on 64-bit mode */
 736	bl	enable_64b_mode
 737
 738	li	r0,0
 739	mfspr	r3,SPRN_HID4
 740	rldimi	r3,r0,40,23	/* clear bit 23 (rm_ci) */
 741	sync
 742	mtspr	SPRN_HID4,r3
 743	isync
 744	sync
 745	slbia
 746
 747	/* get TOC pointer (real address) */
 748	bl	relative_toc
 749	tovirt(r2,r2)
 750
 751	/* Copy some CPU settings from CPU 0 */
 752	bl	__restore_cpu_ppc970
 753
 754	/* pSeries do that early though I don't think we really need it */
 755	mfmsr	r3
 756	ori	r3,r3,MSR_RI
 757	mtmsrd	r3			/* RI on */
 758
 759	/* Set up a paca value for this processor. */
 760	LOAD_REG_ADDR(r4,paca_ptrs)	/* Load paca pointer		*/
 761	ld	r4,0(r4)		/* Get base vaddr of paca_ptrs array */
 762	sldi	r5,r24,3		/* get paca_ptrs[] index from cpu id */
 763	ldx	r13,r5,r4		/* r13 = paca_ptrs[cpu id]       */
 764	SET_PACA(r13)			/* Save vaddr of paca in an SPRG*/
 765
 766	/* Mark interrupts soft and hard disabled (they might be enabled
 767	 * in the PACA when doing hotplug)
 768	 */
 769	li	r0,IRQS_DISABLED
 770	stb	r0,PACAIRQSOFTMASK(r13)
 771	li	r0,PACA_IRQ_HARD_DIS
 772	stb	r0,PACAIRQHAPPENED(r13)
 773
 774	/* Create a temp kernel stack for use before relocation is on.	*/
 775	ld	r1,PACAEMERGSP(r13)
 776	subi	r1,r1,STACK_FRAME_OVERHEAD
 777
 778	b	__secondary_start
 779
 780#endif /* CONFIG_PPC_PMAC */
 781
 782/*
 783 * This function is called after the master CPU has released the
 784 * secondary processors.  The execution environment is relocation off.
 785 * The paca for this processor has the following fields initialized at
 786 * this point:
 787 *   1. Processor number
 788 *   2. Segment table pointer (virtual address)
 789 * On entry the following are set:
 790 *   r1	       = stack pointer (real addr of temp stack)
 791 *   r24       = cpu# (in Linux terms)
 792 *   r13       = paca virtual address
 793 *   SPRG_PACA = paca virtual address
 794 */
 795	.section ".text";
 796	.align 2 ;
 797
 798	.globl	__secondary_start
 799__secondary_start:
 800	/* Set thread priority to MEDIUM */
 801	HMT_MEDIUM
 802
 803	/*
 804	 * Do early setup for this CPU, in particular initialising the MMU so we
 805	 * can turn it on below. This is a call to C, which is OK, we're still
 806	 * running on the emergency stack.
 807	 */
 808	bl	early_setup_secondary
 809
 810	/*
 811	 * The primary has initialized our kernel stack for us in the paca, grab
 812	 * it and put it in r1. We must *not* use it until we turn on the MMU
 813	 * below, because it may not be inside the RMO.
 814	 */
 815	ld	r1, PACAKSAVE(r13)
 816
 817	/* Clear backchain so we get nice backtraces */
 818	li	r7,0
 819	mtlr	r7
 820
 821	/* Mark interrupts soft and hard disabled (they might be enabled
 822	 * in the PACA when doing hotplug)
 823	 */
 824	li	r7,IRQS_DISABLED
 825	stb	r7,PACAIRQSOFTMASK(r13)
 826	li	r0,PACA_IRQ_HARD_DIS
 827	stb	r0,PACAIRQHAPPENED(r13)
 828
 829	/* enable MMU and jump to start_secondary */
 830	LOAD_REG_ADDR(r3, start_secondary_prolog)
 831	LOAD_REG_IMMEDIATE(r4, MSR_KERNEL)
 832
 833	mtspr	SPRN_SRR0,r3
 834	mtspr	SPRN_SRR1,r4
 835	RFI
 836	b	.	/* prevent speculative execution */
 837
 838/* 
 839 * Running with relocation on at this point.  All we want to do is
 840 * zero the stack back-chain pointer and get the TOC virtual address
 841 * before going into C code.
 842 */
 843start_secondary_prolog:
 844	ld	r2,PACATOC(r13)
 845	li	r3,0
 846	std	r3,0(r1)		/* Zero the stack frame pointer	*/
 847	bl	start_secondary
 848	b	.
 849/*
 850 * Reset stack pointer and call start_secondary
 851 * to continue with online operation when woken up
 852 * from cede in cpu offline.
 853 */
 854_GLOBAL(start_secondary_resume)
 855	ld	r1,PACAKSAVE(r13)	/* Reload kernel stack pointer */
 856	li	r3,0
 857	std	r3,0(r1)		/* Zero the stack frame pointer	*/
 858	bl	start_secondary
 859	b	.
 860#endif
 861
 862/*
 863 * This subroutine clobbers r11 and r12
 864 */
 865enable_64b_mode:
 866	mfmsr	r11			/* grab the current MSR */
 867#ifdef CONFIG_PPC_BOOK3E
 868	oris	r11,r11,0x8000		/* CM bit set, we'll set ICM later */
 869	mtmsr	r11
 870#else /* CONFIG_PPC_BOOK3E */
 871	li	r12,(MSR_64BIT | MSR_ISF)@highest
 872	sldi	r12,r12,48
 873	or	r11,r11,r12
 874	mtmsrd	r11
 875	isync
 876#endif
 877	blr
 878
 879/*
 880 * This puts the TOC pointer into r2, offset by 0x8000 (as expected
 881 * by the toolchain).  It computes the correct value for wherever we
 882 * are running at the moment, using position-independent code.
 883 *
 884 * Note: The compiler constructs pointers using offsets from the
 885 * TOC in -mcmodel=medium mode. After we relocate to 0 but before
 886 * the MMU is on we need our TOC to be a virtual address otherwise
 887 * these pointers will be real addresses which may get stored and
 888 * accessed later with the MMU on. We use tovirt() at the call
 889 * sites to handle this.
 890 */
 891_GLOBAL(relative_toc)
 892	mflr	r0
 893	bcl	20,31,$+4
 8940:	mflr	r11
 895	ld	r2,(p_toc - 0b)(r11)
 896	add	r2,r2,r11
 897	mtlr	r0
 898	blr
 899
 900.balign 8
 901p_toc:	.8byte	__toc_start + 0x8000 - 0b
 902
 903/*
 904 * This is where the main kernel code starts.
 905 */
 906__REF
 907start_here_multiplatform:
 908	/* set up the TOC */
 909	bl      relative_toc
 910	tovirt(r2,r2)
 911
 912	/* Clear out the BSS. It may have been done in prom_init,
 913	 * already but that's irrelevant since prom_init will soon
 914	 * be detached from the kernel completely. Besides, we need
 915	 * to clear it now for kexec-style entry.
 916	 */
 917	LOAD_REG_ADDR(r11,__bss_stop)
 918	LOAD_REG_ADDR(r8,__bss_start)
 919	sub	r11,r11,r8		/* bss size			*/
 920	addi	r11,r11,7		/* round up to an even double word */
 921	srdi.	r11,r11,3		/* shift right by 3		*/
 922	beq	4f
 923	addi	r8,r8,-8
 924	li	r0,0
 925	mtctr	r11			/* zero this many doublewords	*/
 9263:	stdu	r0,8(r8)
 927	bdnz	3b
 9284:
 929
 930#ifdef CONFIG_PPC_EARLY_DEBUG_OPAL
 931	/* Setup OPAL entry */
 932	LOAD_REG_ADDR(r11, opal)
 933	std	r28,0(r11);
 934	std	r29,8(r11);
 935#endif
 936
 937#ifndef CONFIG_PPC_BOOK3E
 938	mfmsr	r6
 939	ori	r6,r6,MSR_RI
 940	mtmsrd	r6			/* RI on */
 941#endif
 942
 943#ifdef CONFIG_RELOCATABLE
 944	/* Save the physical address we're running at in kernstart_addr */
 945	LOAD_REG_ADDR(r4, kernstart_addr)
 946	clrldi	r0,r25,2
 947	std	r0,0(r4)
 948#endif
 949
 950	/* The following gets the stack set up with the regs */
 951	/* pointing to the real addr of the kernel stack.  This is   */
 952	/* all done to support the C function call below which sets  */
 953	/* up the htab.  This is done because we have relocated the  */
 954	/* kernel but are still running in real mode. */
 955
 956	LOAD_REG_ADDR(r3,init_thread_union)
 957
 958	/* set up a stack pointer */
 959	LOAD_REG_IMMEDIATE(r1,THREAD_SIZE)
 960	add	r1,r3,r1
 961	li	r0,0
 962	stdu	r0,-STACK_FRAME_OVERHEAD(r1)
 963
 964	/*
 965	 * Do very early kernel initializations, including initial hash table
 966	 * and SLB setup before we turn on relocation.
 967	 */
 968
 969	/* Restore parameters passed from prom_init/kexec */
 970	mr	r3,r31
 971	LOAD_REG_ADDR(r12, DOTSYM(early_setup))
 972	mtctr	r12
 973	bctrl		/* also sets r13 and SPRG_PACA */
 974
 975	LOAD_REG_ADDR(r3, start_here_common)
 976	ld	r4,PACAKMSR(r13)
 977	mtspr	SPRN_SRR0,r3
 978	mtspr	SPRN_SRR1,r4
 979	RFI
 980	b	.	/* prevent speculative execution */
 981
 982	/* This is where all platforms converge execution */
 983
 984start_here_common:
 985	/* relocation is on at this point */
 986	std	r1,PACAKSAVE(r13)
 987
 988	/* Load the TOC (virtual address) */
 989	ld	r2,PACATOC(r13)
 990
 991	/* Mark interrupts soft and hard disabled (they might be enabled
 992	 * in the PACA when doing hotplug)
 993	 */
 994	li	r0,IRQS_DISABLED
 995	stb	r0,PACAIRQSOFTMASK(r13)
 996	li	r0,PACA_IRQ_HARD_DIS
 997	stb	r0,PACAIRQHAPPENED(r13)
 998
 999	/* Generic kernel entry */
1000	bl	start_kernel
1001
1002	/* Not reached */
1003	trap
1004	EMIT_BUG_ENTRY 0b, __FILE__, __LINE__, 0
1005	.previous
1006
1007/*
1008 * We put a few things here that have to be page-aligned.
1009 * This stuff goes at the beginning of the bss, which is page-aligned.
1010 */
1011	.section ".bss"
1012/*
1013 * pgd dir should be aligned to PGD_TABLE_SIZE which is 64K.
1014 * We will need to find a better way to fix this
1015 */
1016	.align	16
1017
1018	.globl	swapper_pg_dir
1019swapper_pg_dir:
1020	.space	PGD_TABLE_SIZE
1021
1022	.globl	empty_zero_page
1023empty_zero_page:
1024	.space	PAGE_SIZE
1025EXPORT_SYMBOL(empty_zero_page)