/arch/x86/kernel/trampoline_64.S

  1/*
  2 *
  3 *	Trampoline.S	Derived from Setup.S by Linus Torvalds
  4 *
  5 *	4 Jan 1997 Michael Chastain: changed to gnu as.
  6 *	15 Sept 2005 Eric Biederman: 64bit PIC support
  7 *
  8 *	Entry: CS:IP point to the start of our code, we are 
  9 *	in real mode with no stack, but the rest of the 
 10 *	trampoline page to make our stack and everything else
 11 *	is a mystery.
 12 *
 13 *	On entry to trampoline_data, the processor is in real mode
 14 *	with 16-bit addressing and 16-bit data.  CS has some value
 15 *	and IP is zero.  Thus, data addresses need to be absolute
 16 *	(no relocation) and are taken with regard to r_base.
 17 *
 18 *	With the addition of trampoline_level4_pgt this code can
 19 *	now enter a 64bit kernel that lives at arbitrary 64bit
 20 *	physical addresses.
 21 *
 22 *	If you work on this file, check the object module with objdump
 23 *	--full-contents --reloc to make sure there are no relocation
 24 *	entries.
 25 */
 26
 27#include <linux/linkage.h>
 28#include <linux/init.h>
 29#include <asm/pgtable_types.h>
 30#include <asm/page_types.h>
 31#include <asm/msr.h>
 32#include <asm/segment.h>
 33#include <asm/processor-flags.h>
 34
 35	.section ".x86_trampoline","a"
 36	.balign PAGE_SIZE
 37	.code16
 38
 39ENTRY(trampoline_data)
 40r_base = .
 41	cli			# We should be safe anyway
 42	wbinvd
 43	mov	%cs, %ax	# Code and data in the same place
 44	mov	%ax, %ds
 45	mov	%ax, %es
 46	mov	%ax, %ss
 47
 48
 49	movl	$0xA5A5A5A5, trampoline_status - r_base
 50				# write marker for master knows we're running
 51
 52					# Setup stack
 53	movw	$(trampoline_stack_end - r_base), %sp
 54
 55	call	verify_cpu		# Verify the cpu supports long mode
 56	testl   %eax, %eax		# Check for return code
 57	jnz	no_longmode
 58
 59	mov	%cs, %ax
 60	movzx	%ax, %esi		# Find the 32bit trampoline location
 61	shll	$4, %esi
 62
 63					# Fixup the absolute vectors
 64	leal	(startup_32 - r_base)(%esi), %eax
 65	movl	%eax, startup_32_vector - r_base
 66	leal	(startup_64 - r_base)(%esi), %eax
 67	movl	%eax, startup_64_vector - r_base
 68	leal	(tgdt - r_base)(%esi), %eax
 69	movl	%eax, (tgdt + 2 - r_base)
 70
 71	/*
 72	 * GDT tables in non default location kernel can be beyond 16MB and
 73	 * lgdt will not be able to load the address as in real mode default
 74	 * operand size is 16bit. Use lgdtl instead to force operand size
 75	 * to 32 bit.
 76	 */
 77
 78	lidtl	tidt - r_base	# load idt with 0, 0
 79	lgdtl	tgdt - r_base	# load gdt with whatever is appropriate
 80
 81	mov	$X86_CR0_PE, %ax	# protected mode (PE) bit
 82	lmsw	%ax			# into protected mode
 83
 84	# flush prefetch and jump to startup_32
 85	ljmpl	*(startup_32_vector - r_base)
 86
 87	.code32
 88	.balign 4
 89startup_32:
 90	movl	$__KERNEL_DS, %eax	# Initialize the %ds segment register
 91	movl	%eax, %ds
 92
 93	movl	$X86_CR4_PAE, %eax
 94	movl	%eax, %cr4		# Enable PAE mode
 95
 96					# Setup trampoline 4 level pagetables
 97	leal	(trampoline_level4_pgt - r_base)(%esi), %eax
 98	movl	%eax, %cr3
 99
100	movl	$MSR_EFER, %ecx
101	movl	$(1 << _EFER_LME), %eax	# Enable Long Mode
102	xorl	%edx, %edx
103	wrmsr
104
105	# Enable paging and in turn activate Long Mode
106	# Enable protected mode
107	movl	$(X86_CR0_PG | X86_CR0_PE), %eax
108	movl	%eax, %cr0
109
110	/*
111	 * At this point we're in long mode but in 32bit compatibility mode
112	 * with EFER.LME = 1, CS.L = 0, CS.D = 1 (and in turn
113	 * EFER.LMA = 1). Now we want to jump in 64bit mode, to do that we use
114	 * the new gdt/idt that has __KERNEL_CS with CS.L = 1.
115	 */
116	ljmp	*(startup_64_vector - r_base)(%esi)
117
118	.code64
119	.balign 4
120startup_64:
121	# Now jump into the kernel using virtual addresses
122	movq	$secondary_startup_64, %rax
123	jmp	*%rax
124
125	.code16
126no_longmode:
127	hlt
128	jmp no_longmode
129#include "verify_cpu.S"
130
131	.balign 4
132	# Careful these need to be in the same 64K segment as the above;
133tidt:
134	.word	0			# idt limit = 0
135	.word	0, 0			# idt base = 0L
136
137	# Duplicate the global descriptor table
138	# so the kernel can live anywhere
139	.balign 4
140tgdt:
141	.short	tgdt_end - tgdt		# gdt limit
142	.long	tgdt - r_base
143	.short 0
144	.quad	0x00cf9b000000ffff	# __KERNEL32_CS
145	.quad	0x00af9b000000ffff	# __KERNEL_CS
146	.quad	0x00cf93000000ffff	# __KERNEL_DS
147tgdt_end:
148
149	.balign 4
150startup_32_vector:
151	.long	startup_32 - r_base
152	.word	__KERNEL32_CS, 0
153
154	.balign 4
155startup_64_vector:
156	.long	startup_64 - r_base
157	.word	__KERNEL_CS, 0
158
159	.balign 4
160ENTRY(trampoline_status)
161	.long	0
162
163trampoline_stack:
164	.org 0x1000
165trampoline_stack_end:
166ENTRY(trampoline_level4_pgt)
167	.quad	level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE
168	.fill	510,8,0
169	.quad	level3_kernel_pgt - __START_KERNEL_map + _KERNPG_TABLE
170
171ENTRY(trampoline_end)