PageRenderTime 57ms CodeModel.GetById 13ms app.highlight 34ms RepoModel.GetById 2ms app.codeStats 0ms

/arch/x86/kernel/entry_32.S

https://gitlab.com/Team-OSE-old/SimpleKernel
Assembly | 1417 lines | 1338 code | 79 blank | 0 comment | 27 complexity | c3630cabe59707589346124533b60b1d MD5 | raw file
   1/*
   2 *
   3 *  Copyright (C) 1991, 1992  Linus Torvalds
   4 */
   5
   6/*
   7 * entry.S contains the system-call and fault low-level handling routines.
   8 * This also contains the timer-interrupt handler, as well as all interrupts
   9 * and faults that can result in a task-switch.
  10 *
  11 * NOTE: This code handles signal-recognition, which happens every time
  12 * after a timer-interrupt and after each system call.
  13 *
  14 * I changed all the .align's to 4 (16 byte alignment), as that's faster
  15 * on a 486.
  16 *
  17 * Stack layout in 'syscall_exit':
  18 * 	ptrace needs to have all regs on the stack.
  19 *	if the order here is changed, it needs to be
  20 *	updated in fork.c:copy_process, signal.c:do_signal,
  21 *	ptrace.c and ptrace.h
  22 *
  23 *	 0(%esp) - %ebx
  24 *	 4(%esp) - %ecx
  25 *	 8(%esp) - %edx
  26 *       C(%esp) - %esi
  27 *	10(%esp) - %edi
  28 *	14(%esp) - %ebp
  29 *	18(%esp) - %eax
  30 *	1C(%esp) - %ds
  31 *	20(%esp) - %es
  32 *	24(%esp) - %fs
  33 *	28(%esp) - %gs		saved iff !CONFIG_X86_32_LAZY_GS
  34 *	2C(%esp) - orig_eax
  35 *	30(%esp) - %eip
  36 *	34(%esp) - %cs
  37 *	38(%esp) - %eflags
  38 *	3C(%esp) - %oldesp
  39 *	40(%esp) - %oldss
  40 *
  41 * "current" is in register %ebx during any slow entries.
  42 */
  43
  44#include <linux/linkage.h>
  45#include <linux/err.h>
  46#include <asm/thread_info.h>
  47#include <asm/irqflags.h>
  48#include <asm/errno.h>
  49#include <asm/segment.h>
  50#include <asm/smp.h>
  51#include <asm/page_types.h>
  52#include <asm/percpu.h>
  53#include <asm/dwarf2.h>
  54#include <asm/processor-flags.h>
  55#include <asm/ftrace.h>
  56#include <asm/irq_vectors.h>
  57#include <asm/cpufeature.h>
  58#include <asm/alternative-asm.h>
  59
  60/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this.  */
  61#include <linux/elf-em.h>
  62#define AUDIT_ARCH_I386		(EM_386|__AUDIT_ARCH_LE)
  63#define __AUDIT_ARCH_LE	   0x40000000
  64
  65#ifndef CONFIG_AUDITSYSCALL
  66#define sysenter_audit	syscall_trace_entry
  67#define sysexit_audit	syscall_exit_work
  68#endif
  69
  70	.section .entry.text, "ax"
  71
  72/*
  73 * We use macros for low-level operations which need to be overridden
  74 * for paravirtualization.  The following will never clobber any registers:
  75 *   INTERRUPT_RETURN (aka. "iret")
  76 *   GET_CR0_INTO_EAX (aka. "movl %cr0, %eax")
  77 *   ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit").
  78 *
  79 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must
  80 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY).
  81 * Allowing a register to be clobbered can shrink the paravirt replacement
  82 * enough to patch inline, increasing performance.
  83 */
  84
  85#ifdef CONFIG_PREEMPT
  86#define preempt_stop(clobbers)	DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF
  87#else
  88#define preempt_stop(clobbers)
  89#define resume_kernel		restore_all
  90#endif
  91
  92.macro TRACE_IRQS_IRET
  93#ifdef CONFIG_TRACE_IRQFLAGS
  94	testl $X86_EFLAGS_IF,PT_EFLAGS(%esp)     # interrupts off?
  95	jz 1f
  96	TRACE_IRQS_ON
  971:
  98#endif
  99.endm
 100
 101/*
 102 * User gs save/restore
 103 *
 104 * %gs is used for userland TLS and kernel only uses it for stack
 105 * canary which is required to be at %gs:20 by gcc.  Read the comment
 106 * at the top of stackprotector.h for more info.
 107 *
 108 * Local labels 98 and 99 are used.
 109 */
 110#ifdef CONFIG_X86_32_LAZY_GS
 111
 112 /* unfortunately push/pop can't be no-op */
 113.macro PUSH_GS
 114	pushl_cfi $0
 115.endm
 116.macro POP_GS pop=0
 117	addl $(4 + \pop), %esp
 118	CFI_ADJUST_CFA_OFFSET -(4 + \pop)
 119.endm
 120.macro POP_GS_EX
 121.endm
 122
 123 /* all the rest are no-op */
 124.macro PTGS_TO_GS
 125.endm
 126.macro PTGS_TO_GS_EX
 127.endm
 128.macro GS_TO_REG reg
 129.endm
 130.macro REG_TO_PTGS reg
 131.endm
 132.macro SET_KERNEL_GS reg
 133.endm
 134
 135#else	/* CONFIG_X86_32_LAZY_GS */
 136
 137.macro PUSH_GS
 138	pushl_cfi %gs
 139	/*CFI_REL_OFFSET gs, 0*/
 140.endm
 141
 142.macro POP_GS pop=0
 14398:	popl_cfi %gs
 144	/*CFI_RESTORE gs*/
 145  .if \pop <> 0
 146	add $\pop, %esp
 147	CFI_ADJUST_CFA_OFFSET -\pop
 148  .endif
 149.endm
 150.macro POP_GS_EX
 151.pushsection .fixup, "ax"
 15299:	movl $0, (%esp)
 153	jmp 98b
 154.section __ex_table, "a"
 155	.align 4
 156	.long 98b, 99b
 157.popsection
 158.endm
 159
 160.macro PTGS_TO_GS
 16198:	mov PT_GS(%esp), %gs
 162.endm
 163.macro PTGS_TO_GS_EX
 164.pushsection .fixup, "ax"
 16599:	movl $0, PT_GS(%esp)
 166	jmp 98b
 167.section __ex_table, "a"
 168	.align 4
 169	.long 98b, 99b
 170.popsection
 171.endm
 172
 173.macro GS_TO_REG reg
 174	movl %gs, \reg
 175	/*CFI_REGISTER gs, \reg*/
 176.endm
 177.macro REG_TO_PTGS reg
 178	movl \reg, PT_GS(%esp)
 179	/*CFI_REL_OFFSET gs, PT_GS*/
 180.endm
 181.macro SET_KERNEL_GS reg
 182	movl $(__KERNEL_STACK_CANARY), \reg
 183	movl \reg, %gs
 184.endm
 185
 186#endif	/* CONFIG_X86_32_LAZY_GS */
 187
 188.macro SAVE_ALL
 189	cld
 190	PUSH_GS
 191	pushl_cfi %fs
 192	/*CFI_REL_OFFSET fs, 0;*/
 193	pushl_cfi %es
 194	/*CFI_REL_OFFSET es, 0;*/
 195	pushl_cfi %ds
 196	/*CFI_REL_OFFSET ds, 0;*/
 197	pushl_cfi %eax
 198	CFI_REL_OFFSET eax, 0
 199	pushl_cfi %ebp
 200	CFI_REL_OFFSET ebp, 0
 201	pushl_cfi %edi
 202	CFI_REL_OFFSET edi, 0
 203	pushl_cfi %esi
 204	CFI_REL_OFFSET esi, 0
 205	pushl_cfi %edx
 206	CFI_REL_OFFSET edx, 0
 207	pushl_cfi %ecx
 208	CFI_REL_OFFSET ecx, 0
 209	pushl_cfi %ebx
 210	CFI_REL_OFFSET ebx, 0
 211	movl $(__USER_DS), %edx
 212	movl %edx, %ds
 213	movl %edx, %es
 214	movl $(__KERNEL_PERCPU), %edx
 215	movl %edx, %fs
 216	SET_KERNEL_GS %edx
 217.endm
 218
 219.macro RESTORE_INT_REGS
 220	popl_cfi %ebx
 221	CFI_RESTORE ebx
 222	popl_cfi %ecx
 223	CFI_RESTORE ecx
 224	popl_cfi %edx
 225	CFI_RESTORE edx
 226	popl_cfi %esi
 227	CFI_RESTORE esi
 228	popl_cfi %edi
 229	CFI_RESTORE edi
 230	popl_cfi %ebp
 231	CFI_RESTORE ebp
 232	popl_cfi %eax
 233	CFI_RESTORE eax
 234.endm
 235
 236.macro RESTORE_REGS pop=0
 237	RESTORE_INT_REGS
 2381:	popl_cfi %ds
 239	/*CFI_RESTORE ds;*/
 2402:	popl_cfi %es
 241	/*CFI_RESTORE es;*/
 2423:	popl_cfi %fs
 243	/*CFI_RESTORE fs;*/
 244	POP_GS \pop
 245.pushsection .fixup, "ax"
 2464:	movl $0, (%esp)
 247	jmp 1b
 2485:	movl $0, (%esp)
 249	jmp 2b
 2506:	movl $0, (%esp)
 251	jmp 3b
 252.section __ex_table, "a"
 253	.align 4
 254	.long 1b, 4b
 255	.long 2b, 5b
 256	.long 3b, 6b
 257.popsection
 258	POP_GS_EX
 259.endm
 260
 261.macro RING0_INT_FRAME
 262	CFI_STARTPROC simple
 263	CFI_SIGNAL_FRAME
 264	CFI_DEF_CFA esp, 3*4
 265	/*CFI_OFFSET cs, -2*4;*/
 266	CFI_OFFSET eip, -3*4
 267.endm
 268
 269.macro RING0_EC_FRAME
 270	CFI_STARTPROC simple
 271	CFI_SIGNAL_FRAME
 272	CFI_DEF_CFA esp, 4*4
 273	/*CFI_OFFSET cs, -2*4;*/
 274	CFI_OFFSET eip, -3*4
 275.endm
 276
 277.macro RING0_PTREGS_FRAME
 278	CFI_STARTPROC simple
 279	CFI_SIGNAL_FRAME
 280	CFI_DEF_CFA esp, PT_OLDESP-PT_EBX
 281	/*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/
 282	CFI_OFFSET eip, PT_EIP-PT_OLDESP
 283	/*CFI_OFFSET es, PT_ES-PT_OLDESP;*/
 284	/*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/
 285	CFI_OFFSET eax, PT_EAX-PT_OLDESP
 286	CFI_OFFSET ebp, PT_EBP-PT_OLDESP
 287	CFI_OFFSET edi, PT_EDI-PT_OLDESP
 288	CFI_OFFSET esi, PT_ESI-PT_OLDESP
 289	CFI_OFFSET edx, PT_EDX-PT_OLDESP
 290	CFI_OFFSET ecx, PT_ECX-PT_OLDESP
 291	CFI_OFFSET ebx, PT_EBX-PT_OLDESP
 292.endm
 293
 294ENTRY(ret_from_fork)
 295	CFI_STARTPROC
 296	pushl_cfi %eax
 297	call schedule_tail
 298	GET_THREAD_INFO(%ebp)
 299	popl_cfi %eax
 300	pushl_cfi $0x0202		# Reset kernel eflags
 301	popfl_cfi
 302	jmp syscall_exit
 303	CFI_ENDPROC
 304END(ret_from_fork)
 305
 306/*
 307 * Interrupt exit functions should be protected against kprobes
 308 */
 309	.pushsection .kprobes.text, "ax"
 310/*
 311 * Return to user mode is not as complex as all this looks,
 312 * but we want the default path for a system call return to
 313 * go as quickly as possible which is why some of this is
 314 * less clear than it otherwise should be.
 315 */
 316
 317	# userspace resumption stub bypassing syscall exit tracing
 318	ALIGN
 319	RING0_PTREGS_FRAME
 320ret_from_exception:
 321	preempt_stop(CLBR_ANY)
 322ret_from_intr:
 323	GET_THREAD_INFO(%ebp)
 324resume_userspace_sig:
 325#ifdef CONFIG_VM86
 326	movl PT_EFLAGS(%esp), %eax	# mix EFLAGS and CS
 327	movb PT_CS(%esp), %al
 328	andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax
 329#else
 330	/*
 331	 * We can be coming here from a syscall done in the kernel space,
 332	 * e.g. a failed kernel_execve().
 333	 */
 334	movl PT_CS(%esp), %eax
 335	andl $SEGMENT_RPL_MASK, %eax
 336#endif
 337	cmpl $USER_RPL, %eax
 338	jb resume_kernel		# not returning to v8086 or userspace
 339
 340ENTRY(resume_userspace)
 341	LOCKDEP_SYS_EXIT
 342 	DISABLE_INTERRUPTS(CLBR_ANY)	# make sure we don't miss an interrupt
 343					# setting need_resched or sigpending
 344					# between sampling and the iret
 345	TRACE_IRQS_OFF
 346	movl TI_flags(%ebp), %ecx
 347	andl $_TIF_WORK_MASK, %ecx	# is there any work to be done on
 348					# int/exception return?
 349	jne work_pending
 350	jmp restore_all
 351END(ret_from_exception)
 352
 353#ifdef CONFIG_PREEMPT
 354ENTRY(resume_kernel)
 355	DISABLE_INTERRUPTS(CLBR_ANY)
 356	cmpl $0,TI_preempt_count(%ebp)	# non-zero preempt_count ?
 357	jnz restore_all
 358need_resched:
 359	movl TI_flags(%ebp), %ecx	# need_resched set ?
 360	testb $_TIF_NEED_RESCHED, %cl
 361	jz restore_all
 362	testl $X86_EFLAGS_IF,PT_EFLAGS(%esp)	# interrupts off (exception path) ?
 363	jz restore_all
 364	call preempt_schedule_irq
 365	jmp need_resched
 366END(resume_kernel)
 367#endif
 368	CFI_ENDPROC
 369/*
 370 * End of kprobes section
 371 */
 372	.popsection
 373
 374/* SYSENTER_RETURN points to after the "sysenter" instruction in
 375   the vsyscall page.  See vsyscall-sysentry.S, which defines the symbol.  */
 376
 377	# sysenter call handler stub
 378ENTRY(ia32_sysenter_target)
 379	CFI_STARTPROC simple
 380	CFI_SIGNAL_FRAME
 381	CFI_DEF_CFA esp, 0
 382	CFI_REGISTER esp, ebp
 383	movl TSS_sysenter_sp0(%esp),%esp
 384sysenter_past_esp:
 385	/*
 386	 * Interrupts are disabled here, but we can't trace it until
 387	 * enough kernel state to call TRACE_IRQS_OFF can be called - but
 388	 * we immediately enable interrupts at that point anyway.
 389	 */
 390	pushl_cfi $__USER_DS
 391	/*CFI_REL_OFFSET ss, 0*/
 392	pushl_cfi %ebp
 393	CFI_REL_OFFSET esp, 0
 394	pushfl_cfi
 395	orl $X86_EFLAGS_IF, (%esp)
 396	pushl_cfi $__USER_CS
 397	/*CFI_REL_OFFSET cs, 0*/
 398	/*
 399	 * Push current_thread_info()->sysenter_return to the stack.
 400	 * A tiny bit of offset fixup is necessary - 4*4 means the 4 words
 401	 * pushed above; +8 corresponds to copy_thread's esp0 setting.
 402	 */
 403	pushl_cfi ((TI_sysenter_return)-THREAD_SIZE+8+4*4)(%esp)
 404	CFI_REL_OFFSET eip, 0
 405
 406	pushl_cfi %eax
 407	SAVE_ALL
 408	ENABLE_INTERRUPTS(CLBR_NONE)
 409
 410/*
 411 * Load the potential sixth argument from user stack.
 412 * Careful about security.
 413 */
 414	cmpl $__PAGE_OFFSET-3,%ebp
 415	jae syscall_fault
 4161:	movl (%ebp),%ebp
 417	movl %ebp,PT_EBP(%esp)
 418.section __ex_table,"a"
 419	.align 4
 420	.long 1b,syscall_fault
 421.previous
 422
 423	GET_THREAD_INFO(%ebp)
 424
 425	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp)
 426	jnz sysenter_audit
 427sysenter_do_call:
 428	cmpl $(NR_syscalls), %eax
 429	jae syscall_badsys
 430	call *sys_call_table(,%eax,4)
 431	movl %eax,PT_EAX(%esp)
 432	LOCKDEP_SYS_EXIT
 433	DISABLE_INTERRUPTS(CLBR_ANY)
 434	TRACE_IRQS_OFF
 435	movl TI_flags(%ebp), %ecx
 436	testl $_TIF_ALLWORK_MASK, %ecx
 437	jne sysexit_audit
 438sysenter_exit:
 439/* if something modifies registers it must also disable sysexit */
 440	movl PT_EIP(%esp), %edx
 441	movl PT_OLDESP(%esp), %ecx
 442	xorl %ebp,%ebp
 443	TRACE_IRQS_ON
 4441:	mov  PT_FS(%esp), %fs
 445	PTGS_TO_GS
 446	ENABLE_INTERRUPTS_SYSEXIT
 447
 448#ifdef CONFIG_AUDITSYSCALL
 449sysenter_audit:
 450	testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
 451	jnz syscall_trace_entry
 452	addl $4,%esp
 453	CFI_ADJUST_CFA_OFFSET -4
 454	/* %esi already in 8(%esp)	   6th arg: 4th syscall arg */
 455	/* %edx already in 4(%esp)	   5th arg: 3rd syscall arg */
 456	/* %ecx already in 0(%esp)	   4th arg: 2nd syscall arg */
 457	movl %ebx,%ecx			/* 3rd arg: 1st syscall arg */
 458	movl %eax,%edx			/* 2nd arg: syscall number */
 459	movl $AUDIT_ARCH_I386,%eax	/* 1st arg: audit arch */
 460	call __audit_syscall_entry
 461	pushl_cfi %ebx
 462	movl PT_EAX(%esp),%eax		/* reload syscall number */
 463	jmp sysenter_do_call
 464
 465sysexit_audit:
 466	testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx
 467	jne syscall_exit_work
 468	TRACE_IRQS_ON
 469	ENABLE_INTERRUPTS(CLBR_ANY)
 470	movl %eax,%edx		/* second arg, syscall return value */
 471	cmpl $-MAX_ERRNO,%eax	/* is it an error ? */
 472	setbe %al		/* 1 if so, 0 if not */
 473	movzbl %al,%eax		/* zero-extend that */
 474	call __audit_syscall_exit
 475	DISABLE_INTERRUPTS(CLBR_ANY)
 476	TRACE_IRQS_OFF
 477	movl TI_flags(%ebp), %ecx
 478	testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx
 479	jne syscall_exit_work
 480	movl PT_EAX(%esp),%eax	/* reload syscall return value */
 481	jmp sysenter_exit
 482#endif
 483
 484	CFI_ENDPROC
 485.pushsection .fixup,"ax"
 4862:	movl $0,PT_FS(%esp)
 487	jmp 1b
 488.section __ex_table,"a"
 489	.align 4
 490	.long 1b,2b
 491.popsection
 492	PTGS_TO_GS_EX
 493ENDPROC(ia32_sysenter_target)
 494
 495/*
 496 * syscall stub including irq exit should be protected against kprobes
 497 */
 498	.pushsection .kprobes.text, "ax"
 499	# system call handler stub
 500ENTRY(system_call)
 501	RING0_INT_FRAME			# can't unwind into user space anyway
 502	pushl_cfi %eax			# save orig_eax
 503	SAVE_ALL
 504	GET_THREAD_INFO(%ebp)
 505					# system call tracing in operation / emulation
 506	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp)
 507	jnz syscall_trace_entry
 508	cmpl $(NR_syscalls), %eax
 509	jae syscall_badsys
 510syscall_call:
 511	call *sys_call_table(,%eax,4)
 512	movl %eax,PT_EAX(%esp)		# store the return value
 513syscall_exit:
 514	LOCKDEP_SYS_EXIT
 515	DISABLE_INTERRUPTS(CLBR_ANY)	# make sure we don't miss an interrupt
 516					# setting need_resched or sigpending
 517					# between sampling and the iret
 518	TRACE_IRQS_OFF
 519	movl TI_flags(%ebp), %ecx
 520	testl $_TIF_ALLWORK_MASK, %ecx	# current->work
 521	jne syscall_exit_work
 522
 523restore_all:
 524	TRACE_IRQS_IRET
 525restore_all_notrace:
 526	movl PT_EFLAGS(%esp), %eax	# mix EFLAGS, SS and CS
 527	# Warning: PT_OLDSS(%esp) contains the wrong/random values if we
 528	# are returning to the kernel.
 529	# See comments in process.c:copy_thread() for details.
 530	movb PT_OLDSS(%esp), %ah
 531	movb PT_CS(%esp), %al
 532	andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax
 533	cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax
 534	CFI_REMEMBER_STATE
 535	je ldt_ss			# returning to user-space with LDT SS
 536restore_nocheck:
 537	RESTORE_REGS 4			# skip orig_eax/error_code
 538irq_return:
 539	INTERRUPT_RETURN
 540.section .fixup,"ax"
 541ENTRY(iret_exc)
 542	pushl $0			# no error code
 543	pushl $do_iret_error
 544	jmp error_code
 545.previous
 546.section __ex_table,"a"
 547	.align 4
 548	.long irq_return,iret_exc
 549.previous
 550
 551	CFI_RESTORE_STATE
 552ldt_ss:
 553	larl PT_OLDSS(%esp), %eax
 554	jnz restore_nocheck
 555	testl $0x00400000, %eax		# returning to 32bit stack?
 556	jnz restore_nocheck		# allright, normal return
 557
 558#ifdef CONFIG_PARAVIRT
 559	/*
 560	 * The kernel can't run on a non-flat stack if paravirt mode
 561	 * is active.  Rather than try to fixup the high bits of
 562	 * ESP, bypass this code entirely.  This may break DOSemu
 563	 * and/or Wine support in a paravirt VM, although the option
 564	 * is still available to implement the setting of the high
 565	 * 16-bits in the INTERRUPT_RETURN paravirt-op.
 566	 */
 567	cmpl $0, pv_info+PARAVIRT_enabled
 568	jne restore_nocheck
 569#endif
 570
 571/*
 572 * Setup and switch to ESPFIX stack
 573 *
 574 * We're returning to userspace with a 16 bit stack. The CPU will not
 575 * restore the high word of ESP for us on executing iret... This is an
 576 * "official" bug of all the x86-compatible CPUs, which we can work
 577 * around to make dosemu and wine happy. We do this by preloading the
 578 * high word of ESP with the high word of the userspace ESP while
 579 * compensating for the offset by changing to the ESPFIX segment with
 580 * a base address that matches for the difference.
 581 */
 582#define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + (GDT_ENTRY_ESPFIX_SS * 8)
 583	mov %esp, %edx			/* load kernel esp */
 584	mov PT_OLDESP(%esp), %eax	/* load userspace esp */
 585	mov %dx, %ax			/* eax: new kernel esp */
 586	sub %eax, %edx			/* offset (low word is 0) */
 587	shr $16, %edx
 588	mov %dl, GDT_ESPFIX_SS + 4 /* bits 16..23 */
 589	mov %dh, GDT_ESPFIX_SS + 7 /* bits 24..31 */
 590	pushl_cfi $__ESPFIX_SS
 591	pushl_cfi %eax			/* new kernel esp */
 592	/* Disable interrupts, but do not irqtrace this section: we
 593	 * will soon execute iret and the tracer was already set to
 594	 * the irqstate after the iret */
 595	DISABLE_INTERRUPTS(CLBR_EAX)
 596	lss (%esp), %esp		/* switch to espfix segment */
 597	CFI_ADJUST_CFA_OFFSET -8
 598	jmp restore_nocheck
 599	CFI_ENDPROC
 600ENDPROC(system_call)
 601
 602	# perform work that needs to be done immediately before resumption
 603	ALIGN
 604	RING0_PTREGS_FRAME		# can't unwind into user space anyway
 605work_pending:
 606	testb $_TIF_NEED_RESCHED, %cl
 607	jz work_notifysig
 608work_resched:
 609	call schedule
 610	LOCKDEP_SYS_EXIT
 611	DISABLE_INTERRUPTS(CLBR_ANY)	# make sure we don't miss an interrupt
 612					# setting need_resched or sigpending
 613					# between sampling and the iret
 614	TRACE_IRQS_OFF
 615	movl TI_flags(%ebp), %ecx
 616	andl $_TIF_WORK_MASK, %ecx	# is there any work to be done other
 617					# than syscall tracing?
 618	jz restore_all
 619	testb $_TIF_NEED_RESCHED, %cl
 620	jnz work_resched
 621
 622work_notifysig:				# deal with pending signals and
 623					# notify-resume requests
 624#ifdef CONFIG_VM86
 625	testl $X86_EFLAGS_VM, PT_EFLAGS(%esp)
 626	movl %esp, %eax
 627	jne work_notifysig_v86		# returning to kernel-space or
 628					# vm86-space
 629	TRACE_IRQS_ON
 630	ENABLE_INTERRUPTS(CLBR_NONE)
 631	xorl %edx, %edx
 632	call do_notify_resume
 633	jmp resume_userspace_sig
 634
 635	ALIGN
 636work_notifysig_v86:
 637	pushl_cfi %ecx			# save ti_flags for do_notify_resume
 638	call save_v86_state		# %eax contains pt_regs pointer
 639	popl_cfi %ecx
 640	movl %eax, %esp
 641#else
 642	movl %esp, %eax
 643#endif
 644	TRACE_IRQS_ON
 645	ENABLE_INTERRUPTS(CLBR_NONE)
 646	xorl %edx, %edx
 647	call do_notify_resume
 648	jmp resume_userspace_sig
 649END(work_pending)
 650
 651	# perform syscall exit tracing
 652	ALIGN
 653syscall_trace_entry:
 654	movl $-ENOSYS,PT_EAX(%esp)
 655	movl %esp, %eax
 656	call syscall_trace_enter
 657	/* What it returned is what we'll actually use.  */
 658	cmpl $(NR_syscalls), %eax
 659	jnae syscall_call
 660	jmp syscall_exit
 661END(syscall_trace_entry)
 662
 663	# perform syscall exit tracing
 664	ALIGN
 665syscall_exit_work:
 666	testl $_TIF_WORK_SYSCALL_EXIT, %ecx
 667	jz work_pending
 668	TRACE_IRQS_ON
 669	ENABLE_INTERRUPTS(CLBR_ANY)	# could let syscall_trace_leave() call
 670					# schedule() instead
 671	movl %esp, %eax
 672	call syscall_trace_leave
 673	jmp resume_userspace
 674END(syscall_exit_work)
 675	CFI_ENDPROC
 676
 677	RING0_INT_FRAME			# can't unwind into user space anyway
 678syscall_fault:
 679	GET_THREAD_INFO(%ebp)
 680	movl $-EFAULT,PT_EAX(%esp)
 681	jmp resume_userspace
 682END(syscall_fault)
 683
 684syscall_badsys:
 685	movl $-ENOSYS,PT_EAX(%esp)
 686	jmp resume_userspace
 687END(syscall_badsys)
 688	CFI_ENDPROC
 689/*
 690 * End of kprobes section
 691 */
 692	.popsection
 693
 694/*
 695 * System calls that need a pt_regs pointer.
 696 */
 697#define PTREGSCALL0(name) \
 698ENTRY(ptregs_##name) ;  \
 699	leal 4(%esp),%eax; \
 700	jmp sys_##name; \
 701ENDPROC(ptregs_##name)
 702
 703#define PTREGSCALL1(name) \
 704ENTRY(ptregs_##name) ; \
 705	leal 4(%esp),%edx; \
 706	movl (PT_EBX+4)(%esp),%eax; \
 707	jmp sys_##name; \
 708ENDPROC(ptregs_##name)
 709
 710#define PTREGSCALL2(name) \
 711ENTRY(ptregs_##name) ; \
 712	leal 4(%esp),%ecx; \
 713	movl (PT_ECX+4)(%esp),%edx; \
 714	movl (PT_EBX+4)(%esp),%eax; \
 715	jmp sys_##name; \
 716ENDPROC(ptregs_##name)
 717
 718#define PTREGSCALL3(name) \
 719ENTRY(ptregs_##name) ; \
 720	CFI_STARTPROC; \
 721	leal 4(%esp),%eax; \
 722	pushl_cfi %eax; \
 723	movl PT_EDX(%eax),%ecx; \
 724	movl PT_ECX(%eax),%edx; \
 725	movl PT_EBX(%eax),%eax; \
 726	call sys_##name; \
 727	addl $4,%esp; \
 728	CFI_ADJUST_CFA_OFFSET -4; \
 729	ret; \
 730	CFI_ENDPROC; \
 731ENDPROC(ptregs_##name)
 732
 733PTREGSCALL1(iopl)
 734PTREGSCALL0(fork)
 735PTREGSCALL0(vfork)
 736PTREGSCALL3(execve)
 737PTREGSCALL2(sigaltstack)
 738PTREGSCALL0(sigreturn)
 739PTREGSCALL0(rt_sigreturn)
 740PTREGSCALL2(vm86)
 741PTREGSCALL1(vm86old)
 742
 743/* Clone is an oddball.  The 4th arg is in %edi */
 744ENTRY(ptregs_clone)
 745	CFI_STARTPROC
 746	leal 4(%esp),%eax
 747	pushl_cfi %eax
 748	pushl_cfi PT_EDI(%eax)
 749	movl PT_EDX(%eax),%ecx
 750	movl PT_ECX(%eax),%edx
 751	movl PT_EBX(%eax),%eax
 752	call sys_clone
 753	addl $8,%esp
 754	CFI_ADJUST_CFA_OFFSET -8
 755	ret
 756	CFI_ENDPROC
 757ENDPROC(ptregs_clone)
 758
 759.macro FIXUP_ESPFIX_STACK
 760/*
 761 * Switch back for ESPFIX stack to the normal zerobased stack
 762 *
 763 * We can't call C functions using the ESPFIX stack. This code reads
 764 * the high word of the segment base from the GDT and swiches to the
 765 * normal stack and adjusts ESP with the matching offset.
 766 */
 767	/* fixup the stack */
 768	mov GDT_ESPFIX_SS + 4, %al /* bits 16..23 */
 769	mov GDT_ESPFIX_SS + 7, %ah /* bits 24..31 */
 770	shl $16, %eax
 771	addl %esp, %eax			/* the adjusted stack pointer */
 772	pushl_cfi $__KERNEL_DS
 773	pushl_cfi %eax
 774	lss (%esp), %esp		/* switch to the normal stack segment */
 775	CFI_ADJUST_CFA_OFFSET -8
 776.endm
 777.macro UNWIND_ESPFIX_STACK
 778	movl %ss, %eax
 779	/* see if on espfix stack */
 780	cmpw $__ESPFIX_SS, %ax
 781	jne 27f
 782	movl $__KERNEL_DS, %eax
 783	movl %eax, %ds
 784	movl %eax, %es
 785	/* switch to normal stack */
 786	FIXUP_ESPFIX_STACK
 78727:
 788.endm
 789
 790/*
 791 * Build the entry stubs and pointer table with some assembler magic.
 792 * We pack 7 stubs into a single 32-byte chunk, which will fit in a
 793 * single cache line on all modern x86 implementations.
 794 */
 795.section .init.rodata,"a"
 796ENTRY(interrupt)
 797.section .entry.text, "ax"
 798	.p2align 5
 799	.p2align CONFIG_X86_L1_CACHE_SHIFT
 800ENTRY(irq_entries_start)
 801	RING0_INT_FRAME
 802vector=FIRST_EXTERNAL_VECTOR
 803.rept (NR_VECTORS-FIRST_EXTERNAL_VECTOR+6)/7
 804	.balign 32
 805  .rept	7
 806    .if vector < NR_VECTORS
 807      .if vector <> FIRST_EXTERNAL_VECTOR
 808	CFI_ADJUST_CFA_OFFSET -4
 809      .endif
 8101:	pushl_cfi $(~vector+0x80)	/* Note: always in signed byte range */
 811      .if ((vector-FIRST_EXTERNAL_VECTOR)%7) <> 6
 812	jmp 2f
 813      .endif
 814      .previous
 815	.long 1b
 816      .section .entry.text, "ax"
 817vector=vector+1
 818    .endif
 819  .endr
 8202:	jmp common_interrupt
 821.endr
 822END(irq_entries_start)
 823
 824.previous
 825END(interrupt)
 826.previous
 827
 828/*
 829 * the CPU automatically disables interrupts when executing an IRQ vector,
 830 * so IRQ-flags tracing has to follow that:
 831 */
 832	.p2align CONFIG_X86_L1_CACHE_SHIFT
 833common_interrupt:
 834	addl $-0x80,(%esp)	/* Adjust vector into the [-256,-1] range */
 835	SAVE_ALL
 836	TRACE_IRQS_OFF
 837	movl %esp,%eax
 838	call do_IRQ
 839	jmp ret_from_intr
 840ENDPROC(common_interrupt)
 841	CFI_ENDPROC
 842
 843/*
 844 *  Irq entries should be protected against kprobes
 845 */
 846	.pushsection .kprobes.text, "ax"
 847#define BUILD_INTERRUPT3(name, nr, fn)	\
 848ENTRY(name)				\
 849	RING0_INT_FRAME;		\
 850	pushl_cfi $~(nr);		\
 851	SAVE_ALL;			\
 852	TRACE_IRQS_OFF			\
 853	movl %esp,%eax;			\
 854	call fn;			\
 855	jmp ret_from_intr;		\
 856	CFI_ENDPROC;			\
 857ENDPROC(name)
 858
 859#define BUILD_INTERRUPT(name, nr)	BUILD_INTERRUPT3(name, nr, smp_##name)
 860
 861/* The include is where all of the SMP etc. interrupts come from */
 862#include <asm/entry_arch.h>
 863
 864ENTRY(coprocessor_error)
 865	RING0_INT_FRAME
 866	pushl_cfi $0
 867	pushl_cfi $do_coprocessor_error
 868	jmp error_code
 869	CFI_ENDPROC
 870END(coprocessor_error)
 871
 872ENTRY(simd_coprocessor_error)
 873	RING0_INT_FRAME
 874	pushl_cfi $0
 875#ifdef CONFIG_X86_INVD_BUG
 876	/* AMD 486 bug: invd from userspace calls exception 19 instead of #GP */
 877661:	pushl_cfi $do_general_protection
 878662:
 879.section .altinstructions,"a"
 880	altinstruction_entry 661b, 663f, X86_FEATURE_XMM, 662b-661b, 664f-663f
 881.previous
 882.section .altinstr_replacement,"ax"
 883663:	pushl $do_simd_coprocessor_error
 884664:
 885.previous
 886#else
 887	pushl_cfi $do_simd_coprocessor_error
 888#endif
 889	jmp error_code
 890	CFI_ENDPROC
 891END(simd_coprocessor_error)
 892
 893ENTRY(device_not_available)
 894	RING0_INT_FRAME
 895	pushl_cfi $-1			# mark this as an int
 896	pushl_cfi $do_device_not_available
 897	jmp error_code
 898	CFI_ENDPROC
 899END(device_not_available)
 900
 901#ifdef CONFIG_PARAVIRT
 902ENTRY(native_iret)
 903	iret
 904.section __ex_table,"a"
 905	.align 4
 906	.long native_iret, iret_exc
 907.previous
 908END(native_iret)
 909
 910ENTRY(native_irq_enable_sysexit)
 911	sti
 912	sysexit
 913END(native_irq_enable_sysexit)
 914#endif
 915
 916ENTRY(overflow)
 917	RING0_INT_FRAME
 918	pushl_cfi $0
 919	pushl_cfi $do_overflow
 920	jmp error_code
 921	CFI_ENDPROC
 922END(overflow)
 923
 924ENTRY(bounds)
 925	RING0_INT_FRAME
 926	pushl_cfi $0
 927	pushl_cfi $do_bounds
 928	jmp error_code
 929	CFI_ENDPROC
 930END(bounds)
 931
 932ENTRY(invalid_op)
 933	RING0_INT_FRAME
 934	pushl_cfi $0
 935	pushl_cfi $do_invalid_op
 936	jmp error_code
 937	CFI_ENDPROC
 938END(invalid_op)
 939
 940ENTRY(coprocessor_segment_overrun)
 941	RING0_INT_FRAME
 942	pushl_cfi $0
 943	pushl_cfi $do_coprocessor_segment_overrun
 944	jmp error_code
 945	CFI_ENDPROC
 946END(coprocessor_segment_overrun)
 947
 948ENTRY(invalid_TSS)
 949	RING0_EC_FRAME
 950	pushl_cfi $do_invalid_TSS
 951	jmp error_code
 952	CFI_ENDPROC
 953END(invalid_TSS)
 954
 955ENTRY(segment_not_present)
 956	RING0_EC_FRAME
 957	pushl_cfi $do_segment_not_present
 958	jmp error_code
 959	CFI_ENDPROC
 960END(segment_not_present)
 961
 962ENTRY(stack_segment)
 963	RING0_EC_FRAME
 964	pushl_cfi $do_stack_segment
 965	jmp error_code
 966	CFI_ENDPROC
 967END(stack_segment)
 968
 969ENTRY(alignment_check)
 970	RING0_EC_FRAME
 971	pushl_cfi $do_alignment_check
 972	jmp error_code
 973	CFI_ENDPROC
 974END(alignment_check)
 975
 976ENTRY(divide_error)
 977	RING0_INT_FRAME
 978	pushl_cfi $0			# no error code
 979	pushl_cfi $do_divide_error
 980	jmp error_code
 981	CFI_ENDPROC
 982END(divide_error)
 983
 984#ifdef CONFIG_X86_MCE
 985ENTRY(machine_check)
 986	RING0_INT_FRAME
 987	pushl_cfi $0
 988	pushl_cfi machine_check_vector
 989	jmp error_code
 990	CFI_ENDPROC
 991END(machine_check)
 992#endif
 993
 994ENTRY(spurious_interrupt_bug)
 995	RING0_INT_FRAME
 996	pushl_cfi $0
 997	pushl_cfi $do_spurious_interrupt_bug
 998	jmp error_code
 999	CFI_ENDPROC
1000END(spurious_interrupt_bug)
1001/*
1002 * End of kprobes section
1003 */
1004	.popsection
1005
1006ENTRY(kernel_thread_helper)
1007	pushl $0		# fake return address for unwinder
1008	CFI_STARTPROC
1009	movl %edi,%eax
1010	call *%esi
1011	call do_exit
1012	ud2			# padding for call trace
1013	CFI_ENDPROC
1014ENDPROC(kernel_thread_helper)
1015
1016#ifdef CONFIG_XEN
1017/* Xen doesn't set %esp to be precisely what the normal sysenter
1018   entrypoint expects, so fix it up before using the normal path. */
1019ENTRY(xen_sysenter_target)
1020	RING0_INT_FRAME
1021	addl $5*4, %esp		/* remove xen-provided frame */
1022	CFI_ADJUST_CFA_OFFSET -5*4
1023	jmp sysenter_past_esp
1024	CFI_ENDPROC
1025
1026ENTRY(xen_hypervisor_callback)
1027	CFI_STARTPROC
1028	pushl_cfi $0
1029	SAVE_ALL
1030	TRACE_IRQS_OFF
1031
1032	/* Check to see if we got the event in the critical
1033	   region in xen_iret_direct, after we've reenabled
1034	   events and checked for pending events.  This simulates
1035	   iret instruction's behaviour where it delivers a
1036	   pending interrupt when enabling interrupts. */
1037	movl PT_EIP(%esp),%eax
1038	cmpl $xen_iret_start_crit,%eax
1039	jb   1f
1040	cmpl $xen_iret_end_crit,%eax
1041	jae  1f
1042
1043	jmp  xen_iret_crit_fixup
1044
1045ENTRY(xen_do_upcall)
10461:	mov %esp, %eax
1047	call xen_evtchn_do_upcall
1048	jmp  ret_from_intr
1049	CFI_ENDPROC
1050ENDPROC(xen_hypervisor_callback)
1051
1052# Hypervisor uses this for application faults while it executes.
1053# We get here for two reasons:
1054#  1. Fault while reloading DS, ES, FS or GS
1055#  2. Fault while executing IRET
1056# Category 1 we fix up by reattempting the load, and zeroing the segment
1057# register if the load fails.
1058# Category 2 we fix up by jumping to do_iret_error. We cannot use the
1059# normal Linux return path in this case because if we use the IRET hypercall
1060# to pop the stack frame we end up in an infinite loop of failsafe callbacks.
1061# We distinguish between categories by maintaining a status value in EAX.
1062ENTRY(xen_failsafe_callback)
1063	CFI_STARTPROC
1064	pushl_cfi %eax
1065	movl $1,%eax
10661:	mov 4(%esp),%ds
10672:	mov 8(%esp),%es
10683:	mov 12(%esp),%fs
10694:	mov 16(%esp),%gs
1070	testl %eax,%eax
1071	popl_cfi %eax
1072	lea 16(%esp),%esp
1073	CFI_ADJUST_CFA_OFFSET -16
1074	jz 5f
1075	addl $16,%esp
1076	jmp iret_exc		# EAX != 0 => Category 2 (Bad IRET)
10775:	pushl_cfi $0		# EAX == 0 => Category 1 (Bad segment)
1078	SAVE_ALL
1079	jmp ret_from_exception
1080	CFI_ENDPROC
1081
1082.section .fixup,"ax"
10836:	xorl %eax,%eax
1084	movl %eax,4(%esp)
1085	jmp 1b
10867:	xorl %eax,%eax
1087	movl %eax,8(%esp)
1088	jmp 2b
10898:	xorl %eax,%eax
1090	movl %eax,12(%esp)
1091	jmp 3b
10929:	xorl %eax,%eax
1093	movl %eax,16(%esp)
1094	jmp 4b
1095.previous
1096.section __ex_table,"a"
1097	.align 4
1098	.long 1b,6b
1099	.long 2b,7b
1100	.long 3b,8b
1101	.long 4b,9b
1102.previous
1103ENDPROC(xen_failsafe_callback)
1104
1105BUILD_INTERRUPT3(xen_hvm_callback_vector, XEN_HVM_EVTCHN_CALLBACK,
1106		xen_evtchn_do_upcall)
1107
1108#endif	/* CONFIG_XEN */
1109
1110#ifdef CONFIG_FUNCTION_TRACER
1111#ifdef CONFIG_DYNAMIC_FTRACE
1112
1113ENTRY(mcount)
1114	ret
1115END(mcount)
1116
1117ENTRY(ftrace_caller)
1118	cmpl $0, function_trace_stop
1119	jne  ftrace_stub
1120
1121	pushl %eax
1122	pushl %ecx
1123	pushl %edx
1124	movl 0xc(%esp), %eax
1125	movl 0x4(%ebp), %edx
1126	subl $MCOUNT_INSN_SIZE, %eax
1127
1128.globl ftrace_call
1129ftrace_call:
1130	call ftrace_stub
1131
1132	popl %edx
1133	popl %ecx
1134	popl %eax
1135#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1136.globl ftrace_graph_call
1137ftrace_graph_call:
1138	jmp ftrace_stub
1139#endif
1140
1141.globl ftrace_stub
1142ftrace_stub:
1143	ret
1144END(ftrace_caller)
1145
1146#else /* ! CONFIG_DYNAMIC_FTRACE */
1147
1148ENTRY(mcount)
1149	cmpl $0, function_trace_stop
1150	jne  ftrace_stub
1151
1152	cmpl $ftrace_stub, ftrace_trace_function
1153	jnz trace
1154#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1155	cmpl $ftrace_stub, ftrace_graph_return
1156	jnz ftrace_graph_caller
1157
1158	cmpl $ftrace_graph_entry_stub, ftrace_graph_entry
1159	jnz ftrace_graph_caller
1160#endif
1161.globl ftrace_stub
1162ftrace_stub:
1163	ret
1164
1165	/* taken from glibc */
1166trace:
1167	pushl %eax
1168	pushl %ecx
1169	pushl %edx
1170	movl 0xc(%esp), %eax
1171	movl 0x4(%ebp), %edx
1172	subl $MCOUNT_INSN_SIZE, %eax
1173
1174	call *ftrace_trace_function
1175
1176	popl %edx
1177	popl %ecx
1178	popl %eax
1179	jmp ftrace_stub
1180END(mcount)
1181#endif /* CONFIG_DYNAMIC_FTRACE */
1182#endif /* CONFIG_FUNCTION_TRACER */
1183
1184#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1185ENTRY(ftrace_graph_caller)
1186	cmpl $0, function_trace_stop
1187	jne ftrace_stub
1188
1189	pushl %eax
1190	pushl %ecx
1191	pushl %edx
1192	movl 0xc(%esp), %edx
1193	lea 0x4(%ebp), %eax
1194	movl (%ebp), %ecx
1195	subl $MCOUNT_INSN_SIZE, %edx
1196	call prepare_ftrace_return
1197	popl %edx
1198	popl %ecx
1199	popl %eax
1200	ret
1201END(ftrace_graph_caller)
1202
1203.globl return_to_handler
1204return_to_handler:
1205	pushl %eax
1206	pushl %edx
1207	movl %ebp, %eax
1208	call ftrace_return_to_handler
1209	movl %eax, %ecx
1210	popl %edx
1211	popl %eax
1212	jmp *%ecx
1213#endif
1214
1215/*
1216 * Some functions should be protected against kprobes
1217 */
1218	.pushsection .kprobes.text, "ax"
1219
1220ENTRY(page_fault)
1221	RING0_EC_FRAME
1222	pushl_cfi $do_page_fault
1223	ALIGN
1224error_code:
1225	/* the function address is in %gs's slot on the stack */
1226	pushl_cfi %fs
1227	/*CFI_REL_OFFSET fs, 0*/
1228	pushl_cfi %es
1229	/*CFI_REL_OFFSET es, 0*/
1230	pushl_cfi %ds
1231	/*CFI_REL_OFFSET ds, 0*/
1232	pushl_cfi %eax
1233	CFI_REL_OFFSET eax, 0
1234	pushl_cfi %ebp
1235	CFI_REL_OFFSET ebp, 0
1236	pushl_cfi %edi
1237	CFI_REL_OFFSET edi, 0
1238	pushl_cfi %esi
1239	CFI_REL_OFFSET esi, 0
1240	pushl_cfi %edx
1241	CFI_REL_OFFSET edx, 0
1242	pushl_cfi %ecx
1243	CFI_REL_OFFSET ecx, 0
1244	pushl_cfi %ebx
1245	CFI_REL_OFFSET ebx, 0
1246	cld
1247	movl $(__KERNEL_PERCPU), %ecx
1248	movl %ecx, %fs
1249	UNWIND_ESPFIX_STACK
1250	GS_TO_REG %ecx
1251	movl PT_GS(%esp), %edi		# get the function address
1252	movl PT_ORIG_EAX(%esp), %edx	# get the error code
1253	movl $-1, PT_ORIG_EAX(%esp)	# no syscall to restart
1254	REG_TO_PTGS %ecx
1255	SET_KERNEL_GS %ecx
1256	movl $(__USER_DS), %ecx
1257	movl %ecx, %ds
1258	movl %ecx, %es
1259	TRACE_IRQS_OFF
1260	movl %esp,%eax			# pt_regs pointer
1261	call *%edi
1262	jmp ret_from_exception
1263	CFI_ENDPROC
1264END(page_fault)
1265
1266/*
1267 * Debug traps and NMI can happen at the one SYSENTER instruction
1268 * that sets up the real kernel stack. Check here, since we can't
1269 * allow the wrong stack to be used.
1270 *
1271 * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have
1272 * already pushed 3 words if it hits on the sysenter instruction:
1273 * eflags, cs and eip.
1274 *
1275 * We just load the right stack, and push the three (known) values
1276 * by hand onto the new stack - while updating the return eip past
1277 * the instruction that would have done it for sysenter.
1278 */
1279.macro FIX_STACK offset ok label
1280	cmpw $__KERNEL_CS, 4(%esp)
1281	jne \ok
1282\label:
1283	movl TSS_sysenter_sp0 + \offset(%esp), %esp
1284	CFI_DEF_CFA esp, 0
1285	CFI_UNDEFINED eip
1286	pushfl_cfi
1287	pushl_cfi $__KERNEL_CS
1288	pushl_cfi $sysenter_past_esp
1289	CFI_REL_OFFSET eip, 0
1290.endm
1291
1292ENTRY(debug)
1293	RING0_INT_FRAME
1294	cmpl $ia32_sysenter_target,(%esp)
1295	jne debug_stack_correct
1296	FIX_STACK 12, debug_stack_correct, debug_esp_fix_insn
1297debug_stack_correct:
1298	pushl_cfi $-1			# mark this as an int
1299	SAVE_ALL
1300	TRACE_IRQS_OFF
1301	xorl %edx,%edx			# error code 0
1302	movl %esp,%eax			# pt_regs pointer
1303	call do_debug
1304	jmp ret_from_exception
1305	CFI_ENDPROC
1306END(debug)
1307
1308/*
1309 * NMI is doubly nasty. It can happen _while_ we're handling
1310 * a debug fault, and the debug fault hasn't yet been able to
1311 * clear up the stack. So we first check whether we got  an
1312 * NMI on the sysenter entry path, but after that we need to
1313 * check whether we got an NMI on the debug path where the debug
1314 * fault happened on the sysenter path.
1315 */
1316ENTRY(nmi)
1317	RING0_INT_FRAME
1318	pushl_cfi %eax
1319	movl %ss, %eax
1320	cmpw $__ESPFIX_SS, %ax
1321	popl_cfi %eax
1322	je nmi_espfix_stack
1323	cmpl $ia32_sysenter_target,(%esp)
1324	je nmi_stack_fixup
1325	pushl_cfi %eax
1326	movl %esp,%eax
1327	/* Do not access memory above the end of our stack page,
1328	 * it might not exist.
1329	 */
1330	andl $(THREAD_SIZE-1),%eax
1331	cmpl $(THREAD_SIZE-20),%eax
1332	popl_cfi %eax
1333	jae nmi_stack_correct
1334	cmpl $ia32_sysenter_target,12(%esp)
1335	je nmi_debug_stack_check
1336nmi_stack_correct:
1337	/* We have a RING0_INT_FRAME here */
1338	pushl_cfi %eax
1339	SAVE_ALL
1340	xorl %edx,%edx		# zero error code
1341	movl %esp,%eax		# pt_regs pointer
1342	call do_nmi
1343	jmp restore_all_notrace
1344	CFI_ENDPROC
1345
1346nmi_stack_fixup:
1347	RING0_INT_FRAME
1348	FIX_STACK 12, nmi_stack_correct, 1
1349	jmp nmi_stack_correct
1350
1351nmi_debug_stack_check:
1352	/* We have a RING0_INT_FRAME here */
1353	cmpw $__KERNEL_CS,16(%esp)
1354	jne nmi_stack_correct
1355	cmpl $debug,(%esp)
1356	jb nmi_stack_correct
1357	cmpl $debug_esp_fix_insn,(%esp)
1358	ja nmi_stack_correct
1359	FIX_STACK 24, nmi_stack_correct, 1
1360	jmp nmi_stack_correct
1361
1362nmi_espfix_stack:
1363	/* We have a RING0_INT_FRAME here.
1364	 *
1365	 * create the pointer to lss back
1366	 */
1367	pushl_cfi %ss
1368	pushl_cfi %esp
1369	addl $4, (%esp)
1370	/* copy the iret frame of 12 bytes */
1371	.rept 3
1372	pushl_cfi 16(%esp)
1373	.endr
1374	pushl_cfi %eax
1375	SAVE_ALL
1376	FIXUP_ESPFIX_STACK		# %eax == %esp
1377	xorl %edx,%edx			# zero error code
1378	call do_nmi
1379	RESTORE_REGS
1380	lss 12+4(%esp), %esp		# back to espfix stack
1381	CFI_ADJUST_CFA_OFFSET -24
1382	jmp irq_return
1383	CFI_ENDPROC
1384END(nmi)
1385
1386ENTRY(int3)
1387	RING0_INT_FRAME
1388	pushl_cfi $-1			# mark this as an int
1389	SAVE_ALL
1390	TRACE_IRQS_OFF
1391	xorl %edx,%edx		# zero error code
1392	movl %esp,%eax		# pt_regs pointer
1393	call do_int3
1394	jmp ret_from_exception
1395	CFI_ENDPROC
1396END(int3)
1397
1398ENTRY(general_protection)
1399	RING0_EC_FRAME
1400	pushl_cfi $do_general_protection
1401	jmp error_code
1402	CFI_ENDPROC
1403END(general_protection)
1404
1405#ifdef CONFIG_KVM_GUEST
1406ENTRY(async_page_fault)
1407	RING0_EC_FRAME
1408	pushl_cfi $do_async_page_fault
1409	jmp error_code
1410	CFI_ENDPROC
1411END(async_page_fault)
1412#endif
1413
1414/*
1415 * End of kprobes section
1416 */
1417	.popsection