/arch/x86/kernel/entry_64.S
Assembly | 1549 lines | 1511 code | 38 blank | 0 comment | 16 complexity | d1908a1393dd0b0f222c2d3405e8e7af MD5 | raw file
Possible License(s): GPL-2.0, LGPL-2.0, AGPL-1.0
1/* 2 * linux/arch/x86_64/entry.S 3 * 4 * Copyright (C) 1991, 1992 Linus Torvalds 5 * Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs 6 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz> 7 */ 8 9/* 10 * entry.S contains the system-call and fault low-level handling routines. 11 * 12 * NOTE: This code handles signal-recognition, which happens every time 13 * after an interrupt and after each system call. 14 * 15 * Normal syscalls and interrupts don't save a full stack frame, this is 16 * only done for syscall tracing, signals or fork/exec et.al. 17 * 18 * A note on terminology: 19 * - top of stack: Architecture defined interrupt frame from SS to RIP 20 * at the top of the kernel process stack. 21 * - partial stack frame: partially saved registers up to R11. 22 * - full stack frame: Like partial stack frame, but all register saved. 23 * 24 * Some macro usage: 25 * - CFI macros are used to generate dwarf2 unwind information for better 26 * backtraces. They don't change any code. 27 * - SAVE_ALL/RESTORE_ALL - Save/restore all registers 28 * - SAVE_ARGS/RESTORE_ARGS - Save/restore registers that C functions modify. 29 * There are unfortunately lots of special cases where some registers 30 * not touched. The macro is a big mess that should be cleaned up. 31 * - SAVE_REST/RESTORE_REST - Handle the registers not saved by SAVE_ARGS. 32 * Gives a full stack frame. 33 * - ENTRY/END Define functions in the symbol table. 34 * - FIXUP_TOP_OF_STACK/RESTORE_TOP_OF_STACK - Fix up the hardware stack 35 * frame that is otherwise undefined after a SYSCALL 36 * - TRACE_IRQ_* - Trace hard interrupt state for lock debugging. 37 * - errorentry/paranoidentry/zeroentry - Define exception entry points. 38 */ 39 40#include <linux/linkage.h> 41#include <asm/segment.h> 42#include <asm/cache.h> 43#include <asm/errno.h> 44#include <asm/dwarf2.h> 45#include <asm/calling.h> 46#include <asm/asm-offsets.h> 47#include <asm/msr.h> 48#include <asm/unistd.h> 49#include <asm/thread_info.h> 50#include <asm/hw_irq.h> 51#include <asm/page_types.h> 52#include <asm/irqflags.h> 53#include <asm/paravirt.h> 54#include <asm/ftrace.h> 55#include <asm/percpu.h> 56 57/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */ 58#include <linux/elf-em.h> 59#define AUDIT_ARCH_X86_64 (EM_X86_64|__AUDIT_ARCH_64BIT|__AUDIT_ARCH_LE) 60#define __AUDIT_ARCH_64BIT 0x80000000 61#define __AUDIT_ARCH_LE 0x40000000 62 63 .code64 64 .section .entry.text, "ax" 65 66#ifdef CONFIG_FUNCTION_TRACER 67#ifdef CONFIG_DYNAMIC_FTRACE 68ENTRY(mcount) 69 retq 70END(mcount) 71 72ENTRY(ftrace_caller) 73 cmpl $0, function_trace_stop 74 jne ftrace_stub 75 76 MCOUNT_SAVE_FRAME 77 78 movq 0x38(%rsp), %rdi 79 movq 8(%rbp), %rsi 80 subq $MCOUNT_INSN_SIZE, %rdi 81 82GLOBAL(ftrace_call) 83 call ftrace_stub 84 85 MCOUNT_RESTORE_FRAME 86 87#ifdef CONFIG_FUNCTION_GRAPH_TRACER 88GLOBAL(ftrace_graph_call) 89 jmp ftrace_stub 90#endif 91 92GLOBAL(ftrace_stub) 93 retq 94END(ftrace_caller) 95 96#else /* ! CONFIG_DYNAMIC_FTRACE */ 97ENTRY(mcount) 98 cmpl $0, function_trace_stop 99 jne ftrace_stub 100 101 cmpq $ftrace_stub, ftrace_trace_function 102 jnz trace 103 104#ifdef CONFIG_FUNCTION_GRAPH_TRACER 105 cmpq $ftrace_stub, ftrace_graph_return 106 jnz ftrace_graph_caller 107 108 cmpq $ftrace_graph_entry_stub, ftrace_graph_entry 109 jnz ftrace_graph_caller 110#endif 111 112GLOBAL(ftrace_stub) 113 retq 114 115trace: 116 MCOUNT_SAVE_FRAME 117 118 movq 0x38(%rsp), %rdi 119 movq 8(%rbp), %rsi 120 subq $MCOUNT_INSN_SIZE, %rdi 121 122 call *ftrace_trace_function 123 124 MCOUNT_RESTORE_FRAME 125 126 jmp ftrace_stub 127END(mcount) 128#endif /* CONFIG_DYNAMIC_FTRACE */ 129#endif /* CONFIG_FUNCTION_TRACER */ 130 131#ifdef CONFIG_FUNCTION_GRAPH_TRACER 132ENTRY(ftrace_graph_caller) 133 cmpl $0, function_trace_stop 134 jne ftrace_stub 135 136 MCOUNT_SAVE_FRAME 137 138 leaq 8(%rbp), %rdi 139 movq 0x38(%rsp), %rsi 140 movq (%rbp), %rdx 141 subq $MCOUNT_INSN_SIZE, %rsi 142 143 call prepare_ftrace_return 144 145 MCOUNT_RESTORE_FRAME 146 147 retq 148END(ftrace_graph_caller) 149 150GLOBAL(return_to_handler) 151 subq $24, %rsp 152 153 /* Save the return values */ 154 movq %rax, (%rsp) 155 movq %rdx, 8(%rsp) 156 movq %rbp, %rdi 157 158 call ftrace_return_to_handler 159 160 movq %rax, %rdi 161 movq 8(%rsp), %rdx 162 movq (%rsp), %rax 163 addq $24, %rsp 164 jmp *%rdi 165#endif 166 167 168#ifndef CONFIG_PREEMPT 169#define retint_kernel retint_restore_args 170#endif 171 172#ifdef CONFIG_PARAVIRT 173ENTRY(native_usergs_sysret64) 174 swapgs 175 sysretq 176ENDPROC(native_usergs_sysret64) 177#endif /* CONFIG_PARAVIRT */ 178 179 180.macro TRACE_IRQS_IRETQ offset=ARGOFFSET 181#ifdef CONFIG_TRACE_IRQFLAGS 182 bt $9,EFLAGS-\offset(%rsp) /* interrupts off? */ 183 jnc 1f 184 TRACE_IRQS_ON 1851: 186#endif 187.endm 188 189/* 190 * C code is not supposed to know about undefined top of stack. Every time 191 * a C function with an pt_regs argument is called from the SYSCALL based 192 * fast path FIXUP_TOP_OF_STACK is needed. 193 * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs 194 * manipulation. 195 */ 196 197 /* %rsp:at FRAMEEND */ 198 .macro FIXUP_TOP_OF_STACK tmp offset=0 199 movq PER_CPU_VAR(old_rsp),\tmp 200 movq \tmp,RSP+\offset(%rsp) 201 movq $__USER_DS,SS+\offset(%rsp) 202 movq $__USER_CS,CS+\offset(%rsp) 203 movq $-1,RCX+\offset(%rsp) 204 movq R11+\offset(%rsp),\tmp /* get eflags */ 205 movq \tmp,EFLAGS+\offset(%rsp) 206 .endm 207 208 .macro RESTORE_TOP_OF_STACK tmp offset=0 209 movq RSP+\offset(%rsp),\tmp 210 movq \tmp,PER_CPU_VAR(old_rsp) 211 movq EFLAGS+\offset(%rsp),\tmp 212 movq \tmp,R11+\offset(%rsp) 213 .endm 214 215 .macro FAKE_STACK_FRAME child_rip 216 /* push in order ss, rsp, eflags, cs, rip */ 217 xorl %eax, %eax 218 pushq_cfi $__KERNEL_DS /* ss */ 219 /*CFI_REL_OFFSET ss,0*/ 220 pushq_cfi %rax /* rsp */ 221 CFI_REL_OFFSET rsp,0 222 pushq_cfi $X86_EFLAGS_IF /* eflags - interrupts on */ 223 /*CFI_REL_OFFSET rflags,0*/ 224 pushq_cfi $__KERNEL_CS /* cs */ 225 /*CFI_REL_OFFSET cs,0*/ 226 pushq_cfi \child_rip /* rip */ 227 CFI_REL_OFFSET rip,0 228 pushq_cfi %rax /* orig rax */ 229 .endm 230 231 .macro UNFAKE_STACK_FRAME 232 addq $8*6, %rsp 233 CFI_ADJUST_CFA_OFFSET -(6*8) 234 .endm 235 236/* 237 * initial frame state for interrupts (and exceptions without error code) 238 */ 239 .macro EMPTY_FRAME start=1 offset=0 240 .if \start 241 CFI_STARTPROC simple 242 CFI_SIGNAL_FRAME 243 CFI_DEF_CFA rsp,8+\offset 244 .else 245 CFI_DEF_CFA_OFFSET 8+\offset 246 .endif 247 .endm 248 249/* 250 * initial frame state for interrupts (and exceptions without error code) 251 */ 252 .macro INTR_FRAME start=1 offset=0 253 EMPTY_FRAME \start, SS+8+\offset-RIP 254 /*CFI_REL_OFFSET ss, SS+\offset-RIP*/ 255 CFI_REL_OFFSET rsp, RSP+\offset-RIP 256 /*CFI_REL_OFFSET rflags, EFLAGS+\offset-RIP*/ 257 /*CFI_REL_OFFSET cs, CS+\offset-RIP*/ 258 CFI_REL_OFFSET rip, RIP+\offset-RIP 259 .endm 260 261/* 262 * initial frame state for exceptions with error code (and interrupts 263 * with vector already pushed) 264 */ 265 .macro XCPT_FRAME start=1 offset=0 266 INTR_FRAME \start, RIP+\offset-ORIG_RAX 267 /*CFI_REL_OFFSET orig_rax, ORIG_RAX-ORIG_RAX*/ 268 .endm 269 270/* 271 * frame that enables calling into C. 272 */ 273 .macro PARTIAL_FRAME start=1 offset=0 274 XCPT_FRAME \start, ORIG_RAX+\offset-ARGOFFSET 275 CFI_REL_OFFSET rdi, RDI+\offset-ARGOFFSET 276 CFI_REL_OFFSET rsi, RSI+\offset-ARGOFFSET 277 CFI_REL_OFFSET rdx, RDX+\offset-ARGOFFSET 278 CFI_REL_OFFSET rcx, RCX+\offset-ARGOFFSET 279 CFI_REL_OFFSET rax, RAX+\offset-ARGOFFSET 280 CFI_REL_OFFSET r8, R8+\offset-ARGOFFSET 281 CFI_REL_OFFSET r9, R9+\offset-ARGOFFSET 282 CFI_REL_OFFSET r10, R10+\offset-ARGOFFSET 283 CFI_REL_OFFSET r11, R11+\offset-ARGOFFSET 284 .endm 285 286/* 287 * frame that enables passing a complete pt_regs to a C function. 288 */ 289 .macro DEFAULT_FRAME start=1 offset=0 290 PARTIAL_FRAME \start, R11+\offset-R15 291 CFI_REL_OFFSET rbx, RBX+\offset 292 CFI_REL_OFFSET rbp, RBP+\offset 293 CFI_REL_OFFSET r12, R12+\offset 294 CFI_REL_OFFSET r13, R13+\offset 295 CFI_REL_OFFSET r14, R14+\offset 296 CFI_REL_OFFSET r15, R15+\offset 297 .endm 298 299/* save partial stack frame */ 300 .pushsection .kprobes.text, "ax" 301ENTRY(save_args) 302 XCPT_FRAME 303 cld 304 /* 305 * start from rbp in pt_regs and jump over 306 * return address. 307 */ 308 movq_cfi rdi, RDI+8-RBP 309 movq_cfi rsi, RSI+8-RBP 310 movq_cfi rdx, RDX+8-RBP 311 movq_cfi rcx, RCX+8-RBP 312 movq_cfi rax, RAX+8-RBP 313 movq_cfi r8, R8+8-RBP 314 movq_cfi r9, R9+8-RBP 315 movq_cfi r10, R10+8-RBP 316 movq_cfi r11, R11+8-RBP 317 318 leaq -RBP+8(%rsp),%rdi /* arg1 for handler */ 319 movq_cfi rbp, 8 /* push %rbp */ 320 leaq 8(%rsp), %rbp /* mov %rsp, %ebp */ 321 testl $3, CS(%rdi) 322 je 1f 323 SWAPGS 324 /* 325 * irq_count is used to check if a CPU is already on an interrupt stack 326 * or not. While this is essentially redundant with preempt_count it is 327 * a little cheaper to use a separate counter in the PDA (short of 328 * moving irq_enter into assembly, which would be too much work) 329 */ 3301: incl PER_CPU_VAR(irq_count) 331 jne 2f 332 popq_cfi %rax /* move return address... */ 333 mov PER_CPU_VAR(irq_stack_ptr),%rsp 334 EMPTY_FRAME 0 335 pushq_cfi %rbp /* backlink for unwinder */ 336 pushq_cfi %rax /* ... to the new stack */ 337 /* 338 * We entered an interrupt context - irqs are off: 339 */ 3402: TRACE_IRQS_OFF 341 ret 342 CFI_ENDPROC 343END(save_args) 344 .popsection 345 346ENTRY(save_rest) 347 PARTIAL_FRAME 1 REST_SKIP+8 348 movq 5*8+16(%rsp), %r11 /* save return address */ 349 movq_cfi rbx, RBX+16 350 movq_cfi rbp, RBP+16 351 movq_cfi r12, R12+16 352 movq_cfi r13, R13+16 353 movq_cfi r14, R14+16 354 movq_cfi r15, R15+16 355 movq %r11, 8(%rsp) /* return address */ 356 FIXUP_TOP_OF_STACK %r11, 16 357 ret 358 CFI_ENDPROC 359END(save_rest) 360 361/* save complete stack frame */ 362 .pushsection .kprobes.text, "ax" 363ENTRY(save_paranoid) 364 XCPT_FRAME 1 RDI+8 365 cld 366 movq_cfi rdi, RDI+8 367 movq_cfi rsi, RSI+8 368 movq_cfi rdx, RDX+8 369 movq_cfi rcx, RCX+8 370 movq_cfi rax, RAX+8 371 movq_cfi r8, R8+8 372 movq_cfi r9, R9+8 373 movq_cfi r10, R10+8 374 movq_cfi r11, R11+8 375 movq_cfi rbx, RBX+8 376 movq_cfi rbp, RBP+8 377 movq_cfi r12, R12+8 378 movq_cfi r13, R13+8 379 movq_cfi r14, R14+8 380 movq_cfi r15, R15+8 381 movl $1,%ebx 382 movl $MSR_GS_BASE,%ecx 383 rdmsr 384 testl %edx,%edx 385 js 1f /* negative -> in kernel */ 386 SWAPGS 387 xorl %ebx,%ebx 3881: ret 389 CFI_ENDPROC 390END(save_paranoid) 391 .popsection 392 393/* 394 * A newly forked process directly context switches into this address. 395 * 396 * rdi: prev task we switched from 397 */ 398ENTRY(ret_from_fork) 399 DEFAULT_FRAME 400 401 LOCK ; btr $TIF_FORK,TI_flags(%r8) 402 403 pushq_cfi kernel_eflags(%rip) 404 popfq_cfi # reset kernel eflags 405 406 call schedule_tail # rdi: 'prev' task parameter 407 408 GET_THREAD_INFO(%rcx) 409 410 RESTORE_REST 411 412 testl $3, CS-ARGOFFSET(%rsp) # from kernel_thread? 413 je int_ret_from_sys_call 414 415 testl $_TIF_IA32, TI_flags(%rcx) # 32-bit compat task needs IRET 416 jnz int_ret_from_sys_call 417 418 RESTORE_TOP_OF_STACK %rdi, -ARGOFFSET 419 jmp ret_from_sys_call # go to the SYSRET fastpath 420 421 CFI_ENDPROC 422END(ret_from_fork) 423 424/* 425 * System call entry. Up to 6 arguments in registers are supported. 426 * 427 * SYSCALL does not save anything on the stack and does not change the 428 * stack pointer. 429 */ 430 431/* 432 * Register setup: 433 * rax system call number 434 * rdi arg0 435 * rcx return address for syscall/sysret, C arg3 436 * rsi arg1 437 * rdx arg2 438 * r10 arg3 (--> moved to rcx for C) 439 * r8 arg4 440 * r9 arg5 441 * r11 eflags for syscall/sysret, temporary for C 442 * r12-r15,rbp,rbx saved by C code, not touched. 443 * 444 * Interrupts are off on entry. 445 * Only called from user space. 446 * 447 * XXX if we had a free scratch register we could save the RSP into the stack frame 448 * and report it properly in ps. Unfortunately we haven't. 449 * 450 * When user can change the frames always force IRET. That is because 451 * it deals with uncanonical addresses better. SYSRET has trouble 452 * with them due to bugs in both AMD and Intel CPUs. 453 */ 454 455ENTRY(system_call) 456 CFI_STARTPROC simple 457 CFI_SIGNAL_FRAME 458 CFI_DEF_CFA rsp,KERNEL_STACK_OFFSET 459 CFI_REGISTER rip,rcx 460 /*CFI_REGISTER rflags,r11*/ 461 SWAPGS_UNSAFE_STACK 462 /* 463 * A hypervisor implementation might want to use a label 464 * after the swapgs, so that it can do the swapgs 465 * for the guest and jump here on syscall. 466 */ 467ENTRY(system_call_after_swapgs) 468 469 movq %rsp,PER_CPU_VAR(old_rsp) 470 movq PER_CPU_VAR(kernel_stack),%rsp 471 /* 472 * No need to follow this irqs off/on section - it's straight 473 * and short: 474 */ 475 ENABLE_INTERRUPTS(CLBR_NONE) 476 SAVE_ARGS 8,1 477 movq %rax,ORIG_RAX-ARGOFFSET(%rsp) 478 movq %rcx,RIP-ARGOFFSET(%rsp) 479 CFI_REL_OFFSET rip,RIP-ARGOFFSET 480 GET_THREAD_INFO(%rcx) 481 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%rcx) 482 jnz tracesys 483system_call_fastpath: 484 cmpq $__NR_syscall_max,%rax 485 ja badsys 486 movq %r10,%rcx 487 call *sys_call_table(,%rax,8) # XXX: rip relative 488 movq %rax,RAX-ARGOFFSET(%rsp) 489/* 490 * Syscall return path ending with SYSRET (fast path) 491 * Has incomplete stack frame and undefined top of stack. 492 */ 493ret_from_sys_call: 494 movl $_TIF_ALLWORK_MASK,%edi 495 /* edi: flagmask */ 496sysret_check: 497 LOCKDEP_SYS_EXIT 498 GET_THREAD_INFO(%rcx) 499 DISABLE_INTERRUPTS(CLBR_NONE) 500 TRACE_IRQS_OFF 501 movl TI_flags(%rcx),%edx 502 andl %edi,%edx 503 jnz sysret_careful 504 CFI_REMEMBER_STATE 505 /* 506 * sysretq will re-enable interrupts: 507 */ 508 TRACE_IRQS_ON 509 movq RIP-ARGOFFSET(%rsp),%rcx 510 CFI_REGISTER rip,rcx 511 RESTORE_ARGS 0,-ARG_SKIP,1 512 /*CFI_REGISTER rflags,r11*/ 513 movq PER_CPU_VAR(old_rsp), %rsp 514 USERGS_SYSRET64 515 516 CFI_RESTORE_STATE 517 /* Handle reschedules */ 518 /* edx: work, edi: workmask */ 519sysret_careful: 520 bt $TIF_NEED_RESCHED,%edx 521 jnc sysret_signal 522 TRACE_IRQS_ON 523 ENABLE_INTERRUPTS(CLBR_NONE) 524 pushq_cfi %rdi 525 call schedule 526 popq_cfi %rdi 527 jmp sysret_check 528 529 /* Handle a signal */ 530sysret_signal: 531 TRACE_IRQS_ON 532 ENABLE_INTERRUPTS(CLBR_NONE) 533#ifdef CONFIG_AUDITSYSCALL 534 bt $TIF_SYSCALL_AUDIT,%edx 535 jc sysret_audit 536#endif 537 /* 538 * We have a signal, or exit tracing or single-step. 539 * These all wind up with the iret return path anyway, 540 * so just join that path right now. 541 */ 542 FIXUP_TOP_OF_STACK %r11, -ARGOFFSET 543 jmp int_check_syscall_exit_work 544 545badsys: 546 movq $-ENOSYS,RAX-ARGOFFSET(%rsp) 547 jmp ret_from_sys_call 548 549#ifdef CONFIG_AUDITSYSCALL 550 /* 551 * Fast path for syscall audit without full syscall trace. 552 * We just call audit_syscall_entry() directly, and then 553 * jump back to the normal fast path. 554 */ 555auditsys: 556 movq %r10,%r9 /* 6th arg: 4th syscall arg */ 557 movq %rdx,%r8 /* 5th arg: 3rd syscall arg */ 558 movq %rsi,%rcx /* 4th arg: 2nd syscall arg */ 559 movq %rdi,%rdx /* 3rd arg: 1st syscall arg */ 560 movq %rax,%rsi /* 2nd arg: syscall number */ 561 movl $AUDIT_ARCH_X86_64,%edi /* 1st arg: audit arch */ 562 call audit_syscall_entry 563 LOAD_ARGS 0 /* reload call-clobbered registers */ 564 jmp system_call_fastpath 565 566 /* 567 * Return fast path for syscall audit. Call audit_syscall_exit() 568 * directly and then jump back to the fast path with TIF_SYSCALL_AUDIT 569 * masked off. 570 */ 571sysret_audit: 572 movq RAX-ARGOFFSET(%rsp),%rsi /* second arg, syscall return value */ 573 cmpq $0,%rsi /* is it < 0? */ 574 setl %al /* 1 if so, 0 if not */ 575 movzbl %al,%edi /* zero-extend that into %edi */ 576 inc %edi /* first arg, 0->1(AUDITSC_SUCCESS), 1->2(AUDITSC_FAILURE) */ 577 call audit_syscall_exit 578 movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi 579 jmp sysret_check 580#endif /* CONFIG_AUDITSYSCALL */ 581 582 /* Do syscall tracing */ 583tracesys: 584#ifdef CONFIG_AUDITSYSCALL 585 testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%rcx) 586 jz auditsys 587#endif 588 SAVE_REST 589 movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */ 590 FIXUP_TOP_OF_STACK %rdi 591 movq %rsp,%rdi 592 call syscall_trace_enter 593 /* 594 * Reload arg registers from stack in case ptrace changed them. 595 * We don't reload %rax because syscall_trace_enter() returned 596 * the value it wants us to use in the table lookup. 597 */ 598 LOAD_ARGS ARGOFFSET, 1 599 RESTORE_REST 600 cmpq $__NR_syscall_max,%rax 601 ja int_ret_from_sys_call /* RAX(%rsp) set to -ENOSYS above */ 602 movq %r10,%rcx /* fixup for C */ 603 call *sys_call_table(,%rax,8) 604 movq %rax,RAX-ARGOFFSET(%rsp) 605 /* Use IRET because user could have changed frame */ 606 607/* 608 * Syscall return path ending with IRET. 609 * Has correct top of stack, but partial stack frame. 610 */ 611GLOBAL(int_ret_from_sys_call) 612 DISABLE_INTERRUPTS(CLBR_NONE) 613 TRACE_IRQS_OFF 614 testl $3,CS-ARGOFFSET(%rsp) 615 je retint_restore_args 616 movl $_TIF_ALLWORK_MASK,%edi 617 /* edi: mask to check */ 618GLOBAL(int_with_check) 619 LOCKDEP_SYS_EXIT_IRQ 620 GET_THREAD_INFO(%rcx) 621 movl TI_flags(%rcx),%edx 622 andl %edi,%edx 623 jnz int_careful 624 andl $~TS_COMPAT,TI_status(%rcx) 625 jmp retint_swapgs 626 627 /* Either reschedule or signal or syscall exit tracking needed. */ 628 /* First do a reschedule test. */ 629 /* edx: work, edi: workmask */ 630int_careful: 631 bt $TIF_NEED_RESCHED,%edx 632 jnc int_very_careful 633 TRACE_IRQS_ON 634 ENABLE_INTERRUPTS(CLBR_NONE) 635 pushq_cfi %rdi 636 call schedule 637 popq_cfi %rdi 638 DISABLE_INTERRUPTS(CLBR_NONE) 639 TRACE_IRQS_OFF 640 jmp int_with_check 641 642 /* handle signals and tracing -- both require a full stack frame */ 643int_very_careful: 644 TRACE_IRQS_ON 645 ENABLE_INTERRUPTS(CLBR_NONE) 646int_check_syscall_exit_work: 647 SAVE_REST 648 /* Check for syscall exit trace */ 649 testl $_TIF_WORK_SYSCALL_EXIT,%edx 650 jz int_signal 651 pushq_cfi %rdi 652 leaq 8(%rsp),%rdi # &ptregs -> arg1 653 call syscall_trace_leave 654 popq_cfi %rdi 655 andl $~(_TIF_WORK_SYSCALL_EXIT|_TIF_SYSCALL_EMU),%edi 656 jmp int_restore_rest 657 658int_signal: 659 testl $_TIF_DO_NOTIFY_MASK,%edx 660 jz 1f 661 movq %rsp,%rdi # &ptregs -> arg1 662 xorl %esi,%esi # oldset -> arg2 663 call do_notify_resume 6641: movl $_TIF_WORK_MASK,%edi 665int_restore_rest: 666 RESTORE_REST 667 DISABLE_INTERRUPTS(CLBR_NONE) 668 TRACE_IRQS_OFF 669 jmp int_with_check 670 CFI_ENDPROC 671END(system_call) 672 673/* 674 * Certain special system calls that need to save a complete full stack frame. 675 */ 676 .macro PTREGSCALL label,func,arg 677ENTRY(\label) 678 PARTIAL_FRAME 1 8 /* offset 8: return address */ 679 subq $REST_SKIP, %rsp 680 CFI_ADJUST_CFA_OFFSET REST_SKIP 681 call save_rest 682 DEFAULT_FRAME 0 8 /* offset 8: return address */ 683 leaq 8(%rsp), \arg /* pt_regs pointer */ 684 call \func 685 jmp ptregscall_common 686 CFI_ENDPROC 687END(\label) 688 .endm 689 690 PTREGSCALL stub_clone, sys_clone, %r8 691 PTREGSCALL stub_fork, sys_fork, %rdi 692 PTREGSCALL stub_vfork, sys_vfork, %rdi 693 PTREGSCALL stub_sigaltstack, sys_sigaltstack, %rdx 694 PTREGSCALL stub_iopl, sys_iopl, %rsi 695 696ENTRY(ptregscall_common) 697 DEFAULT_FRAME 1 8 /* offset 8: return address */ 698 RESTORE_TOP_OF_STACK %r11, 8 699 movq_cfi_restore R15+8, r15 700 movq_cfi_restore R14+8, r14 701 movq_cfi_restore R13+8, r13 702 movq_cfi_restore R12+8, r12 703 movq_cfi_restore RBP+8, rbp 704 movq_cfi_restore RBX+8, rbx 705 ret $REST_SKIP /* pop extended registers */ 706 CFI_ENDPROC 707END(ptregscall_common) 708 709ENTRY(stub_execve) 710 CFI_STARTPROC 711 addq $8, %rsp 712 PARTIAL_FRAME 0 713 SAVE_REST 714 FIXUP_TOP_OF_STACK %r11 715 movq %rsp, %rcx 716 call sys_execve 717 RESTORE_TOP_OF_STACK %r11 718 movq %rax,RAX(%rsp) 719 RESTORE_REST 720 jmp int_ret_from_sys_call 721 CFI_ENDPROC 722END(stub_execve) 723 724/* 725 * sigreturn is special because it needs to restore all registers on return. 726 * This cannot be done with SYSRET, so use the IRET return path instead. 727 */ 728ENTRY(stub_rt_sigreturn) 729 CFI_STARTPROC 730 addq $8, %rsp 731 PARTIAL_FRAME 0 732 SAVE_REST 733 movq %rsp,%rdi 734 FIXUP_TOP_OF_STACK %r11 735 call sys_rt_sigreturn 736 movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer 737 RESTORE_REST 738 jmp int_ret_from_sys_call 739 CFI_ENDPROC 740END(stub_rt_sigreturn) 741 742/* 743 * Build the entry stubs and pointer table with some assembler magic. 744 * We pack 7 stubs into a single 32-byte chunk, which will fit in a 745 * single cache line on all modern x86 implementations. 746 */ 747 .section .init.rodata,"a" 748ENTRY(interrupt) 749 .section .entry.text 750 .p2align 5 751 .p2align CONFIG_X86_L1_CACHE_SHIFT 752ENTRY(irq_entries_start) 753 INTR_FRAME 754vector=FIRST_EXTERNAL_VECTOR 755.rept (NR_VECTORS-FIRST_EXTERNAL_VECTOR+6)/7 756 .balign 32 757 .rept 7 758 .if vector < NR_VECTORS 759 .if vector <> FIRST_EXTERNAL_VECTOR 760 CFI_ADJUST_CFA_OFFSET -8 761 .endif 7621: pushq_cfi $(~vector+0x80) /* Note: always in signed byte range */ 763 .if ((vector-FIRST_EXTERNAL_VECTOR)%7) <> 6 764 jmp 2f 765 .endif 766 .previous 767 .quad 1b 768 .section .entry.text 769vector=vector+1 770 .endif 771 .endr 7722: jmp common_interrupt 773.endr 774 CFI_ENDPROC 775END(irq_entries_start) 776 777.previous 778END(interrupt) 779.previous 780 781/* 782 * Interrupt entry/exit. 783 * 784 * Interrupt entry points save only callee clobbered registers in fast path. 785 * 786 * Entry runs with interrupts off. 787 */ 788 789/* 0(%rsp): ~(interrupt number) */ 790 .macro interrupt func 791 /* reserve pt_regs for scratch regs and rbp */ 792 subq $ORIG_RAX-RBP, %rsp 793 CFI_ADJUST_CFA_OFFSET ORIG_RAX-RBP 794 call save_args 795 PARTIAL_FRAME 0 796 call \func 797 .endm 798 799/* 800 * Interrupt entry/exit should be protected against kprobes 801 */ 802 .pushsection .kprobes.text, "ax" 803 /* 804 * The interrupt stubs push (~vector+0x80) onto the stack and 805 * then jump to common_interrupt. 806 */ 807 .p2align CONFIG_X86_L1_CACHE_SHIFT 808common_interrupt: 809 XCPT_FRAME 810 addq $-0x80,(%rsp) /* Adjust vector to [-256,-1] range */ 811 interrupt do_IRQ 812 /* 0(%rsp): old_rsp-ARGOFFSET */ 813ret_from_intr: 814 DISABLE_INTERRUPTS(CLBR_NONE) 815 TRACE_IRQS_OFF 816 decl PER_CPU_VAR(irq_count) 817 leaveq 818 819 CFI_RESTORE rbp 820 CFI_DEF_CFA_REGISTER rsp 821 CFI_ADJUST_CFA_OFFSET -8 822 823 /* we did not save rbx, restore only from ARGOFFSET */ 824 addq $8, %rsp 825 CFI_ADJUST_CFA_OFFSET -8 826exit_intr: 827 GET_THREAD_INFO(%rcx) 828 testl $3,CS-ARGOFFSET(%rsp) 829 je retint_kernel 830 831 /* Interrupt came from user space */ 832 /* 833 * Has a correct top of stack, but a partial stack frame 834 * %rcx: thread info. Interrupts off. 835 */ 836retint_with_reschedule: 837 movl $_TIF_WORK_MASK,%edi 838retint_check: 839 LOCKDEP_SYS_EXIT_IRQ 840 movl TI_flags(%rcx),%edx 841 andl %edi,%edx 842 CFI_REMEMBER_STATE 843 jnz retint_careful 844 845retint_swapgs: /* return to user-space */ 846 /* 847 * The iretq could re-enable interrupts: 848 */ 849 DISABLE_INTERRUPTS(CLBR_ANY) 850 TRACE_IRQS_IRETQ 851 SWAPGS 852 jmp restore_args 853 854retint_restore_args: /* return to kernel space */ 855 DISABLE_INTERRUPTS(CLBR_ANY) 856 /* 857 * The iretq could re-enable interrupts: 858 */ 859 TRACE_IRQS_IRETQ 860restore_args: 861 RESTORE_ARGS 0,8,0 862 863irq_return: 864 INTERRUPT_RETURN 865 866 .section __ex_table, "a" 867 .quad irq_return, bad_iret 868 .previous 869 870#ifdef CONFIG_PARAVIRT 871ENTRY(native_iret) 872 iretq 873 874 .section __ex_table,"a" 875 .quad native_iret, bad_iret 876 .previous 877#endif 878 879 .section .fixup,"ax" 880bad_iret: 881 /* 882 * The iret traps when the %cs or %ss being restored is bogus. 883 * We've lost the original trap vector and error code. 884 * #GPF is the most likely one to get for an invalid selector. 885 * So pretend we completed the iret and took the #GPF in user mode. 886 * 887 * We are now running with the kernel GS after exception recovery. 888 * But error_entry expects us to have user GS to match the user %cs, 889 * so swap back. 890 */ 891 pushq $0 892 893 SWAPGS 894 jmp general_protection 895 896 .previous 897 898 /* edi: workmask, edx: work */ 899retint_careful: 900 CFI_RESTORE_STATE 901 bt $TIF_NEED_RESCHED,%edx 902 jnc retint_signal 903 TRACE_IRQS_ON 904 ENABLE_INTERRUPTS(CLBR_NONE) 905 pushq_cfi %rdi 906 call schedule 907 popq_cfi %rdi 908 GET_THREAD_INFO(%rcx) 909 DISABLE_INTERRUPTS(CLBR_NONE) 910 TRACE_IRQS_OFF 911 jmp retint_check 912 913retint_signal: 914 testl $_TIF_DO_NOTIFY_MASK,%edx 915 jz retint_swapgs 916 TRACE_IRQS_ON 917 ENABLE_INTERRUPTS(CLBR_NONE) 918 SAVE_REST 919 movq $-1,ORIG_RAX(%rsp) 920 xorl %esi,%esi # oldset 921 movq %rsp,%rdi # &pt_regs 922 call do_notify_resume 923 RESTORE_REST 924 DISABLE_INTERRUPTS(CLBR_NONE) 925 TRACE_IRQS_OFF 926 GET_THREAD_INFO(%rcx) 927 jmp retint_with_reschedule 928 929#ifdef CONFIG_PREEMPT 930 /* Returning to kernel space. Check if we need preemption */ 931 /* rcx: threadinfo. interrupts off. */ 932ENTRY(retint_kernel) 933 cmpl $0,TI_preempt_count(%rcx) 934 jnz retint_restore_args 935 bt $TIF_NEED_RESCHED,TI_flags(%rcx) 936 jnc retint_restore_args 937 bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */ 938 jnc retint_restore_args 939 call preempt_schedule_irq 940 jmp exit_intr 941#endif 942 943 CFI_ENDPROC 944END(common_interrupt) 945/* 946 * End of kprobes section 947 */ 948 .popsection 949 950/* 951 * APIC interrupts. 952 */ 953.macro apicinterrupt num sym do_sym 954ENTRY(\sym) 955 INTR_FRAME 956 pushq_cfi $~(\num) 957 interrupt \do_sym 958 jmp ret_from_intr 959 CFI_ENDPROC 960END(\sym) 961.endm 962 963#ifdef CONFIG_SMP 964apicinterrupt IRQ_MOVE_CLEANUP_VECTOR \ 965 irq_move_cleanup_interrupt smp_irq_move_cleanup_interrupt 966apicinterrupt REBOOT_VECTOR \ 967 reboot_interrupt smp_reboot_interrupt 968#endif 969 970#ifdef CONFIG_X86_UV 971apicinterrupt UV_BAU_MESSAGE \ 972 uv_bau_message_intr1 uv_bau_message_interrupt 973#endif 974apicinterrupt LOCAL_TIMER_VECTOR \ 975 apic_timer_interrupt smp_apic_timer_interrupt 976apicinterrupt X86_PLATFORM_IPI_VECTOR \ 977 x86_platform_ipi smp_x86_platform_ipi 978 979#ifdef CONFIG_SMP 980.irp idx,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15, \ 981 16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 982.if NUM_INVALIDATE_TLB_VECTORS > \idx 983apicinterrupt (INVALIDATE_TLB_VECTOR_START)+\idx \ 984 invalidate_interrupt\idx smp_invalidate_interrupt 985.endif 986.endr 987#endif 988 989apicinterrupt THRESHOLD_APIC_VECTOR \ 990 threshold_interrupt smp_threshold_interrupt 991apicinterrupt THERMAL_APIC_VECTOR \ 992 thermal_interrupt smp_thermal_interrupt 993 994#ifdef CONFIG_X86_MCE 995apicinterrupt MCE_SELF_VECTOR \ 996 mce_self_interrupt smp_mce_self_interrupt 997#endif 998 999#ifdef CONFIG_SMP 1000apicinterrupt CALL_FUNCTION_SINGLE_VECTOR \ 1001 call_function_single_interrupt smp_call_function_single_interrupt 1002apicinterrupt CALL_FUNCTION_VECTOR \ 1003 call_function_interrupt smp_call_function_interrupt 1004apicinterrupt RESCHEDULE_VECTOR \ 1005 reschedule_interrupt smp_reschedule_interrupt 1006#endif 1007 1008apicinterrupt ERROR_APIC_VECTOR \ 1009 error_interrupt smp_error_interrupt 1010apicinterrupt SPURIOUS_APIC_VECTOR \ 1011 spurious_interrupt smp_spurious_interrupt 1012 1013#ifdef CONFIG_IRQ_WORK 1014apicinterrupt IRQ_WORK_VECTOR \ 1015 irq_work_interrupt smp_irq_work_interrupt 1016#endif 1017 1018/* 1019 * Exception entry points. 1020 */ 1021.macro zeroentry sym do_sym 1022ENTRY(\sym) 1023 INTR_FRAME 1024 PARAVIRT_ADJUST_EXCEPTION_FRAME 1025 pushq_cfi $-1 /* ORIG_RAX: no syscall to restart */ 1026 subq $ORIG_RAX-R15, %rsp 1027 CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15 1028 call error_entry 1029 DEFAULT_FRAME 0 1030 movq %rsp,%rdi /* pt_regs pointer */ 1031 xorl %esi,%esi /* no error code */ 1032 call \do_sym 1033 jmp error_exit /* %ebx: no swapgs flag */ 1034 CFI_ENDPROC 1035END(\sym) 1036.endm 1037 1038.macro paranoidzeroentry sym do_sym 1039ENTRY(\sym) 1040 INTR_FRAME 1041 PARAVIRT_ADJUST_EXCEPTION_FRAME 1042 pushq_cfi $-1 /* ORIG_RAX: no syscall to restart */ 1043 subq $ORIG_RAX-R15, %rsp 1044 CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15 1045 call save_paranoid 1046 TRACE_IRQS_OFF 1047 movq %rsp,%rdi /* pt_regs pointer */ 1048 xorl %esi,%esi /* no error code */ 1049 call \do_sym 1050 jmp paranoid_exit /* %ebx: no swapgs flag */ 1051 CFI_ENDPROC 1052END(\sym) 1053.endm 1054 1055#define INIT_TSS_IST(x) PER_CPU_VAR(init_tss) + (TSS_ist + ((x) - 1) * 8) 1056.macro paranoidzeroentry_ist sym do_sym ist 1057ENTRY(\sym) 1058 INTR_FRAME 1059 PARAVIRT_ADJUST_EXCEPTION_FRAME 1060 pushq_cfi $-1 /* ORIG_RAX: no syscall to restart */ 1061 subq $ORIG_RAX-R15, %rsp 1062 CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15 1063 call save_paranoid 1064 TRACE_IRQS_OFF 1065 movq %rsp,%rdi /* pt_regs pointer */ 1066 xorl %esi,%esi /* no error code */ 1067 subq $EXCEPTION_STKSZ, INIT_TSS_IST(\ist) 1068 call \do_sym 1069 addq $EXCEPTION_STKSZ, INIT_TSS_IST(\ist) 1070 jmp paranoid_exit /* %ebx: no swapgs flag */ 1071 CFI_ENDPROC 1072END(\sym) 1073.endm 1074 1075.macro errorentry sym do_sym 1076ENTRY(\sym) 1077 XCPT_FRAME 1078 PARAVIRT_ADJUST_EXCEPTION_FRAME 1079 subq $ORIG_RAX-R15, %rsp 1080 CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15 1081 call error_entry 1082 DEFAULT_FRAME 0 1083 movq %rsp,%rdi /* pt_regs pointer */ 1084 movq ORIG_RAX(%rsp),%rsi /* get error code */ 1085 movq $-1,ORIG_RAX(%rsp) /* no syscall to restart */ 1086 call \do_sym 1087 jmp error_exit /* %ebx: no swapgs flag */ 1088 CFI_ENDPROC 1089END(\sym) 1090.endm 1091 1092 /* error code is on the stack already */ 1093.macro paranoiderrorentry sym do_sym 1094ENTRY(\sym) 1095 XCPT_FRAME 1096 PARAVIRT_ADJUST_EXCEPTION_FRAME 1097 subq $ORIG_RAX-R15, %rsp 1098 CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15 1099 call save_paranoid 1100 DEFAULT_FRAME 0 1101 TRACE_IRQS_OFF 1102 movq %rsp,%rdi /* pt_regs pointer */ 1103 movq ORIG_RAX(%rsp),%rsi /* get error code */ 1104 movq $-1,ORIG_RAX(%rsp) /* no syscall to restart */ 1105 call \do_sym 1106 jmp paranoid_exit /* %ebx: no swapgs flag */ 1107 CFI_ENDPROC 1108END(\sym) 1109.endm 1110 1111zeroentry divide_error do_divide_error 1112zeroentry overflow do_overflow 1113zeroentry bounds do_bounds 1114zeroentry invalid_op do_invalid_op 1115zeroentry device_not_available do_device_not_available 1116paranoiderrorentry double_fault do_double_fault 1117zeroentry coprocessor_segment_overrun do_coprocessor_segment_overrun 1118errorentry invalid_TSS do_invalid_TSS 1119errorentry segment_not_present do_segment_not_present 1120zeroentry spurious_interrupt_bug do_spurious_interrupt_bug 1121zeroentry coprocessor_error do_coprocessor_error 1122errorentry alignment_check do_alignment_check 1123zeroentry simd_coprocessor_error do_simd_coprocessor_error 1124 1125 /* Reload gs selector with exception handling */ 1126 /* edi: new selector */ 1127ENTRY(native_load_gs_index) 1128 CFI_STARTPROC 1129 pushfq_cfi 1130 DISABLE_INTERRUPTS(CLBR_ANY & ~CLBR_RDI) 1131 SWAPGS 1132gs_change: 1133 movl %edi,%gs 11342: mfence /* workaround */ 1135 SWAPGS 1136 popfq_cfi 1137 ret 1138 CFI_ENDPROC 1139END(native_load_gs_index) 1140 1141 .section __ex_table,"a" 1142 .align 8 1143 .quad gs_change,bad_gs 1144 .previous 1145 .section .fixup,"ax" 1146 /* running with kernelgs */ 1147bad_gs: 1148 SWAPGS /* switch back to user gs */ 1149 xorl %eax,%eax 1150 movl %eax,%gs 1151 jmp 2b 1152 .previous 1153 1154ENTRY(kernel_thread_helper) 1155 pushq $0 # fake return address 1156 CFI_STARTPROC 1157 /* 1158 * Here we are in the child and the registers are set as they were 1159 * at kernel_thread() invocation in the parent. 1160 */ 1161 call *%rsi 1162 # exit 1163 mov %eax, %edi 1164 call do_exit 1165 ud2 # padding for call trace 1166 CFI_ENDPROC 1167END(kernel_thread_helper) 1168 1169/* 1170 * execve(). This function needs to use IRET, not SYSRET, to set up all state properly. 1171 * 1172 * C extern interface: 1173 * extern long execve(const char *name, char **argv, char **envp) 1174 * 1175 * asm input arguments: 1176 * rdi: name, rsi: argv, rdx: envp 1177 * 1178 * We want to fallback into: 1179 * extern long sys_execve(const char *name, char **argv,char **envp, struct pt_regs *regs) 1180 * 1181 * do_sys_execve asm fallback arguments: 1182 * rdi: name, rsi: argv, rdx: envp, rcx: fake frame on the stack 1183 */ 1184ENTRY(kernel_execve) 1185 CFI_STARTPROC 1186 FAKE_STACK_FRAME $0 1187 SAVE_ALL 1188 movq %rsp,%rcx 1189 call sys_execve 1190 movq %rax, RAX(%rsp) 1191 RESTORE_REST 1192 testq %rax,%rax 1193 je int_ret_from_sys_call 1194 RESTORE_ARGS 1195 UNFAKE_STACK_FRAME 1196 ret 1197 CFI_ENDPROC 1198END(kernel_execve) 1199 1200/* Call softirq on interrupt stack. Interrupts are off. */ 1201ENTRY(call_softirq) 1202 CFI_STARTPROC 1203 pushq_cfi %rbp 1204 CFI_REL_OFFSET rbp,0 1205 mov %rsp,%rbp 1206 CFI_DEF_CFA_REGISTER rbp 1207 incl PER_CPU_VAR(irq_count) 1208 cmove PER_CPU_VAR(irq_stack_ptr),%rsp 1209 push %rbp # backlink for old unwinder 1210 call __do_softirq 1211 leaveq 1212 CFI_RESTORE rbp 1213 CFI_DEF_CFA_REGISTER rsp 1214 CFI_ADJUST_CFA_OFFSET -8 1215 decl PER_CPU_VAR(irq_count) 1216 ret 1217 CFI_ENDPROC 1218END(call_softirq) 1219 1220#ifdef CONFIG_XEN 1221zeroentry xen_hypervisor_callback xen_do_hypervisor_callback 1222 1223/* 1224 * A note on the "critical region" in our callback handler. 1225 * We want to avoid stacking callback handlers due to events occurring 1226 * during handling of the last event. To do this, we keep events disabled 1227 * until we've done all processing. HOWEVER, we must enable events before 1228 * popping the stack frame (can't be done atomically) and so it would still 1229 * be possible to get enough handler activations to overflow the stack. 1230 * Although unlikely, bugs of that kind are hard to track down, so we'd 1231 * like to avoid the possibility. 1232 * So, on entry to the handler we detect whether we interrupted an 1233 * existing activation in its critical region -- if so, we pop the current 1234 * activation and restart the handler using the previous one. 1235 */ 1236ENTRY(xen_do_hypervisor_callback) # do_hypervisor_callback(struct *pt_regs) 1237 CFI_STARTPROC 1238/* 1239 * Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will 1240 * see the correct pointer to the pt_regs 1241 */ 1242 movq %rdi, %rsp # we don't return, adjust the stack frame 1243 CFI_ENDPROC 1244 DEFAULT_FRAME 124511: incl PER_CPU_VAR(irq_count) 1246 movq %rsp,%rbp 1247 CFI_DEF_CFA_REGISTER rbp 1248 cmovzq PER_CPU_VAR(irq_stack_ptr),%rsp 1249 pushq %rbp # backlink for old unwinder 1250 call xen_evtchn_do_upcall 1251 popq %rsp 1252 CFI_DEF_CFA_REGISTER rsp 1253 decl PER_CPU_VAR(irq_count) 1254 jmp error_exit 1255 CFI_ENDPROC 1256END(xen_do_hypervisor_callback) 1257 1258/* 1259 * Hypervisor uses this for application faults while it executes. 1260 * We get here for two reasons: 1261 * 1. Fault while reloading DS, ES, FS or GS 1262 * 2. Fault while executing IRET 1263 * Category 1 we do not need to fix up as Xen has already reloaded all segment 1264 * registers that could be reloaded and zeroed the others. 1265 * Category 2 we fix up by killing the current process. We cannot use the 1266 * normal Linux return path in this case because if we use the IRET hypercall 1267 * to pop the stack frame we end up in an infinite loop of failsafe callbacks. 1268 * We distinguish between categories by comparing each saved segment register 1269 * with its current contents: any discrepancy means we in category 1. 1270 */ 1271ENTRY(xen_failsafe_callback) 1272 INTR_FRAME 1 (6*8) 1273 /*CFI_REL_OFFSET gs,GS*/ 1274 /*CFI_REL_OFFSET fs,FS*/ 1275 /*CFI_REL_OFFSET es,ES*/ 1276 /*CFI_REL_OFFSET ds,DS*/ 1277 CFI_REL_OFFSET r11,8 1278 CFI_REL_OFFSET rcx,0 1279 movw %ds,%cx 1280 cmpw %cx,0x10(%rsp) 1281 CFI_REMEMBER_STATE 1282 jne 1f 1283 movw %es,%cx 1284 cmpw %cx,0x18(%rsp) 1285 jne 1f 1286 movw %fs,%cx 1287 cmpw %cx,0x20(%rsp) 1288 jne 1f 1289 movw %gs,%cx 1290 cmpw %cx,0x28(%rsp) 1291 jne 1f 1292 /* All segments match their saved values => Category 2 (Bad IRET). */ 1293 movq (%rsp),%rcx 1294 CFI_RESTORE rcx 1295 movq 8(%rsp),%r11 1296 CFI_RESTORE r11 1297 addq $0x30,%rsp 1298 CFI_ADJUST_CFA_OFFSET -0x30 1299 pushq_cfi $0 /* RIP */ 1300 pushq_cfi %r11 1301 pushq_cfi %rcx 1302 jmp general_protection 1303 CFI_RESTORE_STATE 13041: /* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */ 1305 movq (%rsp),%rcx 1306 CFI_RESTORE rcx 1307 movq 8(%rsp),%r11 1308 CFI_RESTORE r11 1309 addq $0x30,%rsp 1310 CFI_ADJUST_CFA_OFFSET -0x30 1311 pushq_cfi $-1 /* orig_ax = -1 => not a system call */ 1312 SAVE_ALL 1313 jmp error_exit 1314 CFI_ENDPROC 1315END(xen_failsafe_callback) 1316 1317apicinterrupt XEN_HVM_EVTCHN_CALLBACK \ 1318 xen_hvm_callback_vector xen_evtchn_do_upcall 1319 1320#endif /* CONFIG_XEN */ 1321 1322/* 1323 * Some functions should be protected against kprobes 1324 */ 1325 .pushsection .kprobes.text, "ax" 1326 1327paranoidzeroentry_ist debug do_debug DEBUG_STACK 1328paranoidzeroentry_ist int3 do_int3 DEBUG_STACK 1329paranoiderrorentry stack_segment do_stack_segment 1330#ifdef CONFIG_XEN 1331zeroentry xen_debug do_debug 1332zeroentry xen_int3 do_int3 1333errorentry xen_stack_segment do_stack_segment 1334#endif 1335errorentry general_protection do_general_protection 1336errorentry page_fault do_page_fault 1337#ifdef CONFIG_KVM_GUEST 1338errorentry async_page_fault do_async_page_fault 1339#endif 1340#ifdef CONFIG_X86_MCE 1341paranoidzeroentry machine_check *machine_check_vector(%rip) 1342#endif 1343 1344 /* 1345 * "Paranoid" exit path from exception stack. 1346 * Paranoid because this is used by NMIs and cannot take 1347 * any kernel state for granted. 1348 * We don't do kernel preemption checks here, because only 1349 * NMI should be common and it does not enable IRQs and 1350 * cannot get reschedule ticks. 1351 * 1352 * "trace" is 0 for the NMI handler only, because irq-tracing 1353 * is fundamentally NMI-unsafe. (we cannot change the soft and 1354 * hard flags at once, atomically) 1355 */ 1356 1357 /* ebx: no swapgs flag */ 1358ENTRY(paranoid_exit) 1359 DEFAULT_FRAME 1360 DISABLE_INTERRUPTS(CLBR_NONE) 1361 TRACE_IRQS_OFF 1362 testl %ebx,%ebx /* swapgs needed? */ 1363 jnz paranoid_restore 1364 testl $3,CS(%rsp) 1365 jnz paranoid_userspace 1366paranoid_swapgs: 1367 TRACE_IRQS_IRETQ 0 1368 SWAPGS_UNSAFE_STACK 1369 RESTORE_ALL 8 1370 jmp irq_return 1371paranoid_restore: 1372 TRACE_IRQS_IRETQ 0 1373 RESTORE_ALL 8 1374 jmp irq_return 1375paranoid_userspace: 1376 GET_THREAD_INFO(%rcx) 1377 movl TI_flags(%rcx),%ebx 1378 andl $_TIF_WORK_MASK,%ebx 1379 jz paranoid_swapgs 1380 movq %rsp,%rdi /* &pt_regs */ 1381 call sync_regs 1382 movq %rax,%rsp /* switch stack for scheduling */ 1383 testl $_TIF_NEED_RESCHED,%ebx 1384 jnz paranoid_schedule 1385 movl %ebx,%edx /* arg3: thread flags */ 1386 TRACE_IRQS_ON 1387 ENABLE_INTERRUPTS(CLBR_NONE) 1388 xorl %esi,%esi /* arg2: oldset */ 1389 movq %rsp,%rdi /* arg1: &pt_regs */ 1390 call do_notify_resume 1391 DISABLE_INTERRUPTS(CLBR_NONE) 1392 TRACE_IRQS_OFF 1393 jmp paranoid_userspace 1394paranoid_schedule: 1395 TRACE_IRQS_ON 1396 ENABLE_INTERRUPTS(CLBR_ANY) 1397 call schedule 1398 DISABLE_INTERRUPTS(CLBR_ANY) 1399 TRACE_IRQS_OFF 1400 jmp paranoid_userspace 1401 CFI_ENDPROC 1402END(paranoid_exit) 1403 1404/* 1405 * Exception entry point. This expects an error code/orig_rax on the stack. 1406 * returns in "no swapgs flag" in %ebx. 1407 */ 1408ENTRY(error_entry) 1409 XCPT_FRAME 1410 CFI_ADJUST_CFA_OFFSET 15*8 1411 /* oldrax contains error code */ 1412 cld 1413 movq_cfi rdi, RDI+8 1414 movq_cfi rsi, RSI+8 1415 movq_cfi rdx, RDX+8 1416 movq_cfi rcx, RCX+8 1417 movq_cfi rax, RAX+8 1418 movq_cfi r8, R8+8 1419 movq_cfi r9, R9+8 1420 movq_cfi r10, R10+8 1421 movq_cfi r11, R11+8 1422 movq_cfi rbx, RBX+8 1423 movq_cfi rbp, RBP+8 1424 movq_cfi r12, R12+8 1425 movq_cfi r13, R13+8 1426 movq_cfi r14, R14+8 1427 movq_cfi r15, R15+8 1428 xorl %ebx,%ebx 1429 testl $3,CS+8(%rsp) 1430 je error_kernelspace 1431error_swapgs: 1432 SWAPGS 1433error_sti: 1434 TRACE_IRQS_OFF 1435 ret 1436 1437/* 1438 * There are two places in the kernel that can potentially fault with 1439 * usergs. Handle them here. The exception handlers after iret run with 1440 * kernel gs again, so don't set the user space flag. B stepping K8s 1441 * sometimes report an truncated RIP for IRET exceptions returning to 1442 * compat mode. Check for these here too. 1443 */ 1444error_kernelspace: 1445 incl %ebx 1446 leaq irq_return(%rip),%rcx 1447 cmpq %rcx,RIP+8(%rsp) 1448 je error_swapgs 1449 movl %ecx,%eax /* zero extend */ 1450 cmpq %rax,RIP+8(%rsp) 1451 je bstep_iret 1452 cmpq $gs_change,RIP+8(%rsp) 1453 je error_swapgs 1454 jmp error_sti 1455 1456bstep_iret: 1457 /* Fix truncated RIP */ 1458 movq %rcx,RIP+8(%rsp) 1459 jmp error_swapgs 1460 CFI_ENDPROC 1461END(error_entry) 1462 1463 1464/* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */ 1465ENTRY(error_exit) 1466 DEFAULT_FRAME 1467 movl %ebx,%eax 1468 RESTORE_REST 1469 DISABLE_INTERRUPTS(CLBR_NONE) 1470 TRACE_IRQS_OFF 1471 GET_THREAD_INFO(%rcx) 1472 testl %eax,%eax 1473 jne retint_kernel 1474 LOCKDEP_SYS_EXIT_IRQ 1475 movl TI_flags(%rcx),%edx 1476 movl $_TIF_WORK_MASK,%edi 1477 andl %edi,%edx 1478 jnz retint_careful 1479 jmp retint_swapgs 1480 CFI_ENDPROC 1481END(error_exit) 1482 1483 1484 /* runs on exception stack */ 1485ENTRY(nmi) 1486 INTR_FRAME 1487 PARAVIRT_ADJUST_EXCEPTION_FRAME 1488 pushq_cfi $-1 1489 subq $ORIG_RAX-R15, %rsp 1490 CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15 1491 call save_paranoid 1492 DEFAULT_FRAME 0 1493 /* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */ 1494 movq %rsp,%rdi 1495 movq $-1,%rsi 1496 call do_nmi 1497#ifdef CONFIG_TRACE_IRQFLAGS 1498 /* paranoidexit; without TRACE_IRQS_OFF */ 1499 /* ebx: no swapgs flag */ 1500 DISABLE_INTERRUPTS(CLBR_NONE) 1501 testl %ebx,%ebx /* swapgs needed? */ 1502 jnz nmi_restore 1503 testl $3,CS(%rsp) 1504 jnz nmi_userspace 1505nmi_swapgs: 1506 SWAPGS_UNSAFE_STACK 1507nmi_restore: 1508 RESTORE_ALL 8 1509 jmp irq_return 1510nmi_userspace: 1511 GET_THREAD_INFO(%rcx) 1512 movl TI_flags(%rcx),%ebx 1513 andl $_TIF_WORK_MASK,%ebx 1514 jz nmi_swapgs 1515 movq %rsp,%rdi /* &pt_regs */ 1516 call sync_regs 1517 movq %rax,%rsp /* switch stack for scheduling */ 1518 testl $_TIF_NEED_RESCHED,%ebx 1519 jnz nmi_schedule 1520 movl %ebx,%edx /* arg3: thread flags */ 1521 ENABLE_INTERRUPTS(CLBR_NONE) 1522 xorl %esi,%esi /* arg2: oldset */ 1523 movq %rsp,%rdi /* arg1: &pt_regs */ 1524 call do_notify_resume 1525 DISABLE_INTERRUPTS(CLBR_NONE) 1526 jmp nmi_userspace 1527nmi_schedule: 1528 ENABLE_INTERRUPTS(CLBR_ANY) 1529 call schedule 1530 DISABLE_INTERRUPTS(CLBR_ANY) 1531 jmp nmi_userspace 1532 CFI_ENDPROC 1533#else 1534 jmp paranoid_exit 1535 CFI_ENDPROC 1536#endif 1537END(nmi) 1538 1539ENTRY(ignore_sysret) 1540 CFI_STARTPROC 1541 mov $-ENOSYS,%eax 1542 sysret 1543 CFI_ENDPROC 1544END(ignore_sysret) 1545 1546/* 1547 * End of kprobes section 1548 */ 1549 .popsection