/arch/x86/kernel/process.c
C | 659 lines | 433 code | 87 blank | 139 comment | 69 complexity | 4cd57fff3b97502cbb37f804990e5e69 MD5 | raw file
Possible License(s): GPL-2.0, LGPL-2.0, AGPL-1.0
1#include <linux/errno.h> 2#include <linux/kernel.h> 3#include <linux/mm.h> 4#include <linux/smp.h> 5#include <linux/prctl.h> 6#include <linux/slab.h> 7#include <linux/sched.h> 8#include <linux/module.h> 9#include <linux/pm.h> 10#include <linux/clockchips.h> 11#include <linux/random.h> 12#include <linux/user-return-notifier.h> 13#include <linux/dmi.h> 14#include <linux/utsname.h> 15#include <trace/events/power.h> 16#include <linux/hw_breakpoint.h> 17#include <asm/cpu.h> 18#include <asm/system.h> 19#include <asm/apic.h> 20#include <asm/syscalls.h> 21#include <asm/idle.h> 22#include <asm/uaccess.h> 23#include <asm/i387.h> 24#include <asm/debugreg.h> 25 26struct kmem_cache *task_xstate_cachep; 27EXPORT_SYMBOL_GPL(task_xstate_cachep); 28 29int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) 30{ 31 int ret; 32 33 *dst = *src; 34 if (fpu_allocated(&src->thread.fpu)) { 35 memset(&dst->thread.fpu, 0, sizeof(dst->thread.fpu)); 36 ret = fpu_alloc(&dst->thread.fpu); 37 if (ret) 38 return ret; 39 fpu_copy(&dst->thread.fpu, &src->thread.fpu); 40 } 41 return 0; 42} 43 44void free_thread_xstate(struct task_struct *tsk) 45{ 46 fpu_free(&tsk->thread.fpu); 47} 48 49void free_thread_info(struct thread_info *ti) 50{ 51 free_thread_xstate(ti->task); 52 free_pages((unsigned long)ti, get_order(THREAD_SIZE)); 53} 54 55void arch_task_cache_init(void) 56{ 57 task_xstate_cachep = 58 kmem_cache_create("task_xstate", xstate_size, 59 __alignof__(union thread_xstate), 60 SLAB_PANIC | SLAB_NOTRACK, NULL); 61} 62 63/* 64 * Free current thread data structures etc.. 65 */ 66void exit_thread(void) 67{ 68 struct task_struct *me = current; 69 struct thread_struct *t = &me->thread; 70 unsigned long *bp = t->io_bitmap_ptr; 71 72 if (bp) { 73 struct tss_struct *tss = &per_cpu(init_tss, get_cpu()); 74 75 t->io_bitmap_ptr = NULL; 76 clear_thread_flag(TIF_IO_BITMAP); 77 /* 78 * Careful, clear this in the TSS too: 79 */ 80 memset(tss->io_bitmap, 0xff, t->io_bitmap_max); 81 t->io_bitmap_max = 0; 82 put_cpu(); 83 kfree(bp); 84 } 85} 86 87void show_regs(struct pt_regs *regs) 88{ 89 show_registers(regs); 90 show_trace(NULL, regs, (unsigned long *)kernel_stack_pointer(regs), 0); 91} 92 93void show_regs_common(void) 94{ 95 const char *vendor, *product, *board; 96 97 vendor = dmi_get_system_info(DMI_SYS_VENDOR); 98 if (!vendor) 99 vendor = ""; 100 product = dmi_get_system_info(DMI_PRODUCT_NAME); 101 if (!product) 102 product = ""; 103 104 /* Board Name is optional */ 105 board = dmi_get_system_info(DMI_BOARD_NAME); 106 107 printk(KERN_CONT "\n"); 108 printk(KERN_DEFAULT "Pid: %d, comm: %.20s %s %s %.*s", 109 current->pid, current->comm, print_tainted(), 110 init_utsname()->release, 111 (int)strcspn(init_utsname()->version, " "), 112 init_utsname()->version); 113 printk(KERN_CONT " %s %s", vendor, product); 114 if (board) 115 printk(KERN_CONT "/%s", board); 116 printk(KERN_CONT "\n"); 117} 118 119void flush_thread(void) 120{ 121 struct task_struct *tsk = current; 122 123 flush_ptrace_hw_breakpoint(tsk); 124 memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array)); 125 /* 126 * Forget coprocessor state.. 127 */ 128 tsk->fpu_counter = 0; 129 clear_fpu(tsk); 130 clear_used_math(); 131} 132 133static void hard_disable_TSC(void) 134{ 135 write_cr4(read_cr4() | X86_CR4_TSD); 136} 137 138void disable_TSC(void) 139{ 140 preempt_disable(); 141 if (!test_and_set_thread_flag(TIF_NOTSC)) 142 /* 143 * Must flip the CPU state synchronously with 144 * TIF_NOTSC in the current running context. 145 */ 146 hard_disable_TSC(); 147 preempt_enable(); 148} 149 150static void hard_enable_TSC(void) 151{ 152 write_cr4(read_cr4() & ~X86_CR4_TSD); 153} 154 155static void enable_TSC(void) 156{ 157 preempt_disable(); 158 if (test_and_clear_thread_flag(TIF_NOTSC)) 159 /* 160 * Must flip the CPU state synchronously with 161 * TIF_NOTSC in the current running context. 162 */ 163 hard_enable_TSC(); 164 preempt_enable(); 165} 166 167int get_tsc_mode(unsigned long adr) 168{ 169 unsigned int val; 170 171 if (test_thread_flag(TIF_NOTSC)) 172 val = PR_TSC_SIGSEGV; 173 else 174 val = PR_TSC_ENABLE; 175 176 return put_user(val, (unsigned int __user *)adr); 177} 178 179int set_tsc_mode(unsigned int val) 180{ 181 if (val == PR_TSC_SIGSEGV) 182 disable_TSC(); 183 else if (val == PR_TSC_ENABLE) 184 enable_TSC(); 185 else 186 return -EINVAL; 187 188 return 0; 189} 190 191void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p, 192 struct tss_struct *tss) 193{ 194 struct thread_struct *prev, *next; 195 196 prev = &prev_p->thread; 197 next = &next_p->thread; 198 199 if (test_tsk_thread_flag(prev_p, TIF_BLOCKSTEP) ^ 200 test_tsk_thread_flag(next_p, TIF_BLOCKSTEP)) { 201 unsigned long debugctl = get_debugctlmsr(); 202 203 debugctl &= ~DEBUGCTLMSR_BTF; 204 if (test_tsk_thread_flag(next_p, TIF_BLOCKSTEP)) 205 debugctl |= DEBUGCTLMSR_BTF; 206 207 update_debugctlmsr(debugctl); 208 } 209 210 if (test_tsk_thread_flag(prev_p, TIF_NOTSC) ^ 211 test_tsk_thread_flag(next_p, TIF_NOTSC)) { 212 /* prev and next are different */ 213 if (test_tsk_thread_flag(next_p, TIF_NOTSC)) 214 hard_disable_TSC(); 215 else 216 hard_enable_TSC(); 217 } 218 219 if (test_tsk_thread_flag(next_p, TIF_IO_BITMAP)) { 220 /* 221 * Copy the relevant range of the IO bitmap. 222 * Normally this is 128 bytes or less: 223 */ 224 memcpy(tss->io_bitmap, next->io_bitmap_ptr, 225 max(prev->io_bitmap_max, next->io_bitmap_max)); 226 } else if (test_tsk_thread_flag(prev_p, TIF_IO_BITMAP)) { 227 /* 228 * Clear any possible leftover bits: 229 */ 230 memset(tss->io_bitmap, 0xff, prev->io_bitmap_max); 231 } 232 propagate_user_return_notify(prev_p, next_p); 233} 234 235int sys_fork(struct pt_regs *regs) 236{ 237 return do_fork(SIGCHLD, regs->sp, regs, 0, NULL, NULL); 238} 239 240/* 241 * This is trivial, and on the face of it looks like it 242 * could equally well be done in user mode. 243 * 244 * Not so, for quite unobvious reasons - register pressure. 245 * In user mode vfork() cannot have a stack frame, and if 246 * done by calling the "clone()" system call directly, you 247 * do not have enough call-clobbered registers to hold all 248 * the information you need. 249 */ 250int sys_vfork(struct pt_regs *regs) 251{ 252 return do_fork(CLONE_VFORK | CLONE_VM | SIGCHLD, regs->sp, regs, 0, 253 NULL, NULL); 254} 255 256long 257sys_clone(unsigned long clone_flags, unsigned long newsp, 258 void __user *parent_tid, void __user *child_tid, struct pt_regs *regs) 259{ 260 if (!newsp) 261 newsp = regs->sp; 262 return do_fork(clone_flags, newsp, regs, 0, parent_tid, child_tid); 263} 264 265/* 266 * This gets run with %si containing the 267 * function to call, and %di containing 268 * the "args". 269 */ 270extern void kernel_thread_helper(void); 271 272/* 273 * Create a kernel thread 274 */ 275int kernel_thread(int (*fn)(void *), void *arg, unsigned long flags) 276{ 277 struct pt_regs regs; 278 279 memset(®s, 0, sizeof(regs)); 280 281 regs.si = (unsigned long) fn; 282 regs.di = (unsigned long) arg; 283 284#ifdef CONFIG_X86_32 285 regs.ds = __USER_DS; 286 regs.es = __USER_DS; 287 regs.fs = __KERNEL_PERCPU; 288 regs.gs = __KERNEL_STACK_CANARY; 289#else 290 regs.ss = __KERNEL_DS; 291#endif 292 293 regs.orig_ax = -1; 294 regs.ip = (unsigned long) kernel_thread_helper; 295 regs.cs = __KERNEL_CS | get_kernel_rpl(); 296 regs.flags = X86_EFLAGS_IF | 0x2; 297 298 /* Ok, create the new process.. */ 299 return do_fork(flags | CLONE_VM | CLONE_UNTRACED, 0, ®s, 0, NULL, NULL); 300} 301EXPORT_SYMBOL(kernel_thread); 302 303/* 304 * sys_execve() executes a new program. 305 */ 306long sys_execve(const char __user *name, 307 const char __user *const __user *argv, 308 const char __user *const __user *envp, struct pt_regs *regs) 309{ 310 long error; 311 char *filename; 312 313 filename = getname(name); 314 error = PTR_ERR(filename); 315 if (IS_ERR(filename)) 316 return error; 317 error = do_execve(filename, argv, envp, regs); 318 319#ifdef CONFIG_X86_32 320 if (error == 0) { 321 /* Make sure we don't return using sysenter.. */ 322 set_thread_flag(TIF_IRET); 323 } 324#endif 325 326 putname(filename); 327 return error; 328} 329 330/* 331 * Idle related variables and functions 332 */ 333unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE; 334EXPORT_SYMBOL(boot_option_idle_override); 335 336/* 337 * Powermanagement idle function, if any.. 338 */ 339void (*pm_idle)(void); 340#ifdef CONFIG_APM_MODULE 341EXPORT_SYMBOL(pm_idle); 342#endif 343 344static inline int hlt_use_halt(void) 345{ 346 return 1; 347} 348 349/* 350 * We use this if we don't have any better 351 * idle routine.. 352 */ 353void default_idle(void) 354{ 355 if (hlt_use_halt()) { 356 trace_power_start(POWER_CSTATE, 1, smp_processor_id()); 357 trace_cpu_idle(1, smp_processor_id()); 358 current_thread_info()->status &= ~TS_POLLING; 359 /* 360 * TS_POLLING-cleared state must be visible before we 361 * test NEED_RESCHED: 362 */ 363 smp_mb(); 364 365 if (!need_resched()) 366 safe_halt(); /* enables interrupts racelessly */ 367 else 368 local_irq_enable(); 369 current_thread_info()->status |= TS_POLLING; 370 trace_power_end(smp_processor_id()); 371 trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id()); 372 } else { 373 local_irq_enable(); 374 /* loop is done by the caller */ 375 cpu_relax(); 376 } 377} 378#ifdef CONFIG_APM_MODULE 379EXPORT_SYMBOL(default_idle); 380#endif 381 382void stop_this_cpu(void *dummy) 383{ 384 local_irq_disable(); 385 /* 386 * Remove this CPU: 387 */ 388 set_cpu_online(smp_processor_id(), false); 389 disable_local_APIC(); 390 391 for (;;) { 392 if (hlt_works(smp_processor_id())) 393 halt(); 394 } 395} 396 397static void do_nothing(void *unused) 398{ 399} 400 401/* 402 * cpu_idle_wait - Used to ensure that all the CPUs discard old value of 403 * pm_idle and update to new pm_idle value. Required while changing pm_idle 404 * handler on SMP systems. 405 * 406 * Caller must have changed pm_idle to the new value before the call. Old 407 * pm_idle value will not be used by any CPU after the return of this function. 408 */ 409void cpu_idle_wait(void) 410{ 411 smp_mb(); 412 /* kick all the CPUs so that they exit out of pm_idle */ 413 smp_call_function(do_nothing, NULL, 1); 414} 415EXPORT_SYMBOL_GPL(cpu_idle_wait); 416 417/* 418 * This uses new MONITOR/MWAIT instructions on P4 processors with PNI, 419 * which can obviate IPI to trigger checking of need_resched. 420 * We execute MONITOR against need_resched and enter optimized wait state 421 * through MWAIT. Whenever someone changes need_resched, we would be woken 422 * up from MWAIT (without an IPI). 423 * 424 * New with Core Duo processors, MWAIT can take some hints based on CPU 425 * capability. 426 */ 427void mwait_idle_with_hints(unsigned long ax, unsigned long cx) 428{ 429 if (!need_resched()) { 430 if (this_cpu_has(X86_FEATURE_CLFLUSH_MONITOR)) 431 clflush((void *)¤t_thread_info()->flags); 432 433 __monitor((void *)¤t_thread_info()->flags, 0, 0); 434 smp_mb(); 435 if (!need_resched()) 436 __mwait(ax, cx); 437 } 438} 439 440/* Default MONITOR/MWAIT with no hints, used for default C1 state */ 441static void mwait_idle(void) 442{ 443 if (!need_resched()) { 444 trace_power_start(POWER_CSTATE, 1, smp_processor_id()); 445 trace_cpu_idle(1, smp_processor_id()); 446 if (this_cpu_has(X86_FEATURE_CLFLUSH_MONITOR)) 447 clflush((void *)¤t_thread_info()->flags); 448 449 __monitor((void *)¤t_thread_info()->flags, 0, 0); 450 smp_mb(); 451 if (!need_resched()) 452 __sti_mwait(0, 0); 453 else 454 local_irq_enable(); 455 trace_power_end(smp_processor_id()); 456 trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id()); 457 } else 458 local_irq_enable(); 459} 460 461/* 462 * On SMP it's slightly faster (but much more power-consuming!) 463 * to poll the ->work.need_resched flag instead of waiting for the 464 * cross-CPU IPI to arrive. Use this option with caution. 465 */ 466static void poll_idle(void) 467{ 468 trace_power_start(POWER_CSTATE, 0, smp_processor_id()); 469 trace_cpu_idle(0, smp_processor_id()); 470 local_irq_enable(); 471 while (!need_resched()) 472 cpu_relax(); 473 trace_power_end(smp_processor_id()); 474 trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id()); 475} 476 477/* 478 * mwait selection logic: 479 * 480 * It depends on the CPU. For AMD CPUs that support MWAIT this is 481 * wrong. Family 0x10 and 0x11 CPUs will enter C1 on HLT. Powersavings 482 * then depend on a clock divisor and current Pstate of the core. If 483 * all cores of a processor are in halt state (C1) the processor can 484 * enter the C1E (C1 enhanced) state. If mwait is used this will never 485 * happen. 486 * 487 * idle=mwait overrides this decision and forces the usage of mwait. 488 */ 489 490#define MWAIT_INFO 0x05 491#define MWAIT_ECX_EXTENDED_INFO 0x01 492#define MWAIT_EDX_C1 0xf0 493 494int mwait_usable(const struct cpuinfo_x86 *c) 495{ 496 u32 eax, ebx, ecx, edx; 497 498 if (boot_option_idle_override == IDLE_FORCE_MWAIT) 499 return 1; 500 501 if (c->cpuid_level < MWAIT_INFO) 502 return 0; 503 504 cpuid(MWAIT_INFO, &eax, &ebx, &ecx, &edx); 505 /* Check, whether EDX has extended info about MWAIT */ 506 if (!(ecx & MWAIT_ECX_EXTENDED_INFO)) 507 return 1; 508 509 /* 510 * edx enumeratios MONITOR/MWAIT extensions. Check, whether 511 * C1 supports MWAIT 512 */ 513 return (edx & MWAIT_EDX_C1); 514} 515 516bool amd_e400_c1e_detected; 517EXPORT_SYMBOL(amd_e400_c1e_detected); 518 519static cpumask_var_t amd_e400_c1e_mask; 520 521void amd_e400_remove_cpu(int cpu) 522{ 523 if (amd_e400_c1e_mask != NULL) 524 cpumask_clear_cpu(cpu, amd_e400_c1e_mask); 525} 526 527/* 528 * AMD Erratum 400 aware idle routine. We check for C1E active in the interrupt 529 * pending message MSR. If we detect C1E, then we handle it the same 530 * way as C3 power states (local apic timer and TSC stop) 531 */ 532static void amd_e400_idle(void) 533{ 534 if (need_resched()) 535 return; 536 537 if (!amd_e400_c1e_detected) { 538 u32 lo, hi; 539 540 rdmsr(MSR_K8_INT_PENDING_MSG, lo, hi); 541 542 if (lo & K8_INTP_C1E_ACTIVE_MASK) { 543 amd_e400_c1e_detected = true; 544 if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) 545 mark_tsc_unstable("TSC halt in AMD C1E"); 546 printk(KERN_INFO "System has AMD C1E enabled\n"); 547 } 548 } 549 550 if (amd_e400_c1e_detected) { 551 int cpu = smp_processor_id(); 552 553 if (!cpumask_test_cpu(cpu, amd_e400_c1e_mask)) { 554 cpumask_set_cpu(cpu, amd_e400_c1e_mask); 555 /* 556 * Force broadcast so ACPI can not interfere. 557 */ 558 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_FORCE, 559 &cpu); 560 printk(KERN_INFO "Switch to broadcast mode on CPU%d\n", 561 cpu); 562 } 563 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &cpu); 564 565 default_idle(); 566 567 /* 568 * The switch back from broadcast mode needs to be 569 * called with interrupts disabled. 570 */ 571 local_irq_disable(); 572 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &cpu); 573 local_irq_enable(); 574 } else 575 default_idle(); 576} 577 578void __cpuinit select_idle_routine(const struct cpuinfo_x86 *c) 579{ 580#ifdef CONFIG_SMP 581 if (pm_idle == poll_idle && smp_num_siblings > 1) { 582 printk_once(KERN_WARNING "WARNING: polling idle and HT enabled," 583 " performance may degrade.\n"); 584 } 585#endif 586 if (pm_idle) 587 return; 588 589 if (cpu_has(c, X86_FEATURE_MWAIT) && mwait_usable(c)) { 590 /* 591 * One CPU supports mwait => All CPUs supports mwait 592 */ 593 printk(KERN_INFO "using mwait in idle threads.\n"); 594 pm_idle = mwait_idle; 595 } else if (cpu_has_amd_erratum(amd_erratum_400)) { 596 /* E400: APIC timer interrupt does not wake up CPU from C1e */ 597 printk(KERN_INFO "using AMD E400 aware idle routine\n"); 598 pm_idle = amd_e400_idle; 599 } else 600 pm_idle = default_idle; 601} 602 603void __init init_amd_e400_c1e_mask(void) 604{ 605 /* If we're using amd_e400_idle, we need to allocate amd_e400_c1e_mask. */ 606 if (pm_idle == amd_e400_idle) 607 zalloc_cpumask_var(&amd_e400_c1e_mask, GFP_KERNEL); 608} 609 610static int __init idle_setup(char *str) 611{ 612 if (!str) 613 return -EINVAL; 614 615 if (!strcmp(str, "poll")) { 616 printk("using polling idle threads.\n"); 617 pm_idle = poll_idle; 618 boot_option_idle_override = IDLE_POLL; 619 } else if (!strcmp(str, "mwait")) { 620 boot_option_idle_override = IDLE_FORCE_MWAIT; 621 WARN_ONCE(1, "\"idle=mwait\" will be removed in 2012\n"); 622 } else if (!strcmp(str, "halt")) { 623 /* 624 * When the boot option of idle=halt is added, halt is 625 * forced to be used for CPU idle. In such case CPU C2/C3 626 * won't be used again. 627 * To continue to load the CPU idle driver, don't touch 628 * the boot_option_idle_override. 629 */ 630 pm_idle = default_idle; 631 boot_option_idle_override = IDLE_HALT; 632 } else if (!strcmp(str, "nomwait")) { 633 /* 634 * If the boot option of "idle=nomwait" is added, 635 * it means that mwait will be disabled for CPU C2/C3 636 * states. In such case it won't touch the variable 637 * of boot_option_idle_override. 638 */ 639 boot_option_idle_override = IDLE_NOMWAIT; 640 } else 641 return -1; 642 643 return 0; 644} 645early_param("idle", idle_setup); 646 647unsigned long arch_align_stack(unsigned long sp) 648{ 649 if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space) 650 sp -= get_random_int() % 8192; 651 return sp & ~0xf; 652} 653 654unsigned long arch_randomize_brk(struct mm_struct *mm) 655{ 656 unsigned long range_end = mm->brk + 0x02000000; 657 return randomize_range(mm->brk, range_end, 0) ? : mm->brk; 658} 659