/arch/x86/include/asm/system.h

https://bitbucket.org/ndreys/linux-sunxi · C++ Header · 515 lines · 345 code · 64 blank · 106 comment · 10 complexity · 71c886b51b7a273e949c162ecde7ee68 MD5 · raw file

  1. #ifndef _ASM_X86_SYSTEM_H
  2. #define _ASM_X86_SYSTEM_H
  3. #include <asm/asm.h>
  4. #include <asm/segment.h>
  5. #include <asm/cpufeature.h>
  6. #include <asm/cmpxchg.h>
  7. #include <asm/nops.h>
  8. #include <linux/kernel.h>
  9. #include <linux/irqflags.h>
  10. /* entries in ARCH_DLINFO: */
  11. #if defined(CONFIG_IA32_EMULATION) || !defined(CONFIG_X86_64)
  12. # define AT_VECTOR_SIZE_ARCH 2
  13. #else /* else it's non-compat x86-64 */
  14. # define AT_VECTOR_SIZE_ARCH 1
  15. #endif
  16. struct task_struct; /* one of the stranger aspects of C forward declarations */
  17. struct task_struct *__switch_to(struct task_struct *prev,
  18. struct task_struct *next);
  19. struct tss_struct;
  20. void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
  21. struct tss_struct *tss);
  22. extern void show_regs_common(void);
  23. #ifdef CONFIG_X86_32
  24. #ifdef CONFIG_CC_STACKPROTECTOR
  25. #define __switch_canary \
  26. "movl %P[task_canary](%[next]), %%ebx\n\t" \
  27. "movl %%ebx, "__percpu_arg([stack_canary])"\n\t"
  28. #define __switch_canary_oparam \
  29. , [stack_canary] "=m" (stack_canary.canary)
  30. #define __switch_canary_iparam \
  31. , [task_canary] "i" (offsetof(struct task_struct, stack_canary))
  32. #else /* CC_STACKPROTECTOR */
  33. #define __switch_canary
  34. #define __switch_canary_oparam
  35. #define __switch_canary_iparam
  36. #endif /* CC_STACKPROTECTOR */
  37. /*
  38. * Saving eflags is important. It switches not only IOPL between tasks,
  39. * it also protects other tasks from NT leaking through sysenter etc.
  40. */
  41. #define switch_to(prev, next, last) \
  42. do { \
  43. /* \
  44. * Context-switching clobbers all registers, so we clobber \
  45. * them explicitly, via unused output variables. \
  46. * (EAX and EBP is not listed because EBP is saved/restored \
  47. * explicitly for wchan access and EAX is the return value of \
  48. * __switch_to()) \
  49. */ \
  50. unsigned long ebx, ecx, edx, esi, edi; \
  51. \
  52. asm volatile("pushfl\n\t" /* save flags */ \
  53. "pushl %%ebp\n\t" /* save EBP */ \
  54. "movl %%esp,%[prev_sp]\n\t" /* save ESP */ \
  55. "movl %[next_sp],%%esp\n\t" /* restore ESP */ \
  56. "movl $1f,%[prev_ip]\n\t" /* save EIP */ \
  57. "pushl %[next_ip]\n\t" /* restore EIP */ \
  58. __switch_canary \
  59. "jmp __switch_to\n" /* regparm call */ \
  60. "1:\t" \
  61. "popl %%ebp\n\t" /* restore EBP */ \
  62. "popfl\n" /* restore flags */ \
  63. \
  64. /* output parameters */ \
  65. : [prev_sp] "=m" (prev->thread.sp), \
  66. [prev_ip] "=m" (prev->thread.ip), \
  67. "=a" (last), \
  68. \
  69. /* clobbered output registers: */ \
  70. "=b" (ebx), "=c" (ecx), "=d" (edx), \
  71. "=S" (esi), "=D" (edi) \
  72. \
  73. __switch_canary_oparam \
  74. \
  75. /* input parameters: */ \
  76. : [next_sp] "m" (next->thread.sp), \
  77. [next_ip] "m" (next->thread.ip), \
  78. \
  79. /* regparm parameters for __switch_to(): */ \
  80. [prev] "a" (prev), \
  81. [next] "d" (next) \
  82. \
  83. __switch_canary_iparam \
  84. \
  85. : /* reloaded segment registers */ \
  86. "memory"); \
  87. } while (0)
  88. #else
  89. /* frame pointer must be last for get_wchan */
  90. #define SAVE_CONTEXT "pushf ; pushq %%rbp ; movq %%rsi,%%rbp\n\t"
  91. #define RESTORE_CONTEXT "movq %%rbp,%%rsi ; popq %%rbp ; popf\t"
  92. #define __EXTRA_CLOBBER \
  93. , "rcx", "rbx", "rdx", "r8", "r9", "r10", "r11", \
  94. "r12", "r13", "r14", "r15"
  95. #ifdef CONFIG_CC_STACKPROTECTOR
  96. #define __switch_canary \
  97. "movq %P[task_canary](%%rsi),%%r8\n\t" \
  98. "movq %%r8,"__percpu_arg([gs_canary])"\n\t"
  99. #define __switch_canary_oparam \
  100. , [gs_canary] "=m" (irq_stack_union.stack_canary)
  101. #define __switch_canary_iparam \
  102. , [task_canary] "i" (offsetof(struct task_struct, stack_canary))
  103. #else /* CC_STACKPROTECTOR */
  104. #define __switch_canary
  105. #define __switch_canary_oparam
  106. #define __switch_canary_iparam
  107. #endif /* CC_STACKPROTECTOR */
  108. /* Save restore flags to clear handle leaking NT */
  109. #define switch_to(prev, next, last) \
  110. asm volatile(SAVE_CONTEXT \
  111. "movq %%rsp,%P[threadrsp](%[prev])\n\t" /* save RSP */ \
  112. "movq %P[threadrsp](%[next]),%%rsp\n\t" /* restore RSP */ \
  113. "call __switch_to\n\t" \
  114. "movq "__percpu_arg([current_task])",%%rsi\n\t" \
  115. __switch_canary \
  116. "movq %P[thread_info](%%rsi),%%r8\n\t" \
  117. "movq %%rax,%%rdi\n\t" \
  118. "testl %[_tif_fork],%P[ti_flags](%%r8)\n\t" \
  119. "jnz ret_from_fork\n\t" \
  120. RESTORE_CONTEXT \
  121. : "=a" (last) \
  122. __switch_canary_oparam \
  123. : [next] "S" (next), [prev] "D" (prev), \
  124. [threadrsp] "i" (offsetof(struct task_struct, thread.sp)), \
  125. [ti_flags] "i" (offsetof(struct thread_info, flags)), \
  126. [_tif_fork] "i" (_TIF_FORK), \
  127. [thread_info] "i" (offsetof(struct task_struct, stack)), \
  128. [current_task] "m" (current_task) \
  129. __switch_canary_iparam \
  130. : "memory", "cc" __EXTRA_CLOBBER)
  131. #endif
  132. #ifdef __KERNEL__
  133. extern void native_load_gs_index(unsigned);
  134. /*
  135. * Load a segment. Fall back on loading the zero
  136. * segment if something goes wrong..
  137. */
  138. #define loadsegment(seg, value) \
  139. do { \
  140. unsigned short __val = (value); \
  141. \
  142. asm volatile(" \n" \
  143. "1: movl %k0,%%" #seg " \n" \
  144. \
  145. ".section .fixup,\"ax\" \n" \
  146. "2: xorl %k0,%k0 \n" \
  147. " jmp 1b \n" \
  148. ".previous \n" \
  149. \
  150. _ASM_EXTABLE(1b, 2b) \
  151. \
  152. : "+r" (__val) : : "memory"); \
  153. } while (0)
  154. /*
  155. * Save a segment register away
  156. */
  157. #define savesegment(seg, value) \
  158. asm("mov %%" #seg ",%0":"=r" (value) : : "memory")
  159. /*
  160. * x86_32 user gs accessors.
  161. */
  162. #ifdef CONFIG_X86_32
  163. #ifdef CONFIG_X86_32_LAZY_GS
  164. #define get_user_gs(regs) (u16)({unsigned long v; savesegment(gs, v); v;})
  165. #define set_user_gs(regs, v) loadsegment(gs, (unsigned long)(v))
  166. #define task_user_gs(tsk) ((tsk)->thread.gs)
  167. #define lazy_save_gs(v) savesegment(gs, (v))
  168. #define lazy_load_gs(v) loadsegment(gs, (v))
  169. #else /* X86_32_LAZY_GS */
  170. #define get_user_gs(regs) (u16)((regs)->gs)
  171. #define set_user_gs(regs, v) do { (regs)->gs = (v); } while (0)
  172. #define task_user_gs(tsk) (task_pt_regs(tsk)->gs)
  173. #define lazy_save_gs(v) do { } while (0)
  174. #define lazy_load_gs(v) do { } while (0)
  175. #endif /* X86_32_LAZY_GS */
  176. #endif /* X86_32 */
  177. static inline unsigned long get_limit(unsigned long segment)
  178. {
  179. unsigned long __limit;
  180. asm("lsll %1,%0" : "=r" (__limit) : "r" (segment));
  181. return __limit + 1;
  182. }
  183. static inline void native_clts(void)
  184. {
  185. asm volatile("clts");
  186. }
  187. /*
  188. * Volatile isn't enough to prevent the compiler from reordering the
  189. * read/write functions for the control registers and messing everything up.
  190. * A memory clobber would solve the problem, but would prevent reordering of
  191. * all loads stores around it, which can hurt performance. Solution is to
  192. * use a variable and mimic reads and writes to it to enforce serialization
  193. */
  194. static unsigned long __force_order;
  195. static inline unsigned long native_read_cr0(void)
  196. {
  197. unsigned long val;
  198. asm volatile("mov %%cr0,%0\n\t" : "=r" (val), "=m" (__force_order));
  199. return val;
  200. }
  201. static inline void native_write_cr0(unsigned long val)
  202. {
  203. asm volatile("mov %0,%%cr0": : "r" (val), "m" (__force_order));
  204. }
  205. static inline unsigned long native_read_cr2(void)
  206. {
  207. unsigned long val;
  208. asm volatile("mov %%cr2,%0\n\t" : "=r" (val), "=m" (__force_order));
  209. return val;
  210. }
  211. static inline void native_write_cr2(unsigned long val)
  212. {
  213. asm volatile("mov %0,%%cr2": : "r" (val), "m" (__force_order));
  214. }
  215. static inline unsigned long native_read_cr3(void)
  216. {
  217. unsigned long val;
  218. asm volatile("mov %%cr3,%0\n\t" : "=r" (val), "=m" (__force_order));
  219. return val;
  220. }
  221. static inline void native_write_cr3(unsigned long val)
  222. {
  223. asm volatile("mov %0,%%cr3": : "r" (val), "m" (__force_order));
  224. }
  225. static inline unsigned long native_read_cr4(void)
  226. {
  227. unsigned long val;
  228. asm volatile("mov %%cr4,%0\n\t" : "=r" (val), "=m" (__force_order));
  229. return val;
  230. }
  231. static inline unsigned long native_read_cr4_safe(void)
  232. {
  233. unsigned long val;
  234. /* This could fault if %cr4 does not exist. In x86_64, a cr4 always
  235. * exists, so it will never fail. */
  236. #ifdef CONFIG_X86_32
  237. asm volatile("1: mov %%cr4, %0\n"
  238. "2:\n"
  239. _ASM_EXTABLE(1b, 2b)
  240. : "=r" (val), "=m" (__force_order) : "0" (0));
  241. #else
  242. val = native_read_cr4();
  243. #endif
  244. return val;
  245. }
  246. static inline void native_write_cr4(unsigned long val)
  247. {
  248. asm volatile("mov %0,%%cr4": : "r" (val), "m" (__force_order));
  249. }
  250. #ifdef CONFIG_X86_64
  251. static inline unsigned long native_read_cr8(void)
  252. {
  253. unsigned long cr8;
  254. asm volatile("movq %%cr8,%0" : "=r" (cr8));
  255. return cr8;
  256. }
  257. static inline void native_write_cr8(unsigned long val)
  258. {
  259. asm volatile("movq %0,%%cr8" :: "r" (val) : "memory");
  260. }
  261. #endif
  262. static inline void native_wbinvd(void)
  263. {
  264. asm volatile("wbinvd": : :"memory");
  265. }
  266. #ifdef CONFIG_PARAVIRT
  267. #include <asm/paravirt.h>
  268. #else
  269. static inline unsigned long read_cr0(void)
  270. {
  271. return native_read_cr0();
  272. }
  273. static inline void write_cr0(unsigned long x)
  274. {
  275. native_write_cr0(x);
  276. }
  277. static inline unsigned long read_cr2(void)
  278. {
  279. return native_read_cr2();
  280. }
  281. static inline void write_cr2(unsigned long x)
  282. {
  283. native_write_cr2(x);
  284. }
  285. static inline unsigned long read_cr3(void)
  286. {
  287. return native_read_cr3();
  288. }
  289. static inline void write_cr3(unsigned long x)
  290. {
  291. native_write_cr3(x);
  292. }
  293. static inline unsigned long read_cr4(void)
  294. {
  295. return native_read_cr4();
  296. }
  297. static inline unsigned long read_cr4_safe(void)
  298. {
  299. return native_read_cr4_safe();
  300. }
  301. static inline void write_cr4(unsigned long x)
  302. {
  303. native_write_cr4(x);
  304. }
  305. static inline void wbinvd(void)
  306. {
  307. native_wbinvd();
  308. }
  309. #ifdef CONFIG_X86_64
  310. static inline unsigned long read_cr8(void)
  311. {
  312. return native_read_cr8();
  313. }
  314. static inline void write_cr8(unsigned long x)
  315. {
  316. native_write_cr8(x);
  317. }
  318. static inline void load_gs_index(unsigned selector)
  319. {
  320. native_load_gs_index(selector);
  321. }
  322. #endif
  323. /* Clear the 'TS' bit */
  324. static inline void clts(void)
  325. {
  326. native_clts();
  327. }
  328. #endif/* CONFIG_PARAVIRT */
  329. #define stts() write_cr0(read_cr0() | X86_CR0_TS)
  330. #endif /* __KERNEL__ */
  331. static inline void clflush(volatile void *__p)
  332. {
  333. asm volatile("clflush %0" : "+m" (*(volatile char __force *)__p));
  334. }
  335. #define nop() asm volatile ("nop")
  336. void cpu_idle_wait(void);
  337. extern unsigned long arch_align_stack(unsigned long sp);
  338. extern void free_init_pages(char *what, unsigned long begin, unsigned long end);
  339. void default_idle(void);
  340. void stop_this_cpu(void *dummy);
  341. /*
  342. * Force strict CPU ordering.
  343. * And yes, this is required on UP too when we're talking
  344. * to devices.
  345. */
  346. #ifdef CONFIG_X86_32
  347. /*
  348. * Some non-Intel clones support out of order store. wmb() ceases to be a
  349. * nop for these.
  350. */
  351. #define mb() alternative("lock; addl $0,0(%%esp)", "mfence", X86_FEATURE_XMM2)
  352. #define rmb() alternative("lock; addl $0,0(%%esp)", "lfence", X86_FEATURE_XMM2)
  353. #define wmb() alternative("lock; addl $0,0(%%esp)", "sfence", X86_FEATURE_XMM)
  354. #else
  355. #define mb() asm volatile("mfence":::"memory")
  356. #define rmb() asm volatile("lfence":::"memory")
  357. #define wmb() asm volatile("sfence" ::: "memory")
  358. #endif
  359. /**
  360. * read_barrier_depends - Flush all pending reads that subsequents reads
  361. * depend on.
  362. *
  363. * No data-dependent reads from memory-like regions are ever reordered
  364. * over this barrier. All reads preceding this primitive are guaranteed
  365. * to access memory (but not necessarily other CPUs' caches) before any
  366. * reads following this primitive that depend on the data return by
  367. * any of the preceding reads. This primitive is much lighter weight than
  368. * rmb() on most CPUs, and is never heavier weight than is
  369. * rmb().
  370. *
  371. * These ordering constraints are respected by both the local CPU
  372. * and the compiler.
  373. *
  374. * Ordering is not guaranteed by anything other than these primitives,
  375. * not even by data dependencies. See the documentation for
  376. * memory_barrier() for examples and URLs to more information.
  377. *
  378. * For example, the following code would force ordering (the initial
  379. * value of "a" is zero, "b" is one, and "p" is "&a"):
  380. *
  381. * <programlisting>
  382. * CPU 0 CPU 1
  383. *
  384. * b = 2;
  385. * memory_barrier();
  386. * p = &b; q = p;
  387. * read_barrier_depends();
  388. * d = *q;
  389. * </programlisting>
  390. *
  391. * because the read of "*q" depends on the read of "p" and these
  392. * two reads are separated by a read_barrier_depends(). However,
  393. * the following code, with the same initial values for "a" and "b":
  394. *
  395. * <programlisting>
  396. * CPU 0 CPU 1
  397. *
  398. * a = 2;
  399. * memory_barrier();
  400. * b = 3; y = b;
  401. * read_barrier_depends();
  402. * x = a;
  403. * </programlisting>
  404. *
  405. * does not enforce ordering, since there is no data dependency between
  406. * the read of "a" and the read of "b". Therefore, on some CPUs, such
  407. * as Alpha, "y" could be set to 3 and "x" to 0. Use rmb()
  408. * in cases like this where there are no data dependencies.
  409. **/
  410. #define read_barrier_depends() do { } while (0)
  411. #ifdef CONFIG_SMP
  412. #define smp_mb() mb()
  413. #ifdef CONFIG_X86_PPRO_FENCE
  414. # define smp_rmb() rmb()
  415. #else
  416. # define smp_rmb() barrier()
  417. #endif
  418. #ifdef CONFIG_X86_OOSTORE
  419. # define smp_wmb() wmb()
  420. #else
  421. # define smp_wmb() barrier()
  422. #endif
  423. #define smp_read_barrier_depends() read_barrier_depends()
  424. #define set_mb(var, value) do { (void)xchg(&var, value); } while (0)
  425. #else
  426. #define smp_mb() barrier()
  427. #define smp_rmb() barrier()
  428. #define smp_wmb() barrier()
  429. #define smp_read_barrier_depends() do { } while (0)
  430. #define set_mb(var, value) do { var = value; barrier(); } while (0)
  431. #endif
  432. /*
  433. * Stop RDTSC speculation. This is needed when you need to use RDTSC
  434. * (or get_cycles or vread that possibly accesses the TSC) in a defined
  435. * code region.
  436. *
  437. * (Could use an alternative three way for this if there was one.)
  438. */
  439. static __always_inline void rdtsc_barrier(void)
  440. {
  441. alternative(ASM_NOP3, "mfence", X86_FEATURE_MFENCE_RDTSC);
  442. alternative(ASM_NOP3, "lfence", X86_FEATURE_LFENCE_RDTSC);
  443. }
  444. /*
  445. * We handle most unaligned accesses in hardware. On the other hand
  446. * unaligned DMA can be quite expensive on some Nehalem processors.
  447. *
  448. * Based on this we disable the IP header alignment in network drivers.
  449. */
  450. #define NET_IP_ALIGN 0
  451. #endif /* _ASM_X86_SYSTEM_H */