PageRenderTime 49ms CodeModel.GetById 12ms RepoModel.GetById 0ms app.codeStats 0ms

/arch/x86/kernel/ftrace.c

https://bitbucket.org/cresqo/cm7-p500-kernel
C | 512 lines | 303 code | 97 blank | 112 comment | 29 complexity | d5cf04d0dab0508041544bb427388970 MD5 | raw file
Possible License(s): LGPL-2.0, AGPL-1.0, GPL-2.0
  1. /*
  2. * Code for replacing ftrace calls with jumps.
  3. *
  4. * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
  5. *
  6. * Thanks goes to Ingo Molnar, for suggesting the idea.
  7. * Mathieu Desnoyers, for suggesting postponing the modifications.
  8. * Arjan van de Ven, for keeping me straight, and explaining to me
  9. * the dangers of modifying code on the run.
  10. */
  11. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  12. #include <linux/spinlock.h>
  13. #include <linux/hardirq.h>
  14. #include <linux/uaccess.h>
  15. #include <linux/ftrace.h>
  16. #include <linux/percpu.h>
  17. #include <linux/sched.h>
  18. #include <linux/init.h>
  19. #include <linux/list.h>
  20. #include <trace/syscall.h>
  21. #include <asm/cacheflush.h>
  22. #include <asm/ftrace.h>
  23. #include <asm/nops.h>
  24. #include <asm/nmi.h>
  25. #ifdef CONFIG_DYNAMIC_FTRACE
  26. /*
  27. * modifying_code is set to notify NMIs that they need to use
  28. * memory barriers when entering or exiting. But we don't want
  29. * to burden NMIs with unnecessary memory barriers when code
  30. * modification is not being done (which is most of the time).
  31. *
  32. * A mutex is already held when ftrace_arch_code_modify_prepare
  33. * and post_process are called. No locks need to be taken here.
  34. *
  35. * Stop machine will make sure currently running NMIs are done
  36. * and new NMIs will see the updated variable before we need
  37. * to worry about NMIs doing memory barriers.
  38. */
  39. static int modifying_code __read_mostly;
  40. static DEFINE_PER_CPU(int, save_modifying_code);
  41. int ftrace_arch_code_modify_prepare(void)
  42. {
  43. set_kernel_text_rw();
  44. modifying_code = 1;
  45. return 0;
  46. }
  47. int ftrace_arch_code_modify_post_process(void)
  48. {
  49. modifying_code = 0;
  50. set_kernel_text_ro();
  51. return 0;
  52. }
  53. union ftrace_code_union {
  54. char code[MCOUNT_INSN_SIZE];
  55. struct {
  56. char e8;
  57. int offset;
  58. } __attribute__((packed));
  59. };
  60. static int ftrace_calc_offset(long ip, long addr)
  61. {
  62. return (int)(addr - ip);
  63. }
  64. static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
  65. {
  66. static union ftrace_code_union calc;
  67. calc.e8 = 0xe8;
  68. calc.offset = ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
  69. /*
  70. * No locking needed, this must be called via kstop_machine
  71. * which in essence is like running on a uniprocessor machine.
  72. */
  73. return calc.code;
  74. }
  75. /*
  76. * Modifying code must take extra care. On an SMP machine, if
  77. * the code being modified is also being executed on another CPU
  78. * that CPU will have undefined results and possibly take a GPF.
  79. * We use kstop_machine to stop other CPUS from exectuing code.
  80. * But this does not stop NMIs from happening. We still need
  81. * to protect against that. We separate out the modification of
  82. * the code to take care of this.
  83. *
  84. * Two buffers are added: An IP buffer and a "code" buffer.
  85. *
  86. * 1) Put the instruction pointer into the IP buffer
  87. * and the new code into the "code" buffer.
  88. * 2) Wait for any running NMIs to finish and set a flag that says
  89. * we are modifying code, it is done in an atomic operation.
  90. * 3) Write the code
  91. * 4) clear the flag.
  92. * 5) Wait for any running NMIs to finish.
  93. *
  94. * If an NMI is executed, the first thing it does is to call
  95. * "ftrace_nmi_enter". This will check if the flag is set to write
  96. * and if it is, it will write what is in the IP and "code" buffers.
  97. *
  98. * The trick is, it does not matter if everyone is writing the same
  99. * content to the code location. Also, if a CPU is executing code
  100. * it is OK to write to that code location if the contents being written
  101. * are the same as what exists.
  102. */
  103. #define MOD_CODE_WRITE_FLAG (1 << 31) /* set when NMI should do the write */
  104. static atomic_t nmi_running = ATOMIC_INIT(0);
  105. static int mod_code_status; /* holds return value of text write */
  106. static void *mod_code_ip; /* holds the IP to write to */
  107. static void *mod_code_newcode; /* holds the text to write to the IP */
  108. static unsigned nmi_wait_count;
  109. static atomic_t nmi_update_count = ATOMIC_INIT(0);
  110. int ftrace_arch_read_dyn_info(char *buf, int size)
  111. {
  112. int r;
  113. r = snprintf(buf, size, "%u %u",
  114. nmi_wait_count,
  115. atomic_read(&nmi_update_count));
  116. return r;
  117. }
  118. static void clear_mod_flag(void)
  119. {
  120. int old = atomic_read(&nmi_running);
  121. for (;;) {
  122. int new = old & ~MOD_CODE_WRITE_FLAG;
  123. if (old == new)
  124. break;
  125. old = atomic_cmpxchg(&nmi_running, old, new);
  126. }
  127. }
  128. static void ftrace_mod_code(void)
  129. {
  130. /*
  131. * Yes, more than one CPU process can be writing to mod_code_status.
  132. * (and the code itself)
  133. * But if one were to fail, then they all should, and if one were
  134. * to succeed, then they all should.
  135. */
  136. mod_code_status = probe_kernel_write(mod_code_ip, mod_code_newcode,
  137. MCOUNT_INSN_SIZE);
  138. /* if we fail, then kill any new writers */
  139. if (mod_code_status)
  140. clear_mod_flag();
  141. }
  142. void ftrace_nmi_enter(void)
  143. {
  144. __get_cpu_var(save_modifying_code) = modifying_code;
  145. if (!__get_cpu_var(save_modifying_code))
  146. return;
  147. if (atomic_inc_return(&nmi_running) & MOD_CODE_WRITE_FLAG) {
  148. smp_rmb();
  149. ftrace_mod_code();
  150. atomic_inc(&nmi_update_count);
  151. }
  152. /* Must have previous changes seen before executions */
  153. smp_mb();
  154. }
  155. void ftrace_nmi_exit(void)
  156. {
  157. if (!__get_cpu_var(save_modifying_code))
  158. return;
  159. /* Finish all executions before clearing nmi_running */
  160. smp_mb();
  161. atomic_dec(&nmi_running);
  162. }
  163. static void wait_for_nmi_and_set_mod_flag(void)
  164. {
  165. if (!atomic_cmpxchg(&nmi_running, 0, MOD_CODE_WRITE_FLAG))
  166. return;
  167. do {
  168. cpu_relax();
  169. } while (atomic_cmpxchg(&nmi_running, 0, MOD_CODE_WRITE_FLAG));
  170. nmi_wait_count++;
  171. }
  172. static void wait_for_nmi(void)
  173. {
  174. if (!atomic_read(&nmi_running))
  175. return;
  176. do {
  177. cpu_relax();
  178. } while (atomic_read(&nmi_running));
  179. nmi_wait_count++;
  180. }
  181. static inline int
  182. within(unsigned long addr, unsigned long start, unsigned long end)
  183. {
  184. return addr >= start && addr < end;
  185. }
  186. static int
  187. do_ftrace_mod_code(unsigned long ip, void *new_code)
  188. {
  189. /*
  190. * On x86_64, kernel text mappings are mapped read-only with
  191. * CONFIG_DEBUG_RODATA. So we use the kernel identity mapping instead
  192. * of the kernel text mapping to modify the kernel text.
  193. *
  194. * For 32bit kernels, these mappings are same and we can use
  195. * kernel identity mapping to modify code.
  196. */
  197. if (within(ip, (unsigned long)_text, (unsigned long)_etext))
  198. ip = (unsigned long)__va(__pa(ip));
  199. mod_code_ip = (void *)ip;
  200. mod_code_newcode = new_code;
  201. /* The buffers need to be visible before we let NMIs write them */
  202. smp_mb();
  203. wait_for_nmi_and_set_mod_flag();
  204. /* Make sure all running NMIs have finished before we write the code */
  205. smp_mb();
  206. ftrace_mod_code();
  207. /* Make sure the write happens before clearing the bit */
  208. smp_mb();
  209. clear_mod_flag();
  210. wait_for_nmi();
  211. return mod_code_status;
  212. }
  213. static unsigned char ftrace_nop[MCOUNT_INSN_SIZE];
  214. static unsigned char *ftrace_nop_replace(void)
  215. {
  216. return ftrace_nop;
  217. }
  218. static int
  219. ftrace_modify_code(unsigned long ip, unsigned char *old_code,
  220. unsigned char *new_code)
  221. {
  222. unsigned char replaced[MCOUNT_INSN_SIZE];
  223. /*
  224. * Note: Due to modules and __init, code can
  225. * disappear and change, we need to protect against faulting
  226. * as well as code changing. We do this by using the
  227. * probe_kernel_* functions.
  228. *
  229. * No real locking needed, this code is run through
  230. * kstop_machine, or before SMP starts.
  231. */
  232. /* read the text we want to modify */
  233. if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
  234. return -EFAULT;
  235. /* Make sure it is what we expect it to be */
  236. if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0)
  237. return -EINVAL;
  238. /* replace the text with the new text */
  239. if (do_ftrace_mod_code(ip, new_code))
  240. return -EPERM;
  241. sync_core();
  242. return 0;
  243. }
  244. int ftrace_make_nop(struct module *mod,
  245. struct dyn_ftrace *rec, unsigned long addr)
  246. {
  247. unsigned char *new, *old;
  248. unsigned long ip = rec->ip;
  249. old = ftrace_call_replace(ip, addr);
  250. new = ftrace_nop_replace();
  251. return ftrace_modify_code(rec->ip, old, new);
  252. }
  253. int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
  254. {
  255. unsigned char *new, *old;
  256. unsigned long ip = rec->ip;
  257. old = ftrace_nop_replace();
  258. new = ftrace_call_replace(ip, addr);
  259. return ftrace_modify_code(rec->ip, old, new);
  260. }
  261. int ftrace_update_ftrace_func(ftrace_func_t func)
  262. {
  263. unsigned long ip = (unsigned long)(&ftrace_call);
  264. unsigned char old[MCOUNT_INSN_SIZE], *new;
  265. int ret;
  266. memcpy(old, &ftrace_call, MCOUNT_INSN_SIZE);
  267. new = ftrace_call_replace(ip, (unsigned long)func);
  268. ret = ftrace_modify_code(ip, old, new);
  269. return ret;
  270. }
  271. int __init ftrace_dyn_arch_init(void *data)
  272. {
  273. extern const unsigned char ftrace_test_p6nop[];
  274. extern const unsigned char ftrace_test_nop5[];
  275. extern const unsigned char ftrace_test_jmp[];
  276. int faulted = 0;
  277. /*
  278. * There is no good nop for all x86 archs.
  279. * We will default to using the P6_NOP5, but first we
  280. * will test to make sure that the nop will actually
  281. * work on this CPU. If it faults, we will then
  282. * go to a lesser efficient 5 byte nop. If that fails
  283. * we then just use a jmp as our nop. This isn't the most
  284. * efficient nop, but we can not use a multi part nop
  285. * since we would then risk being preempted in the middle
  286. * of that nop, and if we enabled tracing then, it might
  287. * cause a system crash.
  288. *
  289. * TODO: check the cpuid to determine the best nop.
  290. */
  291. asm volatile (
  292. "ftrace_test_jmp:"
  293. "jmp ftrace_test_p6nop\n"
  294. "nop\n"
  295. "nop\n"
  296. "nop\n" /* 2 byte jmp + 3 bytes */
  297. "ftrace_test_p6nop:"
  298. P6_NOP5
  299. "jmp 1f\n"
  300. "ftrace_test_nop5:"
  301. ".byte 0x66,0x66,0x66,0x66,0x90\n"
  302. "1:"
  303. ".section .fixup, \"ax\"\n"
  304. "2: movl $1, %0\n"
  305. " jmp ftrace_test_nop5\n"
  306. "3: movl $2, %0\n"
  307. " jmp 1b\n"
  308. ".previous\n"
  309. _ASM_EXTABLE(ftrace_test_p6nop, 2b)
  310. _ASM_EXTABLE(ftrace_test_nop5, 3b)
  311. : "=r"(faulted) : "0" (faulted));
  312. switch (faulted) {
  313. case 0:
  314. pr_info("converting mcount calls to 0f 1f 44 00 00\n");
  315. memcpy(ftrace_nop, ftrace_test_p6nop, MCOUNT_INSN_SIZE);
  316. break;
  317. case 1:
  318. pr_info("converting mcount calls to 66 66 66 66 90\n");
  319. memcpy(ftrace_nop, ftrace_test_nop5, MCOUNT_INSN_SIZE);
  320. break;
  321. case 2:
  322. pr_info("converting mcount calls to jmp . + 5\n");
  323. memcpy(ftrace_nop, ftrace_test_jmp, MCOUNT_INSN_SIZE);
  324. break;
  325. }
  326. /* The return code is retured via data */
  327. *(unsigned long *)data = 0;
  328. return 0;
  329. }
  330. #endif
  331. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  332. #ifdef CONFIG_DYNAMIC_FTRACE
  333. extern void ftrace_graph_call(void);
  334. static int ftrace_mod_jmp(unsigned long ip,
  335. int old_offset, int new_offset)
  336. {
  337. unsigned char code[MCOUNT_INSN_SIZE];
  338. if (probe_kernel_read(code, (void *)ip, MCOUNT_INSN_SIZE))
  339. return -EFAULT;
  340. if (code[0] != 0xe9 || old_offset != *(int *)(&code[1]))
  341. return -EINVAL;
  342. *(int *)(&code[1]) = new_offset;
  343. if (do_ftrace_mod_code(ip, &code))
  344. return -EPERM;
  345. return 0;
  346. }
  347. int ftrace_enable_ftrace_graph_caller(void)
  348. {
  349. unsigned long ip = (unsigned long)(&ftrace_graph_call);
  350. int old_offset, new_offset;
  351. old_offset = (unsigned long)(&ftrace_stub) - (ip + MCOUNT_INSN_SIZE);
  352. new_offset = (unsigned long)(&ftrace_graph_caller) - (ip + MCOUNT_INSN_SIZE);
  353. return ftrace_mod_jmp(ip, old_offset, new_offset);
  354. }
  355. int ftrace_disable_ftrace_graph_caller(void)
  356. {
  357. unsigned long ip = (unsigned long)(&ftrace_graph_call);
  358. int old_offset, new_offset;
  359. old_offset = (unsigned long)(&ftrace_graph_caller) - (ip + MCOUNT_INSN_SIZE);
  360. new_offset = (unsigned long)(&ftrace_stub) - (ip + MCOUNT_INSN_SIZE);
  361. return ftrace_mod_jmp(ip, old_offset, new_offset);
  362. }
  363. #endif /* !CONFIG_DYNAMIC_FTRACE */
  364. /*
  365. * Hook the return address and push it in the stack of return addrs
  366. * in current thread info.
  367. */
  368. void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
  369. unsigned long frame_pointer)
  370. {
  371. unsigned long old;
  372. int faulted;
  373. struct ftrace_graph_ent trace;
  374. unsigned long return_hooker = (unsigned long)
  375. &return_to_handler;
  376. if (unlikely(atomic_read(&current->tracing_graph_pause)))
  377. return;
  378. /*
  379. * Protect against fault, even if it shouldn't
  380. * happen. This tool is too much intrusive to
  381. * ignore such a protection.
  382. */
  383. asm volatile(
  384. "1: " _ASM_MOV " (%[parent]), %[old]\n"
  385. "2: " _ASM_MOV " %[return_hooker], (%[parent])\n"
  386. " movl $0, %[faulted]\n"
  387. "3:\n"
  388. ".section .fixup, \"ax\"\n"
  389. "4: movl $1, %[faulted]\n"
  390. " jmp 3b\n"
  391. ".previous\n"
  392. _ASM_EXTABLE(1b, 4b)
  393. _ASM_EXTABLE(2b, 4b)
  394. : [old] "=&r" (old), [faulted] "=r" (faulted)
  395. : [parent] "r" (parent), [return_hooker] "r" (return_hooker)
  396. : "memory"
  397. );
  398. if (unlikely(faulted)) {
  399. ftrace_graph_stop();
  400. WARN_ON(1);
  401. return;
  402. }
  403. if (ftrace_push_return_trace(old, self_addr, &trace.depth,
  404. frame_pointer) == -EBUSY) {
  405. *parent = old;
  406. return;
  407. }
  408. trace.func = self_addr;
  409. /* Only trace if the calling function expects to */
  410. if (!ftrace_graph_entry(&trace)) {
  411. current->curr_ret_stack--;
  412. *parent = old;
  413. }
  414. }
  415. #endif /* CONFIG_FUNCTION_GRAPH_TRACER */