/arch/x86_64/kernel/kprobes.c

https://bitbucket.org/evzijst/gittest · C · 631 lines · 413 code · 54 blank · 164 comment · 70 complexity · 184dd5179ed62ff60a4d20d2f9674d3c MD5 · raw file

  1. /*
  2. * Kernel Probes (KProbes)
  3. * arch/x86_64/kernel/kprobes.c
  4. *
  5. * This program is free software; you can redistribute it and/or modify
  6. * it under the terms of the GNU General Public License as published by
  7. * the Free Software Foundation; either version 2 of the License, or
  8. * (at your option) any later version.
  9. *
  10. * This program is distributed in the hope that it will be useful,
  11. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. * GNU General Public License for more details.
  14. *
  15. * You should have received a copy of the GNU General Public License
  16. * along with this program; if not, write to the Free Software
  17. * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18. *
  19. * Copyright (C) IBM Corporation, 2002, 2004
  20. *
  21. * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
  22. * Probes initial implementation ( includes contributions from
  23. * Rusty Russell).
  24. * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
  25. * interface to access function arguments.
  26. * 2004-Oct Jim Keniston <kenistoj@us.ibm.com> and Prasanna S Panchamukhi
  27. * <prasanna@in.ibm.com> adapted for x86_64
  28. * 2005-Mar Roland McGrath <roland@redhat.com>
  29. * Fixed to handle %rip-relative addressing mode correctly.
  30. */
  31. #include <linux/config.h>
  32. #include <linux/kprobes.h>
  33. #include <linux/ptrace.h>
  34. #include <linux/spinlock.h>
  35. #include <linux/string.h>
  36. #include <linux/slab.h>
  37. #include <linux/preempt.h>
  38. #include <linux/moduleloader.h>
  39. #include <asm/pgtable.h>
  40. #include <asm/kdebug.h>
  41. static DECLARE_MUTEX(kprobe_mutex);
  42. /* kprobe_status settings */
  43. #define KPROBE_HIT_ACTIVE 0x00000001
  44. #define KPROBE_HIT_SS 0x00000002
  45. static struct kprobe *current_kprobe;
  46. static unsigned long kprobe_status, kprobe_old_rflags, kprobe_saved_rflags;
  47. static struct pt_regs jprobe_saved_regs;
  48. static long *jprobe_saved_rsp;
  49. static kprobe_opcode_t *get_insn_slot(void);
  50. static void free_insn_slot(kprobe_opcode_t *slot);
  51. void jprobe_return_end(void);
  52. /* copy of the kernel stack at the probe fire time */
  53. static kprobe_opcode_t jprobes_stack[MAX_STACK_SIZE];
  54. /*
  55. * returns non-zero if opcode modifies the interrupt flag.
  56. */
  57. static inline int is_IF_modifier(kprobe_opcode_t *insn)
  58. {
  59. switch (*insn) {
  60. case 0xfa: /* cli */
  61. case 0xfb: /* sti */
  62. case 0xcf: /* iret/iretd */
  63. case 0x9d: /* popf/popfd */
  64. return 1;
  65. }
  66. if (*insn >= 0x40 && *insn <= 0x4f && *++insn == 0xcf)
  67. return 1;
  68. return 0;
  69. }
  70. int arch_prepare_kprobe(struct kprobe *p)
  71. {
  72. /* insn: must be on special executable page on x86_64. */
  73. up(&kprobe_mutex);
  74. p->ainsn.insn = get_insn_slot();
  75. down(&kprobe_mutex);
  76. if (!p->ainsn.insn) {
  77. return -ENOMEM;
  78. }
  79. return 0;
  80. }
  81. /*
  82. * Determine if the instruction uses the %rip-relative addressing mode.
  83. * If it does, return the address of the 32-bit displacement word.
  84. * If not, return null.
  85. */
  86. static inline s32 *is_riprel(u8 *insn)
  87. {
  88. #define W(row,b0,b1,b2,b3,b4,b5,b6,b7,b8,b9,ba,bb,bc,bd,be,bf) \
  89. (((b0##UL << 0x0)|(b1##UL << 0x1)|(b2##UL << 0x2)|(b3##UL << 0x3) | \
  90. (b4##UL << 0x4)|(b5##UL << 0x5)|(b6##UL << 0x6)|(b7##UL << 0x7) | \
  91. (b8##UL << 0x8)|(b9##UL << 0x9)|(ba##UL << 0xa)|(bb##UL << 0xb) | \
  92. (bc##UL << 0xc)|(bd##UL << 0xd)|(be##UL << 0xe)|(bf##UL << 0xf)) \
  93. << (row % 64))
  94. static const u64 onebyte_has_modrm[256 / 64] = {
  95. /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
  96. /* ------------------------------- */
  97. W(0x00, 1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0)| /* 00 */
  98. W(0x10, 1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0)| /* 10 */
  99. W(0x20, 1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0)| /* 20 */
  100. W(0x30, 1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0), /* 30 */
  101. W(0x40, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0)| /* 40 */
  102. W(0x50, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0)| /* 50 */
  103. W(0x60, 0,0,1,1,0,0,0,0,0,1,0,1,0,0,0,0)| /* 60 */
  104. W(0x70, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0), /* 70 */
  105. W(0x80, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1)| /* 80 */
  106. W(0x90, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0)| /* 90 */
  107. W(0xa0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0)| /* a0 */
  108. W(0xb0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0), /* b0 */
  109. W(0xc0, 1,1,0,0,1,1,1,1,0,0,0,0,0,0,0,0)| /* c0 */
  110. W(0xd0, 1,1,1,1,0,0,0,0,1,1,1,1,1,1,1,1)| /* d0 */
  111. W(0xe0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0)| /* e0 */
  112. W(0xf0, 0,0,0,0,0,0,1,1,0,0,0,0,0,0,1,1) /* f0 */
  113. /* ------------------------------- */
  114. /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
  115. };
  116. static const u64 twobyte_has_modrm[256 / 64] = {
  117. /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
  118. /* ------------------------------- */
  119. W(0x00, 1,1,1,1,0,0,0,0,0,0,0,0,0,1,0,1)| /* 0f */
  120. W(0x10, 1,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0)| /* 1f */
  121. W(0x20, 1,1,1,1,1,0,1,0,1,1,1,1,1,1,1,1)| /* 2f */
  122. W(0x30, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0), /* 3f */
  123. W(0x40, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1)| /* 4f */
  124. W(0x50, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1)| /* 5f */
  125. W(0x60, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1)| /* 6f */
  126. W(0x70, 1,1,1,1,1,1,1,0,0,0,0,0,1,1,1,1), /* 7f */
  127. W(0x80, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0)| /* 8f */
  128. W(0x90, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1)| /* 9f */
  129. W(0xa0, 0,0,0,1,1,1,1,1,0,0,0,1,1,1,1,1)| /* af */
  130. W(0xb0, 1,1,1,1,1,1,1,1,0,0,1,1,1,1,1,1), /* bf */
  131. W(0xc0, 1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0)| /* cf */
  132. W(0xd0, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1)| /* df */
  133. W(0xe0, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1)| /* ef */
  134. W(0xf0, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0) /* ff */
  135. /* ------------------------------- */
  136. /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
  137. };
  138. #undef W
  139. int need_modrm;
  140. /* Skip legacy instruction prefixes. */
  141. while (1) {
  142. switch (*insn) {
  143. case 0x66:
  144. case 0x67:
  145. case 0x2e:
  146. case 0x3e:
  147. case 0x26:
  148. case 0x64:
  149. case 0x65:
  150. case 0x36:
  151. case 0xf0:
  152. case 0xf3:
  153. case 0xf2:
  154. ++insn;
  155. continue;
  156. }
  157. break;
  158. }
  159. /* Skip REX instruction prefix. */
  160. if ((*insn & 0xf0) == 0x40)
  161. ++insn;
  162. if (*insn == 0x0f) { /* Two-byte opcode. */
  163. ++insn;
  164. need_modrm = test_bit(*insn, twobyte_has_modrm);
  165. } else { /* One-byte opcode. */
  166. need_modrm = test_bit(*insn, onebyte_has_modrm);
  167. }
  168. if (need_modrm) {
  169. u8 modrm = *++insn;
  170. if ((modrm & 0xc7) == 0x05) { /* %rip+disp32 addressing mode */
  171. /* Displacement follows ModRM byte. */
  172. return (s32 *) ++insn;
  173. }
  174. }
  175. /* No %rip-relative addressing mode here. */
  176. return NULL;
  177. }
  178. void arch_copy_kprobe(struct kprobe *p)
  179. {
  180. s32 *ripdisp;
  181. memcpy(p->ainsn.insn, p->addr, MAX_INSN_SIZE);
  182. ripdisp = is_riprel(p->ainsn.insn);
  183. if (ripdisp) {
  184. /*
  185. * The copied instruction uses the %rip-relative
  186. * addressing mode. Adjust the displacement for the
  187. * difference between the original location of this
  188. * instruction and the location of the copy that will
  189. * actually be run. The tricky bit here is making sure
  190. * that the sign extension happens correctly in this
  191. * calculation, since we need a signed 32-bit result to
  192. * be sign-extended to 64 bits when it's added to the
  193. * %rip value and yield the same 64-bit result that the
  194. * sign-extension of the original signed 32-bit
  195. * displacement would have given.
  196. */
  197. s64 disp = (u8 *) p->addr + *ripdisp - (u8 *) p->ainsn.insn;
  198. BUG_ON((s64) (s32) disp != disp); /* Sanity check. */
  199. *ripdisp = disp;
  200. }
  201. }
  202. void arch_remove_kprobe(struct kprobe *p)
  203. {
  204. up(&kprobe_mutex);
  205. free_insn_slot(p->ainsn.insn);
  206. down(&kprobe_mutex);
  207. }
  208. static inline void disarm_kprobe(struct kprobe *p, struct pt_regs *regs)
  209. {
  210. *p->addr = p->opcode;
  211. regs->rip = (unsigned long)p->addr;
  212. }
  213. static void prepare_singlestep(struct kprobe *p, struct pt_regs *regs)
  214. {
  215. regs->eflags |= TF_MASK;
  216. regs->eflags &= ~IF_MASK;
  217. /*single step inline if the instruction is an int3*/
  218. if (p->opcode == BREAKPOINT_INSTRUCTION)
  219. regs->rip = (unsigned long)p->addr;
  220. else
  221. regs->rip = (unsigned long)p->ainsn.insn;
  222. }
  223. /*
  224. * Interrupts are disabled on entry as trap3 is an interrupt gate and they
  225. * remain disabled thorough out this function.
  226. */
  227. int kprobe_handler(struct pt_regs *regs)
  228. {
  229. struct kprobe *p;
  230. int ret = 0;
  231. kprobe_opcode_t *addr = (kprobe_opcode_t *)(regs->rip - sizeof(kprobe_opcode_t));
  232. /* We're in an interrupt, but this is clear and BUG()-safe. */
  233. preempt_disable();
  234. /* Check we're not actually recursing */
  235. if (kprobe_running()) {
  236. /* We *are* holding lock here, so this is safe.
  237. Disarm the probe we just hit, and ignore it. */
  238. p = get_kprobe(addr);
  239. if (p) {
  240. if (kprobe_status == KPROBE_HIT_SS) {
  241. regs->eflags &= ~TF_MASK;
  242. regs->eflags |= kprobe_saved_rflags;
  243. unlock_kprobes();
  244. goto no_kprobe;
  245. }
  246. disarm_kprobe(p, regs);
  247. ret = 1;
  248. } else {
  249. p = current_kprobe;
  250. if (p->break_handler && p->break_handler(p, regs)) {
  251. goto ss_probe;
  252. }
  253. }
  254. /* If it's not ours, can't be delete race, (we hold lock). */
  255. goto no_kprobe;
  256. }
  257. lock_kprobes();
  258. p = get_kprobe(addr);
  259. if (!p) {
  260. unlock_kprobes();
  261. if (*addr != BREAKPOINT_INSTRUCTION) {
  262. /*
  263. * The breakpoint instruction was removed right
  264. * after we hit it. Another cpu has removed
  265. * either a probepoint or a debugger breakpoint
  266. * at this address. In either case, no further
  267. * handling of this interrupt is appropriate.
  268. */
  269. ret = 1;
  270. }
  271. /* Not one of ours: let kernel handle it */
  272. goto no_kprobe;
  273. }
  274. kprobe_status = KPROBE_HIT_ACTIVE;
  275. current_kprobe = p;
  276. kprobe_saved_rflags = kprobe_old_rflags
  277. = (regs->eflags & (TF_MASK | IF_MASK));
  278. if (is_IF_modifier(p->ainsn.insn))
  279. kprobe_saved_rflags &= ~IF_MASK;
  280. if (p->pre_handler && p->pre_handler(p, regs))
  281. /* handler has already set things up, so skip ss setup */
  282. return 1;
  283. ss_probe:
  284. prepare_singlestep(p, regs);
  285. kprobe_status = KPROBE_HIT_SS;
  286. return 1;
  287. no_kprobe:
  288. preempt_enable_no_resched();
  289. return ret;
  290. }
  291. /*
  292. * Called after single-stepping. p->addr is the address of the
  293. * instruction whose first byte has been replaced by the "int 3"
  294. * instruction. To avoid the SMP problems that can occur when we
  295. * temporarily put back the original opcode to single-step, we
  296. * single-stepped a copy of the instruction. The address of this
  297. * copy is p->ainsn.insn.
  298. *
  299. * This function prepares to return from the post-single-step
  300. * interrupt. We have to fix up the stack as follows:
  301. *
  302. * 0) Except in the case of absolute or indirect jump or call instructions,
  303. * the new rip is relative to the copied instruction. We need to make
  304. * it relative to the original instruction.
  305. *
  306. * 1) If the single-stepped instruction was pushfl, then the TF and IF
  307. * flags are set in the just-pushed eflags, and may need to be cleared.
  308. *
  309. * 2) If the single-stepped instruction was a call, the return address
  310. * that is atop the stack is the address following the copied instruction.
  311. * We need to make it the address following the original instruction.
  312. */
  313. static void resume_execution(struct kprobe *p, struct pt_regs *regs)
  314. {
  315. unsigned long *tos = (unsigned long *)regs->rsp;
  316. unsigned long next_rip = 0;
  317. unsigned long copy_rip = (unsigned long)p->ainsn.insn;
  318. unsigned long orig_rip = (unsigned long)p->addr;
  319. kprobe_opcode_t *insn = p->ainsn.insn;
  320. /*skip the REX prefix*/
  321. if (*insn >= 0x40 && *insn <= 0x4f)
  322. insn++;
  323. switch (*insn) {
  324. case 0x9c: /* pushfl */
  325. *tos &= ~(TF_MASK | IF_MASK);
  326. *tos |= kprobe_old_rflags;
  327. break;
  328. case 0xe8: /* call relative - Fix return addr */
  329. *tos = orig_rip + (*tos - copy_rip);
  330. break;
  331. case 0xff:
  332. if ((*insn & 0x30) == 0x10) {
  333. /* call absolute, indirect */
  334. /* Fix return addr; rip is correct. */
  335. next_rip = regs->rip;
  336. *tos = orig_rip + (*tos - copy_rip);
  337. } else if (((*insn & 0x31) == 0x20) || /* jmp near, absolute indirect */
  338. ((*insn & 0x31) == 0x21)) { /* jmp far, absolute indirect */
  339. /* rip is correct. */
  340. next_rip = regs->rip;
  341. }
  342. break;
  343. case 0xea: /* jmp absolute -- rip is correct */
  344. next_rip = regs->rip;
  345. break;
  346. default:
  347. break;
  348. }
  349. regs->eflags &= ~TF_MASK;
  350. if (next_rip) {
  351. regs->rip = next_rip;
  352. } else {
  353. regs->rip = orig_rip + (regs->rip - copy_rip);
  354. }
  355. }
  356. /*
  357. * Interrupts are disabled on entry as trap1 is an interrupt gate and they
  358. * remain disabled thoroughout this function. And we hold kprobe lock.
  359. */
  360. int post_kprobe_handler(struct pt_regs *regs)
  361. {
  362. if (!kprobe_running())
  363. return 0;
  364. if (current_kprobe->post_handler)
  365. current_kprobe->post_handler(current_kprobe, regs, 0);
  366. resume_execution(current_kprobe, regs);
  367. regs->eflags |= kprobe_saved_rflags;
  368. unlock_kprobes();
  369. preempt_enable_no_resched();
  370. /*
  371. * if somebody else is singlestepping across a probe point, eflags
  372. * will have TF set, in which case, continue the remaining processing
  373. * of do_debug, as if this is not a probe hit.
  374. */
  375. if (regs->eflags & TF_MASK)
  376. return 0;
  377. return 1;
  378. }
  379. /* Interrupts disabled, kprobe_lock held. */
  380. int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
  381. {
  382. if (current_kprobe->fault_handler
  383. && current_kprobe->fault_handler(current_kprobe, regs, trapnr))
  384. return 1;
  385. if (kprobe_status & KPROBE_HIT_SS) {
  386. resume_execution(current_kprobe, regs);
  387. regs->eflags |= kprobe_old_rflags;
  388. unlock_kprobes();
  389. preempt_enable_no_resched();
  390. }
  391. return 0;
  392. }
  393. /*
  394. * Wrapper routine for handling exceptions.
  395. */
  396. int kprobe_exceptions_notify(struct notifier_block *self, unsigned long val,
  397. void *data)
  398. {
  399. struct die_args *args = (struct die_args *)data;
  400. switch (val) {
  401. case DIE_INT3:
  402. if (kprobe_handler(args->regs))
  403. return NOTIFY_STOP;
  404. break;
  405. case DIE_DEBUG:
  406. if (post_kprobe_handler(args->regs))
  407. return NOTIFY_STOP;
  408. break;
  409. case DIE_GPF:
  410. if (kprobe_running() &&
  411. kprobe_fault_handler(args->regs, args->trapnr))
  412. return NOTIFY_STOP;
  413. break;
  414. case DIE_PAGE_FAULT:
  415. if (kprobe_running() &&
  416. kprobe_fault_handler(args->regs, args->trapnr))
  417. return NOTIFY_STOP;
  418. break;
  419. default:
  420. break;
  421. }
  422. return NOTIFY_DONE;
  423. }
  424. int setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
  425. {
  426. struct jprobe *jp = container_of(p, struct jprobe, kp);
  427. unsigned long addr;
  428. jprobe_saved_regs = *regs;
  429. jprobe_saved_rsp = (long *) regs->rsp;
  430. addr = (unsigned long)jprobe_saved_rsp;
  431. /*
  432. * As Linus pointed out, gcc assumes that the callee
  433. * owns the argument space and could overwrite it, e.g.
  434. * tailcall optimization. So, to be absolutely safe
  435. * we also save and restore enough stack bytes to cover
  436. * the argument area.
  437. */
  438. memcpy(jprobes_stack, (kprobe_opcode_t *) addr, MIN_STACK_SIZE(addr));
  439. regs->eflags &= ~IF_MASK;
  440. regs->rip = (unsigned long)(jp->entry);
  441. return 1;
  442. }
  443. void jprobe_return(void)
  444. {
  445. preempt_enable_no_resched();
  446. asm volatile (" xchg %%rbx,%%rsp \n"
  447. " int3 \n"
  448. " .globl jprobe_return_end \n"
  449. " jprobe_return_end: \n"
  450. " nop \n"::"b"
  451. (jprobe_saved_rsp):"memory");
  452. }
  453. int longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
  454. {
  455. u8 *addr = (u8 *) (regs->rip - 1);
  456. unsigned long stack_addr = (unsigned long)jprobe_saved_rsp;
  457. struct jprobe *jp = container_of(p, struct jprobe, kp);
  458. if ((addr > (u8 *) jprobe_return) && (addr < (u8 *) jprobe_return_end)) {
  459. if ((long *)regs->rsp != jprobe_saved_rsp) {
  460. struct pt_regs *saved_regs =
  461. container_of(jprobe_saved_rsp, struct pt_regs, rsp);
  462. printk("current rsp %p does not match saved rsp %p\n",
  463. (long *)regs->rsp, jprobe_saved_rsp);
  464. printk("Saved registers for jprobe %p\n", jp);
  465. show_registers(saved_regs);
  466. printk("Current registers\n");
  467. show_registers(regs);
  468. BUG();
  469. }
  470. *regs = jprobe_saved_regs;
  471. memcpy((kprobe_opcode_t *) stack_addr, jprobes_stack,
  472. MIN_STACK_SIZE(stack_addr));
  473. return 1;
  474. }
  475. return 0;
  476. }
  477. /*
  478. * kprobe->ainsn.insn points to the copy of the instruction to be single-stepped.
  479. * By default on x86_64, pages we get from kmalloc or vmalloc are not
  480. * executable. Single-stepping an instruction on such a page yields an
  481. * oops. So instead of storing the instruction copies in their respective
  482. * kprobe objects, we allocate a page, map it executable, and store all the
  483. * instruction copies there. (We can allocate additional pages if somebody
  484. * inserts a huge number of probes.) Each page can hold up to INSNS_PER_PAGE
  485. * instruction slots, each of which is MAX_INSN_SIZE*sizeof(kprobe_opcode_t)
  486. * bytes.
  487. */
  488. #define INSNS_PER_PAGE (PAGE_SIZE/(MAX_INSN_SIZE*sizeof(kprobe_opcode_t)))
  489. struct kprobe_insn_page {
  490. struct hlist_node hlist;
  491. kprobe_opcode_t *insns; /* page of instruction slots */
  492. char slot_used[INSNS_PER_PAGE];
  493. int nused;
  494. };
  495. static struct hlist_head kprobe_insn_pages;
  496. /**
  497. * get_insn_slot() - Find a slot on an executable page for an instruction.
  498. * We allocate an executable page if there's no room on existing ones.
  499. */
  500. static kprobe_opcode_t *get_insn_slot(void)
  501. {
  502. struct kprobe_insn_page *kip;
  503. struct hlist_node *pos;
  504. hlist_for_each(pos, &kprobe_insn_pages) {
  505. kip = hlist_entry(pos, struct kprobe_insn_page, hlist);
  506. if (kip->nused < INSNS_PER_PAGE) {
  507. int i;
  508. for (i = 0; i < INSNS_PER_PAGE; i++) {
  509. if (!kip->slot_used[i]) {
  510. kip->slot_used[i] = 1;
  511. kip->nused++;
  512. return kip->insns + (i*MAX_INSN_SIZE);
  513. }
  514. }
  515. /* Surprise! No unused slots. Fix kip->nused. */
  516. kip->nused = INSNS_PER_PAGE;
  517. }
  518. }
  519. /* All out of space. Need to allocate a new page. Use slot 0.*/
  520. kip = kmalloc(sizeof(struct kprobe_insn_page), GFP_KERNEL);
  521. if (!kip) {
  522. return NULL;
  523. }
  524. /*
  525. * For the %rip-relative displacement fixups to be doable, we
  526. * need our instruction copy to be within +/- 2GB of any data it
  527. * might access via %rip. That is, within 2GB of where the
  528. * kernel image and loaded module images reside. So we allocate
  529. * a page in the module loading area.
  530. */
  531. kip->insns = module_alloc(PAGE_SIZE);
  532. if (!kip->insns) {
  533. kfree(kip);
  534. return NULL;
  535. }
  536. INIT_HLIST_NODE(&kip->hlist);
  537. hlist_add_head(&kip->hlist, &kprobe_insn_pages);
  538. memset(kip->slot_used, 0, INSNS_PER_PAGE);
  539. kip->slot_used[0] = 1;
  540. kip->nused = 1;
  541. return kip->insns;
  542. }
  543. /**
  544. * free_insn_slot() - Free instruction slot obtained from get_insn_slot().
  545. */
  546. static void free_insn_slot(kprobe_opcode_t *slot)
  547. {
  548. struct kprobe_insn_page *kip;
  549. struct hlist_node *pos;
  550. hlist_for_each(pos, &kprobe_insn_pages) {
  551. kip = hlist_entry(pos, struct kprobe_insn_page, hlist);
  552. if (kip->insns <= slot
  553. && slot < kip->insns+(INSNS_PER_PAGE*MAX_INSN_SIZE)) {
  554. int i = (slot - kip->insns) / MAX_INSN_SIZE;
  555. kip->slot_used[i] = 0;
  556. kip->nused--;
  557. if (kip->nused == 0) {
  558. /*
  559. * Page is no longer in use. Free it unless
  560. * it's the last one. We keep the last one
  561. * so as not to have to set it up again the
  562. * next time somebody inserts a probe.
  563. */
  564. hlist_del(&kip->hlist);
  565. if (hlist_empty(&kprobe_insn_pages)) {
  566. INIT_HLIST_NODE(&kip->hlist);
  567. hlist_add_head(&kip->hlist,
  568. &kprobe_insn_pages);
  569. } else {
  570. module_free(NULL, kip->insns);
  571. kfree(kip);
  572. }
  573. }
  574. return;
  575. }
  576. }
  577. }