PageRenderTime 77ms CodeModel.GetById 16ms RepoModel.GetById 0ms app.codeStats 1ms

/arch/ia64/kernel/perfmon.c

https://bitbucket.org/evzijst/gittest
C | 6676 lines | 3710 code | 1108 blank | 1858 comment | 795 complexity | 1d0f29d4a70baa02df7e7151feb78ee8 MD5 | raw file
Possible License(s): CC-BY-SA-3.0, GPL-2.0, LGPL-2.0
  1. /*
  2. * This file implements the perfmon-2 subsystem which is used
  3. * to program the IA-64 Performance Monitoring Unit (PMU).
  4. *
  5. * The initial version of perfmon.c was written by
  6. * Ganesh Venkitachalam, IBM Corp.
  7. *
  8. * Then it was modified for perfmon-1.x by Stephane Eranian and
  9. * David Mosberger, Hewlett Packard Co.
  10. *
  11. * Version Perfmon-2.x is a rewrite of perfmon-1.x
  12. * by Stephane Eranian, Hewlett Packard Co.
  13. *
  14. * Copyright (C) 1999-2003, 2005 Hewlett Packard Co
  15. * Stephane Eranian <eranian@hpl.hp.com>
  16. * David Mosberger-Tang <davidm@hpl.hp.com>
  17. *
  18. * More information about perfmon available at:
  19. * http://www.hpl.hp.com/research/linux/perfmon
  20. */
  21. #include <linux/config.h>
  22. #include <linux/module.h>
  23. #include <linux/kernel.h>
  24. #include <linux/sched.h>
  25. #include <linux/interrupt.h>
  26. #include <linux/smp_lock.h>
  27. #include <linux/proc_fs.h>
  28. #include <linux/seq_file.h>
  29. #include <linux/init.h>
  30. #include <linux/vmalloc.h>
  31. #include <linux/mm.h>
  32. #include <linux/sysctl.h>
  33. #include <linux/list.h>
  34. #include <linux/file.h>
  35. #include <linux/poll.h>
  36. #include <linux/vfs.h>
  37. #include <linux/pagemap.h>
  38. #include <linux/mount.h>
  39. #include <linux/version.h>
  40. #include <linux/bitops.h>
  41. #include <asm/errno.h>
  42. #include <asm/intrinsics.h>
  43. #include <asm/page.h>
  44. #include <asm/perfmon.h>
  45. #include <asm/processor.h>
  46. #include <asm/signal.h>
  47. #include <asm/system.h>
  48. #include <asm/uaccess.h>
  49. #include <asm/delay.h>
  50. #ifdef CONFIG_PERFMON
  51. /*
  52. * perfmon context state
  53. */
  54. #define PFM_CTX_UNLOADED 1 /* context is not loaded onto any task */
  55. #define PFM_CTX_LOADED 2 /* context is loaded onto a task */
  56. #define PFM_CTX_MASKED 3 /* context is loaded but monitoring is masked due to overflow */
  57. #define PFM_CTX_ZOMBIE 4 /* owner of the context is closing it */
  58. #define PFM_INVALID_ACTIVATION (~0UL)
  59. /*
  60. * depth of message queue
  61. */
  62. #define PFM_MAX_MSGS 32
  63. #define PFM_CTXQ_EMPTY(g) ((g)->ctx_msgq_head == (g)->ctx_msgq_tail)
  64. /*
  65. * type of a PMU register (bitmask).
  66. * bitmask structure:
  67. * bit0 : register implemented
  68. * bit1 : end marker
  69. * bit2-3 : reserved
  70. * bit4 : pmc has pmc.pm
  71. * bit5 : pmc controls a counter (has pmc.oi), pmd is used as counter
  72. * bit6-7 : register type
  73. * bit8-31: reserved
  74. */
  75. #define PFM_REG_NOTIMPL 0x0 /* not implemented at all */
  76. #define PFM_REG_IMPL 0x1 /* register implemented */
  77. #define PFM_REG_END 0x2 /* end marker */
  78. #define PFM_REG_MONITOR (0x1<<4|PFM_REG_IMPL) /* a PMC with a pmc.pm field only */
  79. #define PFM_REG_COUNTING (0x2<<4|PFM_REG_MONITOR) /* a monitor + pmc.oi+ PMD used as a counter */
  80. #define PFM_REG_CONTROL (0x4<<4|PFM_REG_IMPL) /* PMU control register */
  81. #define PFM_REG_CONFIG (0x8<<4|PFM_REG_IMPL) /* configuration register */
  82. #define PFM_REG_BUFFER (0xc<<4|PFM_REG_IMPL) /* PMD used as buffer */
  83. #define PMC_IS_LAST(i) (pmu_conf->pmc_desc[i].type & PFM_REG_END)
  84. #define PMD_IS_LAST(i) (pmu_conf->pmd_desc[i].type & PFM_REG_END)
  85. #define PMC_OVFL_NOTIFY(ctx, i) ((ctx)->ctx_pmds[i].flags & PFM_REGFL_OVFL_NOTIFY)
  86. /* i assumed unsigned */
  87. #define PMC_IS_IMPL(i) (i< PMU_MAX_PMCS && (pmu_conf->pmc_desc[i].type & PFM_REG_IMPL))
  88. #define PMD_IS_IMPL(i) (i< PMU_MAX_PMDS && (pmu_conf->pmd_desc[i].type & PFM_REG_IMPL))
  89. /* XXX: these assume that register i is implemented */
  90. #define PMD_IS_COUNTING(i) ((pmu_conf->pmd_desc[i].type & PFM_REG_COUNTING) == PFM_REG_COUNTING)
  91. #define PMC_IS_COUNTING(i) ((pmu_conf->pmc_desc[i].type & PFM_REG_COUNTING) == PFM_REG_COUNTING)
  92. #define PMC_IS_MONITOR(i) ((pmu_conf->pmc_desc[i].type & PFM_REG_MONITOR) == PFM_REG_MONITOR)
  93. #define PMC_IS_CONTROL(i) ((pmu_conf->pmc_desc[i].type & PFM_REG_CONTROL) == PFM_REG_CONTROL)
  94. #define PMC_DFL_VAL(i) pmu_conf->pmc_desc[i].default_value
  95. #define PMC_RSVD_MASK(i) pmu_conf->pmc_desc[i].reserved_mask
  96. #define PMD_PMD_DEP(i) pmu_conf->pmd_desc[i].dep_pmd[0]
  97. #define PMC_PMD_DEP(i) pmu_conf->pmc_desc[i].dep_pmd[0]
  98. #define PFM_NUM_IBRS IA64_NUM_DBG_REGS
  99. #define PFM_NUM_DBRS IA64_NUM_DBG_REGS
  100. #define CTX_OVFL_NOBLOCK(c) ((c)->ctx_fl_block == 0)
  101. #define CTX_HAS_SMPL(c) ((c)->ctx_fl_is_sampling)
  102. #define PFM_CTX_TASK(h) (h)->ctx_task
  103. #define PMU_PMC_OI 5 /* position of pmc.oi bit */
  104. /* XXX: does not support more than 64 PMDs */
  105. #define CTX_USED_PMD(ctx, mask) (ctx)->ctx_used_pmds[0] |= (mask)
  106. #define CTX_IS_USED_PMD(ctx, c) (((ctx)->ctx_used_pmds[0] & (1UL << (c))) != 0UL)
  107. #define CTX_USED_MONITOR(ctx, mask) (ctx)->ctx_used_monitors[0] |= (mask)
  108. #define CTX_USED_IBR(ctx,n) (ctx)->ctx_used_ibrs[(n)>>6] |= 1UL<< ((n) % 64)
  109. #define CTX_USED_DBR(ctx,n) (ctx)->ctx_used_dbrs[(n)>>6] |= 1UL<< ((n) % 64)
  110. #define CTX_USES_DBREGS(ctx) (((pfm_context_t *)(ctx))->ctx_fl_using_dbreg==1)
  111. #define PFM_CODE_RR 0 /* requesting code range restriction */
  112. #define PFM_DATA_RR 1 /* requestion data range restriction */
  113. #define PFM_CPUINFO_CLEAR(v) pfm_get_cpu_var(pfm_syst_info) &= ~(v)
  114. #define PFM_CPUINFO_SET(v) pfm_get_cpu_var(pfm_syst_info) |= (v)
  115. #define PFM_CPUINFO_GET() pfm_get_cpu_var(pfm_syst_info)
  116. #define RDEP(x) (1UL<<(x))
  117. /*
  118. * context protection macros
  119. * in SMP:
  120. * - we need to protect against CPU concurrency (spin_lock)
  121. * - we need to protect against PMU overflow interrupts (local_irq_disable)
  122. * in UP:
  123. * - we need to protect against PMU overflow interrupts (local_irq_disable)
  124. *
  125. * spin_lock_irqsave()/spin_lock_irqrestore():
  126. * in SMP: local_irq_disable + spin_lock
  127. * in UP : local_irq_disable
  128. *
  129. * spin_lock()/spin_lock():
  130. * in UP : removed automatically
  131. * in SMP: protect against context accesses from other CPU. interrupts
  132. * are not masked. This is useful for the PMU interrupt handler
  133. * because we know we will not get PMU concurrency in that code.
  134. */
  135. #define PROTECT_CTX(c, f) \
  136. do { \
  137. DPRINT(("spinlock_irq_save ctx %p by [%d]\n", c, current->pid)); \
  138. spin_lock_irqsave(&(c)->ctx_lock, f); \
  139. DPRINT(("spinlocked ctx %p by [%d]\n", c, current->pid)); \
  140. } while(0)
  141. #define UNPROTECT_CTX(c, f) \
  142. do { \
  143. DPRINT(("spinlock_irq_restore ctx %p by [%d]\n", c, current->pid)); \
  144. spin_unlock_irqrestore(&(c)->ctx_lock, f); \
  145. } while(0)
  146. #define PROTECT_CTX_NOPRINT(c, f) \
  147. do { \
  148. spin_lock_irqsave(&(c)->ctx_lock, f); \
  149. } while(0)
  150. #define UNPROTECT_CTX_NOPRINT(c, f) \
  151. do { \
  152. spin_unlock_irqrestore(&(c)->ctx_lock, f); \
  153. } while(0)
  154. #define PROTECT_CTX_NOIRQ(c) \
  155. do { \
  156. spin_lock(&(c)->ctx_lock); \
  157. } while(0)
  158. #define UNPROTECT_CTX_NOIRQ(c) \
  159. do { \
  160. spin_unlock(&(c)->ctx_lock); \
  161. } while(0)
  162. #ifdef CONFIG_SMP
  163. #define GET_ACTIVATION() pfm_get_cpu_var(pmu_activation_number)
  164. #define INC_ACTIVATION() pfm_get_cpu_var(pmu_activation_number)++
  165. #define SET_ACTIVATION(c) (c)->ctx_last_activation = GET_ACTIVATION()
  166. #else /* !CONFIG_SMP */
  167. #define SET_ACTIVATION(t) do {} while(0)
  168. #define GET_ACTIVATION(t) do {} while(0)
  169. #define INC_ACTIVATION(t) do {} while(0)
  170. #endif /* CONFIG_SMP */
  171. #define SET_PMU_OWNER(t, c) do { pfm_get_cpu_var(pmu_owner) = (t); pfm_get_cpu_var(pmu_ctx) = (c); } while(0)
  172. #define GET_PMU_OWNER() pfm_get_cpu_var(pmu_owner)
  173. #define GET_PMU_CTX() pfm_get_cpu_var(pmu_ctx)
  174. #define LOCK_PFS(g) spin_lock_irqsave(&pfm_sessions.pfs_lock, g)
  175. #define UNLOCK_PFS(g) spin_unlock_irqrestore(&pfm_sessions.pfs_lock, g)
  176. #define PFM_REG_RETFLAG_SET(flags, val) do { flags &= ~PFM_REG_RETFL_MASK; flags |= (val); } while(0)
  177. /*
  178. * cmp0 must be the value of pmc0
  179. */
  180. #define PMC0_HAS_OVFL(cmp0) (cmp0 & ~0x1UL)
  181. #define PFMFS_MAGIC 0xa0b4d889
  182. /*
  183. * debugging
  184. */
  185. #define PFM_DEBUGGING 1
  186. #ifdef PFM_DEBUGGING
  187. #define DPRINT(a) \
  188. do { \
  189. if (unlikely(pfm_sysctl.debug >0)) { printk("%s.%d: CPU%d [%d] ", __FUNCTION__, __LINE__, smp_processor_id(), current->pid); printk a; } \
  190. } while (0)
  191. #define DPRINT_ovfl(a) \
  192. do { \
  193. if (unlikely(pfm_sysctl.debug > 0 && pfm_sysctl.debug_ovfl >0)) { printk("%s.%d: CPU%d [%d] ", __FUNCTION__, __LINE__, smp_processor_id(), current->pid); printk a; } \
  194. } while (0)
  195. #endif
  196. /*
  197. * 64-bit software counter structure
  198. *
  199. * the next_reset_type is applied to the next call to pfm_reset_regs()
  200. */
  201. typedef struct {
  202. unsigned long val; /* virtual 64bit counter value */
  203. unsigned long lval; /* last reset value */
  204. unsigned long long_reset; /* reset value on sampling overflow */
  205. unsigned long short_reset; /* reset value on overflow */
  206. unsigned long reset_pmds[4]; /* which other pmds to reset when this counter overflows */
  207. unsigned long smpl_pmds[4]; /* which pmds are accessed when counter overflow */
  208. unsigned long seed; /* seed for random-number generator */
  209. unsigned long mask; /* mask for random-number generator */
  210. unsigned int flags; /* notify/do not notify */
  211. unsigned long eventid; /* overflow event identifier */
  212. } pfm_counter_t;
  213. /*
  214. * context flags
  215. */
  216. typedef struct {
  217. unsigned int block:1; /* when 1, task will blocked on user notifications */
  218. unsigned int system:1; /* do system wide monitoring */
  219. unsigned int using_dbreg:1; /* using range restrictions (debug registers) */
  220. unsigned int is_sampling:1; /* true if using a custom format */
  221. unsigned int excl_idle:1; /* exclude idle task in system wide session */
  222. unsigned int going_zombie:1; /* context is zombie (MASKED+blocking) */
  223. unsigned int trap_reason:2; /* reason for going into pfm_handle_work() */
  224. unsigned int no_msg:1; /* no message sent on overflow */
  225. unsigned int can_restart:1; /* allowed to issue a PFM_RESTART */
  226. unsigned int reserved:22;
  227. } pfm_context_flags_t;
  228. #define PFM_TRAP_REASON_NONE 0x0 /* default value */
  229. #define PFM_TRAP_REASON_BLOCK 0x1 /* we need to block on overflow */
  230. #define PFM_TRAP_REASON_RESET 0x2 /* we need to reset PMDs */
  231. /*
  232. * perfmon context: encapsulates all the state of a monitoring session
  233. */
  234. typedef struct pfm_context {
  235. spinlock_t ctx_lock; /* context protection */
  236. pfm_context_flags_t ctx_flags; /* bitmask of flags (block reason incl.) */
  237. unsigned int ctx_state; /* state: active/inactive (no bitfield) */
  238. struct task_struct *ctx_task; /* task to which context is attached */
  239. unsigned long ctx_ovfl_regs[4]; /* which registers overflowed (notification) */
  240. struct semaphore ctx_restart_sem; /* use for blocking notification mode */
  241. unsigned long ctx_used_pmds[4]; /* bitmask of PMD used */
  242. unsigned long ctx_all_pmds[4]; /* bitmask of all accessible PMDs */
  243. unsigned long ctx_reload_pmds[4]; /* bitmask of force reload PMD on ctxsw in */
  244. unsigned long ctx_all_pmcs[4]; /* bitmask of all accessible PMCs */
  245. unsigned long ctx_reload_pmcs[4]; /* bitmask of force reload PMC on ctxsw in */
  246. unsigned long ctx_used_monitors[4]; /* bitmask of monitor PMC being used */
  247. unsigned long ctx_pmcs[IA64_NUM_PMC_REGS]; /* saved copies of PMC values */
  248. unsigned int ctx_used_ibrs[1]; /* bitmask of used IBR (speedup ctxsw in) */
  249. unsigned int ctx_used_dbrs[1]; /* bitmask of used DBR (speedup ctxsw in) */
  250. unsigned long ctx_dbrs[IA64_NUM_DBG_REGS]; /* DBR values (cache) when not loaded */
  251. unsigned long ctx_ibrs[IA64_NUM_DBG_REGS]; /* IBR values (cache) when not loaded */
  252. pfm_counter_t ctx_pmds[IA64_NUM_PMD_REGS]; /* software state for PMDS */
  253. u64 ctx_saved_psr_up; /* only contains psr.up value */
  254. unsigned long ctx_last_activation; /* context last activation number for last_cpu */
  255. unsigned int ctx_last_cpu; /* CPU id of current or last CPU used (SMP only) */
  256. unsigned int ctx_cpu; /* cpu to which perfmon is applied (system wide) */
  257. int ctx_fd; /* file descriptor used my this context */
  258. pfm_ovfl_arg_t ctx_ovfl_arg; /* argument to custom buffer format handler */
  259. pfm_buffer_fmt_t *ctx_buf_fmt; /* buffer format callbacks */
  260. void *ctx_smpl_hdr; /* points to sampling buffer header kernel vaddr */
  261. unsigned long ctx_smpl_size; /* size of sampling buffer */
  262. void *ctx_smpl_vaddr; /* user level virtual address of smpl buffer */
  263. wait_queue_head_t ctx_msgq_wait;
  264. pfm_msg_t ctx_msgq[PFM_MAX_MSGS];
  265. int ctx_msgq_head;
  266. int ctx_msgq_tail;
  267. struct fasync_struct *ctx_async_queue;
  268. wait_queue_head_t ctx_zombieq; /* termination cleanup wait queue */
  269. } pfm_context_t;
  270. /*
  271. * magic number used to verify that structure is really
  272. * a perfmon context
  273. */
  274. #define PFM_IS_FILE(f) ((f)->f_op == &pfm_file_ops)
  275. #define PFM_GET_CTX(t) ((pfm_context_t *)(t)->thread.pfm_context)
  276. #ifdef CONFIG_SMP
  277. #define SET_LAST_CPU(ctx, v) (ctx)->ctx_last_cpu = (v)
  278. #define GET_LAST_CPU(ctx) (ctx)->ctx_last_cpu
  279. #else
  280. #define SET_LAST_CPU(ctx, v) do {} while(0)
  281. #define GET_LAST_CPU(ctx) do {} while(0)
  282. #endif
  283. #define ctx_fl_block ctx_flags.block
  284. #define ctx_fl_system ctx_flags.system
  285. #define ctx_fl_using_dbreg ctx_flags.using_dbreg
  286. #define ctx_fl_is_sampling ctx_flags.is_sampling
  287. #define ctx_fl_excl_idle ctx_flags.excl_idle
  288. #define ctx_fl_going_zombie ctx_flags.going_zombie
  289. #define ctx_fl_trap_reason ctx_flags.trap_reason
  290. #define ctx_fl_no_msg ctx_flags.no_msg
  291. #define ctx_fl_can_restart ctx_flags.can_restart
  292. #define PFM_SET_WORK_PENDING(t, v) do { (t)->thread.pfm_needs_checking = v; } while(0);
  293. #define PFM_GET_WORK_PENDING(t) (t)->thread.pfm_needs_checking
  294. /*
  295. * global information about all sessions
  296. * mostly used to synchronize between system wide and per-process
  297. */
  298. typedef struct {
  299. spinlock_t pfs_lock; /* lock the structure */
  300. unsigned int pfs_task_sessions; /* number of per task sessions */
  301. unsigned int pfs_sys_sessions; /* number of per system wide sessions */
  302. unsigned int pfs_sys_use_dbregs; /* incremented when a system wide session uses debug regs */
  303. unsigned int pfs_ptrace_use_dbregs; /* incremented when a process uses debug regs */
  304. struct task_struct *pfs_sys_session[NR_CPUS]; /* point to task owning a system-wide session */
  305. } pfm_session_t;
  306. /*
  307. * information about a PMC or PMD.
  308. * dep_pmd[]: a bitmask of dependent PMD registers
  309. * dep_pmc[]: a bitmask of dependent PMC registers
  310. */
  311. typedef int (*pfm_reg_check_t)(struct task_struct *task, pfm_context_t *ctx, unsigned int cnum, unsigned long *val, struct pt_regs *regs);
  312. typedef struct {
  313. unsigned int type;
  314. int pm_pos;
  315. unsigned long default_value; /* power-on default value */
  316. unsigned long reserved_mask; /* bitmask of reserved bits */
  317. pfm_reg_check_t read_check;
  318. pfm_reg_check_t write_check;
  319. unsigned long dep_pmd[4];
  320. unsigned long dep_pmc[4];
  321. } pfm_reg_desc_t;
  322. /* assume cnum is a valid monitor */
  323. #define PMC_PM(cnum, val) (((val) >> (pmu_conf->pmc_desc[cnum].pm_pos)) & 0x1)
  324. /*
  325. * This structure is initialized at boot time and contains
  326. * a description of the PMU main characteristics.
  327. *
  328. * If the probe function is defined, detection is based
  329. * on its return value:
  330. * - 0 means recognized PMU
  331. * - anything else means not supported
  332. * When the probe function is not defined, then the pmu_family field
  333. * is used and it must match the host CPU family such that:
  334. * - cpu->family & config->pmu_family != 0
  335. */
  336. typedef struct {
  337. unsigned long ovfl_val; /* overflow value for counters */
  338. pfm_reg_desc_t *pmc_desc; /* detailed PMC register dependencies descriptions */
  339. pfm_reg_desc_t *pmd_desc; /* detailed PMD register dependencies descriptions */
  340. unsigned int num_pmcs; /* number of PMCS: computed at init time */
  341. unsigned int num_pmds; /* number of PMDS: computed at init time */
  342. unsigned long impl_pmcs[4]; /* bitmask of implemented PMCS */
  343. unsigned long impl_pmds[4]; /* bitmask of implemented PMDS */
  344. char *pmu_name; /* PMU family name */
  345. unsigned int pmu_family; /* cpuid family pattern used to identify pmu */
  346. unsigned int flags; /* pmu specific flags */
  347. unsigned int num_ibrs; /* number of IBRS: computed at init time */
  348. unsigned int num_dbrs; /* number of DBRS: computed at init time */
  349. unsigned int num_counters; /* PMC/PMD counting pairs : computed at init time */
  350. int (*probe)(void); /* customized probe routine */
  351. unsigned int use_rr_dbregs:1; /* set if debug registers used for range restriction */
  352. } pmu_config_t;
  353. /*
  354. * PMU specific flags
  355. */
  356. #define PFM_PMU_IRQ_RESEND 1 /* PMU needs explicit IRQ resend */
  357. /*
  358. * debug register related type definitions
  359. */
  360. typedef struct {
  361. unsigned long ibr_mask:56;
  362. unsigned long ibr_plm:4;
  363. unsigned long ibr_ig:3;
  364. unsigned long ibr_x:1;
  365. } ibr_mask_reg_t;
  366. typedef struct {
  367. unsigned long dbr_mask:56;
  368. unsigned long dbr_plm:4;
  369. unsigned long dbr_ig:2;
  370. unsigned long dbr_w:1;
  371. unsigned long dbr_r:1;
  372. } dbr_mask_reg_t;
  373. typedef union {
  374. unsigned long val;
  375. ibr_mask_reg_t ibr;
  376. dbr_mask_reg_t dbr;
  377. } dbreg_t;
  378. /*
  379. * perfmon command descriptions
  380. */
  381. typedef struct {
  382. int (*cmd_func)(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs);
  383. char *cmd_name;
  384. int cmd_flags;
  385. unsigned int cmd_narg;
  386. size_t cmd_argsize;
  387. int (*cmd_getsize)(void *arg, size_t *sz);
  388. } pfm_cmd_desc_t;
  389. #define PFM_CMD_FD 0x01 /* command requires a file descriptor */
  390. #define PFM_CMD_ARG_READ 0x02 /* command must read argument(s) */
  391. #define PFM_CMD_ARG_RW 0x04 /* command must read/write argument(s) */
  392. #define PFM_CMD_STOP 0x08 /* command does not work on zombie context */
  393. #define PFM_CMD_NAME(cmd) pfm_cmd_tab[(cmd)].cmd_name
  394. #define PFM_CMD_READ_ARG(cmd) (pfm_cmd_tab[(cmd)].cmd_flags & PFM_CMD_ARG_READ)
  395. #define PFM_CMD_RW_ARG(cmd) (pfm_cmd_tab[(cmd)].cmd_flags & PFM_CMD_ARG_RW)
  396. #define PFM_CMD_USE_FD(cmd) (pfm_cmd_tab[(cmd)].cmd_flags & PFM_CMD_FD)
  397. #define PFM_CMD_STOPPED(cmd) (pfm_cmd_tab[(cmd)].cmd_flags & PFM_CMD_STOP)
  398. #define PFM_CMD_ARG_MANY -1 /* cannot be zero */
  399. typedef struct {
  400. int debug; /* turn on/off debugging via syslog */
  401. int debug_ovfl; /* turn on/off debug printk in overflow handler */
  402. int fastctxsw; /* turn on/off fast (unsecure) ctxsw */
  403. int expert_mode; /* turn on/off value checking */
  404. int debug_pfm_read;
  405. } pfm_sysctl_t;
  406. typedef struct {
  407. unsigned long pfm_spurious_ovfl_intr_count; /* keep track of spurious ovfl interrupts */
  408. unsigned long pfm_replay_ovfl_intr_count; /* keep track of replayed ovfl interrupts */
  409. unsigned long pfm_ovfl_intr_count; /* keep track of ovfl interrupts */
  410. unsigned long pfm_ovfl_intr_cycles; /* cycles spent processing ovfl interrupts */
  411. unsigned long pfm_ovfl_intr_cycles_min; /* min cycles spent processing ovfl interrupts */
  412. unsigned long pfm_ovfl_intr_cycles_max; /* max cycles spent processing ovfl interrupts */
  413. unsigned long pfm_smpl_handler_calls;
  414. unsigned long pfm_smpl_handler_cycles;
  415. char pad[SMP_CACHE_BYTES] ____cacheline_aligned;
  416. } pfm_stats_t;
  417. /*
  418. * perfmon internal variables
  419. */
  420. static pfm_stats_t pfm_stats[NR_CPUS];
  421. static pfm_session_t pfm_sessions; /* global sessions information */
  422. static struct proc_dir_entry *perfmon_dir;
  423. static pfm_uuid_t pfm_null_uuid = {0,};
  424. static spinlock_t pfm_buffer_fmt_lock;
  425. static LIST_HEAD(pfm_buffer_fmt_list);
  426. static pmu_config_t *pmu_conf;
  427. /* sysctl() controls */
  428. static pfm_sysctl_t pfm_sysctl;
  429. int pfm_debug_var;
  430. static ctl_table pfm_ctl_table[]={
  431. {1, "debug", &pfm_sysctl.debug, sizeof(int), 0666, NULL, &proc_dointvec, NULL,},
  432. {2, "debug_ovfl", &pfm_sysctl.debug_ovfl, sizeof(int), 0666, NULL, &proc_dointvec, NULL,},
  433. {3, "fastctxsw", &pfm_sysctl.fastctxsw, sizeof(int), 0600, NULL, &proc_dointvec, NULL,},
  434. {4, "expert_mode", &pfm_sysctl.expert_mode, sizeof(int), 0600, NULL, &proc_dointvec, NULL,},
  435. { 0, },
  436. };
  437. static ctl_table pfm_sysctl_dir[] = {
  438. {1, "perfmon", NULL, 0, 0755, pfm_ctl_table, },
  439. {0,},
  440. };
  441. static ctl_table pfm_sysctl_root[] = {
  442. {1, "kernel", NULL, 0, 0755, pfm_sysctl_dir, },
  443. {0,},
  444. };
  445. static struct ctl_table_header *pfm_sysctl_header;
  446. static int pfm_context_unload(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs);
  447. static int pfm_flush(struct file *filp);
  448. #define pfm_get_cpu_var(v) __ia64_per_cpu_var(v)
  449. #define pfm_get_cpu_data(a,b) per_cpu(a, b)
  450. static inline void
  451. pfm_put_task(struct task_struct *task)
  452. {
  453. if (task != current) put_task_struct(task);
  454. }
  455. static inline void
  456. pfm_set_task_notify(struct task_struct *task)
  457. {
  458. struct thread_info *info;
  459. info = (struct thread_info *) ((char *) task + IA64_TASK_SIZE);
  460. set_bit(TIF_NOTIFY_RESUME, &info->flags);
  461. }
  462. static inline void
  463. pfm_clear_task_notify(void)
  464. {
  465. clear_thread_flag(TIF_NOTIFY_RESUME);
  466. }
  467. static inline void
  468. pfm_reserve_page(unsigned long a)
  469. {
  470. SetPageReserved(vmalloc_to_page((void *)a));
  471. }
  472. static inline void
  473. pfm_unreserve_page(unsigned long a)
  474. {
  475. ClearPageReserved(vmalloc_to_page((void*)a));
  476. }
  477. static inline unsigned long
  478. pfm_protect_ctx_ctxsw(pfm_context_t *x)
  479. {
  480. spin_lock(&(x)->ctx_lock);
  481. return 0UL;
  482. }
  483. static inline unsigned long
  484. pfm_unprotect_ctx_ctxsw(pfm_context_t *x, unsigned long f)
  485. {
  486. spin_unlock(&(x)->ctx_lock);
  487. }
  488. static inline unsigned int
  489. pfm_do_munmap(struct mm_struct *mm, unsigned long addr, size_t len, int acct)
  490. {
  491. return do_munmap(mm, addr, len);
  492. }
  493. static inline unsigned long
  494. pfm_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags, unsigned long exec)
  495. {
  496. return get_unmapped_area(file, addr, len, pgoff, flags);
  497. }
  498. static struct super_block *
  499. pfmfs_get_sb(struct file_system_type *fs_type, int flags, const char *dev_name, void *data)
  500. {
  501. return get_sb_pseudo(fs_type, "pfm:", NULL, PFMFS_MAGIC);
  502. }
  503. static struct file_system_type pfm_fs_type = {
  504. .name = "pfmfs",
  505. .get_sb = pfmfs_get_sb,
  506. .kill_sb = kill_anon_super,
  507. };
  508. DEFINE_PER_CPU(unsigned long, pfm_syst_info);
  509. DEFINE_PER_CPU(struct task_struct *, pmu_owner);
  510. DEFINE_PER_CPU(pfm_context_t *, pmu_ctx);
  511. DEFINE_PER_CPU(unsigned long, pmu_activation_number);
  512. /* forward declaration */
  513. static struct file_operations pfm_file_ops;
  514. /*
  515. * forward declarations
  516. */
  517. #ifndef CONFIG_SMP
  518. static void pfm_lazy_save_regs (struct task_struct *ta);
  519. #endif
  520. void dump_pmu_state(const char *);
  521. static int pfm_write_ibr_dbr(int mode, pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs);
  522. #include "perfmon_itanium.h"
  523. #include "perfmon_mckinley.h"
  524. #include "perfmon_generic.h"
  525. static pmu_config_t *pmu_confs[]={
  526. &pmu_conf_mck,
  527. &pmu_conf_ita,
  528. &pmu_conf_gen, /* must be last */
  529. NULL
  530. };
  531. static int pfm_end_notify_user(pfm_context_t *ctx);
  532. static inline void
  533. pfm_clear_psr_pp(void)
  534. {
  535. ia64_rsm(IA64_PSR_PP);
  536. ia64_srlz_i();
  537. }
  538. static inline void
  539. pfm_set_psr_pp(void)
  540. {
  541. ia64_ssm(IA64_PSR_PP);
  542. ia64_srlz_i();
  543. }
  544. static inline void
  545. pfm_clear_psr_up(void)
  546. {
  547. ia64_rsm(IA64_PSR_UP);
  548. ia64_srlz_i();
  549. }
  550. static inline void
  551. pfm_set_psr_up(void)
  552. {
  553. ia64_ssm(IA64_PSR_UP);
  554. ia64_srlz_i();
  555. }
  556. static inline unsigned long
  557. pfm_get_psr(void)
  558. {
  559. unsigned long tmp;
  560. tmp = ia64_getreg(_IA64_REG_PSR);
  561. ia64_srlz_i();
  562. return tmp;
  563. }
  564. static inline void
  565. pfm_set_psr_l(unsigned long val)
  566. {
  567. ia64_setreg(_IA64_REG_PSR_L, val);
  568. ia64_srlz_i();
  569. }
  570. static inline void
  571. pfm_freeze_pmu(void)
  572. {
  573. ia64_set_pmc(0,1UL);
  574. ia64_srlz_d();
  575. }
  576. static inline void
  577. pfm_unfreeze_pmu(void)
  578. {
  579. ia64_set_pmc(0,0UL);
  580. ia64_srlz_d();
  581. }
  582. static inline void
  583. pfm_restore_ibrs(unsigned long *ibrs, unsigned int nibrs)
  584. {
  585. int i;
  586. for (i=0; i < nibrs; i++) {
  587. ia64_set_ibr(i, ibrs[i]);
  588. ia64_dv_serialize_instruction();
  589. }
  590. ia64_srlz_i();
  591. }
  592. static inline void
  593. pfm_restore_dbrs(unsigned long *dbrs, unsigned int ndbrs)
  594. {
  595. int i;
  596. for (i=0; i < ndbrs; i++) {
  597. ia64_set_dbr(i, dbrs[i]);
  598. ia64_dv_serialize_data();
  599. }
  600. ia64_srlz_d();
  601. }
  602. /*
  603. * PMD[i] must be a counter. no check is made
  604. */
  605. static inline unsigned long
  606. pfm_read_soft_counter(pfm_context_t *ctx, int i)
  607. {
  608. return ctx->ctx_pmds[i].val + (ia64_get_pmd(i) & pmu_conf->ovfl_val);
  609. }
  610. /*
  611. * PMD[i] must be a counter. no check is made
  612. */
  613. static inline void
  614. pfm_write_soft_counter(pfm_context_t *ctx, int i, unsigned long val)
  615. {
  616. unsigned long ovfl_val = pmu_conf->ovfl_val;
  617. ctx->ctx_pmds[i].val = val & ~ovfl_val;
  618. /*
  619. * writing to unimplemented part is ignore, so we do not need to
  620. * mask off top part
  621. */
  622. ia64_set_pmd(i, val & ovfl_val);
  623. }
  624. static pfm_msg_t *
  625. pfm_get_new_msg(pfm_context_t *ctx)
  626. {
  627. int idx, next;
  628. next = (ctx->ctx_msgq_tail+1) % PFM_MAX_MSGS;
  629. DPRINT(("ctx_fd=%p head=%d tail=%d\n", ctx, ctx->ctx_msgq_head, ctx->ctx_msgq_tail));
  630. if (next == ctx->ctx_msgq_head) return NULL;
  631. idx = ctx->ctx_msgq_tail;
  632. ctx->ctx_msgq_tail = next;
  633. DPRINT(("ctx=%p head=%d tail=%d msg=%d\n", ctx, ctx->ctx_msgq_head, ctx->ctx_msgq_tail, idx));
  634. return ctx->ctx_msgq+idx;
  635. }
  636. static pfm_msg_t *
  637. pfm_get_next_msg(pfm_context_t *ctx)
  638. {
  639. pfm_msg_t *msg;
  640. DPRINT(("ctx=%p head=%d tail=%d\n", ctx, ctx->ctx_msgq_head, ctx->ctx_msgq_tail));
  641. if (PFM_CTXQ_EMPTY(ctx)) return NULL;
  642. /*
  643. * get oldest message
  644. */
  645. msg = ctx->ctx_msgq+ctx->ctx_msgq_head;
  646. /*
  647. * and move forward
  648. */
  649. ctx->ctx_msgq_head = (ctx->ctx_msgq_head+1) % PFM_MAX_MSGS;
  650. DPRINT(("ctx=%p head=%d tail=%d type=%d\n", ctx, ctx->ctx_msgq_head, ctx->ctx_msgq_tail, msg->pfm_gen_msg.msg_type));
  651. return msg;
  652. }
  653. static void
  654. pfm_reset_msgq(pfm_context_t *ctx)
  655. {
  656. ctx->ctx_msgq_head = ctx->ctx_msgq_tail = 0;
  657. DPRINT(("ctx=%p msgq reset\n", ctx));
  658. }
  659. static void *
  660. pfm_rvmalloc(unsigned long size)
  661. {
  662. void *mem;
  663. unsigned long addr;
  664. size = PAGE_ALIGN(size);
  665. mem = vmalloc(size);
  666. if (mem) {
  667. //printk("perfmon: CPU%d pfm_rvmalloc(%ld)=%p\n", smp_processor_id(), size, mem);
  668. memset(mem, 0, size);
  669. addr = (unsigned long)mem;
  670. while (size > 0) {
  671. pfm_reserve_page(addr);
  672. addr+=PAGE_SIZE;
  673. size-=PAGE_SIZE;
  674. }
  675. }
  676. return mem;
  677. }
  678. static void
  679. pfm_rvfree(void *mem, unsigned long size)
  680. {
  681. unsigned long addr;
  682. if (mem) {
  683. DPRINT(("freeing physical buffer @%p size=%lu\n", mem, size));
  684. addr = (unsigned long) mem;
  685. while ((long) size > 0) {
  686. pfm_unreserve_page(addr);
  687. addr+=PAGE_SIZE;
  688. size-=PAGE_SIZE;
  689. }
  690. vfree(mem);
  691. }
  692. return;
  693. }
  694. static pfm_context_t *
  695. pfm_context_alloc(void)
  696. {
  697. pfm_context_t *ctx;
  698. /*
  699. * allocate context descriptor
  700. * must be able to free with interrupts disabled
  701. */
  702. ctx = kmalloc(sizeof(pfm_context_t), GFP_KERNEL);
  703. if (ctx) {
  704. memset(ctx, 0, sizeof(pfm_context_t));
  705. DPRINT(("alloc ctx @%p\n", ctx));
  706. }
  707. return ctx;
  708. }
  709. static void
  710. pfm_context_free(pfm_context_t *ctx)
  711. {
  712. if (ctx) {
  713. DPRINT(("free ctx @%p\n", ctx));
  714. kfree(ctx);
  715. }
  716. }
  717. static void
  718. pfm_mask_monitoring(struct task_struct *task)
  719. {
  720. pfm_context_t *ctx = PFM_GET_CTX(task);
  721. struct thread_struct *th = &task->thread;
  722. unsigned long mask, val, ovfl_mask;
  723. int i;
  724. DPRINT_ovfl(("masking monitoring for [%d]\n", task->pid));
  725. ovfl_mask = pmu_conf->ovfl_val;
  726. /*
  727. * monitoring can only be masked as a result of a valid
  728. * counter overflow. In UP, it means that the PMU still
  729. * has an owner. Note that the owner can be different
  730. * from the current task. However the PMU state belongs
  731. * to the owner.
  732. * In SMP, a valid overflow only happens when task is
  733. * current. Therefore if we come here, we know that
  734. * the PMU state belongs to the current task, therefore
  735. * we can access the live registers.
  736. *
  737. * So in both cases, the live register contains the owner's
  738. * state. We can ONLY touch the PMU registers and NOT the PSR.
  739. *
  740. * As a consequence to this call, the thread->pmds[] array
  741. * contains stale information which must be ignored
  742. * when context is reloaded AND monitoring is active (see
  743. * pfm_restart).
  744. */
  745. mask = ctx->ctx_used_pmds[0];
  746. for (i = 0; mask; i++, mask>>=1) {
  747. /* skip non used pmds */
  748. if ((mask & 0x1) == 0) continue;
  749. val = ia64_get_pmd(i);
  750. if (PMD_IS_COUNTING(i)) {
  751. /*
  752. * we rebuild the full 64 bit value of the counter
  753. */
  754. ctx->ctx_pmds[i].val += (val & ovfl_mask);
  755. } else {
  756. ctx->ctx_pmds[i].val = val;
  757. }
  758. DPRINT_ovfl(("pmd[%d]=0x%lx hw_pmd=0x%lx\n",
  759. i,
  760. ctx->ctx_pmds[i].val,
  761. val & ovfl_mask));
  762. }
  763. /*
  764. * mask monitoring by setting the privilege level to 0
  765. * we cannot use psr.pp/psr.up for this, it is controlled by
  766. * the user
  767. *
  768. * if task is current, modify actual registers, otherwise modify
  769. * thread save state, i.e., what will be restored in pfm_load_regs()
  770. */
  771. mask = ctx->ctx_used_monitors[0] >> PMU_FIRST_COUNTER;
  772. for(i= PMU_FIRST_COUNTER; mask; i++, mask>>=1) {
  773. if ((mask & 0x1) == 0UL) continue;
  774. ia64_set_pmc(i, th->pmcs[i] & ~0xfUL);
  775. th->pmcs[i] &= ~0xfUL;
  776. DPRINT_ovfl(("pmc[%d]=0x%lx\n", i, th->pmcs[i]));
  777. }
  778. /*
  779. * make all of this visible
  780. */
  781. ia64_srlz_d();
  782. }
  783. /*
  784. * must always be done with task == current
  785. *
  786. * context must be in MASKED state when calling
  787. */
  788. static void
  789. pfm_restore_monitoring(struct task_struct *task)
  790. {
  791. pfm_context_t *ctx = PFM_GET_CTX(task);
  792. struct thread_struct *th = &task->thread;
  793. unsigned long mask, ovfl_mask;
  794. unsigned long psr, val;
  795. int i, is_system;
  796. is_system = ctx->ctx_fl_system;
  797. ovfl_mask = pmu_conf->ovfl_val;
  798. if (task != current) {
  799. printk(KERN_ERR "perfmon.%d: invalid task[%d] current[%d]\n", __LINE__, task->pid, current->pid);
  800. return;
  801. }
  802. if (ctx->ctx_state != PFM_CTX_MASKED) {
  803. printk(KERN_ERR "perfmon.%d: task[%d] current[%d] invalid state=%d\n", __LINE__,
  804. task->pid, current->pid, ctx->ctx_state);
  805. return;
  806. }
  807. psr = pfm_get_psr();
  808. /*
  809. * monitoring is masked via the PMC.
  810. * As we restore their value, we do not want each counter to
  811. * restart right away. We stop monitoring using the PSR,
  812. * restore the PMC (and PMD) and then re-establish the psr
  813. * as it was. Note that there can be no pending overflow at
  814. * this point, because monitoring was MASKED.
  815. *
  816. * system-wide session are pinned and self-monitoring
  817. */
  818. if (is_system && (PFM_CPUINFO_GET() & PFM_CPUINFO_DCR_PP)) {
  819. /* disable dcr pp */
  820. ia64_setreg(_IA64_REG_CR_DCR, ia64_getreg(_IA64_REG_CR_DCR) & ~IA64_DCR_PP);
  821. pfm_clear_psr_pp();
  822. } else {
  823. pfm_clear_psr_up();
  824. }
  825. /*
  826. * first, we restore the PMD
  827. */
  828. mask = ctx->ctx_used_pmds[0];
  829. for (i = 0; mask; i++, mask>>=1) {
  830. /* skip non used pmds */
  831. if ((mask & 0x1) == 0) continue;
  832. if (PMD_IS_COUNTING(i)) {
  833. /*
  834. * we split the 64bit value according to
  835. * counter width
  836. */
  837. val = ctx->ctx_pmds[i].val & ovfl_mask;
  838. ctx->ctx_pmds[i].val &= ~ovfl_mask;
  839. } else {
  840. val = ctx->ctx_pmds[i].val;
  841. }
  842. ia64_set_pmd(i, val);
  843. DPRINT(("pmd[%d]=0x%lx hw_pmd=0x%lx\n",
  844. i,
  845. ctx->ctx_pmds[i].val,
  846. val));
  847. }
  848. /*
  849. * restore the PMCs
  850. */
  851. mask = ctx->ctx_used_monitors[0] >> PMU_FIRST_COUNTER;
  852. for(i= PMU_FIRST_COUNTER; mask; i++, mask>>=1) {
  853. if ((mask & 0x1) == 0UL) continue;
  854. th->pmcs[i] = ctx->ctx_pmcs[i];
  855. ia64_set_pmc(i, th->pmcs[i]);
  856. DPRINT(("[%d] pmc[%d]=0x%lx\n", task->pid, i, th->pmcs[i]));
  857. }
  858. ia64_srlz_d();
  859. /*
  860. * must restore DBR/IBR because could be modified while masked
  861. * XXX: need to optimize
  862. */
  863. if (ctx->ctx_fl_using_dbreg) {
  864. pfm_restore_ibrs(ctx->ctx_ibrs, pmu_conf->num_ibrs);
  865. pfm_restore_dbrs(ctx->ctx_dbrs, pmu_conf->num_dbrs);
  866. }
  867. /*
  868. * now restore PSR
  869. */
  870. if (is_system && (PFM_CPUINFO_GET() & PFM_CPUINFO_DCR_PP)) {
  871. /* enable dcr pp */
  872. ia64_setreg(_IA64_REG_CR_DCR, ia64_getreg(_IA64_REG_CR_DCR) | IA64_DCR_PP);
  873. ia64_srlz_i();
  874. }
  875. pfm_set_psr_l(psr);
  876. }
  877. static inline void
  878. pfm_save_pmds(unsigned long *pmds, unsigned long mask)
  879. {
  880. int i;
  881. ia64_srlz_d();
  882. for (i=0; mask; i++, mask>>=1) {
  883. if (mask & 0x1) pmds[i] = ia64_get_pmd(i);
  884. }
  885. }
  886. /*
  887. * reload from thread state (used for ctxw only)
  888. */
  889. static inline void
  890. pfm_restore_pmds(unsigned long *pmds, unsigned long mask)
  891. {
  892. int i;
  893. unsigned long val, ovfl_val = pmu_conf->ovfl_val;
  894. for (i=0; mask; i++, mask>>=1) {
  895. if ((mask & 0x1) == 0) continue;
  896. val = PMD_IS_COUNTING(i) ? pmds[i] & ovfl_val : pmds[i];
  897. ia64_set_pmd(i, val);
  898. }
  899. ia64_srlz_d();
  900. }
  901. /*
  902. * propagate PMD from context to thread-state
  903. */
  904. static inline void
  905. pfm_copy_pmds(struct task_struct *task, pfm_context_t *ctx)
  906. {
  907. struct thread_struct *thread = &task->thread;
  908. unsigned long ovfl_val = pmu_conf->ovfl_val;
  909. unsigned long mask = ctx->ctx_all_pmds[0];
  910. unsigned long val;
  911. int i;
  912. DPRINT(("mask=0x%lx\n", mask));
  913. for (i=0; mask; i++, mask>>=1) {
  914. val = ctx->ctx_pmds[i].val;
  915. /*
  916. * We break up the 64 bit value into 2 pieces
  917. * the lower bits go to the machine state in the
  918. * thread (will be reloaded on ctxsw in).
  919. * The upper part stays in the soft-counter.
  920. */
  921. if (PMD_IS_COUNTING(i)) {
  922. ctx->ctx_pmds[i].val = val & ~ovfl_val;
  923. val &= ovfl_val;
  924. }
  925. thread->pmds[i] = val;
  926. DPRINT(("pmd[%d]=0x%lx soft_val=0x%lx\n",
  927. i,
  928. thread->pmds[i],
  929. ctx->ctx_pmds[i].val));
  930. }
  931. }
  932. /*
  933. * propagate PMC from context to thread-state
  934. */
  935. static inline void
  936. pfm_copy_pmcs(struct task_struct *task, pfm_context_t *ctx)
  937. {
  938. struct thread_struct *thread = &task->thread;
  939. unsigned long mask = ctx->ctx_all_pmcs[0];
  940. int i;
  941. DPRINT(("mask=0x%lx\n", mask));
  942. for (i=0; mask; i++, mask>>=1) {
  943. /* masking 0 with ovfl_val yields 0 */
  944. thread->pmcs[i] = ctx->ctx_pmcs[i];
  945. DPRINT(("pmc[%d]=0x%lx\n", i, thread->pmcs[i]));
  946. }
  947. }
  948. static inline void
  949. pfm_restore_pmcs(unsigned long *pmcs, unsigned long mask)
  950. {
  951. int i;
  952. for (i=0; mask; i++, mask>>=1) {
  953. if ((mask & 0x1) == 0) continue;
  954. ia64_set_pmc(i, pmcs[i]);
  955. }
  956. ia64_srlz_d();
  957. }
  958. static inline int
  959. pfm_uuid_cmp(pfm_uuid_t a, pfm_uuid_t b)
  960. {
  961. return memcmp(a, b, sizeof(pfm_uuid_t));
  962. }
  963. static inline int
  964. pfm_buf_fmt_exit(pfm_buffer_fmt_t *fmt, struct task_struct *task, void *buf, struct pt_regs *regs)
  965. {
  966. int ret = 0;
  967. if (fmt->fmt_exit) ret = (*fmt->fmt_exit)(task, buf, regs);
  968. return ret;
  969. }
  970. static inline int
  971. pfm_buf_fmt_getsize(pfm_buffer_fmt_t *fmt, struct task_struct *task, unsigned int flags, int cpu, void *arg, unsigned long *size)
  972. {
  973. int ret = 0;
  974. if (fmt->fmt_getsize) ret = (*fmt->fmt_getsize)(task, flags, cpu, arg, size);
  975. return ret;
  976. }
  977. static inline int
  978. pfm_buf_fmt_validate(pfm_buffer_fmt_t *fmt, struct task_struct *task, unsigned int flags,
  979. int cpu, void *arg)
  980. {
  981. int ret = 0;
  982. if (fmt->fmt_validate) ret = (*fmt->fmt_validate)(task, flags, cpu, arg);
  983. return ret;
  984. }
  985. static inline int
  986. pfm_buf_fmt_init(pfm_buffer_fmt_t *fmt, struct task_struct *task, void *buf, unsigned int flags,
  987. int cpu, void *arg)
  988. {
  989. int ret = 0;
  990. if (fmt->fmt_init) ret = (*fmt->fmt_init)(task, buf, flags, cpu, arg);
  991. return ret;
  992. }
  993. static inline int
  994. pfm_buf_fmt_restart(pfm_buffer_fmt_t *fmt, struct task_struct *task, pfm_ovfl_ctrl_t *ctrl, void *buf, struct pt_regs *regs)
  995. {
  996. int ret = 0;
  997. if (fmt->fmt_restart) ret = (*fmt->fmt_restart)(task, ctrl, buf, regs);
  998. return ret;
  999. }
  1000. static inline int
  1001. pfm_buf_fmt_restart_active(pfm_buffer_fmt_t *fmt, struct task_struct *task, pfm_ovfl_ctrl_t *ctrl, void *buf, struct pt_regs *regs)
  1002. {
  1003. int ret = 0;
  1004. if (fmt->fmt_restart_active) ret = (*fmt->fmt_restart_active)(task, ctrl, buf, regs);
  1005. return ret;
  1006. }
  1007. static pfm_buffer_fmt_t *
  1008. __pfm_find_buffer_fmt(pfm_uuid_t uuid)
  1009. {
  1010. struct list_head * pos;
  1011. pfm_buffer_fmt_t * entry;
  1012. list_for_each(pos, &pfm_buffer_fmt_list) {
  1013. entry = list_entry(pos, pfm_buffer_fmt_t, fmt_list);
  1014. if (pfm_uuid_cmp(uuid, entry->fmt_uuid) == 0)
  1015. return entry;
  1016. }
  1017. return NULL;
  1018. }
  1019. /*
  1020. * find a buffer format based on its uuid
  1021. */
  1022. static pfm_buffer_fmt_t *
  1023. pfm_find_buffer_fmt(pfm_uuid_t uuid)
  1024. {
  1025. pfm_buffer_fmt_t * fmt;
  1026. spin_lock(&pfm_buffer_fmt_lock);
  1027. fmt = __pfm_find_buffer_fmt(uuid);
  1028. spin_unlock(&pfm_buffer_fmt_lock);
  1029. return fmt;
  1030. }
  1031. int
  1032. pfm_register_buffer_fmt(pfm_buffer_fmt_t *fmt)
  1033. {
  1034. int ret = 0;
  1035. /* some sanity checks */
  1036. if (fmt == NULL || fmt->fmt_name == NULL) return -EINVAL;
  1037. /* we need at least a handler */
  1038. if (fmt->fmt_handler == NULL) return -EINVAL;
  1039. /*
  1040. * XXX: need check validity of fmt_arg_size
  1041. */
  1042. spin_lock(&pfm_buffer_fmt_lock);
  1043. if (__pfm_find_buffer_fmt(fmt->fmt_uuid)) {
  1044. printk(KERN_ERR "perfmon: duplicate sampling format: %s\n", fmt->fmt_name);
  1045. ret = -EBUSY;
  1046. goto out;
  1047. }
  1048. list_add(&fmt->fmt_list, &pfm_buffer_fmt_list);
  1049. printk(KERN_INFO "perfmon: added sampling format %s\n", fmt->fmt_name);
  1050. out:
  1051. spin_unlock(&pfm_buffer_fmt_lock);
  1052. return ret;
  1053. }
  1054. EXPORT_SYMBOL(pfm_register_buffer_fmt);
  1055. int
  1056. pfm_unregister_buffer_fmt(pfm_uuid_t uuid)
  1057. {
  1058. pfm_buffer_fmt_t *fmt;
  1059. int ret = 0;
  1060. spin_lock(&pfm_buffer_fmt_lock);
  1061. fmt = __pfm_find_buffer_fmt(uuid);
  1062. if (!fmt) {
  1063. printk(KERN_ERR "perfmon: cannot unregister format, not found\n");
  1064. ret = -EINVAL;
  1065. goto out;
  1066. }
  1067. list_del_init(&fmt->fmt_list);
  1068. printk(KERN_INFO "perfmon: removed sampling format: %s\n", fmt->fmt_name);
  1069. out:
  1070. spin_unlock(&pfm_buffer_fmt_lock);
  1071. return ret;
  1072. }
  1073. EXPORT_SYMBOL(pfm_unregister_buffer_fmt);
  1074. static int
  1075. pfm_reserve_session(struct task_struct *task, int is_syswide, unsigned int cpu)
  1076. {
  1077. unsigned long flags;
  1078. /*
  1079. * validy checks on cpu_mask have been done upstream
  1080. */
  1081. LOCK_PFS(flags);
  1082. DPRINT(("in sys_sessions=%u task_sessions=%u dbregs=%u syswide=%d cpu=%u\n",
  1083. pfm_sessions.pfs_sys_sessions,
  1084. pfm_sessions.pfs_task_sessions,
  1085. pfm_sessions.pfs_sys_use_dbregs,
  1086. is_syswide,
  1087. cpu));
  1088. if (is_syswide) {
  1089. /*
  1090. * cannot mix system wide and per-task sessions
  1091. */
  1092. if (pfm_sessions.pfs_task_sessions > 0UL) {
  1093. DPRINT(("system wide not possible, %u conflicting task_sessions\n",
  1094. pfm_sessions.pfs_task_sessions));
  1095. goto abort;
  1096. }
  1097. if (pfm_sessions.pfs_sys_session[cpu]) goto error_conflict;
  1098. DPRINT(("reserving system wide session on CPU%u currently on CPU%u\n", cpu, smp_processor_id()));
  1099. pfm_sessions.pfs_sys_session[cpu] = task;
  1100. pfm_sessions.pfs_sys_sessions++ ;
  1101. } else {
  1102. if (pfm_sessions.pfs_sys_sessions) goto abort;
  1103. pfm_sessions.pfs_task_sessions++;
  1104. }
  1105. DPRINT(("out sys_sessions=%u task_sessions=%u dbregs=%u syswide=%d cpu=%u\n",
  1106. pfm_sessions.pfs_sys_sessions,
  1107. pfm_sessions.pfs_task_sessions,
  1108. pfm_sessions.pfs_sys_use_dbregs,
  1109. is_syswide,
  1110. cpu));
  1111. UNLOCK_PFS(flags);
  1112. return 0;
  1113. error_conflict:
  1114. DPRINT(("system wide not possible, conflicting session [%d] on CPU%d\n",
  1115. pfm_sessions.pfs_sys_session[cpu]->pid,
  1116. smp_processor_id()));
  1117. abort:
  1118. UNLOCK_PFS(flags);
  1119. return -EBUSY;
  1120. }
  1121. static int
  1122. pfm_unreserve_session(pfm_context_t *ctx, int is_syswide, unsigned int cpu)
  1123. {
  1124. unsigned long flags;
  1125. /*
  1126. * validy checks on cpu_mask have been done upstream
  1127. */
  1128. LOCK_PFS(flags);
  1129. DPRINT(("in sys_sessions=%u task_sessions=%u dbregs=%u syswide=%d cpu=%u\n",
  1130. pfm_sessions.pfs_sys_sessions,
  1131. pfm_sessions.pfs_task_sessions,
  1132. pfm_sessions.pfs_sys_use_dbregs,
  1133. is_syswide,
  1134. cpu));
  1135. if (is_syswide) {
  1136. pfm_sessions.pfs_sys_session[cpu] = NULL;
  1137. /*
  1138. * would not work with perfmon+more than one bit in cpu_mask
  1139. */
  1140. if (ctx && ctx->ctx_fl_using_dbreg) {
  1141. if (pfm_sessions.pfs_sys_use_dbregs == 0) {
  1142. printk(KERN_ERR "perfmon: invalid release for ctx %p sys_use_dbregs=0\n", ctx);
  1143. } else {
  1144. pfm_sessions.pfs_sys_use_dbregs--;
  1145. }
  1146. }
  1147. pfm_sessions.pfs_sys_sessions--;
  1148. } else {
  1149. pfm_sessions.pfs_task_sessions--;
  1150. }
  1151. DPRINT(("out sys_sessions=%u task_sessions=%u dbregs=%u syswide=%d cpu=%u\n",
  1152. pfm_sessions.pfs_sys_sessions,
  1153. pfm_sessions.pfs_task_sessions,
  1154. pfm_sessions.pfs_sys_use_dbregs,
  1155. is_syswide,
  1156. cpu));
  1157. UNLOCK_PFS(flags);
  1158. return 0;
  1159. }
  1160. /*
  1161. * removes virtual mapping of the sampling buffer.
  1162. * IMPORTANT: cannot be called with interrupts disable, e.g. inside
  1163. * a PROTECT_CTX() section.
  1164. */
  1165. static int
  1166. pfm_remove_smpl_mapping(struct task_struct *task, void *vaddr, unsigned long size)
  1167. {
  1168. int r;
  1169. /* sanity checks */
  1170. if (task->mm == NULL || size == 0UL || vaddr == NULL) {
  1171. printk(KERN_ERR "perfmon: pfm_remove_smpl_mapping [%d] invalid context mm=%p\n", task->pid, task->mm);
  1172. return -EINVAL;
  1173. }
  1174. DPRINT(("smpl_vaddr=%p size=%lu\n", vaddr, size));
  1175. /*
  1176. * does the actual unmapping
  1177. */
  1178. down_write(&task->mm->mmap_sem);
  1179. DPRINT(("down_write done smpl_vaddr=%p size=%lu\n", vaddr, size));
  1180. r = pfm_do_munmap(task->mm, (unsigned long)vaddr, size, 0);
  1181. up_write(&task->mm->mmap_sem);
  1182. if (r !=0) {
  1183. printk(KERN_ERR "perfmon: [%d] unable to unmap sampling buffer @%p size=%lu\n", task->pid, vaddr, size);
  1184. }
  1185. DPRINT(("do_unmap(%p, %lu)=%d\n", vaddr, size, r));
  1186. return 0;
  1187. }
  1188. /*
  1189. * free actual physical storage used by sampling buffer
  1190. */
  1191. #if 0
  1192. static int
  1193. pfm_free_smpl_buffer(pfm_context_t *ctx)
  1194. {
  1195. pfm_buffer_fmt_t *fmt;
  1196. if (ctx->ctx_smpl_hdr == NULL) goto invalid_free;
  1197. /*
  1198. * we won't use the buffer format anymore
  1199. */
  1200. fmt = ctx->ctx_buf_fmt;
  1201. DPRINT(("sampling buffer @%p size %lu vaddr=%p\n",
  1202. ctx->ctx_smpl_hdr,
  1203. ctx->ctx_smpl_size,
  1204. ctx->ctx_smpl_vaddr));
  1205. pfm_buf_fmt_exit(fmt, current, NULL, NULL);
  1206. /*
  1207. * free the buffer
  1208. */
  1209. pfm_rvfree(ctx->ctx_smpl_hdr, ctx->ctx_smpl_size);
  1210. ctx->ctx_smpl_hdr = NULL;
  1211. ctx->ctx_smpl_size = 0UL;
  1212. return 0;
  1213. invalid_free:
  1214. printk(KERN_ERR "perfmon: pfm_free_smpl_buffer [%d] no buffer\n", current->pid);
  1215. return -EINVAL;
  1216. }
  1217. #endif
  1218. static inline void
  1219. pfm_exit_smpl_buffer(pfm_buffer_fmt_t *fmt)
  1220. {
  1221. if (fmt == NULL) return;
  1222. pfm_buf_fmt_exit(fmt, current, NULL, NULL);
  1223. }
  1224. /*
  1225. * pfmfs should _never_ be mounted by userland - too much of security hassle,
  1226. * no real gain from having the whole whorehouse mounted. So we don't need
  1227. * any operations on the root directory. However, we need a non-trivial
  1228. * d_name - pfm: will go nicely and kill the special-casing in procfs.
  1229. */
  1230. static struct vfsmount *pfmfs_mnt;
  1231. static int __init
  1232. init_pfm_fs(void)
  1233. {
  1234. int err = register_filesystem(&pfm_fs_type);
  1235. if (!err) {
  1236. pfmfs_mnt = kern_mount(&pfm_fs_type);
  1237. err = PTR_ERR(pfmfs_mnt);
  1238. if (IS_ERR(pfmfs_mnt))
  1239. unregister_filesystem(&pfm_fs_type);
  1240. else
  1241. err = 0;
  1242. }
  1243. return err;
  1244. }
  1245. static void __exit
  1246. exit_pfm_fs(void)
  1247. {
  1248. unregister_filesystem(&pfm_fs_type);
  1249. mntput(pfmfs_mnt);
  1250. }
  1251. static ssize_t
  1252. pfm_read(struct file *filp, char __user *buf, size_t size, loff_t *ppos)
  1253. {
  1254. pfm_context_t *ctx;
  1255. pfm_msg_t *msg;
  1256. ssize_t ret;
  1257. unsigned long flags;
  1258. DECLARE_WAITQUEUE(wait, current);
  1259. if (PFM_IS_FILE(filp) == 0) {
  1260. printk(KERN_ERR "perfmon: pfm_poll: bad magic [%d]\n", current->pid);
  1261. return -EINVAL;
  1262. }
  1263. ctx = (pfm_context_t *)filp->private_data;
  1264. if (ctx == NULL) {
  1265. printk(KERN_ERR "perfmon: pfm_read: NULL ctx [%d]\n", current->pid);
  1266. return -EINVAL;
  1267. }
  1268. /*
  1269. * check even when there is no message
  1270. */
  1271. if (size < sizeof(pfm_msg_t)) {
  1272. DPRINT(("message is too small ctx=%p (>=%ld)\n", ctx, sizeof(pfm_msg_t)));
  1273. return -EINVAL;
  1274. }
  1275. PROTECT_CTX(ctx, flags);
  1276. /*
  1277. * put ourselves on the wait queue
  1278. */
  1279. add_wait_queue(&ctx->ctx_msgq_wait, &wait);
  1280. for(;;) {
  1281. /*
  1282. * check wait queue
  1283. */
  1284. set_current_state(TASK_INTERRUPTIBLE);
  1285. DPRINT(("head=%d tail=%d\n", ctx->ctx_msgq_head, ctx->ctx_msgq_tail));
  1286. ret = 0;
  1287. if(PFM_CTXQ_EMPTY(ctx) == 0) break;
  1288. UNPROTECT_CTX(ctx, flags);
  1289. /*
  1290. * check non-blocking read
  1291. */
  1292. ret = -EAGAIN;
  1293. if(filp->f_flags & O_NONBLOCK) break;
  1294. /*
  1295. * check pending signals
  1296. */
  1297. if(signal_pending(current)) {
  1298. ret = -EINTR;
  1299. break;
  1300. }
  1301. /*
  1302. * no message, so wait
  1303. */
  1304. schedule();
  1305. PROTECT_CTX(ctx, flags);
  1306. }
  1307. DPRINT(("[%d] back to running ret=%ld\n", current->pid, ret));
  1308. set_current_state(TASK_RUNNING);
  1309. remove_wait_queue(&ctx->ctx_msgq_wait, &wait);
  1310. if (ret < 0) goto abort;
  1311. ret = -EINVAL;
  1312. msg = pfm_get_next_msg(ctx);
  1313. if (msg == NULL) {
  1314. printk(KERN_ERR "perfmon: pfm_read no msg for ctx=%p [%d]\n", ctx, current->pid);
  1315. goto abort_locked;
  1316. }
  1317. DPRINT(("[%d] fd=%d type=%d\n", current->pid, msg->pfm_gen_msg.msg_ctx_fd, msg->pfm_gen_msg.msg_type));
  1318. ret = -EFAULT;
  1319. if(copy_to_user(buf, msg, sizeof(pfm_msg_t)) == 0) ret = sizeof(pfm_msg_t);
  1320. abort_locked:
  1321. UNPROTECT_CTX(ctx, flags);
  1322. abort:
  1323. return ret;
  1324. }
  1325. static ssize_t
  1326. pfm_write(struct file *file, const char __user *ubuf,
  1327. size_t size, loff_t *ppos)
  1328. {
  1329. DPRINT(("pfm_write called\n"));
  1330. return -EINVAL;
  1331. }
  1332. static unsigned int
  1333. pfm_poll(struct file *filp, poll_table * wait)
  1334. {
  1335. pfm_context_t *ctx;
  1336. unsigned long flags;
  1337. unsigned int mask = 0;
  1338. if (PFM_IS_FILE(filp) == 0) {
  1339. printk(KERN_ERR "perfmon: pfm_poll: bad magic [%d]\n", current->pid);
  1340. return 0;
  1341. }
  1342. ctx = (pfm_context_t *)filp->private_data;
  1343. if (ctx == NULL) {
  1344. printk(KERN_ERR "perfmon: pfm_poll: NULL ctx [%d]\n", current->pid);
  1345. return 0;
  1346. }
  1347. DPRINT(("pfm_poll ctx_fd=%d before poll_wait\n", ctx->ctx_fd));
  1348. poll_wait(filp, &ctx->ctx_msgq_wait, wait);
  1349. PROTECT_CTX(ctx, flags);
  1350. if (PFM_CTXQ_EMPTY(ctx) == 0)
  1351. mask = POLLIN | POLLRDNORM;
  1352. UNPROTECT_CTX(ctx, flags);
  1353. DPRINT(("pfm_poll ctx_fd=%d mask=0x%x\n", ctx->ctx_fd, mask));
  1354. return mask;
  1355. }
  1356. static int
  1357. pfm_ioctl(struct inode *inode, struct file *file, unsigned int cmd, unsigned long arg)
  1358. {
  1359. DPRINT(("pfm_ioctl called\n"));
  1360. return -EINVAL;
  1361. }
  1362. /*
  1363. * interrupt cannot be masked when coming here
  1364. */
  1365. static inline int
  1366. pfm_do_fasync(int fd, struct file *filp, pfm_context_t *ctx, int on)
  1367. {
  1368. int ret;
  1369. ret = fasync_helper (fd, filp, on, &ctx->ctx_async_queue);
  1370. DPRINT(("pfm_fasync called by [%d] on ctx_fd=%d on=%d async_queue=%p ret=%d\n",
  1371. current->pid,
  1372. fd,
  1373. on,
  1374. ctx->ctx_async_queue, ret));
  1375. return ret;
  1376. }
  1377. static int
  1378. pfm_fasync(int fd, struct file *filp, int on)
  1379. {
  1380. pfm_context_t *ctx;
  1381. int ret;
  1382. if (PFM_IS_FILE(filp) == 0) {
  1383. printk(KERN_ERR "perfmon: pfm_fasync bad magic [%d]\n", current->pid);
  1384. return -EBADF;
  1385. }
  1386. ctx = (pfm_context_t *)filp->private_data;
  1387. if (ctx == NULL) {
  1388. printk(KERN_ERR "perfmon: pfm_fasync NULL ctx [%d]\n", current->pid);
  1389. return -EBADF;
  1390. }
  1391. /*
  1392. * we cannot mask interrupts during this call because this may
  1393. * may go to sleep if memory is not readily avalaible.
  1394. *
  1395. * We are protected from the conetxt disappearing by the get_fd()/put_fd()
  1396. * done in caller. Serialization of this function is ensured by caller.
  1397. */
  1398. ret = pfm_do_fasync(fd, filp, ctx, on);
  1399. DPRINT(("pfm_fasync called on ctx_fd=%d on=%d async_queue=%p ret=%d\n",
  1400. fd,
  1401. on,
  1402. ctx->ctx_async_queue, ret));
  1403. return ret;
  1404. }
  1405. #ifdef CONFIG_SMP
  1406. /*
  1407. * this function is exclusively called from pfm_close().
  1408. * The context is not protected at that time, nor are interrupts
  1409. * on the remote CPU. That's necessary to avoid deadlocks.
  1410. */
  1411. static void
  1412. pfm_syswide_force_stop(void *info)
  1413. {
  1414. pfm_context_t *ctx = (pfm_context_t *)info;
  1415. struct pt_regs *regs = ia64_task_regs(current);
  1416. struct task_struct *owner;
  1417. unsigned long flags;
  1418. int ret;
  1419. if (ctx->ctx_cpu != smp_processor_id()) {
  1420. printk(KERN_ERR "perfmon: pfm_syswide_force_stop for CPU%d but on CPU%d\n",
  1421. ctx->ctx_cpu,
  1422. smp_processor_id());
  1423. return;
  1424. }
  1425. owner = GET_PMU_OWNER();
  1426. if (owner != ctx->ctx_task) {
  1427. printk(KERN_ERR "perfmon: pfm_syswide_force_stop CPU%d unexpected owner [%d] instead of [%d]\n",
  1428. smp_processor_id(),
  1429. owner->pid, ctx->ctx_task->pid);
  1430. return;
  1431. }
  1432. if (GET_PMU_CTX() != ctx) {
  1433. printk(KERN_ERR "perfmon: pfm_syswide_force_stop CPU%d unexpected ctx %p instead of %p\n",
  1434. smp_processor_id(),
  1435. GET_PMU_CTX(), ctx);
  1436. return;
  1437. }
  1438. DPRINT(("on CPU%d forcing system wide stop for [%d]\n", smp_processor_id(), ctx->ctx_task->pid));
  1439. /*
  1440. * the context is already protected in pfm_close(), we simply
  1441. * need to mask interrupts to avoid a PMU interrupt race on
  1442. * this CPU
  1443. */
  1444. local_irq_save(flags);
  1445. ret = pfm_context_unload(ctx, NULL, 0, regs);
  1446. if (ret) {
  1447. DPRINT(("context_unload returned %d\n", ret));
  1448. }
  1449. /*
  1450. * unmask interrupts, PMU interrupts are now spurious here
  1451. */
  1452. local_irq_restore(flags);
  1453. }
  1454. static void
  1455. pfm_syswide_cleanup_other_cpu(pfm_context_t *ctx)
  1456. {
  1457. int ret;
  1458. DPRINT(("calling CPU%d for cleanup\n", ctx->ctx_cpu));
  1459. ret = smp_call_function_single(ctx->ctx_cpu, pfm_syswide_force_stop, ctx, 0, 1);
  1460. DPRINT(("called CPU%d for cleanup ret=%d\n", ctx->ctx_cpu, ret));
  1461. }
  1462. #endif /* CONFIG_SMP */
  1463. /*
  1464. * called for each close(). Partially free resources.
  1465. * When caller is self-monitoring, the context is unloaded.
  1466. */
  1467. static int
  1468. pfm_flush(struct file *filp)
  1469. {
  1470. pfm_context_t *ctx;
  1471. struct task_struct *task;
  1472. struct pt_regs *regs;
  1473. unsigned long flags;
  1474. unsigned long smpl_buf_size = 0UL;
  1475. void *smpl_buf_vaddr = NULL;
  1476. int state, is_system;
  1477. if (PFM_IS_FILE(filp) == 0) {
  1478. DPRINT(("bad magic for\n"));
  1479. return -EBADF;
  1480. }
  1481. ctx = (pfm_context_t *)filp->private_data;
  1482. if (ctx == NULL) {
  1483. printk(KERN_ERR "perfmon: pfm_flush: NULL ctx [%d]\n", current->pid);
  1484. return -EBADF;
  1485. }
  1486. /*
  1487. * remove our file from the async queue, if we use this mode.
  1488. * This can be done without the context being protected. We come
  1489. * here when the context has become unreacheable by other tasks.
  1490. *
  1491. * We may still have active monitoring at this point and we may
  1492. * end up in pfm_overflow_handler(). However, fasync_helper()
  1493. * operates with interrupts disabled and it cleans up the
  1494. * queue. If the PMU handler is called prior to entering
  1495. * fasync_helper() then it will send a signal. If it is
  1496. * invoked after, it will find an empty queue and no
  1497. * signal will be sent. In both case, we are safe
  1498. */
  1499. if (filp->f_flags & FASYNC) {
  1500. DPRINT(("cleaning up async_queue=%p\n", ctx->ctx_async_queue));
  1501. pfm_do_fasync (-1, filp, ctx, 0);
  1502. }
  1503. PROTECT_CTX(ctx, flags);
  1504. state = ctx->ctx_state;
  1505. is_system = ctx->ctx_fl_system;
  1506. task = PFM_CTX_TASK(ctx);
  1507. regs = ia64_task_regs(task);
  1508. DPRINT(("ctx_state=%d is_current=%d\n",
  1509. state,
  1510. task == current ? 1 : 0));
  1511. /*
  1512. * if state == UNLOADED, then task is NULL
  1513. */
  1514. /*
  1515. * we must stop and unload because we are losing access to the context.
  1516. */
  1517. if (task == current) {
  1518. #ifdef CONFIG_SMP
  1519. /*
  1520. * the task IS the owner but it migrated to another CPU: that's bad
  1521. * but we must handle this cleanly. Unfortunately, the kernel does
  1522. * not provide a mechanism to block migration (while the context is loaded).
  1523. *
  1524. * We need to release the resource on the ORIGINAL cpu.
  1525. */
  1526. if (is_system && ctx->ctx_cpu != smp_processor_id()) {
  1527. DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
  1528. /*
  1529. * keep context protected but unmask interrupt for IPI
  1530. */
  1531. local_irq_restore(flags);
  1532. pfm_syswide_cleanup_other_cpu(ctx);
  1533. /*
  1534. * restore interrupt masking
  1535. */
  1536. local_irq_save(flags);
  1537. /*
  1538. * context is unloaded at this point
  1539. */
  1540. } else
  1541. #endif /* CONFIG_SMP */
  1542. {
  1543. DPRINT(("forcing unload\n"));
  1544. /*
  1545. * stop and unload, returning with state UNLOADED
  1546. * and session unreserved.
  1547. */
  1548. pfm_context_unload(ctx, NULL, 0, regs);
  1549. DPRINT(("ctx_state=%d\n", ctx->ctx_state));
  1550. }
  1551. }
  1552. /*
  1553. * remove virtual mapping, if any, for the calling task.
  1554. * cannot reset ctx field until last user is calling close().
  1555. *
  1556. * ctx_smpl_vaddr must never be cleared because it is needed
  1557. * by every task with access to the context
  1558. *
  1559. * When called from do_exit(), the mm context is gone already, therefore
  1560. * mm is NULL, i.e., the VMA is already gone and we do not have to
  1561. * do anything here
  1562. */
  1563. if (ctx->ctx_smpl_vaddr && current->mm) {
  1564. smpl_buf_vaddr = ctx->ctx_smpl_vaddr;
  1565. smpl_buf_size = ctx->ctx_smpl_size;
  1566. }
  1567. UNPROTECT_CTX(ctx, flags);
  1568. /*
  1569. * if there was a mapping, then we systematically remove it
  1570. * at this point. Cannot be done inside critical section
  1571. * because some VM function reenables interrupts.
  1572. *
  1573. */
  1574. if (smpl_buf_vaddr) pfm_remove_smpl_mapping(current, smpl_buf_vaddr, smpl_buf_size);
  1575. return 0;
  1576. }
  1577. /*
  1578. * called either on explicit close() or from exit_files().
  1579. * Only the LAST user of the file gets to this point, i.e., it is
  1580. * called only ONCE.
  1581. *
  1582. * IMPORTANT: we get called ONLY when the refcnt on the file gets to zero
  1583. * (fput()),i.e, last task to access the file. Nobody else can access the
  1584. * file at this point.
  1585. *
  1586. * When called from exit_files(), the VMA has been freed because exit_mm()
  1587. * is executed before exit_files().
  1588. *
  1589. * When called from exit_files(), the current task is not yet ZOMBIE but we
  1590. * flush the PMU state to the context.
  1591. */
  1592. static int
  1593. pfm_close(struct inode *inode, struct file *filp)
  1594. {
  1595. pfm_context_t *ctx;
  1596. struct task_struct *task;
  1597. struct pt_regs *regs;
  1598. DECLARE_WAITQUEUE(wait, current);
  1599. unsigned long flags;
  1600. unsigned long smpl_buf_size = 0UL;
  1601. void *smpl_buf_addr = NULL;
  1602. int free_possible = 1;
  1603. int state, is_system;
  1604. DPRINT(("pfm_close called private=%p\n", filp->private_data));
  1605. if (PFM_IS_FILE(filp) == 0) {
  1606. DPRINT(("bad magic\n"));
  1607. return -EBADF;
  1608. }
  1609. ctx = (pfm_context_t *)filp->private_data;
  1610. if (ctx == NULL) {
  1611. printk(KERN_ERR "perfmon: pfm_close: NULL ctx [%d]\n", current->pid);
  1612. return -EBADF;
  1613. }
  1614. PROTECT_CTX(ctx, flags);
  1615. state = ctx->ctx_state;
  1616. is_system = ctx->ctx_fl_system;
  1617. task = PFM_CTX_TASK(ctx);
  1618. regs = ia64_task_regs(task);
  1619. DPRINT(("ctx_state=%d is_current=%d\n",
  1620. state,
  1621. task == current ? 1 : 0));
  1622. /*
  1623. * if task == current, then pfm_flush() unloaded the context
  1624. */
  1625. if (state == PFM_CTX_UNLOADED) goto doit;
  1626. /*
  1627. * context is loaded/masked and task != current, we need to
  1628. * either force an unload or go zombie
  1629. */
  1630. /*
  1631. * The task is currently blocked or will block after an overflow.
  1632. * we must force it to wakeup to get out of the
  1633. * MASKED state and transition to the unloaded state by itself.
  1634. *
  1635. * This situation is only possible for per-task mode
  1636. */
  1637. if (state == PFM_CTX_MASKED && CTX_OVFL_NOBLOCK(ctx) == 0) {
  1638. /*
  1639. * set a "partial" zombie state to be checked
  1640. * upon return from down() in pfm_handle_work().
  1641. *
  1642. * We cannot use the ZOMBIE state, because it is checked
  1643. * by pfm_load_regs() which is called upon wakeup from down().
  1644. * In such case, it would free the context and then we would
  1645. * return to pfm_handle_work() which would access the
  1646. * stale context. Instead, we set a flag invisible to pfm_load_regs()
  1647. * but visible to pfm_handle_work().
  1648. *
  1649. * For some window of time, we have a zombie context with
  1650. * ctx_state = MASKED and not ZOMBIE
  1651. */
  1652. ctx->ctx_fl_going_zombie = 1;
  1653. /*
  1654. * force task to wake up from MASKED state
  1655. */
  1656. up(&ctx->ctx_restart_sem);
  1657. DPRINT(("waking up ctx_state=%d\n", state));
  1658. /*
  1659. * put ourself to sleep waiting for the other
  1660. * task to report completion
  1661. *
  1662. * the context is protected by mutex, therefore there
  1663. * is no risk of being notified of completion before
  1664. * begin actually on the waitq.
  1665. */
  1666. set_current_state(TASK_INTERRUPTIBLE);
  1667. add_wait_queue(&ctx->ctx_zombieq, &wait);
  1668. UNPROTECT_CTX(ctx, flags);
  1669. /*
  1670. * XXX: check for signals :
  1671. * - ok for explicit close
  1672. * - not ok when coming from exit_files()
  1673. */
  1674. schedule();
  1675. PROTECT_CTX(ctx, flags);
  1676. remove_wait_queue(&ctx->ctx_zombieq, &wait);
  1677. set_current_state(TASK_RUNNING);
  1678. /*
  1679. * context is unloaded at this point
  1680. */
  1681. DPRINT(("after zombie wakeup ctx_state=%d for\n", state));
  1682. }
  1683. else if (task != current) {
  1684. #ifdef CONFIG_SMP
  1685. /*
  1686. * switch context to zombie state
  1687. */
  1688. ctx->ctx_state = PFM_CTX_ZOMBIE;
  1689. DPRINT(("zombie ctx for [%d]\n", task->pid));
  1690. /*
  1691. * cannot free the context on the spot. deferred until
  1692. * the task notices the ZOMBIE state
  1693. */
  1694. free_possible = 0;
  1695. #else
  1696. pfm_context_unload(ctx, NULL, 0, regs);
  1697. #endif
  1698. }
  1699. doit:
  1700. /* reload state, may have changed during opening of critical section */
  1701. state = ctx->ctx_state;
  1702. /*
  1703. * the context is still attached to a task (possibly current)
  1704. * we cannot destroy it right now
  1705. */
  1706. /*
  1707. * we must free the sampling buffer right here because
  1708. * we cannot rely on it being cleaned up later by the
  1709. * monitored task. It is not possible to free vmalloc'ed
  1710. * memory in pfm_load_regs(). Instead, we remove the buffer
  1711. * now. should there be subsequent PMU overflow originally
  1712. * meant for sampling, the will be converted to spurious
  1713. * and that's fine because the monitoring tools is gone anyway.
  1714. */
  1715. if (ctx->ctx_smpl_hdr) {
  1716. smpl_buf_addr = ctx->ctx_smpl_hdr;
  1717. smpl_buf_size = ctx->ctx_smpl_size;
  1718. /* no more sampling */
  1719. ctx->ctx_smpl_hdr = NULL;
  1720. ctx->ctx_fl_is_sampling = 0;
  1721. }
  1722. DPRINT(("ctx_state=%d free_possible=%d addr=%p size=%lu\n",
  1723. state,
  1724. free_possible,
  1725. smpl_buf_addr,
  1726. smpl_buf_size));
  1727. if (smpl_buf_addr) pfm_exit_smpl_buffer(ctx->ctx_buf_fmt);
  1728. /*
  1729. * UNLOADED that the session has already been unreserved.
  1730. */
  1731. if (state == PFM_CTX_ZOMBIE) {
  1732. pfm_unreserve_session(ctx, ctx->ctx_fl_system , ctx->ctx_cpu);
  1733. }
  1734. /*
  1735. * disconnect file descriptor from context must be done
  1736. * before we unlock.
  1737. */
  1738. filp->private_data = NULL;
  1739. /*
  1740. * if we free on the spot, the context is now completely unreacheable
  1741. * from the callers side. The monitored task side is also cut, so we
  1742. * can freely cut.
  1743. *
  1744. * If we have a deferred free, only the caller side is disconnected.
  1745. */
  1746. UNPROTECT_CTX(ctx, flags);
  1747. /*
  1748. * All memory free operations (especially for vmalloc'ed memory)
  1749. * MUST be done with interrupts ENABLED.
  1750. */
  1751. if (smpl_buf_addr) pfm_rvfree(smpl_buf_addr, smpl_buf_size);
  1752. /*
  1753. * return the memory used by the context
  1754. */
  1755. if (free_possible) pfm_context_free(ctx);
  1756. return 0;
  1757. }
  1758. static int
  1759. pfm_no_open(struct inode *irrelevant, struct file *dontcare)
  1760. {
  1761. DPRINT(("pfm_no_open called\n"));
  1762. return -ENXIO;
  1763. }
  1764. static struct file_operations pfm_file_ops = {
  1765. .llseek = no_llseek,
  1766. .read = pfm_read,
  1767. .write = pfm_write,
  1768. .poll = pfm_poll,
  1769. .ioctl = pfm_ioctl,
  1770. .open = pfm_no_open, /* special open code to disallow open via /proc */
  1771. .fasync = pfm_fasync,
  1772. .release = pfm_close,
  1773. .flush = pfm_flush
  1774. };
  1775. static int
  1776. pfmfs_delete_dentry(struct dentry *dentry)
  1777. {
  1778. return 1;
  1779. }
  1780. static struct dentry_operations pfmfs_dentry_operations = {
  1781. .d_delete = pfmfs_delete_dentry,
  1782. };
  1783. static int
  1784. pfm_alloc_fd(struct file **cfile)
  1785. {
  1786. int fd, ret = 0;
  1787. struct file *file = NULL;
  1788. struct inode * inode;
  1789. char name[32];
  1790. struct qstr this;
  1791. fd = get_unused_fd();
  1792. if (fd < 0) return -ENFILE;
  1793. ret = -ENFILE;
  1794. file = get_empty_filp();
  1795. if (!file) goto out;
  1796. /*
  1797. * allocate a new inode
  1798. */
  1799. inode = new_inode(pfmfs_mnt->mnt_sb);
  1800. if (!inode) goto out;
  1801. DPRINT(("new inode ino=%ld @%p\n", inode->i_ino, inode));
  1802. inode->i_mode = S_IFCHR|S_IRUGO;
  1803. inode->i_uid = current->fsuid;
  1804. inode->i_gid = current->fsgid;
  1805. sprintf(name, "[%lu]", inode->i_ino);
  1806. this.name = name;
  1807. this.len = strlen(name);
  1808. this.hash = inode->i_ino;
  1809. ret = -ENOMEM;
  1810. /*
  1811. * allocate a new dcache entry
  1812. */
  1813. file->f_dentry = d_alloc(pfmfs_mnt->mnt_sb->s_root, &this);
  1814. if (!file->f_dentry) goto out;
  1815. file->f_dentry->d_op = &pfmfs_dentry_operations;
  1816. d_add(file->f_dentry, inode);
  1817. file->f_vfsmnt = mntget(pfmfs_mnt);
  1818. file->f_mapping = inode->i_mapping;
  1819. file->f_op = &pfm_file_ops;
  1820. file->f_mode = FMODE_READ;
  1821. file->f_flags = O_RDONLY;
  1822. file->f_pos = 0;
  1823. /*
  1824. * may have to delay until context is attached?
  1825. */
  1826. fd_install(fd, file);
  1827. /*
  1828. * the file structure we will use
  1829. */
  1830. *cfile = file;
  1831. return fd;
  1832. out:
  1833. if (file) put_filp(file);
  1834. put_unused_fd(fd);
  1835. return ret;
  1836. }
  1837. static void
  1838. pfm_free_fd(int fd, struct file *file)
  1839. {
  1840. struct files_struct *files = current->files;
  1841. /*
  1842. * there ie no fd_uninstall(), so we do it here
  1843. */
  1844. spin_lock(&files->file_lock);
  1845. files->fd[fd] = NULL;
  1846. spin_unlock(&files->file_lock);
  1847. if (file) put_filp(file);
  1848. put_unused_fd(fd);
  1849. }
  1850. static int
  1851. pfm_remap_buffer(struct vm_area_struct *vma, unsigned long buf, unsigned long addr, unsigned long size)
  1852. {
  1853. DPRINT(("CPU%d buf=0x%lx addr=0x%lx size=%ld\n", smp_processor_id(), buf, addr, size));
  1854. while (size > 0) {
  1855. unsigned long pfn = ia64_tpa(buf) >> PAGE_SHIFT;
  1856. if (remap_pfn_range(vma, addr, pfn, PAGE_SIZE, PAGE_READONLY))
  1857. return -ENOMEM;
  1858. addr += PAGE_SIZE;
  1859. buf += PAGE_SIZE;
  1860. size -= PAGE_SIZE;
  1861. }
  1862. return 0;
  1863. }
  1864. /*
  1865. * allocate a sampling buffer and remaps it into the user address space of the task
  1866. */
  1867. static int
  1868. pfm_smpl_buffer_alloc(struct task_struct *task, pfm_context_t *ctx, unsigned long rsize, void **user_vaddr)
  1869. {
  1870. struct mm_struct *mm = task->mm;
  1871. struct vm_area_struct *vma = NULL;
  1872. unsigned long size;
  1873. void *smpl_buf;
  1874. /*
  1875. * the fixed header + requested size and align to page boundary
  1876. */
  1877. size = PAGE_ALIGN(rsize);
  1878. DPRINT(("sampling buffer rsize=%lu size=%lu bytes\n", rsize, size));
  1879. /*
  1880. * check requested size to avoid Denial-of-service attacks
  1881. * XXX: may have to refine this test
  1882. * Check against address space limit.
  1883. *
  1884. * if ((mm->total_vm << PAGE_SHIFT) + len> task->rlim[RLIMIT_AS].rlim_cur)
  1885. * return -ENOMEM;
  1886. */
  1887. if (size > task->signal->rlim[RLIMIT_MEMLOCK].rlim_cur)
  1888. return -ENOMEM;
  1889. /*
  1890. * We do the easy to undo allocations first.
  1891. *
  1892. * pfm_rvmalloc(), clears the buffer, so there is no leak
  1893. */
  1894. smpl_buf = pfm_rvmalloc(size);
  1895. if (smpl_buf == NULL) {
  1896. DPRINT(("Can't allocate sampling buffer\n"));
  1897. return -ENOMEM;
  1898. }
  1899. DPRINT(("smpl_buf @%p\n", smpl_buf));
  1900. /* allocate vma */
  1901. vma = kmem_cache_alloc(vm_area_cachep, SLAB_KERNEL);
  1902. if (!vma) {
  1903. DPRINT(("Cannot allocate vma\n"));
  1904. goto error_kmem;
  1905. }
  1906. memset(vma, 0, sizeof(*vma));
  1907. /*
  1908. * partially initialize the vma for the sampling buffer
  1909. */
  1910. vma->vm_mm = mm;
  1911. vma->vm_flags = VM_READ| VM_MAYREAD |VM_RESERVED;
  1912. vma->vm_page_prot = PAGE_READONLY; /* XXX may need to change */
  1913. /*
  1914. * Now we have everything we need and we can initialize
  1915. * and connect all the data structures
  1916. */
  1917. ctx->ctx_smpl_hdr = smpl_buf;
  1918. ctx->ctx_smpl_size = size; /* aligned size */
  1919. /*
  1920. * Let's do the difficult operations next.
  1921. *
  1922. * now we atomically find some area in the address space and
  1923. * remap the buffer in it.
  1924. */
  1925. down_write(&task->mm->mmap_sem);
  1926. /* find some free area in address space, must have mmap sem held */
  1927. vma->vm_start = pfm_get_unmapped_area(NULL, 0, size, 0, MAP_PRIVATE|MAP_ANONYMOUS, 0);
  1928. if (vma->vm_start == 0UL) {
  1929. DPRINT(("Cannot find unmapped area for size %ld\n", size));
  1930. up_write(&task->mm->mmap_sem);
  1931. goto error;
  1932. }
  1933. vma->vm_end = vma->vm_start + size;
  1934. vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT;
  1935. DPRINT(("aligned size=%ld, hdr=%p mapped @0x%lx\n", size, ctx->ctx_smpl_hdr, vma->vm_start));
  1936. /* can only be applied to current task, need to have the mm semaphore held when called */
  1937. if (pfm_remap_buffer(vma, (unsigned long)smpl_buf, vma->vm_start, size)) {
  1938. DPRINT(("Can't remap buffer\n"));
  1939. up_write(&task->mm->mmap_sem);
  1940. goto error;
  1941. }
  1942. /*
  1943. * now insert the vma in the vm list for the process, must be
  1944. * done with mmap lock held
  1945. */
  1946. insert_vm_struct(mm, vma);
  1947. mm->total_vm += size >> PAGE_SHIFT;
  1948. vm_stat_account(vma);
  1949. up_write(&task->mm->mmap_sem);
  1950. /*
  1951. * keep track of user level virtual address
  1952. */
  1953. ctx->ctx_smpl_vaddr = (void *)vma->vm_start;
  1954. *(unsigned long *)user_vaddr = vma->vm_start;
  1955. return 0;
  1956. error:
  1957. kmem_cache_free(vm_area_cachep, vma);
  1958. error_kmem:
  1959. pfm_rvfree(smpl_buf, size);
  1960. return -ENOMEM;
  1961. }
  1962. /*
  1963. * XXX: do something better here
  1964. */
  1965. static int
  1966. pfm_bad_permissions(struct task_struct *task)
  1967. {
  1968. /* inspired by ptrace_attach() */
  1969. DPRINT(("cur: uid=%d gid=%d task: euid=%d suid=%d uid=%d egid=%d sgid=%d\n",
  1970. current->uid,
  1971. current->gid,
  1972. task->euid,
  1973. task->suid,
  1974. task->uid,
  1975. task->egid,
  1976. task->sgid));
  1977. return ((current->uid != task->euid)
  1978. || (current->uid != task->suid)
  1979. || (current->uid != task->uid)
  1980. || (current->gid != task->egid)
  1981. || (current->gid != task->sgid)
  1982. || (current->gid != task->gid)) && !capable(CAP_SYS_PTRACE);
  1983. }
  1984. static int
  1985. pfarg_is_sane(struct task_struct *task, pfarg_context_t *pfx)
  1986. {
  1987. int ctx_flags;
  1988. /* valid signal */
  1989. ctx_flags = pfx->ctx_flags;
  1990. if (ctx_flags & PFM_FL_SYSTEM_WIDE) {
  1991. /*
  1992. * cannot block in this mode
  1993. */
  1994. if (ctx_flags & PFM_FL_NOTIFY_BLOCK) {
  1995. DPRINT(("cannot use blocking mode when in system wide monitoring\n"));
  1996. return -EINVAL;
  1997. }
  1998. } else {
  1999. }
  2000. /* probably more to add here */
  2001. return 0;
  2002. }
  2003. static int
  2004. pfm_setup_buffer_fmt(struct task_struct *task, pfm_context_t *ctx, unsigned int ctx_flags,
  2005. unsigned int cpu, pfarg_context_t *arg)
  2006. {
  2007. pfm_buffer_fmt_t *fmt = NULL;
  2008. unsigned long size = 0UL;
  2009. void *uaddr = NULL;
  2010. void *fmt_arg = NULL;
  2011. int ret = 0;
  2012. #define PFM_CTXARG_BUF_ARG(a) (pfm_buffer_fmt_t *)(a+1)
  2013. /* invoke and lock buffer format, if found */
  2014. fmt = pfm_find_buffer_fmt(arg->ctx_smpl_buf_id);
  2015. if (fmt == NULL) {
  2016. DPRINT(("[%d] cannot find buffer format\n", task->pid));
  2017. return -EINVAL;
  2018. }
  2019. /*
  2020. * buffer argument MUST be contiguous to pfarg_context_t
  2021. */
  2022. if (fmt->fmt_arg_size) fmt_arg = PFM_CTXARG_BUF_ARG(arg);
  2023. ret = pfm_buf_fmt_validate(fmt, task, ctx_flags, cpu, fmt_arg);
  2024. DPRINT(("[%d] after validate(0x%x,%d,%p)=%d\n", task->pid, ctx_flags, cpu, fmt_arg, ret));
  2025. if (ret) goto error;
  2026. /* link buffer format and context */
  2027. ctx->ctx_buf_fmt = fmt;
  2028. /*
  2029. * check if buffer format wants to use perfmon buffer allocation/mapping service
  2030. */
  2031. ret = pfm_buf_fmt_getsize(fmt, task, ctx_flags, cpu, fmt_arg, &size);
  2032. if (ret) goto error;
  2033. if (size) {
  2034. /*
  2035. * buffer is always remapped into the caller's address space
  2036. */
  2037. ret = pfm_smpl_buffer_alloc(current, ctx, size, &uaddr);
  2038. if (ret) goto error;
  2039. /* keep track of user address of buffer */
  2040. arg->ctx_smpl_vaddr = uaddr;
  2041. }
  2042. ret = pfm_buf_fmt_init(fmt, task, ctx->ctx_smpl_hdr, ctx_flags, cpu, fmt_arg);
  2043. error:
  2044. return ret;
  2045. }
  2046. static void
  2047. pfm_reset_pmu_state(pfm_context_t *ctx)
  2048. {
  2049. int i;
  2050. /*
  2051. * install reset values for PMC.
  2052. */
  2053. for (i=1; PMC_IS_LAST(i) == 0; i++) {
  2054. if (PMC_IS_IMPL(i) == 0) continue;
  2055. ctx->ctx_pmcs[i] = PMC_DFL_VAL(i);
  2056. DPRINT(("pmc[%d]=0x%lx\n", i, ctx->ctx_pmcs[i]));
  2057. }
  2058. /*
  2059. * PMD registers are set to 0UL when the context in memset()
  2060. */
  2061. /*
  2062. * On context switched restore, we must restore ALL pmc and ALL pmd even
  2063. * when they are not actively used by the task. In UP, the incoming process
  2064. * may otherwise pick up left over PMC, PMD state from the previous process.
  2065. * As opposed to PMD, stale PMC can cause harm to the incoming
  2066. * process because they may change what is being measured.
  2067. * Therefore, we must systematically reinstall the entire
  2068. * PMC state. In SMP, the same thing is possible on the
  2069. * same CPU but also on between 2 CPUs.
  2070. *
  2071. * The problem with PMD is information leaking especially
  2072. * to user level when psr.sp=0
  2073. *
  2074. * There is unfortunately no easy way to avoid this problem
  2075. * on either UP or SMP. This definitively slows down the
  2076. * pfm_load_regs() function.
  2077. */
  2078. /*
  2079. * bitmask of all PMCs accessible to this context
  2080. *
  2081. * PMC0 is treated differently.
  2082. */
  2083. ctx->ctx_all_pmcs[0] = pmu_conf->impl_pmcs[0] & ~0x1;
  2084. /*
  2085. * bitmask of all PMDs that are accesible to this context
  2086. */
  2087. ctx->ctx_all_pmds[0] = pmu_conf->impl_pmds[0];
  2088. DPRINT(("<%d> all_pmcs=0x%lx all_pmds=0x%lx\n", ctx->ctx_fd, ctx->ctx_all_pmcs[0],ctx->ctx_all_pmds[0]));
  2089. /*
  2090. * useful in case of re-enable after disable
  2091. */
  2092. ctx->ctx_used_ibrs[0] = 0UL;
  2093. ctx->ctx_used_dbrs[0] = 0UL;
  2094. }
  2095. static int
  2096. pfm_ctx_getsize(void *arg, size_t *sz)
  2097. {
  2098. pfarg_context_t *req = (pfarg_context_t *)arg;
  2099. pfm_buffer_fmt_t *fmt;
  2100. *sz = 0;
  2101. if (!pfm_uuid_cmp(req->ctx_smpl_buf_id, pfm_null_uuid)) return 0;
  2102. fmt = pfm_find_buffer_fmt(req->ctx_smpl_buf_id);
  2103. if (fmt == NULL) {
  2104. DPRINT(("cannot find buffer format\n"));
  2105. return -EINVAL;
  2106. }
  2107. /* get just enough to copy in user parameters */
  2108. *sz = fmt->fmt_arg_size;
  2109. DPRINT(("arg_size=%lu\n", *sz));
  2110. return 0;
  2111. }
  2112. /*
  2113. * cannot attach if :
  2114. * - kernel task
  2115. * - task not owned by caller
  2116. * - task incompatible with context mode
  2117. */
  2118. static int
  2119. pfm_task_incompatible(pfm_context_t *ctx, struct task_struct *task)
  2120. {
  2121. /*
  2122. * no kernel task or task not owner by caller
  2123. */
  2124. if (task->mm == NULL) {
  2125. DPRINT(("task [%d] has not memory context (kernel thread)\n", task->pid));
  2126. return -EPERM;
  2127. }
  2128. if (pfm_bad_permissions(task)) {
  2129. DPRINT(("no permission to attach to [%d]\n", task->pid));
  2130. return -EPERM;
  2131. }
  2132. /*
  2133. * cannot block in self-monitoring mode
  2134. */
  2135. if (CTX_OVFL_NOBLOCK(ctx) == 0 && task == current) {
  2136. DPRINT(("cannot load a blocking context on self for [%d]\n", task->pid));
  2137. return -EINVAL;
  2138. }
  2139. if (task->exit_state == EXIT_ZOMBIE) {
  2140. DPRINT(("cannot attach to zombie task [%d]\n", task->pid));
  2141. return -EBUSY;
  2142. }
  2143. /*
  2144. * always ok for self
  2145. */
  2146. if (task == current) return 0;
  2147. if ((task->state != TASK_STOPPED) && (task->state != TASK_TRACED)) {
  2148. DPRINT(("cannot attach to non-stopped task [%d] state=%ld\n", task->pid, task->state));
  2149. return -EBUSY;
  2150. }
  2151. /*
  2152. * make sure the task is off any CPU
  2153. */
  2154. wait_task_inactive(task);
  2155. /* more to come... */
  2156. return 0;
  2157. }
  2158. static int
  2159. pfm_get_task(pfm_context_t *ctx, pid_t pid, struct task_struct **task)
  2160. {
  2161. struct task_struct *p = current;
  2162. int ret;
  2163. /* XXX: need to add more checks here */
  2164. if (pid < 2) return -EPERM;
  2165. if (pid != current->pid) {
  2166. read_lock(&tasklist_lock);
  2167. p = find_task_by_pid(pid);
  2168. /* make sure task cannot go away while we operate on it */
  2169. if (p) get_task_struct(p);
  2170. read_unlock(&tasklist_lock);
  2171. if (p == NULL) return -ESRCH;
  2172. }
  2173. ret = pfm_task_incompatible(ctx, p);
  2174. if (ret == 0) {
  2175. *task = p;
  2176. } else if (p != current) {
  2177. pfm_put_task(p);
  2178. }
  2179. return ret;
  2180. }
  2181. static int
  2182. pfm_context_create(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
  2183. {
  2184. pfarg_context_t *req = (pfarg_context_t *)arg;
  2185. struct file *filp;
  2186. int ctx_flags;
  2187. int ret;
  2188. /* let's check the arguments first */
  2189. ret = pfarg_is_sane(current, req);
  2190. if (ret < 0) return ret;
  2191. ctx_flags = req->ctx_flags;
  2192. ret = -ENOMEM;
  2193. ctx = pfm_context_alloc();
  2194. if (!ctx) goto error;
  2195. ret = pfm_alloc_fd(&filp);
  2196. if (ret < 0) goto error_file;
  2197. req->ctx_fd = ctx->ctx_fd = ret;
  2198. /*
  2199. * attach context to file
  2200. */
  2201. filp->private_data = ctx;
  2202. /*
  2203. * does the user want to sample?
  2204. */
  2205. if (pfm_uuid_cmp(req->ctx_smpl_buf_id, pfm_null_uuid)) {
  2206. ret = pfm_setup_buffer_fmt(current, ctx, ctx_flags, 0, req);
  2207. if (ret) goto buffer_error;
  2208. }
  2209. /*
  2210. * init context protection lock
  2211. */
  2212. spin_lock_init(&ctx->ctx_lock);
  2213. /*
  2214. * context is unloaded
  2215. */
  2216. ctx->ctx_state = PFM_CTX_UNLOADED;
  2217. /*
  2218. * initialization of context's flags
  2219. */
  2220. ctx->ctx_fl_block = (ctx_flags & PFM_FL_NOTIFY_BLOCK) ? 1 : 0;
  2221. ctx->ctx_fl_system = (ctx_flags & PFM_FL_SYSTEM_WIDE) ? 1: 0;
  2222. ctx->ctx_fl_is_sampling = ctx->ctx_buf_fmt ? 1 : 0; /* assume record() is defined */
  2223. ctx->ctx_fl_no_msg = (ctx_flags & PFM_FL_OVFL_NO_MSG) ? 1: 0;
  2224. /*
  2225. * will move to set properties
  2226. * ctx->ctx_fl_excl_idle = (ctx_flags & PFM_FL_EXCL_IDLE) ? 1: 0;
  2227. */
  2228. /*
  2229. * init restart semaphore to locked
  2230. */
  2231. sema_init(&ctx->ctx_restart_sem, 0);
  2232. /*
  2233. * activation is used in SMP only
  2234. */
  2235. ctx->ctx_last_activation = PFM_INVALID_ACTIVATION;
  2236. SET_LAST_CPU(ctx, -1);
  2237. /*
  2238. * initialize notification message queue
  2239. */
  2240. ctx->ctx_msgq_head = ctx->ctx_msgq_tail = 0;
  2241. init_waitqueue_head(&ctx->ctx_msgq_wait);
  2242. init_waitqueue_head(&ctx->ctx_zombieq);
  2243. DPRINT(("ctx=%p flags=0x%x system=%d notify_block=%d excl_idle=%d no_msg=%d ctx_fd=%d \n",
  2244. ctx,
  2245. ctx_flags,
  2246. ctx->ctx_fl_system,
  2247. ctx->ctx_fl_block,
  2248. ctx->ctx_fl_excl_idle,
  2249. ctx->ctx_fl_no_msg,
  2250. ctx->ctx_fd));
  2251. /*
  2252. * initialize soft PMU state
  2253. */
  2254. pfm_reset_pmu_state(ctx);
  2255. return 0;
  2256. buffer_error:
  2257. pfm_free_fd(ctx->ctx_fd, filp);
  2258. if (ctx->ctx_buf_fmt) {
  2259. pfm_buf_fmt_exit(ctx->ctx_buf_fmt, current, NULL, regs);
  2260. }
  2261. error_file:
  2262. pfm_context_free(ctx);
  2263. error:
  2264. return ret;
  2265. }
  2266. static inline unsigned long
  2267. pfm_new_counter_value (pfm_counter_t *reg, int is_long_reset)
  2268. {
  2269. unsigned long val = is_long_reset ? reg->long_reset : reg->short_reset;
  2270. unsigned long new_seed, old_seed = reg->seed, mask = reg->mask;
  2271. extern unsigned long carta_random32 (unsigned long seed);
  2272. if (reg->flags & PFM_REGFL_RANDOM) {
  2273. new_seed = carta_random32(old_seed);
  2274. val -= (old_seed & mask); /* counter values are negative numbers! */
  2275. if ((mask >> 32) != 0)
  2276. /* construct a full 64-bit random value: */
  2277. new_seed |= carta_random32(old_seed >> 32) << 32;
  2278. reg->seed = new_seed;
  2279. }
  2280. reg->lval = val;
  2281. return val;
  2282. }
  2283. static void
  2284. pfm_reset_regs_masked(pfm_context_t *ctx, unsigned long *ovfl_regs, int is_long_reset)
  2285. {
  2286. unsigned long mask = ovfl_regs[0];
  2287. unsigned long reset_others = 0UL;
  2288. unsigned long val;
  2289. int i;
  2290. /*
  2291. * now restore reset value on sampling overflowed counters
  2292. */
  2293. mask >>= PMU_FIRST_COUNTER;
  2294. for(i = PMU_FIRST_COUNTER; mask; i++, mask >>= 1) {
  2295. if ((mask & 0x1UL) == 0UL) continue;
  2296. ctx->ctx_pmds[i].val = val = pfm_new_counter_value(ctx->ctx_pmds+ i, is_long_reset);
  2297. reset_others |= ctx->ctx_pmds[i].reset_pmds[0];
  2298. DPRINT_ovfl((" %s reset ctx_pmds[%d]=%lx\n", is_long_reset ? "long" : "short", i, val));
  2299. }
  2300. /*
  2301. * Now take care of resetting the other registers
  2302. */
  2303. for(i = 0; reset_others; i++, reset_others >>= 1) {
  2304. if ((reset_others & 0x1) == 0) continue;
  2305. ctx->ctx_pmds[i].val = val = pfm_new_counter_value(ctx->ctx_pmds + i, is_long_reset);
  2306. DPRINT_ovfl(("%s reset_others pmd[%d]=%lx\n",
  2307. is_long_reset ? "long" : "short", i, val));
  2308. }
  2309. }
  2310. static void
  2311. pfm_reset_regs(pfm_context_t *ctx, unsigned long *ovfl_regs, int is_long_reset)
  2312. {
  2313. unsigned long mask = ovfl_regs[0];
  2314. unsigned long reset_others = 0UL;
  2315. unsigned long val;
  2316. int i;
  2317. DPRINT_ovfl(("ovfl_regs=0x%lx is_long_reset=%d\n", ovfl_regs[0], is_long_reset));
  2318. if (ctx->ctx_state == PFM_CTX_MASKED) {
  2319. pfm_reset_regs_masked(ctx, ovfl_regs, is_long_reset);
  2320. return;
  2321. }
  2322. /*
  2323. * now restore reset value on sampling overflowed counters
  2324. */
  2325. mask >>= PMU_FIRST_COUNTER;
  2326. for(i = PMU_FIRST_COUNTER; mask; i++, mask >>= 1) {
  2327. if ((mask & 0x1UL) == 0UL) continue;
  2328. val = pfm_new_counter_value(ctx->ctx_pmds+ i, is_long_reset);
  2329. reset_others |= ctx->ctx_pmds[i].reset_pmds[0];
  2330. DPRINT_ovfl((" %s reset ctx_pmds[%d]=%lx\n", is_long_reset ? "long" : "short", i, val));
  2331. pfm_write_soft_counter(ctx, i, val);
  2332. }
  2333. /*
  2334. * Now take care of resetting the other registers
  2335. */
  2336. for(i = 0; reset_others; i++, reset_others >>= 1) {
  2337. if ((reset_others & 0x1) == 0) continue;
  2338. val = pfm_new_counter_value(ctx->ctx_pmds + i, is_long_reset);
  2339. if (PMD_IS_COUNTING(i)) {
  2340. pfm_write_soft_counter(ctx, i, val);
  2341. } else {
  2342. ia64_set_pmd(i, val);
  2343. }
  2344. DPRINT_ovfl(("%s reset_others pmd[%d]=%lx\n",
  2345. is_long_reset ? "long" : "short", i, val));
  2346. }
  2347. ia64_srlz_d();
  2348. }
  2349. static int
  2350. pfm_write_pmcs(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
  2351. {
  2352. struct thread_struct *thread = NULL;
  2353. struct task_struct *task;
  2354. pfarg_reg_t *req = (pfarg_reg_t *)arg;
  2355. unsigned long value, pmc_pm;
  2356. unsigned long smpl_pmds, reset_pmds, impl_pmds;
  2357. unsigned int cnum, reg_flags, flags, pmc_type;
  2358. int i, can_access_pmu = 0, is_loaded, is_system, expert_mode;
  2359. int is_monitor, is_counting, state;
  2360. int ret = -EINVAL;
  2361. pfm_reg_check_t wr_func;
  2362. #define PFM_CHECK_PMC_PM(x, y, z) ((x)->ctx_fl_system ^ PMC_PM(y, z))
  2363. state = ctx->ctx_state;
  2364. is_loaded = state == PFM_CTX_LOADED ? 1 : 0;
  2365. is_system = ctx->ctx_fl_system;
  2366. task = ctx->ctx_task;
  2367. impl_pmds = pmu_conf->impl_pmds[0];
  2368. if (state == PFM_CTX_ZOMBIE) return -EINVAL;
  2369. if (is_loaded) {
  2370. thread = &task->thread;
  2371. /*
  2372. * In system wide and when the context is loaded, access can only happen
  2373. * when the caller is running on the CPU being monitored by the session.
  2374. * It does not have to be the owner (ctx_task) of the context per se.
  2375. */
  2376. if (is_system && ctx->ctx_cpu != smp_processor_id()) {
  2377. DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
  2378. return -EBUSY;
  2379. }
  2380. can_access_pmu = GET_PMU_OWNER() == task || is_system ? 1 : 0;
  2381. }
  2382. expert_mode = pfm_sysctl.expert_mode;
  2383. for (i = 0; i < count; i++, req++) {
  2384. cnum = req->reg_num;
  2385. reg_flags = req->reg_flags;
  2386. value = req->reg_value;
  2387. smpl_pmds = req->reg_smpl_pmds[0];
  2388. reset_pmds = req->reg_reset_pmds[0];
  2389. flags = 0;
  2390. if (cnum >= PMU_MAX_PMCS) {
  2391. DPRINT(("pmc%u is invalid\n", cnum));
  2392. goto error;
  2393. }
  2394. pmc_type = pmu_conf->pmc_desc[cnum].type;
  2395. pmc_pm = (value >> pmu_conf->pmc_desc[cnum].pm_pos) & 0x1;
  2396. is_counting = (pmc_type & PFM_REG_COUNTING) == PFM_REG_COUNTING ? 1 : 0;
  2397. is_monitor = (pmc_type & PFM_REG_MONITOR) == PFM_REG_MONITOR ? 1 : 0;
  2398. /*
  2399. * we reject all non implemented PMC as well
  2400. * as attempts to modify PMC[0-3] which are used
  2401. * as status registers by the PMU
  2402. */
  2403. if ((pmc_type & PFM_REG_IMPL) == 0 || (pmc_type & PFM_REG_CONTROL) == PFM_REG_CONTROL) {
  2404. DPRINT(("pmc%u is unimplemented or no-access pmc_type=%x\n", cnum, pmc_type));
  2405. goto error;
  2406. }
  2407. wr_func = pmu_conf->pmc_desc[cnum].write_check;
  2408. /*
  2409. * If the PMC is a monitor, then if the value is not the default:
  2410. * - system-wide session: PMCx.pm=1 (privileged monitor)
  2411. * - per-task : PMCx.pm=0 (user monitor)
  2412. */
  2413. if (is_monitor && value != PMC_DFL_VAL(cnum) && is_system ^ pmc_pm) {
  2414. DPRINT(("pmc%u pmc_pm=%lu is_system=%d\n",
  2415. cnum,
  2416. pmc_pm,
  2417. is_system));
  2418. goto error;
  2419. }
  2420. if (is_counting) {
  2421. /*
  2422. * enforce generation of overflow interrupt. Necessary on all
  2423. * CPUs.
  2424. */
  2425. value |= 1 << PMU_PMC_OI;
  2426. if (reg_flags & PFM_REGFL_OVFL_NOTIFY) {
  2427. flags |= PFM_REGFL_OVFL_NOTIFY;
  2428. }
  2429. if (reg_flags & PFM_REGFL_RANDOM) flags |= PFM_REGFL_RANDOM;
  2430. /* verify validity of smpl_pmds */
  2431. if ((smpl_pmds & impl_pmds) != smpl_pmds) {
  2432. DPRINT(("invalid smpl_pmds 0x%lx for pmc%u\n", smpl_pmds, cnum));
  2433. goto error;
  2434. }
  2435. /* verify validity of reset_pmds */
  2436. if ((reset_pmds & impl_pmds) != reset_pmds) {
  2437. DPRINT(("invalid reset_pmds 0x%lx for pmc%u\n", reset_pmds, cnum));
  2438. goto error;
  2439. }
  2440. } else {
  2441. if (reg_flags & (PFM_REGFL_OVFL_NOTIFY|PFM_REGFL_RANDOM)) {
  2442. DPRINT(("cannot set ovfl_notify or random on pmc%u\n", cnum));
  2443. goto error;
  2444. }
  2445. /* eventid on non-counting monitors are ignored */
  2446. }
  2447. /*
  2448. * execute write checker, if any
  2449. */
  2450. if (likely(expert_mode == 0 && wr_func)) {
  2451. ret = (*wr_func)(task, ctx, cnum, &value, regs);
  2452. if (ret) goto error;
  2453. ret = -EINVAL;
  2454. }
  2455. /*
  2456. * no error on this register
  2457. */
  2458. PFM_REG_RETFLAG_SET(req->reg_flags, 0);
  2459. /*
  2460. * Now we commit the changes to the software state
  2461. */
  2462. /*
  2463. * update overflow information
  2464. */
  2465. if (is_counting) {
  2466. /*
  2467. * full flag update each time a register is programmed
  2468. */
  2469. ctx->ctx_pmds[cnum].flags = flags;
  2470. ctx->ctx_pmds[cnum].reset_pmds[0] = reset_pmds;
  2471. ctx->ctx_pmds[cnum].smpl_pmds[0] = smpl_pmds;
  2472. ctx->ctx_pmds[cnum].eventid = req->reg_smpl_eventid;
  2473. /*
  2474. * Mark all PMDS to be accessed as used.
  2475. *
  2476. * We do not keep track of PMC because we have to
  2477. * systematically restore ALL of them.
  2478. *
  2479. * We do not update the used_monitors mask, because
  2480. * if we have not programmed them, then will be in
  2481. * a quiescent state, therefore we will not need to
  2482. * mask/restore then when context is MASKED.
  2483. */
  2484. CTX_USED_PMD(ctx, reset_pmds);
  2485. CTX_USED_PMD(ctx, smpl_pmds);
  2486. /*
  2487. * make sure we do not try to reset on
  2488. * restart because we have established new values
  2489. */
  2490. if (state == PFM_CTX_MASKED) ctx->ctx_ovfl_regs[0] &= ~1UL << cnum;
  2491. }
  2492. /*
  2493. * Needed in case the user does not initialize the equivalent
  2494. * PMD. Clearing is done indirectly via pfm_reset_pmu_state() so there is no
  2495. * possible leak here.
  2496. */
  2497. CTX_USED_PMD(ctx, pmu_conf->pmc_desc[cnum].dep_pmd[0]);
  2498. /*
  2499. * keep track of the monitor PMC that we are using.
  2500. * we save the value of the pmc in ctx_pmcs[] and if
  2501. * the monitoring is not stopped for the context we also
  2502. * place it in the saved state area so that it will be
  2503. * picked up later by the context switch code.
  2504. *
  2505. * The value in ctx_pmcs[] can only be changed in pfm_write_pmcs().
  2506. *
  2507. * The value in thread->pmcs[] may be modified on overflow, i.e., when
  2508. * monitoring needs to be stopped.
  2509. */
  2510. if (is_monitor) CTX_USED_MONITOR(ctx, 1UL << cnum);
  2511. /*
  2512. * update context state
  2513. */
  2514. ctx->ctx_pmcs[cnum] = value;
  2515. if (is_loaded) {
  2516. /*
  2517. * write thread state
  2518. */
  2519. if (is_system == 0) thread->pmcs[cnum] = value;
  2520. /*
  2521. * write hardware register if we can
  2522. */
  2523. if (can_access_pmu) {
  2524. ia64_set_pmc(cnum, value);
  2525. }
  2526. #ifdef CONFIG_SMP
  2527. else {
  2528. /*
  2529. * per-task SMP only here
  2530. *
  2531. * we are guaranteed that the task is not running on the other CPU,
  2532. * we indicate that this PMD will need to be reloaded if the task
  2533. * is rescheduled on the CPU it ran last on.
  2534. */
  2535. ctx->ctx_reload_pmcs[0] |= 1UL << cnum;
  2536. }
  2537. #endif
  2538. }
  2539. DPRINT(("pmc[%u]=0x%lx ld=%d apmu=%d flags=0x%x all_pmcs=0x%lx used_pmds=0x%lx eventid=%ld smpl_pmds=0x%lx reset_pmds=0x%lx reloads_pmcs=0x%lx used_monitors=0x%lx ovfl_regs=0x%lx\n",
  2540. cnum,
  2541. value,
  2542. is_loaded,
  2543. can_access_pmu,
  2544. flags,
  2545. ctx->ctx_all_pmcs[0],
  2546. ctx->ctx_used_pmds[0],
  2547. ctx->ctx_pmds[cnum].eventid,
  2548. smpl_pmds,
  2549. reset_pmds,
  2550. ctx->ctx_reload_pmcs[0],
  2551. ctx->ctx_used_monitors[0],
  2552. ctx->ctx_ovfl_regs[0]));
  2553. }
  2554. /*
  2555. * make sure the changes are visible
  2556. */
  2557. if (can_access_pmu) ia64_srlz_d();
  2558. return 0;
  2559. error:
  2560. PFM_REG_RETFLAG_SET(req->reg_flags, PFM_REG_RETFL_EINVAL);
  2561. return ret;
  2562. }
  2563. static int
  2564. pfm_write_pmds(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
  2565. {
  2566. struct thread_struct *thread = NULL;
  2567. struct task_struct *task;
  2568. pfarg_reg_t *req = (pfarg_reg_t *)arg;
  2569. unsigned long value, hw_value, ovfl_mask;
  2570. unsigned int cnum;
  2571. int i, can_access_pmu = 0, state;
  2572. int is_counting, is_loaded, is_system, expert_mode;
  2573. int ret = -EINVAL;
  2574. pfm_reg_check_t wr_func;
  2575. state = ctx->ctx_state;
  2576. is_loaded = state == PFM_CTX_LOADED ? 1 : 0;
  2577. is_system = ctx->ctx_fl_system;
  2578. ovfl_mask = pmu_conf->ovfl_val;
  2579. task = ctx->ctx_task;
  2580. if (unlikely(state == PFM_CTX_ZOMBIE)) return -EINVAL;
  2581. /*
  2582. * on both UP and SMP, we can only write to the PMC when the task is
  2583. * the owner of the local PMU.
  2584. */
  2585. if (likely(is_loaded)) {
  2586. thread = &task->thread;
  2587. /*
  2588. * In system wide and when the context is loaded, access can only happen
  2589. * when the caller is running on the CPU being monitored by the session.
  2590. * It does not have to be the owner (ctx_task) of the context per se.
  2591. */
  2592. if (unlikely(is_system && ctx->ctx_cpu != smp_processor_id())) {
  2593. DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
  2594. return -EBUSY;
  2595. }
  2596. can_access_pmu = GET_PMU_OWNER() == task || is_system ? 1 : 0;
  2597. }
  2598. expert_mode = pfm_sysctl.expert_mode;
  2599. for (i = 0; i < count; i++, req++) {
  2600. cnum = req->reg_num;
  2601. value = req->reg_value;
  2602. if (!PMD_IS_IMPL(cnum)) {
  2603. DPRINT(("pmd[%u] is unimplemented or invalid\n", cnum));
  2604. goto abort_mission;
  2605. }
  2606. is_counting = PMD_IS_COUNTING(cnum);
  2607. wr_func = pmu_conf->pmd_desc[cnum].write_check;
  2608. /*
  2609. * execute write checker, if any
  2610. */
  2611. if (unlikely(expert_mode == 0 && wr_func)) {
  2612. unsigned long v = value;
  2613. ret = (*wr_func)(task, ctx, cnum, &v, regs);
  2614. if (ret) goto abort_mission;
  2615. value = v;
  2616. ret = -EINVAL;
  2617. }
  2618. /*
  2619. * no error on this register
  2620. */
  2621. PFM_REG_RETFLAG_SET(req->reg_flags, 0);
  2622. /*
  2623. * now commit changes to software state
  2624. */
  2625. hw_value = value;
  2626. /*
  2627. * update virtualized (64bits) counter
  2628. */
  2629. if (is_counting) {
  2630. /*
  2631. * write context state
  2632. */
  2633. ctx->ctx_pmds[cnum].lval = value;
  2634. /*
  2635. * when context is load we use the split value
  2636. */
  2637. if (is_loaded) {
  2638. hw_value = value & ovfl_mask;
  2639. value = value & ~ovfl_mask;
  2640. }
  2641. }
  2642. /*
  2643. * update reset values (not just for counters)
  2644. */
  2645. ctx->ctx_pmds[cnum].long_reset = req->reg_long_reset;
  2646. ctx->ctx_pmds[cnum].short_reset = req->reg_short_reset;
  2647. /*
  2648. * update randomization parameters (not just for counters)
  2649. */
  2650. ctx->ctx_pmds[cnum].seed = req->reg_random_seed;
  2651. ctx->ctx_pmds[cnum].mask = req->reg_random_mask;
  2652. /*
  2653. * update context value
  2654. */
  2655. ctx->ctx_pmds[cnum].val = value;
  2656. /*
  2657. * Keep track of what we use
  2658. *
  2659. * We do not keep track of PMC because we have to
  2660. * systematically restore ALL of them.
  2661. */
  2662. CTX_USED_PMD(ctx, PMD_PMD_DEP(cnum));
  2663. /*
  2664. * mark this PMD register used as well
  2665. */
  2666. CTX_USED_PMD(ctx, RDEP(cnum));
  2667. /*
  2668. * make sure we do not try to reset on
  2669. * restart because we have established new values
  2670. */
  2671. if (is_counting && state == PFM_CTX_MASKED) {
  2672. ctx->ctx_ovfl_regs[0] &= ~1UL << cnum;
  2673. }
  2674. if (is_loaded) {
  2675. /*
  2676. * write thread state
  2677. */
  2678. if (is_system == 0) thread->pmds[cnum] = hw_value;
  2679. /*
  2680. * write hardware register if we can
  2681. */
  2682. if (can_access_pmu) {
  2683. ia64_set_pmd(cnum, hw_value);
  2684. } else {
  2685. #ifdef CONFIG_SMP
  2686. /*
  2687. * we are guaranteed that the task is not running on the other CPU,
  2688. * we indicate that this PMD will need to be reloaded if the task
  2689. * is rescheduled on the CPU it ran last on.
  2690. */
  2691. ctx->ctx_reload_pmds[0] |= 1UL << cnum;
  2692. #endif
  2693. }
  2694. }
  2695. DPRINT(("pmd[%u]=0x%lx ld=%d apmu=%d, hw_value=0x%lx ctx_pmd=0x%lx short_reset=0x%lx "
  2696. "long_reset=0x%lx notify=%c seed=0x%lx mask=0x%lx used_pmds=0x%lx reset_pmds=0x%lx reload_pmds=0x%lx all_pmds=0x%lx ovfl_regs=0x%lx\n",
  2697. cnum,
  2698. value,
  2699. is_loaded,
  2700. can_access_pmu,
  2701. hw_value,
  2702. ctx->ctx_pmds[cnum].val,
  2703. ctx->ctx_pmds[cnum].short_reset,
  2704. ctx->ctx_pmds[cnum].long_reset,
  2705. PMC_OVFL_NOTIFY(ctx, cnum) ? 'Y':'N',
  2706. ctx->ctx_pmds[cnum].seed,
  2707. ctx->ctx_pmds[cnum].mask,
  2708. ctx->ctx_used_pmds[0],
  2709. ctx->ctx_pmds[cnum].reset_pmds[0],
  2710. ctx->ctx_reload_pmds[0],
  2711. ctx->ctx_all_pmds[0],
  2712. ctx->ctx_ovfl_regs[0]));
  2713. }
  2714. /*
  2715. * make changes visible
  2716. */
  2717. if (can_access_pmu) ia64_srlz_d();
  2718. return 0;
  2719. abort_mission:
  2720. /*
  2721. * for now, we have only one possibility for error
  2722. */
  2723. PFM_REG_RETFLAG_SET(req->reg_flags, PFM_REG_RETFL_EINVAL);
  2724. return ret;
  2725. }
  2726. /*
  2727. * By the way of PROTECT_CONTEXT(), interrupts are masked while we are in this function.
  2728. * Therefore we know, we do not have to worry about the PMU overflow interrupt. If an
  2729. * interrupt is delivered during the call, it will be kept pending until we leave, making
  2730. * it appears as if it had been generated at the UNPROTECT_CONTEXT(). At least we are
  2731. * guaranteed to return consistent data to the user, it may simply be old. It is not
  2732. * trivial to treat the overflow while inside the call because you may end up in
  2733. * some module sampling buffer code causing deadlocks.
  2734. */
  2735. static int
  2736. pfm_read_pmds(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
  2737. {
  2738. struct thread_struct *thread = NULL;
  2739. struct task_struct *task;
  2740. unsigned long val = 0UL, lval, ovfl_mask, sval;
  2741. pfarg_reg_t *req = (pfarg_reg_t *)arg;
  2742. unsigned int cnum, reg_flags = 0;
  2743. int i, can_access_pmu = 0, state;
  2744. int is_loaded, is_system, is_counting, expert_mode;
  2745. int ret = -EINVAL;
  2746. pfm_reg_check_t rd_func;
  2747. /*
  2748. * access is possible when loaded only for
  2749. * self-monitoring tasks or in UP mode
  2750. */
  2751. state = ctx->ctx_state;
  2752. is_loaded = state == PFM_CTX_LOADED ? 1 : 0;
  2753. is_system = ctx->ctx_fl_system;
  2754. ovfl_mask = pmu_conf->ovfl_val;
  2755. task = ctx->ctx_task;
  2756. if (state == PFM_CTX_ZOMBIE) return -EINVAL;
  2757. if (likely(is_loaded)) {
  2758. thread = &task->thread;
  2759. /*
  2760. * In system wide and when the context is loaded, access can only happen
  2761. * when the caller is running on the CPU being monitored by the session.
  2762. * It does not have to be the owner (ctx_task) of the context per se.
  2763. */
  2764. if (unlikely(is_system && ctx->ctx_cpu != smp_processor_id())) {
  2765. DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
  2766. return -EBUSY;
  2767. }
  2768. /*
  2769. * this can be true when not self-monitoring only in UP
  2770. */
  2771. can_access_pmu = GET_PMU_OWNER() == task || is_system ? 1 : 0;
  2772. if (can_access_pmu) ia64_srlz_d();
  2773. }
  2774. expert_mode = pfm_sysctl.expert_mode;
  2775. DPRINT(("ld=%d apmu=%d ctx_state=%d\n",
  2776. is_loaded,
  2777. can_access_pmu,
  2778. state));
  2779. /*
  2780. * on both UP and SMP, we can only read the PMD from the hardware register when
  2781. * the task is the owner of the local PMU.
  2782. */
  2783. for (i = 0; i < count; i++, req++) {
  2784. cnum = req->reg_num;
  2785. reg_flags = req->reg_flags;
  2786. if (unlikely(!PMD_IS_IMPL(cnum))) goto error;
  2787. /*
  2788. * we can only read the register that we use. That includes
  2789. * the one we explicitely initialize AND the one we want included
  2790. * in the sampling buffer (smpl_regs).
  2791. *
  2792. * Having this restriction allows optimization in the ctxsw routine
  2793. * without compromising security (leaks)
  2794. */
  2795. if (unlikely(!CTX_IS_USED_PMD(ctx, cnum))) goto error;
  2796. sval = ctx->ctx_pmds[cnum].val;
  2797. lval = ctx->ctx_pmds[cnum].lval;
  2798. is_counting = PMD_IS_COUNTING(cnum);
  2799. /*
  2800. * If the task is not the current one, then we check if the
  2801. * PMU state is still in the local live register due to lazy ctxsw.
  2802. * If true, then we read directly from the registers.
  2803. */
  2804. if (can_access_pmu){
  2805. val = ia64_get_pmd(cnum);
  2806. } else {
  2807. /*
  2808. * context has been saved
  2809. * if context is zombie, then task does not exist anymore.
  2810. * In this case, we use the full value saved in the context (pfm_flush_regs()).
  2811. */
  2812. val = is_loaded ? thread->pmds[cnum] : 0UL;
  2813. }
  2814. rd_func = pmu_conf->pmd_desc[cnum].read_check;
  2815. if (is_counting) {
  2816. /*
  2817. * XXX: need to check for overflow when loaded
  2818. */
  2819. val &= ovfl_mask;
  2820. val += sval;
  2821. }
  2822. /*
  2823. * execute read checker, if any
  2824. */
  2825. if (unlikely(expert_mode == 0 && rd_func)) {
  2826. unsigned long v = val;
  2827. ret = (*rd_func)(ctx->ctx_task, ctx, cnum, &v, regs);
  2828. if (ret) goto error;
  2829. val = v;
  2830. ret = -EINVAL;
  2831. }
  2832. PFM_REG_RETFLAG_SET(reg_flags, 0);
  2833. DPRINT(("pmd[%u]=0x%lx\n", cnum, val));
  2834. /*
  2835. * update register return value, abort all if problem during copy.
  2836. * we only modify the reg_flags field. no check mode is fine because
  2837. * access has been verified upfront in sys_perfmonctl().
  2838. */
  2839. req->reg_value = val;
  2840. req->reg_flags = reg_flags;
  2841. req->reg_last_reset_val = lval;
  2842. }
  2843. return 0;
  2844. error:
  2845. PFM_REG_RETFLAG_SET(req->reg_flags, PFM_REG_RETFL_EINVAL);
  2846. return ret;
  2847. }
  2848. int
  2849. pfm_mod_write_pmcs(struct task_struct *task, void *req, unsigned int nreq, struct pt_regs *regs)
  2850. {
  2851. pfm_context_t *ctx;
  2852. if (req == NULL) return -EINVAL;
  2853. ctx = GET_PMU_CTX();
  2854. if (ctx == NULL) return -EINVAL;
  2855. /*
  2856. * for now limit to current task, which is enough when calling
  2857. * from overflow handler
  2858. */
  2859. if (task != current && ctx->ctx_fl_system == 0) return -EBUSY;
  2860. return pfm_write_pmcs(ctx, req, nreq, regs);
  2861. }
  2862. EXPORT_SYMBOL(pfm_mod_write_pmcs);
  2863. int
  2864. pfm_mod_read_pmds(struct task_struct *task, void *req, unsigned int nreq, struct pt_regs *regs)
  2865. {
  2866. pfm_context_t *ctx;
  2867. if (req == NULL) return -EINVAL;
  2868. ctx = GET_PMU_CTX();
  2869. if (ctx == NULL) return -EINVAL;
  2870. /*
  2871. * for now limit to current task, which is enough when calling
  2872. * from overflow handler
  2873. */
  2874. if (task != current && ctx->ctx_fl_system == 0) return -EBUSY;
  2875. return pfm_read_pmds(ctx, req, nreq, regs);
  2876. }
  2877. EXPORT_SYMBOL(pfm_mod_read_pmds);
  2878. /*
  2879. * Only call this function when a process it trying to
  2880. * write the debug registers (reading is always allowed)
  2881. */
  2882. int
  2883. pfm_use_debug_registers(struct task_struct *task)
  2884. {
  2885. pfm_context_t *ctx = task->thread.pfm_context;
  2886. unsigned long flags;
  2887. int ret = 0;
  2888. if (pmu_conf->use_rr_dbregs == 0) return 0;
  2889. DPRINT(("called for [%d]\n", task->pid));
  2890. /*
  2891. * do it only once
  2892. */
  2893. if (task->thread.flags & IA64_THREAD_DBG_VALID) return 0;
  2894. /*
  2895. * Even on SMP, we do not need to use an atomic here because
  2896. * the only way in is via ptrace() and this is possible only when the
  2897. * process is stopped. Even in the case where the ctxsw out is not totally
  2898. * completed by the time we come here, there is no way the 'stopped' process
  2899. * could be in the middle of fiddling with the pfm_write_ibr_dbr() routine.
  2900. * So this is always safe.
  2901. */
  2902. if (ctx && ctx->ctx_fl_using_dbreg == 1) return -1;
  2903. LOCK_PFS(flags);
  2904. /*
  2905. * We cannot allow setting breakpoints when system wide monitoring
  2906. * sessions are using the debug registers.
  2907. */
  2908. if (pfm_sessions.pfs_sys_use_dbregs> 0)
  2909. ret = -1;
  2910. else
  2911. pfm_sessions.pfs_ptrace_use_dbregs++;
  2912. DPRINT(("ptrace_use_dbregs=%u sys_use_dbregs=%u by [%d] ret = %d\n",
  2913. pfm_sessions.pfs_ptrace_use_dbregs,
  2914. pfm_sessions.pfs_sys_use_dbregs,
  2915. task->pid, ret));
  2916. UNLOCK_PFS(flags);
  2917. return ret;
  2918. }
  2919. /*
  2920. * This function is called for every task that exits with the
  2921. * IA64_THREAD_DBG_VALID set. This indicates a task which was
  2922. * able to use the debug registers for debugging purposes via
  2923. * ptrace(). Therefore we know it was not using them for
  2924. * perfmormance monitoring, so we only decrement the number
  2925. * of "ptraced" debug register users to keep the count up to date
  2926. */
  2927. int
  2928. pfm_release_debug_registers(struct task_struct *task)
  2929. {
  2930. unsigned long flags;
  2931. int ret;
  2932. if (pmu_conf->use_rr_dbregs == 0) return 0;
  2933. LOCK_PFS(flags);
  2934. if (pfm_sessions.pfs_ptrace_use_dbregs == 0) {
  2935. printk(KERN_ERR "perfmon: invalid release for [%d] ptrace_use_dbregs=0\n", task->pid);
  2936. ret = -1;
  2937. } else {
  2938. pfm_sessions.pfs_ptrace_use_dbregs--;
  2939. ret = 0;
  2940. }
  2941. UNLOCK_PFS(flags);
  2942. return ret;
  2943. }
  2944. static int
  2945. pfm_restart(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
  2946. {
  2947. struct task_struct *task;
  2948. pfm_buffer_fmt_t *fmt;
  2949. pfm_ovfl_ctrl_t rst_ctrl;
  2950. int state, is_system;
  2951. int ret = 0;
  2952. state = ctx->ctx_state;
  2953. fmt = ctx->ctx_buf_fmt;
  2954. is_system = ctx->ctx_fl_system;
  2955. task = PFM_CTX_TASK(ctx);
  2956. switch(state) {
  2957. case PFM_CTX_MASKED:
  2958. break;
  2959. case PFM_CTX_LOADED:
  2960. if (CTX_HAS_SMPL(ctx) && fmt->fmt_restart_active) break;
  2961. /* fall through */
  2962. case PFM_CTX_UNLOADED:
  2963. case PFM_CTX_ZOMBIE:
  2964. DPRINT(("invalid state=%d\n", state));
  2965. return -EBUSY;
  2966. default:
  2967. DPRINT(("state=%d, cannot operate (no active_restart handler)\n", state));
  2968. return -EINVAL;
  2969. }
  2970. /*
  2971. * In system wide and when the context is loaded, access can only happen
  2972. * when the caller is running on the CPU being monitored by the session.
  2973. * It does not have to be the owner (ctx_task) of the context per se.
  2974. */
  2975. if (is_system && ctx->ctx_cpu != smp_processor_id()) {
  2976. DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
  2977. return -EBUSY;
  2978. }
  2979. /* sanity check */
  2980. if (unlikely(task == NULL)) {
  2981. printk(KERN_ERR "perfmon: [%d] pfm_restart no task\n", current->pid);
  2982. return -EINVAL;
  2983. }
  2984. if (task == current || is_system) {
  2985. fmt = ctx->ctx_buf_fmt;
  2986. DPRINT(("restarting self %d ovfl=0x%lx\n",
  2987. task->pid,
  2988. ctx->ctx_ovfl_regs[0]));
  2989. if (CTX_HAS_SMPL(ctx)) {
  2990. prefetch(ctx->ctx_smpl_hdr);
  2991. rst_ctrl.bits.mask_monitoring = 0;
  2992. rst_ctrl.bits.reset_ovfl_pmds = 0;
  2993. if (state == PFM_CTX_LOADED)
  2994. ret = pfm_buf_fmt_restart_active(fmt, task, &rst_ctrl, ctx->ctx_smpl_hdr, regs);
  2995. else
  2996. ret = pfm_buf_fmt_restart(fmt, task, &rst_ctrl, ctx->ctx_smpl_hdr, regs);
  2997. } else {
  2998. rst_ctrl.bits.mask_monitoring = 0;
  2999. rst_ctrl.bits.reset_ovfl_pmds = 1;
  3000. }
  3001. if (ret == 0) {
  3002. if (rst_ctrl.bits.reset_ovfl_pmds)
  3003. pfm_reset_regs(ctx, ctx->ctx_ovfl_regs, PFM_PMD_LONG_RESET);
  3004. if (rst_ctrl.bits.mask_monitoring == 0) {
  3005. DPRINT(("resuming monitoring for [%d]\n", task->pid));
  3006. if (state == PFM_CTX_MASKED) pfm_restore_monitoring(task);
  3007. } else {
  3008. DPRINT(("keeping monitoring stopped for [%d]\n", task->pid));
  3009. // cannot use pfm_stop_monitoring(task, regs);
  3010. }
  3011. }
  3012. /*
  3013. * clear overflowed PMD mask to remove any stale information
  3014. */
  3015. ctx->ctx_ovfl_regs[0] = 0UL;
  3016. /*
  3017. * back to LOADED state
  3018. */
  3019. ctx->ctx_state = PFM_CTX_LOADED;
  3020. /*
  3021. * XXX: not really useful for self monitoring
  3022. */
  3023. ctx->ctx_fl_can_restart = 0;
  3024. return 0;
  3025. }
  3026. /*
  3027. * restart another task
  3028. */
  3029. /*
  3030. * When PFM_CTX_MASKED, we cannot issue a restart before the previous
  3031. * one is seen by the task.
  3032. */
  3033. if (state == PFM_CTX_MASKED) {
  3034. if (ctx->ctx_fl_can_restart == 0) return -EINVAL;
  3035. /*
  3036. * will prevent subsequent restart before this one is
  3037. * seen by other task
  3038. */
  3039. ctx->ctx_fl_can_restart = 0;
  3040. }
  3041. /*
  3042. * if blocking, then post the semaphore is PFM_CTX_MASKED, i.e.
  3043. * the task is blocked or on its way to block. That's the normal
  3044. * restart path. If the monitoring is not masked, then the task
  3045. * can be actively monitoring and we cannot directly intervene.
  3046. * Therefore we use the trap mechanism to catch the task and
  3047. * force it to reset the buffer/reset PMDs.
  3048. *
  3049. * if non-blocking, then we ensure that the task will go into
  3050. * pfm_handle_work() before returning to user mode.
  3051. *
  3052. * We cannot explicitely reset another task, it MUST always
  3053. * be done by the task itself. This works for system wide because
  3054. * the tool that is controlling the session is logically doing
  3055. * "self-monitoring".
  3056. */
  3057. if (CTX_OVFL_NOBLOCK(ctx) == 0 && state == PFM_CTX_MASKED) {
  3058. DPRINT(("unblocking [%d] \n", task->pid));
  3059. up(&ctx->ctx_restart_sem);
  3060. } else {
  3061. DPRINT(("[%d] armed exit trap\n", task->pid));
  3062. ctx->ctx_fl_trap_reason = PFM_TRAP_REASON_RESET;
  3063. PFM_SET_WORK_PENDING(task, 1);
  3064. pfm_set_task_notify(task);
  3065. /*
  3066. * XXX: send reschedule if task runs on another CPU
  3067. */
  3068. }
  3069. return 0;
  3070. }
  3071. static int
  3072. pfm_debug(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
  3073. {
  3074. unsigned int m = *(unsigned int *)arg;
  3075. pfm_sysctl.debug = m == 0 ? 0 : 1;
  3076. pfm_debug_var = pfm_sysctl.debug;
  3077. printk(KERN_INFO "perfmon debugging %s (timing reset)\n", pfm_sysctl.debug ? "on" : "off");
  3078. if (m == 0) {
  3079. memset(pfm_stats, 0, sizeof(pfm_stats));
  3080. for(m=0; m < NR_CPUS; m++) pfm_stats[m].pfm_ovfl_intr_cycles_min = ~0UL;
  3081. }
  3082. return 0;
  3083. }
  3084. /*
  3085. * arg can be NULL and count can be zero for this function
  3086. */
  3087. static int
  3088. pfm_write_ibr_dbr(int mode, pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
  3089. {
  3090. struct thread_struct *thread = NULL;
  3091. struct task_struct *task;
  3092. pfarg_dbreg_t *req = (pfarg_dbreg_t *)arg;
  3093. unsigned long flags;
  3094. dbreg_t dbreg;
  3095. unsigned int rnum;
  3096. int first_time;
  3097. int ret = 0, state;
  3098. int i, can_access_pmu = 0;
  3099. int is_system, is_loaded;
  3100. if (pmu_conf->use_rr_dbregs == 0) return -EINVAL;
  3101. state = ctx->ctx_state;
  3102. is_loaded = state == PFM_CTX_LOADED ? 1 : 0;
  3103. is_system = ctx->ctx_fl_system;
  3104. task = ctx->ctx_task;
  3105. if (state == PFM_CTX_ZOMBIE) return -EINVAL;
  3106. /*
  3107. * on both UP and SMP, we can only write to the PMC when the task is
  3108. * the owner of the local PMU.
  3109. */
  3110. if (is_loaded) {
  3111. thread = &task->thread;
  3112. /*
  3113. * In system wide and when the context is loaded, access can only happen
  3114. * when the caller is running on the CPU being monitored by the session.
  3115. * It does not have to be the owner (ctx_task) of the context per se.
  3116. */
  3117. if (unlikely(is_system && ctx->ctx_cpu != smp_processor_id())) {
  3118. DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
  3119. return -EBUSY;
  3120. }
  3121. can_access_pmu = GET_PMU_OWNER() == task || is_system ? 1 : 0;
  3122. }
  3123. /*
  3124. * we do not need to check for ipsr.db because we do clear ibr.x, dbr.r, and dbr.w
  3125. * ensuring that no real breakpoint can be installed via this call.
  3126. *
  3127. * IMPORTANT: regs can be NULL in this function
  3128. */
  3129. first_time = ctx->ctx_fl_using_dbreg == 0;
  3130. /*
  3131. * don't bother if we are loaded and task is being debugged
  3132. */
  3133. if (is_loaded && (thread->flags & IA64_THREAD_DBG_VALID) != 0) {
  3134. DPRINT(("debug registers already in use for [%d]\n", task->pid));
  3135. return -EBUSY;
  3136. }
  3137. /*
  3138. * check for debug registers in system wide mode
  3139. *
  3140. * If though a check is done in pfm_context_load(),
  3141. * we must repeat it here, in case the registers are
  3142. * written after the context is loaded
  3143. */
  3144. if (is_loaded) {
  3145. LOCK_PFS(flags);
  3146. if (first_time && is_system) {
  3147. if (pfm_sessions.pfs_ptrace_use_dbregs)
  3148. ret = -EBUSY;
  3149. else
  3150. pfm_sessions.pfs_sys_use_dbregs++;
  3151. }
  3152. UNLOCK_PFS(flags);
  3153. }
  3154. if (ret != 0) return ret;
  3155. /*
  3156. * mark ourself as user of the debug registers for
  3157. * perfmon purposes.
  3158. */
  3159. ctx->ctx_fl_using_dbreg = 1;
  3160. /*
  3161. * clear hardware registers to make sure we don't
  3162. * pick up stale state.
  3163. *
  3164. * for a system wide session, we do not use
  3165. * thread.dbr, thread.ibr because this process
  3166. * never leaves the current CPU and the state
  3167. * is shared by all processes running on it
  3168. */
  3169. if (first_time && can_access_pmu) {
  3170. DPRINT(("[%d] clearing ibrs, dbrs\n", task->pid));
  3171. for (i=0; i < pmu_conf->num_ibrs; i++) {
  3172. ia64_set_ibr(i, 0UL);
  3173. ia64_dv_serialize_instruction();
  3174. }
  3175. ia64_srlz_i();
  3176. for (i=0; i < pmu_conf->num_dbrs; i++) {
  3177. ia64_set_dbr(i, 0UL);
  3178. ia64_dv_serialize_data();
  3179. }
  3180. ia64_srlz_d();
  3181. }
  3182. /*
  3183. * Now install the values into the registers
  3184. */
  3185. for (i = 0; i < count; i++, req++) {
  3186. rnum = req->dbreg_num;
  3187. dbreg.val = req->dbreg_value;
  3188. ret = -EINVAL;
  3189. if ((mode == PFM_CODE_RR && rnum >= PFM_NUM_IBRS) || ((mode == PFM_DATA_RR) && rnum >= PFM_NUM_DBRS)) {
  3190. DPRINT(("invalid register %u val=0x%lx mode=%d i=%d count=%d\n",
  3191. rnum, dbreg.val, mode, i, count));
  3192. goto abort_mission;
  3193. }
  3194. /*
  3195. * make sure we do not install enabled breakpoint
  3196. */
  3197. if (rnum & 0x1) {
  3198. if (mode == PFM_CODE_RR)
  3199. dbreg.ibr.ibr_x = 0;
  3200. else
  3201. dbreg.dbr.dbr_r = dbreg.dbr.dbr_w = 0;
  3202. }
  3203. PFM_REG_RETFLAG_SET(req->dbreg_flags, 0);
  3204. /*
  3205. * Debug registers, just like PMC, can only be modified
  3206. * by a kernel call. Moreover, perfmon() access to those
  3207. * registers are centralized in this routine. The hardware
  3208. * does not modify the value of these registers, therefore,
  3209. * if we save them as they are written, we can avoid having
  3210. * to save them on context switch out. This is made possible
  3211. * by the fact that when perfmon uses debug registers, ptrace()
  3212. * won't be able to modify them concurrently.
  3213. */
  3214. if (mode == PFM_CODE_RR) {
  3215. CTX_USED_IBR(ctx, rnum);
  3216. if (can_access_pmu) {
  3217. ia64_set_ibr(rnum, dbreg.val);
  3218. ia64_dv_serialize_instruction();
  3219. }
  3220. ctx->ctx_ibrs[rnum] = dbreg.val;
  3221. DPRINT(("write ibr%u=0x%lx used_ibrs=0x%x ld=%d apmu=%d\n",
  3222. rnum, dbreg.val, ctx->ctx_used_ibrs[0], is_loaded, can_access_pmu));
  3223. } else {
  3224. CTX_USED_DBR(ctx, rnum);
  3225. if (can_access_pmu) {
  3226. ia64_set_dbr(rnum, dbreg.val);
  3227. ia64_dv_serialize_data();
  3228. }
  3229. ctx->ctx_dbrs[rnum] = dbreg.val;
  3230. DPRINT(("write dbr%u=0x%lx used_dbrs=0x%x ld=%d apmu=%d\n",
  3231. rnum, dbreg.val, ctx->ctx_used_dbrs[0], is_loaded, can_access_pmu));
  3232. }
  3233. }
  3234. return 0;
  3235. abort_mission:
  3236. /*
  3237. * in case it was our first attempt, we undo the global modifications
  3238. */
  3239. if (first_time) {
  3240. LOCK_PFS(flags);
  3241. if (ctx->ctx_fl_system) {
  3242. pfm_sessions.pfs_sys_use_dbregs--;
  3243. }
  3244. UNLOCK_PFS(flags);
  3245. ctx->ctx_fl_using_dbreg = 0;
  3246. }
  3247. /*
  3248. * install error return flag
  3249. */
  3250. PFM_REG_RETFLAG_SET(req->dbreg_flags, PFM_REG_RETFL_EINVAL);
  3251. return ret;
  3252. }
  3253. static int
  3254. pfm_write_ibrs(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
  3255. {
  3256. return pfm_write_ibr_dbr(PFM_CODE_RR, ctx, arg, count, regs);
  3257. }
  3258. static int
  3259. pfm_write_dbrs(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
  3260. {
  3261. return pfm_write_ibr_dbr(PFM_DATA_RR, ctx, arg, count, regs);
  3262. }
  3263. int
  3264. pfm_mod_write_ibrs(struct task_struct *task, void *req, unsigned int nreq, struct pt_regs *regs)
  3265. {
  3266. pfm_context_t *ctx;
  3267. if (req == NULL) return -EINVAL;
  3268. ctx = GET_PMU_CTX();
  3269. if (ctx == NULL) return -EINVAL;
  3270. /*
  3271. * for now limit to current task, which is enough when calling
  3272. * from overflow handler
  3273. */
  3274. if (task != current && ctx->ctx_fl_system == 0) return -EBUSY;
  3275. return pfm_write_ibrs(ctx, req, nreq, regs);
  3276. }
  3277. EXPORT_SYMBOL(pfm_mod_write_ibrs);
  3278. int
  3279. pfm_mod_write_dbrs(struct task_struct *task, void *req, unsigned int nreq, struct pt_regs *regs)
  3280. {
  3281. pfm_context_t *ctx;
  3282. if (req == NULL) return -EINVAL;
  3283. ctx = GET_PMU_CTX();
  3284. if (ctx == NULL) return -EINVAL;
  3285. /*
  3286. * for now limit to current task, which is enough when calling
  3287. * from overflow handler
  3288. */
  3289. if (task != current && ctx->ctx_fl_system == 0) return -EBUSY;
  3290. return pfm_write_dbrs(ctx, req, nreq, regs);
  3291. }
  3292. EXPORT_SYMBOL(pfm_mod_write_dbrs);
  3293. static int
  3294. pfm_get_features(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
  3295. {
  3296. pfarg_features_t *req = (pfarg_features_t *)arg;
  3297. req->ft_version = PFM_VERSION;
  3298. return 0;
  3299. }
  3300. static int
  3301. pfm_stop(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
  3302. {
  3303. struct pt_regs *tregs;
  3304. struct task_struct *task = PFM_CTX_TASK(ctx);
  3305. int state, is_system;
  3306. state = ctx->ctx_state;
  3307. is_system = ctx->ctx_fl_system;
  3308. /*
  3309. * context must be attached to issue the stop command (includes LOADED,MASKED,ZOMBIE)
  3310. */
  3311. if (state == PFM_CTX_UNLOADED) return -EINVAL;
  3312. /*
  3313. * In system wide and when the context is loaded, access can only happen
  3314. * when the caller is running on the CPU being monitored by the session.
  3315. * It does not have to be the owner (ctx_task) of the context per se.
  3316. */
  3317. if (is_system && ctx->ctx_cpu != smp_processor_id()) {
  3318. DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
  3319. return -EBUSY;
  3320. }
  3321. DPRINT(("task [%d] ctx_state=%d is_system=%d\n",
  3322. PFM_CTX_TASK(ctx)->pid,
  3323. state,
  3324. is_system));
  3325. /*
  3326. * in system mode, we need to update the PMU directly
  3327. * and the user level state of the caller, which may not
  3328. * necessarily be the creator of the context.
  3329. */
  3330. if (is_system) {
  3331. /*
  3332. * Update local PMU first
  3333. *
  3334. * disable dcr pp
  3335. */
  3336. ia64_setreg(_IA64_REG_CR_DCR, ia64_getreg(_IA64_REG_CR_DCR) & ~IA64_DCR_PP);
  3337. ia64_srlz_i();
  3338. /*
  3339. * update local cpuinfo
  3340. */
  3341. PFM_CPUINFO_CLEAR(PFM_CPUINFO_DCR_PP);
  3342. /*
  3343. * stop monitoring, does srlz.i
  3344. */
  3345. pfm_clear_psr_pp();
  3346. /*
  3347. * stop monitoring in the caller
  3348. */
  3349. ia64_psr(regs)->pp = 0;
  3350. return 0;
  3351. }
  3352. /*
  3353. * per-task mode
  3354. */
  3355. if (task == current) {
  3356. /* stop monitoring at kernel level */
  3357. pfm_clear_psr_up();
  3358. /*
  3359. * stop monitoring at the user level
  3360. */
  3361. ia64_psr(regs)->up = 0;
  3362. } else {
  3363. tregs = ia64_task_regs(task);
  3364. /*
  3365. * stop monitoring at the user level
  3366. */
  3367. ia64_psr(tregs)->up = 0;
  3368. /*
  3369. * monitoring disabled in kernel at next reschedule
  3370. */
  3371. ctx->ctx_saved_psr_up = 0;
  3372. DPRINT(("task=[%d]\n", task->pid));
  3373. }
  3374. return 0;
  3375. }
  3376. static int
  3377. pfm_start(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
  3378. {
  3379. struct pt_regs *tregs;
  3380. int state, is_system;
  3381. state = ctx->ctx_state;
  3382. is_system = ctx->ctx_fl_system;
  3383. if (state != PFM_CTX_LOADED) return -EINVAL;
  3384. /*
  3385. * In system wide and when the context is loaded, access can only happen
  3386. * when the caller is running on the CPU being monitored by the session.
  3387. * It does not have to be the owner (ctx_task) of the context per se.
  3388. */
  3389. if (is_system && ctx->ctx_cpu != smp_processor_id()) {
  3390. DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
  3391. return -EBUSY;
  3392. }
  3393. /*
  3394. * in system mode, we need to update the PMU directly
  3395. * and the user level state of the caller, which may not
  3396. * necessarily be the creator of the context.
  3397. */
  3398. if (is_system) {
  3399. /*
  3400. * set user level psr.pp for the caller
  3401. */
  3402. ia64_psr(regs)->pp = 1;
  3403. /*
  3404. * now update the local PMU and cpuinfo
  3405. */
  3406. PFM_CPUINFO_SET(PFM_CPUINFO_DCR_PP);
  3407. /*
  3408. * start monitoring at kernel level
  3409. */
  3410. pfm_set_psr_pp();
  3411. /* enable dcr pp */
  3412. ia64_setreg(_IA64_REG_CR_DCR, ia64_getreg(_IA64_REG_CR_DCR) | IA64_DCR_PP);
  3413. ia64_srlz_i();
  3414. return 0;
  3415. }
  3416. /*
  3417. * per-process mode
  3418. */
  3419. if (ctx->ctx_task == current) {
  3420. /* start monitoring at kernel level */
  3421. pfm_set_psr_up();
  3422. /*
  3423. * activate monitoring at user level
  3424. */
  3425. ia64_psr(regs)->up = 1;
  3426. } else {
  3427. tregs = ia64_task_regs(ctx->ctx_task);
  3428. /*
  3429. * start monitoring at the kernel level the next
  3430. * time the task is scheduled
  3431. */
  3432. ctx->ctx_saved_psr_up = IA64_PSR_UP;
  3433. /*
  3434. * activate monitoring at user level
  3435. */
  3436. ia64_psr(tregs)->up = 1;
  3437. }
  3438. return 0;
  3439. }
  3440. static int
  3441. pfm_get_pmc_reset(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
  3442. {
  3443. pfarg_reg_t *req = (pfarg_reg_t *)arg;
  3444. unsigned int cnum;
  3445. int i;
  3446. int ret = -EINVAL;
  3447. for (i = 0; i < count; i++, req++) {
  3448. cnum = req->reg_num;
  3449. if (!PMC_IS_IMPL(cnum)) goto abort_mission;
  3450. req->reg_value = PMC_DFL_VAL(cnum);
  3451. PFM_REG_RETFLAG_SET(req->reg_flags, 0);
  3452. DPRINT(("pmc_reset_val pmc[%u]=0x%lx\n", cnum, req->reg_value));
  3453. }
  3454. return 0;
  3455. abort_mission:
  3456. PFM_REG_RETFLAG_SET(req->reg_flags, PFM_REG_RETFL_EINVAL);
  3457. return ret;
  3458. }
  3459. static int
  3460. pfm_check_task_exist(pfm_context_t *ctx)
  3461. {
  3462. struct task_struct *g, *t;
  3463. int ret = -ESRCH;
  3464. read_lock(&tasklist_lock);
  3465. do_each_thread (g, t) {
  3466. if (t->thread.pfm_context == ctx) {
  3467. ret = 0;
  3468. break;
  3469. }
  3470. } while_each_thread (g, t);
  3471. read_unlock(&tasklist_lock);
  3472. DPRINT(("pfm_check_task_exist: ret=%d ctx=%p\n", ret, ctx));
  3473. return ret;
  3474. }
  3475. static int
  3476. pfm_context_load(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
  3477. {
  3478. struct task_struct *task;
  3479. struct thread_struct *thread;
  3480. struct pfm_context_t *old;
  3481. unsigned long flags;
  3482. #ifndef CONFIG_SMP
  3483. struct task_struct *owner_task = NULL;
  3484. #endif
  3485. pfarg_load_t *req = (pfarg_load_t *)arg;
  3486. unsigned long *pmcs_source, *pmds_source;
  3487. int the_cpu;
  3488. int ret = 0;
  3489. int state, is_system, set_dbregs = 0;
  3490. state = ctx->ctx_state;
  3491. is_system = ctx->ctx_fl_system;
  3492. /*
  3493. * can only load from unloaded or terminated state
  3494. */
  3495. if (state != PFM_CTX_UNLOADED) {
  3496. DPRINT(("cannot load to [%d], invalid ctx_state=%d\n",
  3497. req->load_pid,
  3498. ctx->ctx_state));
  3499. return -EINVAL;
  3500. }
  3501. DPRINT(("load_pid [%d] using_dbreg=%d\n", req->load_pid, ctx->ctx_fl_using_dbreg));
  3502. if (CTX_OVFL_NOBLOCK(ctx) == 0 && req->load_pid == current->pid) {
  3503. DPRINT(("cannot use blocking mode on self\n"));
  3504. return -EINVAL;
  3505. }
  3506. ret = pfm_get_task(ctx, req->load_pid, &task);
  3507. if (ret) {
  3508. DPRINT(("load_pid [%d] get_task=%d\n", req->load_pid, ret));
  3509. return ret;
  3510. }
  3511. ret = -EINVAL;
  3512. /*
  3513. * system wide is self monitoring only
  3514. */
  3515. if (is_system && task != current) {
  3516. DPRINT(("system wide is self monitoring only load_pid=%d\n",
  3517. req->load_pid));
  3518. goto error;
  3519. }
  3520. thread = &task->thread;
  3521. ret = 0;
  3522. /*
  3523. * cannot load a context which is using range restrictions,
  3524. * into a task that is being debugged.
  3525. */
  3526. if (ctx->ctx_fl_using_dbreg) {
  3527. if (thread->flags & IA64_THREAD_DBG_VALID) {
  3528. ret = -EBUSY;
  3529. DPRINT(("load_pid [%d] task is debugged, cannot load range restrictions\n", req->load_pid));
  3530. goto error;
  3531. }
  3532. LOCK_PFS(flags);
  3533. if (is_system) {
  3534. if (pfm_sessions.pfs_ptrace_use_dbregs) {
  3535. DPRINT(("cannot load [%d] dbregs in use\n", task->pid));
  3536. ret = -EBUSY;
  3537. } else {
  3538. pfm_sessions.pfs_sys_use_dbregs++;
  3539. DPRINT(("load [%d] increased sys_use_dbreg=%u\n", task->pid, pfm_sessions.pfs_sys_use_dbregs));
  3540. set_dbregs = 1;
  3541. }
  3542. }
  3543. UNLOCK_PFS(flags);
  3544. if (ret) goto error;
  3545. }
  3546. /*
  3547. * SMP system-wide monitoring implies self-monitoring.
  3548. *
  3549. * The programming model expects the task to
  3550. * be pinned on a CPU throughout the session.
  3551. * Here we take note of the current CPU at the
  3552. * time the context is loaded. No call from
  3553. * another CPU will be allowed.
  3554. *
  3555. * The pinning via shed_setaffinity()
  3556. * must be done by the calling task prior
  3557. * to this call.
  3558. *
  3559. * systemwide: keep track of CPU this session is supposed to run on
  3560. */
  3561. the_cpu = ctx->ctx_cpu = smp_processor_id();
  3562. ret = -EBUSY;
  3563. /*
  3564. * now reserve the session
  3565. */
  3566. ret = pfm_reserve_session(current, is_system, the_cpu);
  3567. if (ret) goto error;
  3568. /*
  3569. * task is necessarily stopped at this point.
  3570. *
  3571. * If the previous context was zombie, then it got removed in
  3572. * pfm_save_regs(). Therefore we should not see it here.
  3573. * If we see a context, then this is an active context
  3574. *
  3575. * XXX: needs to be atomic
  3576. */
  3577. DPRINT(("before cmpxchg() old_ctx=%p new_ctx=%p\n",
  3578. thread->pfm_context, ctx));
  3579. old = ia64_cmpxchg(acq, &thread->pfm_context, NULL, ctx, sizeof(pfm_context_t *));
  3580. if (old != NULL) {
  3581. DPRINT(("load_pid [%d] already has a context\n", req->load_pid));
  3582. goto error_unres;
  3583. }
  3584. pfm_reset_msgq(ctx);
  3585. ctx->ctx_state = PFM_CTX_LOADED;
  3586. /*
  3587. * link context to task
  3588. */
  3589. ctx->ctx_task = task;
  3590. if (is_system) {
  3591. /*
  3592. * we load as stopped
  3593. */
  3594. PFM_CPUINFO_SET(PFM_CPUINFO_SYST_WIDE);
  3595. PFM_CPUINFO_CLEAR(PFM_CPUINFO_DCR_PP);
  3596. if (ctx->ctx_fl_excl_idle) PFM_CPUINFO_SET(PFM_CPUINFO_EXCL_IDLE);
  3597. } else {
  3598. thread->flags |= IA64_THREAD_PM_VALID;
  3599. }
  3600. /*
  3601. * propagate into thread-state
  3602. */
  3603. pfm_copy_pmds(task, ctx);
  3604. pfm_copy_pmcs(task, ctx);
  3605. pmcs_source = thread->pmcs;
  3606. pmds_source = thread->pmds;
  3607. /*
  3608. * always the case for system-wide
  3609. */
  3610. if (task == current) {
  3611. if (is_system == 0) {
  3612. /* allow user level control */
  3613. ia64_psr(regs)->sp = 0;
  3614. DPRINT(("clearing psr.sp for [%d]\n", task->pid));
  3615. SET_LAST_CPU(ctx, smp_processor_id());
  3616. INC_ACTIVATION();
  3617. SET_ACTIVATION(ctx);
  3618. #ifndef CONFIG_SMP
  3619. /*
  3620. * push the other task out, if any
  3621. */
  3622. owner_task = GET_PMU_OWNER();
  3623. if (owner_task) pfm_lazy_save_regs(owner_task);
  3624. #endif
  3625. }
  3626. /*
  3627. * load all PMD from ctx to PMU (as opposed to thread state)
  3628. * restore all PMC from ctx to PMU
  3629. */
  3630. pfm_restore_pmds(pmds_source, ctx->ctx_all_pmds[0]);
  3631. pfm_restore_pmcs(pmcs_source, ctx->ctx_all_pmcs[0]);
  3632. ctx->ctx_reload_pmcs[0] = 0UL;
  3633. ctx->ctx_reload_pmds[0] = 0UL;
  3634. /*
  3635. * guaranteed safe by earlier check against DBG_VALID
  3636. */
  3637. if (ctx->ctx_fl_using_dbreg) {
  3638. pfm_restore_ibrs(ctx->ctx_ibrs, pmu_conf->num_ibrs);
  3639. pfm_restore_dbrs(ctx->ctx_dbrs, pmu_conf->num_dbrs);
  3640. }
  3641. /*
  3642. * set new ownership
  3643. */
  3644. SET_PMU_OWNER(task, ctx);
  3645. DPRINT(("context loaded on PMU for [%d]\n", task->pid));
  3646. } else {
  3647. /*
  3648. * when not current, task MUST be stopped, so this is safe
  3649. */
  3650. regs = ia64_task_regs(task);
  3651. /* force a full reload */
  3652. ctx->ctx_last_activation = PFM_INVALID_ACTIVATION;
  3653. SET_LAST_CPU(ctx, -1);
  3654. /* initial saved psr (stopped) */
  3655. ctx->ctx_saved_psr_up = 0UL;
  3656. ia64_psr(regs)->up = ia64_psr(regs)->pp = 0;
  3657. }
  3658. ret = 0;
  3659. error_unres:
  3660. if (ret) pfm_unreserve_session(ctx, ctx->ctx_fl_system, the_cpu);
  3661. error:
  3662. /*
  3663. * we must undo the dbregs setting (for system-wide)
  3664. */
  3665. if (ret && set_dbregs) {
  3666. LOCK_PFS(flags);
  3667. pfm_sessions.pfs_sys_use_dbregs--;
  3668. UNLOCK_PFS(flags);
  3669. }
  3670. /*
  3671. * release task, there is now a link with the context
  3672. */
  3673. if (is_system == 0 && task != current) {
  3674. pfm_put_task(task);
  3675. if (ret == 0) {
  3676. ret = pfm_check_task_exist(ctx);
  3677. if (ret) {
  3678. ctx->ctx_state = PFM_CTX_UNLOADED;
  3679. ctx->ctx_task = NULL;
  3680. }
  3681. }
  3682. }
  3683. return ret;
  3684. }
  3685. /*
  3686. * in this function, we do not need to increase the use count
  3687. * for the task via get_task_struct(), because we hold the
  3688. * context lock. If the task were to disappear while having
  3689. * a context attached, it would go through pfm_exit_thread()
  3690. * which also grabs the context lock and would therefore be blocked
  3691. * until we are here.
  3692. */
  3693. static void pfm_flush_pmds(struct task_struct *, pfm_context_t *ctx);
  3694. static int
  3695. pfm_context_unload(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
  3696. {
  3697. struct task_struct *task = PFM_CTX_TASK(ctx);
  3698. struct pt_regs *tregs;
  3699. int prev_state, is_system;
  3700. int ret;
  3701. DPRINT(("ctx_state=%d task [%d]\n", ctx->ctx_state, task ? task->pid : -1));
  3702. prev_state = ctx->ctx_state;
  3703. is_system = ctx->ctx_fl_system;
  3704. /*
  3705. * unload only when necessary
  3706. */
  3707. if (prev_state == PFM_CTX_UNLOADED) {
  3708. DPRINT(("ctx_state=%d, nothing to do\n", prev_state));
  3709. return 0;
  3710. }
  3711. /*
  3712. * clear psr and dcr bits
  3713. */
  3714. ret = pfm_stop(ctx, NULL, 0, regs);
  3715. if (ret) return ret;
  3716. ctx->ctx_state = PFM_CTX_UNLOADED;
  3717. /*
  3718. * in system mode, we need to update the PMU directly
  3719. * and the user level state of the caller, which may not
  3720. * necessarily be the creator of the context.
  3721. */
  3722. if (is_system) {
  3723. /*
  3724. * Update cpuinfo
  3725. *
  3726. * local PMU is taken care of in pfm_stop()
  3727. */
  3728. PFM_CPUINFO_CLEAR(PFM_CPUINFO_SYST_WIDE);
  3729. PFM_CPUINFO_CLEAR(PFM_CPUINFO_EXCL_IDLE);
  3730. /*
  3731. * save PMDs in context
  3732. * release ownership
  3733. */
  3734. pfm_flush_pmds(current, ctx);
  3735. /*
  3736. * at this point we are done with the PMU
  3737. * so we can unreserve the resource.
  3738. */
  3739. if (prev_state != PFM_CTX_ZOMBIE)
  3740. pfm_unreserve_session(ctx, 1 , ctx->ctx_cpu);
  3741. /*
  3742. * disconnect context from task
  3743. */
  3744. task->thread.pfm_context = NULL;
  3745. /*
  3746. * disconnect task from context
  3747. */
  3748. ctx->ctx_task = NULL;
  3749. /*
  3750. * There is nothing more to cleanup here.
  3751. */
  3752. return 0;
  3753. }
  3754. /*
  3755. * per-task mode
  3756. */
  3757. tregs = task == current ? regs : ia64_task_regs(task);
  3758. if (task == current) {
  3759. /*
  3760. * cancel user level control
  3761. */
  3762. ia64_psr(regs)->sp = 1;
  3763. DPRINT(("setting psr.sp for [%d]\n", task->pid));
  3764. }
  3765. /*
  3766. * save PMDs to context
  3767. * release ownership
  3768. */
  3769. pfm_flush_pmds(task, ctx);
  3770. /*
  3771. * at this point we are done with the PMU
  3772. * so we can unreserve the resource.
  3773. *
  3774. * when state was ZOMBIE, we have already unreserved.
  3775. */
  3776. if (prev_state != PFM_CTX_ZOMBIE)
  3777. pfm_unreserve_session(ctx, 0 , ctx->ctx_cpu);
  3778. /*
  3779. * reset activation counter and psr
  3780. */
  3781. ctx->ctx_last_activation = PFM_INVALID_ACTIVATION;
  3782. SET_LAST_CPU(ctx, -1);
  3783. /*
  3784. * PMU state will not be restored
  3785. */
  3786. task->thread.flags &= ~IA64_THREAD_PM_VALID;
  3787. /*
  3788. * break links between context and task
  3789. */
  3790. task->thread.pfm_context = NULL;
  3791. ctx->ctx_task = NULL;
  3792. PFM_SET_WORK_PENDING(task, 0);
  3793. ctx->ctx_fl_trap_reason = PFM_TRAP_REASON_NONE;
  3794. ctx->ctx_fl_can_restart = 0;
  3795. ctx->ctx_fl_going_zombie = 0;
  3796. DPRINT(("disconnected [%d] from context\n", task->pid));
  3797. return 0;
  3798. }
  3799. /*
  3800. * called only from exit_thread(): task == current
  3801. * we come here only if current has a context attached (loaded or masked)
  3802. */
  3803. void
  3804. pfm_exit_thread(struct task_struct *task)
  3805. {
  3806. pfm_context_t *ctx;
  3807. unsigned long flags;
  3808. struct pt_regs *regs = ia64_task_regs(task);
  3809. int ret, state;
  3810. int free_ok = 0;
  3811. ctx = PFM_GET_CTX(task);
  3812. PROTECT_CTX(ctx, flags);
  3813. DPRINT(("state=%d task [%d]\n", ctx->ctx_state, task->pid));
  3814. state = ctx->ctx_state;
  3815. switch(state) {
  3816. case PFM_CTX_UNLOADED:
  3817. /*
  3818. * only comes to thios function if pfm_context is not NULL, i.e., cannot
  3819. * be in unloaded state
  3820. */
  3821. printk(KERN_ERR "perfmon: pfm_exit_thread [%d] ctx unloaded\n", task->pid);
  3822. break;
  3823. case PFM_CTX_LOADED:
  3824. case PFM_CTX_MASKED:
  3825. ret = pfm_context_unload(ctx, NULL, 0, regs);
  3826. if (ret) {
  3827. printk(KERN_ERR "perfmon: pfm_exit_thread [%d] state=%d unload failed %d\n", task->pid, state, ret);
  3828. }
  3829. DPRINT(("ctx unloaded for current state was %d\n", state));
  3830. pfm_end_notify_user(ctx);
  3831. break;
  3832. case PFM_CTX_ZOMBIE:
  3833. ret = pfm_context_unload(ctx, NULL, 0, regs);
  3834. if (ret) {
  3835. printk(KERN_ERR "perfmon: pfm_exit_thread [%d] state=%d unload failed %d\n", task->pid, state, ret);
  3836. }
  3837. free_ok = 1;
  3838. break;
  3839. default:
  3840. printk(KERN_ERR "perfmon: pfm_exit_thread [%d] unexpected state=%d\n", task->pid, state);
  3841. break;
  3842. }
  3843. UNPROTECT_CTX(ctx, flags);
  3844. { u64 psr = pfm_get_psr();
  3845. BUG_ON(psr & (IA64_PSR_UP|IA64_PSR_PP));
  3846. BUG_ON(GET_PMU_OWNER());
  3847. BUG_ON(ia64_psr(regs)->up);
  3848. BUG_ON(ia64_psr(regs)->pp);
  3849. }
  3850. /*
  3851. * All memory free operations (especially for vmalloc'ed memory)
  3852. * MUST be done with interrupts ENABLED.
  3853. */
  3854. if (free_ok) pfm_context_free(ctx);
  3855. }
  3856. /*
  3857. * functions MUST be listed in the increasing order of their index (see permfon.h)
  3858. */
  3859. #define PFM_CMD(name, flags, arg_count, arg_type, getsz) { name, #name, flags, arg_count, sizeof(arg_type), getsz }
  3860. #define PFM_CMD_S(name, flags) { name, #name, flags, 0, 0, NULL }
  3861. #define PFM_CMD_PCLRWS (PFM_CMD_FD|PFM_CMD_ARG_RW|PFM_CMD_STOP)
  3862. #define PFM_CMD_PCLRW (PFM_CMD_FD|PFM_CMD_ARG_RW)
  3863. #define PFM_CMD_NONE { NULL, "no-cmd", 0, 0, 0, NULL}
  3864. static pfm_cmd_desc_t pfm_cmd_tab[]={
  3865. /* 0 */PFM_CMD_NONE,
  3866. /* 1 */PFM_CMD(pfm_write_pmcs, PFM_CMD_PCLRWS, PFM_CMD_ARG_MANY, pfarg_reg_t, NULL),
  3867. /* 2 */PFM_CMD(pfm_write_pmds, PFM_CMD_PCLRWS, PFM_CMD_ARG_MANY, pfarg_reg_t, NULL),
  3868. /* 3 */PFM_CMD(pfm_read_pmds, PFM_CMD_PCLRWS, PFM_CMD_ARG_MANY, pfarg_reg_t, NULL),
  3869. /* 4 */PFM_CMD_S(pfm_stop, PFM_CMD_PCLRWS),
  3870. /* 5 */PFM_CMD_S(pfm_start, PFM_CMD_PCLRWS),
  3871. /* 6 */PFM_CMD_NONE,
  3872. /* 7 */PFM_CMD_NONE,
  3873. /* 8 */PFM_CMD(pfm_context_create, PFM_CMD_ARG_RW, 1, pfarg_context_t, pfm_ctx_getsize),
  3874. /* 9 */PFM_CMD_NONE,
  3875. /* 10 */PFM_CMD_S(pfm_restart, PFM_CMD_PCLRW),
  3876. /* 11 */PFM_CMD_NONE,
  3877. /* 12 */PFM_CMD(pfm_get_features, PFM_CMD_ARG_RW, 1, pfarg_features_t, NULL),
  3878. /* 13 */PFM_CMD(pfm_debug, 0, 1, unsigned int, NULL),
  3879. /* 14 */PFM_CMD_NONE,
  3880. /* 15 */PFM_CMD(pfm_get_pmc_reset, PFM_CMD_ARG_RW, PFM_CMD_ARG_MANY, pfarg_reg_t, NULL),
  3881. /* 16 */PFM_CMD(pfm_context_load, PFM_CMD_PCLRWS, 1, pfarg_load_t, NULL),
  3882. /* 17 */PFM_CMD_S(pfm_context_unload, PFM_CMD_PCLRWS),
  3883. /* 18 */PFM_CMD_NONE,
  3884. /* 19 */PFM_CMD_NONE,
  3885. /* 20 */PFM_CMD_NONE,
  3886. /* 21 */PFM_CMD_NONE,
  3887. /* 22 */PFM_CMD_NONE,
  3888. /* 23 */PFM_CMD_NONE,
  3889. /* 24 */PFM_CMD_NONE,
  3890. /* 25 */PFM_CMD_NONE,
  3891. /* 26 */PFM_CMD_NONE,
  3892. /* 27 */PFM_CMD_NONE,
  3893. /* 28 */PFM_CMD_NONE,
  3894. /* 29 */PFM_CMD_NONE,
  3895. /* 30 */PFM_CMD_NONE,
  3896. /* 31 */PFM_CMD_NONE,
  3897. /* 32 */PFM_CMD(pfm_write_ibrs, PFM_CMD_PCLRWS, PFM_CMD_ARG_MANY, pfarg_dbreg_t, NULL),
  3898. /* 33 */PFM_CMD(pfm_write_dbrs, PFM_CMD_PCLRWS, PFM_CMD_ARG_MANY, pfarg_dbreg_t, NULL)
  3899. };
  3900. #define PFM_CMD_COUNT (sizeof(pfm_cmd_tab)/sizeof(pfm_cmd_desc_t))
  3901. static int
  3902. pfm_check_task_state(pfm_context_t *ctx, int cmd, unsigned long flags)
  3903. {
  3904. struct task_struct *task;
  3905. int state, old_state;
  3906. recheck:
  3907. state = ctx->ctx_state;
  3908. task = ctx->ctx_task;
  3909. if (task == NULL) {
  3910. DPRINT(("context %d no task, state=%d\n", ctx->ctx_fd, state));
  3911. return 0;
  3912. }
  3913. DPRINT(("context %d state=%d [%d] task_state=%ld must_stop=%d\n",
  3914. ctx->ctx_fd,
  3915. state,
  3916. task->pid,
  3917. task->state, PFM_CMD_STOPPED(cmd)));
  3918. /*
  3919. * self-monitoring always ok.
  3920. *
  3921. * for system-wide the caller can either be the creator of the
  3922. * context (to one to which the context is attached to) OR
  3923. * a task running on the same CPU as the session.
  3924. */
  3925. if (task == current || ctx->ctx_fl_system) return 0;
  3926. /*
  3927. * if context is UNLOADED we are safe to go
  3928. */
  3929. if (state == PFM_CTX_UNLOADED) return 0;
  3930. /*
  3931. * no command can operate on a zombie context
  3932. */
  3933. if (state == PFM_CTX_ZOMBIE) {
  3934. DPRINT(("cmd %d state zombie cannot operate on context\n", cmd));
  3935. return -EINVAL;
  3936. }
  3937. /*
  3938. * context is LOADED or MASKED. Some commands may need to have
  3939. * the task stopped.
  3940. *
  3941. * We could lift this restriction for UP but it would mean that
  3942. * the user has no guarantee the task would not run between
  3943. * two successive calls to perfmonctl(). That's probably OK.
  3944. * If this user wants to ensure the task does not run, then
  3945. * the task must be stopped.
  3946. */
  3947. if (PFM_CMD_STOPPED(cmd)) {
  3948. if ((task->state != TASK_STOPPED) && (task->state != TASK_TRACED)) {
  3949. DPRINT(("[%d] task not in stopped state\n", task->pid));
  3950. return -EBUSY;
  3951. }
  3952. /*
  3953. * task is now stopped, wait for ctxsw out
  3954. *
  3955. * This is an interesting point in the code.
  3956. * We need to unprotect the context because
  3957. * the pfm_save_regs() routines needs to grab
  3958. * the same lock. There are danger in doing
  3959. * this because it leaves a window open for
  3960. * another task to get access to the context
  3961. * and possibly change its state. The one thing
  3962. * that is not possible is for the context to disappear
  3963. * because we are protected by the VFS layer, i.e.,
  3964. * get_fd()/put_fd().
  3965. */
  3966. old_state = state;
  3967. UNPROTECT_CTX(ctx, flags);
  3968. wait_task_inactive(task);
  3969. PROTECT_CTX(ctx, flags);
  3970. /*
  3971. * we must recheck to verify if state has changed
  3972. */
  3973. if (ctx->ctx_state != old_state) {
  3974. DPRINT(("old_state=%d new_state=%d\n", old_state, ctx->ctx_state));
  3975. goto recheck;
  3976. }
  3977. }
  3978. return 0;
  3979. }
  3980. /*
  3981. * system-call entry point (must return long)
  3982. */
  3983. asmlinkage long
  3984. sys_perfmonctl (int fd, int cmd, void __user *arg, int count)
  3985. {
  3986. struct file *file = NULL;
  3987. pfm_context_t *ctx = NULL;
  3988. unsigned long flags = 0UL;
  3989. void *args_k = NULL;
  3990. long ret; /* will expand int return types */
  3991. size_t base_sz, sz, xtra_sz = 0;
  3992. int narg, completed_args = 0, call_made = 0, cmd_flags;
  3993. int (*func)(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs);
  3994. int (*getsize)(void *arg, size_t *sz);
  3995. #define PFM_MAX_ARGSIZE 4096
  3996. /*
  3997. * reject any call if perfmon was disabled at initialization
  3998. */
  3999. if (unlikely(pmu_conf == NULL)) return -ENOSYS;
  4000. if (unlikely(cmd < 0 || cmd >= PFM_CMD_COUNT)) {
  4001. DPRINT(("invalid cmd=%d\n", cmd));
  4002. return -EINVAL;
  4003. }
  4004. func = pfm_cmd_tab[cmd].cmd_func;
  4005. narg = pfm_cmd_tab[cmd].cmd_narg;
  4006. base_sz = pfm_cmd_tab[cmd].cmd_argsize;
  4007. getsize = pfm_cmd_tab[cmd].cmd_getsize;
  4008. cmd_flags = pfm_cmd_tab[cmd].cmd_flags;
  4009. if (unlikely(func == NULL)) {
  4010. DPRINT(("invalid cmd=%d\n", cmd));
  4011. return -EINVAL;
  4012. }
  4013. DPRINT(("cmd=%s idx=%d narg=0x%x argsz=%lu count=%d\n",
  4014. PFM_CMD_NAME(cmd),
  4015. cmd,
  4016. narg,
  4017. base_sz,
  4018. count));
  4019. /*
  4020. * check if number of arguments matches what the command expects
  4021. */
  4022. if (unlikely((narg == PFM_CMD_ARG_MANY && count <= 0) || (narg > 0 && narg != count)))
  4023. return -EINVAL;
  4024. restart_args:
  4025. sz = xtra_sz + base_sz*count;
  4026. /*
  4027. * limit abuse to min page size
  4028. */
  4029. if (unlikely(sz > PFM_MAX_ARGSIZE)) {
  4030. printk(KERN_ERR "perfmon: [%d] argument too big %lu\n", current->pid, sz);
  4031. return -E2BIG;
  4032. }
  4033. /*
  4034. * allocate default-sized argument buffer
  4035. */
  4036. if (likely(count && args_k == NULL)) {
  4037. args_k = kmalloc(PFM_MAX_ARGSIZE, GFP_KERNEL);
  4038. if (args_k == NULL) return -ENOMEM;
  4039. }
  4040. ret = -EFAULT;
  4041. /*
  4042. * copy arguments
  4043. *
  4044. * assume sz = 0 for command without parameters
  4045. */
  4046. if (sz && copy_from_user(args_k, arg, sz)) {
  4047. DPRINT(("cannot copy_from_user %lu bytes @%p\n", sz, arg));
  4048. goto error_args;
  4049. }
  4050. /*
  4051. * check if command supports extra parameters
  4052. */
  4053. if (completed_args == 0 && getsize) {
  4054. /*
  4055. * get extra parameters size (based on main argument)
  4056. */
  4057. ret = (*getsize)(args_k, &xtra_sz);
  4058. if (ret) goto error_args;
  4059. completed_args = 1;
  4060. DPRINT(("restart_args sz=%lu xtra_sz=%lu\n", sz, xtra_sz));
  4061. /* retry if necessary */
  4062. if (likely(xtra_sz)) goto restart_args;
  4063. }
  4064. if (unlikely((cmd_flags & PFM_CMD_FD) == 0)) goto skip_fd;
  4065. ret = -EBADF;
  4066. file = fget(fd);
  4067. if (unlikely(file == NULL)) {
  4068. DPRINT(("invalid fd %d\n", fd));
  4069. goto error_args;
  4070. }
  4071. if (unlikely(PFM_IS_FILE(file) == 0)) {
  4072. DPRINT(("fd %d not related to perfmon\n", fd));
  4073. goto error_args;
  4074. }
  4075. ctx = (pfm_context_t *)file->private_data;
  4076. if (unlikely(ctx == NULL)) {
  4077. DPRINT(("no context for fd %d\n", fd));
  4078. goto error_args;
  4079. }
  4080. prefetch(&ctx->ctx_state);
  4081. PROTECT_CTX(ctx, flags);
  4082. /*
  4083. * check task is stopped
  4084. */
  4085. ret = pfm_check_task_state(ctx, cmd, flags);
  4086. if (unlikely(ret)) goto abort_locked;
  4087. skip_fd:
  4088. ret = (*func)(ctx, args_k, count, ia64_task_regs(current));
  4089. call_made = 1;
  4090. abort_locked:
  4091. if (likely(ctx)) {
  4092. DPRINT(("context unlocked\n"));
  4093. UNPROTECT_CTX(ctx, flags);
  4094. fput(file);
  4095. }
  4096. /* copy argument back to user, if needed */
  4097. if (call_made && PFM_CMD_RW_ARG(cmd) && copy_to_user(arg, args_k, base_sz*count)) ret = -EFAULT;
  4098. error_args:
  4099. if (args_k) kfree(args_k);
  4100. DPRINT(("cmd=%s ret=%ld\n", PFM_CMD_NAME(cmd), ret));
  4101. return ret;
  4102. }
  4103. static void
  4104. pfm_resume_after_ovfl(pfm_context_t *ctx, unsigned long ovfl_regs, struct pt_regs *regs)
  4105. {
  4106. pfm_buffer_fmt_t *fmt = ctx->ctx_buf_fmt;
  4107. pfm_ovfl_ctrl_t rst_ctrl;
  4108. int state;
  4109. int ret = 0;
  4110. state = ctx->ctx_state;
  4111. /*
  4112. * Unlock sampling buffer and reset index atomically
  4113. * XXX: not really needed when blocking
  4114. */
  4115. if (CTX_HAS_SMPL(ctx)) {
  4116. rst_ctrl.bits.mask_monitoring = 0;
  4117. rst_ctrl.bits.reset_ovfl_pmds = 0;
  4118. if (state == PFM_CTX_LOADED)
  4119. ret = pfm_buf_fmt_restart_active(fmt, current, &rst_ctrl, ctx->ctx_smpl_hdr, regs);
  4120. else
  4121. ret = pfm_buf_fmt_restart(fmt, current, &rst_ctrl, ctx->ctx_smpl_hdr, regs);
  4122. } else {
  4123. rst_ctrl.bits.mask_monitoring = 0;
  4124. rst_ctrl.bits.reset_ovfl_pmds = 1;
  4125. }
  4126. if (ret == 0) {
  4127. if (rst_ctrl.bits.reset_ovfl_pmds) {
  4128. pfm_reset_regs(ctx, &ovfl_regs, PFM_PMD_LONG_RESET);
  4129. }
  4130. if (rst_ctrl.bits.mask_monitoring == 0) {
  4131. DPRINT(("resuming monitoring\n"));
  4132. if (ctx->ctx_state == PFM_CTX_MASKED) pfm_restore_monitoring(current);
  4133. } else {
  4134. DPRINT(("stopping monitoring\n"));
  4135. //pfm_stop_monitoring(current, regs);
  4136. }
  4137. ctx->ctx_state = PFM_CTX_LOADED;
  4138. }
  4139. }
  4140. /*
  4141. * context MUST BE LOCKED when calling
  4142. * can only be called for current
  4143. */
  4144. static void
  4145. pfm_context_force_terminate(pfm_context_t *ctx, struct pt_regs *regs)
  4146. {
  4147. int ret;
  4148. DPRINT(("entering for [%d]\n", current->pid));
  4149. ret = pfm_context_unload(ctx, NULL, 0, regs);
  4150. if (ret) {
  4151. printk(KERN_ERR "pfm_context_force_terminate: [%d] unloaded failed with %d\n", current->pid, ret);
  4152. }
  4153. /*
  4154. * and wakeup controlling task, indicating we are now disconnected
  4155. */
  4156. wake_up_interruptible(&ctx->ctx_zombieq);
  4157. /*
  4158. * given that context is still locked, the controlling
  4159. * task will only get access when we return from
  4160. * pfm_handle_work().
  4161. */
  4162. }
  4163. static int pfm_ovfl_notify_user(pfm_context_t *ctx, unsigned long ovfl_pmds);
  4164. void
  4165. pfm_handle_work(void)
  4166. {
  4167. pfm_context_t *ctx;
  4168. struct pt_regs *regs;
  4169. unsigned long flags;
  4170. unsigned long ovfl_regs;
  4171. unsigned int reason;
  4172. int ret;
  4173. ctx = PFM_GET_CTX(current);
  4174. if (ctx == NULL) {
  4175. printk(KERN_ERR "perfmon: [%d] has no PFM context\n", current->pid);
  4176. return;
  4177. }
  4178. PROTECT_CTX(ctx, flags);
  4179. PFM_SET_WORK_PENDING(current, 0);
  4180. pfm_clear_task_notify();
  4181. regs = ia64_task_regs(current);
  4182. /*
  4183. * extract reason for being here and clear
  4184. */
  4185. reason = ctx->ctx_fl_trap_reason;
  4186. ctx->ctx_fl_trap_reason = PFM_TRAP_REASON_NONE;
  4187. ovfl_regs = ctx->ctx_ovfl_regs[0];
  4188. DPRINT(("reason=%d state=%d\n", reason, ctx->ctx_state));
  4189. /*
  4190. * must be done before we check for simple-reset mode
  4191. */
  4192. if (ctx->ctx_fl_going_zombie || ctx->ctx_state == PFM_CTX_ZOMBIE) goto do_zombie;
  4193. //if (CTX_OVFL_NOBLOCK(ctx)) goto skip_blocking;
  4194. if (reason == PFM_TRAP_REASON_RESET) goto skip_blocking;
  4195. UNPROTECT_CTX(ctx, flags);
  4196. /*
  4197. * pfm_handle_work() is currently called with interrupts disabled.
  4198. * The down_interruptible call may sleep, therefore we
  4199. * must re-enable interrupts to avoid deadlocks. It is
  4200. * safe to do so because this function is called ONLY
  4201. * when returning to user level (PUStk=1), in which case
  4202. * there is no risk of kernel stack overflow due to deep
  4203. * interrupt nesting.
  4204. */
  4205. BUG_ON(flags & IA64_PSR_I);
  4206. local_irq_enable();
  4207. DPRINT(("before block sleeping\n"));
  4208. /*
  4209. * may go through without blocking on SMP systems
  4210. * if restart has been received already by the time we call down()
  4211. */
  4212. ret = down_interruptible(&ctx->ctx_restart_sem);
  4213. DPRINT(("after block sleeping ret=%d\n", ret));
  4214. /*
  4215. * disable interrupts to restore state we had upon entering
  4216. * this function
  4217. */
  4218. local_irq_disable();
  4219. PROTECT_CTX(ctx, flags);
  4220. /*
  4221. * we need to read the ovfl_regs only after wake-up
  4222. * because we may have had pfm_write_pmds() in between
  4223. * and that can changed PMD values and therefore
  4224. * ovfl_regs is reset for these new PMD values.
  4225. */
  4226. ovfl_regs = ctx->ctx_ovfl_regs[0];
  4227. if (ctx->ctx_fl_going_zombie) {
  4228. do_zombie:
  4229. DPRINT(("context is zombie, bailing out\n"));
  4230. pfm_context_force_terminate(ctx, regs);
  4231. goto nothing_to_do;
  4232. }
  4233. /*
  4234. * in case of interruption of down() we don't restart anything
  4235. */
  4236. if (ret < 0) goto nothing_to_do;
  4237. skip_blocking:
  4238. pfm_resume_after_ovfl(ctx, ovfl_regs, regs);
  4239. ctx->ctx_ovfl_regs[0] = 0UL;
  4240. nothing_to_do:
  4241. UNPROTECT_CTX(ctx, flags);
  4242. }
  4243. static int
  4244. pfm_notify_user(pfm_context_t *ctx, pfm_msg_t *msg)
  4245. {
  4246. if (ctx->ctx_state == PFM_CTX_ZOMBIE) {
  4247. DPRINT(("ignoring overflow notification, owner is zombie\n"));
  4248. return 0;
  4249. }
  4250. DPRINT(("waking up somebody\n"));
  4251. if (msg) wake_up_interruptible(&ctx->ctx_msgq_wait);
  4252. /*
  4253. * safe, we are not in intr handler, nor in ctxsw when
  4254. * we come here
  4255. */
  4256. kill_fasync (&ctx->ctx_async_queue, SIGIO, POLL_IN);
  4257. return 0;
  4258. }
  4259. static int
  4260. pfm_ovfl_notify_user(pfm_context_t *ctx, unsigned long ovfl_pmds)
  4261. {
  4262. pfm_msg_t *msg = NULL;
  4263. if (ctx->ctx_fl_no_msg == 0) {
  4264. msg = pfm_get_new_msg(ctx);
  4265. if (msg == NULL) {
  4266. printk(KERN_ERR "perfmon: pfm_ovfl_notify_user no more notification msgs\n");
  4267. return -1;
  4268. }
  4269. msg->pfm_ovfl_msg.msg_type = PFM_MSG_OVFL;
  4270. msg->pfm_ovfl_msg.msg_ctx_fd = ctx->ctx_fd;
  4271. msg->pfm_ovfl_msg.msg_active_set = 0;
  4272. msg->pfm_ovfl_msg.msg_ovfl_pmds[0] = ovfl_pmds;
  4273. msg->pfm_ovfl_msg.msg_ovfl_pmds[1] = 0UL;
  4274. msg->pfm_ovfl_msg.msg_ovfl_pmds[2] = 0UL;
  4275. msg->pfm_ovfl_msg.msg_ovfl_pmds[3] = 0UL;
  4276. msg->pfm_ovfl_msg.msg_tstamp = 0UL;
  4277. }
  4278. DPRINT(("ovfl msg: msg=%p no_msg=%d fd=%d ovfl_pmds=0x%lx\n",
  4279. msg,
  4280. ctx->ctx_fl_no_msg,
  4281. ctx->ctx_fd,
  4282. ovfl_pmds));
  4283. return pfm_notify_user(ctx, msg);
  4284. }
  4285. static int
  4286. pfm_end_notify_user(pfm_context_t *ctx)
  4287. {
  4288. pfm_msg_t *msg;
  4289. msg = pfm_get_new_msg(ctx);
  4290. if (msg == NULL) {
  4291. printk(KERN_ERR "perfmon: pfm_end_notify_user no more notification msgs\n");
  4292. return -1;
  4293. }
  4294. /* no leak */
  4295. memset(msg, 0, sizeof(*msg));
  4296. msg->pfm_end_msg.msg_type = PFM_MSG_END;
  4297. msg->pfm_end_msg.msg_ctx_fd = ctx->ctx_fd;
  4298. msg->pfm_ovfl_msg.msg_tstamp = 0UL;
  4299. DPRINT(("end msg: msg=%p no_msg=%d ctx_fd=%d\n",
  4300. msg,
  4301. ctx->ctx_fl_no_msg,
  4302. ctx->ctx_fd));
  4303. return pfm_notify_user(ctx, msg);
  4304. }
  4305. /*
  4306. * main overflow processing routine.
  4307. * it can be called from the interrupt path or explicitely during the context switch code
  4308. */
  4309. static void
  4310. pfm_overflow_handler(struct task_struct *task, pfm_context_t *ctx, u64 pmc0, struct pt_regs *regs)
  4311. {
  4312. pfm_ovfl_arg_t *ovfl_arg;
  4313. unsigned long mask;
  4314. unsigned long old_val, ovfl_val, new_val;
  4315. unsigned long ovfl_notify = 0UL, ovfl_pmds = 0UL, smpl_pmds = 0UL, reset_pmds;
  4316. unsigned long tstamp;
  4317. pfm_ovfl_ctrl_t ovfl_ctrl;
  4318. unsigned int i, has_smpl;
  4319. int must_notify = 0;
  4320. if (unlikely(ctx->ctx_state == PFM_CTX_ZOMBIE)) goto stop_monitoring;
  4321. /*
  4322. * sanity test. Should never happen
  4323. */
  4324. if (unlikely((pmc0 & 0x1) == 0)) goto sanity_check;
  4325. tstamp = ia64_get_itc();
  4326. mask = pmc0 >> PMU_FIRST_COUNTER;
  4327. ovfl_val = pmu_conf->ovfl_val;
  4328. has_smpl = CTX_HAS_SMPL(ctx);
  4329. DPRINT_ovfl(("pmc0=0x%lx pid=%d iip=0x%lx, %s "
  4330. "used_pmds=0x%lx\n",
  4331. pmc0,
  4332. task ? task->pid: -1,
  4333. (regs ? regs->cr_iip : 0),
  4334. CTX_OVFL_NOBLOCK(ctx) ? "nonblocking" : "blocking",
  4335. ctx->ctx_used_pmds[0]));
  4336. /*
  4337. * first we update the virtual counters
  4338. * assume there was a prior ia64_srlz_d() issued
  4339. */
  4340. for (i = PMU_FIRST_COUNTER; mask ; i++, mask >>= 1) {
  4341. /* skip pmd which did not overflow */
  4342. if ((mask & 0x1) == 0) continue;
  4343. /*
  4344. * Note that the pmd is not necessarily 0 at this point as qualified events
  4345. * may have happened before the PMU was frozen. The residual count is not
  4346. * taken into consideration here but will be with any read of the pmd via
  4347. * pfm_read_pmds().
  4348. */
  4349. old_val = new_val = ctx->ctx_pmds[i].val;
  4350. new_val += 1 + ovfl_val;
  4351. ctx->ctx_pmds[i].val = new_val;
  4352. /*
  4353. * check for overflow condition
  4354. */
  4355. if (likely(old_val > new_val)) {
  4356. ovfl_pmds |= 1UL << i;
  4357. if (PMC_OVFL_NOTIFY(ctx, i)) ovfl_notify |= 1UL << i;
  4358. }
  4359. DPRINT_ovfl(("ctx_pmd[%d].val=0x%lx old_val=0x%lx pmd=0x%lx ovfl_pmds=0x%lx ovfl_notify=0x%lx\n",
  4360. i,
  4361. new_val,
  4362. old_val,
  4363. ia64_get_pmd(i) & ovfl_val,
  4364. ovfl_pmds,
  4365. ovfl_notify));
  4366. }
  4367. /*
  4368. * there was no 64-bit overflow, nothing else to do
  4369. */
  4370. if (ovfl_pmds == 0UL) return;
  4371. /*
  4372. * reset all control bits
  4373. */
  4374. ovfl_ctrl.val = 0;
  4375. reset_pmds = 0UL;
  4376. /*
  4377. * if a sampling format module exists, then we "cache" the overflow by
  4378. * calling the module's handler() routine.
  4379. */
  4380. if (has_smpl) {
  4381. unsigned long start_cycles, end_cycles;
  4382. unsigned long pmd_mask;
  4383. int j, k, ret = 0;
  4384. int this_cpu = smp_processor_id();
  4385. pmd_mask = ovfl_pmds >> PMU_FIRST_COUNTER;
  4386. ovfl_arg = &ctx->ctx_ovfl_arg;
  4387. prefetch(ctx->ctx_smpl_hdr);
  4388. for(i=PMU_FIRST_COUNTER; pmd_mask && ret == 0; i++, pmd_mask >>=1) {
  4389. mask = 1UL << i;
  4390. if ((pmd_mask & 0x1) == 0) continue;
  4391. ovfl_arg->ovfl_pmd = (unsigned char )i;
  4392. ovfl_arg->ovfl_notify = ovfl_notify & mask ? 1 : 0;
  4393. ovfl_arg->active_set = 0;
  4394. ovfl_arg->ovfl_ctrl.val = 0; /* module must fill in all fields */
  4395. ovfl_arg->smpl_pmds[0] = smpl_pmds = ctx->ctx_pmds[i].smpl_pmds[0];
  4396. ovfl_arg->pmd_value = ctx->ctx_pmds[i].val;
  4397. ovfl_arg->pmd_last_reset = ctx->ctx_pmds[i].lval;
  4398. ovfl_arg->pmd_eventid = ctx->ctx_pmds[i].eventid;
  4399. /*
  4400. * copy values of pmds of interest. Sampling format may copy them
  4401. * into sampling buffer.
  4402. */
  4403. if (smpl_pmds) {
  4404. for(j=0, k=0; smpl_pmds; j++, smpl_pmds >>=1) {
  4405. if ((smpl_pmds & 0x1) == 0) continue;
  4406. ovfl_arg->smpl_pmds_values[k++] = PMD_IS_COUNTING(j) ? pfm_read_soft_counter(ctx, j) : ia64_get_pmd(j);
  4407. DPRINT_ovfl(("smpl_pmd[%d]=pmd%u=0x%lx\n", k-1, j, ovfl_arg->smpl_pmds_values[k-1]));
  4408. }
  4409. }
  4410. pfm_stats[this_cpu].pfm_smpl_handler_calls++;
  4411. start_cycles = ia64_get_itc();
  4412. /*
  4413. * call custom buffer format record (handler) routine
  4414. */
  4415. ret = (*ctx->ctx_buf_fmt->fmt_handler)(task, ctx->ctx_smpl_hdr, ovfl_arg, regs, tstamp);
  4416. end_cycles = ia64_get_itc();
  4417. /*
  4418. * For those controls, we take the union because they have
  4419. * an all or nothing behavior.
  4420. */
  4421. ovfl_ctrl.bits.notify_user |= ovfl_arg->ovfl_ctrl.bits.notify_user;
  4422. ovfl_ctrl.bits.block_task |= ovfl_arg->ovfl_ctrl.bits.block_task;
  4423. ovfl_ctrl.bits.mask_monitoring |= ovfl_arg->ovfl_ctrl.bits.mask_monitoring;
  4424. /*
  4425. * build the bitmask of pmds to reset now
  4426. */
  4427. if (ovfl_arg->ovfl_ctrl.bits.reset_ovfl_pmds) reset_pmds |= mask;
  4428. pfm_stats[this_cpu].pfm_smpl_handler_cycles += end_cycles - start_cycles;
  4429. }
  4430. /*
  4431. * when the module cannot handle the rest of the overflows, we abort right here
  4432. */
  4433. if (ret && pmd_mask) {
  4434. DPRINT(("handler aborts leftover ovfl_pmds=0x%lx\n",
  4435. pmd_mask<<PMU_FIRST_COUNTER));
  4436. }
  4437. /*
  4438. * remove the pmds we reset now from the set of pmds to reset in pfm_restart()
  4439. */
  4440. ovfl_pmds &= ~reset_pmds;
  4441. } else {
  4442. /*
  4443. * when no sampling module is used, then the default
  4444. * is to notify on overflow if requested by user
  4445. */
  4446. ovfl_ctrl.bits.notify_user = ovfl_notify ? 1 : 0;
  4447. ovfl_ctrl.bits.block_task = ovfl_notify ? 1 : 0;
  4448. ovfl_ctrl.bits.mask_monitoring = ovfl_notify ? 1 : 0; /* XXX: change for saturation */
  4449. ovfl_ctrl.bits.reset_ovfl_pmds = ovfl_notify ? 0 : 1;
  4450. /*
  4451. * if needed, we reset all overflowed pmds
  4452. */
  4453. if (ovfl_notify == 0) reset_pmds = ovfl_pmds;
  4454. }
  4455. DPRINT_ovfl(("ovfl_pmds=0x%lx reset_pmds=0x%lx\n", ovfl_pmds, reset_pmds));
  4456. /*
  4457. * reset the requested PMD registers using the short reset values
  4458. */
  4459. if (reset_pmds) {
  4460. unsigned long bm = reset_pmds;
  4461. pfm_reset_regs(ctx, &bm, PFM_PMD_SHORT_RESET);
  4462. }
  4463. if (ovfl_notify && ovfl_ctrl.bits.notify_user) {
  4464. /*
  4465. * keep track of what to reset when unblocking
  4466. */
  4467. ctx->ctx_ovfl_regs[0] = ovfl_pmds;
  4468. /*
  4469. * check for blocking context
  4470. */
  4471. if (CTX_OVFL_NOBLOCK(ctx) == 0 && ovfl_ctrl.bits.block_task) {
  4472. ctx->ctx_fl_trap_reason = PFM_TRAP_REASON_BLOCK;
  4473. /*
  4474. * set the perfmon specific checking pending work for the task
  4475. */
  4476. PFM_SET_WORK_PENDING(task, 1);
  4477. /*
  4478. * when coming from ctxsw, current still points to the
  4479. * previous task, therefore we must work with task and not current.
  4480. */
  4481. pfm_set_task_notify(task);
  4482. }
  4483. /*
  4484. * defer until state is changed (shorten spin window). the context is locked
  4485. * anyway, so the signal receiver would come spin for nothing.
  4486. */
  4487. must_notify = 1;
  4488. }
  4489. DPRINT_ovfl(("owner [%d] pending=%ld reason=%u ovfl_pmds=0x%lx ovfl_notify=0x%lx masked=%d\n",
  4490. GET_PMU_OWNER() ? GET_PMU_OWNER()->pid : -1,
  4491. PFM_GET_WORK_PENDING(task),
  4492. ctx->ctx_fl_trap_reason,
  4493. ovfl_pmds,
  4494. ovfl_notify,
  4495. ovfl_ctrl.bits.mask_monitoring ? 1 : 0));
  4496. /*
  4497. * in case monitoring must be stopped, we toggle the psr bits
  4498. */
  4499. if (ovfl_ctrl.bits.mask_monitoring) {
  4500. pfm_mask_monitoring(task);
  4501. ctx->ctx_state = PFM_CTX_MASKED;
  4502. ctx->ctx_fl_can_restart = 1;
  4503. }
  4504. /*
  4505. * send notification now
  4506. */
  4507. if (must_notify) pfm_ovfl_notify_user(ctx, ovfl_notify);
  4508. return;
  4509. sanity_check:
  4510. printk(KERN_ERR "perfmon: CPU%d overflow handler [%d] pmc0=0x%lx\n",
  4511. smp_processor_id(),
  4512. task ? task->pid : -1,
  4513. pmc0);
  4514. return;
  4515. stop_monitoring:
  4516. /*
  4517. * in SMP, zombie context is never restored but reclaimed in pfm_load_regs().
  4518. * Moreover, zombies are also reclaimed in pfm_save_regs(). Therefore we can
  4519. * come here as zombie only if the task is the current task. In which case, we
  4520. * can access the PMU hardware directly.
  4521. *
  4522. * Note that zombies do have PM_VALID set. So here we do the minimal.
  4523. *
  4524. * In case the context was zombified it could not be reclaimed at the time
  4525. * the monitoring program exited. At this point, the PMU reservation has been
  4526. * returned, the sampiing buffer has been freed. We must convert this call
  4527. * into a spurious interrupt. However, we must also avoid infinite overflows
  4528. * by stopping monitoring for this task. We can only come here for a per-task
  4529. * context. All we need to do is to stop monitoring using the psr bits which
  4530. * are always task private. By re-enabling secure montioring, we ensure that
  4531. * the monitored task will not be able to re-activate monitoring.
  4532. * The task will eventually be context switched out, at which point the context
  4533. * will be reclaimed (that includes releasing ownership of the PMU).
  4534. *
  4535. * So there might be a window of time where the number of per-task session is zero
  4536. * yet one PMU might have a owner and get at most one overflow interrupt for a zombie
  4537. * context. This is safe because if a per-task session comes in, it will push this one
  4538. * out and by the virtue on pfm_save_regs(), this one will disappear. If a system wide
  4539. * session is force on that CPU, given that we use task pinning, pfm_save_regs() will
  4540. * also push our zombie context out.
  4541. *
  4542. * Overall pretty hairy stuff....
  4543. */
  4544. DPRINT(("ctx is zombie for [%d], converted to spurious\n", task ? task->pid: -1));
  4545. pfm_clear_psr_up();
  4546. ia64_psr(regs)->up = 0;
  4547. ia64_psr(regs)->sp = 1;
  4548. return;
  4549. }
  4550. static int
  4551. pfm_do_interrupt_handler(int irq, void *arg, struct pt_regs *regs)
  4552. {
  4553. struct task_struct *task;
  4554. pfm_context_t *ctx;
  4555. unsigned long flags;
  4556. u64 pmc0;
  4557. int this_cpu = smp_processor_id();
  4558. int retval = 0;
  4559. pfm_stats[this_cpu].pfm_ovfl_intr_count++;
  4560. /*
  4561. * srlz.d done before arriving here
  4562. */
  4563. pmc0 = ia64_get_pmc(0);
  4564. task = GET_PMU_OWNER();
  4565. ctx = GET_PMU_CTX();
  4566. /*
  4567. * if we have some pending bits set
  4568. * assumes : if any PMC0.bit[63-1] is set, then PMC0.fr = 1
  4569. */
  4570. if (PMC0_HAS_OVFL(pmc0) && task) {
  4571. /*
  4572. * we assume that pmc0.fr is always set here
  4573. */
  4574. /* sanity check */
  4575. if (!ctx) goto report_spurious1;
  4576. if (ctx->ctx_fl_system == 0 && (task->thread.flags & IA64_THREAD_PM_VALID) == 0)
  4577. goto report_spurious2;
  4578. PROTECT_CTX_NOPRINT(ctx, flags);
  4579. pfm_overflow_handler(task, ctx, pmc0, regs);
  4580. UNPROTECT_CTX_NOPRINT(ctx, flags);
  4581. } else {
  4582. pfm_stats[this_cpu].pfm_spurious_ovfl_intr_count++;
  4583. retval = -1;
  4584. }
  4585. /*
  4586. * keep it unfrozen at all times
  4587. */
  4588. pfm_unfreeze_pmu();
  4589. return retval;
  4590. report_spurious1:
  4591. printk(KERN_INFO "perfmon: spurious overflow interrupt on CPU%d: process %d has no PFM context\n",
  4592. this_cpu, task->pid);
  4593. pfm_unfreeze_pmu();
  4594. return -1;
  4595. report_spurious2:
  4596. printk(KERN_INFO "perfmon: spurious overflow interrupt on CPU%d: process %d, invalid flag\n",
  4597. this_cpu,
  4598. task->pid);
  4599. pfm_unfreeze_pmu();
  4600. return -1;
  4601. }
  4602. static irqreturn_t
  4603. pfm_interrupt_handler(int irq, void *arg, struct pt_regs *regs)
  4604. {
  4605. unsigned long start_cycles, total_cycles;
  4606. unsigned long min, max;
  4607. int this_cpu;
  4608. int ret;
  4609. this_cpu = get_cpu();
  4610. min = pfm_stats[this_cpu].pfm_ovfl_intr_cycles_min;
  4611. max = pfm_stats[this_cpu].pfm_ovfl_intr_cycles_max;
  4612. start_cycles = ia64_get_itc();
  4613. ret = pfm_do_interrupt_handler(irq, arg, regs);
  4614. total_cycles = ia64_get_itc();
  4615. /*
  4616. * don't measure spurious interrupts
  4617. */
  4618. if (likely(ret == 0)) {
  4619. total_cycles -= start_cycles;
  4620. if (total_cycles < min) pfm_stats[this_cpu].pfm_ovfl_intr_cycles_min = total_cycles;
  4621. if (total_cycles > max) pfm_stats[this_cpu].pfm_ovfl_intr_cycles_max = total_cycles;
  4622. pfm_stats[this_cpu].pfm_ovfl_intr_cycles += total_cycles;
  4623. }
  4624. put_cpu_no_resched();
  4625. return IRQ_HANDLED;
  4626. }
  4627. /*
  4628. * /proc/perfmon interface, for debug only
  4629. */
  4630. #define PFM_PROC_SHOW_HEADER ((void *)NR_CPUS+1)
  4631. static void *
  4632. pfm_proc_start(struct seq_file *m, loff_t *pos)
  4633. {
  4634. if (*pos == 0) {
  4635. return PFM_PROC_SHOW_HEADER;
  4636. }
  4637. while (*pos <= NR_CPUS) {
  4638. if (cpu_online(*pos - 1)) {
  4639. return (void *)*pos;
  4640. }
  4641. ++*pos;
  4642. }
  4643. return NULL;
  4644. }
  4645. static void *
  4646. pfm_proc_next(struct seq_file *m, void *v, loff_t *pos)
  4647. {
  4648. ++*pos;
  4649. return pfm_proc_start(m, pos);
  4650. }
  4651. static void
  4652. pfm_proc_stop(struct seq_file *m, void *v)
  4653. {
  4654. }
  4655. static void
  4656. pfm_proc_show_header(struct seq_file *m)
  4657. {
  4658. struct list_head * pos;
  4659. pfm_buffer_fmt_t * entry;
  4660. unsigned long flags;
  4661. seq_printf(m,
  4662. "perfmon version : %u.%u\n"
  4663. "model : %s\n"
  4664. "fastctxsw : %s\n"
  4665. "expert mode : %s\n"
  4666. "ovfl_mask : 0x%lx\n"
  4667. "PMU flags : 0x%x\n",
  4668. PFM_VERSION_MAJ, PFM_VERSION_MIN,
  4669. pmu_conf->pmu_name,
  4670. pfm_sysctl.fastctxsw > 0 ? "Yes": "No",
  4671. pfm_sysctl.expert_mode > 0 ? "Yes": "No",
  4672. pmu_conf->ovfl_val,
  4673. pmu_conf->flags);
  4674. LOCK_PFS(flags);
  4675. seq_printf(m,
  4676. "proc_sessions : %u\n"
  4677. "sys_sessions : %u\n"
  4678. "sys_use_dbregs : %u\n"
  4679. "ptrace_use_dbregs : %u\n",
  4680. pfm_sessions.pfs_task_sessions,
  4681. pfm_sessions.pfs_sys_sessions,
  4682. pfm_sessions.pfs_sys_use_dbregs,
  4683. pfm_sessions.pfs_ptrace_use_dbregs);
  4684. UNLOCK_PFS(flags);
  4685. spin_lock(&pfm_buffer_fmt_lock);
  4686. list_for_each(pos, &pfm_buffer_fmt_list) {
  4687. entry = list_entry(pos, pfm_buffer_fmt_t, fmt_list);
  4688. seq_printf(m, "format : %02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x %s\n",
  4689. entry->fmt_uuid[0],
  4690. entry->fmt_uuid[1],
  4691. entry->fmt_uuid[2],
  4692. entry->fmt_uuid[3],
  4693. entry->fmt_uuid[4],
  4694. entry->fmt_uuid[5],
  4695. entry->fmt_uuid[6],
  4696. entry->fmt_uuid[7],
  4697. entry->fmt_uuid[8],
  4698. entry->fmt_uuid[9],
  4699. entry->fmt_uuid[10],
  4700. entry->fmt_uuid[11],
  4701. entry->fmt_uuid[12],
  4702. entry->fmt_uuid[13],
  4703. entry->fmt_uuid[14],
  4704. entry->fmt_uuid[15],
  4705. entry->fmt_name);
  4706. }
  4707. spin_unlock(&pfm_buffer_fmt_lock);
  4708. }
  4709. static int
  4710. pfm_proc_show(struct seq_file *m, void *v)
  4711. {
  4712. unsigned long psr;
  4713. unsigned int i;
  4714. int cpu;
  4715. if (v == PFM_PROC_SHOW_HEADER) {
  4716. pfm_proc_show_header(m);
  4717. return 0;
  4718. }
  4719. /* show info for CPU (v - 1) */
  4720. cpu = (long)v - 1;
  4721. seq_printf(m,
  4722. "CPU%-2d overflow intrs : %lu\n"
  4723. "CPU%-2d overflow cycles : %lu\n"
  4724. "CPU%-2d overflow min : %lu\n"
  4725. "CPU%-2d overflow max : %lu\n"
  4726. "CPU%-2d smpl handler calls : %lu\n"
  4727. "CPU%-2d smpl handler cycles : %lu\n"
  4728. "CPU%-2d spurious intrs : %lu\n"
  4729. "CPU%-2d replay intrs : %lu\n"
  4730. "CPU%-2d syst_wide : %d\n"
  4731. "CPU%-2d dcr_pp : %d\n"
  4732. "CPU%-2d exclude idle : %d\n"
  4733. "CPU%-2d owner : %d\n"
  4734. "CPU%-2d context : %p\n"
  4735. "CPU%-2d activations : %lu\n",
  4736. cpu, pfm_stats[cpu].pfm_ovfl_intr_count,
  4737. cpu, pfm_stats[cpu].pfm_ovfl_intr_cycles,
  4738. cpu, pfm_stats[cpu].pfm_ovfl_intr_cycles_min,
  4739. cpu, pfm_stats[cpu].pfm_ovfl_intr_cycles_max,
  4740. cpu, pfm_stats[cpu].pfm_smpl_handler_calls,
  4741. cpu, pfm_stats[cpu].pfm_smpl_handler_cycles,
  4742. cpu, pfm_stats[cpu].pfm_spurious_ovfl_intr_count,
  4743. cpu, pfm_stats[cpu].pfm_replay_ovfl_intr_count,
  4744. cpu, pfm_get_cpu_data(pfm_syst_info, cpu) & PFM_CPUINFO_SYST_WIDE ? 1 : 0,
  4745. cpu, pfm_get_cpu_data(pfm_syst_info, cpu) & PFM_CPUINFO_DCR_PP ? 1 : 0,
  4746. cpu, pfm_get_cpu_data(pfm_syst_info, cpu) & PFM_CPUINFO_EXCL_IDLE ? 1 : 0,
  4747. cpu, pfm_get_cpu_data(pmu_owner, cpu) ? pfm_get_cpu_data(pmu_owner, cpu)->pid: -1,
  4748. cpu, pfm_get_cpu_data(pmu_ctx, cpu),
  4749. cpu, pfm_get_cpu_data(pmu_activation_number, cpu));
  4750. if (num_online_cpus() == 1 && pfm_sysctl.debug > 0) {
  4751. psr = pfm_get_psr();
  4752. ia64_srlz_d();
  4753. seq_printf(m,
  4754. "CPU%-2d psr : 0x%lx\n"
  4755. "CPU%-2d pmc0 : 0x%lx\n",
  4756. cpu, psr,
  4757. cpu, ia64_get_pmc(0));
  4758. for (i=0; PMC_IS_LAST(i) == 0; i++) {
  4759. if (PMC_IS_COUNTING(i) == 0) continue;
  4760. seq_printf(m,
  4761. "CPU%-2d pmc%u : 0x%lx\n"
  4762. "CPU%-2d pmd%u : 0x%lx\n",
  4763. cpu, i, ia64_get_pmc(i),
  4764. cpu, i, ia64_get_pmd(i));
  4765. }
  4766. }
  4767. return 0;
  4768. }
  4769. struct seq_operations pfm_seq_ops = {
  4770. .start = pfm_proc_start,
  4771. .next = pfm_proc_next,
  4772. .stop = pfm_proc_stop,
  4773. .show = pfm_proc_show
  4774. };
  4775. static int
  4776. pfm_proc_open(struct inode *inode, struct file *file)
  4777. {
  4778. return seq_open(file, &pfm_seq_ops);
  4779. }
  4780. /*
  4781. * we come here as soon as local_cpu_data->pfm_syst_wide is set. this happens
  4782. * during pfm_enable() hence before pfm_start(). We cannot assume monitoring
  4783. * is active or inactive based on mode. We must rely on the value in
  4784. * local_cpu_data->pfm_syst_info
  4785. */
  4786. void
  4787. pfm_syst_wide_update_task(struct task_struct *task, unsigned long info, int is_ctxswin)
  4788. {
  4789. struct pt_regs *regs;
  4790. unsigned long dcr;
  4791. unsigned long dcr_pp;
  4792. dcr_pp = info & PFM_CPUINFO_DCR_PP ? 1 : 0;
  4793. /*
  4794. * pid 0 is guaranteed to be the idle task. There is one such task with pid 0
  4795. * on every CPU, so we can rely on the pid to identify the idle task.
  4796. */
  4797. if ((info & PFM_CPUINFO_EXCL_IDLE) == 0 || task->pid) {
  4798. regs = ia64_task_regs(task);
  4799. ia64_psr(regs)->pp = is_ctxswin ? dcr_pp : 0;
  4800. return;
  4801. }
  4802. /*
  4803. * if monitoring has started
  4804. */
  4805. if (dcr_pp) {
  4806. dcr = ia64_getreg(_IA64_REG_CR_DCR);
  4807. /*
  4808. * context switching in?
  4809. */
  4810. if (is_ctxswin) {
  4811. /* mask monitoring for the idle task */
  4812. ia64_setreg(_IA64_REG_CR_DCR, dcr & ~IA64_DCR_PP);
  4813. pfm_clear_psr_pp();
  4814. ia64_srlz_i();
  4815. return;
  4816. }
  4817. /*
  4818. * context switching out
  4819. * restore monitoring for next task
  4820. *
  4821. * Due to inlining this odd if-then-else construction generates
  4822. * better code.
  4823. */
  4824. ia64_setreg(_IA64_REG_CR_DCR, dcr |IA64_DCR_PP);
  4825. pfm_set_psr_pp();
  4826. ia64_srlz_i();
  4827. }
  4828. }
  4829. #ifdef CONFIG_SMP
  4830. static void
  4831. pfm_force_cleanup(pfm_context_t *ctx, struct pt_regs *regs)
  4832. {
  4833. struct task_struct *task = ctx->ctx_task;
  4834. ia64_psr(regs)->up = 0;
  4835. ia64_psr(regs)->sp = 1;
  4836. if (GET_PMU_OWNER() == task) {
  4837. DPRINT(("cleared ownership for [%d]\n", ctx->ctx_task->pid));
  4838. SET_PMU_OWNER(NULL, NULL);
  4839. }
  4840. /*
  4841. * disconnect the task from the context and vice-versa
  4842. */
  4843. PFM_SET_WORK_PENDING(task, 0);
  4844. task->thread.pfm_context = NULL;
  4845. task->thread.flags &= ~IA64_THREAD_PM_VALID;
  4846. DPRINT(("force cleanup for [%d]\n", task->pid));
  4847. }
  4848. /*
  4849. * in 2.6, interrupts are masked when we come here and the runqueue lock is held
  4850. */
  4851. void
  4852. pfm_save_regs(struct task_struct *task)
  4853. {
  4854. pfm_context_t *ctx;
  4855. struct thread_struct *t;
  4856. unsigned long flags;
  4857. u64 psr;
  4858. ctx = PFM_GET_CTX(task);
  4859. if (ctx == NULL) return;
  4860. t = &task->thread;
  4861. /*
  4862. * we always come here with interrupts ALREADY disabled by
  4863. * the scheduler. So we simply need to protect against concurrent
  4864. * access, not CPU concurrency.
  4865. */
  4866. flags = pfm_protect_ctx_ctxsw(ctx);
  4867. if (ctx->ctx_state == PFM_CTX_ZOMBIE) {
  4868. struct pt_regs *regs = ia64_task_regs(task);
  4869. pfm_clear_psr_up();
  4870. pfm_force_cleanup(ctx, regs);
  4871. BUG_ON(ctx->ctx_smpl_hdr);
  4872. pfm_unprotect_ctx_ctxsw(ctx, flags);
  4873. pfm_context_free(ctx);
  4874. return;
  4875. }
  4876. /*
  4877. * save current PSR: needed because we modify it
  4878. */
  4879. ia64_srlz_d();
  4880. psr = pfm_get_psr();
  4881. BUG_ON(psr & (IA64_PSR_I));
  4882. /*
  4883. * stop monitoring:
  4884. * This is the last instruction which may generate an overflow
  4885. *
  4886. * We do not need to set psr.sp because, it is irrelevant in kernel.
  4887. * It will be restored from ipsr when going back to user level
  4888. */
  4889. pfm_clear_psr_up();
  4890. /*
  4891. * keep a copy of psr.up (for reload)
  4892. */
  4893. ctx->ctx_saved_psr_up = psr & IA64_PSR_UP;
  4894. /*
  4895. * release ownership of this PMU.
  4896. * PM interrupts are masked, so nothing
  4897. * can happen.
  4898. */
  4899. SET_PMU_OWNER(NULL, NULL);
  4900. /*
  4901. * we systematically save the PMD as we have no
  4902. * guarantee we will be schedule at that same
  4903. * CPU again.
  4904. */
  4905. pfm_save_pmds(t->pmds, ctx->ctx_used_pmds[0]);
  4906. /*
  4907. * save pmc0 ia64_srlz_d() done in pfm_save_pmds()
  4908. * we will need it on the restore path to check
  4909. * for pending overflow.
  4910. */
  4911. t->pmcs[0] = ia64_get_pmc(0);
  4912. /*
  4913. * unfreeze PMU if had pending overflows
  4914. */
  4915. if (t->pmcs[0] & ~0x1UL) pfm_unfreeze_pmu();
  4916. /*
  4917. * finally, allow context access.
  4918. * interrupts will still be masked after this call.
  4919. */
  4920. pfm_unprotect_ctx_ctxsw(ctx, flags);
  4921. }
  4922. #else /* !CONFIG_SMP */
  4923. void
  4924. pfm_save_regs(struct task_struct *task)
  4925. {
  4926. pfm_context_t *ctx;
  4927. u64 psr;
  4928. ctx = PFM_GET_CTX(task);
  4929. if (ctx == NULL) return;
  4930. /*
  4931. * save current PSR: needed because we modify it
  4932. */
  4933. psr = pfm_get_psr();
  4934. BUG_ON(psr & (IA64_PSR_I));
  4935. /*
  4936. * stop monitoring:
  4937. * This is the last instruction which may generate an overflow
  4938. *
  4939. * We do not need to set psr.sp because, it is irrelevant in kernel.
  4940. * It will be restored from ipsr when going back to user level
  4941. */
  4942. pfm_clear_psr_up();
  4943. /*
  4944. * keep a copy of psr.up (for reload)
  4945. */
  4946. ctx->ctx_saved_psr_up = psr & IA64_PSR_UP;
  4947. }
  4948. static void
  4949. pfm_lazy_save_regs (struct task_struct *task)
  4950. {
  4951. pfm_context_t *ctx;
  4952. struct thread_struct *t;
  4953. unsigned long flags;
  4954. { u64 psr = pfm_get_psr();
  4955. BUG_ON(psr & IA64_PSR_UP);
  4956. }
  4957. ctx = PFM_GET_CTX(task);
  4958. t = &task->thread;
  4959. /*
  4960. * we need to mask PMU overflow here to
  4961. * make sure that we maintain pmc0 until
  4962. * we save it. overflow interrupts are
  4963. * treated as spurious if there is no
  4964. * owner.
  4965. *
  4966. * XXX: I don't think this is necessary
  4967. */
  4968. PROTECT_CTX(ctx,flags);
  4969. /*
  4970. * release ownership of this PMU.
  4971. * must be done before we save the registers.
  4972. *
  4973. * after this call any PMU interrupt is treated
  4974. * as spurious.
  4975. */
  4976. SET_PMU_OWNER(NULL, NULL);
  4977. /*
  4978. * save all the pmds we use
  4979. */
  4980. pfm_save_pmds(t->pmds, ctx->ctx_used_pmds[0]);
  4981. /*
  4982. * save pmc0 ia64_srlz_d() done in pfm_save_pmds()
  4983. * it is needed to check for pended overflow
  4984. * on the restore path
  4985. */
  4986. t->pmcs[0] = ia64_get_pmc(0);
  4987. /*
  4988. * unfreeze PMU if had pending overflows
  4989. */
  4990. if (t->pmcs[0] & ~0x1UL) pfm_unfreeze_pmu();
  4991. /*
  4992. * now get can unmask PMU interrupts, they will
  4993. * be treated as purely spurious and we will not
  4994. * lose any information
  4995. */
  4996. UNPROTECT_CTX(ctx,flags);
  4997. }
  4998. #endif /* CONFIG_SMP */
  4999. #ifdef CONFIG_SMP
  5000. /*
  5001. * in 2.6, interrupts are masked when we come here and the runqueue lock is held
  5002. */
  5003. void
  5004. pfm_load_regs (struct task_struct *task)
  5005. {
  5006. pfm_context_t *ctx;
  5007. struct thread_struct *t;
  5008. unsigned long pmc_mask = 0UL, pmd_mask = 0UL;
  5009. unsigned long flags;
  5010. u64 psr, psr_up;
  5011. int need_irq_resend;
  5012. ctx = PFM_GET_CTX(task);
  5013. if (unlikely(ctx == NULL)) return;
  5014. BUG_ON(GET_PMU_OWNER());
  5015. t = &task->thread;
  5016. /*
  5017. * possible on unload
  5018. */
  5019. if (unlikely((t->flags & IA64_THREAD_PM_VALID) == 0)) return;
  5020. /*
  5021. * we always come here with interrupts ALREADY disabled by
  5022. * the scheduler. So we simply need to protect against concurrent
  5023. * access, not CPU concurrency.
  5024. */
  5025. flags = pfm_protect_ctx_ctxsw(ctx);
  5026. psr = pfm_get_psr();
  5027. need_irq_resend = pmu_conf->flags & PFM_PMU_IRQ_RESEND;
  5028. BUG_ON(psr & (IA64_PSR_UP|IA64_PSR_PP));
  5029. BUG_ON(psr & IA64_PSR_I);
  5030. if (unlikely(ctx->ctx_state == PFM_CTX_ZOMBIE)) {
  5031. struct pt_regs *regs = ia64_task_regs(task);
  5032. BUG_ON(ctx->ctx_smpl_hdr);
  5033. pfm_force_cleanup(ctx, regs);
  5034. pfm_unprotect_ctx_ctxsw(ctx, flags);
  5035. /*
  5036. * this one (kmalloc'ed) is fine with interrupts disabled
  5037. */
  5038. pfm_context_free(ctx);
  5039. return;
  5040. }
  5041. /*
  5042. * we restore ALL the debug registers to avoid picking up
  5043. * stale state.
  5044. */
  5045. if (ctx->ctx_fl_using_dbreg) {
  5046. pfm_restore_ibrs(ctx->ctx_ibrs, pmu_conf->num_ibrs);
  5047. pfm_restore_dbrs(ctx->ctx_dbrs, pmu_conf->num_dbrs);
  5048. }
  5049. /*
  5050. * retrieve saved psr.up
  5051. */
  5052. psr_up = ctx->ctx_saved_psr_up;
  5053. /*
  5054. * if we were the last user of the PMU on that CPU,
  5055. * then nothing to do except restore psr
  5056. */
  5057. if (GET_LAST_CPU(ctx) == smp_processor_id() && ctx->ctx_last_activation == GET_ACTIVATION()) {
  5058. /*
  5059. * retrieve partial reload masks (due to user modifications)
  5060. */
  5061. pmc_mask = ctx->ctx_reload_pmcs[0];
  5062. pmd_mask = ctx->ctx_reload_pmds[0];
  5063. } else {
  5064. /*
  5065. * To avoid leaking information to the user level when psr.sp=0,
  5066. * we must reload ALL implemented pmds (even the ones we don't use).
  5067. * In the kernel we only allow PFM_READ_PMDS on registers which
  5068. * we initialized or requested (sampling) so there is no risk there.
  5069. */
  5070. pmd_mask = pfm_sysctl.fastctxsw ? ctx->ctx_used_pmds[0] : ctx->ctx_all_pmds[0];
  5071. /*
  5072. * ALL accessible PMCs are systematically reloaded, unused registers
  5073. * get their default (from pfm_reset_pmu_state()) values to avoid picking
  5074. * up stale configuration.
  5075. *
  5076. * PMC0 is never in the mask. It is always restored separately.
  5077. */
  5078. pmc_mask = ctx->ctx_all_pmcs[0];
  5079. }
  5080. /*
  5081. * when context is MASKED, we will restore PMC with plm=0
  5082. * and PMD with stale information, but that's ok, nothing
  5083. * will be captured.
  5084. *
  5085. * XXX: optimize here
  5086. */
  5087. if (pmd_mask) pfm_restore_pmds(t->pmds, pmd_mask);
  5088. if (pmc_mask) pfm_restore_pmcs(t->pmcs, pmc_mask);
  5089. /*
  5090. * check for pending overflow at the time the state
  5091. * was saved.
  5092. */
  5093. if (unlikely(PMC0_HAS_OVFL(t->pmcs[0]))) {
  5094. /*
  5095. * reload pmc0 with the overflow information
  5096. * On McKinley PMU, this will trigger a PMU interrupt
  5097. */
  5098. ia64_set_pmc(0, t->pmcs[0]);
  5099. ia64_srlz_d();
  5100. t->pmcs[0] = 0UL;
  5101. /*
  5102. * will replay the PMU interrupt
  5103. */
  5104. if (need_irq_resend) hw_resend_irq(NULL, IA64_PERFMON_VECTOR);
  5105. pfm_stats[smp_processor_id()].pfm_replay_ovfl_intr_count++;
  5106. }
  5107. /*
  5108. * we just did a reload, so we reset the partial reload fields
  5109. */
  5110. ctx->ctx_reload_pmcs[0] = 0UL;
  5111. ctx->ctx_reload_pmds[0] = 0UL;
  5112. SET_LAST_CPU(ctx, smp_processor_id());
  5113. /*
  5114. * dump activation value for this PMU
  5115. */
  5116. INC_ACTIVATION();
  5117. /*
  5118. * record current activation for this context
  5119. */
  5120. SET_ACTIVATION(ctx);
  5121. /*
  5122. * establish new ownership.
  5123. */
  5124. SET_PMU_OWNER(task, ctx);
  5125. /*
  5126. * restore the psr.up bit. measurement
  5127. * is active again.
  5128. * no PMU interrupt can happen at this point
  5129. * because we still have interrupts disabled.
  5130. */
  5131. if (likely(psr_up)) pfm_set_psr_up();
  5132. /*
  5133. * allow concurrent access to context
  5134. */
  5135. pfm_unprotect_ctx_ctxsw(ctx, flags);
  5136. }
  5137. #else /* !CONFIG_SMP */
  5138. /*
  5139. * reload PMU state for UP kernels
  5140. * in 2.5 we come here with interrupts disabled
  5141. */
  5142. void
  5143. pfm_load_regs (struct task_struct *task)
  5144. {
  5145. struct thread_struct *t;
  5146. pfm_context_t *ctx;
  5147. struct task_struct *owner;
  5148. unsigned long pmd_mask, pmc_mask;
  5149. u64 psr, psr_up;
  5150. int need_irq_resend;
  5151. owner = GET_PMU_OWNER();
  5152. ctx = PFM_GET_CTX(task);
  5153. t = &task->thread;
  5154. psr = pfm_get_psr();
  5155. BUG_ON(psr & (IA64_PSR_UP|IA64_PSR_PP));
  5156. BUG_ON(psr & IA64_PSR_I);
  5157. /*
  5158. * we restore ALL the debug registers to avoid picking up
  5159. * stale state.
  5160. *
  5161. * This must be done even when the task is still the owner
  5162. * as the registers may have been modified via ptrace()
  5163. * (not perfmon) by the previous task.
  5164. */
  5165. if (ctx->ctx_fl_using_dbreg) {
  5166. pfm_restore_ibrs(ctx->ctx_ibrs, pmu_conf->num_ibrs);
  5167. pfm_restore_dbrs(ctx->ctx_dbrs, pmu_conf->num_dbrs);
  5168. }
  5169. /*
  5170. * retrieved saved psr.up
  5171. */
  5172. psr_up = ctx->ctx_saved_psr_up;
  5173. need_irq_resend = pmu_conf->flags & PFM_PMU_IRQ_RESEND;
  5174. /*
  5175. * short path, our state is still there, just
  5176. * need to restore psr and we go
  5177. *
  5178. * we do not touch either PMC nor PMD. the psr is not touched
  5179. * by the overflow_handler. So we are safe w.r.t. to interrupt
  5180. * concurrency even without interrupt masking.
  5181. */
  5182. if (likely(owner == task)) {
  5183. if (likely(psr_up)) pfm_set_psr_up();
  5184. return;
  5185. }
  5186. /*
  5187. * someone else is still using the PMU, first push it out and
  5188. * then we'll be able to install our stuff !
  5189. *
  5190. * Upon return, there will be no owner for the current PMU
  5191. */
  5192. if (owner) pfm_lazy_save_regs(owner);
  5193. /*
  5194. * To avoid leaking information to the user level when psr.sp=0,
  5195. * we must reload ALL implemented pmds (even the ones we don't use).
  5196. * In the kernel we only allow PFM_READ_PMDS on registers which
  5197. * we initialized or requested (sampling) so there is no risk there.
  5198. */
  5199. pmd_mask = pfm_sysctl.fastctxsw ? ctx->ctx_used_pmds[0] : ctx->ctx_all_pmds[0];
  5200. /*
  5201. * ALL accessible PMCs are systematically reloaded, unused registers
  5202. * get their default (from pfm_reset_pmu_state()) values to avoid picking
  5203. * up stale configuration.
  5204. *
  5205. * PMC0 is never in the mask. It is always restored separately
  5206. */
  5207. pmc_mask = ctx->ctx_all_pmcs[0];
  5208. pfm_restore_pmds(t->pmds, pmd_mask);
  5209. pfm_restore_pmcs(t->pmcs, pmc_mask);
  5210. /*
  5211. * check for pending overflow at the time the state
  5212. * was saved.
  5213. */
  5214. if (unlikely(PMC0_HAS_OVFL(t->pmcs[0]))) {
  5215. /*
  5216. * reload pmc0 with the overflow information
  5217. * On McKinley PMU, this will trigger a PMU interrupt
  5218. */
  5219. ia64_set_pmc(0, t->pmcs[0]);
  5220. ia64_srlz_d();
  5221. t->pmcs[0] = 0UL;
  5222. /*
  5223. * will replay the PMU interrupt
  5224. */
  5225. if (need_irq_resend) hw_resend_irq(NULL, IA64_PERFMON_VECTOR);
  5226. pfm_stats[smp_processor_id()].pfm_replay_ovfl_intr_count++;
  5227. }
  5228. /*
  5229. * establish new ownership.
  5230. */
  5231. SET_PMU_OWNER(task, ctx);
  5232. /*
  5233. * restore the psr.up bit. measurement
  5234. * is active again.
  5235. * no PMU interrupt can happen at this point
  5236. * because we still have interrupts disabled.
  5237. */
  5238. if (likely(psr_up)) pfm_set_psr_up();
  5239. }
  5240. #endif /* CONFIG_SMP */
  5241. /*
  5242. * this function assumes monitoring is stopped
  5243. */
  5244. static void
  5245. pfm_flush_pmds(struct task_struct *task, pfm_context_t *ctx)
  5246. {
  5247. u64 pmc0;
  5248. unsigned long mask2, val, pmd_val, ovfl_val;
  5249. int i, can_access_pmu = 0;
  5250. int is_self;
  5251. /*
  5252. * is the caller the task being monitored (or which initiated the
  5253. * session for system wide measurements)
  5254. */
  5255. is_self = ctx->ctx_task == task ? 1 : 0;
  5256. /*
  5257. * can access PMU is task is the owner of the PMU state on the current CPU
  5258. * or if we are running on the CPU bound to the context in system-wide mode
  5259. * (that is not necessarily the task the context is attached to in this mode).
  5260. * In system-wide we always have can_access_pmu true because a task running on an
  5261. * invalid processor is flagged earlier in the call stack (see pfm_stop).
  5262. */
  5263. can_access_pmu = (GET_PMU_OWNER() == task) || (ctx->ctx_fl_system && ctx->ctx_cpu == smp_processor_id());
  5264. if (can_access_pmu) {
  5265. /*
  5266. * Mark the PMU as not owned
  5267. * This will cause the interrupt handler to do nothing in case an overflow
  5268. * interrupt was in-flight
  5269. * This also guarantees that pmc0 will contain the final state
  5270. * It virtually gives us full control on overflow processing from that point
  5271. * on.
  5272. */
  5273. SET_PMU_OWNER(NULL, NULL);
  5274. DPRINT(("releasing ownership\n"));
  5275. /*
  5276. * read current overflow status:
  5277. *
  5278. * we are guaranteed to read the final stable state
  5279. */
  5280. ia64_srlz_d();
  5281. pmc0 = ia64_get_pmc(0); /* slow */
  5282. /*
  5283. * reset freeze bit, overflow status information destroyed
  5284. */
  5285. pfm_unfreeze_pmu();
  5286. } else {
  5287. pmc0 = task->thread.pmcs[0];
  5288. /*
  5289. * clear whatever overflow status bits there were
  5290. */
  5291. task->thread.pmcs[0] = 0;
  5292. }
  5293. ovfl_val = pmu_conf->ovfl_val;
  5294. /*
  5295. * we save all the used pmds
  5296. * we take care of overflows for counting PMDs
  5297. *
  5298. * XXX: sampling situation is not taken into account here
  5299. */
  5300. mask2 = ctx->ctx_used_pmds[0];
  5301. DPRINT(("is_self=%d ovfl_val=0x%lx mask2=0x%lx\n", is_self, ovfl_val, mask2));
  5302. for (i = 0; mask2; i++, mask2>>=1) {
  5303. /* skip non used pmds */
  5304. if ((mask2 & 0x1) == 0) continue;
  5305. /*
  5306. * can access PMU always true in system wide mode
  5307. */
  5308. val = pmd_val = can_access_pmu ? ia64_get_pmd(i) : task->thread.pmds[i];
  5309. if (PMD_IS_COUNTING(i)) {
  5310. DPRINT(("[%d] pmd[%d] ctx_pmd=0x%lx hw_pmd=0x%lx\n",
  5311. task->pid,
  5312. i,
  5313. ctx->ctx_pmds[i].val,
  5314. val & ovfl_val));
  5315. /*
  5316. * we rebuild the full 64 bit value of the counter
  5317. */
  5318. val = ctx->ctx_pmds[i].val + (val & ovfl_val);
  5319. /*
  5320. * now everything is in ctx_pmds[] and we need
  5321. * to clear the saved context from save_regs() such that
  5322. * pfm_read_pmds() gets the correct value
  5323. */
  5324. pmd_val = 0UL;
  5325. /*
  5326. * take care of overflow inline
  5327. */
  5328. if (pmc0 & (1UL << i)) {
  5329. val += 1 + ovfl_val;
  5330. DPRINT(("[%d] pmd[%d] overflowed\n", task->pid, i));
  5331. }
  5332. }
  5333. DPRINT(("[%d] ctx_pmd[%d]=0x%lx pmd_val=0x%lx\n", task->pid, i, val, pmd_val));
  5334. if (is_self) task->thread.pmds[i] = pmd_val;
  5335. ctx->ctx_pmds[i].val = val;
  5336. }
  5337. }
  5338. static struct irqaction perfmon_irqaction = {
  5339. .handler = pfm_interrupt_handler,
  5340. .flags = SA_INTERRUPT,
  5341. .name = "perfmon"
  5342. };
  5343. /*
  5344. * perfmon initialization routine, called from the initcall() table
  5345. */
  5346. static int init_pfm_fs(void);
  5347. static int __init
  5348. pfm_probe_pmu(void)
  5349. {
  5350. pmu_config_t **p;
  5351. int family;
  5352. family = local_cpu_data->family;
  5353. p = pmu_confs;
  5354. while(*p) {
  5355. if ((*p)->probe) {
  5356. if ((*p)->probe() == 0) goto found;
  5357. } else if ((*p)->pmu_family == family || (*p)->pmu_family == 0xff) {
  5358. goto found;
  5359. }
  5360. p++;
  5361. }
  5362. return -1;
  5363. found:
  5364. pmu_conf = *p;
  5365. return 0;
  5366. }
  5367. static struct file_operations pfm_proc_fops = {
  5368. .open = pfm_proc_open,
  5369. .read = seq_read,
  5370. .llseek = seq_lseek,
  5371. .release = seq_release,
  5372. };
  5373. int __init
  5374. pfm_init(void)
  5375. {
  5376. unsigned int n, n_counters, i;
  5377. printk("perfmon: version %u.%u IRQ %u\n",
  5378. PFM_VERSION_MAJ,
  5379. PFM_VERSION_MIN,
  5380. IA64_PERFMON_VECTOR);
  5381. if (pfm_probe_pmu()) {
  5382. printk(KERN_INFO "perfmon: disabled, there is no support for processor family %d\n",
  5383. local_cpu_data->family);
  5384. return -ENODEV;
  5385. }
  5386. /*
  5387. * compute the number of implemented PMD/PMC from the
  5388. * description tables
  5389. */
  5390. n = 0;
  5391. for (i=0; PMC_IS_LAST(i) == 0; i++) {
  5392. if (PMC_IS_IMPL(i) == 0) continue;
  5393. pmu_conf->impl_pmcs[i>>6] |= 1UL << (i&63);
  5394. n++;
  5395. }
  5396. pmu_conf->num_pmcs = n;
  5397. n = 0; n_counters = 0;
  5398. for (i=0; PMD_IS_LAST(i) == 0; i++) {
  5399. if (PMD_IS_IMPL(i) == 0) continue;
  5400. pmu_conf->impl_pmds[i>>6] |= 1UL << (i&63);
  5401. n++;
  5402. if (PMD_IS_COUNTING(i)) n_counters++;
  5403. }
  5404. pmu_conf->num_pmds = n;
  5405. pmu_conf->num_counters = n_counters;
  5406. /*
  5407. * sanity checks on the number of debug registers
  5408. */
  5409. if (pmu_conf->use_rr_dbregs) {
  5410. if (pmu_conf->num_ibrs > IA64_NUM_DBG_REGS) {
  5411. printk(KERN_INFO "perfmon: unsupported number of code debug registers (%u)\n", pmu_conf->num_ibrs);
  5412. pmu_conf = NULL;
  5413. return -1;
  5414. }
  5415. if (pmu_conf->num_dbrs > IA64_NUM_DBG_REGS) {
  5416. printk(KERN_INFO "perfmon: unsupported number of data debug registers (%u)\n", pmu_conf->num_ibrs);
  5417. pmu_conf = NULL;
  5418. return -1;
  5419. }
  5420. }
  5421. printk("perfmon: %s PMU detected, %u PMCs, %u PMDs, %u counters (%lu bits)\n",
  5422. pmu_conf->pmu_name,
  5423. pmu_conf->num_pmcs,
  5424. pmu_conf->num_pmds,
  5425. pmu_conf->num_counters,
  5426. ffz(pmu_conf->ovfl_val));
  5427. /* sanity check */
  5428. if (pmu_conf->num_pmds >= IA64_NUM_PMD_REGS || pmu_conf->num_pmcs >= IA64_NUM_PMC_REGS) {
  5429. printk(KERN_ERR "perfmon: not enough pmc/pmd, perfmon disabled\n");
  5430. pmu_conf = NULL;
  5431. return -1;
  5432. }
  5433. /*
  5434. * create /proc/perfmon (mostly for debugging purposes)
  5435. */
  5436. perfmon_dir = create_proc_entry("perfmon", S_IRUGO, NULL);
  5437. if (perfmon_dir == NULL) {
  5438. printk(KERN_ERR "perfmon: cannot create /proc entry, perfmon disabled\n");
  5439. pmu_conf = NULL;
  5440. return -1;
  5441. }
  5442. /*
  5443. * install customized file operations for /proc/perfmon entry
  5444. */
  5445. perfmon_dir->proc_fops = &pfm_proc_fops;
  5446. /*
  5447. * create /proc/sys/kernel/perfmon (for debugging purposes)
  5448. */
  5449. pfm_sysctl_header = register_sysctl_table(pfm_sysctl_root, 0);
  5450. /*
  5451. * initialize all our spinlocks
  5452. */
  5453. spin_lock_init(&pfm_sessions.pfs_lock);
  5454. spin_lock_init(&pfm_buffer_fmt_lock);
  5455. init_pfm_fs();
  5456. for(i=0; i < NR_CPUS; i++) pfm_stats[i].pfm_ovfl_intr_cycles_min = ~0UL;
  5457. return 0;
  5458. }
  5459. __initcall(pfm_init);
  5460. /*
  5461. * this function is called before pfm_init()
  5462. */
  5463. void
  5464. pfm_init_percpu (void)
  5465. {
  5466. /*
  5467. * make sure no measurement is active
  5468. * (may inherit programmed PMCs from EFI).
  5469. */
  5470. pfm_clear_psr_pp();
  5471. pfm_clear_psr_up();
  5472. /*
  5473. * we run with the PMU not frozen at all times
  5474. */
  5475. pfm_unfreeze_pmu();
  5476. if (smp_processor_id() == 0)
  5477. register_percpu_irq(IA64_PERFMON_VECTOR, &perfmon_irqaction);
  5478. ia64_setreg(_IA64_REG_CR_PMV, IA64_PERFMON_VECTOR);
  5479. ia64_srlz_d();
  5480. }
  5481. /*
  5482. * used for debug purposes only
  5483. */
  5484. void
  5485. dump_pmu_state(const char *from)
  5486. {
  5487. struct task_struct *task;
  5488. struct thread_struct *t;
  5489. struct pt_regs *regs;
  5490. pfm_context_t *ctx;
  5491. unsigned long psr, dcr, info, flags;
  5492. int i, this_cpu;
  5493. local_irq_save(flags);
  5494. this_cpu = smp_processor_id();
  5495. regs = ia64_task_regs(current);
  5496. info = PFM_CPUINFO_GET();
  5497. dcr = ia64_getreg(_IA64_REG_CR_DCR);
  5498. if (info == 0 && ia64_psr(regs)->pp == 0 && (dcr & IA64_DCR_PP) == 0) {
  5499. local_irq_restore(flags);
  5500. return;
  5501. }
  5502. printk("CPU%d from %s() current [%d] iip=0x%lx %s\n",
  5503. this_cpu,
  5504. from,
  5505. current->pid,
  5506. regs->cr_iip,
  5507. current->comm);
  5508. task = GET_PMU_OWNER();
  5509. ctx = GET_PMU_CTX();
  5510. printk("->CPU%d owner [%d] ctx=%p\n", this_cpu, task ? task->pid : -1, ctx);
  5511. psr = pfm_get_psr();
  5512. printk("->CPU%d pmc0=0x%lx psr.pp=%d psr.up=%d dcr.pp=%d syst_info=0x%lx user_psr.up=%d user_psr.pp=%d\n",
  5513. this_cpu,
  5514. ia64_get_pmc(0),
  5515. psr & IA64_PSR_PP ? 1 : 0,
  5516. psr & IA64_PSR_UP ? 1 : 0,
  5517. dcr & IA64_DCR_PP ? 1 : 0,
  5518. info,
  5519. ia64_psr(regs)->up,
  5520. ia64_psr(regs)->pp);
  5521. ia64_psr(regs)->up = 0;
  5522. ia64_psr(regs)->pp = 0;
  5523. t = &current->thread;
  5524. for (i=1; PMC_IS_LAST(i) == 0; i++) {
  5525. if (PMC_IS_IMPL(i) == 0) continue;
  5526. printk("->CPU%d pmc[%d]=0x%lx thread_pmc[%d]=0x%lx\n", this_cpu, i, ia64_get_pmc(i), i, t->pmcs[i]);
  5527. }
  5528. for (i=1; PMD_IS_LAST(i) == 0; i++) {
  5529. if (PMD_IS_IMPL(i) == 0) continue;
  5530. printk("->CPU%d pmd[%d]=0x%lx thread_pmd[%d]=0x%lx\n", this_cpu, i, ia64_get_pmd(i), i, t->pmds[i]);
  5531. }
  5532. if (ctx) {
  5533. printk("->CPU%d ctx_state=%d vaddr=%p addr=%p fd=%d ctx_task=[%d] saved_psr_up=0x%lx\n",
  5534. this_cpu,
  5535. ctx->ctx_state,
  5536. ctx->ctx_smpl_vaddr,
  5537. ctx->ctx_smpl_hdr,
  5538. ctx->ctx_msgq_head,
  5539. ctx->ctx_msgq_tail,
  5540. ctx->ctx_saved_psr_up);
  5541. }
  5542. local_irq_restore(flags);
  5543. }
  5544. /*
  5545. * called from process.c:copy_thread(). task is new child.
  5546. */
  5547. void
  5548. pfm_inherit(struct task_struct *task, struct pt_regs *regs)
  5549. {
  5550. struct thread_struct *thread;
  5551. DPRINT(("perfmon: pfm_inherit clearing state for [%d]\n", task->pid));
  5552. thread = &task->thread;
  5553. /*
  5554. * cut links inherited from parent (current)
  5555. */
  5556. thread->pfm_context = NULL;
  5557. PFM_SET_WORK_PENDING(task, 0);
  5558. /*
  5559. * the psr bits are already set properly in copy_threads()
  5560. */
  5561. }
  5562. #else /* !CONFIG_PERFMON */
  5563. asmlinkage long
  5564. sys_perfmonctl (int fd, int cmd, void *arg, int count)
  5565. {
  5566. return -ENOSYS;
  5567. }
  5568. #endif /* CONFIG_PERFMON */