/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c

https://gitlab.com/sunny256/linux · C · 850 lines · 621 code · 166 blank · 63 comment · 41 complexity · 179fbdd593e3197334cd4441d38815b8 MD5 · raw file 

    

  1. /*
    2. 

  2.  * Copyright 2014 Advanced Micro Devices, Inc.
    3. 

  3.  *
    4. 

  4.  * Permission is hereby granted, free of charge, to any person obtaining a
    5. 

  5.  * copy of this software and associated documentation files (the "Software"),
    6. 

  6.  * to deal in the Software without restriction, including without limitation
    7. 

  7.  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
    8. 

  8.  * and/or sell copies of the Software, and to permit persons to whom the
    9. 

  9.  * Software is furnished to do so, subject to the following conditions:
    10. 

  10.  *
    11. 

  11.  * The above copyright notice and this permission notice shall be included in
    12. 

  12.  * all copies or substantial portions of the Software.
    13. 

  13.  *
    14. 

  14.  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    15. 

  15.  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    16. 

  16.  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
    17. 

  17.  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
    18. 

  18.  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
    19. 

  19.  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
    20. 

  20.  * OTHER DEALINGS IN THE SOFTWARE.
    21. 

  21.  *
    22. 

  22.  */
    23. 

  23. 

  24. #include <linux/types.h>
    25. 

  25. #include <linux/kernel.h>
    26. 

  26. #include <linux/log2.h>
    27. 

  27. #include <linux/sched.h>
    28. 

  28. #include <linux/slab.h>
    29. 

  29. #include <linux/mutex.h>
    30. 

  30. #include <linux/device.h>
    31. 

  31. 

  32. #include "kfd_pm4_headers.h"
    33. 

  33. #include "kfd_pm4_headers_diq.h"
    34. 

  34. #include "kfd_kernel_queue.h"
    35. 

  35. #include "kfd_priv.h"
    36. 

  36. #include "kfd_pm4_opcodes.h"
    37. 

  37. #include "cik_regs.h"
    38. 

  38. #include "kfd_dbgmgr.h"
    39. 

  39. #include "kfd_dbgdev.h"
    40. 

  40. #include "kfd_device_queue_manager.h"
    41. 

  41. #include "../../radeon/cik_reg.h"
    42. 

  42. 

  43. static void dbgdev_address_watch_disable_nodiq(struct kfd_dev *dev)
    44. 

  44. {
    45. 

  45. 	dev->kfd2kgd->address_watch_disable(dev->kgd);
    46. 

  46. }
    47. 

  47. 

  48. static int dbgdev_diq_submit_ib(struct kfd_dbgdev *dbgdev,
    49. 

  49. 				unsigned int pasid, uint64_t vmid0_address,
    50. 

  50. 				uint32_t *packet_buff, size_t size_in_bytes)
    51. 

  51. {
    52. 

  52. 	struct pm4__release_mem *rm_packet;
    53. 

  53. 	struct pm4__indirect_buffer_pasid *ib_packet;
    54. 

  54. 	struct kfd_mem_obj *mem_obj;
    55. 

  55. 	size_t pq_packets_size_in_bytes;
    56. 

  56. 	union ULARGE_INTEGER *largep;
    57. 

  57. 	union ULARGE_INTEGER addr;
    58. 

  58. 	struct kernel_queue *kq;
    59. 

  59. 	uint64_t *rm_state;
    60. 

  60. 	unsigned int *ib_packet_buff;
    61. 

  61. 	int status;
    62. 

  62. 

  63. 	if (WARN_ON(!size_in_bytes))
    64. 

  64. 		return -EINVAL;
    65. 

  65. 

  66. 	kq = dbgdev->kq;
    67. 

  67. 

  68. 	pq_packets_size_in_bytes = sizeof(struct pm4__release_mem) +
    69. 

  69. 				sizeof(struct pm4__indirect_buffer_pasid);
    70. 

  70. 

  71. 	/*
    72. 

  72. 	 * We acquire a buffer from DIQ
    73. 

  73. 	 * The receive packet buff will be sitting on the Indirect Buffer
    74. 

  74. 	 * and in the PQ we put the IB packet + sync packet(s).
    75. 

  75. 	 */
    76. 

  76. 	status = kq->ops.acquire_packet_buffer(kq,
    77. 

  77. 				pq_packets_size_in_bytes / sizeof(uint32_t),
    78. 

  78. 				&ib_packet_buff);
    79. 

  79. 	if (status) {
    80. 

  80. 		pr_err("acquire_packet_buffer failed\n");
    81. 

  81. 		return status;
    82. 

  82. 	}
    83. 

  83. 

  84. 	memset(ib_packet_buff, 0, pq_packets_size_in_bytes);
    85. 

  85. 

  86. 	ib_packet = (struct pm4__indirect_buffer_pasid *) (ib_packet_buff);
    87. 

  87. 

  88. 	ib_packet->header.count = 3;
    89. 

  89. 	ib_packet->header.opcode = IT_INDIRECT_BUFFER_PASID;
    90. 

  90. 	ib_packet->header.type = PM4_TYPE_3;
    91. 

  91. 

  92. 	largep = (union ULARGE_INTEGER *) &vmid0_address;
    93. 

  93. 

  94. 	ib_packet->bitfields2.ib_base_lo = largep->u.low_part >> 2;
    95. 

  95. 	ib_packet->bitfields3.ib_base_hi = largep->u.high_part;
    96. 

  96. 

  97. 	ib_packet->control = (1 << 23) | (1 << 31) |
    98. 

  98. 			((size_in_bytes / sizeof(uint32_t)) & 0xfffff);
    99. 

  99. 

  100. 	ib_packet->bitfields5.pasid = pasid;
    101. 

  101. 

  102. 	/*
    103. 

  103. 	 * for now we use release mem for GPU-CPU synchronization
    104. 

  104. 	 * Consider WaitRegMem + WriteData as a better alternative
    105. 

  105. 	 * we get a GART allocations ( gpu/cpu mapping),
    106. 

  106. 	 * for the sync variable, and wait until:
    107. 

  107. 	 * (a) Sync with HW
    108. 

  108. 	 * (b) Sync var is written by CP to mem.
    109. 

  109. 	 */
    110. 

  110. 	rm_packet = (struct pm4__release_mem *) (ib_packet_buff +
    111. 

  111. 			(sizeof(struct pm4__indirect_buffer_pasid) /
    112. 

  112. 					sizeof(unsigned int)));
    113. 

  113. 

  114. 	status = kfd_gtt_sa_allocate(dbgdev->dev, sizeof(uint64_t),
    115. 

  115. 					&mem_obj);
    116. 

  116. 

  117. 	if (status) {
    118. 

  118. 		pr_err("Failed to allocate GART memory\n");
    119. 

  119. 		kq->ops.rollback_packet(kq);
    120. 

  120. 		return status;
    121. 

  121. 	}
    122. 

  122. 

  123. 	rm_state = (uint64_t *) mem_obj->cpu_ptr;
    124. 

  124. 

  125. 	*rm_state = QUEUESTATE__ACTIVE_COMPLETION_PENDING;
    126. 

  126. 

  127. 	rm_packet->header.opcode = IT_RELEASE_MEM;
    128. 

  128. 	rm_packet->header.type = PM4_TYPE_3;
    129. 

  129. 	rm_packet->header.count = sizeof(struct pm4__release_mem) /
    130. 

  130. 					sizeof(unsigned int) - 2;
    131. 

  131. 

  132. 	rm_packet->bitfields2.event_type = CACHE_FLUSH_AND_INV_TS_EVENT;
    133. 

  133. 	rm_packet->bitfields2.event_index =
    134. 

  134. 				event_index___release_mem__end_of_pipe;
    135. 

  135. 

  136. 	rm_packet->bitfields2.cache_policy = cache_policy___release_mem__lru;
    137. 

  137. 	rm_packet->bitfields2.atc = 0;
    138. 

  138. 	rm_packet->bitfields2.tc_wb_action_ena = 1;
    139. 

  139. 

  140. 	addr.quad_part = mem_obj->gpu_addr;
    141. 

  141. 

  142. 	rm_packet->bitfields4.address_lo_32b = addr.u.low_part >> 2;
    143. 

  143. 	rm_packet->address_hi = addr.u.high_part;
    144. 

  144. 

  145. 	rm_packet->bitfields3.data_sel =
    146. 

  146. 				data_sel___release_mem__send_64_bit_data;
    147. 

  147. 

  148. 	rm_packet->bitfields3.int_sel =
    149. 

  149. 			int_sel___release_mem__send_data_after_write_confirm;
    150. 

  150. 

  151. 	rm_packet->bitfields3.dst_sel =
    152. 

  152. 			dst_sel___release_mem__memory_controller;
    153. 

  153. 

  154. 	rm_packet->data_lo = QUEUESTATE__ACTIVE;
    155. 

  155. 

  156. 	kq->ops.submit_packet(kq);
    157. 

  157. 

  158. 	/* Wait till CP writes sync code: */
    159. 

  159. 	status = amdkfd_fence_wait_timeout(
    160. 

  160. 			(unsigned int *) rm_state,
    161. 

  161. 			QUEUESTATE__ACTIVE, 1500);
    162. 

  162. 

  163. 	kfd_gtt_sa_free(dbgdev->dev, mem_obj);
    164. 

  164. 

  165. 	return status;
    166. 

  166. }
    167. 

  167. 

  168. static int dbgdev_register_nodiq(struct kfd_dbgdev *dbgdev)
    169. 

  169. {
    170. 

  170. 	/*
    171. 

  171. 	 * no action is needed in this case,
    172. 

  172. 	 * just make sure diq will not be used
    173. 

  173. 	 */
    174. 

  174. 

  175. 	dbgdev->kq = NULL;
    176. 

  176. 

  177. 	return 0;
    178. 

  178. }
    179. 

  179. 

  180. static int dbgdev_register_diq(struct kfd_dbgdev *dbgdev)
    181. 

  181. {
    182. 

  182. 	struct queue_properties properties;
    183. 

  183. 	unsigned int qid;
    184. 

  184. 	struct kernel_queue *kq = NULL;
    185. 

  185. 	int status;
    186. 

  186. 

  187. 	properties.type = KFD_QUEUE_TYPE_DIQ;
    188. 

  188. 

  189. 	status = pqm_create_queue(dbgdev->pqm, dbgdev->dev, NULL,
    190. 

  190. 				&properties, &qid);
    191. 

  191. 

  192. 	if (status) {
    193. 

  193. 		pr_err("Failed to create DIQ\n");
    194. 

  194. 		return status;
    195. 

  195. 	}
    196. 

  196. 

  197. 	pr_debug("DIQ Created with queue id: %d\n", qid);
    198. 

  198. 

  199. 	kq = pqm_get_kernel_queue(dbgdev->pqm, qid);
    200. 

  200. 

  201. 	if (!kq) {
    202. 

  202. 		pr_err("Error getting DIQ\n");
    203. 

  203. 		pqm_destroy_queue(dbgdev->pqm, qid);
    204. 

  204. 		return -EFAULT;
    205. 

  205. 	}
    206. 

  206. 

  207. 	dbgdev->kq = kq;
    208. 

  208. 

  209. 	return status;
    210. 

  210. }
    211. 

  211. 

  212. static int dbgdev_unregister_nodiq(struct kfd_dbgdev *dbgdev)
    213. 

  213. {
    214. 

  214. 	/* disable watch address */
    215. 

  215. 	dbgdev_address_watch_disable_nodiq(dbgdev->dev);
    216. 

  216. 	return 0;
    217. 

  217. }
    218. 

  218. 

  219. static int dbgdev_unregister_diq(struct kfd_dbgdev *dbgdev)
    220. 

  220. {
    221. 

  221. 	/* todo - disable address watch */
    222. 

  222. 	int status;
    223. 

  223. 

  224. 	status = pqm_destroy_queue(dbgdev->pqm,
    225. 

  225. 			dbgdev->kq->queue->properties.queue_id);
    226. 

  226. 	dbgdev->kq = NULL;
    227. 

  227. 

  228. 	return status;
    229. 

  229. }
    230. 

  230. 

  231. static void dbgdev_address_watch_set_registers(
    232. 

  232. 			const struct dbg_address_watch_info *adw_info,
    233. 

  233. 			union TCP_WATCH_ADDR_H_BITS *addrHi,
    234. 

  234. 			union TCP_WATCH_ADDR_L_BITS *addrLo,
    235. 

  235. 			union TCP_WATCH_CNTL_BITS *cntl,
    236. 

  236. 			unsigned int index, unsigned int vmid)
    237. 

  237. {
    238. 

  238. 	union ULARGE_INTEGER addr;
    239. 

  239. 

  240. 	addr.quad_part = 0;
    241. 

  241. 	addrHi->u32All = 0;
    242. 

  242. 	addrLo->u32All = 0;
    243. 

  243. 	cntl->u32All = 0;
    244. 

  244. 

  245. 	if (adw_info->watch_mask)
    246. 

  246. 		cntl->bitfields.mask =
    247. 

  247. 			(uint32_t) (adw_info->watch_mask[index] &
    248. 

  248. 					ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK);
    249. 

  249. 	else
    250. 

  250. 		cntl->bitfields.mask = ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK;
    251. 

  251. 

  252. 	addr.quad_part = (unsigned long long) adw_info->watch_address[index];
    253. 

  253. 

  254. 	addrHi->bitfields.addr = addr.u.high_part &
    255. 

  255. 					ADDRESS_WATCH_REG_ADDHIGH_MASK;
    256. 

  256. 	addrLo->bitfields.addr =
    257. 

  257. 			(addr.u.low_part >> ADDRESS_WATCH_REG_ADDLOW_SHIFT);
    258. 

  258. 

  259. 	cntl->bitfields.mode = adw_info->watch_mode[index];
    260. 

  260. 	cntl->bitfields.vmid = (uint32_t) vmid;
    261. 

  261. 	/* for now assume it is an ATC address */
    262. 

  262. 	cntl->u32All |= ADDRESS_WATCH_REG_CNTL_ATC_BIT;
    263. 

  263. 

  264. 	pr_debug("\t\t%20s %08x\n", "set reg mask :", cntl->bitfields.mask);
    265. 

  265. 	pr_debug("\t\t%20s %08x\n", "set reg add high :",
    266. 

  266. 			addrHi->bitfields.addr);
    267. 

  267. 	pr_debug("\t\t%20s %08x\n", "set reg add low :",
    268. 

  268. 			addrLo->bitfields.addr);
    269. 

  269. }
    270. 

  270. 

  271. static int dbgdev_address_watch_nodiq(struct kfd_dbgdev *dbgdev,
    272. 

  272. 				      struct dbg_address_watch_info *adw_info)
    273. 

  273. {
    274. 

  274. 	union TCP_WATCH_ADDR_H_BITS addrHi;
    275. 

  275. 	union TCP_WATCH_ADDR_L_BITS addrLo;
    276. 

  276. 	union TCP_WATCH_CNTL_BITS cntl;
    277. 

  277. 	struct kfd_process_device *pdd;
    278. 

  278. 	unsigned int i;
    279. 

  279. 

  280. 	/* taking the vmid for that process on the safe way using pdd */
    281. 

  281. 	pdd = kfd_get_process_device_data(dbgdev->dev,
    282. 

  282. 					adw_info->process);
    283. 

  283. 	if (!pdd) {
    284. 

  284. 		pr_err("Failed to get pdd for wave control no DIQ\n");
    285. 

  285. 		return -EFAULT;
    286. 

  286. 	}
    287. 

  287. 

  288. 	addrHi.u32All = 0;
    289. 

  289. 	addrLo.u32All = 0;
    290. 

  290. 	cntl.u32All = 0;
    291. 

  291. 

  292. 	if ((adw_info->num_watch_points > MAX_WATCH_ADDRESSES) ||
    293. 

  293. 			(adw_info->num_watch_points == 0)) {
    294. 

  294. 		pr_err("num_watch_points is invalid\n");
    295. 

  295. 		return -EINVAL;
    296. 

  296. 	}
    297. 

  297. 

  298. 	if (!adw_info->watch_mode || !adw_info->watch_address) {
    299. 

  299. 		pr_err("adw_info fields are not valid\n");
    300. 

  300. 		return -EINVAL;
    301. 

  301. 	}
    302. 

  302. 

  303. 	for (i = 0; i < adw_info->num_watch_points; i++) {
    304. 

  304. 		dbgdev_address_watch_set_registers(adw_info, &addrHi, &addrLo,
    305. 

  305. 						&cntl, i, pdd->qpd.vmid);
    306. 

  306. 

  307. 		pr_debug("\t\t%30s\n", "* * * * * * * * * * * * * * * * * *");
    308. 

  308. 		pr_debug("\t\t%20s %08x\n", "register index :", i);
    309. 

  309. 		pr_debug("\t\t%20s %08x\n", "vmid is :", pdd->qpd.vmid);
    310. 

  310. 		pr_debug("\t\t%20s %08x\n", "Address Low is :",
    311. 

  311. 				addrLo.bitfields.addr);
    312. 

  312. 		pr_debug("\t\t%20s %08x\n", "Address high is :",
    313. 

  313. 				addrHi.bitfields.addr);
    314. 

  314. 		pr_debug("\t\t%20s %08x\n", "Address high is :",
    315. 

  315. 				addrHi.bitfields.addr);
    316. 

  316. 		pr_debug("\t\t%20s %08x\n", "Control Mask is :",
    317. 

  317. 				cntl.bitfields.mask);
    318. 

  318. 		pr_debug("\t\t%20s %08x\n", "Control Mode is :",
    319. 

  319. 				cntl.bitfields.mode);
    320. 

  320. 		pr_debug("\t\t%20s %08x\n", "Control Vmid is :",
    321. 

  321. 				cntl.bitfields.vmid);
    322. 

  322. 		pr_debug("\t\t%20s %08x\n", "Control atc  is :",
    323. 

  323. 				cntl.bitfields.atc);
    324. 

  324. 		pr_debug("\t\t%30s\n", "* * * * * * * * * * * * * * * * * *");
    325. 

  325. 

  326. 		pdd->dev->kfd2kgd->address_watch_execute(
    327. 

  327. 						dbgdev->dev->kgd,
    328. 

  328. 						i,
    329. 

  329. 						cntl.u32All,
    330. 

  330. 						addrHi.u32All,
    331. 

  331. 						addrLo.u32All);
    332. 

  332. 	}
    333. 

  333. 

  334. 	return 0;
    335. 

  335. }
    336. 

  336. 

  337. static int dbgdev_address_watch_diq(struct kfd_dbgdev *dbgdev,
    338. 

  338. 				    struct dbg_address_watch_info *adw_info)
    339. 

  339. {
    340. 

  340. 	struct pm4__set_config_reg *packets_vec;
    341. 

  341. 	union TCP_WATCH_ADDR_H_BITS addrHi;
    342. 

  342. 	union TCP_WATCH_ADDR_L_BITS addrLo;
    343. 

  343. 	union TCP_WATCH_CNTL_BITS cntl;
    344. 

  344. 	struct kfd_mem_obj *mem_obj;
    345. 

  345. 	unsigned int aw_reg_add_dword;
    346. 

  346. 	uint32_t *packet_buff_uint;
    347. 

  347. 	unsigned int i;
    348. 

  348. 	int status;
    349. 

  349. 	size_t ib_size = sizeof(struct pm4__set_config_reg) * 4;
    350. 

  350. 	/* we do not control the vmid in DIQ mode, just a place holder */
    351. 

  351. 	unsigned int vmid = 0;
    352. 

  352. 

  353. 	addrHi.u32All = 0;
    354. 

  354. 	addrLo.u32All = 0;
    355. 

  355. 	cntl.u32All = 0;
    356. 

  356. 

  357. 	if ((adw_info->num_watch_points > MAX_WATCH_ADDRESSES) ||
    358. 

  358. 			(adw_info->num_watch_points == 0)) {
    359. 

  359. 		pr_err("num_watch_points is invalid\n");
    360. 

  360. 		return -EINVAL;
    361. 

  361. 	}
    362. 

  362. 

  363. 	if (!adw_info->watch_mode || !adw_info->watch_address) {
    364. 

  364. 		pr_err("adw_info fields are not valid\n");
    365. 

  365. 		return -EINVAL;
    366. 

  366. 	}
    367. 

  367. 

  368. 	status = kfd_gtt_sa_allocate(dbgdev->dev, ib_size, &mem_obj);
    369. 

  369. 

  370. 	if (status) {
    371. 

  371. 		pr_err("Failed to allocate GART memory\n");
    372. 

  372. 		return status;
    373. 

  373. 	}
    374. 

  374. 

  375. 	packet_buff_uint = mem_obj->cpu_ptr;
    376. 

  376. 

  377. 	memset(packet_buff_uint, 0, ib_size);
    378. 

  378. 

  379. 	packets_vec = (struct pm4__set_config_reg *) (packet_buff_uint);
    380. 

  380. 

  381. 	packets_vec[0].header.count = 1;
    382. 

  382. 	packets_vec[0].header.opcode = IT_SET_CONFIG_REG;
    383. 

  383. 	packets_vec[0].header.type = PM4_TYPE_3;
    384. 

  384. 	packets_vec[0].bitfields2.vmid_shift = ADDRESS_WATCH_CNTL_OFFSET;
    385. 

  385. 	packets_vec[0].bitfields2.insert_vmid = 1;
    386. 

  386. 	packets_vec[1].ordinal1 = packets_vec[0].ordinal1;
    387. 

  387. 	packets_vec[1].bitfields2.insert_vmid = 0;
    388. 

  388. 	packets_vec[2].ordinal1 = packets_vec[0].ordinal1;
    389. 

  389. 	packets_vec[2].bitfields2.insert_vmid = 0;
    390. 

  390. 	packets_vec[3].ordinal1 = packets_vec[0].ordinal1;
    391. 

  391. 	packets_vec[3].bitfields2.vmid_shift = ADDRESS_WATCH_CNTL_OFFSET;
    392. 

  392. 	packets_vec[3].bitfields2.insert_vmid = 1;
    393. 

  393. 

  394. 	for (i = 0; i < adw_info->num_watch_points; i++) {
    395. 

  395. 		dbgdev_address_watch_set_registers(adw_info,
    396. 

  396. 						&addrHi,
    397. 

  397. 						&addrLo,
    398. 

  398. 						&cntl,
    399. 

  399. 						i,
    400. 

  400. 						vmid);
    401. 

  401. 

  402. 		pr_debug("\t\t%30s\n", "* * * * * * * * * * * * * * * * * *");
    403. 

  403. 		pr_debug("\t\t%20s %08x\n", "register index :", i);
    404. 

  404. 		pr_debug("\t\t%20s %08x\n", "vmid is :", vmid);
    405. 

  405. 		pr_debug("\t\t%20s %p\n", "Add ptr is :",
    406. 

  406. 				adw_info->watch_address);
    407. 

  407. 		pr_debug("\t\t%20s %08llx\n", "Add     is :",
    408. 

  408. 				adw_info->watch_address[i]);
    409. 

  409. 		pr_debug("\t\t%20s %08x\n", "Address Low is :",
    410. 

  410. 				addrLo.bitfields.addr);
    411. 

  411. 		pr_debug("\t\t%20s %08x\n", "Address high is :",
    412. 

  412. 				addrHi.bitfields.addr);
    413. 

  413. 		pr_debug("\t\t%20s %08x\n", "Control Mask is :",
    414. 

  414. 				cntl.bitfields.mask);
    415. 

  415. 		pr_debug("\t\t%20s %08x\n", "Control Mode is :",
    416. 

  416. 				cntl.bitfields.mode);
    417. 

  417. 		pr_debug("\t\t%20s %08x\n", "Control Vmid is :",
    418. 

  418. 				cntl.bitfields.vmid);
    419. 

  419. 		pr_debug("\t\t%20s %08x\n", "Control atc  is :",
    420. 

  420. 				cntl.bitfields.atc);
    421. 

  421. 		pr_debug("\t\t%30s\n", "* * * * * * * * * * * * * * * * * *");
    422. 

  422. 

  423. 		aw_reg_add_dword =
    424. 

  424. 				dbgdev->dev->kfd2kgd->address_watch_get_offset(
    425. 

  425. 					dbgdev->dev->kgd,
    426. 

  426. 					i,
    427. 

  427. 					ADDRESS_WATCH_REG_CNTL);
    428. 

  428. 

  429. 		packets_vec[0].bitfields2.reg_offset =
    430. 

  430. 					aw_reg_add_dword - AMD_CONFIG_REG_BASE;
    431. 

  431. 

  432. 		packets_vec[0].reg_data[0] = cntl.u32All;
    433. 

  433. 

  434. 		aw_reg_add_dword =
    435. 

  435. 				dbgdev->dev->kfd2kgd->address_watch_get_offset(
    436. 

  436. 					dbgdev->dev->kgd,
    437. 

  437. 					i,
    438. 

  438. 					ADDRESS_WATCH_REG_ADDR_HI);
    439. 

  439. 

  440. 		packets_vec[1].bitfields2.reg_offset =
    441. 

  441. 					aw_reg_add_dword - AMD_CONFIG_REG_BASE;
    442. 

  442. 		packets_vec[1].reg_data[0] = addrHi.u32All;
    443. 

  443. 

  444. 		aw_reg_add_dword =
    445. 

  445. 				dbgdev->dev->kfd2kgd->address_watch_get_offset(
    446. 

  446. 					dbgdev->dev->kgd,
    447. 

  447. 					i,
    448. 

  448. 					ADDRESS_WATCH_REG_ADDR_LO);
    449. 

  449. 

  450. 		packets_vec[2].bitfields2.reg_offset =
    451. 

  451. 				aw_reg_add_dword - AMD_CONFIG_REG_BASE;
    452. 

  452. 		packets_vec[2].reg_data[0] = addrLo.u32All;
    453. 

  453. 

  454. 		/* enable watch flag if address is not zero*/
    455. 

  455. 		if (adw_info->watch_address[i] > 0)
    456. 

  456. 			cntl.bitfields.valid = 1;
    457. 

  457. 		else
    458. 

  458. 			cntl.bitfields.valid = 0;
    459. 

  459. 

  460. 		aw_reg_add_dword =
    461. 

  461. 				dbgdev->dev->kfd2kgd->address_watch_get_offset(
    462. 

  462. 					dbgdev->dev->kgd,
    463. 

  463. 					i,
    464. 

  464. 					ADDRESS_WATCH_REG_CNTL);
    465. 

  465. 

  466. 		packets_vec[3].bitfields2.reg_offset =
    467. 

  467. 					aw_reg_add_dword - AMD_CONFIG_REG_BASE;
    468. 

  468. 		packets_vec[3].reg_data[0] = cntl.u32All;
    469. 

  469. 

  470. 		status = dbgdev_diq_submit_ib(
    471. 

  471. 					dbgdev,
    472. 

  472. 					adw_info->process->pasid,
    473. 

  473. 					mem_obj->gpu_addr,
    474. 

  474. 					packet_buff_uint,
    475. 

  475. 					ib_size);
    476. 

  476. 

  477. 		if (status) {
    478. 

  478. 			pr_err("Failed to submit IB to DIQ\n");
    479. 

  479. 			break;
    480. 

  480. 		}
    481. 

  481. 	}
    482. 

  482. 

  483. 	kfd_gtt_sa_free(dbgdev->dev, mem_obj);
    484. 

  484. 	return status;
    485. 

  485. }
    486. 

  486. 

  487. static int dbgdev_wave_control_set_registers(
    488. 

  488. 				struct dbg_wave_control_info *wac_info,
    489. 

  489. 				union SQ_CMD_BITS *in_reg_sq_cmd,
    490. 

  490. 				union GRBM_GFX_INDEX_BITS *in_reg_gfx_index)
    491. 

  491. {
    492. 

  492. 	int status = 0;
    493. 

  493. 	union SQ_CMD_BITS reg_sq_cmd;
    494. 

  494. 	union GRBM_GFX_INDEX_BITS reg_gfx_index;
    495. 

  495. 	struct HsaDbgWaveMsgAMDGen2 *pMsg;
    496. 

  496. 

  497. 	reg_sq_cmd.u32All = 0;
    498. 

  498. 	reg_gfx_index.u32All = 0;
    499. 

  499. 	pMsg = &wac_info->dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2;
    500. 

  500. 

  501. 	switch (wac_info->mode) {
    502. 

  502. 	/* Send command to single wave */
    503. 

  503. 	case HSA_DBG_WAVEMODE_SINGLE:
    504. 

  504. 		/*
    505. 

  505. 		 * Limit access to the process waves only,
    506. 

  506. 		 * by setting vmid check
    507. 

  507. 		 */
    508. 

  508. 		reg_sq_cmd.bits.check_vmid = 1;
    509. 

  509. 		reg_sq_cmd.bits.simd_id = pMsg->ui32.SIMD;
    510. 

  510. 		reg_sq_cmd.bits.wave_id = pMsg->ui32.WaveId;
    511. 

  511. 		reg_sq_cmd.bits.mode = SQ_IND_CMD_MODE_SINGLE;
    512. 

  512. 

  513. 		reg_gfx_index.bits.sh_index = pMsg->ui32.ShaderArray;
    514. 

  514. 		reg_gfx_index.bits.se_index = pMsg->ui32.ShaderEngine;
    515. 

  515. 		reg_gfx_index.bits.instance_index = pMsg->ui32.HSACU;
    516. 

  516. 

  517. 		break;
    518. 

  518. 

  519. 	/* Send command to all waves with matching VMID */
    520. 

  520. 	case HSA_DBG_WAVEMODE_BROADCAST_PROCESS:
    521. 

  521. 

  522. 		reg_gfx_index.bits.sh_broadcast_writes = 1;
    523. 

  523. 		reg_gfx_index.bits.se_broadcast_writes = 1;
    524. 

  524. 		reg_gfx_index.bits.instance_broadcast_writes = 1;
    525. 

  525. 

  526. 		reg_sq_cmd.bits.mode = SQ_IND_CMD_MODE_BROADCAST;
    527. 

  527. 

  528. 		break;
    529. 

  529. 

  530. 	/* Send command to all CU waves with matching VMID */
    531. 

  531. 	case HSA_DBG_WAVEMODE_BROADCAST_PROCESS_CU:
    532. 

  532. 

  533. 		reg_sq_cmd.bits.check_vmid = 1;
    534. 

  534. 		reg_sq_cmd.bits.mode = SQ_IND_CMD_MODE_BROADCAST;
    535. 

  535. 

  536. 		reg_gfx_index.bits.sh_index = pMsg->ui32.ShaderArray;
    537. 

  537. 		reg_gfx_index.bits.se_index = pMsg->ui32.ShaderEngine;
    538. 

  538. 		reg_gfx_index.bits.instance_index = pMsg->ui32.HSACU;
    539. 

  539. 

  540. 		break;
    541. 

  541. 

  542. 	default:
    543. 

  543. 		return -EINVAL;
    544. 

  544. 	}
    545. 

  545. 

  546. 	switch (wac_info->operand) {
    547. 

  547. 	case HSA_DBG_WAVEOP_HALT:
    548. 

  548. 		reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_HALT;
    549. 

  549. 		break;
    550. 

  550. 

  551. 	case HSA_DBG_WAVEOP_RESUME:
    552. 

  552. 		reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_RESUME;
    553. 

  553. 		break;
    554. 

  554. 

  555. 	case HSA_DBG_WAVEOP_KILL:
    556. 

  556. 		reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_KILL;
    557. 

  557. 		break;
    558. 

  558. 

  559. 	case HSA_DBG_WAVEOP_DEBUG:
    560. 

  560. 		reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_DEBUG;
    561. 

  561. 		break;
    562. 

  562. 

  563. 	case HSA_DBG_WAVEOP_TRAP:
    564. 

  564. 		if (wac_info->trapId < MAX_TRAPID) {
    565. 

  565. 			reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_TRAP;
    566. 

  566. 			reg_sq_cmd.bits.trap_id = wac_info->trapId;
    567. 

  567. 		} else {
    568. 

  568. 			status = -EINVAL;
    569. 

  569. 		}
    570. 

  570. 		break;
    571. 

  571. 

  572. 	default:
    573. 

  573. 		status = -EINVAL;
    574. 

  574. 		break;
    575. 

  575. 	}
    576. 

  576. 

  577. 	if (status == 0) {
    578. 

  578. 		*in_reg_sq_cmd = reg_sq_cmd;
    579. 

  579. 		*in_reg_gfx_index = reg_gfx_index;
    580. 

  580. 	}
    581. 

  581. 

  582. 	return status;
    583. 

  583. }
    584. 

  584. 

  585. static int dbgdev_wave_control_diq(struct kfd_dbgdev *dbgdev,
    586. 

  586. 					struct dbg_wave_control_info *wac_info)
    587. 

  587. {
    588. 

  588. 

  589. 	int status;
    590. 

  590. 	union SQ_CMD_BITS reg_sq_cmd;
    591. 

  591. 	union GRBM_GFX_INDEX_BITS reg_gfx_index;
    592. 

  592. 	struct kfd_mem_obj *mem_obj;
    593. 

  593. 	uint32_t *packet_buff_uint;
    594. 

  594. 	struct pm4__set_config_reg *packets_vec;
    595. 

  595. 	size_t ib_size = sizeof(struct pm4__set_config_reg) * 3;
    596. 

  596. 

  597. 	reg_sq_cmd.u32All = 0;
    598. 

  598. 

  599. 	status = dbgdev_wave_control_set_registers(wac_info, &reg_sq_cmd,
    600. 

  600. 							&reg_gfx_index);
    601. 

  601. 	if (status) {
    602. 

  602. 		pr_err("Failed to set wave control registers\n");
    603. 

  603. 		return status;
    604. 

  604. 	}
    605. 

  605. 

  606. 	/* we do not control the VMID in DIQ, so reset it to a known value */
    607. 

  607. 	reg_sq_cmd.bits.vm_id = 0;
    608. 

  608. 

  609. 	pr_debug("\t\t %30s\n", "* * * * * * * * * * * * * * * * * *");
    610. 

  610. 

  611. 	pr_debug("\t\t mode      is: %u\n", wac_info->mode);
    612. 

  612. 	pr_debug("\t\t operand   is: %u\n", wac_info->operand);
    613. 

  613. 	pr_debug("\t\t trap id   is: %u\n", wac_info->trapId);
    614. 

  614. 	pr_debug("\t\t msg value is: %u\n",
    615. 

  615. 			wac_info->dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value);
    616. 

  616. 	pr_debug("\t\t vmid      is: N/A\n");
    617. 

  617. 

  618. 	pr_debug("\t\t chk_vmid  is : %u\n", reg_sq_cmd.bitfields.check_vmid);
    619. 

  619. 	pr_debug("\t\t command   is : %u\n", reg_sq_cmd.bitfields.cmd);
    620. 

  620. 	pr_debug("\t\t queue id  is : %u\n", reg_sq_cmd.bitfields.queue_id);
    621. 

  621. 	pr_debug("\t\t simd id   is : %u\n", reg_sq_cmd.bitfields.simd_id);
    622. 

  622. 	pr_debug("\t\t mode      is : %u\n", reg_sq_cmd.bitfields.mode);
    623. 

  623. 	pr_debug("\t\t vm_id     is : %u\n", reg_sq_cmd.bitfields.vm_id);
    624. 

  624. 	pr_debug("\t\t wave_id   is : %u\n", reg_sq_cmd.bitfields.wave_id);
    625. 

  625. 

  626. 	pr_debug("\t\t ibw       is : %u\n",
    627. 

  627. 			reg_gfx_index.bitfields.instance_broadcast_writes);
    628. 

  628. 	pr_debug("\t\t ii        is : %u\n",
    629. 

  629. 			reg_gfx_index.bitfields.instance_index);
    630. 

  630. 	pr_debug("\t\t sebw      is : %u\n",
    631. 

  631. 			reg_gfx_index.bitfields.se_broadcast_writes);
    632. 

  632. 	pr_debug("\t\t se_ind    is : %u\n", reg_gfx_index.bitfields.se_index);
    633. 

  633. 	pr_debug("\t\t sh_ind    is : %u\n", reg_gfx_index.bitfields.sh_index);
    634. 

  634. 	pr_debug("\t\t sbw       is : %u\n",
    635. 

  635. 			reg_gfx_index.bitfields.sh_broadcast_writes);
    636. 

  636. 

  637. 	pr_debug("\t\t %30s\n", "* * * * * * * * * * * * * * * * * *");
    638. 

  638. 

  639. 	status = kfd_gtt_sa_allocate(dbgdev->dev, ib_size, &mem_obj);
    640. 

  640. 

  641. 	if (status != 0) {
    642. 

  642. 		pr_err("Failed to allocate GART memory\n");
    643. 

  643. 		return status;
    644. 

  644. 	}
    645. 

  645. 

  646. 	packet_buff_uint = mem_obj->cpu_ptr;
    647. 

  647. 

  648. 	memset(packet_buff_uint, 0, ib_size);
    649. 

  649. 

  650. 	packets_vec =  (struct pm4__set_config_reg *) packet_buff_uint;
    651. 

  651. 	packets_vec[0].header.count = 1;
    652. 

  652. 	packets_vec[0].header.opcode = IT_SET_UCONFIG_REG;
    653. 

  653. 	packets_vec[0].header.type = PM4_TYPE_3;
    654. 

  654. 	packets_vec[0].bitfields2.reg_offset =
    655. 

  655. 			GRBM_GFX_INDEX / (sizeof(uint32_t)) -
    656. 

  656. 				USERCONFIG_REG_BASE;
    657. 

  657. 

  658. 	packets_vec[0].bitfields2.insert_vmid = 0;
    659. 

  659. 	packets_vec[0].reg_data[0] = reg_gfx_index.u32All;
    660. 

  660. 

  661. 	packets_vec[1].header.count = 1;
    662. 

  662. 	packets_vec[1].header.opcode = IT_SET_CONFIG_REG;
    663. 

  663. 	packets_vec[1].header.type = PM4_TYPE_3;
    664. 

  664. 	packets_vec[1].bitfields2.reg_offset = SQ_CMD / (sizeof(uint32_t)) -
    665. 

  665. 						AMD_CONFIG_REG_BASE;
    666. 

  666. 

  667. 	packets_vec[1].bitfields2.vmid_shift = SQ_CMD_VMID_OFFSET;
    668. 

  668. 	packets_vec[1].bitfields2.insert_vmid = 1;
    669. 

  669. 	packets_vec[1].reg_data[0] = reg_sq_cmd.u32All;
    670. 

  670. 

  671. 	/* Restore the GRBM_GFX_INDEX register */
    672. 

  672. 

  673. 	reg_gfx_index.u32All = 0;
    674. 

  674. 	reg_gfx_index.bits.sh_broadcast_writes = 1;
    675. 

  675. 	reg_gfx_index.bits.instance_broadcast_writes = 1;
    676. 

  676. 	reg_gfx_index.bits.se_broadcast_writes = 1;
    677. 

  677. 

  678. 

  679. 	packets_vec[2].ordinal1 = packets_vec[0].ordinal1;
    680. 

  680. 	packets_vec[2].bitfields2.reg_offset =
    681. 

  681. 				GRBM_GFX_INDEX / (sizeof(uint32_t)) -
    682. 

  682. 					USERCONFIG_REG_BASE;
    683. 

  683. 

  684. 	packets_vec[2].bitfields2.insert_vmid = 0;
    685. 

  685. 	packets_vec[2].reg_data[0] = reg_gfx_index.u32All;
    686. 

  686. 

  687. 	status = dbgdev_diq_submit_ib(
    688. 

  688. 			dbgdev,
    689. 

  689. 			wac_info->process->pasid,
    690. 

  690. 			mem_obj->gpu_addr,
    691. 

  691. 			packet_buff_uint,
    692. 

  692. 			ib_size);
    693. 

  693. 

  694. 	if (status)
    695. 

  695. 		pr_err("Failed to submit IB to DIQ\n");
    696. 

  696. 

  697. 	kfd_gtt_sa_free(dbgdev->dev, mem_obj);
    698. 

  698. 

  699. 	return status;
    700. 

  700. }
    701. 

  701. 

  702. static int dbgdev_wave_control_nodiq(struct kfd_dbgdev *dbgdev,
    703. 

  703. 					struct dbg_wave_control_info *wac_info)
    704. 

  704. {
    705. 

  705. 	int status;
    706. 

  706. 	union SQ_CMD_BITS reg_sq_cmd;
    707. 

  707. 	union GRBM_GFX_INDEX_BITS reg_gfx_index;
    708. 

  708. 	struct kfd_process_device *pdd;
    709. 

  709. 

  710. 	reg_sq_cmd.u32All = 0;
    711. 

  711. 

  712. 	/* taking the VMID for that process on the safe way using PDD */
    713. 

  713. 	pdd = kfd_get_process_device_data(dbgdev->dev, wac_info->process);
    714. 

  714. 

  715. 	if (!pdd) {
    716. 

  716. 		pr_err("Failed to get pdd for wave control no DIQ\n");
    717. 

  717. 		return -EFAULT;
    718. 

  718. 	}
    719. 

  719. 	status = dbgdev_wave_control_set_registers(wac_info, &reg_sq_cmd,
    720. 

  720. 							&reg_gfx_index);
    721. 

  721. 	if (status) {
    722. 

  722. 		pr_err("Failed to set wave control registers\n");
    723. 

  723. 		return status;
    724. 

  724. 	}
    725. 

  725. 

  726. 	/* for non DIQ we need to patch the VMID: */
    727. 

  727. 

  728. 	reg_sq_cmd.bits.vm_id = pdd->qpd.vmid;
    729. 

  729. 

  730. 	pr_debug("\t\t %30s\n", "* * * * * * * * * * * * * * * * * *");
    731. 

  731. 

  732. 	pr_debug("\t\t mode      is: %u\n", wac_info->mode);
    733. 

  733. 	pr_debug("\t\t operand   is: %u\n", wac_info->operand);
    734. 

  734. 	pr_debug("\t\t trap id   is: %u\n", wac_info->trapId);
    735. 

  735. 	pr_debug("\t\t msg value is: %u\n",
    736. 

  736. 			wac_info->dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value);
    737. 

  737. 	pr_debug("\t\t vmid      is: %u\n", pdd->qpd.vmid);
    738. 

  738. 

  739. 	pr_debug("\t\t chk_vmid  is : %u\n", reg_sq_cmd.bitfields.check_vmid);
    740. 

  740. 	pr_debug("\t\t command   is : %u\n", reg_sq_cmd.bitfields.cmd);
    741. 

  741. 	pr_debug("\t\t queue id  is : %u\n", reg_sq_cmd.bitfields.queue_id);
    742. 

  742. 	pr_debug("\t\t simd id   is : %u\n", reg_sq_cmd.bitfields.simd_id);
    743. 

  743. 	pr_debug("\t\t mode      is : %u\n", reg_sq_cmd.bitfields.mode);
    744. 

  744. 	pr_debug("\t\t vm_id     is : %u\n", reg_sq_cmd.bitfields.vm_id);
    745. 

  745. 	pr_debug("\t\t wave_id   is : %u\n", reg_sq_cmd.bitfields.wave_id);
    746. 

  746. 

  747. 	pr_debug("\t\t ibw       is : %u\n",
    748. 

  748. 			reg_gfx_index.bitfields.instance_broadcast_writes);
    749. 

  749. 	pr_debug("\t\t ii        is : %u\n",
    750. 

  750. 			reg_gfx_index.bitfields.instance_index);
    751. 

  751. 	pr_debug("\t\t sebw      is : %u\n",
    752. 

  752. 			reg_gfx_index.bitfields.se_broadcast_writes);
    753. 

  753. 	pr_debug("\t\t se_ind    is : %u\n", reg_gfx_index.bitfields.se_index);
    754. 

  754. 	pr_debug("\t\t sh_ind    is : %u\n", reg_gfx_index.bitfields.sh_index);
    755. 

  755. 	pr_debug("\t\t sbw       is : %u\n",
    756. 

  756. 			reg_gfx_index.bitfields.sh_broadcast_writes);
    757. 

  757. 

  758. 	pr_debug("\t\t %30s\n", "* * * * * * * * * * * * * * * * * *");
    759. 

  759. 

  760. 	return dbgdev->dev->kfd2kgd->wave_control_execute(dbgdev->dev->kgd,
    761. 

  761. 							reg_gfx_index.u32All,
    762. 

  762. 							reg_sq_cmd.u32All);
    763. 

  763. }
    764. 

  764. 

  765. int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p)
    766. 

  766. {
    767. 

  767. 	int status = 0;
    768. 

  768. 	unsigned int vmid;
    769. 

  769. 	union SQ_CMD_BITS reg_sq_cmd;
    770. 

  770. 	union GRBM_GFX_INDEX_BITS reg_gfx_index;
    771. 

  771. 	struct kfd_process_device *pdd;
    772. 

  772. 	struct dbg_wave_control_info wac_info;
    773. 

  773. 	int first_vmid_to_scan = dev->vm_info.first_vmid_kfd;
    774. 

  774. 	int last_vmid_to_scan = dev->vm_info.last_vmid_kfd;
    775. 

  775. 

  776. 	reg_sq_cmd.u32All = 0;
    777. 

  777. 	status = 0;
    778. 

  778. 

  779. 	wac_info.mode = HSA_DBG_WAVEMODE_BROADCAST_PROCESS;
    780. 

  780. 	wac_info.operand = HSA_DBG_WAVEOP_KILL;
    781. 

  781. 

  782. 	pr_debug("Killing all process wavefronts\n");
    783. 

  783. 

  784. 	/* Scan all registers in the range ATC_VMID8_PASID_MAPPING ..
    785. 

  785. 	 * ATC_VMID15_PASID_MAPPING
    786. 

  786. 	 * to check which VMID the current process is mapped to.
    787. 

  787. 	 */
    788. 

  788. 

  789. 	for (vmid = first_vmid_to_scan; vmid <= last_vmid_to_scan; vmid++) {
    790. 

  790. 		if (dev->kfd2kgd->get_atc_vmid_pasid_mapping_valid
    791. 

  791. 				(dev->kgd, vmid)) {
    792. 

  792. 			if (dev->kfd2kgd->get_atc_vmid_pasid_mapping_pasid
    793. 

  793. 					(dev->kgd, vmid) == p->pasid) {
    794. 

  794. 				pr_debug("Killing wave fronts of vmid %d and pasid %d\n",
    795. 

  795. 						vmid, p->pasid);
    796. 

  796. 				break;
    797. 

  797. 			}
    798. 

  798. 		}
    799. 

  799. 	}
    800. 

  800. 

  801. 	if (vmid > last_vmid_to_scan) {
    802. 

  802. 		pr_err("Didn't find vmid for pasid %d\n", p->pasid);
    803. 

  803. 		return -EFAULT;
    804. 

  804. 	}
    805. 

  805. 

  806. 	/* taking the VMID for that process on the safe way using PDD */
    807. 

  807. 	pdd = kfd_get_process_device_data(dev, p);
    808. 

  808. 	if (!pdd)
    809. 

  809. 		return -EFAULT;
    810. 

  810. 

  811. 	status = dbgdev_wave_control_set_registers(&wac_info, &reg_sq_cmd,
    812. 

  812. 			&reg_gfx_index);
    813. 

  813. 	if (status != 0)
    814. 

  814. 		return -EINVAL;
    815. 

  815. 

  816. 	/* for non DIQ we need to patch the VMID: */
    817. 

  817. 	reg_sq_cmd.bits.vm_id = vmid;
    818. 

  818. 

  819. 	dev->kfd2kgd->wave_control_execute(dev->kgd,
    820. 

  820. 					reg_gfx_index.u32All,
    821. 

  821. 					reg_sq_cmd.u32All);
    822. 

  822. 

  823. 	return 0;
    824. 

  824. }
    825. 

  825. 

  826. void kfd_dbgdev_init(struct kfd_dbgdev *pdbgdev, struct kfd_dev *pdev,
    827. 

  827. 			enum DBGDEV_TYPE type)
    828. 

  828. {
    829. 

  829. 	pdbgdev->dev = pdev;
    830. 

  830. 	pdbgdev->kq = NULL;
    831. 

  831. 	pdbgdev->type = type;
    832. 

  832. 	pdbgdev->pqm = NULL;
    833. 

  833. 

  834. 	switch (type) {
    835. 

  835. 	case DBGDEV_TYPE_NODIQ:
    836. 

  836. 		pdbgdev->dbgdev_register = dbgdev_register_nodiq;
    837. 

  837. 		pdbgdev->dbgdev_unregister = dbgdev_unregister_nodiq;
    838. 

  838. 		pdbgdev->dbgdev_wave_control = dbgdev_wave_control_nodiq;
    839. 

  839. 		pdbgdev->dbgdev_address_watch = dbgdev_address_watch_nodiq;
    840. 

  840. 		break;
    841. 

  841. 	case DBGDEV_TYPE_DIQ:
    842. 

  842. 	default:
    843. 

  843. 		pdbgdev->dbgdev_register = dbgdev_register_diq;
    844. 

  844. 		pdbgdev->dbgdev_unregister = dbgdev_unregister_diq;
    845. 

  845. 		pdbgdev->dbgdev_wave_control =  dbgdev_wave_control_diq;
    846. 

  846. 		pdbgdev->dbgdev_address_watch = dbgdev_address_watch_diq;
    847. 

  847. 		break;
    848. 

  848. 	}
    849. 

  849. 

  850. }
    851.