/drivers/infiniband/hw/mthca/mthca_eq.c

https://bitbucket.org/evzijst/gittest · C · 964 lines · 733 code · 148 blank · 83 comment · 89 complexity · ee47e43e4205283a41f9410db202c017 MD5 · raw file 

    

  1. /*
    2. 

  2.  * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
    3. 

  3.  *
    4. 

  4.  * This software is available to you under a choice of one of two
    5. 

  5.  * licenses.  You may choose to be licensed under the terms of the GNU
    6. 

  6.  * General Public License (GPL) Version 2, available from the file
    7. 

  7.  * COPYING in the main directory of this source tree, or the
    8. 

  8.  * OpenIB.org BSD license below:
    9. 

  9.  *
    10. 

  10.  *     Redistribution and use in source and binary forms, with or
    11. 

  11.  *     without modification, are permitted provided that the following
    12. 

  12.  *     conditions are met:
    13. 

  13.  *
    14. 

  14.  *      - Redistributions of source code must retain the above
    15. 

  15.  *        copyright notice, this list of conditions and the following
    16. 

  16.  *        disclaimer.
    17. 

  17.  *
    18. 

  18.  *      - Redistributions in binary form must reproduce the above
    19. 

  19.  *        copyright notice, this list of conditions and the following
    20. 

  20.  *        disclaimer in the documentation and/or other materials
    21. 

  21.  *        provided with the distribution.
    22. 

  22.  *
    23. 

  23.  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
    24. 

  24.  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
    25. 

  25.  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
    26. 

  26.  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
    27. 

  27.  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
    28. 

  28.  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
    29. 

  29.  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
    30. 

  30.  * SOFTWARE.
    31. 

  31.  *
    32. 

  32.  * $Id: mthca_eq.c 1382 2004-12-24 02:21:02Z roland $
    33. 

  33.  */
    34. 

  34. 

  35. #include <linux/init.h>
    36. 

  36. #include <linux/errno.h>
    37. 

  37. #include <linux/interrupt.h>
    38. 

  38. #include <linux/pci.h>
    39. 

  39. 

  40. #include "mthca_dev.h"
    41. 

  41. #include "mthca_cmd.h"
    42. 

  42. #include "mthca_config_reg.h"
    43. 

  43. 

  44. enum {
    45. 

  45. 	MTHCA_NUM_ASYNC_EQE = 0x80,
    46. 

  46. 	MTHCA_NUM_CMD_EQE   = 0x80,
    47. 

  47. 	MTHCA_EQ_ENTRY_SIZE = 0x20
    48. 

  48. };
    49. 

  49. 

  50. /*
    51. 

  51.  * Must be packed because start is 64 bits but only aligned to 32 bits.
    52. 

  52.  */
    53. 

  53. struct mthca_eq_context {
    54. 

  54. 	u32 flags;
    55. 

  55. 	u64 start;
    56. 

  56. 	u32 logsize_usrpage;
    57. 

  57. 	u32 tavor_pd;		/* reserved for Arbel */
    58. 

  58. 	u8  reserved1[3];
    59. 

  59. 	u8  intr;
    60. 

  60. 	u32 arbel_pd;		/* lost_count for Tavor */
    61. 

  61. 	u32 lkey;
    62. 

  62. 	u32 reserved2[2];
    63. 

  63. 	u32 consumer_index;
    64. 

  64. 	u32 producer_index;
    65. 

  65. 	u32 reserved3[4];
    66. 

  66. } __attribute__((packed));
    67. 

  67. 

  68. #define MTHCA_EQ_STATUS_OK          ( 0 << 28)
    69. 

  69. #define MTHCA_EQ_STATUS_OVERFLOW    ( 9 << 28)
    70. 

  70. #define MTHCA_EQ_STATUS_WRITE_FAIL  (10 << 28)
    71. 

  71. #define MTHCA_EQ_OWNER_SW           ( 0 << 24)
    72. 

  72. #define MTHCA_EQ_OWNER_HW           ( 1 << 24)
    73. 

  73. #define MTHCA_EQ_FLAG_TR            ( 1 << 18)
    74. 

  74. #define MTHCA_EQ_FLAG_OI            ( 1 << 17)
    75. 

  75. #define MTHCA_EQ_STATE_ARMED        ( 1 <<  8)
    76. 

  76. #define MTHCA_EQ_STATE_FIRED        ( 2 <<  8)
    77. 

  77. #define MTHCA_EQ_STATE_ALWAYS_ARMED ( 3 <<  8)
    78. 

  78. #define MTHCA_EQ_STATE_ARBEL        ( 8 <<  8)
    79. 

  79. 

  80. enum {
    81. 

  81. 	MTHCA_EVENT_TYPE_COMP       	    = 0x00,
    82. 

  82. 	MTHCA_EVENT_TYPE_PATH_MIG   	    = 0x01,
    83. 

  83. 	MTHCA_EVENT_TYPE_COMM_EST   	    = 0x02,
    84. 

  84. 	MTHCA_EVENT_TYPE_SQ_DRAINED 	    = 0x03,
    85. 

  85. 	MTHCA_EVENT_TYPE_SRQ_LAST_WQE       = 0x13,
    86. 

  86. 	MTHCA_EVENT_TYPE_CQ_ERROR   	    = 0x04,
    87. 

  87. 	MTHCA_EVENT_TYPE_WQ_CATAS_ERROR     = 0x05,
    88. 

  88. 	MTHCA_EVENT_TYPE_EEC_CATAS_ERROR    = 0x06,
    89. 

  89. 	MTHCA_EVENT_TYPE_PATH_MIG_FAILED    = 0x07,
    90. 

  90. 	MTHCA_EVENT_TYPE_WQ_INVAL_REQ_ERROR = 0x10,
    91. 

  91. 	MTHCA_EVENT_TYPE_WQ_ACCESS_ERROR    = 0x11,
    92. 

  92. 	MTHCA_EVENT_TYPE_SRQ_CATAS_ERROR    = 0x12,
    93. 

  93. 	MTHCA_EVENT_TYPE_LOCAL_CATAS_ERROR  = 0x08,
    94. 

  94. 	MTHCA_EVENT_TYPE_PORT_CHANGE        = 0x09,
    95. 

  95. 	MTHCA_EVENT_TYPE_EQ_OVERFLOW        = 0x0f,
    96. 

  96. 	MTHCA_EVENT_TYPE_ECC_DETECT         = 0x0e,
    97. 

  97. 	MTHCA_EVENT_TYPE_CMD                = 0x0a
    98. 

  98. };
    99. 

  99. 

  100. #define MTHCA_ASYNC_EVENT_MASK ((1ULL << MTHCA_EVENT_TYPE_PATH_MIG)           | \
    101. 

  101. 				(1ULL << MTHCA_EVENT_TYPE_COMM_EST)           | \
    102. 

  102. 				(1ULL << MTHCA_EVENT_TYPE_SQ_DRAINED)         | \
    103. 

  103. 				(1ULL << MTHCA_EVENT_TYPE_CQ_ERROR)           | \
    104. 

  104. 				(1ULL << MTHCA_EVENT_TYPE_WQ_CATAS_ERROR)     | \
    105. 

  105. 				(1ULL << MTHCA_EVENT_TYPE_EEC_CATAS_ERROR)    | \
    106. 

  106. 				(1ULL << MTHCA_EVENT_TYPE_PATH_MIG_FAILED)    | \
    107. 

  107. 				(1ULL << MTHCA_EVENT_TYPE_WQ_INVAL_REQ_ERROR) | \
    108. 

  108. 				(1ULL << MTHCA_EVENT_TYPE_WQ_ACCESS_ERROR)    | \
    109. 

  109. 				(1ULL << MTHCA_EVENT_TYPE_LOCAL_CATAS_ERROR)  | \
    110. 

  110. 				(1ULL << MTHCA_EVENT_TYPE_PORT_CHANGE)        | \
    111. 

  111. 				(1ULL << MTHCA_EVENT_TYPE_ECC_DETECT))
    112. 

  112. #define MTHCA_SRQ_EVENT_MASK    (1ULL << MTHCA_EVENT_TYPE_SRQ_CATAS_ERROR)    | \
    113. 

  113. 				(1ULL << MTHCA_EVENT_TYPE_SRQ_LAST_WQE)
    114. 

  114. #define MTHCA_CMD_EVENT_MASK    (1ULL << MTHCA_EVENT_TYPE_CMD)
    115. 

  115. 

  116. #define MTHCA_EQ_DB_INC_CI     (1 << 24)
    117. 

  117. #define MTHCA_EQ_DB_REQ_NOT    (2 << 24)
    118. 

  118. #define MTHCA_EQ_DB_DISARM_CQ  (3 << 24)
    119. 

  119. #define MTHCA_EQ_DB_SET_CI     (4 << 24)
    120. 

  120. #define MTHCA_EQ_DB_ALWAYS_ARM (5 << 24)
    121. 

  121. 

  122. struct mthca_eqe {
    123. 

  123. 	u8 reserved1;
    124. 

  124. 	u8 type;
    125. 

  125. 	u8 reserved2;
    126. 

  126. 	u8 subtype;
    127. 

  127. 	union {
    128. 

  128. 		u32 raw[6];
    129. 

  129. 		struct {
    130. 

  130. 			u32 cqn;
    131. 

  131. 		} __attribute__((packed)) comp;
    132. 

  132. 		struct {
    133. 

  133. 			u16 reserved1;
    134. 

  134. 			u16 token;
    135. 

  135. 			u32 reserved2;
    136. 

  136. 			u8  reserved3[3];
    137. 

  137. 			u8  status;
    138. 

  138. 			u64 out_param;
    139. 

  139. 		} __attribute__((packed)) cmd;
    140. 

  140. 		struct {
    141. 

  141. 			u32 qpn;
    142. 

  142. 		} __attribute__((packed)) qp;
    143. 

  143. 		struct {
    144. 

  144. 			u32 cqn;
    145. 

  145. 			u32 reserved1;
    146. 

  146. 			u8  reserved2[3];
    147. 

  147. 			u8  syndrome;
    148. 

  148. 		} __attribute__((packed)) cq_err;
    149. 

  149. 		struct {
    150. 

  150. 			u32 reserved1[2];
    151. 

  151. 			u32 port;
    152. 

  152. 		} __attribute__((packed)) port_change;
    153. 

  153. 	} event;
    154. 

  154. 	u8 reserved3[3];
    155. 

  155. 	u8 owner;
    156. 

  156. } __attribute__((packed));
    157. 

  157. 

  158. #define  MTHCA_EQ_ENTRY_OWNER_SW      (0 << 7)
    159. 

  159. #define  MTHCA_EQ_ENTRY_OWNER_HW      (1 << 7)
    160. 

  160. 

  161. static inline u64 async_mask(struct mthca_dev *dev)
    162. 

  162. {
    163. 

  163. 	return dev->mthca_flags & MTHCA_FLAG_SRQ ?
    164. 

  164. 		MTHCA_ASYNC_EVENT_MASK | MTHCA_SRQ_EVENT_MASK :
    165. 

  165. 		MTHCA_ASYNC_EVENT_MASK;
    166. 

  166. }
    167. 

  167. 

  168. static inline void tavor_set_eq_ci(struct mthca_dev *dev, struct mthca_eq *eq, u32 ci)
    169. 

  169. {
    170. 

  170. 	u32 doorbell[2];
    171. 

  171. 

  172. 	doorbell[0] = cpu_to_be32(MTHCA_EQ_DB_SET_CI | eq->eqn);
    173. 

  173. 	doorbell[1] = cpu_to_be32(ci & (eq->nent - 1));
    174. 

  174. 

  175. 	/*
    176. 

  176. 	 * This barrier makes sure that all updates to ownership bits
    177. 

  177. 	 * done by set_eqe_hw() hit memory before the consumer index
    178. 

  178. 	 * is updated.  set_eq_ci() allows the HCA to possibly write
    179. 

  179. 	 * more EQ entries, and we want to avoid the exceedingly
    180. 

  180. 	 * unlikely possibility of the HCA writing an entry and then
    181. 

  181. 	 * having set_eqe_hw() overwrite the owner field.
    182. 

  182. 	 */
    183. 

  183. 	wmb();
    184. 

  184. 	mthca_write64(doorbell,
    185. 

  185. 		      dev->kar + MTHCA_EQ_DOORBELL,
    186. 

  186. 		      MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
    187. 

  187. }
    188. 

  188. 

  189. static inline void arbel_set_eq_ci(struct mthca_dev *dev, struct mthca_eq *eq, u32 ci)
    190. 

  190. {
    191. 

  191. 	/* See comment in tavor_set_eq_ci() above. */
    192. 

  192. 	wmb();
    193. 

  193. 	__raw_writel(cpu_to_be32(ci), dev->eq_regs.arbel.eq_set_ci_base +
    194. 

  194. 		     eq->eqn * 8);
    195. 

  195. 	/* We still want ordering, just not swabbing, so add a barrier */
    196. 

  196. 	mb();
    197. 

  197. }
    198. 

  198. 

  199. static inline void set_eq_ci(struct mthca_dev *dev, struct mthca_eq *eq, u32 ci)
    200. 

  200. {
    201. 

  201. 	if (dev->hca_type == ARBEL_NATIVE)
    202. 

  202. 		arbel_set_eq_ci(dev, eq, ci);
    203. 

  203. 	else
    204. 

  204. 		tavor_set_eq_ci(dev, eq, ci);
    205. 

  205. }
    206. 

  206. 

  207. static inline void tavor_eq_req_not(struct mthca_dev *dev, int eqn)
    208. 

  208. {
    209. 

  209. 	u32 doorbell[2];
    210. 

  210. 

  211. 	doorbell[0] = cpu_to_be32(MTHCA_EQ_DB_REQ_NOT | eqn);
    212. 

  212. 	doorbell[1] = 0;
    213. 

  213. 

  214. 	mthca_write64(doorbell,
    215. 

  215. 		      dev->kar + MTHCA_EQ_DOORBELL,
    216. 

  216. 		      MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
    217. 

  217. }
    218. 

  218. 

  219. static inline void arbel_eq_req_not(struct mthca_dev *dev, u32 eqn_mask)
    220. 

  220. {
    221. 

  221. 	writel(eqn_mask, dev->eq_regs.arbel.eq_arm);
    222. 

  222. }
    223. 

  223. 

  224. static inline void disarm_cq(struct mthca_dev *dev, int eqn, int cqn)
    225. 

  225. {
    226. 

  226. 	if (dev->hca_type != ARBEL_NATIVE) {
    227. 

  227. 		u32 doorbell[2];
    228. 

  228. 

  229. 		doorbell[0] = cpu_to_be32(MTHCA_EQ_DB_DISARM_CQ | eqn);
    230. 

  230. 		doorbell[1] = cpu_to_be32(cqn);
    231. 

  231. 

  232. 		mthca_write64(doorbell,
    233. 

  233. 			      dev->kar + MTHCA_EQ_DOORBELL,
    234. 

  234. 			      MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
    235. 

  235. 	}
    236. 

  236. }
    237. 

  237. 

  238. static inline struct mthca_eqe *get_eqe(struct mthca_eq *eq, u32 entry)
    239. 

  239. {
    240. 

  240. 	unsigned long off = (entry & (eq->nent - 1)) * MTHCA_EQ_ENTRY_SIZE;
    241. 

  241. 	return eq->page_list[off / PAGE_SIZE].buf + off % PAGE_SIZE;
    242. 

  242. }
    243. 

  243. 

  244. static inline struct mthca_eqe* next_eqe_sw(struct mthca_eq *eq)
    245. 

  245. {
    246. 

  246. 	struct mthca_eqe* eqe;
    247. 

  247. 	eqe = get_eqe(eq, eq->cons_index);
    248. 

  248. 	return (MTHCA_EQ_ENTRY_OWNER_HW & eqe->owner) ? NULL : eqe;
    249. 

  249. }
    250. 

  250. 

  251. static inline void set_eqe_hw(struct mthca_eqe *eqe)
    252. 

  252. {
    253. 

  253. 	eqe->owner =  MTHCA_EQ_ENTRY_OWNER_HW;
    254. 

  254. }
    255. 

  255. 

  256. static void port_change(struct mthca_dev *dev, int port, int active)
    257. 

  257. {
    258. 

  258. 	struct ib_event record;
    259. 

  259. 

  260. 	mthca_dbg(dev, "Port change to %s for port %d\n",
    261. 

  261. 		  active ? "active" : "down", port);
    262. 

  262. 

  263. 	record.device = &dev->ib_dev;
    264. 

  264. 	record.event  = active ? IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
    265. 

  265. 	record.element.port_num = port;
    266. 

  266. 

  267. 	ib_dispatch_event(&record);
    268. 

  268. }
    269. 

  269. 

  270. static int mthca_eq_int(struct mthca_dev *dev, struct mthca_eq *eq)
    271. 

  271. {
    272. 

  272. 	struct mthca_eqe *eqe;
    273. 

  273. 	int disarm_cqn;
    274. 

  274. 	int  eqes_found = 0;
    275. 

  275. 

  276. 	while ((eqe = next_eqe_sw(eq))) {
    277. 

  277. 		int set_ci = 0;
    278. 

  278. 

  279. 		/*
    280. 

  280. 		 * Make sure we read EQ entry contents after we've
    281. 

  281. 		 * checked the ownership bit.
    282. 

  282. 		 */
    283. 

  283. 		rmb();
    284. 

  284. 

  285. 		switch (eqe->type) {
    286. 

  286. 		case MTHCA_EVENT_TYPE_COMP:
    287. 

  287. 			disarm_cqn = be32_to_cpu(eqe->event.comp.cqn) & 0xffffff;
    288. 

  288. 			disarm_cq(dev, eq->eqn, disarm_cqn);
    289. 

  289. 			mthca_cq_event(dev, disarm_cqn);
    290. 

  290. 			break;
    291. 

  291. 

  292. 		case MTHCA_EVENT_TYPE_PATH_MIG:
    293. 

  293. 			mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,
    294. 

  294. 				       IB_EVENT_PATH_MIG);
    295. 

  295. 			break;
    296. 

  296. 

  297. 		case MTHCA_EVENT_TYPE_COMM_EST:
    298. 

  298. 			mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,
    299. 

  299. 				       IB_EVENT_COMM_EST);
    300. 

  300. 			break;
    301. 

  301. 

  302. 		case MTHCA_EVENT_TYPE_SQ_DRAINED:
    303. 

  303. 			mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,
    304. 

  304. 				       IB_EVENT_SQ_DRAINED);
    305. 

  305. 			break;
    306. 

  306. 

  307. 		case MTHCA_EVENT_TYPE_WQ_CATAS_ERROR:
    308. 

  308. 			mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,
    309. 

  309. 				       IB_EVENT_QP_FATAL);
    310. 

  310. 			break;
    311. 

  311. 

  312. 		case MTHCA_EVENT_TYPE_PATH_MIG_FAILED:
    313. 

  313. 			mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,
    314. 

  314. 				       IB_EVENT_PATH_MIG_ERR);
    315. 

  315. 			break;
    316. 

  316. 

  317. 		case MTHCA_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
    318. 

  318. 			mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,
    319. 

  319. 				       IB_EVENT_QP_REQ_ERR);
    320. 

  320. 			break;
    321. 

  321. 

  322. 		case MTHCA_EVENT_TYPE_WQ_ACCESS_ERROR:
    323. 

  323. 			mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,
    324. 

  324. 				       IB_EVENT_QP_ACCESS_ERR);
    325. 

  325. 			break;
    326. 

  326. 

  327. 		case MTHCA_EVENT_TYPE_CMD:
    328. 

  328. 			mthca_cmd_event(dev,
    329. 

  329. 					be16_to_cpu(eqe->event.cmd.token),
    330. 

  330. 					eqe->event.cmd.status,
    331. 

  331. 					be64_to_cpu(eqe->event.cmd.out_param));
    332. 

  332. 			/*
    333. 

  333. 			 * cmd_event() may add more commands.
    334. 

  334. 			 * The card will think the queue has overflowed if
    335. 

  335. 			 * we don't tell it we've been processing events.
    336. 

  336. 			 */
    337. 

  337. 			set_ci = 1;
    338. 

  338. 			break;
    339. 

  339. 

  340. 		case MTHCA_EVENT_TYPE_PORT_CHANGE:
    341. 

  341. 			port_change(dev,
    342. 

  342. 				    (be32_to_cpu(eqe->event.port_change.port) >> 28) & 3,
    343. 

  343. 				    eqe->subtype == 0x4);
    344. 

  344. 			break;
    345. 

  345. 

  346. 		case MTHCA_EVENT_TYPE_CQ_ERROR:
    347. 

  347. 			mthca_warn(dev, "CQ %s on CQN %08x\n",
    348. 

  348. 				   eqe->event.cq_err.syndrome == 1 ?
    349. 

  349. 				   "overrun" : "access violation",
    350. 

  350. 				   be32_to_cpu(eqe->event.cq_err.cqn));
    351. 

  351. 			break;
    352. 

  352. 

  353. 		case MTHCA_EVENT_TYPE_EQ_OVERFLOW:
    354. 

  354. 			mthca_warn(dev, "EQ overrun on EQN %d\n", eq->eqn);
    355. 

  355. 			break;
    356. 

  356. 

  357. 		case MTHCA_EVENT_TYPE_EEC_CATAS_ERROR:
    358. 

  358. 		case MTHCA_EVENT_TYPE_SRQ_CATAS_ERROR:
    359. 

  359. 		case MTHCA_EVENT_TYPE_LOCAL_CATAS_ERROR:
    360. 

  360. 		case MTHCA_EVENT_TYPE_ECC_DETECT:
    361. 

  361. 		default:
    362. 

  362. 			mthca_warn(dev, "Unhandled event %02x(%02x) on EQ %d\n",
    363. 

  363. 				   eqe->type, eqe->subtype, eq->eqn);
    364. 

  364. 			break;
    365. 

  365. 		};
    366. 

  366. 

  367. 		set_eqe_hw(eqe);
    368. 

  368. 		++eq->cons_index;
    369. 

  369. 		eqes_found = 1;
    370. 

  370. 

  371. 		if (unlikely(set_ci)) {
    372. 

  372. 			/*
    373. 

  373. 			 * Conditional on hca_type is OK here because
    374. 

  374. 			 * this is a rare case, not the fast path.
    375. 

  375. 			 */
    376. 

  376. 			set_eq_ci(dev, eq, eq->cons_index);
    377. 

  377. 			set_ci = 0;
    378. 

  378. 		}
    379. 

  379. 	}
    380. 

  380. 

  381. 	/*
    382. 

  382. 	 * Rely on caller to set consumer index so that we don't have
    383. 

  383. 	 * to test hca_type in our interrupt handling fast path.
    384. 

  384. 	 */
    385. 

  385. 	return eqes_found;
    386. 

  386. }
    387. 

  387. 

  388. static irqreturn_t mthca_tavor_interrupt(int irq, void *dev_ptr, struct pt_regs *regs)
    389. 

  389. {
    390. 

  390. 	struct mthca_dev *dev = dev_ptr;
    391. 

  391. 	u32 ecr;
    392. 

  392. 	int i;
    393. 

  393. 

  394. 	if (dev->eq_table.clr_mask)
    395. 

  395. 		writel(dev->eq_table.clr_mask, dev->eq_table.clr_int);
    396. 

  396. 

  397. 	ecr = readl(dev->eq_regs.tavor.ecr_base + 4);
    398. 

  398. 	if (ecr) {
    399. 

  399. 		writel(ecr, dev->eq_regs.tavor.ecr_base +
    400. 

  400. 		       MTHCA_ECR_CLR_BASE - MTHCA_ECR_BASE + 4);
    401. 

  401. 

  402. 		for (i = 0; i < MTHCA_NUM_EQ; ++i)
    403. 

  403. 			if (ecr & dev->eq_table.eq[i].eqn_mask &&
    404. 

  404. 			    mthca_eq_int(dev, &dev->eq_table.eq[i])) {
    405. 

  405. 				tavor_set_eq_ci(dev, &dev->eq_table.eq[i],
    406. 

  406. 						dev->eq_table.eq[i].cons_index);
    407. 

  407. 				tavor_eq_req_not(dev, dev->eq_table.eq[i].eqn);
    408. 

  408. 			}
    409. 

  409. 	}
    410. 

  410. 

  411. 	return IRQ_RETVAL(ecr);
    412. 

  412. }
    413. 

  413. 

  414. static irqreturn_t mthca_tavor_msi_x_interrupt(int irq, void *eq_ptr,
    415. 

  415. 					 struct pt_regs *regs)
    416. 

  416. {
    417. 

  417. 	struct mthca_eq  *eq  = eq_ptr;
    418. 

  418. 	struct mthca_dev *dev = eq->dev;
    419. 

  419. 

  420. 	mthca_eq_int(dev, eq);
    421. 

  421. 	tavor_set_eq_ci(dev, eq, eq->cons_index);
    422. 

  422. 	tavor_eq_req_not(dev, eq->eqn);
    423. 

  423. 

  424. 	/* MSI-X vectors always belong to us */
    425. 

  425. 	return IRQ_HANDLED;
    426. 

  426. }
    427. 

  427. 

  428. static irqreturn_t mthca_arbel_interrupt(int irq, void *dev_ptr, struct pt_regs *regs)
    429. 

  429. {
    430. 

  430. 	struct mthca_dev *dev = dev_ptr;
    431. 

  431. 	int work = 0;
    432. 

  432. 	int i;
    433. 

  433. 

  434. 	if (dev->eq_table.clr_mask)
    435. 

  435. 		writel(dev->eq_table.clr_mask, dev->eq_table.clr_int);
    436. 

  436. 

  437. 	for (i = 0; i < MTHCA_NUM_EQ; ++i)
    438. 

  438. 		if (mthca_eq_int(dev, &dev->eq_table.eq[i])) {
    439. 

  439. 			work = 1;
    440. 

  440. 			arbel_set_eq_ci(dev, &dev->eq_table.eq[i],
    441. 

  441. 					dev->eq_table.eq[i].cons_index);
    442. 

  442. 		}
    443. 

  443. 

  444. 	arbel_eq_req_not(dev, dev->eq_table.arm_mask);
    445. 

  445. 

  446. 	return IRQ_RETVAL(work);
    447. 

  447. }
    448. 

  448. 

  449. static irqreturn_t mthca_arbel_msi_x_interrupt(int irq, void *eq_ptr,
    450. 

  450. 					       struct pt_regs *regs)
    451. 

  451. {
    452. 

  452. 	struct mthca_eq  *eq  = eq_ptr;
    453. 

  453. 	struct mthca_dev *dev = eq->dev;
    454. 

  454. 

  455. 	mthca_eq_int(dev, eq);
    456. 

  456. 	arbel_set_eq_ci(dev, eq, eq->cons_index);
    457. 

  457. 	arbel_eq_req_not(dev, eq->eqn_mask);
    458. 

  458. 

  459. 	/* MSI-X vectors always belong to us */
    460. 

  460. 	return IRQ_HANDLED;
    461. 

  461. }
    462. 

  462. 

  463. static int __devinit mthca_create_eq(struct mthca_dev *dev,
    464. 

  464. 				     int nent,
    465. 

  465. 				     u8 intr,
    466. 

  466. 				     struct mthca_eq *eq)
    467. 

  467. {
    468. 

  468. 	int npages = (nent * MTHCA_EQ_ENTRY_SIZE + PAGE_SIZE - 1) /
    469. 

  469. 		PAGE_SIZE;
    470. 

  470. 	u64 *dma_list = NULL;
    471. 

  471. 	dma_addr_t t;
    472. 

  472. 	void *mailbox = NULL;
    473. 

  473. 	struct mthca_eq_context *eq_context;
    474. 

  474. 	int err = -ENOMEM;
    475. 

  475. 	int i;
    476. 

  476. 	u8 status;
    477. 

  477. 

  478. 	/* Make sure EQ size is aligned to a power of 2 size. */
    479. 

  479. 	for (i = 1; i < nent; i <<= 1)
    480. 

  480. 		; /* nothing */
    481. 

  481. 	nent = i;
    482. 

  482. 

  483. 	eq->dev = dev;
    484. 

  484. 

  485. 	eq->page_list = kmalloc(npages * sizeof *eq->page_list,
    486. 

  486. 				GFP_KERNEL);
    487. 

  487. 	if (!eq->page_list)
    488. 

  488. 		goto err_out;
    489. 

  489. 

  490. 	for (i = 0; i < npages; ++i)
    491. 

  491. 		eq->page_list[i].buf = NULL;
    492. 

  492. 

  493. 	dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL);
    494. 

  494. 	if (!dma_list)
    495. 

  495. 		goto err_out_free;
    496. 

  496. 

  497. 	mailbox = kmalloc(sizeof *eq_context + MTHCA_CMD_MAILBOX_EXTRA,
    498. 

  498. 			  GFP_KERNEL);
    499. 

  499. 	if (!mailbox)
    500. 

  500. 		goto err_out_free;
    501. 

  501. 	eq_context = MAILBOX_ALIGN(mailbox);
    502. 

  502. 

  503. 	for (i = 0; i < npages; ++i) {
    504. 

  504. 		eq->page_list[i].buf = pci_alloc_consistent(dev->pdev,
    505. 

  505. 							    PAGE_SIZE, &t);
    506. 

  506. 		if (!eq->page_list[i].buf)
    507. 

  507. 			goto err_out_free;
    508. 

  508. 

  509. 		dma_list[i] = t;
    510. 

  510. 		pci_unmap_addr_set(&eq->page_list[i], mapping, t);
    511. 

  511. 

  512. 		memset(eq->page_list[i].buf, 0, PAGE_SIZE);
    513. 

  513. 	}
    514. 

  514. 

  515. 	for (i = 0; i < nent; ++i)
    516. 

  516. 		set_eqe_hw(get_eqe(eq, i));
    517. 

  517. 

  518. 	eq->eqn = mthca_alloc(&dev->eq_table.alloc);
    519. 

  519. 	if (eq->eqn == -1)
    520. 

  520. 		goto err_out_free;
    521. 

  521. 

  522. 	err = mthca_mr_alloc_phys(dev, dev->driver_pd.pd_num,
    523. 

  523. 				  dma_list, PAGE_SHIFT, npages,
    524. 

  524. 				  0, npages * PAGE_SIZE,
    525. 

  525. 				  MTHCA_MPT_FLAG_LOCAL_WRITE |
    526. 

  526. 				  MTHCA_MPT_FLAG_LOCAL_READ,
    527. 

  527. 				  &eq->mr);
    528. 

  528. 	if (err)
    529. 

  529. 		goto err_out_free_eq;
    530. 

  530. 

  531. 	eq->nent = nent;
    532. 

  532. 

  533. 	memset(eq_context, 0, sizeof *eq_context);
    534. 

  534. 	eq_context->flags           = cpu_to_be32(MTHCA_EQ_STATUS_OK   |
    535. 

  535. 						  MTHCA_EQ_OWNER_HW    |
    536. 

  536. 						  MTHCA_EQ_STATE_ARMED |
    537. 

  537. 						  MTHCA_EQ_FLAG_TR);
    538. 

  538. 	if (dev->hca_type == ARBEL_NATIVE)
    539. 

  539. 		eq_context->flags  |= cpu_to_be32(MTHCA_EQ_STATE_ARBEL);
    540. 

  540. 

  541. 	eq_context->logsize_usrpage = cpu_to_be32((ffs(nent) - 1) << 24);
    542. 

  542. 	if (dev->hca_type == ARBEL_NATIVE) {
    543. 

  543. 		eq_context->arbel_pd = cpu_to_be32(dev->driver_pd.pd_num);
    544. 

  544. 	} else {
    545. 

  545. 		eq_context->logsize_usrpage |= cpu_to_be32(dev->driver_uar.index);
    546. 

  546. 		eq_context->tavor_pd         = cpu_to_be32(dev->driver_pd.pd_num);
    547. 

  547. 	}
    548. 

  548. 	eq_context->intr            = intr;
    549. 

  549. 	eq_context->lkey            = cpu_to_be32(eq->mr.ibmr.lkey);
    550. 

  550. 

  551. 	err = mthca_SW2HW_EQ(dev, eq_context, eq->eqn, &status);
    552. 

  552. 	if (err) {
    553. 

  553. 		mthca_warn(dev, "SW2HW_EQ failed (%d)\n", err);
    554. 

  554. 		goto err_out_free_mr;
    555. 

  555. 	}
    556. 

  556. 	if (status) {
    557. 

  557. 		mthca_warn(dev, "SW2HW_EQ returned status 0x%02x\n",
    558. 

  558. 			   status);
    559. 

  559. 		err = -EINVAL;
    560. 

  560. 		goto err_out_free_mr;
    561. 

  561. 	}
    562. 

  562. 

  563. 	kfree(dma_list);
    564. 

  564. 	kfree(mailbox);
    565. 

  565. 

  566. 	eq->eqn_mask   = swab32(1 << eq->eqn);
    567. 

  567. 	eq->cons_index = 0;
    568. 

  568. 

  569. 	dev->eq_table.arm_mask |= eq->eqn_mask;
    570. 

  570. 

  571. 	mthca_dbg(dev, "Allocated EQ %d with %d entries\n",
    572. 

  572. 		  eq->eqn, nent);
    573. 

  573. 

  574. 	return err;
    575. 

  575. 

  576.  err_out_free_mr:
    577. 

  577. 	mthca_free_mr(dev, &eq->mr);
    578. 

  578. 

  579.  err_out_free_eq:
    580. 

  580. 	mthca_free(&dev->eq_table.alloc, eq->eqn);
    581. 

  581. 

  582.  err_out_free:
    583. 

  583. 	for (i = 0; i < npages; ++i)
    584. 

  584. 		if (eq->page_list[i].buf)
    585. 

  585. 			pci_free_consistent(dev->pdev, PAGE_SIZE,
    586. 

  586. 					    eq->page_list[i].buf,
    587. 

  587. 					    pci_unmap_addr(&eq->page_list[i],
    588. 

  588. 							   mapping));
    589. 

  589. 

  590. 	kfree(eq->page_list);
    591. 

  591. 	kfree(dma_list);
    592. 

  592. 	kfree(mailbox);
    593. 

  593. 

  594.  err_out:
    595. 

  595. 	return err;
    596. 

  596. }
    597. 

  597. 

  598. static void mthca_free_eq(struct mthca_dev *dev,
    599. 

  599. 			  struct mthca_eq *eq)
    600. 

  600. {
    601. 

  601. 	void *mailbox = NULL;
    602. 

  602. 	int err;
    603. 

  603. 	u8 status;
    604. 

  604. 	int npages = (eq->nent * MTHCA_EQ_ENTRY_SIZE + PAGE_SIZE - 1) /
    605. 

  605. 		PAGE_SIZE;
    606. 

  606. 	int i;
    607. 

  607. 

  608. 	mailbox = kmalloc(sizeof (struct mthca_eq_context) + MTHCA_CMD_MAILBOX_EXTRA,
    609. 

  609. 			  GFP_KERNEL);
    610. 

  610. 	if (!mailbox)
    611. 

  611. 		return;
    612. 

  612. 

  613. 	err = mthca_HW2SW_EQ(dev, MAILBOX_ALIGN(mailbox),
    614. 

  614. 			     eq->eqn, &status);
    615. 

  615. 	if (err)
    616. 

  616. 		mthca_warn(dev, "HW2SW_EQ failed (%d)\n", err);
    617. 

  617. 	if (status)
    618. 

  618. 		mthca_warn(dev, "HW2SW_EQ returned status 0x%02x\n",
    619. 

  619. 			   status);
    620. 

  620. 

  621. 	dev->eq_table.arm_mask &= ~eq->eqn_mask;
    622. 

  622. 

  623. 	if (0) {
    624. 

  624. 		mthca_dbg(dev, "Dumping EQ context %02x:\n", eq->eqn);
    625. 

  625. 		for (i = 0; i < sizeof (struct mthca_eq_context) / 4; ++i) {
    626. 

  626. 			if (i % 4 == 0)
    627. 

  627. 				printk("[%02x] ", i * 4);
    628. 

  628. 			printk(" %08x", be32_to_cpup(MAILBOX_ALIGN(mailbox) + i * 4));
    629. 

  629. 			if ((i + 1) % 4 == 0)
    630. 

  630. 				printk("\n");
    631. 

  631. 		}
    632. 

  632. 	}
    633. 

  633. 

  634. 	mthca_free_mr(dev, &eq->mr);
    635. 

  635. 	for (i = 0; i < npages; ++i)
    636. 

  636. 		pci_free_consistent(dev->pdev, PAGE_SIZE,
    637. 

  637. 				    eq->page_list[i].buf,
    638. 

  638. 				    pci_unmap_addr(&eq->page_list[i], mapping));
    639. 

  639. 

  640. 	kfree(eq->page_list);
    641. 

  641. 	kfree(mailbox);
    642. 

  642. }
    643. 

  643. 

  644. static void mthca_free_irqs(struct mthca_dev *dev)
    645. 

  645. {
    646. 

  646. 	int i;
    647. 

  647. 

  648. 	if (dev->eq_table.have_irq)
    649. 

  649. 		free_irq(dev->pdev->irq, dev);
    650. 

  650. 	for (i = 0; i < MTHCA_NUM_EQ; ++i)
    651. 

  651. 		if (dev->eq_table.eq[i].have_irq)
    652. 

  652. 			free_irq(dev->eq_table.eq[i].msi_x_vector,
    653. 

  653. 				 dev->eq_table.eq + i);
    654. 

  654. }
    655. 

  655. 

  656. static int __devinit mthca_map_reg(struct mthca_dev *dev,
    657. 

  657. 				   unsigned long offset, unsigned long size,
    658. 

  658. 				   void __iomem **map)
    659. 

  659. {
    660. 

  660. 	unsigned long base = pci_resource_start(dev->pdev, 0);
    661. 

  661. 

  662. 	if (!request_mem_region(base + offset, size, DRV_NAME))
    663. 

  663. 		return -EBUSY;
    664. 

  664. 

  665. 	*map = ioremap(base + offset, size);
    666. 

  666. 	if (!*map) {
    667. 

  667. 		release_mem_region(base + offset, size);
    668. 

  668. 		return -ENOMEM;
    669. 

  669. 	}
    670. 

  670. 

  671. 	return 0;
    672. 

  672. }
    673. 

  673. 

  674. static void mthca_unmap_reg(struct mthca_dev *dev, unsigned long offset,
    675. 

  675. 			    unsigned long size, void __iomem *map)
    676. 

  676. {
    677. 

  677. 	unsigned long base = pci_resource_start(dev->pdev, 0);
    678. 

  678. 

  679. 	release_mem_region(base + offset, size);
    680. 

  680. 	iounmap(map);
    681. 

  681. }
    682. 

  682. 

  683. static int __devinit mthca_map_eq_regs(struct mthca_dev *dev)
    684. 

  684. {
    685. 

  685. 	unsigned long mthca_base;
    686. 

  686. 

  687. 	mthca_base = pci_resource_start(dev->pdev, 0);
    688. 

  688. 

  689. 	if (dev->hca_type == ARBEL_NATIVE) {
    690. 

  690. 		/*
    691. 

  691. 		 * We assume that the EQ arm and EQ set CI registers
    692. 

  692. 		 * fall within the first BAR.  We can't trust the
    693. 

  693. 		 * values firmware gives us, since those addresses are
    694. 

  694. 		 * valid on the HCA's side of the PCI bus but not
    695. 

  695. 		 * necessarily the host side.
    696. 

  696. 		 */
    697. 

  697. 		if (mthca_map_reg(dev, (pci_resource_len(dev->pdev, 0) - 1) &
    698. 

  698. 				  dev->fw.arbel.clr_int_base, MTHCA_CLR_INT_SIZE,
    699. 

  699. 				  &dev->clr_base)) {
    700. 

  700. 			mthca_err(dev, "Couldn't map interrupt clear register, "
    701. 

  701. 				  "aborting.\n");
    702. 

  702. 			return -ENOMEM;
    703. 

  703. 		}
    704. 

  704. 

  705. 		/*
    706. 

  706. 		 * Add 4 because we limit ourselves to EQs 0 ... 31,
    707. 

  707. 		 * so we only need the low word of the register.
    708. 

  708. 		 */
    709. 

  709. 		if (mthca_map_reg(dev, ((pci_resource_len(dev->pdev, 0) - 1) &
    710. 

  710. 					dev->fw.arbel.eq_arm_base) + 4, 4,
    711. 

  711. 				  &dev->eq_regs.arbel.eq_arm)) {
    712. 

  712. 			mthca_err(dev, "Couldn't map interrupt clear register, "
    713. 

  713. 				  "aborting.\n");
    714. 

  714. 			mthca_unmap_reg(dev, (pci_resource_len(dev->pdev, 0) - 1) &
    715. 

  715. 					dev->fw.arbel.clr_int_base, MTHCA_CLR_INT_SIZE,
    716. 

  716. 					dev->clr_base);
    717. 

  717. 			return -ENOMEM;
    718. 

  718. 		}
    719. 

  719. 

  720. 		if (mthca_map_reg(dev, (pci_resource_len(dev->pdev, 0) - 1) &
    721. 

  721. 				  dev->fw.arbel.eq_set_ci_base,
    722. 

  722. 				  MTHCA_EQ_SET_CI_SIZE,
    723. 

  723. 				  &dev->eq_regs.arbel.eq_set_ci_base)) {
    724. 

  724. 			mthca_err(dev, "Couldn't map interrupt clear register, "
    725. 

  725. 				  "aborting.\n");
    726. 

  726. 			mthca_unmap_reg(dev, ((pci_resource_len(dev->pdev, 0) - 1) &
    727. 

  727. 					      dev->fw.arbel.eq_arm_base) + 4, 4,
    728. 

  728. 					dev->eq_regs.arbel.eq_arm);
    729. 

  729. 			mthca_unmap_reg(dev, (pci_resource_len(dev->pdev, 0) - 1) &
    730. 

  730. 					dev->fw.arbel.clr_int_base, MTHCA_CLR_INT_SIZE,
    731. 

  731. 					dev->clr_base);
    732. 

  732. 			return -ENOMEM;
    733. 

  733. 		}
    734. 

  734. 	} else {
    735. 

  735. 		if (mthca_map_reg(dev, MTHCA_CLR_INT_BASE, MTHCA_CLR_INT_SIZE,
    736. 

  736. 				  &dev->clr_base)) {
    737. 

  737. 			mthca_err(dev, "Couldn't map interrupt clear register, "
    738. 

  738. 				  "aborting.\n");
    739. 

  739. 			return -ENOMEM;
    740. 

  740. 		}
    741. 

  741. 

  742. 		if (mthca_map_reg(dev, MTHCA_ECR_BASE,
    743. 

  743. 				  MTHCA_ECR_SIZE + MTHCA_ECR_CLR_SIZE,
    744. 

  744. 				  &dev->eq_regs.tavor.ecr_base)) {
    745. 

  745. 			mthca_err(dev, "Couldn't map ecr register, "
    746. 

  746. 				  "aborting.\n");
    747. 

  747. 			mthca_unmap_reg(dev, MTHCA_CLR_INT_BASE, MTHCA_CLR_INT_SIZE,
    748. 

  748. 					dev->clr_base);
    749. 

  749. 			return -ENOMEM;
    750. 

  750. 		}
    751. 

  751. 	}
    752. 

  752. 

  753. 	return 0;
    754. 

  754. 

  755. }
    756. 

  756. 

  757. static void __devexit mthca_unmap_eq_regs(struct mthca_dev *dev)
    758. 

  758. {
    759. 

  759. 	if (dev->hca_type == ARBEL_NATIVE) {
    760. 

  760. 		mthca_unmap_reg(dev, (pci_resource_len(dev->pdev, 0) - 1) &
    761. 

  761. 				dev->fw.arbel.eq_set_ci_base,
    762. 

  762. 				MTHCA_EQ_SET_CI_SIZE,
    763. 

  763. 				dev->eq_regs.arbel.eq_set_ci_base);
    764. 

  764. 		mthca_unmap_reg(dev, ((pci_resource_len(dev->pdev, 0) - 1) &
    765. 

  765. 				      dev->fw.arbel.eq_arm_base) + 4, 4,
    766. 

  766. 				dev->eq_regs.arbel.eq_arm);
    767. 

  767. 		mthca_unmap_reg(dev, (pci_resource_len(dev->pdev, 0) - 1) &
    768. 

  768. 				dev->fw.arbel.clr_int_base, MTHCA_CLR_INT_SIZE,
    769. 

  769. 				dev->clr_base);
    770. 

  770. 	} else {
    771. 

  771. 		mthca_unmap_reg(dev, MTHCA_ECR_BASE,
    772. 

  772. 				MTHCA_ECR_SIZE + MTHCA_ECR_CLR_SIZE,
    773. 

  773. 				dev->eq_regs.tavor.ecr_base);
    774. 

  774. 		mthca_unmap_reg(dev, MTHCA_CLR_INT_BASE, MTHCA_CLR_INT_SIZE,
    775. 

  775. 				dev->clr_base);
    776. 

  776. 	}
    777. 

  777. }
    778. 

  778. 

  779. int __devinit mthca_map_eq_icm(struct mthca_dev *dev, u64 icm_virt)
    780. 

  780. {
    781. 

  781. 	int ret;
    782. 

  782. 	u8 status;
    783. 

  783. 

  784. 	/*
    785. 

  785. 	 * We assume that mapping one page is enough for the whole EQ
    786. 

  786. 	 * context table.  This is fine with all current HCAs, because
    787. 

  787. 	 * we only use 32 EQs and each EQ uses 32 bytes of context
    788. 

  788. 	 * memory, or 1 KB total.
    789. 

  789. 	 */
    790. 

  790. 	dev->eq_table.icm_virt = icm_virt;
    791. 

  791. 	dev->eq_table.icm_page = alloc_page(GFP_HIGHUSER);
    792. 

  792. 	if (!dev->eq_table.icm_page)
    793. 

  793. 		return -ENOMEM;
    794. 

  794. 	dev->eq_table.icm_dma  = pci_map_page(dev->pdev, dev->eq_table.icm_page, 0,
    795. 

  795. 					      PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
    796. 

  796. 	if (pci_dma_mapping_error(dev->eq_table.icm_dma)) {
    797. 

  797. 		__free_page(dev->eq_table.icm_page);
    798. 

  798. 		return -ENOMEM;
    799. 

  799. 	}
    800. 

  800. 

  801. 	ret = mthca_MAP_ICM_page(dev, dev->eq_table.icm_dma, icm_virt, &status);
    802. 

  802. 	if (!ret && status)
    803. 

  803. 		ret = -EINVAL;
    804. 

  804. 	if (ret) {
    805. 

  805. 		pci_unmap_page(dev->pdev, dev->eq_table.icm_dma, PAGE_SIZE,
    806. 

  806. 			       PCI_DMA_BIDIRECTIONAL);
    807. 

  807. 		__free_page(dev->eq_table.icm_page);
    808. 

  808. 	}
    809. 

  809. 

  810. 	return ret;
    811. 

  811. }
    812. 

  812. 

  813. void __devexit mthca_unmap_eq_icm(struct mthca_dev *dev)
    814. 

  814. {
    815. 

  815. 	u8 status;
    816. 

  816. 

  817. 	mthca_UNMAP_ICM(dev, dev->eq_table.icm_virt, PAGE_SIZE / 4096, &status);
    818. 

  818. 	pci_unmap_page(dev->pdev, dev->eq_table.icm_dma, PAGE_SIZE,
    819. 

  819. 		       PCI_DMA_BIDIRECTIONAL);
    820. 

  820. 	__free_page(dev->eq_table.icm_page);
    821. 

  821. }
    822. 

  822. 

  823. int __devinit mthca_init_eq_table(struct mthca_dev *dev)
    824. 

  824. {
    825. 

  825. 	int err;
    826. 

  826. 	u8 status;
    827. 

  827. 	u8 intr;
    828. 

  828. 	int i;
    829. 

  829. 

  830. 	err = mthca_alloc_init(&dev->eq_table.alloc,
    831. 

  831. 			       dev->limits.num_eqs,
    832. 

  832. 			       dev->limits.num_eqs - 1,
    833. 

  833. 			       dev->limits.reserved_eqs);
    834. 

  834. 	if (err)
    835. 

  835. 		return err;
    836. 

  836. 

  837. 	err = mthca_map_eq_regs(dev);
    838. 

  838. 	if (err)
    839. 

  839. 		goto err_out_free;
    840. 

  840. 

  841. 	if (dev->mthca_flags & MTHCA_FLAG_MSI ||
    842. 

  842. 	    dev->mthca_flags & MTHCA_FLAG_MSI_X) {
    843. 

  843. 		dev->eq_table.clr_mask = 0;
    844. 

  844. 	} else {
    845. 

  845. 		dev->eq_table.clr_mask =
    846. 

  846. 			swab32(1 << (dev->eq_table.inta_pin & 31));
    847. 

  847. 		dev->eq_table.clr_int  = dev->clr_base +
    848. 

  848. 			(dev->eq_table.inta_pin < 31 ? 4 : 0);
    849. 

  849. 	}
    850. 

  850. 

  851. 	dev->eq_table.arm_mask = 0;
    852. 

  852. 

  853. 	intr = (dev->mthca_flags & MTHCA_FLAG_MSI) ?
    854. 

  854. 		128 : dev->eq_table.inta_pin;
    855. 

  855. 

  856. 	err = mthca_create_eq(dev, dev->limits.num_cqs,
    857. 

  857. 			      (dev->mthca_flags & MTHCA_FLAG_MSI_X) ? 128 : intr,
    858. 

  858. 			      &dev->eq_table.eq[MTHCA_EQ_COMP]);
    859. 

  859. 	if (err)
    860. 

  860. 		goto err_out_unmap;
    861. 

  861. 

  862. 	err = mthca_create_eq(dev, MTHCA_NUM_ASYNC_EQE,
    863. 

  863. 			      (dev->mthca_flags & MTHCA_FLAG_MSI_X) ? 129 : intr,
    864. 

  864. 			      &dev->eq_table.eq[MTHCA_EQ_ASYNC]);
    865. 

  865. 	if (err)
    866. 

  866. 		goto err_out_comp;
    867. 

  867. 

  868. 	err = mthca_create_eq(dev, MTHCA_NUM_CMD_EQE,
    869. 

  869. 			      (dev->mthca_flags & MTHCA_FLAG_MSI_X) ? 130 : intr,
    870. 

  870. 			      &dev->eq_table.eq[MTHCA_EQ_CMD]);
    871. 

  871. 	if (err)
    872. 

  872. 		goto err_out_async;
    873. 

  873. 

  874. 	if (dev->mthca_flags & MTHCA_FLAG_MSI_X) {
    875. 

  875. 		static const char *eq_name[] = {
    876. 

  876. 			[MTHCA_EQ_COMP]  = DRV_NAME " (comp)",
    877. 

  877. 			[MTHCA_EQ_ASYNC] = DRV_NAME " (async)",
    878. 

  878. 			[MTHCA_EQ_CMD]   = DRV_NAME " (cmd)"
    879. 

  879. 		};
    880. 

  880. 

  881. 		for (i = 0; i < MTHCA_NUM_EQ; ++i) {
    882. 

  882. 			err = request_irq(dev->eq_table.eq[i].msi_x_vector,
    883. 

  883. 					  dev->hca_type == ARBEL_NATIVE ?
    884. 

  884. 					  mthca_arbel_msi_x_interrupt :
    885. 

  885. 					  mthca_tavor_msi_x_interrupt,
    886. 

  886. 					  0, eq_name[i], dev->eq_table.eq + i);
    887. 

  887. 			if (err)
    888. 

  888. 				goto err_out_cmd;
    889. 

  889. 			dev->eq_table.eq[i].have_irq = 1;
    890. 

  890. 		}
    891. 

  891. 	} else {
    892. 

  892. 		err = request_irq(dev->pdev->irq,
    893. 

  893. 				  dev->hca_type == ARBEL_NATIVE ?
    894. 

  894. 				  mthca_arbel_interrupt :
    895. 

  895. 				  mthca_tavor_interrupt,
    896. 

  896. 				  SA_SHIRQ, DRV_NAME, dev);
    897. 

  897. 		if (err)
    898. 

  898. 			goto err_out_cmd;
    899. 

  899. 		dev->eq_table.have_irq = 1;
    900. 

  900. 	}
    901. 

  901. 

  902. 	err = mthca_MAP_EQ(dev, async_mask(dev),
    903. 

  903. 			   0, dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn, &status);
    904. 

  904. 	if (err)
    905. 

  905. 		mthca_warn(dev, "MAP_EQ for async EQ %d failed (%d)\n",
    906. 

  906. 			   dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn, err);
    907. 

  907. 	if (status)
    908. 

  908. 		mthca_warn(dev, "MAP_EQ for async EQ %d returned status 0x%02x\n",
    909. 

  909. 			   dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn, status);
    910. 

  910. 

  911. 	err = mthca_MAP_EQ(dev, MTHCA_CMD_EVENT_MASK,
    912. 

  912. 			   0, dev->eq_table.eq[MTHCA_EQ_CMD].eqn, &status);
    913. 

  913. 	if (err)
    914. 

  914. 		mthca_warn(dev, "MAP_EQ for cmd EQ %d failed (%d)\n",
    915. 

  915. 			   dev->eq_table.eq[MTHCA_EQ_CMD].eqn, err);
    916. 

  916. 	if (status)
    917. 

  917. 		mthca_warn(dev, "MAP_EQ for cmd EQ %d returned status 0x%02x\n",
    918. 

  918. 			   dev->eq_table.eq[MTHCA_EQ_CMD].eqn, status);
    919. 

  919. 

  920. 	for (i = 0; i < MTHCA_EQ_CMD; ++i)
    921. 

  921. 		if (dev->hca_type == ARBEL_NATIVE)
    922. 

  922. 			arbel_eq_req_not(dev, dev->eq_table.eq[i].eqn_mask);
    923. 

  923. 		else
    924. 

  924. 			tavor_eq_req_not(dev, dev->eq_table.eq[i].eqn);
    925. 

  925. 

  926. 	return 0;
    927. 

  927. 

  928. err_out_cmd:
    929. 

  929. 	mthca_free_irqs(dev);
    930. 

  930. 	mthca_free_eq(dev, &dev->eq_table.eq[MTHCA_EQ_CMD]);
    931. 

  931. 

  932. err_out_async:
    933. 

  933. 	mthca_free_eq(dev, &dev->eq_table.eq[MTHCA_EQ_ASYNC]);
    934. 

  934. 

  935. err_out_comp:
    936. 

  936. 	mthca_free_eq(dev, &dev->eq_table.eq[MTHCA_EQ_COMP]);
    937. 

  937. 

  938. err_out_unmap:
    939. 

  939. 	mthca_unmap_eq_regs(dev);
    940. 

  940. 

  941. err_out_free:
    942. 

  942. 	mthca_alloc_cleanup(&dev->eq_table.alloc);
    943. 

  943. 	return err;
    944. 

  944. }
    945. 

  945. 

  946. void __devexit mthca_cleanup_eq_table(struct mthca_dev *dev)
    947. 

  947. {
    948. 

  948. 	u8 status;
    949. 

  949. 	int i;
    950. 

  950. 

  951. 	mthca_free_irqs(dev);
    952. 

  952. 

  953. 	mthca_MAP_EQ(dev, async_mask(dev),
    954. 

  954. 		     1, dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn, &status);
    955. 

  955. 	mthca_MAP_EQ(dev, MTHCA_CMD_EVENT_MASK,
    956. 

  956. 		     1, dev->eq_table.eq[MTHCA_EQ_CMD].eqn, &status);
    957. 

  957. 

  958. 	for (i = 0; i < MTHCA_NUM_EQ; ++i)
    959. 

  959. 		mthca_free_eq(dev, &dev->eq_table.eq[i]);
    960. 

  960. 

  961. 	mthca_unmap_eq_regs(dev);
    962. 

  962. 

  963. 	mthca_alloc_cleanup(&dev->eq_table.alloc);
    964. 

  964. }
    965.