PageRenderTime 116ms CodeModel.GetById 31ms RepoModel.GetById 1ms app.codeStats 0ms

/drivers/infiniband/hw/cxgb4/cm.c

https://bitbucket.org/cresqo/cm7-p500-kernel
C | 2376 lines | 1917 code | 279 blank | 180 comment | 181 complexity | e1f1177c8a14e73d09e77e80a03f78a6 MD5 | raw file
Possible License(s): LGPL-2.0, AGPL-1.0, GPL-2.0
  1. /*
  2. * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved.
  3. *
  4. * This software is available to you under a choice of one of two
  5. * licenses. You may choose to be licensed under the terms of the GNU
  6. * General Public License (GPL) Version 2, available from the file
  7. * COPYING in the main directory of this source tree, or the
  8. * OpenIB.org BSD license below:
  9. *
  10. * Redistribution and use in source and binary forms, with or
  11. * without modification, are permitted provided that the following
  12. * conditions are met:
  13. *
  14. * - Redistributions of source code must retain the above
  15. * copyright notice, this list of conditions and the following
  16. * disclaimer.
  17. *
  18. * - Redistributions in binary form must reproduce the above
  19. * copyright notice, this list of conditions and the following
  20. * disclaimer in the documentation and/or other materials
  21. * provided with the distribution.
  22. *
  23. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30. * SOFTWARE.
  31. */
  32. #include <linux/module.h>
  33. #include <linux/list.h>
  34. #include <linux/workqueue.h>
  35. #include <linux/skbuff.h>
  36. #include <linux/timer.h>
  37. #include <linux/notifier.h>
  38. #include <linux/inetdevice.h>
  39. #include <linux/ip.h>
  40. #include <linux/tcp.h>
  41. #include <net/neighbour.h>
  42. #include <net/netevent.h>
  43. #include <net/route.h>
  44. #include "iw_cxgb4.h"
  45. static char *states[] = {
  46. "idle",
  47. "listen",
  48. "connecting",
  49. "mpa_wait_req",
  50. "mpa_req_sent",
  51. "mpa_req_rcvd",
  52. "mpa_rep_sent",
  53. "fpdu_mode",
  54. "aborting",
  55. "closing",
  56. "moribund",
  57. "dead",
  58. NULL,
  59. };
  60. int c4iw_max_read_depth = 8;
  61. module_param(c4iw_max_read_depth, int, 0644);
  62. MODULE_PARM_DESC(c4iw_max_read_depth, "Per-connection max ORD/IRD (default=8)");
  63. static int enable_tcp_timestamps;
  64. module_param(enable_tcp_timestamps, int, 0644);
  65. MODULE_PARM_DESC(enable_tcp_timestamps, "Enable tcp timestamps (default=0)");
  66. static int enable_tcp_sack;
  67. module_param(enable_tcp_sack, int, 0644);
  68. MODULE_PARM_DESC(enable_tcp_sack, "Enable tcp SACK (default=0)");
  69. static int enable_tcp_window_scaling = 1;
  70. module_param(enable_tcp_window_scaling, int, 0644);
  71. MODULE_PARM_DESC(enable_tcp_window_scaling,
  72. "Enable tcp window scaling (default=1)");
  73. int c4iw_debug;
  74. module_param(c4iw_debug, int, 0644);
  75. MODULE_PARM_DESC(c4iw_debug, "Enable debug logging (default=0)");
  76. static int peer2peer;
  77. module_param(peer2peer, int, 0644);
  78. MODULE_PARM_DESC(peer2peer, "Support peer2peer ULPs (default=0)");
  79. static int p2p_type = FW_RI_INIT_P2PTYPE_READ_REQ;
  80. module_param(p2p_type, int, 0644);
  81. MODULE_PARM_DESC(p2p_type, "RDMAP opcode to use for the RTR message: "
  82. "1=RDMA_READ 0=RDMA_WRITE (default 1)");
  83. static int ep_timeout_secs = 60;
  84. module_param(ep_timeout_secs, int, 0644);
  85. MODULE_PARM_DESC(ep_timeout_secs, "CM Endpoint operation timeout "
  86. "in seconds (default=60)");
  87. static int mpa_rev = 1;
  88. module_param(mpa_rev, int, 0644);
  89. MODULE_PARM_DESC(mpa_rev, "MPA Revision, 0 supports amso1100, "
  90. "1 is spec compliant. (default=1)");
  91. static int markers_enabled;
  92. module_param(markers_enabled, int, 0644);
  93. MODULE_PARM_DESC(markers_enabled, "Enable MPA MARKERS (default(0)=disabled)");
  94. static int crc_enabled = 1;
  95. module_param(crc_enabled, int, 0644);
  96. MODULE_PARM_DESC(crc_enabled, "Enable MPA CRC (default(1)=enabled)");
  97. static int rcv_win = 256 * 1024;
  98. module_param(rcv_win, int, 0644);
  99. MODULE_PARM_DESC(rcv_win, "TCP receive window in bytes (default=256KB)");
  100. static int snd_win = 32 * 1024;
  101. module_param(snd_win, int, 0644);
  102. MODULE_PARM_DESC(snd_win, "TCP send window in bytes (default=32KB)");
  103. static struct workqueue_struct *workq;
  104. static struct sk_buff_head rxq;
  105. static struct sk_buff *get_skb(struct sk_buff *skb, int len, gfp_t gfp);
  106. static void ep_timeout(unsigned long arg);
  107. static void connect_reply_upcall(struct c4iw_ep *ep, int status);
  108. static LIST_HEAD(timeout_list);
  109. static spinlock_t timeout_lock;
  110. static void start_ep_timer(struct c4iw_ep *ep)
  111. {
  112. PDBG("%s ep %p\n", __func__, ep);
  113. if (timer_pending(&ep->timer)) {
  114. PDBG("%s stopped / restarted timer ep %p\n", __func__, ep);
  115. del_timer_sync(&ep->timer);
  116. } else
  117. c4iw_get_ep(&ep->com);
  118. ep->timer.expires = jiffies + ep_timeout_secs * HZ;
  119. ep->timer.data = (unsigned long)ep;
  120. ep->timer.function = ep_timeout;
  121. add_timer(&ep->timer);
  122. }
  123. static void stop_ep_timer(struct c4iw_ep *ep)
  124. {
  125. PDBG("%s ep %p\n", __func__, ep);
  126. if (!timer_pending(&ep->timer)) {
  127. printk(KERN_ERR "%s timer stopped when its not running! "
  128. "ep %p state %u\n", __func__, ep, ep->com.state);
  129. WARN_ON(1);
  130. return;
  131. }
  132. del_timer_sync(&ep->timer);
  133. c4iw_put_ep(&ep->com);
  134. }
  135. static int c4iw_l2t_send(struct c4iw_rdev *rdev, struct sk_buff *skb,
  136. struct l2t_entry *l2e)
  137. {
  138. int error = 0;
  139. if (c4iw_fatal_error(rdev)) {
  140. kfree_skb(skb);
  141. PDBG("%s - device in error state - dropping\n", __func__);
  142. return -EIO;
  143. }
  144. error = cxgb4_l2t_send(rdev->lldi.ports[0], skb, l2e);
  145. if (error < 0)
  146. kfree_skb(skb);
  147. return error;
  148. }
  149. int c4iw_ofld_send(struct c4iw_rdev *rdev, struct sk_buff *skb)
  150. {
  151. int error = 0;
  152. if (c4iw_fatal_error(rdev)) {
  153. kfree_skb(skb);
  154. PDBG("%s - device in error state - dropping\n", __func__);
  155. return -EIO;
  156. }
  157. error = cxgb4_ofld_send(rdev->lldi.ports[0], skb);
  158. if (error < 0)
  159. kfree_skb(skb);
  160. return error;
  161. }
  162. static void release_tid(struct c4iw_rdev *rdev, u32 hwtid, struct sk_buff *skb)
  163. {
  164. struct cpl_tid_release *req;
  165. skb = get_skb(skb, sizeof *req, GFP_KERNEL);
  166. if (!skb)
  167. return;
  168. req = (struct cpl_tid_release *) skb_put(skb, sizeof(*req));
  169. INIT_TP_WR(req, hwtid);
  170. OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_TID_RELEASE, hwtid));
  171. set_wr_txq(skb, CPL_PRIORITY_SETUP, 0);
  172. c4iw_ofld_send(rdev, skb);
  173. return;
  174. }
  175. static void set_emss(struct c4iw_ep *ep, u16 opt)
  176. {
  177. ep->emss = ep->com.dev->rdev.lldi.mtus[GET_TCPOPT_MSS(opt)] - 40;
  178. ep->mss = ep->emss;
  179. if (GET_TCPOPT_TSTAMP(opt))
  180. ep->emss -= 12;
  181. if (ep->emss < 128)
  182. ep->emss = 128;
  183. PDBG("%s mss_idx %u mss %u emss=%u\n", __func__, GET_TCPOPT_MSS(opt),
  184. ep->mss, ep->emss);
  185. }
  186. static enum c4iw_ep_state state_read(struct c4iw_ep_common *epc)
  187. {
  188. unsigned long flags;
  189. enum c4iw_ep_state state;
  190. spin_lock_irqsave(&epc->lock, flags);
  191. state = epc->state;
  192. spin_unlock_irqrestore(&epc->lock, flags);
  193. return state;
  194. }
  195. static void __state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new)
  196. {
  197. epc->state = new;
  198. }
  199. static void state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new)
  200. {
  201. unsigned long flags;
  202. spin_lock_irqsave(&epc->lock, flags);
  203. PDBG("%s - %s -> %s\n", __func__, states[epc->state], states[new]);
  204. __state_set(epc, new);
  205. spin_unlock_irqrestore(&epc->lock, flags);
  206. return;
  207. }
  208. static void *alloc_ep(int size, gfp_t gfp)
  209. {
  210. struct c4iw_ep_common *epc;
  211. epc = kzalloc(size, gfp);
  212. if (epc) {
  213. kref_init(&epc->kref);
  214. spin_lock_init(&epc->lock);
  215. init_waitqueue_head(&epc->waitq);
  216. }
  217. PDBG("%s alloc ep %p\n", __func__, epc);
  218. return epc;
  219. }
  220. void _c4iw_free_ep(struct kref *kref)
  221. {
  222. struct c4iw_ep *ep;
  223. ep = container_of(kref, struct c4iw_ep, com.kref);
  224. PDBG("%s ep %p state %s\n", __func__, ep, states[state_read(&ep->com)]);
  225. if (test_bit(RELEASE_RESOURCES, &ep->com.flags)) {
  226. cxgb4_remove_tid(ep->com.dev->rdev.lldi.tids, 0, ep->hwtid);
  227. dst_release(ep->dst);
  228. cxgb4_l2t_release(ep->l2t);
  229. }
  230. kfree(ep);
  231. }
  232. static void release_ep_resources(struct c4iw_ep *ep)
  233. {
  234. set_bit(RELEASE_RESOURCES, &ep->com.flags);
  235. c4iw_put_ep(&ep->com);
  236. }
  237. static int status2errno(int status)
  238. {
  239. switch (status) {
  240. case CPL_ERR_NONE:
  241. return 0;
  242. case CPL_ERR_CONN_RESET:
  243. return -ECONNRESET;
  244. case CPL_ERR_ARP_MISS:
  245. return -EHOSTUNREACH;
  246. case CPL_ERR_CONN_TIMEDOUT:
  247. return -ETIMEDOUT;
  248. case CPL_ERR_TCAM_FULL:
  249. return -ENOMEM;
  250. case CPL_ERR_CONN_EXIST:
  251. return -EADDRINUSE;
  252. default:
  253. return -EIO;
  254. }
  255. }
  256. /*
  257. * Try and reuse skbs already allocated...
  258. */
  259. static struct sk_buff *get_skb(struct sk_buff *skb, int len, gfp_t gfp)
  260. {
  261. if (skb && !skb_is_nonlinear(skb) && !skb_cloned(skb)) {
  262. skb_trim(skb, 0);
  263. skb_get(skb);
  264. skb_reset_transport_header(skb);
  265. } else {
  266. skb = alloc_skb(len, gfp);
  267. }
  268. return skb;
  269. }
  270. static struct rtable *find_route(struct c4iw_dev *dev, __be32 local_ip,
  271. __be32 peer_ip, __be16 local_port,
  272. __be16 peer_port, u8 tos)
  273. {
  274. struct rtable *rt;
  275. struct flowi fl = {
  276. .oif = 0,
  277. .nl_u = {
  278. .ip4_u = {
  279. .daddr = peer_ip,
  280. .saddr = local_ip,
  281. .tos = tos}
  282. },
  283. .proto = IPPROTO_TCP,
  284. .uli_u = {
  285. .ports = {
  286. .sport = local_port,
  287. .dport = peer_port}
  288. }
  289. };
  290. if (ip_route_output_flow(&init_net, &rt, &fl, NULL, 0))
  291. return NULL;
  292. return rt;
  293. }
  294. static void arp_failure_discard(void *handle, struct sk_buff *skb)
  295. {
  296. PDBG("%s c4iw_dev %p\n", __func__, handle);
  297. kfree_skb(skb);
  298. }
  299. /*
  300. * Handle an ARP failure for an active open.
  301. */
  302. static void act_open_req_arp_failure(void *handle, struct sk_buff *skb)
  303. {
  304. printk(KERN_ERR MOD "ARP failure duing connect\n");
  305. kfree_skb(skb);
  306. }
  307. /*
  308. * Handle an ARP failure for a CPL_ABORT_REQ. Change it into a no RST variant
  309. * and send it along.
  310. */
  311. static void abort_arp_failure(void *handle, struct sk_buff *skb)
  312. {
  313. struct c4iw_rdev *rdev = handle;
  314. struct cpl_abort_req *req = cplhdr(skb);
  315. PDBG("%s rdev %p\n", __func__, rdev);
  316. req->cmd = CPL_ABORT_NO_RST;
  317. c4iw_ofld_send(rdev, skb);
  318. }
  319. static void send_flowc(struct c4iw_ep *ep, struct sk_buff *skb)
  320. {
  321. unsigned int flowclen = 80;
  322. struct fw_flowc_wr *flowc;
  323. int i;
  324. skb = get_skb(skb, flowclen, GFP_KERNEL);
  325. flowc = (struct fw_flowc_wr *)__skb_put(skb, flowclen);
  326. flowc->op_to_nparams = cpu_to_be32(FW_WR_OP(FW_FLOWC_WR) |
  327. FW_FLOWC_WR_NPARAMS(8));
  328. flowc->flowid_len16 = cpu_to_be32(FW_WR_LEN16(DIV_ROUND_UP(flowclen,
  329. 16)) | FW_WR_FLOWID(ep->hwtid));
  330. flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN;
  331. flowc->mnemval[0].val = cpu_to_be32(0);
  332. flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH;
  333. flowc->mnemval[1].val = cpu_to_be32(ep->tx_chan);
  334. flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT;
  335. flowc->mnemval[2].val = cpu_to_be32(ep->tx_chan);
  336. flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID;
  337. flowc->mnemval[3].val = cpu_to_be32(ep->rss_qid);
  338. flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT;
  339. flowc->mnemval[4].val = cpu_to_be32(ep->snd_seq);
  340. flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT;
  341. flowc->mnemval[5].val = cpu_to_be32(ep->rcv_seq);
  342. flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF;
  343. flowc->mnemval[6].val = cpu_to_be32(snd_win);
  344. flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS;
  345. flowc->mnemval[7].val = cpu_to_be32(ep->emss);
  346. /* Pad WR to 16 byte boundary */
  347. flowc->mnemval[8].mnemonic = 0;
  348. flowc->mnemval[8].val = 0;
  349. for (i = 0; i < 9; i++) {
  350. flowc->mnemval[i].r4[0] = 0;
  351. flowc->mnemval[i].r4[1] = 0;
  352. flowc->mnemval[i].r4[2] = 0;
  353. }
  354. set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx);
  355. c4iw_ofld_send(&ep->com.dev->rdev, skb);
  356. }
  357. static int send_halfclose(struct c4iw_ep *ep, gfp_t gfp)
  358. {
  359. struct cpl_close_con_req *req;
  360. struct sk_buff *skb;
  361. int wrlen = roundup(sizeof *req, 16);
  362. PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
  363. skb = get_skb(NULL, wrlen, gfp);
  364. if (!skb) {
  365. printk(KERN_ERR MOD "%s - failed to alloc skb\n", __func__);
  366. return -ENOMEM;
  367. }
  368. set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx);
  369. t4_set_arp_err_handler(skb, NULL, arp_failure_discard);
  370. req = (struct cpl_close_con_req *) skb_put(skb, wrlen);
  371. memset(req, 0, wrlen);
  372. INIT_TP_WR(req, ep->hwtid);
  373. OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_CON_REQ,
  374. ep->hwtid));
  375. return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t);
  376. }
  377. static int send_abort(struct c4iw_ep *ep, struct sk_buff *skb, gfp_t gfp)
  378. {
  379. struct cpl_abort_req *req;
  380. int wrlen = roundup(sizeof *req, 16);
  381. PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
  382. skb = get_skb(skb, wrlen, gfp);
  383. if (!skb) {
  384. printk(KERN_ERR MOD "%s - failed to alloc skb.\n",
  385. __func__);
  386. return -ENOMEM;
  387. }
  388. set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx);
  389. t4_set_arp_err_handler(skb, &ep->com.dev->rdev, abort_arp_failure);
  390. req = (struct cpl_abort_req *) skb_put(skb, wrlen);
  391. memset(req, 0, wrlen);
  392. INIT_TP_WR(req, ep->hwtid);
  393. OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ, ep->hwtid));
  394. req->cmd = CPL_ABORT_SEND_RST;
  395. return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t);
  396. }
  397. static int send_connect(struct c4iw_ep *ep)
  398. {
  399. struct cpl_act_open_req *req;
  400. struct sk_buff *skb;
  401. u64 opt0;
  402. u32 opt2;
  403. unsigned int mtu_idx;
  404. int wscale;
  405. int wrlen = roundup(sizeof *req, 16);
  406. PDBG("%s ep %p atid %u\n", __func__, ep, ep->atid);
  407. skb = get_skb(NULL, wrlen, GFP_KERNEL);
  408. if (!skb) {
  409. printk(KERN_ERR MOD "%s - failed to alloc skb.\n",
  410. __func__);
  411. return -ENOMEM;
  412. }
  413. set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->txq_idx);
  414. cxgb4_best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx);
  415. wscale = compute_wscale(rcv_win);
  416. opt0 = KEEP_ALIVE(1) |
  417. WND_SCALE(wscale) |
  418. MSS_IDX(mtu_idx) |
  419. L2T_IDX(ep->l2t->idx) |
  420. TX_CHAN(ep->tx_chan) |
  421. SMAC_SEL(ep->smac_idx) |
  422. DSCP(ep->tos) |
  423. RCV_BUFSIZ(rcv_win>>10);
  424. opt2 = RX_CHANNEL(0) |
  425. RSS_QUEUE_VALID | RSS_QUEUE(ep->rss_qid);
  426. if (enable_tcp_timestamps)
  427. opt2 |= TSTAMPS_EN(1);
  428. if (enable_tcp_sack)
  429. opt2 |= SACK_EN(1);
  430. if (wscale && enable_tcp_window_scaling)
  431. opt2 |= WND_SCALE_EN(1);
  432. t4_set_arp_err_handler(skb, NULL, act_open_req_arp_failure);
  433. req = (struct cpl_act_open_req *) skb_put(skb, wrlen);
  434. INIT_TP_WR(req, 0);
  435. OPCODE_TID(req) = cpu_to_be32(
  436. MK_OPCODE_TID(CPL_ACT_OPEN_REQ, ((ep->rss_qid<<14)|ep->atid)));
  437. req->local_port = ep->com.local_addr.sin_port;
  438. req->peer_port = ep->com.remote_addr.sin_port;
  439. req->local_ip = ep->com.local_addr.sin_addr.s_addr;
  440. req->peer_ip = ep->com.remote_addr.sin_addr.s_addr;
  441. req->opt0 = cpu_to_be64(opt0);
  442. req->params = 0;
  443. req->opt2 = cpu_to_be32(opt2);
  444. return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t);
  445. }
  446. static void send_mpa_req(struct c4iw_ep *ep, struct sk_buff *skb)
  447. {
  448. int mpalen, wrlen;
  449. struct fw_ofld_tx_data_wr *req;
  450. struct mpa_message *mpa;
  451. PDBG("%s ep %p tid %u pd_len %d\n", __func__, ep, ep->hwtid, ep->plen);
  452. BUG_ON(skb_cloned(skb));
  453. mpalen = sizeof(*mpa) + ep->plen;
  454. wrlen = roundup(mpalen + sizeof *req, 16);
  455. skb = get_skb(skb, wrlen, GFP_KERNEL);
  456. if (!skb) {
  457. connect_reply_upcall(ep, -ENOMEM);
  458. return;
  459. }
  460. set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx);
  461. req = (struct fw_ofld_tx_data_wr *)skb_put(skb, wrlen);
  462. memset(req, 0, wrlen);
  463. req->op_to_immdlen = cpu_to_be32(
  464. FW_WR_OP(FW_OFLD_TX_DATA_WR) |
  465. FW_WR_COMPL(1) |
  466. FW_WR_IMMDLEN(mpalen));
  467. req->flowid_len16 = cpu_to_be32(
  468. FW_WR_FLOWID(ep->hwtid) |
  469. FW_WR_LEN16(wrlen >> 4));
  470. req->plen = cpu_to_be32(mpalen);
  471. req->tunnel_to_proxy = cpu_to_be32(
  472. FW_OFLD_TX_DATA_WR_FLUSH(1) |
  473. FW_OFLD_TX_DATA_WR_SHOVE(1));
  474. mpa = (struct mpa_message *)(req + 1);
  475. memcpy(mpa->key, MPA_KEY_REQ, sizeof(mpa->key));
  476. mpa->flags = (crc_enabled ? MPA_CRC : 0) |
  477. (markers_enabled ? MPA_MARKERS : 0);
  478. mpa->private_data_size = htons(ep->plen);
  479. mpa->revision = mpa_rev;
  480. if (ep->plen)
  481. memcpy(mpa->private_data, ep->mpa_pkt + sizeof(*mpa), ep->plen);
  482. /*
  483. * Reference the mpa skb. This ensures the data area
  484. * will remain in memory until the hw acks the tx.
  485. * Function fw4_ack() will deref it.
  486. */
  487. skb_get(skb);
  488. t4_set_arp_err_handler(skb, NULL, arp_failure_discard);
  489. BUG_ON(ep->mpa_skb);
  490. ep->mpa_skb = skb;
  491. c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t);
  492. start_ep_timer(ep);
  493. state_set(&ep->com, MPA_REQ_SENT);
  494. ep->mpa_attr.initiator = 1;
  495. return;
  496. }
  497. static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen)
  498. {
  499. int mpalen, wrlen;
  500. struct fw_ofld_tx_data_wr *req;
  501. struct mpa_message *mpa;
  502. struct sk_buff *skb;
  503. PDBG("%s ep %p tid %u pd_len %d\n", __func__, ep, ep->hwtid, ep->plen);
  504. mpalen = sizeof(*mpa) + plen;
  505. wrlen = roundup(mpalen + sizeof *req, 16);
  506. skb = get_skb(NULL, wrlen, GFP_KERNEL);
  507. if (!skb) {
  508. printk(KERN_ERR MOD "%s - cannot alloc skb!\n", __func__);
  509. return -ENOMEM;
  510. }
  511. set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx);
  512. req = (struct fw_ofld_tx_data_wr *)skb_put(skb, wrlen);
  513. memset(req, 0, wrlen);
  514. req->op_to_immdlen = cpu_to_be32(
  515. FW_WR_OP(FW_OFLD_TX_DATA_WR) |
  516. FW_WR_COMPL(1) |
  517. FW_WR_IMMDLEN(mpalen));
  518. req->flowid_len16 = cpu_to_be32(
  519. FW_WR_FLOWID(ep->hwtid) |
  520. FW_WR_LEN16(wrlen >> 4));
  521. req->plen = cpu_to_be32(mpalen);
  522. req->tunnel_to_proxy = cpu_to_be32(
  523. FW_OFLD_TX_DATA_WR_FLUSH(1) |
  524. FW_OFLD_TX_DATA_WR_SHOVE(1));
  525. mpa = (struct mpa_message *)(req + 1);
  526. memset(mpa, 0, sizeof(*mpa));
  527. memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key));
  528. mpa->flags = MPA_REJECT;
  529. mpa->revision = mpa_rev;
  530. mpa->private_data_size = htons(plen);
  531. if (plen)
  532. memcpy(mpa->private_data, pdata, plen);
  533. /*
  534. * Reference the mpa skb again. This ensures the data area
  535. * will remain in memory until the hw acks the tx.
  536. * Function fw4_ack() will deref it.
  537. */
  538. skb_get(skb);
  539. set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx);
  540. t4_set_arp_err_handler(skb, NULL, arp_failure_discard);
  541. BUG_ON(ep->mpa_skb);
  542. ep->mpa_skb = skb;
  543. return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t);
  544. }
  545. static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen)
  546. {
  547. int mpalen, wrlen;
  548. struct fw_ofld_tx_data_wr *req;
  549. struct mpa_message *mpa;
  550. struct sk_buff *skb;
  551. PDBG("%s ep %p tid %u pd_len %d\n", __func__, ep, ep->hwtid, ep->plen);
  552. mpalen = sizeof(*mpa) + plen;
  553. wrlen = roundup(mpalen + sizeof *req, 16);
  554. skb = get_skb(NULL, wrlen, GFP_KERNEL);
  555. if (!skb) {
  556. printk(KERN_ERR MOD "%s - cannot alloc skb!\n", __func__);
  557. return -ENOMEM;
  558. }
  559. set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx);
  560. req = (struct fw_ofld_tx_data_wr *) skb_put(skb, wrlen);
  561. memset(req, 0, wrlen);
  562. req->op_to_immdlen = cpu_to_be32(
  563. FW_WR_OP(FW_OFLD_TX_DATA_WR) |
  564. FW_WR_COMPL(1) |
  565. FW_WR_IMMDLEN(mpalen));
  566. req->flowid_len16 = cpu_to_be32(
  567. FW_WR_FLOWID(ep->hwtid) |
  568. FW_WR_LEN16(wrlen >> 4));
  569. req->plen = cpu_to_be32(mpalen);
  570. req->tunnel_to_proxy = cpu_to_be32(
  571. FW_OFLD_TX_DATA_WR_FLUSH(1) |
  572. FW_OFLD_TX_DATA_WR_SHOVE(1));
  573. mpa = (struct mpa_message *)(req + 1);
  574. memset(mpa, 0, sizeof(*mpa));
  575. memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key));
  576. mpa->flags = (ep->mpa_attr.crc_enabled ? MPA_CRC : 0) |
  577. (markers_enabled ? MPA_MARKERS : 0);
  578. mpa->revision = mpa_rev;
  579. mpa->private_data_size = htons(plen);
  580. if (plen)
  581. memcpy(mpa->private_data, pdata, plen);
  582. /*
  583. * Reference the mpa skb. This ensures the data area
  584. * will remain in memory until the hw acks the tx.
  585. * Function fw4_ack() will deref it.
  586. */
  587. skb_get(skb);
  588. t4_set_arp_err_handler(skb, NULL, arp_failure_discard);
  589. ep->mpa_skb = skb;
  590. state_set(&ep->com, MPA_REP_SENT);
  591. return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t);
  592. }
  593. static int act_establish(struct c4iw_dev *dev, struct sk_buff *skb)
  594. {
  595. struct c4iw_ep *ep;
  596. struct cpl_act_establish *req = cplhdr(skb);
  597. unsigned int tid = GET_TID(req);
  598. unsigned int atid = GET_TID_TID(ntohl(req->tos_atid));
  599. struct tid_info *t = dev->rdev.lldi.tids;
  600. ep = lookup_atid(t, atid);
  601. PDBG("%s ep %p tid %u snd_isn %u rcv_isn %u\n", __func__, ep, tid,
  602. be32_to_cpu(req->snd_isn), be32_to_cpu(req->rcv_isn));
  603. dst_confirm(ep->dst);
  604. /* setup the hwtid for this connection */
  605. ep->hwtid = tid;
  606. cxgb4_insert_tid(t, ep, tid);
  607. ep->snd_seq = be32_to_cpu(req->snd_isn);
  608. ep->rcv_seq = be32_to_cpu(req->rcv_isn);
  609. set_emss(ep, ntohs(req->tcp_opt));
  610. /* dealloc the atid */
  611. cxgb4_free_atid(t, atid);
  612. /* start MPA negotiation */
  613. send_flowc(ep, NULL);
  614. send_mpa_req(ep, skb);
  615. return 0;
  616. }
  617. static void close_complete_upcall(struct c4iw_ep *ep)
  618. {
  619. struct iw_cm_event event;
  620. PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
  621. memset(&event, 0, sizeof(event));
  622. event.event = IW_CM_EVENT_CLOSE;
  623. if (ep->com.cm_id) {
  624. PDBG("close complete delivered ep %p cm_id %p tid %u\n",
  625. ep, ep->com.cm_id, ep->hwtid);
  626. ep->com.cm_id->event_handler(ep->com.cm_id, &event);
  627. ep->com.cm_id->rem_ref(ep->com.cm_id);
  628. ep->com.cm_id = NULL;
  629. ep->com.qp = NULL;
  630. }
  631. }
  632. static int abort_connection(struct c4iw_ep *ep, struct sk_buff *skb, gfp_t gfp)
  633. {
  634. PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
  635. close_complete_upcall(ep);
  636. state_set(&ep->com, ABORTING);
  637. return send_abort(ep, skb, gfp);
  638. }
  639. static void peer_close_upcall(struct c4iw_ep *ep)
  640. {
  641. struct iw_cm_event event;
  642. PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
  643. memset(&event, 0, sizeof(event));
  644. event.event = IW_CM_EVENT_DISCONNECT;
  645. if (ep->com.cm_id) {
  646. PDBG("peer close delivered ep %p cm_id %p tid %u\n",
  647. ep, ep->com.cm_id, ep->hwtid);
  648. ep->com.cm_id->event_handler(ep->com.cm_id, &event);
  649. }
  650. }
  651. static void peer_abort_upcall(struct c4iw_ep *ep)
  652. {
  653. struct iw_cm_event event;
  654. PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
  655. memset(&event, 0, sizeof(event));
  656. event.event = IW_CM_EVENT_CLOSE;
  657. event.status = -ECONNRESET;
  658. if (ep->com.cm_id) {
  659. PDBG("abort delivered ep %p cm_id %p tid %u\n", ep,
  660. ep->com.cm_id, ep->hwtid);
  661. ep->com.cm_id->event_handler(ep->com.cm_id, &event);
  662. ep->com.cm_id->rem_ref(ep->com.cm_id);
  663. ep->com.cm_id = NULL;
  664. ep->com.qp = NULL;
  665. }
  666. }
  667. static void connect_reply_upcall(struct c4iw_ep *ep, int status)
  668. {
  669. struct iw_cm_event event;
  670. PDBG("%s ep %p tid %u status %d\n", __func__, ep, ep->hwtid, status);
  671. memset(&event, 0, sizeof(event));
  672. event.event = IW_CM_EVENT_CONNECT_REPLY;
  673. event.status = status;
  674. event.local_addr = ep->com.local_addr;
  675. event.remote_addr = ep->com.remote_addr;
  676. if ((status == 0) || (status == -ECONNREFUSED)) {
  677. event.private_data_len = ep->plen;
  678. event.private_data = ep->mpa_pkt + sizeof(struct mpa_message);
  679. }
  680. if (ep->com.cm_id) {
  681. PDBG("%s ep %p tid %u status %d\n", __func__, ep,
  682. ep->hwtid, status);
  683. ep->com.cm_id->event_handler(ep->com.cm_id, &event);
  684. }
  685. if (status < 0) {
  686. ep->com.cm_id->rem_ref(ep->com.cm_id);
  687. ep->com.cm_id = NULL;
  688. ep->com.qp = NULL;
  689. }
  690. }
  691. static void connect_request_upcall(struct c4iw_ep *ep)
  692. {
  693. struct iw_cm_event event;
  694. PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
  695. memset(&event, 0, sizeof(event));
  696. event.event = IW_CM_EVENT_CONNECT_REQUEST;
  697. event.local_addr = ep->com.local_addr;
  698. event.remote_addr = ep->com.remote_addr;
  699. event.private_data_len = ep->plen;
  700. event.private_data = ep->mpa_pkt + sizeof(struct mpa_message);
  701. event.provider_data = ep;
  702. if (state_read(&ep->parent_ep->com) != DEAD) {
  703. c4iw_get_ep(&ep->com);
  704. ep->parent_ep->com.cm_id->event_handler(
  705. ep->parent_ep->com.cm_id,
  706. &event);
  707. }
  708. c4iw_put_ep(&ep->parent_ep->com);
  709. ep->parent_ep = NULL;
  710. }
  711. static void established_upcall(struct c4iw_ep *ep)
  712. {
  713. struct iw_cm_event event;
  714. PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
  715. memset(&event, 0, sizeof(event));
  716. event.event = IW_CM_EVENT_ESTABLISHED;
  717. if (ep->com.cm_id) {
  718. PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
  719. ep->com.cm_id->event_handler(ep->com.cm_id, &event);
  720. }
  721. }
  722. static int update_rx_credits(struct c4iw_ep *ep, u32 credits)
  723. {
  724. struct cpl_rx_data_ack *req;
  725. struct sk_buff *skb;
  726. int wrlen = roundup(sizeof *req, 16);
  727. PDBG("%s ep %p tid %u credits %u\n", __func__, ep, ep->hwtid, credits);
  728. skb = get_skb(NULL, wrlen, GFP_KERNEL);
  729. if (!skb) {
  730. printk(KERN_ERR MOD "update_rx_credits - cannot alloc skb!\n");
  731. return 0;
  732. }
  733. req = (struct cpl_rx_data_ack *) skb_put(skb, wrlen);
  734. memset(req, 0, wrlen);
  735. INIT_TP_WR(req, ep->hwtid);
  736. OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK,
  737. ep->hwtid));
  738. req->credit_dack = cpu_to_be32(credits);
  739. set_wr_txq(skb, CPL_PRIORITY_ACK, ep->txq_idx);
  740. c4iw_ofld_send(&ep->com.dev->rdev, skb);
  741. return credits;
  742. }
  743. static void process_mpa_reply(struct c4iw_ep *ep, struct sk_buff *skb)
  744. {
  745. struct mpa_message *mpa;
  746. u16 plen;
  747. struct c4iw_qp_attributes attrs;
  748. enum c4iw_qp_attr_mask mask;
  749. int err;
  750. PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
  751. /*
  752. * Stop mpa timer. If it expired, then the state has
  753. * changed and we bail since ep_timeout already aborted
  754. * the connection.
  755. */
  756. stop_ep_timer(ep);
  757. if (state_read(&ep->com) != MPA_REQ_SENT)
  758. return;
  759. /*
  760. * If we get more than the supported amount of private data
  761. * then we must fail this connection.
  762. */
  763. if (ep->mpa_pkt_len + skb->len > sizeof(ep->mpa_pkt)) {
  764. err = -EINVAL;
  765. goto err;
  766. }
  767. /*
  768. * copy the new data into our accumulation buffer.
  769. */
  770. skb_copy_from_linear_data(skb, &(ep->mpa_pkt[ep->mpa_pkt_len]),
  771. skb->len);
  772. ep->mpa_pkt_len += skb->len;
  773. /*
  774. * if we don't even have the mpa message, then bail.
  775. */
  776. if (ep->mpa_pkt_len < sizeof(*mpa))
  777. return;
  778. mpa = (struct mpa_message *) ep->mpa_pkt;
  779. /* Validate MPA header. */
  780. if (mpa->revision != mpa_rev) {
  781. err = -EPROTO;
  782. goto err;
  783. }
  784. if (memcmp(mpa->key, MPA_KEY_REP, sizeof(mpa->key))) {
  785. err = -EPROTO;
  786. goto err;
  787. }
  788. plen = ntohs(mpa->private_data_size);
  789. /*
  790. * Fail if there's too much private data.
  791. */
  792. if (plen > MPA_MAX_PRIVATE_DATA) {
  793. err = -EPROTO;
  794. goto err;
  795. }
  796. /*
  797. * If plen does not account for pkt size
  798. */
  799. if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) {
  800. err = -EPROTO;
  801. goto err;
  802. }
  803. ep->plen = (u8) plen;
  804. /*
  805. * If we don't have all the pdata yet, then bail.
  806. * We'll continue process when more data arrives.
  807. */
  808. if (ep->mpa_pkt_len < (sizeof(*mpa) + plen))
  809. return;
  810. if (mpa->flags & MPA_REJECT) {
  811. err = -ECONNREFUSED;
  812. goto err;
  813. }
  814. /*
  815. * If we get here we have accumulated the entire mpa
  816. * start reply message including private data. And
  817. * the MPA header is valid.
  818. */
  819. state_set(&ep->com, FPDU_MODE);
  820. ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0;
  821. ep->mpa_attr.recv_marker_enabled = markers_enabled;
  822. ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0;
  823. ep->mpa_attr.version = mpa_rev;
  824. ep->mpa_attr.p2p_type = peer2peer ? p2p_type :
  825. FW_RI_INIT_P2PTYPE_DISABLED;
  826. PDBG("%s - crc_enabled=%d, recv_marker_enabled=%d, "
  827. "xmit_marker_enabled=%d, version=%d\n", __func__,
  828. ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled,
  829. ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version);
  830. attrs.mpa_attr = ep->mpa_attr;
  831. attrs.max_ird = ep->ird;
  832. attrs.max_ord = ep->ord;
  833. attrs.llp_stream_handle = ep;
  834. attrs.next_state = C4IW_QP_STATE_RTS;
  835. mask = C4IW_QP_ATTR_NEXT_STATE |
  836. C4IW_QP_ATTR_LLP_STREAM_HANDLE | C4IW_QP_ATTR_MPA_ATTR |
  837. C4IW_QP_ATTR_MAX_IRD | C4IW_QP_ATTR_MAX_ORD;
  838. /* bind QP and TID with INIT_WR */
  839. err = c4iw_modify_qp(ep->com.qp->rhp,
  840. ep->com.qp, mask, &attrs, 1);
  841. if (err)
  842. goto err;
  843. goto out;
  844. err:
  845. state_set(&ep->com, ABORTING);
  846. send_abort(ep, skb, GFP_KERNEL);
  847. out:
  848. connect_reply_upcall(ep, err);
  849. return;
  850. }
  851. static void process_mpa_request(struct c4iw_ep *ep, struct sk_buff *skb)
  852. {
  853. struct mpa_message *mpa;
  854. u16 plen;
  855. PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
  856. if (state_read(&ep->com) != MPA_REQ_WAIT)
  857. return;
  858. /*
  859. * If we get more than the supported amount of private data
  860. * then we must fail this connection.
  861. */
  862. if (ep->mpa_pkt_len + skb->len > sizeof(ep->mpa_pkt)) {
  863. stop_ep_timer(ep);
  864. abort_connection(ep, skb, GFP_KERNEL);
  865. return;
  866. }
  867. PDBG("%s enter (%s line %u)\n", __func__, __FILE__, __LINE__);
  868. /*
  869. * Copy the new data into our accumulation buffer.
  870. */
  871. skb_copy_from_linear_data(skb, &(ep->mpa_pkt[ep->mpa_pkt_len]),
  872. skb->len);
  873. ep->mpa_pkt_len += skb->len;
  874. /*
  875. * If we don't even have the mpa message, then bail.
  876. * We'll continue process when more data arrives.
  877. */
  878. if (ep->mpa_pkt_len < sizeof(*mpa))
  879. return;
  880. PDBG("%s enter (%s line %u)\n", __func__, __FILE__, __LINE__);
  881. stop_ep_timer(ep);
  882. mpa = (struct mpa_message *) ep->mpa_pkt;
  883. /*
  884. * Validate MPA Header.
  885. */
  886. if (mpa->revision != mpa_rev) {
  887. abort_connection(ep, skb, GFP_KERNEL);
  888. return;
  889. }
  890. if (memcmp(mpa->key, MPA_KEY_REQ, sizeof(mpa->key))) {
  891. abort_connection(ep, skb, GFP_KERNEL);
  892. return;
  893. }
  894. plen = ntohs(mpa->private_data_size);
  895. /*
  896. * Fail if there's too much private data.
  897. */
  898. if (plen > MPA_MAX_PRIVATE_DATA) {
  899. abort_connection(ep, skb, GFP_KERNEL);
  900. return;
  901. }
  902. /*
  903. * If plen does not account for pkt size
  904. */
  905. if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) {
  906. abort_connection(ep, skb, GFP_KERNEL);
  907. return;
  908. }
  909. ep->plen = (u8) plen;
  910. /*
  911. * If we don't have all the pdata yet, then bail.
  912. */
  913. if (ep->mpa_pkt_len < (sizeof(*mpa) + plen))
  914. return;
  915. /*
  916. * If we get here we have accumulated the entire mpa
  917. * start reply message including private data.
  918. */
  919. ep->mpa_attr.initiator = 0;
  920. ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0;
  921. ep->mpa_attr.recv_marker_enabled = markers_enabled;
  922. ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0;
  923. ep->mpa_attr.version = mpa_rev;
  924. ep->mpa_attr.p2p_type = peer2peer ? p2p_type :
  925. FW_RI_INIT_P2PTYPE_DISABLED;
  926. PDBG("%s - crc_enabled=%d, recv_marker_enabled=%d, "
  927. "xmit_marker_enabled=%d, version=%d p2p_type=%d\n", __func__,
  928. ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled,
  929. ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version,
  930. ep->mpa_attr.p2p_type);
  931. state_set(&ep->com, MPA_REQ_RCVD);
  932. /* drive upcall */
  933. connect_request_upcall(ep);
  934. return;
  935. }
  936. static int rx_data(struct c4iw_dev *dev, struct sk_buff *skb)
  937. {
  938. struct c4iw_ep *ep;
  939. struct cpl_rx_data *hdr = cplhdr(skb);
  940. unsigned int dlen = ntohs(hdr->len);
  941. unsigned int tid = GET_TID(hdr);
  942. struct tid_info *t = dev->rdev.lldi.tids;
  943. ep = lookup_tid(t, tid);
  944. PDBG("%s ep %p tid %u dlen %u\n", __func__, ep, ep->hwtid, dlen);
  945. skb_pull(skb, sizeof(*hdr));
  946. skb_trim(skb, dlen);
  947. ep->rcv_seq += dlen;
  948. BUG_ON(ep->rcv_seq != (ntohl(hdr->seq) + dlen));
  949. /* update RX credits */
  950. update_rx_credits(ep, dlen);
  951. switch (state_read(&ep->com)) {
  952. case MPA_REQ_SENT:
  953. process_mpa_reply(ep, skb);
  954. break;
  955. case MPA_REQ_WAIT:
  956. process_mpa_request(ep, skb);
  957. break;
  958. case MPA_REP_SENT:
  959. break;
  960. default:
  961. printk(KERN_ERR MOD "%s Unexpected streaming data."
  962. " ep %p state %d tid %u\n",
  963. __func__, ep, state_read(&ep->com), ep->hwtid);
  964. /*
  965. * The ep will timeout and inform the ULP of the failure.
  966. * See ep_timeout().
  967. */
  968. break;
  969. }
  970. return 0;
  971. }
  972. static int abort_rpl(struct c4iw_dev *dev, struct sk_buff *skb)
  973. {
  974. struct c4iw_ep *ep;
  975. struct cpl_abort_rpl_rss *rpl = cplhdr(skb);
  976. unsigned long flags;
  977. int release = 0;
  978. unsigned int tid = GET_TID(rpl);
  979. struct tid_info *t = dev->rdev.lldi.tids;
  980. ep = lookup_tid(t, tid);
  981. PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
  982. BUG_ON(!ep);
  983. spin_lock_irqsave(&ep->com.lock, flags);
  984. switch (ep->com.state) {
  985. case ABORTING:
  986. __state_set(&ep->com, DEAD);
  987. release = 1;
  988. break;
  989. default:
  990. printk(KERN_ERR "%s ep %p state %d\n",
  991. __func__, ep, ep->com.state);
  992. break;
  993. }
  994. spin_unlock_irqrestore(&ep->com.lock, flags);
  995. if (release)
  996. release_ep_resources(ep);
  997. return 0;
  998. }
  999. /*
  1000. * Return whether a failed active open has allocated a TID
  1001. */
  1002. static inline int act_open_has_tid(int status)
  1003. {
  1004. return status != CPL_ERR_TCAM_FULL && status != CPL_ERR_CONN_EXIST &&
  1005. status != CPL_ERR_ARP_MISS;
  1006. }
  1007. static int act_open_rpl(struct c4iw_dev *dev, struct sk_buff *skb)
  1008. {
  1009. struct c4iw_ep *ep;
  1010. struct cpl_act_open_rpl *rpl = cplhdr(skb);
  1011. unsigned int atid = GET_TID_TID(GET_AOPEN_ATID(
  1012. ntohl(rpl->atid_status)));
  1013. struct tid_info *t = dev->rdev.lldi.tids;
  1014. int status = GET_AOPEN_STATUS(ntohl(rpl->atid_status));
  1015. ep = lookup_atid(t, atid);
  1016. PDBG("%s ep %p atid %u status %u errno %d\n", __func__, ep, atid,
  1017. status, status2errno(status));
  1018. if (status == CPL_ERR_RTX_NEG_ADVICE) {
  1019. printk(KERN_WARNING MOD "Connection problems for atid %u\n",
  1020. atid);
  1021. return 0;
  1022. }
  1023. connect_reply_upcall(ep, status2errno(status));
  1024. state_set(&ep->com, DEAD);
  1025. if (status && act_open_has_tid(status))
  1026. cxgb4_remove_tid(ep->com.dev->rdev.lldi.tids, 0, GET_TID(rpl));
  1027. cxgb4_free_atid(t, atid);
  1028. dst_release(ep->dst);
  1029. cxgb4_l2t_release(ep->l2t);
  1030. c4iw_put_ep(&ep->com);
  1031. return 0;
  1032. }
  1033. static int pass_open_rpl(struct c4iw_dev *dev, struct sk_buff *skb)
  1034. {
  1035. struct cpl_pass_open_rpl *rpl = cplhdr(skb);
  1036. struct tid_info *t = dev->rdev.lldi.tids;
  1037. unsigned int stid = GET_TID(rpl);
  1038. struct c4iw_listen_ep *ep = lookup_stid(t, stid);
  1039. if (!ep) {
  1040. printk(KERN_ERR MOD "stid %d lookup failure!\n", stid);
  1041. return 0;
  1042. }
  1043. PDBG("%s ep %p status %d error %d\n", __func__, ep,
  1044. rpl->status, status2errno(rpl->status));
  1045. ep->com.rpl_err = status2errno(rpl->status);
  1046. ep->com.rpl_done = 1;
  1047. wake_up(&ep->com.waitq);
  1048. return 0;
  1049. }
  1050. static int listen_stop(struct c4iw_listen_ep *ep)
  1051. {
  1052. struct sk_buff *skb;
  1053. struct cpl_close_listsvr_req *req;
  1054. PDBG("%s ep %p\n", __func__, ep);
  1055. skb = get_skb(NULL, sizeof(*req), GFP_KERNEL);
  1056. if (!skb) {
  1057. printk(KERN_ERR MOD "%s - failed to alloc skb\n", __func__);
  1058. return -ENOMEM;
  1059. }
  1060. req = (struct cpl_close_listsvr_req *) skb_put(skb, sizeof(*req));
  1061. INIT_TP_WR(req, 0);
  1062. OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ,
  1063. ep->stid));
  1064. req->reply_ctrl = cpu_to_be16(
  1065. QUEUENO(ep->com.dev->rdev.lldi.rxq_ids[0]));
  1066. set_wr_txq(skb, CPL_PRIORITY_SETUP, 0);
  1067. return c4iw_ofld_send(&ep->com.dev->rdev, skb);
  1068. }
  1069. static int close_listsrv_rpl(struct c4iw_dev *dev, struct sk_buff *skb)
  1070. {
  1071. struct cpl_close_listsvr_rpl *rpl = cplhdr(skb);
  1072. struct tid_info *t = dev->rdev.lldi.tids;
  1073. unsigned int stid = GET_TID(rpl);
  1074. struct c4iw_listen_ep *ep = lookup_stid(t, stid);
  1075. PDBG("%s ep %p\n", __func__, ep);
  1076. ep->com.rpl_err = status2errno(rpl->status);
  1077. ep->com.rpl_done = 1;
  1078. wake_up(&ep->com.waitq);
  1079. return 0;
  1080. }
  1081. static void accept_cr(struct c4iw_ep *ep, __be32 peer_ip, struct sk_buff *skb,
  1082. struct cpl_pass_accept_req *req)
  1083. {
  1084. struct cpl_pass_accept_rpl *rpl;
  1085. unsigned int mtu_idx;
  1086. u64 opt0;
  1087. u32 opt2;
  1088. int wscale;
  1089. PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
  1090. BUG_ON(skb_cloned(skb));
  1091. skb_trim(skb, sizeof(*rpl));
  1092. skb_get(skb);
  1093. cxgb4_best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx);
  1094. wscale = compute_wscale(rcv_win);
  1095. opt0 = KEEP_ALIVE(1) |
  1096. WND_SCALE(wscale) |
  1097. MSS_IDX(mtu_idx) |
  1098. L2T_IDX(ep->l2t->idx) |
  1099. TX_CHAN(ep->tx_chan) |
  1100. SMAC_SEL(ep->smac_idx) |
  1101. DSCP(ep->tos) |
  1102. RCV_BUFSIZ(rcv_win>>10);
  1103. opt2 = RX_CHANNEL(0) |
  1104. RSS_QUEUE_VALID | RSS_QUEUE(ep->rss_qid);
  1105. if (enable_tcp_timestamps && req->tcpopt.tstamp)
  1106. opt2 |= TSTAMPS_EN(1);
  1107. if (enable_tcp_sack && req->tcpopt.sack)
  1108. opt2 |= SACK_EN(1);
  1109. if (wscale && enable_tcp_window_scaling)
  1110. opt2 |= WND_SCALE_EN(1);
  1111. rpl = cplhdr(skb);
  1112. INIT_TP_WR(rpl, ep->hwtid);
  1113. OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL,
  1114. ep->hwtid));
  1115. rpl->opt0 = cpu_to_be64(opt0);
  1116. rpl->opt2 = cpu_to_be32(opt2);
  1117. set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->txq_idx);
  1118. c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t);
  1119. return;
  1120. }
  1121. static void reject_cr(struct c4iw_dev *dev, u32 hwtid, __be32 peer_ip,
  1122. struct sk_buff *skb)
  1123. {
  1124. PDBG("%s c4iw_dev %p tid %u peer_ip %x\n", __func__, dev, hwtid,
  1125. peer_ip);
  1126. BUG_ON(skb_cloned(skb));
  1127. skb_trim(skb, sizeof(struct cpl_tid_release));
  1128. skb_get(skb);
  1129. release_tid(&dev->rdev, hwtid, skb);
  1130. return;
  1131. }
  1132. static void get_4tuple(struct cpl_pass_accept_req *req,
  1133. __be32 *local_ip, __be32 *peer_ip,
  1134. __be16 *local_port, __be16 *peer_port)
  1135. {
  1136. int eth_len = G_ETH_HDR_LEN(be32_to_cpu(req->hdr_len));
  1137. int ip_len = G_IP_HDR_LEN(be32_to_cpu(req->hdr_len));
  1138. struct iphdr *ip = (struct iphdr *)((u8 *)(req + 1) + eth_len);
  1139. struct tcphdr *tcp = (struct tcphdr *)
  1140. ((u8 *)(req + 1) + eth_len + ip_len);
  1141. PDBG("%s saddr 0x%x daddr 0x%x sport %u dport %u\n", __func__,
  1142. ntohl(ip->saddr), ntohl(ip->daddr), ntohs(tcp->source),
  1143. ntohs(tcp->dest));
  1144. *peer_ip = ip->saddr;
  1145. *local_ip = ip->daddr;
  1146. *peer_port = tcp->source;
  1147. *local_port = tcp->dest;
  1148. return;
  1149. }
  1150. static int pass_accept_req(struct c4iw_dev *dev, struct sk_buff *skb)
  1151. {
  1152. struct c4iw_ep *child_ep, *parent_ep;
  1153. struct cpl_pass_accept_req *req = cplhdr(skb);
  1154. unsigned int stid = GET_POPEN_TID(ntohl(req->tos_stid));
  1155. struct tid_info *t = dev->rdev.lldi.tids;
  1156. unsigned int hwtid = GET_TID(req);
  1157. struct dst_entry *dst;
  1158. struct l2t_entry *l2t;
  1159. struct rtable *rt;
  1160. __be32 local_ip, peer_ip;
  1161. __be16 local_port, peer_port;
  1162. struct net_device *pdev;
  1163. u32 tx_chan, smac_idx;
  1164. u16 rss_qid;
  1165. u32 mtu;
  1166. int step;
  1167. int txq_idx;
  1168. parent_ep = lookup_stid(t, stid);
  1169. PDBG("%s parent ep %p tid %u\n", __func__, parent_ep, hwtid);
  1170. get_4tuple(req, &local_ip, &peer_ip, &local_port, &peer_port);
  1171. if (state_read(&parent_ep->com) != LISTEN) {
  1172. printk(KERN_ERR "%s - listening ep not in LISTEN\n",
  1173. __func__);
  1174. goto reject;
  1175. }
  1176. /* Find output route */
  1177. rt = find_route(dev, local_ip, peer_ip, local_port, peer_port,
  1178. GET_POPEN_TOS(ntohl(req->tos_stid)));
  1179. if (!rt) {
  1180. printk(KERN_ERR MOD "%s - failed to find dst entry!\n",
  1181. __func__);
  1182. goto reject;
  1183. }
  1184. dst = &rt->u.dst;
  1185. if (dst->neighbour->dev->flags & IFF_LOOPBACK) {
  1186. pdev = ip_dev_find(&init_net, peer_ip);
  1187. BUG_ON(!pdev);
  1188. l2t = cxgb4_l2t_get(dev->rdev.lldi.l2t, dst->neighbour,
  1189. pdev, 0);
  1190. mtu = pdev->mtu;
  1191. tx_chan = cxgb4_port_chan(pdev);
  1192. smac_idx = (cxgb4_port_viid(pdev) & 0x7F) << 1;
  1193. step = dev->rdev.lldi.ntxq / dev->rdev.lldi.nchan;
  1194. txq_idx = cxgb4_port_idx(pdev) * step;
  1195. step = dev->rdev.lldi.nrxq / dev->rdev.lldi.nchan;
  1196. rss_qid = dev->rdev.lldi.rxq_ids[cxgb4_port_idx(pdev) * step];
  1197. dev_put(pdev);
  1198. } else {
  1199. l2t = cxgb4_l2t_get(dev->rdev.lldi.l2t, dst->neighbour,
  1200. dst->neighbour->dev, 0);
  1201. mtu = dst_mtu(dst);
  1202. tx_chan = cxgb4_port_chan(dst->neighbour->dev);
  1203. smac_idx = (cxgb4_port_viid(dst->neighbour->dev) & 0x7F) << 1;
  1204. step = dev->rdev.lldi.ntxq / dev->rdev.lldi.nchan;
  1205. txq_idx = cxgb4_port_idx(dst->neighbour->dev) * step;
  1206. step = dev->rdev.lldi.nrxq / dev->rdev.lldi.nchan;
  1207. rss_qid = dev->rdev.lldi.rxq_ids[
  1208. cxgb4_port_idx(dst->neighbour->dev) * step];
  1209. }
  1210. if (!l2t) {
  1211. printk(KERN_ERR MOD "%s - failed to allocate l2t entry!\n",
  1212. __func__);
  1213. dst_release(dst);
  1214. goto reject;
  1215. }
  1216. child_ep = alloc_ep(sizeof(*child_ep), GFP_KERNEL);
  1217. if (!child_ep) {
  1218. printk(KERN_ERR MOD "%s - failed to allocate ep entry!\n",
  1219. __func__);
  1220. cxgb4_l2t_release(l2t);
  1221. dst_release(dst);
  1222. goto reject;
  1223. }
  1224. state_set(&child_ep->com, CONNECTING);
  1225. child_ep->com.dev = dev;
  1226. child_ep->com.cm_id = NULL;
  1227. child_ep->com.local_addr.sin_family = PF_INET;
  1228. child_ep->com.local_addr.sin_port = local_port;
  1229. child_ep->com.local_addr.sin_addr.s_addr = local_ip;
  1230. child_ep->com.remote_addr.sin_family = PF_INET;
  1231. child_ep->com.remote_addr.sin_port = peer_port;
  1232. child_ep->com.remote_addr.sin_addr.s_addr = peer_ip;
  1233. c4iw_get_ep(&parent_ep->com);
  1234. child_ep->parent_ep = parent_ep;
  1235. child_ep->tos = GET_POPEN_TOS(ntohl(req->tos_stid));
  1236. child_ep->l2t = l2t;
  1237. child_ep->dst = dst;
  1238. child_ep->hwtid = hwtid;
  1239. child_ep->tx_chan = tx_chan;
  1240. child_ep->smac_idx = smac_idx;
  1241. child_ep->rss_qid = rss_qid;
  1242. child_ep->mtu = mtu;
  1243. child_ep->txq_idx = txq_idx;
  1244. PDBG("%s tx_chan %u smac_idx %u rss_qid %u\n", __func__,
  1245. tx_chan, smac_idx, rss_qid);
  1246. init_timer(&child_ep->timer);
  1247. cxgb4_insert_tid(t, child_ep, hwtid);
  1248. accept_cr(child_ep, peer_ip, skb, req);
  1249. goto out;
  1250. reject:
  1251. reject_cr(dev, hwtid, peer_ip, skb);
  1252. out:
  1253. return 0;
  1254. }
  1255. static int pass_establish(struct c4iw_dev *dev, struct sk_buff *skb)
  1256. {
  1257. struct c4iw_ep *ep;
  1258. struct cpl_pass_establish *req = cplhdr(skb);
  1259. struct tid_info *t = dev->rdev.lldi.tids;
  1260. unsigned int tid = GET_TID(req);
  1261. ep = lookup_tid(t, tid);
  1262. PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
  1263. ep->snd_seq = be32_to_cpu(req->snd_isn);
  1264. ep->rcv_seq = be32_to_cpu(req->rcv_isn);
  1265. set_emss(ep, ntohs(req->tcp_opt));
  1266. dst_confirm(ep->dst);
  1267. state_set(&ep->com, MPA_REQ_WAIT);
  1268. start_ep_timer(ep);
  1269. send_flowc(ep, skb);
  1270. return 0;
  1271. }
  1272. static int peer_close(struct c4iw_dev *dev, struct sk_buff *skb)
  1273. {
  1274. struct cpl_peer_close *hdr = cplhdr(skb);
  1275. struct c4iw_ep *ep;
  1276. struct c4iw_qp_attributes attrs;
  1277. unsigned long flags;
  1278. int disconnect = 1;
  1279. int release = 0;
  1280. int closing = 0;
  1281. struct tid_info *t = dev->rdev.lldi.tids;
  1282. unsigned int tid = GET_TID(hdr);
  1283. int start_timer = 0;
  1284. int stop_timer = 0;
  1285. ep = lookup_tid(t, tid);
  1286. PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
  1287. dst_confirm(ep->dst);
  1288. spin_lock_irqsave(&ep->com.lock, flags);
  1289. switch (ep->com.state) {
  1290. case MPA_REQ_WAIT:
  1291. __state_set(&ep->com, CLOSING);
  1292. break;
  1293. case MPA_REQ_SENT:
  1294. __state_set(&ep->com, CLOSING);
  1295. connect_reply_upcall(ep, -ECONNRESET);
  1296. break;
  1297. case MPA_REQ_RCVD:
  1298. /*
  1299. * We're gonna mark this puppy DEAD, but keep
  1300. * the reference on it until the ULP accepts or
  1301. * rejects the CR. Also wake up anyone waiting
  1302. * in rdma connection migration (see c4iw_accept_cr()).
  1303. */
  1304. __state_set(&ep->com, CLOSING);
  1305. ep->com.rpl_done = 1;
  1306. ep->com.rpl_err = -ECONNRESET;
  1307. PDBG("waking up ep %p tid %u\n", ep, ep->hwtid);
  1308. wake_up(&ep->com.waitq);
  1309. break;
  1310. case MPA_REP_SENT:
  1311. __state_set(&ep->com, CLOSING);
  1312. ep->com.rpl_done = 1;
  1313. ep->com.rpl_err = -ECONNRESET;
  1314. PDBG("waking up ep %p tid %u\n", ep, ep->hwtid);
  1315. wake_up(&ep->com.waitq);
  1316. break;
  1317. case FPDU_MODE:
  1318. start_timer = 1;
  1319. __state_set(&ep->com, CLOSING);
  1320. closing = 1;
  1321. peer_close_upcall(ep);
  1322. break;
  1323. case ABORTING:
  1324. disconnect = 0;
  1325. break;
  1326. case CLOSING:
  1327. __state_set(&ep->com, MORIBUND);
  1328. disconnect = 0;
  1329. break;
  1330. case MORIBUND:
  1331. stop_timer = 1;
  1332. if (ep->com.cm_id && ep->com.qp) {
  1333. attrs.next_state = C4IW_QP_STATE_IDLE;
  1334. c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp,
  1335. C4IW_QP_ATTR_NEXT_STATE, &attrs, 1);
  1336. }
  1337. close_complete_upcall(ep);
  1338. __state_set(&ep->com, DEAD);
  1339. release = 1;
  1340. disconnect = 0;
  1341. break;
  1342. case DEAD:
  1343. disconnect = 0;
  1344. break;
  1345. default:
  1346. BUG_ON(1);
  1347. }
  1348. spin_unlock_irqrestore(&ep->com.lock, flags);
  1349. if (closing) {
  1350. attrs.next_state = C4IW_QP_STATE_CLOSING;
  1351. c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp,
  1352. C4IW_QP_ATTR_NEXT_STATE, &attrs, 1);
  1353. }
  1354. if (start_timer)
  1355. start_ep_timer(ep);
  1356. if (stop_timer)
  1357. stop_ep_timer(ep);
  1358. if (disconnect)
  1359. c4iw_ep_disconnect(ep, 0, GFP_KERNEL);
  1360. if (release)
  1361. release_ep_resources(ep);
  1362. return 0;
  1363. }
  1364. /*
  1365. * Returns whether an ABORT_REQ_RSS message is a negative advice.
  1366. */
  1367. static int is_neg_adv_abort(unsigned int status)
  1368. {
  1369. return status == CPL_ERR_RTX_NEG_ADVICE ||
  1370. status == CPL_ERR_PERSIST_NEG_ADVICE;
  1371. }
  1372. static int peer_abort(struct c4iw_dev *dev, struct sk_buff *skb)
  1373. {
  1374. struct cpl_abort_req_rss *req = cplhdr(skb);
  1375. struct c4iw_ep *ep;
  1376. struct cpl_abort_rpl *rpl;
  1377. struct sk_buff *rpl_skb;
  1378. struct c4iw_qp_attributes attrs;
  1379. int ret;
  1380. int release = 0;
  1381. unsigned long flags;
  1382. struct tid_info *t = dev->rdev.lldi.tids;
  1383. unsigned int tid = GET_TID(req);
  1384. int stop_timer = 0;
  1385. ep = lookup_tid(t, tid);
  1386. if (is_neg_adv_abort(req->status)) {
  1387. PDBG("%s neg_adv_abort ep %p tid %u\n", __func__, ep,
  1388. ep->hwtid);
  1389. return 0;
  1390. }
  1391. spin_lock_irqsave(&ep->com.lock, flags);
  1392. PDBG("%s ep %p tid %u state %u\n", __func__, ep, ep->hwtid,
  1393. ep->com.state);
  1394. switch (ep->com.state) {
  1395. case CONNECTING:
  1396. break;
  1397. case MPA_REQ_WAIT:
  1398. stop_timer = 1;
  1399. break;
  1400. case MPA_REQ_SENT:
  1401. stop_timer = 1;
  1402. connect_reply_upcall(ep, -ECONNRESET);
  1403. break;
  1404. case MPA_REP_SENT:
  1405. ep->com.rpl_done = 1;
  1406. ep->com.rpl_err = -ECONNRESET;
  1407. PDBG("waking up ep %p\n", ep);
  1408. wake_up(&ep->com.waitq);
  1409. break;
  1410. case MPA_REQ_RCVD:
  1411. /*
  1412. * We're gonna mark this puppy DEAD, but keep
  1413. * the reference on it until the ULP accepts or
  1414. * rejects the CR. Also wake up anyone waiting
  1415. * in rdma connection migration (see c4iw_accept_cr()).
  1416. */
  1417. ep->com.rpl_done = 1;
  1418. ep->com.rpl_err = -ECONNRESET;
  1419. PDBG("waking up ep %p tid %u\n", ep, ep->hwtid);
  1420. wake_up(&ep->com.waitq);
  1421. break;
  1422. case MORIBUND:
  1423. case CLOSING:
  1424. stop_timer = 1;
  1425. /*FALLTHROUGH*/
  1426. case FPDU_MODE:
  1427. if (ep->com.cm_id && ep->com.qp) {
  1428. attrs.next_state = C4IW_QP_STATE_ERROR;
  1429. ret = c4iw_modify_qp(ep->com.qp->rhp,
  1430. ep->com.qp, C4IW_QP_ATTR_NEXT_STATE,
  1431. &attrs, 1);
  1432. if (ret)
  1433. printk(KERN_ERR MOD
  1434. "%s - qp <- error failed!\n",
  1435. __func__);
  1436. }
  1437. peer_abort_upcall(ep);
  1438. break;
  1439. case ABORTING:
  1440. break;
  1441. case DEAD:
  1442. PDBG("%s PEER_ABORT IN DEAD STATE!!!!\n", __func__);
  1443. spin_unlock_irqrestore(&ep->com.lock, flags);
  1444. return 0;
  1445. default:
  1446. BUG_ON(1);
  1447. break;
  1448. }
  1449. dst_confirm(ep->dst);
  1450. if (ep->com.state != ABORTING) {
  1451. __state_set(&ep->com, DEAD);
  1452. release = 1;
  1453. }
  1454. spin_unlock_irqrestore(&ep->com.lock, flags);
  1455. rpl_skb = get_skb(skb, sizeof(*rpl), GFP_KERNEL);
  1456. if (!rpl_skb) {
  1457. printk(KERN_ERR MOD "%s - cannot allocate skb!\n",
  1458. __func__);
  1459. release = 1;
  1460. goto out;
  1461. }
  1462. set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx);
  1463. rpl = (struct cpl_abort_rpl *) skb_put(rpl_skb, sizeof(*rpl));
  1464. INIT_TP_WR(rpl, ep->hwtid);
  1465. OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL, ep->hwtid));
  1466. rpl->cmd = CPL_ABORT_NO_RST;
  1467. c4iw_ofld_send(&ep->com.dev->rdev, rpl_skb);
  1468. out:
  1469. if (stop_timer)
  1470. stop_ep_timer(ep);
  1471. if (release)
  1472. release_ep_resources(ep);
  1473. return 0;
  1474. }
  1475. static int close_con_rpl(struct c4iw_dev *dev, struct sk_buff *skb)
  1476. {
  1477. struct c4iw_ep *ep;
  1478. struct c4iw_qp_attributes attrs;
  1479. struct cpl_close_con_rpl *rpl = cplhdr(skb);
  1480. unsigned long flags;
  1481. int release = 0;
  1482. struct tid_info *t = dev->rdev.lldi.tids;
  1483. unsigned int tid = GET_TID(rpl);
  1484. int stop_timer = 0;
  1485. ep = lookup_tid(t, tid);
  1486. PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
  1487. BUG_ON(!ep);
  1488. /* The cm_id may be null if we failed to connect */
  1489. spin_lock_irqsave(&ep->com.lock, flags);
  1490. switch (ep->com.state) {
  1491. case CLOSING:
  1492. __state_set(&ep->com, MORIBUND);
  1493. break;
  1494. case MORIBUND:
  1495. stop_timer = 1;
  1496. if ((ep->com.cm_id) && (ep->com.qp)) {
  1497. attrs.next_state = C4IW_QP_STATE_IDLE;
  1498. c4iw_modify_qp(ep->com.qp->rhp,
  1499. ep->com.qp,
  1500. C4IW_QP_ATTR_NEXT_STATE,
  1501. &attrs, 1);
  1502. }
  1503. close_complete_upcall(ep);
  1504. __state_set(&ep->com, DEAD);
  1505. release = 1;
  1506. break;
  1507. case ABORTING:
  1508. case DEAD:
  1509. break;
  1510. default:
  1511. BUG_ON(1);
  1512. break;
  1513. }
  1514. spin_unlock_irqrestore(&ep->com.lock, flags);
  1515. if (stop_timer)
  1516. stop_ep_timer(ep);
  1517. if (release)
  1518. release_ep_resources(ep);
  1519. return 0;
  1520. }
  1521. static int terminate(struct c4iw_dev *dev, struct sk_buff *skb)
  1522. {
  1523. struct c4iw_ep *ep;
  1524. struct cpl_rdma_terminate *term = cplhdr(skb);
  1525. struct tid_info *t = dev->rdev.lldi.tids;
  1526. unsigned int tid = GET_TID(term);
  1527. ep = lookup_tid(t, tid);
  1528. if (state_read(&ep->com) != FPDU_MODE)
  1529. return 0;
  1530. PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
  1531. skb_pull(skb, sizeof *term);
  1532. PDBG("%s saving %d bytes of term msg\n", __func__, skb->len);
  1533. skb_copy_from_linear_data(skb, ep->com.qp->attr.terminate_buffer,
  1534. skb->len);
  1535. ep->com.qp->attr.terminate_msg_len = skb->len;
  1536. ep->com.qp->attr.is_terminate_local = 0;
  1537. return 0;
  1538. }
  1539. /*
  1540. * Upcall from the adapter indicating data has been transmitted.
  1541. * For us its just the single MPA request or reply. We can now free
  1542. * the skb holding the mpa message.
  1543. */
  1544. static int fw4_ack(struct c4iw_dev *dev, struct sk_buff *skb)
  1545. {
  1546. struct c4iw_ep *ep;
  1547. struct cpl_fw4_ack *hdr = cplhdr(skb);
  1548. u8 credits = hdr->credits;
  1549. unsigned int tid = GET_TID(hdr);
  1550. struct tid_info *t = dev->rdev.lldi.tids;
  1551. ep = lookup_tid(t, tid);
  1552. PDBG("%s ep %p tid %u credits %u\n", __func__, ep, ep->hwtid, credits);
  1553. if (credits == 0) {
  1554. PDBG(KERN_ERR "%s 0 credit ack ep %p tid %u state %u\n",
  1555. __func__, ep, ep->hwtid, state_read(&ep->com));
  1556. return 0;
  1557. }
  1558. dst_confirm(ep->dst);
  1559. if (ep->mpa_skb) {
  1560. PDBG("%s last streaming msg ack ep %p tid %u state %u "
  1561. "initiator %u freeing skb\n", __func__, ep, ep->hwtid,
  1562. state_read(&ep->com), ep->mpa_attr.initiator ? 1 : 0);
  1563. kfree_skb(ep->mpa_skb);
  1564. ep->mpa_skb = NULL;
  1565. }
  1566. return 0;
  1567. }
  1568. int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len)
  1569. {
  1570. int err;
  1571. struct c4iw_ep *ep = to_ep(cm_id);
  1572. PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
  1573. if (state_read(&ep->com) == DEAD) {
  1574. c4iw_put_ep(&ep->com);
  1575. return -ECONNRESET;
  1576. }
  1577. BUG_ON(state_read(&ep->com) != MPA_REQ_RCVD);
  1578. if (mpa_rev == 0)
  1579. abort_connection(ep, NULL, GFP_KERNEL);
  1580. else {
  1581. err = send_mpa_reject(ep, pdata, pdata_len);
  1582. err = c4iw_ep_disconnect(ep, 0, GFP_KERNEL);
  1583. }
  1584. c4iw_put_ep(&ep->com);
  1585. return 0;
  1586. }
  1587. int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
  1588. {
  1589. int err;
  1590. struct c4iw_qp_attributes attrs;
  1591. enum c4iw_qp_attr_mask mask;
  1592. struct c4iw_ep *ep = to_ep(cm_id);
  1593. struct c4iw_dev *h = to_c4iw_dev(cm_id->device);
  1594. struct c4iw_qp *qp = get_qhp(h, conn_param->qpn);
  1595. PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
  1596. if (state_read(&ep->com) == DEAD) {
  1597. err = -ECONNRESET;
  1598. goto err;
  1599. }
  1600. BUG_ON(state_read(&ep->com) != MPA_REQ_RCVD);
  1601. BUG_ON(!qp);
  1602. if ((conn_param->ord > c4iw_max_read_depth) ||
  1603. (conn_param->ird > c4iw_max_read_depth)) {
  1604. abort_connection(ep, NULL, GFP_KERNEL);
  1605. err = -EINVAL;
  1606. goto err;
  1607. }
  1608. cm_id->add_ref(cm_id);
  1609. ep->com.cm_id = cm_id;
  1610. ep->com.qp = qp;
  1611. ep->ird = conn_param->ird;
  1612. ep->ord = conn_param->ord;
  1613. if (peer2peer && ep->ird == 0)
  1614. ep->ird = 1;
  1615. PDBG("%s %d ird %d ord %d\n", __func__, __LINE__, ep->ird, ep->ord);
  1616. /* bind QP to EP and move to RTS */
  1617. attrs.mpa_attr = ep->mpa_attr;
  1618. attrs.max_ird = ep->ird;
  1619. attrs.max_ord = ep->ord;
  1620. attrs.llp_stream_handle = ep;
  1621. attrs.next_state = C4IW_QP_STATE_RTS;
  1622. /* bind QP and TID with INIT_WR */
  1623. mask = C4IW_QP_ATTR_NEXT_STATE |
  1624. C4IW_QP_ATTR_LLP_STREAM_HANDLE |
  1625. C4IW_QP_ATTR_MPA_ATTR |
  1626. C4IW_QP_ATTR_MAX_IRD |
  1627. C4IW_QP_ATTR_MAX_ORD;
  1628. err = c4iw_modify_qp(ep->com.qp->rhp,
  1629. ep->com.qp, mask, &attrs, 1);
  1630. if (err)
  1631. goto err1;
  1632. err = send_mpa_reply(ep, conn_param->private_data,
  1633. conn_param->private_data_len);
  1634. if (err)
  1635. goto err1;
  1636. state_set(&ep->com, FPDU_MODE);
  1637. established_upcall(ep);
  1638. c4iw_put_ep(&ep->com);
  1639. return 0;
  1640. err1:
  1641. ep->com.cm_id = NULL;
  1642. ep->com.qp = NULL;
  1643. cm_id->rem_ref(cm_id);
  1644. err:
  1645. c4iw_put_ep(&ep->com);
  1646. return err;
  1647. }
  1648. int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
  1649. {
  1650. int err = 0;
  1651. struct c4iw_dev *dev = to_c4iw_dev(cm_id->device);
  1652. struct c4iw_ep *ep;
  1653. struct rtable *rt;
  1654. struct net_device *pdev;
  1655. int step;
  1656. if ((conn_param->ord > c4iw_max_read_depth) ||
  1657. (conn_param->ird > c4iw_max_read_depth)) {
  1658. err = -EINVAL;
  1659. goto out;
  1660. }
  1661. ep = alloc_ep(sizeof(*ep), GFP_KERNEL);
  1662. if (!ep) {
  1663. printk(KERN_ERR MOD "%s - cannot alloc ep.\n", __func__);
  1664. err = -ENOMEM;
  1665. goto out;
  1666. }
  1667. init_timer(&ep->timer);
  1668. ep->plen = conn_param->private_data_len;
  1669. if (ep->plen)
  1670. memcpy(ep->mpa_pkt + sizeof(struct mpa_message),
  1671. conn_param->private_data, ep->plen);
  1672. ep->ird = conn_param->ird;
  1673. ep->ord = conn_param->ord;
  1674. if (peer2peer && ep->ord == 0)
  1675. ep->ord = 1;
  1676. cm_id->add_ref(cm_id);
  1677. ep->com.dev = dev;
  1678. ep->com.cm_id = cm_id;
  1679. ep->com.qp = get_qhp(dev, conn_param->qpn);
  1680. BUG_ON(!ep->com.qp);
  1681. PDBG("%s qpn 0x%x qp %p cm_id %p\n", __func__, conn_param->qpn,
  1682. ep->com.qp, cm_id);
  1683. /*
  1684. * Allocate an active TID to initiate a TCP connection.
  1685. */
  1686. ep->atid = cxgb4_alloc_atid(dev->rdev.lldi.tids, ep);
  1687. if (ep->atid == -1) {
  1688. printk(KERN_ERR MOD "%s - cannot alloc atid.\n", __func__);
  1689. err = -ENOMEM;
  1690. goto fail2;
  1691. }
  1692. PDBG("%s saddr 0x%x sport 0x%x raddr 0x%x rport 0x%x\n", __func__,
  1693. ntohl(cm_id->local_addr.sin_addr.s_addr),
  1694. ntohs(cm_id->local_addr.sin_port),
  1695. ntohl(cm_id->remote_addr.sin_addr.s_addr),
  1696. ntohs(cm_id->remote_addr.sin_port));
  1697. /* find a route */
  1698. rt = find_route(dev,
  1699. cm_id->local_addr.sin_addr.s_addr,
  1700. cm_id->remote_addr.sin_addr.s_addr,
  1701. cm_id->local_addr.sin_port,
  1702. cm_id->remote_addr.sin_port, 0);
  1703. if (!rt) {
  1704. printk(KERN_ERR MOD "%s - cannot find route.\n", __func__);
  1705. err = -EHOSTUNREACH;
  1706. goto fail3;
  1707. }
  1708. ep->dst = &rt->u.dst;
  1709. /* get a l2t entry */
  1710. if (ep->dst->neighbour->dev->flags & IFF_LOOPBACK) {
  1711. PDBG("%s LOOPBACK\n", __func__);
  1712. pdev = ip_dev_find(&init_net,
  1713. cm_id->remote_addr.sin_addr.s_addr);
  1714. ep->l2t = cxgb4_l2t_get(ep->com.dev->rdev.lldi.l2t,
  1715. ep->dst->neighbour,
  1716. pdev, 0);
  1717. ep->mtu = pdev->mtu;
  1718. ep->tx_chan = cxgb4_port_chan(pdev);
  1719. ep->smac_idx = (cxgb4_port_viid(pdev) & 0x7F) << 1;
  1720. step = ep->com.dev->rdev.lldi.ntxq /
  1721. ep->com.dev->rdev.lldi.nchan;
  1722. ep->txq_idx = cxgb4_port_idx(pdev) * step;
  1723. step = ep->com.dev->rdev.lldi.nrxq /
  1724. ep->com.dev->rdev.lldi.nchan;
  1725. ep->rss_qid = ep->com.dev->rdev.lldi.rxq_ids[
  1726. cxgb4_port_idx(pdev) * step];
  1727. dev_put(pdev);
  1728. } else {
  1729. ep->l2t = cxgb4_l2t_get(ep->com.dev->rdev.lldi.l2t,
  1730. ep->dst->neighbour,
  1731. ep->dst->neighbour->dev, 0);
  1732. ep->mtu = dst_mtu(ep->dst);
  1733. ep->tx_chan = cxgb4_port_chan(ep->dst->neighbour->dev);
  1734. ep->smac_idx = (cxgb4_port_viid(ep->dst->neighbour->dev) &
  1735. 0x7F) << 1;
  1736. step = ep->com.dev->rdev.lldi.ntxq /
  1737. ep->com.dev->rdev.lldi.nchan;
  1738. ep->txq_idx = cxgb4_port_idx(ep->dst->neighbour->dev) * step;
  1739. step = ep->com.dev->rdev.lldi.nrxq /
  1740. ep->com.dev->rdev.lldi.nchan;
  1741. ep->rss_qid = ep->com.dev->rdev.lldi.rxq_ids[
  1742. cxgb4_port_idx(ep->dst->neighbour->dev) * step];
  1743. }
  1744. if (!ep->l2t) {
  1745. printk(KERN_ERR MOD "%s - cannot alloc l2e.\n", __func__);
  1746. err = -ENOMEM;
  1747. goto fail4;
  1748. }
  1749. PDBG("%s txq_idx %u tx_chan %u smac_idx %u rss_qid %u l2t_idx %u\n",
  1750. __func__, ep->txq_idx, ep->tx_chan, ep->smac_idx, ep->rss_qid,
  1751. ep->l2t->idx);
  1752. state_set(&ep->com, CONNECTING);
  1753. ep->tos = 0;
  1754. ep->com.local_addr = cm_id->local_addr;
  1755. ep->com.remote_addr = cm_id->remote_addr;
  1756. /* send connect request to rnic */
  1757. err = send_connect(ep);
  1758. if (!err)
  1759. goto out;
  1760. cxgb4_l2t_release(ep->l2t);
  1761. fail4:
  1762. dst_release(ep->dst);
  1763. fail3:
  1764. cxgb4_free_atid(ep->com.dev->rdev.lldi.tids, ep->atid);
  1765. fail2:
  1766. cm_id->rem_ref(cm_id);
  1767. c4iw_put_ep(&ep->com);
  1768. out:
  1769. return err;
  1770. }
  1771. int c4iw_create_listen(struct iw_cm_id *cm_id, int backlog)
  1772. {
  1773. int err = 0;
  1774. struct c4iw_dev *dev = to_c4iw_dev(cm_id->device);
  1775. struct c4iw_listen_ep *ep;
  1776. might_sleep();
  1777. ep = alloc_ep(sizeof(*ep), GFP_KERNEL);
  1778. if (!ep) {
  1779. printk(KERN_ERR MOD "%s - cannot alloc ep.\n", __func__);
  1780. err = -ENOMEM;
  1781. goto fail1;
  1782. }
  1783. PDBG("%s ep %p\n", __func__, ep);
  1784. cm_id->add_ref(cm_id);
  1785. ep->com.cm_id = cm_id;
  1786. ep->com.dev = dev;
  1787. ep->backlog = backlog;
  1788. ep->com.local_addr = cm_id->local_addr;
  1789. /*
  1790. * Allocate a server TID.
  1791. */
  1792. ep->stid = cxgb4_alloc_stid(dev->rdev.lldi.tids, PF_INET, ep);
  1793. if (ep->stid == -1) {
  1794. printk(KERN_ERR MOD "%s - cannot alloc stid.\n", __func__);
  1795. err = -ENOMEM;
  1796. goto fail2;
  1797. }
  1798. state_set(&ep->com, LISTEN);
  1799. err = cxgb4_create_server(ep->com.dev->rdev.lldi.ports[0], ep->stid,
  1800. ep->com.local_addr.sin_addr.s_addr,
  1801. ep->com.local_addr.sin_port,
  1802. ep->com.dev->rdev.lldi.rxq_ids[0]);
  1803. if (err)
  1804. goto fail3;
  1805. /* wait for pass_open_rpl */
  1806. wait_event(ep->com.waitq, ep->com.rpl_done);
  1807. err = ep->com.rpl_err;
  1808. if (!err) {
  1809. cm_id->provider_data = ep;
  1810. goto out;
  1811. }
  1812. fail3:
  1813. cxgb4_free_stid(ep->com.dev->rdev.lldi.tids, ep->stid, PF_INET);
  1814. fail2:
  1815. cm_id->rem_ref(cm_id);
  1816. c4iw_put_ep(&ep->com);
  1817. fail1:
  1818. out:
  1819. return err;
  1820. }
  1821. int c4iw_destroy_listen(struct iw_cm_id *cm_id)
  1822. {
  1823. int err;
  1824. struct c4iw_listen_ep *ep = to_listen_ep(cm_id);
  1825. PDBG("%s ep %p\n", __func__, ep);
  1826. might_sleep();
  1827. state_set(&ep->com, DEAD);
  1828. ep->com.rpl_done = 0;
  1829. ep->com.rpl_err = 0;
  1830. err = listen_stop(ep);
  1831. if (err)
  1832. goto done;
  1833. wait_event(ep->com.waitq, ep->com.rpl_done);
  1834. cxgb4_free_stid(ep->com.dev->rdev.lldi.tids, ep->stid, PF_INET);
  1835. done:
  1836. err = ep->com.rpl_err;
  1837. cm_id->rem_ref(cm_id);
  1838. c4iw_put_ep(&ep->com);
  1839. return err;
  1840. }
  1841. int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp)
  1842. {
  1843. int ret = 0;
  1844. unsigned long flags;
  1845. int close = 0;
  1846. int fatal = 0;
  1847. struct c4iw_rdev *rdev;
  1848. int start_timer = 0;
  1849. int stop_timer = 0;
  1850. spin_lock_irqsave(&ep->com.lock, flags);
  1851. PDBG("%s ep %p state %s, abrupt %d\n", __func__, ep,
  1852. states[ep->com.state], abrupt);
  1853. rdev = &ep->com.dev->rdev;
  1854. if (c4iw_fatal_error(rdev)) {
  1855. fatal = 1;
  1856. close_complete_upcall(ep);
  1857. ep->com.state = DEAD;
  1858. }
  1859. switch (ep->com.state) {
  1860. case MPA_REQ_WAIT:
  1861. case MPA_REQ_SENT:
  1862. case MPA_REQ_RCVD:
  1863. case MPA_REP_SENT:
  1864. case FPDU_MODE:
  1865. close = 1;
  1866. if (abrupt)
  1867. ep->com.state = ABORTING;
  1868. else {
  1869. ep->com.state = CLOSING;
  1870. start_timer = 1;
  1871. }
  1872. set_bit(CLOSE_SENT, &ep->com.flags);
  1873. break;
  1874. case CLOSING:
  1875. if (!test_and_set_bit(CLOSE_SENT, &ep->com.flags)) {
  1876. close = 1;
  1877. if (abrupt) {
  1878. stop_timer = 1;
  1879. ep->com.state = ABORTING;
  1880. } else
  1881. ep->com.state = MORIBUND;
  1882. }
  1883. break;
  1884. case MORIBUND:
  1885. case ABORTING:
  1886. case DEAD:
  1887. PDBG("%s ignoring disconnect ep %p state %u\n",
  1888. __func__, ep, ep->com.state);
  1889. break;
  1890. default:
  1891. BUG();
  1892. break;
  1893. }
  1894. spin_unlock_irqrestore(&ep->com.lock, flags);
  1895. if (start_timer)
  1896. start_ep_timer(ep);
  1897. if (stop_timer)
  1898. stop_ep_timer(ep);
  1899. if (close) {
  1900. if (abrupt)
  1901. ret = abort_connection(ep, NULL, gfp);
  1902. else
  1903. ret = send_halfclose(ep, gfp);
  1904. if (ret)
  1905. fatal = 1;
  1906. }
  1907. if (fatal)
  1908. release_ep_resources(ep);
  1909. return ret;
  1910. }
  1911. /*
  1912. * These are the real handlers that are called from a
  1913. * work queue.
  1914. */
  1915. static c4iw_handler_func work_handlers[NUM_CPL_CMDS] = {
  1916. [CPL_ACT_ESTABLISH] = act_establish,
  1917. [CPL_ACT_OPEN_RPL] = act_open_rpl,
  1918. [CPL_RX_DATA] = rx_data,
  1919. [CPL_ABORT_RPL_RSS] = abort_rpl,
  1920. [CPL_ABORT_RPL] = abort_rpl,
  1921. [CPL_PASS_OPEN_RPL] = pass_open_rpl,
  1922. [CPL_CLOSE_LISTSRV_RPL] = close_listsrv_rpl,
  1923. [CPL_PASS_ACCEPT_REQ] = pass_accept_req,
  1924. [CPL_PASS_ESTABLISH] = pass_establish,
  1925. [CPL_PEER_CLOSE] = peer_close,
  1926. [CPL_ABORT_REQ_RSS] = peer_abort,
  1927. [CPL_CLOSE_CON_RPL] = close_con_rpl,
  1928. [CPL_RDMA_TERMINATE] = terminate,
  1929. [CPL_FW4_ACK] = fw4_ack
  1930. };
  1931. static void process_timeout(struct c4iw_ep *ep)
  1932. {
  1933. struct c4iw_qp_attributes attrs;
  1934. int abort = 1;
  1935. spin_lock_irq(&ep->com.lock);
  1936. PDBG("%s ep %p tid %u state %d\n", __func__, ep, ep->hwtid,
  1937. ep->com.state);
  1938. switch (ep->com.state) {
  1939. case MPA_REQ_SENT:
  1940. __state_set(&ep->com, ABORTING);
  1941. connect_reply_upcall(ep, -ETIMEDOUT);
  1942. break;
  1943. case MPA_REQ_WAIT:
  1944. __state_set(&ep->com, ABORTING);
  1945. break;
  1946. case CLOSING:
  1947. case MORIBUND:
  1948. if (ep->com.cm_id && ep->com.qp) {
  1949. attrs.next_state = C4IW_QP_STATE_ERROR;
  1950. c4iw_modify_qp(ep->com.qp->rhp,
  1951. ep->com.qp, C4IW_QP_ATTR_NEXT_STATE,
  1952. &attrs, 1);
  1953. }
  1954. __state_set(&ep->com, ABORTING);
  1955. break;
  1956. default:
  1957. printk(KERN_ERR "%s unexpected state ep %p tid %u state %u\n",
  1958. __func__, ep, ep->hwtid, ep->com.state);
  1959. WARN_ON(1);
  1960. abort = 0;
  1961. }
  1962. spin_unlock_irq(&ep->com.lock);
  1963. if (abort)
  1964. abort_connection(ep, NULL, GFP_KERNEL);
  1965. c4iw_put_ep(&ep->com);
  1966. }
  1967. static void process_timedout_eps(void)
  1968. {
  1969. struct c4iw_ep *ep;
  1970. spin_lock_irq(&timeout_lock);
  1971. while (!list_empty(&timeout_list)) {
  1972. struct list_head *tmp;
  1973. tmp = timeout_list.next;
  1974. list_del(tmp);
  1975. spin_unlock_irq(&timeout_lock);
  1976. ep = list_entry(tmp, struct c4iw_ep, entry);
  1977. process_timeout(ep);
  1978. spin_lock_irq(&timeout_lock);
  1979. }
  1980. spin_unlock_irq(&timeout_lock);
  1981. }
  1982. static void process_work(struct work_struct *work)
  1983. {
  1984. struct sk_buff *skb = NULL;
  1985. struct c4iw_dev *dev;
  1986. struct cpl_act_establish *rpl = cplhdr(skb);
  1987. unsigned int opcode;
  1988. int ret;
  1989. while ((skb = skb_dequeue(&rxq))) {
  1990. rpl = cplhdr(skb);
  1991. dev = *((struct c4iw_dev **) (skb->cb + sizeof(void *)));
  1992. opcode = rpl->ot.opcode;
  1993. BUG_ON(!work_handlers[opcode]);
  1994. ret = work_handlers[opcode](dev, skb);
  1995. if (!ret)
  1996. kfree_skb(skb);
  1997. }
  1998. process_timedout_eps();
  1999. }
  2000. static DECLARE_WORK(skb_work, process_work);
  2001. static void ep_timeout(unsigned long arg)
  2002. {
  2003. struct c4iw_ep *ep = (struct c4iw_ep *)arg;
  2004. spin_lock(&timeout_lock);
  2005. list_add_tail(&ep->entry, &timeout_list);
  2006. spin_unlock(&timeout_lock);
  2007. queue_work(workq, &skb_work);
  2008. }
  2009. /*
  2010. * All the CM events are handled on a work queue to have a safe context.
  2011. */
  2012. static int sched(struct c4iw_dev *dev, struct sk_buff *skb)
  2013. {
  2014. /*
  2015. * Save dev in the skb->cb area.
  2016. */
  2017. *((struct c4iw_dev **) (skb->cb + sizeof(void *))) = dev;
  2018. /*
  2019. * Queue the skb and schedule the worker thread.
  2020. */
  2021. skb_queue_tail(&rxq, skb);
  2022. queue_work(workq, &skb_work);
  2023. return 0;
  2024. }
  2025. static int set_tcb_rpl(struct c4iw_dev *dev, struct sk_buff *skb)
  2026. {
  2027. struct cpl_set_tcb_rpl *rpl = cplhdr(skb);
  2028. if (rpl->status != CPL_ERR_NONE) {
  2029. printk(KERN_ERR MOD "Unexpected SET_TCB_RPL status %u "
  2030. "for tid %u\n", rpl->status, GET_TID(rpl));
  2031. }
  2032. return 0;
  2033. }
  2034. static int fw6_msg(struct c4iw_dev *dev, struct sk_buff *skb)
  2035. {
  2036. struct cpl_fw6_msg *rpl = cplhdr(skb);
  2037. struct c4iw_wr_wait *wr_waitp;
  2038. int ret;
  2039. PDBG("%s type %u\n", __func__, rpl->type);
  2040. switch (rpl->type) {
  2041. case 1:
  2042. ret = (int)((be64_to_cpu(rpl->data[0]) >> 8) & 0xff);
  2043. wr_waitp = (__force struct c4iw_wr_wait *)rpl->data[1];
  2044. PDBG("%s wr_waitp %p ret %u\n", __func__, wr_waitp, ret);
  2045. if (wr_waitp) {
  2046. wr_waitp->ret = ret;
  2047. wr_waitp->done = 1;
  2048. wake_up(&wr_waitp->wait);
  2049. }
  2050. break;
  2051. case 2:
  2052. c4iw_ev_dispatch(dev, (struct t4_cqe *)&rpl->data[0]);
  2053. break;
  2054. default:
  2055. printk(KERN_ERR MOD "%s unexpected fw6 msg type %u\n", __func__,
  2056. rpl->type);
  2057. break;
  2058. }
  2059. return 0;
  2060. }
  2061. /*
  2062. * Most upcalls from the T4 Core go to sched() to
  2063. * schedule the processing on a work queue.
  2064. */
  2065. c4iw_handler_func c4iw_handlers[NUM_CPL_CMDS] = {
  2066. [CPL_ACT_ESTABLISH] = sched,
  2067. [CPL_ACT_OPEN_RPL] = sched,
  2068. [CPL_RX_DATA] = sched,
  2069. [CPL_ABORT_RPL_RSS] = sched,
  2070. [CPL_ABORT_RPL] = sched,
  2071. [CPL_PASS_OPEN_RPL] = sched,
  2072. [CPL_CLOSE_LISTSRV_RPL] = sched,
  2073. [CPL_PASS_ACCEPT_REQ] = sched,
  2074. [CPL_PASS_ESTABLISH] = sched,
  2075. [CPL_PEER_CLOSE] = sched,
  2076. [CPL_CLOSE_CON_RPL] = sched,
  2077. [CPL_ABORT_REQ_RSS] = sched,
  2078. [CPL_RDMA_TERMINATE] = sched,
  2079. [CPL_FW4_ACK] = sched,
  2080. [CPL_SET_TCB_RPL] = set_tcb_rpl,
  2081. [CPL_FW6_MSG] = fw6_msg
  2082. };
  2083. int __init c4iw_cm_init(void)
  2084. {
  2085. spin_lock_init(&timeout_lock);
  2086. skb_queue_head_init(&rxq);
  2087. workq = create_singlethread_workqueue("iw_cxgb4");
  2088. if (!workq)
  2089. return -ENOMEM;
  2090. return 0;
  2091. }
  2092. void __exit c4iw_cm_term(void)
  2093. {
  2094. WARN_ON(!list_empty(&timeout_list));
  2095. flush_workqueue(workq);
  2096. destroy_workqueue(workq);
  2097. }