/drivers/scsi/be2iscsi/be_main.c

https://bitbucket.org/cyanogenmod/android_kernel_asus_tf300t · C · 4461 lines · 3901 code · 471 blank · 89 comment · 425 complexity · 9f338297faf2b91c3eafe7613b4ab8e4 MD5 · raw file

Large files are truncated click here to view the full file

  1. /**
  2. * Copyright (C) 2005 - 2011 Emulex
  3. * All rights reserved.
  4. *
  5. * This program is free software; you can redistribute it and/or
  6. * modify it under the terms of the GNU General Public License version 2
  7. * as published by the Free Software Foundation. The full GNU General
  8. * Public License is included in this distribution in the file called COPYING.
  9. *
  10. * Written by: Jayamohan Kallickal (jayamohan.kallickal@emulex.com)
  11. *
  12. * Contact Information:
  13. * linux-drivers@emulex.com
  14. *
  15. * Emulex
  16. * 3333 Susan Street
  17. * Costa Mesa, CA 92626
  18. */
  19. #include <linux/reboot.h>
  20. #include <linux/delay.h>
  21. #include <linux/slab.h>
  22. #include <linux/interrupt.h>
  23. #include <linux/blkdev.h>
  24. #include <linux/pci.h>
  25. #include <linux/string.h>
  26. #include <linux/kernel.h>
  27. #include <linux/semaphore.h>
  28. #include <linux/iscsi_boot_sysfs.h>
  29. #include <scsi/libiscsi.h>
  30. #include <scsi/scsi_transport_iscsi.h>
  31. #include <scsi/scsi_transport.h>
  32. #include <scsi/scsi_cmnd.h>
  33. #include <scsi/scsi_device.h>
  34. #include <scsi/scsi_host.h>
  35. #include <scsi/scsi.h>
  36. #include "be_main.h"
  37. #include "be_iscsi.h"
  38. #include "be_mgmt.h"
  39. static unsigned int be_iopoll_budget = 10;
  40. static unsigned int be_max_phys_size = 64;
  41. static unsigned int enable_msix = 1;
  42. static unsigned int gcrashmode = 0;
  43. static unsigned int num_hba = 0;
  44. MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
  45. MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR);
  46. MODULE_AUTHOR("ServerEngines Corporation");
  47. MODULE_LICENSE("GPL");
  48. module_param(be_iopoll_budget, int, 0);
  49. module_param(enable_msix, int, 0);
  50. module_param(be_max_phys_size, uint, S_IRUGO);
  51. MODULE_PARM_DESC(be_max_phys_size, "Maximum Size (In Kilobytes) of physically"
  52. "contiguous memory that can be allocated."
  53. "Range is 16 - 128");
  54. static int beiscsi_slave_configure(struct scsi_device *sdev)
  55. {
  56. blk_queue_max_segment_size(sdev->request_queue, 65536);
  57. return 0;
  58. }
  59. static int beiscsi_eh_abort(struct scsi_cmnd *sc)
  60. {
  61. struct iscsi_cls_session *cls_session;
  62. struct iscsi_task *aborted_task = (struct iscsi_task *)sc->SCp.ptr;
  63. struct beiscsi_io_task *aborted_io_task;
  64. struct iscsi_conn *conn;
  65. struct beiscsi_conn *beiscsi_conn;
  66. struct beiscsi_hba *phba;
  67. struct iscsi_session *session;
  68. struct invalidate_command_table *inv_tbl;
  69. struct be_dma_mem nonemb_cmd;
  70. unsigned int cid, tag, num_invalidate;
  71. cls_session = starget_to_session(scsi_target(sc->device));
  72. session = cls_session->dd_data;
  73. spin_lock_bh(&session->lock);
  74. if (!aborted_task || !aborted_task->sc) {
  75. /* we raced */
  76. spin_unlock_bh(&session->lock);
  77. return SUCCESS;
  78. }
  79. aborted_io_task = aborted_task->dd_data;
  80. if (!aborted_io_task->scsi_cmnd) {
  81. /* raced or invalid command */
  82. spin_unlock_bh(&session->lock);
  83. return SUCCESS;
  84. }
  85. spin_unlock_bh(&session->lock);
  86. conn = aborted_task->conn;
  87. beiscsi_conn = conn->dd_data;
  88. phba = beiscsi_conn->phba;
  89. /* invalidate iocb */
  90. cid = beiscsi_conn->beiscsi_conn_cid;
  91. inv_tbl = phba->inv_tbl;
  92. memset(inv_tbl, 0x0, sizeof(*inv_tbl));
  93. inv_tbl->cid = cid;
  94. inv_tbl->icd = aborted_io_task->psgl_handle->sgl_index;
  95. num_invalidate = 1;
  96. nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
  97. sizeof(struct invalidate_commands_params_in),
  98. &nonemb_cmd.dma);
  99. if (nonemb_cmd.va == NULL) {
  100. SE_DEBUG(DBG_LVL_1,
  101. "Failed to allocate memory for"
  102. "mgmt_invalidate_icds\n");
  103. return FAILED;
  104. }
  105. nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
  106. tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
  107. cid, &nonemb_cmd);
  108. if (!tag) {
  109. shost_printk(KERN_WARNING, phba->shost,
  110. "mgmt_invalidate_icds could not be"
  111. " submitted\n");
  112. pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
  113. nonemb_cmd.va, nonemb_cmd.dma);
  114. return FAILED;
  115. } else {
  116. wait_event_interruptible(phba->ctrl.mcc_wait[tag],
  117. phba->ctrl.mcc_numtag[tag]);
  118. free_mcc_tag(&phba->ctrl, tag);
  119. }
  120. pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
  121. nonemb_cmd.va, nonemb_cmd.dma);
  122. return iscsi_eh_abort(sc);
  123. }
  124. static int beiscsi_eh_device_reset(struct scsi_cmnd *sc)
  125. {
  126. struct iscsi_task *abrt_task;
  127. struct beiscsi_io_task *abrt_io_task;
  128. struct iscsi_conn *conn;
  129. struct beiscsi_conn *beiscsi_conn;
  130. struct beiscsi_hba *phba;
  131. struct iscsi_session *session;
  132. struct iscsi_cls_session *cls_session;
  133. struct invalidate_command_table *inv_tbl;
  134. struct be_dma_mem nonemb_cmd;
  135. unsigned int cid, tag, i, num_invalidate;
  136. int rc = FAILED;
  137. /* invalidate iocbs */
  138. cls_session = starget_to_session(scsi_target(sc->device));
  139. session = cls_session->dd_data;
  140. spin_lock_bh(&session->lock);
  141. if (!session->leadconn || session->state != ISCSI_STATE_LOGGED_IN)
  142. goto unlock;
  143. conn = session->leadconn;
  144. beiscsi_conn = conn->dd_data;
  145. phba = beiscsi_conn->phba;
  146. cid = beiscsi_conn->beiscsi_conn_cid;
  147. inv_tbl = phba->inv_tbl;
  148. memset(inv_tbl, 0x0, sizeof(*inv_tbl) * BE2_CMDS_PER_CXN);
  149. num_invalidate = 0;
  150. for (i = 0; i < conn->session->cmds_max; i++) {
  151. abrt_task = conn->session->cmds[i];
  152. abrt_io_task = abrt_task->dd_data;
  153. if (!abrt_task->sc || abrt_task->state == ISCSI_TASK_FREE)
  154. continue;
  155. if (abrt_task->sc->device->lun != abrt_task->sc->device->lun)
  156. continue;
  157. inv_tbl->cid = cid;
  158. inv_tbl->icd = abrt_io_task->psgl_handle->sgl_index;
  159. num_invalidate++;
  160. inv_tbl++;
  161. }
  162. spin_unlock_bh(&session->lock);
  163. inv_tbl = phba->inv_tbl;
  164. nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
  165. sizeof(struct invalidate_commands_params_in),
  166. &nonemb_cmd.dma);
  167. if (nonemb_cmd.va == NULL) {
  168. SE_DEBUG(DBG_LVL_1,
  169. "Failed to allocate memory for"
  170. "mgmt_invalidate_icds\n");
  171. return FAILED;
  172. }
  173. nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
  174. memset(nonemb_cmd.va, 0, nonemb_cmd.size);
  175. tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
  176. cid, &nonemb_cmd);
  177. if (!tag) {
  178. shost_printk(KERN_WARNING, phba->shost,
  179. "mgmt_invalidate_icds could not be"
  180. " submitted\n");
  181. pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
  182. nonemb_cmd.va, nonemb_cmd.dma);
  183. return FAILED;
  184. } else {
  185. wait_event_interruptible(phba->ctrl.mcc_wait[tag],
  186. phba->ctrl.mcc_numtag[tag]);
  187. free_mcc_tag(&phba->ctrl, tag);
  188. }
  189. pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
  190. nonemb_cmd.va, nonemb_cmd.dma);
  191. return iscsi_eh_device_reset(sc);
  192. unlock:
  193. spin_unlock_bh(&session->lock);
  194. return rc;
  195. }
  196. static ssize_t beiscsi_show_boot_tgt_info(void *data, int type, char *buf)
  197. {
  198. struct beiscsi_hba *phba = data;
  199. struct mgmt_session_info *boot_sess = &phba->boot_sess;
  200. struct mgmt_conn_info *boot_conn = &boot_sess->conn_list[0];
  201. char *str = buf;
  202. int rc;
  203. switch (type) {
  204. case ISCSI_BOOT_TGT_NAME:
  205. rc = sprintf(buf, "%.*s\n",
  206. (int)strlen(boot_sess->target_name),
  207. (char *)&boot_sess->target_name);
  208. break;
  209. case ISCSI_BOOT_TGT_IP_ADDR:
  210. if (boot_conn->dest_ipaddr.ip_type == 0x1)
  211. rc = sprintf(buf, "%pI4\n",
  212. (char *)&boot_conn->dest_ipaddr.ip_address);
  213. else
  214. rc = sprintf(str, "%pI6\n",
  215. (char *)&boot_conn->dest_ipaddr.ip_address);
  216. break;
  217. case ISCSI_BOOT_TGT_PORT:
  218. rc = sprintf(str, "%d\n", boot_conn->dest_port);
  219. break;
  220. case ISCSI_BOOT_TGT_CHAP_NAME:
  221. rc = sprintf(str, "%.*s\n",
  222. boot_conn->negotiated_login_options.auth_data.chap.
  223. target_chap_name_length,
  224. (char *)&boot_conn->negotiated_login_options.
  225. auth_data.chap.target_chap_name);
  226. break;
  227. case ISCSI_BOOT_TGT_CHAP_SECRET:
  228. rc = sprintf(str, "%.*s\n",
  229. boot_conn->negotiated_login_options.auth_data.chap.
  230. target_secret_length,
  231. (char *)&boot_conn->negotiated_login_options.
  232. auth_data.chap.target_secret);
  233. break;
  234. case ISCSI_BOOT_TGT_REV_CHAP_NAME:
  235. rc = sprintf(str, "%.*s\n",
  236. boot_conn->negotiated_login_options.auth_data.chap.
  237. intr_chap_name_length,
  238. (char *)&boot_conn->negotiated_login_options.
  239. auth_data.chap.intr_chap_name);
  240. break;
  241. case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
  242. rc = sprintf(str, "%.*s\n",
  243. boot_conn->negotiated_login_options.auth_data.chap.
  244. intr_secret_length,
  245. (char *)&boot_conn->negotiated_login_options.
  246. auth_data.chap.intr_secret);
  247. break;
  248. case ISCSI_BOOT_TGT_FLAGS:
  249. rc = sprintf(str, "2\n");
  250. break;
  251. case ISCSI_BOOT_TGT_NIC_ASSOC:
  252. rc = sprintf(str, "0\n");
  253. break;
  254. default:
  255. rc = -ENOSYS;
  256. break;
  257. }
  258. return rc;
  259. }
  260. static ssize_t beiscsi_show_boot_ini_info(void *data, int type, char *buf)
  261. {
  262. struct beiscsi_hba *phba = data;
  263. char *str = buf;
  264. int rc;
  265. switch (type) {
  266. case ISCSI_BOOT_INI_INITIATOR_NAME:
  267. rc = sprintf(str, "%s\n", phba->boot_sess.initiator_iscsiname);
  268. break;
  269. default:
  270. rc = -ENOSYS;
  271. break;
  272. }
  273. return rc;
  274. }
  275. static ssize_t beiscsi_show_boot_eth_info(void *data, int type, char *buf)
  276. {
  277. struct beiscsi_hba *phba = data;
  278. char *str = buf;
  279. int rc;
  280. switch (type) {
  281. case ISCSI_BOOT_ETH_FLAGS:
  282. rc = sprintf(str, "2\n");
  283. break;
  284. case ISCSI_BOOT_ETH_INDEX:
  285. rc = sprintf(str, "0\n");
  286. break;
  287. case ISCSI_BOOT_ETH_MAC:
  288. rc = beiscsi_get_macaddr(buf, phba);
  289. if (rc < 0) {
  290. SE_DEBUG(DBG_LVL_1, "beiscsi_get_macaddr Failed\n");
  291. return rc;
  292. }
  293. break;
  294. default:
  295. rc = -ENOSYS;
  296. break;
  297. }
  298. return rc;
  299. }
  300. static mode_t beiscsi_tgt_get_attr_visibility(void *data, int type)
  301. {
  302. int rc;
  303. switch (type) {
  304. case ISCSI_BOOT_TGT_NAME:
  305. case ISCSI_BOOT_TGT_IP_ADDR:
  306. case ISCSI_BOOT_TGT_PORT:
  307. case ISCSI_BOOT_TGT_CHAP_NAME:
  308. case ISCSI_BOOT_TGT_CHAP_SECRET:
  309. case ISCSI_BOOT_TGT_REV_CHAP_NAME:
  310. case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
  311. case ISCSI_BOOT_TGT_NIC_ASSOC:
  312. case ISCSI_BOOT_TGT_FLAGS:
  313. rc = S_IRUGO;
  314. break;
  315. default:
  316. rc = 0;
  317. break;
  318. }
  319. return rc;
  320. }
  321. static mode_t beiscsi_ini_get_attr_visibility(void *data, int type)
  322. {
  323. int rc;
  324. switch (type) {
  325. case ISCSI_BOOT_INI_INITIATOR_NAME:
  326. rc = S_IRUGO;
  327. break;
  328. default:
  329. rc = 0;
  330. break;
  331. }
  332. return rc;
  333. }
  334. static mode_t beiscsi_eth_get_attr_visibility(void *data, int type)
  335. {
  336. int rc;
  337. switch (type) {
  338. case ISCSI_BOOT_ETH_FLAGS:
  339. case ISCSI_BOOT_ETH_MAC:
  340. case ISCSI_BOOT_ETH_INDEX:
  341. rc = S_IRUGO;
  342. break;
  343. default:
  344. rc = 0;
  345. break;
  346. }
  347. return rc;
  348. }
  349. /*------------------- PCI Driver operations and data ----------------- */
  350. static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = {
  351. { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
  352. { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
  353. { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
  354. { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
  355. { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) },
  356. { 0 }
  357. };
  358. MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
  359. static struct scsi_host_template beiscsi_sht = {
  360. .module = THIS_MODULE,
  361. .name = "ServerEngines 10Gbe open-iscsi Initiator Driver",
  362. .proc_name = DRV_NAME,
  363. .queuecommand = iscsi_queuecommand,
  364. .change_queue_depth = iscsi_change_queue_depth,
  365. .slave_configure = beiscsi_slave_configure,
  366. .target_alloc = iscsi_target_alloc,
  367. .eh_abort_handler = beiscsi_eh_abort,
  368. .eh_device_reset_handler = beiscsi_eh_device_reset,
  369. .eh_target_reset_handler = iscsi_eh_session_reset,
  370. .sg_tablesize = BEISCSI_SGLIST_ELEMENTS,
  371. .can_queue = BE2_IO_DEPTH,
  372. .this_id = -1,
  373. .max_sectors = BEISCSI_MAX_SECTORS,
  374. .cmd_per_lun = BEISCSI_CMD_PER_LUN,
  375. .use_clustering = ENABLE_CLUSTERING,
  376. };
  377. static struct scsi_transport_template *beiscsi_scsi_transport;
  378. static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev)
  379. {
  380. struct beiscsi_hba *phba;
  381. struct Scsi_Host *shost;
  382. shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0);
  383. if (!shost) {
  384. dev_err(&pcidev->dev, "beiscsi_hba_alloc -"
  385. "iscsi_host_alloc failed\n");
  386. return NULL;
  387. }
  388. shost->dma_boundary = pcidev->dma_mask;
  389. shost->max_id = BE2_MAX_SESSIONS;
  390. shost->max_channel = 0;
  391. shost->max_cmd_len = BEISCSI_MAX_CMD_LEN;
  392. shost->max_lun = BEISCSI_NUM_MAX_LUN;
  393. shost->transportt = beiscsi_scsi_transport;
  394. phba = iscsi_host_priv(shost);
  395. memset(phba, 0, sizeof(*phba));
  396. phba->shost = shost;
  397. phba->pcidev = pci_dev_get(pcidev);
  398. pci_set_drvdata(pcidev, phba);
  399. if (iscsi_host_add(shost, &phba->pcidev->dev))
  400. goto free_devices;
  401. return phba;
  402. free_devices:
  403. pci_dev_put(phba->pcidev);
  404. iscsi_host_free(phba->shost);
  405. return NULL;
  406. }
  407. static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba)
  408. {
  409. if (phba->csr_va) {
  410. iounmap(phba->csr_va);
  411. phba->csr_va = NULL;
  412. }
  413. if (phba->db_va) {
  414. iounmap(phba->db_va);
  415. phba->db_va = NULL;
  416. }
  417. if (phba->pci_va) {
  418. iounmap(phba->pci_va);
  419. phba->pci_va = NULL;
  420. }
  421. }
  422. static int beiscsi_map_pci_bars(struct beiscsi_hba *phba,
  423. struct pci_dev *pcidev)
  424. {
  425. u8 __iomem *addr;
  426. int pcicfg_reg;
  427. addr = ioremap_nocache(pci_resource_start(pcidev, 2),
  428. pci_resource_len(pcidev, 2));
  429. if (addr == NULL)
  430. return -ENOMEM;
  431. phba->ctrl.csr = addr;
  432. phba->csr_va = addr;
  433. phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2);
  434. addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024);
  435. if (addr == NULL)
  436. goto pci_map_err;
  437. phba->ctrl.db = addr;
  438. phba->db_va = addr;
  439. phba->db_pa.u.a64.address = pci_resource_start(pcidev, 4);
  440. if (phba->generation == BE_GEN2)
  441. pcicfg_reg = 1;
  442. else
  443. pcicfg_reg = 0;
  444. addr = ioremap_nocache(pci_resource_start(pcidev, pcicfg_reg),
  445. pci_resource_len(pcidev, pcicfg_reg));
  446. if (addr == NULL)
  447. goto pci_map_err;
  448. phba->ctrl.pcicfg = addr;
  449. phba->pci_va = addr;
  450. phba->pci_pa.u.a64.address = pci_resource_start(pcidev, pcicfg_reg);
  451. return 0;
  452. pci_map_err:
  453. beiscsi_unmap_pci_function(phba);
  454. return -ENOMEM;
  455. }
  456. static int beiscsi_enable_pci(struct pci_dev *pcidev)
  457. {
  458. int ret;
  459. ret = pci_enable_device(pcidev);
  460. if (ret) {
  461. dev_err(&pcidev->dev, "beiscsi_enable_pci - enable device "
  462. "failed. Returning -ENODEV\n");
  463. return ret;
  464. }
  465. pci_set_master(pcidev);
  466. if (pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64))) {
  467. ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32));
  468. if (ret) {
  469. dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");
  470. pci_disable_device(pcidev);
  471. return ret;
  472. }
  473. }
  474. return 0;
  475. }
  476. static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev)
  477. {
  478. struct be_ctrl_info *ctrl = &phba->ctrl;
  479. struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced;
  480. struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem;
  481. int status = 0;
  482. ctrl->pdev = pdev;
  483. status = beiscsi_map_pci_bars(phba, pdev);
  484. if (status)
  485. return status;
  486. mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
  487. mbox_mem_alloc->va = pci_alloc_consistent(pdev,
  488. mbox_mem_alloc->size,
  489. &mbox_mem_alloc->dma);
  490. if (!mbox_mem_alloc->va) {
  491. beiscsi_unmap_pci_function(phba);
  492. status = -ENOMEM;
  493. return status;
  494. }
  495. mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
  496. mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
  497. mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
  498. memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
  499. spin_lock_init(&ctrl->mbox_lock);
  500. spin_lock_init(&phba->ctrl.mcc_lock);
  501. spin_lock_init(&phba->ctrl.mcc_cq_lock);
  502. return status;
  503. }
  504. static void beiscsi_get_params(struct beiscsi_hba *phba)
  505. {
  506. phba->params.ios_per_ctrl = (phba->fw_config.iscsi_icd_count
  507. - (phba->fw_config.iscsi_cid_count
  508. + BE2_TMFS
  509. + BE2_NOPOUT_REQ));
  510. phba->params.cxns_per_ctrl = phba->fw_config.iscsi_cid_count;
  511. phba->params.asyncpdus_per_ctrl = phba->fw_config.iscsi_cid_count * 2;
  512. phba->params.icds_per_ctrl = phba->fw_config.iscsi_icd_count;
  513. phba->params.num_sge_per_io = BE2_SGE;
  514. phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ;
  515. phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ;
  516. phba->params.eq_timer = 64;
  517. phba->params.num_eq_entries =
  518. (((BE2_CMDS_PER_CXN * 2 + phba->fw_config.iscsi_cid_count * 2
  519. + BE2_TMFS) / 512) + 1) * 512;
  520. phba->params.num_eq_entries = (phba->params.num_eq_entries < 1024)
  521. ? 1024 : phba->params.num_eq_entries;
  522. SE_DEBUG(DBG_LVL_8, "phba->params.num_eq_entries=%d\n",
  523. phba->params.num_eq_entries);
  524. phba->params.num_cq_entries =
  525. (((BE2_CMDS_PER_CXN * 2 + phba->fw_config.iscsi_cid_count * 2
  526. + BE2_TMFS) / 512) + 1) * 512;
  527. phba->params.wrbs_per_cxn = 256;
  528. }
  529. static void hwi_ring_eq_db(struct beiscsi_hba *phba,
  530. unsigned int id, unsigned int clr_interrupt,
  531. unsigned int num_processed,
  532. unsigned char rearm, unsigned char event)
  533. {
  534. u32 val = 0;
  535. val |= id & DB_EQ_RING_ID_MASK;
  536. if (rearm)
  537. val |= 1 << DB_EQ_REARM_SHIFT;
  538. if (clr_interrupt)
  539. val |= 1 << DB_EQ_CLR_SHIFT;
  540. if (event)
  541. val |= 1 << DB_EQ_EVNT_SHIFT;
  542. val |= num_processed << DB_EQ_NUM_POPPED_SHIFT;
  543. iowrite32(val, phba->db_va + DB_EQ_OFFSET);
  544. }
  545. /**
  546. * be_isr_mcc - The isr routine of the driver.
  547. * @irq: Not used
  548. * @dev_id: Pointer to host adapter structure
  549. */
  550. static irqreturn_t be_isr_mcc(int irq, void *dev_id)
  551. {
  552. struct beiscsi_hba *phba;
  553. struct be_eq_entry *eqe = NULL;
  554. struct be_queue_info *eq;
  555. struct be_queue_info *mcc;
  556. unsigned int num_eq_processed;
  557. struct be_eq_obj *pbe_eq;
  558. unsigned long flags;
  559. pbe_eq = dev_id;
  560. eq = &pbe_eq->q;
  561. phba = pbe_eq->phba;
  562. mcc = &phba->ctrl.mcc_obj.cq;
  563. eqe = queue_tail_node(eq);
  564. if (!eqe)
  565. SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
  566. num_eq_processed = 0;
  567. while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
  568. & EQE_VALID_MASK) {
  569. if (((eqe->dw[offsetof(struct amap_eq_entry,
  570. resource_id) / 32] &
  571. EQE_RESID_MASK) >> 16) == mcc->id) {
  572. spin_lock_irqsave(&phba->isr_lock, flags);
  573. phba->todo_mcc_cq = 1;
  574. spin_unlock_irqrestore(&phba->isr_lock, flags);
  575. }
  576. AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
  577. queue_tail_inc(eq);
  578. eqe = queue_tail_node(eq);
  579. num_eq_processed++;
  580. }
  581. if (phba->todo_mcc_cq)
  582. queue_work(phba->wq, &phba->work_cqs);
  583. if (num_eq_processed)
  584. hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
  585. return IRQ_HANDLED;
  586. }
  587. /**
  588. * be_isr_msix - The isr routine of the driver.
  589. * @irq: Not used
  590. * @dev_id: Pointer to host adapter structure
  591. */
  592. static irqreturn_t be_isr_msix(int irq, void *dev_id)
  593. {
  594. struct beiscsi_hba *phba;
  595. struct be_eq_entry *eqe = NULL;
  596. struct be_queue_info *eq;
  597. struct be_queue_info *cq;
  598. unsigned int num_eq_processed;
  599. struct be_eq_obj *pbe_eq;
  600. unsigned long flags;
  601. pbe_eq = dev_id;
  602. eq = &pbe_eq->q;
  603. cq = pbe_eq->cq;
  604. eqe = queue_tail_node(eq);
  605. if (!eqe)
  606. SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
  607. phba = pbe_eq->phba;
  608. num_eq_processed = 0;
  609. if (blk_iopoll_enabled) {
  610. while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
  611. & EQE_VALID_MASK) {
  612. if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
  613. blk_iopoll_sched(&pbe_eq->iopoll);
  614. AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
  615. queue_tail_inc(eq);
  616. eqe = queue_tail_node(eq);
  617. num_eq_processed++;
  618. }
  619. if (num_eq_processed)
  620. hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 0, 1);
  621. return IRQ_HANDLED;
  622. } else {
  623. while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
  624. & EQE_VALID_MASK) {
  625. spin_lock_irqsave(&phba->isr_lock, flags);
  626. phba->todo_cq = 1;
  627. spin_unlock_irqrestore(&phba->isr_lock, flags);
  628. AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
  629. queue_tail_inc(eq);
  630. eqe = queue_tail_node(eq);
  631. num_eq_processed++;
  632. }
  633. if (phba->todo_cq)
  634. queue_work(phba->wq, &phba->work_cqs);
  635. if (num_eq_processed)
  636. hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
  637. return IRQ_HANDLED;
  638. }
  639. }
  640. /**
  641. * be_isr - The isr routine of the driver.
  642. * @irq: Not used
  643. * @dev_id: Pointer to host adapter structure
  644. */
  645. static irqreturn_t be_isr(int irq, void *dev_id)
  646. {
  647. struct beiscsi_hba *phba;
  648. struct hwi_controller *phwi_ctrlr;
  649. struct hwi_context_memory *phwi_context;
  650. struct be_eq_entry *eqe = NULL;
  651. struct be_queue_info *eq;
  652. struct be_queue_info *cq;
  653. struct be_queue_info *mcc;
  654. unsigned long flags, index;
  655. unsigned int num_mcceq_processed, num_ioeq_processed;
  656. struct be_ctrl_info *ctrl;
  657. struct be_eq_obj *pbe_eq;
  658. int isr;
  659. phba = dev_id;
  660. ctrl = &phba->ctrl;
  661. isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET +
  662. (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE));
  663. if (!isr)
  664. return IRQ_NONE;
  665. phwi_ctrlr = phba->phwi_ctrlr;
  666. phwi_context = phwi_ctrlr->phwi_ctxt;
  667. pbe_eq = &phwi_context->be_eq[0];
  668. eq = &phwi_context->be_eq[0].q;
  669. mcc = &phba->ctrl.mcc_obj.cq;
  670. index = 0;
  671. eqe = queue_tail_node(eq);
  672. if (!eqe)
  673. SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
  674. num_ioeq_processed = 0;
  675. num_mcceq_processed = 0;
  676. if (blk_iopoll_enabled) {
  677. while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
  678. & EQE_VALID_MASK) {
  679. if (((eqe->dw[offsetof(struct amap_eq_entry,
  680. resource_id) / 32] &
  681. EQE_RESID_MASK) >> 16) == mcc->id) {
  682. spin_lock_irqsave(&phba->isr_lock, flags);
  683. phba->todo_mcc_cq = 1;
  684. spin_unlock_irqrestore(&phba->isr_lock, flags);
  685. num_mcceq_processed++;
  686. } else {
  687. if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
  688. blk_iopoll_sched(&pbe_eq->iopoll);
  689. num_ioeq_processed++;
  690. }
  691. AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
  692. queue_tail_inc(eq);
  693. eqe = queue_tail_node(eq);
  694. }
  695. if (num_ioeq_processed || num_mcceq_processed) {
  696. if (phba->todo_mcc_cq)
  697. queue_work(phba->wq, &phba->work_cqs);
  698. if ((num_mcceq_processed) && (!num_ioeq_processed))
  699. hwi_ring_eq_db(phba, eq->id, 0,
  700. (num_ioeq_processed +
  701. num_mcceq_processed) , 1, 1);
  702. else
  703. hwi_ring_eq_db(phba, eq->id, 0,
  704. (num_ioeq_processed +
  705. num_mcceq_processed), 0, 1);
  706. return IRQ_HANDLED;
  707. } else
  708. return IRQ_NONE;
  709. } else {
  710. cq = &phwi_context->be_cq[0];
  711. while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
  712. & EQE_VALID_MASK) {
  713. if (((eqe->dw[offsetof(struct amap_eq_entry,
  714. resource_id) / 32] &
  715. EQE_RESID_MASK) >> 16) != cq->id) {
  716. spin_lock_irqsave(&phba->isr_lock, flags);
  717. phba->todo_mcc_cq = 1;
  718. spin_unlock_irqrestore(&phba->isr_lock, flags);
  719. } else {
  720. spin_lock_irqsave(&phba->isr_lock, flags);
  721. phba->todo_cq = 1;
  722. spin_unlock_irqrestore(&phba->isr_lock, flags);
  723. }
  724. AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
  725. queue_tail_inc(eq);
  726. eqe = queue_tail_node(eq);
  727. num_ioeq_processed++;
  728. }
  729. if (phba->todo_cq || phba->todo_mcc_cq)
  730. queue_work(phba->wq, &phba->work_cqs);
  731. if (num_ioeq_processed) {
  732. hwi_ring_eq_db(phba, eq->id, 0,
  733. num_ioeq_processed, 1, 1);
  734. return IRQ_HANDLED;
  735. } else
  736. return IRQ_NONE;
  737. }
  738. }
  739. static int beiscsi_init_irqs(struct beiscsi_hba *phba)
  740. {
  741. struct pci_dev *pcidev = phba->pcidev;
  742. struct hwi_controller *phwi_ctrlr;
  743. struct hwi_context_memory *phwi_context;
  744. int ret, msix_vec, i, j;
  745. char desc[32];
  746. phwi_ctrlr = phba->phwi_ctrlr;
  747. phwi_context = phwi_ctrlr->phwi_ctxt;
  748. if (phba->msix_enabled) {
  749. for (i = 0; i < phba->num_cpus; i++) {
  750. sprintf(desc, "beiscsi_msix_%04x", i);
  751. msix_vec = phba->msix_entries[i].vector;
  752. ret = request_irq(msix_vec, be_isr_msix, 0, desc,
  753. &phwi_context->be_eq[i]);
  754. if (ret) {
  755. shost_printk(KERN_ERR, phba->shost,
  756. "beiscsi_init_irqs-Failed to"
  757. "register msix for i = %d\n", i);
  758. if (!i)
  759. return ret;
  760. goto free_msix_irqs;
  761. }
  762. }
  763. msix_vec = phba->msix_entries[i].vector;
  764. ret = request_irq(msix_vec, be_isr_mcc, 0, "beiscsi_msix_mcc",
  765. &phwi_context->be_eq[i]);
  766. if (ret) {
  767. shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-"
  768. "Failed to register beiscsi_msix_mcc\n");
  769. i++;
  770. goto free_msix_irqs;
  771. }
  772. } else {
  773. ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED,
  774. "beiscsi", phba);
  775. if (ret) {
  776. shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-"
  777. "Failed to register irq\\n");
  778. return ret;
  779. }
  780. }
  781. return 0;
  782. free_msix_irqs:
  783. for (j = i - 1; j == 0; j++)
  784. free_irq(msix_vec, &phwi_context->be_eq[j]);
  785. return ret;
  786. }
  787. static void hwi_ring_cq_db(struct beiscsi_hba *phba,
  788. unsigned int id, unsigned int num_processed,
  789. unsigned char rearm, unsigned char event)
  790. {
  791. u32 val = 0;
  792. val |= id & DB_CQ_RING_ID_MASK;
  793. if (rearm)
  794. val |= 1 << DB_CQ_REARM_SHIFT;
  795. val |= num_processed << DB_CQ_NUM_POPPED_SHIFT;
  796. iowrite32(val, phba->db_va + DB_CQ_OFFSET);
  797. }
  798. static unsigned int
  799. beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
  800. struct beiscsi_hba *phba,
  801. unsigned short cid,
  802. struct pdu_base *ppdu,
  803. unsigned long pdu_len,
  804. void *pbuffer, unsigned long buf_len)
  805. {
  806. struct iscsi_conn *conn = beiscsi_conn->conn;
  807. struct iscsi_session *session = conn->session;
  808. struct iscsi_task *task;
  809. struct beiscsi_io_task *io_task;
  810. struct iscsi_hdr *login_hdr;
  811. switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] &
  812. PDUBASE_OPCODE_MASK) {
  813. case ISCSI_OP_NOOP_IN:
  814. pbuffer = NULL;
  815. buf_len = 0;
  816. break;
  817. case ISCSI_OP_ASYNC_EVENT:
  818. break;
  819. case ISCSI_OP_REJECT:
  820. WARN_ON(!pbuffer);
  821. WARN_ON(!(buf_len == 48));
  822. SE_DEBUG(DBG_LVL_1, "In ISCSI_OP_REJECT\n");
  823. break;
  824. case ISCSI_OP_LOGIN_RSP:
  825. case ISCSI_OP_TEXT_RSP:
  826. task = conn->login_task;
  827. io_task = task->dd_data;
  828. login_hdr = (struct iscsi_hdr *)ppdu;
  829. login_hdr->itt = io_task->libiscsi_itt;
  830. break;
  831. default:
  832. shost_printk(KERN_WARNING, phba->shost,
  833. "Unrecognized opcode 0x%x in async msg\n",
  834. (ppdu->
  835. dw[offsetof(struct amap_pdu_base, opcode) / 32]
  836. & PDUBASE_OPCODE_MASK));
  837. return 1;
  838. }
  839. spin_lock_bh(&session->lock);
  840. __iscsi_complete_pdu(conn, (struct iscsi_hdr *)ppdu, pbuffer, buf_len);
  841. spin_unlock_bh(&session->lock);
  842. return 0;
  843. }
  844. static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
  845. {
  846. struct sgl_handle *psgl_handle;
  847. if (phba->io_sgl_hndl_avbl) {
  848. SE_DEBUG(DBG_LVL_8,
  849. "In alloc_io_sgl_handle,io_sgl_alloc_index=%d\n",
  850. phba->io_sgl_alloc_index);
  851. psgl_handle = phba->io_sgl_hndl_base[phba->
  852. io_sgl_alloc_index];
  853. phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL;
  854. phba->io_sgl_hndl_avbl--;
  855. if (phba->io_sgl_alloc_index == (phba->params.
  856. ios_per_ctrl - 1))
  857. phba->io_sgl_alloc_index = 0;
  858. else
  859. phba->io_sgl_alloc_index++;
  860. } else
  861. psgl_handle = NULL;
  862. return psgl_handle;
  863. }
  864. static void
  865. free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
  866. {
  867. SE_DEBUG(DBG_LVL_8, "In free_,io_sgl_free_index=%d\n",
  868. phba->io_sgl_free_index);
  869. if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) {
  870. /*
  871. * this can happen if clean_task is called on a task that
  872. * failed in xmit_task or alloc_pdu.
  873. */
  874. SE_DEBUG(DBG_LVL_8,
  875. "Double Free in IO SGL io_sgl_free_index=%d,"
  876. "value there=%p\n", phba->io_sgl_free_index,
  877. phba->io_sgl_hndl_base[phba->io_sgl_free_index]);
  878. return;
  879. }
  880. phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle;
  881. phba->io_sgl_hndl_avbl++;
  882. if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1))
  883. phba->io_sgl_free_index = 0;
  884. else
  885. phba->io_sgl_free_index++;
  886. }
  887. /**
  888. * alloc_wrb_handle - To allocate a wrb handle
  889. * @phba: The hba pointer
  890. * @cid: The cid to use for allocation
  891. *
  892. * This happens under session_lock until submission to chip
  893. */
  894. struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid)
  895. {
  896. struct hwi_wrb_context *pwrb_context;
  897. struct hwi_controller *phwi_ctrlr;
  898. struct wrb_handle *pwrb_handle, *pwrb_handle_tmp;
  899. phwi_ctrlr = phba->phwi_ctrlr;
  900. pwrb_context = &phwi_ctrlr->wrb_context[cid];
  901. if (pwrb_context->wrb_handles_available >= 2) {
  902. pwrb_handle = pwrb_context->pwrb_handle_base[
  903. pwrb_context->alloc_index];
  904. pwrb_context->wrb_handles_available--;
  905. if (pwrb_context->alloc_index ==
  906. (phba->params.wrbs_per_cxn - 1))
  907. pwrb_context->alloc_index = 0;
  908. else
  909. pwrb_context->alloc_index++;
  910. pwrb_handle_tmp = pwrb_context->pwrb_handle_base[
  911. pwrb_context->alloc_index];
  912. pwrb_handle->nxt_wrb_index = pwrb_handle_tmp->wrb_index;
  913. } else
  914. pwrb_handle = NULL;
  915. return pwrb_handle;
  916. }
  917. /**
  918. * free_wrb_handle - To free the wrb handle back to pool
  919. * @phba: The hba pointer
  920. * @pwrb_context: The context to free from
  921. * @pwrb_handle: The wrb_handle to free
  922. *
  923. * This happens under session_lock until submission to chip
  924. */
  925. static void
  926. free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context,
  927. struct wrb_handle *pwrb_handle)
  928. {
  929. pwrb_context->pwrb_handle_base[pwrb_context->free_index] = pwrb_handle;
  930. pwrb_context->wrb_handles_available++;
  931. if (pwrb_context->free_index == (phba->params.wrbs_per_cxn - 1))
  932. pwrb_context->free_index = 0;
  933. else
  934. pwrb_context->free_index++;
  935. SE_DEBUG(DBG_LVL_8,
  936. "FREE WRB: pwrb_handle=%p free_index=0x%x"
  937. "wrb_handles_available=%d\n",
  938. pwrb_handle, pwrb_context->free_index,
  939. pwrb_context->wrb_handles_available);
  940. }
  941. static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba)
  942. {
  943. struct sgl_handle *psgl_handle;
  944. if (phba->eh_sgl_hndl_avbl) {
  945. psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index];
  946. phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL;
  947. SE_DEBUG(DBG_LVL_8, "mgmt_sgl_alloc_index=%d=0x%x\n",
  948. phba->eh_sgl_alloc_index, phba->eh_sgl_alloc_index);
  949. phba->eh_sgl_hndl_avbl--;
  950. if (phba->eh_sgl_alloc_index ==
  951. (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl -
  952. 1))
  953. phba->eh_sgl_alloc_index = 0;
  954. else
  955. phba->eh_sgl_alloc_index++;
  956. } else
  957. psgl_handle = NULL;
  958. return psgl_handle;
  959. }
  960. void
  961. free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
  962. {
  963. SE_DEBUG(DBG_LVL_8, "In free_mgmt_sgl_handle,eh_sgl_free_index=%d\n",
  964. phba->eh_sgl_free_index);
  965. if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) {
  966. /*
  967. * this can happen if clean_task is called on a task that
  968. * failed in xmit_task or alloc_pdu.
  969. */
  970. SE_DEBUG(DBG_LVL_8,
  971. "Double Free in eh SGL ,eh_sgl_free_index=%d\n",
  972. phba->eh_sgl_free_index);
  973. return;
  974. }
  975. phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle;
  976. phba->eh_sgl_hndl_avbl++;
  977. if (phba->eh_sgl_free_index ==
  978. (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1))
  979. phba->eh_sgl_free_index = 0;
  980. else
  981. phba->eh_sgl_free_index++;
  982. }
  983. static void
  984. be_complete_io(struct beiscsi_conn *beiscsi_conn,
  985. struct iscsi_task *task, struct sol_cqe *psol)
  986. {
  987. struct beiscsi_io_task *io_task = task->dd_data;
  988. struct be_status_bhs *sts_bhs =
  989. (struct be_status_bhs *)io_task->cmd_bhs;
  990. struct iscsi_conn *conn = beiscsi_conn->conn;
  991. unsigned int sense_len;
  992. unsigned char *sense;
  993. u32 resid = 0, exp_cmdsn, max_cmdsn;
  994. u8 rsp, status, flags;
  995. exp_cmdsn = (psol->
  996. dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
  997. & SOL_EXP_CMD_SN_MASK);
  998. max_cmdsn = ((psol->
  999. dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
  1000. & SOL_EXP_CMD_SN_MASK) +
  1001. ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
  1002. / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
  1003. rsp = ((psol->dw[offsetof(struct amap_sol_cqe, i_resp) / 32]
  1004. & SOL_RESP_MASK) >> 16);
  1005. status = ((psol->dw[offsetof(struct amap_sol_cqe, i_sts) / 32]
  1006. & SOL_STS_MASK) >> 8);
  1007. flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
  1008. & SOL_FLAGS_MASK) >> 24) | 0x80;
  1009. task->sc->result = (DID_OK << 16) | status;
  1010. if (rsp != ISCSI_STATUS_CMD_COMPLETED) {
  1011. task->sc->result = DID_ERROR << 16;
  1012. goto unmap;
  1013. }
  1014. /* bidi not initially supported */
  1015. if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) {
  1016. resid = (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) /
  1017. 32] & SOL_RES_CNT_MASK);
  1018. if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW))
  1019. task->sc->result = DID_ERROR << 16;
  1020. if (flags & ISCSI_FLAG_CMD_UNDERFLOW) {
  1021. scsi_set_resid(task->sc, resid);
  1022. if (!status && (scsi_bufflen(task->sc) - resid <
  1023. task->sc->underflow))
  1024. task->sc->result = DID_ERROR << 16;
  1025. }
  1026. }
  1027. if (status == SAM_STAT_CHECK_CONDITION) {
  1028. unsigned short *slen = (unsigned short *)sts_bhs->sense_info;
  1029. sense = sts_bhs->sense_info + sizeof(unsigned short);
  1030. sense_len = cpu_to_be16(*slen);
  1031. memcpy(task->sc->sense_buffer, sense,
  1032. min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE));
  1033. }
  1034. if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ) {
  1035. if (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
  1036. & SOL_RES_CNT_MASK)
  1037. conn->rxdata_octets += (psol->
  1038. dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
  1039. & SOL_RES_CNT_MASK);
  1040. }
  1041. unmap:
  1042. scsi_dma_unmap(io_task->scsi_cmnd);
  1043. iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn);
  1044. }
  1045. static void
  1046. be_complete_logout(struct beiscsi_conn *beiscsi_conn,
  1047. struct iscsi_task *task, struct sol_cqe *psol)
  1048. {
  1049. struct iscsi_logout_rsp *hdr;
  1050. struct beiscsi_io_task *io_task = task->dd_data;
  1051. struct iscsi_conn *conn = beiscsi_conn->conn;
  1052. hdr = (struct iscsi_logout_rsp *)task->hdr;
  1053. hdr->opcode = ISCSI_OP_LOGOUT_RSP;
  1054. hdr->t2wait = 5;
  1055. hdr->t2retain = 0;
  1056. hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
  1057. & SOL_FLAGS_MASK) >> 24) | 0x80;
  1058. hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
  1059. 32] & SOL_RESP_MASK);
  1060. hdr->exp_cmdsn = cpu_to_be32(psol->
  1061. dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
  1062. & SOL_EXP_CMD_SN_MASK);
  1063. hdr->max_cmdsn = be32_to_cpu((psol->
  1064. dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
  1065. & SOL_EXP_CMD_SN_MASK) +
  1066. ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
  1067. / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
  1068. hdr->dlength[0] = 0;
  1069. hdr->dlength[1] = 0;
  1070. hdr->dlength[2] = 0;
  1071. hdr->hlength = 0;
  1072. hdr->itt = io_task->libiscsi_itt;
  1073. __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
  1074. }
  1075. static void
  1076. be_complete_tmf(struct beiscsi_conn *beiscsi_conn,
  1077. struct iscsi_task *task, struct sol_cqe *psol)
  1078. {
  1079. struct iscsi_tm_rsp *hdr;
  1080. struct iscsi_conn *conn = beiscsi_conn->conn;
  1081. struct beiscsi_io_task *io_task = task->dd_data;
  1082. hdr = (struct iscsi_tm_rsp *)task->hdr;
  1083. hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP;
  1084. hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
  1085. & SOL_FLAGS_MASK) >> 24) | 0x80;
  1086. hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
  1087. 32] & SOL_RESP_MASK);
  1088. hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
  1089. i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
  1090. hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
  1091. i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
  1092. ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
  1093. / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
  1094. hdr->itt = io_task->libiscsi_itt;
  1095. __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
  1096. }
  1097. static void
  1098. hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
  1099. struct beiscsi_hba *phba, struct sol_cqe *psol)
  1100. {
  1101. struct hwi_wrb_context *pwrb_context;
  1102. struct wrb_handle *pwrb_handle = NULL;
  1103. struct hwi_controller *phwi_ctrlr;
  1104. struct iscsi_task *task;
  1105. struct beiscsi_io_task *io_task;
  1106. struct iscsi_conn *conn = beiscsi_conn->conn;
  1107. struct iscsi_session *session = conn->session;
  1108. phwi_ctrlr = phba->phwi_ctrlr;
  1109. pwrb_context = &phwi_ctrlr->wrb_context[((psol->
  1110. dw[offsetof(struct amap_sol_cqe, cid) / 32] &
  1111. SOL_CID_MASK) >> 6) -
  1112. phba->fw_config.iscsi_cid_start];
  1113. pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
  1114. dw[offsetof(struct amap_sol_cqe, wrb_index) /
  1115. 32] & SOL_WRB_INDEX_MASK) >> 16)];
  1116. task = pwrb_handle->pio_handle;
  1117. io_task = task->dd_data;
  1118. spin_lock(&phba->mgmt_sgl_lock);
  1119. free_mgmt_sgl_handle(phba, io_task->psgl_handle);
  1120. spin_unlock(&phba->mgmt_sgl_lock);
  1121. spin_lock_bh(&session->lock);
  1122. free_wrb_handle(phba, pwrb_context, pwrb_handle);
  1123. spin_unlock_bh(&session->lock);
  1124. }
  1125. static void
  1126. be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn,
  1127. struct iscsi_task *task, struct sol_cqe *psol)
  1128. {
  1129. struct iscsi_nopin *hdr;
  1130. struct iscsi_conn *conn = beiscsi_conn->conn;
  1131. struct beiscsi_io_task *io_task = task->dd_data;
  1132. hdr = (struct iscsi_nopin *)task->hdr;
  1133. hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
  1134. & SOL_FLAGS_MASK) >> 24) | 0x80;
  1135. hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
  1136. i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
  1137. hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
  1138. i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
  1139. ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
  1140. / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
  1141. hdr->opcode = ISCSI_OP_NOOP_IN;
  1142. hdr->itt = io_task->libiscsi_itt;
  1143. __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
  1144. }
  1145. static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
  1146. struct beiscsi_hba *phba, struct sol_cqe *psol)
  1147. {
  1148. struct hwi_wrb_context *pwrb_context;
  1149. struct wrb_handle *pwrb_handle;
  1150. struct iscsi_wrb *pwrb = NULL;
  1151. struct hwi_controller *phwi_ctrlr;
  1152. struct iscsi_task *task;
  1153. unsigned int type;
  1154. struct iscsi_conn *conn = beiscsi_conn->conn;
  1155. struct iscsi_session *session = conn->session;
  1156. phwi_ctrlr = phba->phwi_ctrlr;
  1157. pwrb_context = &phwi_ctrlr->wrb_context[((psol->dw[offsetof
  1158. (struct amap_sol_cqe, cid) / 32]
  1159. & SOL_CID_MASK) >> 6) -
  1160. phba->fw_config.iscsi_cid_start];
  1161. pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
  1162. dw[offsetof(struct amap_sol_cqe, wrb_index) /
  1163. 32] & SOL_WRB_INDEX_MASK) >> 16)];
  1164. task = pwrb_handle->pio_handle;
  1165. pwrb = pwrb_handle->pwrb;
  1166. type = (pwrb->dw[offsetof(struct amap_iscsi_wrb, type) / 32] &
  1167. WRB_TYPE_MASK) >> 28;
  1168. spin_lock_bh(&session->lock);
  1169. switch (type) {
  1170. case HWH_TYPE_IO:
  1171. case HWH_TYPE_IO_RD:
  1172. if ((task->hdr->opcode & ISCSI_OPCODE_MASK) ==
  1173. ISCSI_OP_NOOP_OUT)
  1174. be_complete_nopin_resp(beiscsi_conn, task, psol);
  1175. else
  1176. be_complete_io(beiscsi_conn, task, psol);
  1177. break;
  1178. case HWH_TYPE_LOGOUT:
  1179. if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGOUT)
  1180. be_complete_logout(beiscsi_conn, task, psol);
  1181. else
  1182. be_complete_tmf(beiscsi_conn, task, psol);
  1183. break;
  1184. case HWH_TYPE_LOGIN:
  1185. SE_DEBUG(DBG_LVL_1,
  1186. "\t\t No HWH_TYPE_LOGIN Expected in hwi_complete_cmd"
  1187. "- Solicited path\n");
  1188. break;
  1189. case HWH_TYPE_NOP:
  1190. be_complete_nopin_resp(beiscsi_conn, task, psol);
  1191. break;
  1192. default:
  1193. shost_printk(KERN_WARNING, phba->shost,
  1194. "In hwi_complete_cmd, unknown type = %d"
  1195. "wrb_index 0x%x CID 0x%x\n", type,
  1196. ((psol->dw[offsetof(struct amap_iscsi_wrb,
  1197. type) / 32] & SOL_WRB_INDEX_MASK) >> 16),
  1198. ((psol->dw[offsetof(struct amap_sol_cqe,
  1199. cid) / 32] & SOL_CID_MASK) >> 6));
  1200. break;
  1201. }
  1202. spin_unlock_bh(&session->lock);
  1203. }
  1204. static struct list_head *hwi_get_async_busy_list(struct hwi_async_pdu_context
  1205. *pasync_ctx, unsigned int is_header,
  1206. unsigned int host_write_ptr)
  1207. {
  1208. if (is_header)
  1209. return &pasync_ctx->async_entry[host_write_ptr].
  1210. header_busy_list;
  1211. else
  1212. return &pasync_ctx->async_entry[host_write_ptr].data_busy_list;
  1213. }
  1214. static struct async_pdu_handle *
  1215. hwi_get_async_handle(struct beiscsi_hba *phba,
  1216. struct beiscsi_conn *beiscsi_conn,
  1217. struct hwi_async_pdu_context *pasync_ctx,
  1218. struct i_t_dpdu_cqe *pdpdu_cqe, unsigned int *pcq_index)
  1219. {
  1220. struct be_bus_address phys_addr;
  1221. struct list_head *pbusy_list;
  1222. struct async_pdu_handle *pasync_handle = NULL;
  1223. int buffer_len = 0;
  1224. unsigned char buffer_index = -1;
  1225. unsigned char is_header = 0;
  1226. phys_addr.u.a32.address_lo =
  1227. pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_lo) / 32] -
  1228. ((pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
  1229. & PDUCQE_DPL_MASK) >> 16);
  1230. phys_addr.u.a32.address_hi =
  1231. pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_hi) / 32];
  1232. phys_addr.u.a64.address =
  1233. *((unsigned long long *)(&phys_addr.u.a64.address));
  1234. switch (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, code) / 32]
  1235. & PDUCQE_CODE_MASK) {
  1236. case UNSOL_HDR_NOTIFY:
  1237. is_header = 1;
  1238. pbusy_list = hwi_get_async_busy_list(pasync_ctx, 1,
  1239. (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
  1240. index) / 32] & PDUCQE_INDEX_MASK));
  1241. buffer_len = (unsigned int)(phys_addr.u.a64.address -
  1242. pasync_ctx->async_header.pa_base.u.a64.address);
  1243. buffer_index = buffer_len /
  1244. pasync_ctx->async_header.buffer_size;
  1245. break;
  1246. case UNSOL_DATA_NOTIFY:
  1247. pbusy_list = hwi_get_async_busy_list(pasync_ctx, 0, (pdpdu_cqe->
  1248. dw[offsetof(struct amap_i_t_dpdu_cqe,
  1249. index) / 32] & PDUCQE_INDEX_MASK));
  1250. buffer_len = (unsigned long)(phys_addr.u.a64.address -
  1251. pasync_ctx->async_data.pa_base.u.
  1252. a64.address);
  1253. buffer_index = buffer_len / pasync_ctx->async_data.buffer_size;
  1254. break;
  1255. default:
  1256. pbusy_list = NULL;
  1257. shost_printk(KERN_WARNING, phba->shost,
  1258. "Unexpected code=%d\n",
  1259. pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
  1260. code) / 32] & PDUCQE_CODE_MASK);
  1261. return NULL;
  1262. }
  1263. WARN_ON(!(buffer_index <= pasync_ctx->async_data.num_entries));
  1264. WARN_ON(list_empty(pbusy_list));
  1265. list_for_each_entry(pasync_handle, pbusy_list, link) {
  1266. WARN_ON(pasync_handle->consumed);
  1267. if (pasync_handle->index == buffer_index)
  1268. break;
  1269. }
  1270. WARN_ON(!pasync_handle);
  1271. pasync_handle->cri = (unsigned short)beiscsi_conn->beiscsi_conn_cid -
  1272. phba->fw_config.iscsi_cid_start;
  1273. pasync_handle->is_header = is_header;
  1274. pasync_handle->buffer_len = ((pdpdu_cqe->
  1275. dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
  1276. & PDUCQE_DPL_MASK) >> 16);
  1277. *pcq_index = (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
  1278. index) / 32] & PDUCQE_INDEX_MASK);
  1279. return pasync_handle;
  1280. }
  1281. static unsigned int
  1282. hwi_update_async_writables(struct hwi_async_pdu_context *pasync_ctx,
  1283. unsigned int is_header, unsigned int cq_index)
  1284. {
  1285. struct list_head *pbusy_list;
  1286. struct async_pdu_handle *pasync_handle;
  1287. unsigned int num_entries, writables = 0;
  1288. unsigned int *pep_read_ptr, *pwritables;
  1289. if (is_header) {
  1290. pep_read_ptr = &pasync_ctx->async_header.ep_read_ptr;
  1291. pwritables = &pasync_ctx->async_header.writables;
  1292. num_entries = pasync_ctx->async_header.num_entries;
  1293. } else {
  1294. pep_read_ptr = &pasync_ctx->async_data.ep_read_ptr;
  1295. pwritables = &pasync_ctx->async_data.writables;
  1296. num_entries = pasync_ctx->async_data.num_entries;
  1297. }
  1298. while ((*pep_read_ptr) != cq_index) {
  1299. (*pep_read_ptr)++;
  1300. *pep_read_ptr = (*pep_read_ptr) % num_entries;
  1301. pbusy_list = hwi_get_async_busy_list(pasync_ctx, is_header,
  1302. *pep_read_ptr);
  1303. if (writables == 0)
  1304. WARN_ON(list_empty(pbusy_list));
  1305. if (!list_empty(pbusy_list)) {
  1306. pasync_handle = list_entry(pbusy_list->next,
  1307. struct async_pdu_handle,
  1308. link);
  1309. WARN_ON(!pasync_handle);
  1310. pasync_handle->consumed = 1;
  1311. }
  1312. writables++;
  1313. }
  1314. if (!writables) {
  1315. SE_DEBUG(DBG_LVL_1,
  1316. "Duplicate notification received - index 0x%x!!\n",
  1317. cq_index);
  1318. WARN_ON(1);
  1319. }
  1320. *pwritables = *pwritables + writables;
  1321. return 0;
  1322. }
  1323. static unsigned int hwi_free_async_msg(struct beiscsi_hba *phba,
  1324. unsigned int cri)
  1325. {
  1326. struct hwi_controller *phwi_ctrlr;
  1327. struct hwi_async_pdu_context *pasync_ctx;
  1328. struct async_pdu_handle *pasync_handle, *tmp_handle;
  1329. struct list_head *plist;
  1330. unsigned int i = 0;
  1331. phwi_ctrlr = phba->phwi_ctrlr;
  1332. pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
  1333. plist = &pasync_ctx->async_entry[cri].wait_queue.list;
  1334. list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) {
  1335. list_del(&pasync_handle->link);
  1336. if (i == 0) {
  1337. list_add_tail(&pasync_handle->link,
  1338. &pasync_ctx->async_header.free_list);
  1339. pasync_ctx->async_header.free_entries++;
  1340. i++;
  1341. } else {
  1342. list_add_tail(&pasync_handle->link,
  1343. &pasync_ctx->async_data.free_list);
  1344. pasync_ctx->async_data.free_entries++;
  1345. i++;
  1346. }
  1347. }
  1348. INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wait_queue.list);
  1349. pasync_ctx->async_entry[cri].wait_queue.hdr_received = 0;
  1350. pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
  1351. return 0;
  1352. }
  1353. static struct phys_addr *
  1354. hwi_get_ring_address(struct hwi_async_pdu_context *pasync_ctx,
  1355. unsigned int is_header, unsigned int host_write_ptr)
  1356. {
  1357. struct phys_addr *pasync_sge = NULL;
  1358. if (is_header)
  1359. pasync_sge = pasync_ctx->async_header.ring_base;
  1360. else
  1361. pasync_sge = pasync_ctx->async_data.ring_base;
  1362. return pasync_sge + host_write_ptr;
  1363. }
  1364. static void hwi_post_async_buffers(struct beiscsi_hba *phba,
  1365. unsigned int is_header)
  1366. {
  1367. struct hwi_controller *phwi_ctrlr;
  1368. struct hwi_async_pdu_context *pasync_ctx;
  1369. struct async_pdu_handle *pasync_handle;
  1370. struct list_head *pfree_link, *pbusy_list;
  1371. struct phys_addr *pasync_sge;
  1372. unsigned int ring_id, num_entries;
  1373. unsigned int host_write_num;
  1374. unsigned int writables;
  1375. unsigned int i = 0;
  1376. u32 doorbell = 0;
  1377. phwi_ctrlr = phba->phwi_ctrlr;
  1378. pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
  1379. if (is_header) {
  1380. num_entries = pasync_ctx->async_header.num_entries;
  1381. writables = min(pasync_ctx->async_header.writables,
  1382. pasync_ctx->async_header.free_entries);
  1383. pfree_link = pasync_ctx->async_header.free_list.next;
  1384. host_write_num = pasync_ctx->async_header.host_write_ptr;
  1385. ring_id = phwi_ctrlr->default_pdu_hdr.id;
  1386. } else {
  1387. num_entries = pasync_ctx->async_data.num_entries;
  1388. writables = min(pasync_ctx->async_data.writables,
  1389. pasync_ctx->async_data.free_entries);
  1390. pfree_link = pasync_ctx->async_data.free_list.next;
  1391. host_write_num = pasync_ctx->async_data.host_write_ptr;
  1392. ring_id = phwi_ctrlr->default_pdu_data.id;
  1393. }
  1394. writables = (writables / 8) * 8;
  1395. if (writables) {
  1396. for (i = 0; i < writables; i++) {
  1397. pbusy_list =
  1398. hwi_get_async_busy_list(pasync_ctx, is_header,
  1399. host_write_num);
  1400. pasync_handle =
  1401. list_entry(pfree_link, struct async_pdu_handle,
  1402. link);
  1403. WARN_ON(!pasync_handle);
  1404. pasync_handle->consumed = 0;
  1405. pfree_link = pfree_link->next;
  1406. pasync_sge = hwi_get_ring_address(pasync_ctx,
  1407. is_header, host_write_num);
  1408. pasync_sge->hi = pasync_handle->pa.u.a32.address_lo;
  1409. pasync_sge->lo = pasync_handle->pa.u.a32.address_hi;
  1410. list_move(&pasync_handle->link, pbusy_list);
  1411. host_write_num++;
  1412. host_write_num = host_write_num % num_entries;
  1413. }
  1414. if (is_header) {
  1415. pasync_ctx->async_header.host_write_ptr =
  1416. host_write_num;
  1417. pasync_ctx->async_header.free_entries -= writables;
  1418. pasync_ctx->async_header.writables -= writables;
  1419. pasync_ctx->async_header.busy_entries += writables;
  1420. } else {
  1421. pasync_ctx->async_data.host_write_ptr = host_write_num;
  1422. pasync_ctx->async_data.free_entries -= writables;
  1423. pasync_ctx->async_data.writables -= writables;
  1424. pasync_ctx->async_data.busy_entries += writables;
  1425. }
  1426. doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK;
  1427. doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT;
  1428. doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT;
  1429. doorbell |= (writables & DB_DEF_PDU_CQPROC_MASK)
  1430. << DB_DEF_PDU_CQPROC_SHIFT;
  1431. iowrite32(doorbell, phba->db_va + DB_RXULP0_OFFSET);
  1432. }
  1433. }
  1434. static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba,
  1435. struct beiscsi_conn *beiscsi_conn,
  1436. struct i_t_dpdu_cqe *pdpdu_cqe)
  1437. {
  1438. struct hwi_controller *phwi_ctrlr;
  1439. struct hwi_async_pdu_context *pasync_ctx;
  1440. struct async_pdu_handle *pasync_handle = NULL;
  1441. unsigned int cq_index = -1;
  1442. phwi_ctrlr = phba->phwi_ctrlr;
  1443. pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
  1444. pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
  1445. pdpdu_cqe, &cq_index);
  1446. BUG_ON(pasync_handle->is_header != 0);
  1447. if (pasync_handle->consumed == 0)
  1448. hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
  1449. cq_index);
  1450. hwi_free_async_msg(phba, pasync_handle->cri);
  1451. hwi_post_async_buffers(phba, pasync_handle->is_header);
  1452. }
  1453. static unsigned int
  1454. hwi_fwd_async_msg(struct beiscsi_conn *beiscsi_conn,
  1455. struct beiscsi_hba *phba,
  1456. struct hwi_async_pdu_context *pasync_ctx, unsigned short cri)
  1457. {
  1458. struct list_head *plist;
  1459. struct async_pdu_handle *pasync_handle;
  1460. void *phdr = NULL;
  1461. unsigned int hdr_len = 0, buf_len = 0;
  1462. unsigned int status, index = 0, offset = 0;
  1463. void *pfirst_buffer = NULL;
  1464. unsigned int num_buf = 0;
  1465. plist = &pasync_ctx->async_entry[cri].wait_queue.list;
  1466. list_for_each_entry(pasync_handle, plist, link) {
  1467. if (index == 0) {
  1468. phdr = pasync_handle->pbuffer;
  1469. hdr_len = pasync_handle->buffer_len;
  1470. } else {
  1471. buf_len = pasync_handle->buffer_len;
  1472. if (!num_buf) {
  1473. pfirst_buffer = pasync_handle->pbuffer;
  1474. num_buf++;
  1475. }
  1476. memcpy(pfirst_buffer + offset,
  1477. pasync_handle->pbuffer, buf_len);
  1478. offset = buf_len;
  1479. }
  1480. index++;
  1481. }
  1482. status = beiscsi_process_async_pdu(beiscsi_conn, phba,
  1483. (beiscsi_conn->beiscsi_conn_cid -
  1484. phba->fw_config.iscsi_cid_start),
  1485. phdr, hdr_len, pfirst_buffer,
  1486. buf_len);
  1487. if (status == 0)
  1488. hwi_free_async_msg(phba, cri);
  1489. return 0;
  1490. }
  1491. static unsigned int
  1492. hwi_gather_async_pdu(struct beiscsi_conn *beiscsi_conn,
  1493. struct beiscsi_hba *phba,
  1494. struct async_pdu_handle *pasync_handle)
  1495. {
  1496. struct hwi_async_pdu_context *pasync_ctx;
  1497. struct hwi_controller *phwi_ctrlr;
  1498. unsigned int bytes_needed = 0, status = 0;
  1499. unsigned short cri = pasync_handle->cri;
  1500. struct pdu_base *ppdu;
  1501. phwi_ctrlr = phba->phwi_ctrlr;
  1502. pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
  1503. list_del(&pasync_handle->link);
  1504. if (pasync_handle->is_header) {
  1505. pasync_ctx->async_header.busy_entries--;
  1506. if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
  1507. hwi_free_async_msg(phba, cri);
  1508. BUG();
  1509. }
  1510. pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
  1511. pasync_ctx->async_entry[cri].wait_queue.hdr_received = 1;
  1512. pasync_ctx->async_entry[cri].wait_queue.hdr_len =
  1513. (unsigned short)pasync_handle->buffer_len;
  1514. list_add_tail(&pasync_handle->link,
  1515. &pasync_ctx->async_entry[cri].wait_queue.list);
  1516. ppdu = pasync_handle->pbuffer;
  1517. bytes_needed = ((((ppdu->dw[offsetof(struct amap_pdu_base,
  1518. data_len_hi) / 32] & PDUBASE_DATALENHI_MASK) << 8) &
  1519. 0xFFFF0000) | ((be16_to_cpu((ppdu->
  1520. dw[offsetof(struct amap_pdu_base, data_len_lo) / 32]
  1521. & PDUBASE_DATALENLO_MASK) >> 16)) & 0x0000FFFF));
  1522. if (status == 0) {
  1523. pasync_ctx->async_entry[cri].wait_queue.bytes_needed =
  1524. bytes_needed;
  1525. if (bytes_needed == 0)
  1526. status = hwi_fwd_async_msg(beiscsi_conn, phba,
  1527. pasync_ctx, cri);
  1528. }
  1529. } else {
  1530. pasync_ctx->async_data.busy_entries--;
  1531. if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
  1532. list_add_tail(&pasync_handle->link,
  1533. &pasync_ctx->async_entry[cri].wait_queue.
  1534. list);
  1535. pasync_ctx->async_entry[cri].wait_queue.
  1536. bytes_received +=
  1537. (unsigned short)pasync_handle->buffer_len;
  1538. if (pasync_ctx->async_entry[cri].wait_queue.
  1539. bytes_received >=
  1540. pasync_ctx->async_entry[cri].wait_queue.
  1541. bytes_needed)
  1542. status = hwi_fwd_asy