/drivers/scsi/be2iscsi/be_main.c
https://bitbucket.org/cyanogenmod/android_kernel_asus_tf300t · C · 4461 lines · 3901 code · 471 blank · 89 comment · 425 complexity · 9f338297faf2b91c3eafe7613b4ab8e4 MD5 · raw file
Large files are truncated click here to view the full file
- /**
- * Copyright (C) 2005 - 2011 Emulex
- * All rights reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version 2
- * as published by the Free Software Foundation. The full GNU General
- * Public License is included in this distribution in the file called COPYING.
- *
- * Written by: Jayamohan Kallickal (jayamohan.kallickal@emulex.com)
- *
- * Contact Information:
- * linux-drivers@emulex.com
- *
- * Emulex
- * 3333 Susan Street
- * Costa Mesa, CA 92626
- */
- #include <linux/reboot.h>
- #include <linux/delay.h>
- #include <linux/slab.h>
- #include <linux/interrupt.h>
- #include <linux/blkdev.h>
- #include <linux/pci.h>
- #include <linux/string.h>
- #include <linux/kernel.h>
- #include <linux/semaphore.h>
- #include <linux/iscsi_boot_sysfs.h>
- #include <scsi/libiscsi.h>
- #include <scsi/scsi_transport_iscsi.h>
- #include <scsi/scsi_transport.h>
- #include <scsi/scsi_cmnd.h>
- #include <scsi/scsi_device.h>
- #include <scsi/scsi_host.h>
- #include <scsi/scsi.h>
- #include "be_main.h"
- #include "be_iscsi.h"
- #include "be_mgmt.h"
- static unsigned int be_iopoll_budget = 10;
- static unsigned int be_max_phys_size = 64;
- static unsigned int enable_msix = 1;
- static unsigned int gcrashmode = 0;
- static unsigned int num_hba = 0;
- MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
- MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR);
- MODULE_AUTHOR("ServerEngines Corporation");
- MODULE_LICENSE("GPL");
- module_param(be_iopoll_budget, int, 0);
- module_param(enable_msix, int, 0);
- module_param(be_max_phys_size, uint, S_IRUGO);
- MODULE_PARM_DESC(be_max_phys_size, "Maximum Size (In Kilobytes) of physically"
- "contiguous memory that can be allocated."
- "Range is 16 - 128");
- static int beiscsi_slave_configure(struct scsi_device *sdev)
- {
- blk_queue_max_segment_size(sdev->request_queue, 65536);
- return 0;
- }
- static int beiscsi_eh_abort(struct scsi_cmnd *sc)
- {
- struct iscsi_cls_session *cls_session;
- struct iscsi_task *aborted_task = (struct iscsi_task *)sc->SCp.ptr;
- struct beiscsi_io_task *aborted_io_task;
- struct iscsi_conn *conn;
- struct beiscsi_conn *beiscsi_conn;
- struct beiscsi_hba *phba;
- struct iscsi_session *session;
- struct invalidate_command_table *inv_tbl;
- struct be_dma_mem nonemb_cmd;
- unsigned int cid, tag, num_invalidate;
- cls_session = starget_to_session(scsi_target(sc->device));
- session = cls_session->dd_data;
- spin_lock_bh(&session->lock);
- if (!aborted_task || !aborted_task->sc) {
- /* we raced */
- spin_unlock_bh(&session->lock);
- return SUCCESS;
- }
- aborted_io_task = aborted_task->dd_data;
- if (!aborted_io_task->scsi_cmnd) {
- /* raced or invalid command */
- spin_unlock_bh(&session->lock);
- return SUCCESS;
- }
- spin_unlock_bh(&session->lock);
- conn = aborted_task->conn;
- beiscsi_conn = conn->dd_data;
- phba = beiscsi_conn->phba;
- /* invalidate iocb */
- cid = beiscsi_conn->beiscsi_conn_cid;
- inv_tbl = phba->inv_tbl;
- memset(inv_tbl, 0x0, sizeof(*inv_tbl));
- inv_tbl->cid = cid;
- inv_tbl->icd = aborted_io_task->psgl_handle->sgl_index;
- num_invalidate = 1;
- nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
- sizeof(struct invalidate_commands_params_in),
- &nonemb_cmd.dma);
- if (nonemb_cmd.va == NULL) {
- SE_DEBUG(DBG_LVL_1,
- "Failed to allocate memory for"
- "mgmt_invalidate_icds\n");
- return FAILED;
- }
- nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
- tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
- cid, &nonemb_cmd);
- if (!tag) {
- shost_printk(KERN_WARNING, phba->shost,
- "mgmt_invalidate_icds could not be"
- " submitted\n");
- pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
- nonemb_cmd.va, nonemb_cmd.dma);
- return FAILED;
- } else {
- wait_event_interruptible(phba->ctrl.mcc_wait[tag],
- phba->ctrl.mcc_numtag[tag]);
- free_mcc_tag(&phba->ctrl, tag);
- }
- pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
- nonemb_cmd.va, nonemb_cmd.dma);
- return iscsi_eh_abort(sc);
- }
- static int beiscsi_eh_device_reset(struct scsi_cmnd *sc)
- {
- struct iscsi_task *abrt_task;
- struct beiscsi_io_task *abrt_io_task;
- struct iscsi_conn *conn;
- struct beiscsi_conn *beiscsi_conn;
- struct beiscsi_hba *phba;
- struct iscsi_session *session;
- struct iscsi_cls_session *cls_session;
- struct invalidate_command_table *inv_tbl;
- struct be_dma_mem nonemb_cmd;
- unsigned int cid, tag, i, num_invalidate;
- int rc = FAILED;
- /* invalidate iocbs */
- cls_session = starget_to_session(scsi_target(sc->device));
- session = cls_session->dd_data;
- spin_lock_bh(&session->lock);
- if (!session->leadconn || session->state != ISCSI_STATE_LOGGED_IN)
- goto unlock;
- conn = session->leadconn;
- beiscsi_conn = conn->dd_data;
- phba = beiscsi_conn->phba;
- cid = beiscsi_conn->beiscsi_conn_cid;
- inv_tbl = phba->inv_tbl;
- memset(inv_tbl, 0x0, sizeof(*inv_tbl) * BE2_CMDS_PER_CXN);
- num_invalidate = 0;
- for (i = 0; i < conn->session->cmds_max; i++) {
- abrt_task = conn->session->cmds[i];
- abrt_io_task = abrt_task->dd_data;
- if (!abrt_task->sc || abrt_task->state == ISCSI_TASK_FREE)
- continue;
- if (abrt_task->sc->device->lun != abrt_task->sc->device->lun)
- continue;
- inv_tbl->cid = cid;
- inv_tbl->icd = abrt_io_task->psgl_handle->sgl_index;
- num_invalidate++;
- inv_tbl++;
- }
- spin_unlock_bh(&session->lock);
- inv_tbl = phba->inv_tbl;
- nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
- sizeof(struct invalidate_commands_params_in),
- &nonemb_cmd.dma);
- if (nonemb_cmd.va == NULL) {
- SE_DEBUG(DBG_LVL_1,
- "Failed to allocate memory for"
- "mgmt_invalidate_icds\n");
- return FAILED;
- }
- nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
- memset(nonemb_cmd.va, 0, nonemb_cmd.size);
- tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
- cid, &nonemb_cmd);
- if (!tag) {
- shost_printk(KERN_WARNING, phba->shost,
- "mgmt_invalidate_icds could not be"
- " submitted\n");
- pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
- nonemb_cmd.va, nonemb_cmd.dma);
- return FAILED;
- } else {
- wait_event_interruptible(phba->ctrl.mcc_wait[tag],
- phba->ctrl.mcc_numtag[tag]);
- free_mcc_tag(&phba->ctrl, tag);
- }
- pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
- nonemb_cmd.va, nonemb_cmd.dma);
- return iscsi_eh_device_reset(sc);
- unlock:
- spin_unlock_bh(&session->lock);
- return rc;
- }
- static ssize_t beiscsi_show_boot_tgt_info(void *data, int type, char *buf)
- {
- struct beiscsi_hba *phba = data;
- struct mgmt_session_info *boot_sess = &phba->boot_sess;
- struct mgmt_conn_info *boot_conn = &boot_sess->conn_list[0];
- char *str = buf;
- int rc;
- switch (type) {
- case ISCSI_BOOT_TGT_NAME:
- rc = sprintf(buf, "%.*s\n",
- (int)strlen(boot_sess->target_name),
- (char *)&boot_sess->target_name);
- break;
- case ISCSI_BOOT_TGT_IP_ADDR:
- if (boot_conn->dest_ipaddr.ip_type == 0x1)
- rc = sprintf(buf, "%pI4\n",
- (char *)&boot_conn->dest_ipaddr.ip_address);
- else
- rc = sprintf(str, "%pI6\n",
- (char *)&boot_conn->dest_ipaddr.ip_address);
- break;
- case ISCSI_BOOT_TGT_PORT:
- rc = sprintf(str, "%d\n", boot_conn->dest_port);
- break;
- case ISCSI_BOOT_TGT_CHAP_NAME:
- rc = sprintf(str, "%.*s\n",
- boot_conn->negotiated_login_options.auth_data.chap.
- target_chap_name_length,
- (char *)&boot_conn->negotiated_login_options.
- auth_data.chap.target_chap_name);
- break;
- case ISCSI_BOOT_TGT_CHAP_SECRET:
- rc = sprintf(str, "%.*s\n",
- boot_conn->negotiated_login_options.auth_data.chap.
- target_secret_length,
- (char *)&boot_conn->negotiated_login_options.
- auth_data.chap.target_secret);
- break;
- case ISCSI_BOOT_TGT_REV_CHAP_NAME:
- rc = sprintf(str, "%.*s\n",
- boot_conn->negotiated_login_options.auth_data.chap.
- intr_chap_name_length,
- (char *)&boot_conn->negotiated_login_options.
- auth_data.chap.intr_chap_name);
- break;
- case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
- rc = sprintf(str, "%.*s\n",
- boot_conn->negotiated_login_options.auth_data.chap.
- intr_secret_length,
- (char *)&boot_conn->negotiated_login_options.
- auth_data.chap.intr_secret);
- break;
- case ISCSI_BOOT_TGT_FLAGS:
- rc = sprintf(str, "2\n");
- break;
- case ISCSI_BOOT_TGT_NIC_ASSOC:
- rc = sprintf(str, "0\n");
- break;
- default:
- rc = -ENOSYS;
- break;
- }
- return rc;
- }
- static ssize_t beiscsi_show_boot_ini_info(void *data, int type, char *buf)
- {
- struct beiscsi_hba *phba = data;
- char *str = buf;
- int rc;
- switch (type) {
- case ISCSI_BOOT_INI_INITIATOR_NAME:
- rc = sprintf(str, "%s\n", phba->boot_sess.initiator_iscsiname);
- break;
- default:
- rc = -ENOSYS;
- break;
- }
- return rc;
- }
- static ssize_t beiscsi_show_boot_eth_info(void *data, int type, char *buf)
- {
- struct beiscsi_hba *phba = data;
- char *str = buf;
- int rc;
- switch (type) {
- case ISCSI_BOOT_ETH_FLAGS:
- rc = sprintf(str, "2\n");
- break;
- case ISCSI_BOOT_ETH_INDEX:
- rc = sprintf(str, "0\n");
- break;
- case ISCSI_BOOT_ETH_MAC:
- rc = beiscsi_get_macaddr(buf, phba);
- if (rc < 0) {
- SE_DEBUG(DBG_LVL_1, "beiscsi_get_macaddr Failed\n");
- return rc;
- }
- break;
- default:
- rc = -ENOSYS;
- break;
- }
- return rc;
- }
- static mode_t beiscsi_tgt_get_attr_visibility(void *data, int type)
- {
- int rc;
- switch (type) {
- case ISCSI_BOOT_TGT_NAME:
- case ISCSI_BOOT_TGT_IP_ADDR:
- case ISCSI_BOOT_TGT_PORT:
- case ISCSI_BOOT_TGT_CHAP_NAME:
- case ISCSI_BOOT_TGT_CHAP_SECRET:
- case ISCSI_BOOT_TGT_REV_CHAP_NAME:
- case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
- case ISCSI_BOOT_TGT_NIC_ASSOC:
- case ISCSI_BOOT_TGT_FLAGS:
- rc = S_IRUGO;
- break;
- default:
- rc = 0;
- break;
- }
- return rc;
- }
- static mode_t beiscsi_ini_get_attr_visibility(void *data, int type)
- {
- int rc;
- switch (type) {
- case ISCSI_BOOT_INI_INITIATOR_NAME:
- rc = S_IRUGO;
- break;
- default:
- rc = 0;
- break;
- }
- return rc;
- }
- static mode_t beiscsi_eth_get_attr_visibility(void *data, int type)
- {
- int rc;
- switch (type) {
- case ISCSI_BOOT_ETH_FLAGS:
- case ISCSI_BOOT_ETH_MAC:
- case ISCSI_BOOT_ETH_INDEX:
- rc = S_IRUGO;
- break;
- default:
- rc = 0;
- break;
- }
- return rc;
- }
- /*------------------- PCI Driver operations and data ----------------- */
- static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = {
- { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
- { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
- { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
- { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
- { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) },
- { 0 }
- };
- MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
- static struct scsi_host_template beiscsi_sht = {
- .module = THIS_MODULE,
- .name = "ServerEngines 10Gbe open-iscsi Initiator Driver",
- .proc_name = DRV_NAME,
- .queuecommand = iscsi_queuecommand,
- .change_queue_depth = iscsi_change_queue_depth,
- .slave_configure = beiscsi_slave_configure,
- .target_alloc = iscsi_target_alloc,
- .eh_abort_handler = beiscsi_eh_abort,
- .eh_device_reset_handler = beiscsi_eh_device_reset,
- .eh_target_reset_handler = iscsi_eh_session_reset,
- .sg_tablesize = BEISCSI_SGLIST_ELEMENTS,
- .can_queue = BE2_IO_DEPTH,
- .this_id = -1,
- .max_sectors = BEISCSI_MAX_SECTORS,
- .cmd_per_lun = BEISCSI_CMD_PER_LUN,
- .use_clustering = ENABLE_CLUSTERING,
- };
- static struct scsi_transport_template *beiscsi_scsi_transport;
- static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev)
- {
- struct beiscsi_hba *phba;
- struct Scsi_Host *shost;
- shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0);
- if (!shost) {
- dev_err(&pcidev->dev, "beiscsi_hba_alloc -"
- "iscsi_host_alloc failed\n");
- return NULL;
- }
- shost->dma_boundary = pcidev->dma_mask;
- shost->max_id = BE2_MAX_SESSIONS;
- shost->max_channel = 0;
- shost->max_cmd_len = BEISCSI_MAX_CMD_LEN;
- shost->max_lun = BEISCSI_NUM_MAX_LUN;
- shost->transportt = beiscsi_scsi_transport;
- phba = iscsi_host_priv(shost);
- memset(phba, 0, sizeof(*phba));
- phba->shost = shost;
- phba->pcidev = pci_dev_get(pcidev);
- pci_set_drvdata(pcidev, phba);
- if (iscsi_host_add(shost, &phba->pcidev->dev))
- goto free_devices;
- return phba;
- free_devices:
- pci_dev_put(phba->pcidev);
- iscsi_host_free(phba->shost);
- return NULL;
- }
- static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba)
- {
- if (phba->csr_va) {
- iounmap(phba->csr_va);
- phba->csr_va = NULL;
- }
- if (phba->db_va) {
- iounmap(phba->db_va);
- phba->db_va = NULL;
- }
- if (phba->pci_va) {
- iounmap(phba->pci_va);
- phba->pci_va = NULL;
- }
- }
- static int beiscsi_map_pci_bars(struct beiscsi_hba *phba,
- struct pci_dev *pcidev)
- {
- u8 __iomem *addr;
- int pcicfg_reg;
- addr = ioremap_nocache(pci_resource_start(pcidev, 2),
- pci_resource_len(pcidev, 2));
- if (addr == NULL)
- return -ENOMEM;
- phba->ctrl.csr = addr;
- phba->csr_va = addr;
- phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2);
- addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024);
- if (addr == NULL)
- goto pci_map_err;
- phba->ctrl.db = addr;
- phba->db_va = addr;
- phba->db_pa.u.a64.address = pci_resource_start(pcidev, 4);
- if (phba->generation == BE_GEN2)
- pcicfg_reg = 1;
- else
- pcicfg_reg = 0;
- addr = ioremap_nocache(pci_resource_start(pcidev, pcicfg_reg),
- pci_resource_len(pcidev, pcicfg_reg));
- if (addr == NULL)
- goto pci_map_err;
- phba->ctrl.pcicfg = addr;
- phba->pci_va = addr;
- phba->pci_pa.u.a64.address = pci_resource_start(pcidev, pcicfg_reg);
- return 0;
- pci_map_err:
- beiscsi_unmap_pci_function(phba);
- return -ENOMEM;
- }
- static int beiscsi_enable_pci(struct pci_dev *pcidev)
- {
- int ret;
- ret = pci_enable_device(pcidev);
- if (ret) {
- dev_err(&pcidev->dev, "beiscsi_enable_pci - enable device "
- "failed. Returning -ENODEV\n");
- return ret;
- }
- pci_set_master(pcidev);
- if (pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64))) {
- ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32));
- if (ret) {
- dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");
- pci_disable_device(pcidev);
- return ret;
- }
- }
- return 0;
- }
- static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev)
- {
- struct be_ctrl_info *ctrl = &phba->ctrl;
- struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced;
- struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem;
- int status = 0;
- ctrl->pdev = pdev;
- status = beiscsi_map_pci_bars(phba, pdev);
- if (status)
- return status;
- mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
- mbox_mem_alloc->va = pci_alloc_consistent(pdev,
- mbox_mem_alloc->size,
- &mbox_mem_alloc->dma);
- if (!mbox_mem_alloc->va) {
- beiscsi_unmap_pci_function(phba);
- status = -ENOMEM;
- return status;
- }
- mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
- mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
- mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
- memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
- spin_lock_init(&ctrl->mbox_lock);
- spin_lock_init(&phba->ctrl.mcc_lock);
- spin_lock_init(&phba->ctrl.mcc_cq_lock);
- return status;
- }
- static void beiscsi_get_params(struct beiscsi_hba *phba)
- {
- phba->params.ios_per_ctrl = (phba->fw_config.iscsi_icd_count
- - (phba->fw_config.iscsi_cid_count
- + BE2_TMFS
- + BE2_NOPOUT_REQ));
- phba->params.cxns_per_ctrl = phba->fw_config.iscsi_cid_count;
- phba->params.asyncpdus_per_ctrl = phba->fw_config.iscsi_cid_count * 2;
- phba->params.icds_per_ctrl = phba->fw_config.iscsi_icd_count;
- phba->params.num_sge_per_io = BE2_SGE;
- phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ;
- phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ;
- phba->params.eq_timer = 64;
- phba->params.num_eq_entries =
- (((BE2_CMDS_PER_CXN * 2 + phba->fw_config.iscsi_cid_count * 2
- + BE2_TMFS) / 512) + 1) * 512;
- phba->params.num_eq_entries = (phba->params.num_eq_entries < 1024)
- ? 1024 : phba->params.num_eq_entries;
- SE_DEBUG(DBG_LVL_8, "phba->params.num_eq_entries=%d\n",
- phba->params.num_eq_entries);
- phba->params.num_cq_entries =
- (((BE2_CMDS_PER_CXN * 2 + phba->fw_config.iscsi_cid_count * 2
- + BE2_TMFS) / 512) + 1) * 512;
- phba->params.wrbs_per_cxn = 256;
- }
- static void hwi_ring_eq_db(struct beiscsi_hba *phba,
- unsigned int id, unsigned int clr_interrupt,
- unsigned int num_processed,
- unsigned char rearm, unsigned char event)
- {
- u32 val = 0;
- val |= id & DB_EQ_RING_ID_MASK;
- if (rearm)
- val |= 1 << DB_EQ_REARM_SHIFT;
- if (clr_interrupt)
- val |= 1 << DB_EQ_CLR_SHIFT;
- if (event)
- val |= 1 << DB_EQ_EVNT_SHIFT;
- val |= num_processed << DB_EQ_NUM_POPPED_SHIFT;
- iowrite32(val, phba->db_va + DB_EQ_OFFSET);
- }
- /**
- * be_isr_mcc - The isr routine of the driver.
- * @irq: Not used
- * @dev_id: Pointer to host adapter structure
- */
- static irqreturn_t be_isr_mcc(int irq, void *dev_id)
- {
- struct beiscsi_hba *phba;
- struct be_eq_entry *eqe = NULL;
- struct be_queue_info *eq;
- struct be_queue_info *mcc;
- unsigned int num_eq_processed;
- struct be_eq_obj *pbe_eq;
- unsigned long flags;
- pbe_eq = dev_id;
- eq = &pbe_eq->q;
- phba = pbe_eq->phba;
- mcc = &phba->ctrl.mcc_obj.cq;
- eqe = queue_tail_node(eq);
- if (!eqe)
- SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
- num_eq_processed = 0;
- while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
- & EQE_VALID_MASK) {
- if (((eqe->dw[offsetof(struct amap_eq_entry,
- resource_id) / 32] &
- EQE_RESID_MASK) >> 16) == mcc->id) {
- spin_lock_irqsave(&phba->isr_lock, flags);
- phba->todo_mcc_cq = 1;
- spin_unlock_irqrestore(&phba->isr_lock, flags);
- }
- AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
- queue_tail_inc(eq);
- eqe = queue_tail_node(eq);
- num_eq_processed++;
- }
- if (phba->todo_mcc_cq)
- queue_work(phba->wq, &phba->work_cqs);
- if (num_eq_processed)
- hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
- return IRQ_HANDLED;
- }
- /**
- * be_isr_msix - The isr routine of the driver.
- * @irq: Not used
- * @dev_id: Pointer to host adapter structure
- */
- static irqreturn_t be_isr_msix(int irq, void *dev_id)
- {
- struct beiscsi_hba *phba;
- struct be_eq_entry *eqe = NULL;
- struct be_queue_info *eq;
- struct be_queue_info *cq;
- unsigned int num_eq_processed;
- struct be_eq_obj *pbe_eq;
- unsigned long flags;
- pbe_eq = dev_id;
- eq = &pbe_eq->q;
- cq = pbe_eq->cq;
- eqe = queue_tail_node(eq);
- if (!eqe)
- SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
- phba = pbe_eq->phba;
- num_eq_processed = 0;
- if (blk_iopoll_enabled) {
- while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
- & EQE_VALID_MASK) {
- if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
- blk_iopoll_sched(&pbe_eq->iopoll);
- AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
- queue_tail_inc(eq);
- eqe = queue_tail_node(eq);
- num_eq_processed++;
- }
- if (num_eq_processed)
- hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 0, 1);
- return IRQ_HANDLED;
- } else {
- while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
- & EQE_VALID_MASK) {
- spin_lock_irqsave(&phba->isr_lock, flags);
- phba->todo_cq = 1;
- spin_unlock_irqrestore(&phba->isr_lock, flags);
- AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
- queue_tail_inc(eq);
- eqe = queue_tail_node(eq);
- num_eq_processed++;
- }
- if (phba->todo_cq)
- queue_work(phba->wq, &phba->work_cqs);
- if (num_eq_processed)
- hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
- return IRQ_HANDLED;
- }
- }
- /**
- * be_isr - The isr routine of the driver.
- * @irq: Not used
- * @dev_id: Pointer to host adapter structure
- */
- static irqreturn_t be_isr(int irq, void *dev_id)
- {
- struct beiscsi_hba *phba;
- struct hwi_controller *phwi_ctrlr;
- struct hwi_context_memory *phwi_context;
- struct be_eq_entry *eqe = NULL;
- struct be_queue_info *eq;
- struct be_queue_info *cq;
- struct be_queue_info *mcc;
- unsigned long flags, index;
- unsigned int num_mcceq_processed, num_ioeq_processed;
- struct be_ctrl_info *ctrl;
- struct be_eq_obj *pbe_eq;
- int isr;
- phba = dev_id;
- ctrl = &phba->ctrl;
- isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET +
- (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE));
- if (!isr)
- return IRQ_NONE;
- phwi_ctrlr = phba->phwi_ctrlr;
- phwi_context = phwi_ctrlr->phwi_ctxt;
- pbe_eq = &phwi_context->be_eq[0];
- eq = &phwi_context->be_eq[0].q;
- mcc = &phba->ctrl.mcc_obj.cq;
- index = 0;
- eqe = queue_tail_node(eq);
- if (!eqe)
- SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
- num_ioeq_processed = 0;
- num_mcceq_processed = 0;
- if (blk_iopoll_enabled) {
- while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
- & EQE_VALID_MASK) {
- if (((eqe->dw[offsetof(struct amap_eq_entry,
- resource_id) / 32] &
- EQE_RESID_MASK) >> 16) == mcc->id) {
- spin_lock_irqsave(&phba->isr_lock, flags);
- phba->todo_mcc_cq = 1;
- spin_unlock_irqrestore(&phba->isr_lock, flags);
- num_mcceq_processed++;
- } else {
- if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
- blk_iopoll_sched(&pbe_eq->iopoll);
- num_ioeq_processed++;
- }
- AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
- queue_tail_inc(eq);
- eqe = queue_tail_node(eq);
- }
- if (num_ioeq_processed || num_mcceq_processed) {
- if (phba->todo_mcc_cq)
- queue_work(phba->wq, &phba->work_cqs);
- if ((num_mcceq_processed) && (!num_ioeq_processed))
- hwi_ring_eq_db(phba, eq->id, 0,
- (num_ioeq_processed +
- num_mcceq_processed) , 1, 1);
- else
- hwi_ring_eq_db(phba, eq->id, 0,
- (num_ioeq_processed +
- num_mcceq_processed), 0, 1);
- return IRQ_HANDLED;
- } else
- return IRQ_NONE;
- } else {
- cq = &phwi_context->be_cq[0];
- while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
- & EQE_VALID_MASK) {
- if (((eqe->dw[offsetof(struct amap_eq_entry,
- resource_id) / 32] &
- EQE_RESID_MASK) >> 16) != cq->id) {
- spin_lock_irqsave(&phba->isr_lock, flags);
- phba->todo_mcc_cq = 1;
- spin_unlock_irqrestore(&phba->isr_lock, flags);
- } else {
- spin_lock_irqsave(&phba->isr_lock, flags);
- phba->todo_cq = 1;
- spin_unlock_irqrestore(&phba->isr_lock, flags);
- }
- AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
- queue_tail_inc(eq);
- eqe = queue_tail_node(eq);
- num_ioeq_processed++;
- }
- if (phba->todo_cq || phba->todo_mcc_cq)
- queue_work(phba->wq, &phba->work_cqs);
- if (num_ioeq_processed) {
- hwi_ring_eq_db(phba, eq->id, 0,
- num_ioeq_processed, 1, 1);
- return IRQ_HANDLED;
- } else
- return IRQ_NONE;
- }
- }
- static int beiscsi_init_irqs(struct beiscsi_hba *phba)
- {
- struct pci_dev *pcidev = phba->pcidev;
- struct hwi_controller *phwi_ctrlr;
- struct hwi_context_memory *phwi_context;
- int ret, msix_vec, i, j;
- char desc[32];
- phwi_ctrlr = phba->phwi_ctrlr;
- phwi_context = phwi_ctrlr->phwi_ctxt;
- if (phba->msix_enabled) {
- for (i = 0; i < phba->num_cpus; i++) {
- sprintf(desc, "beiscsi_msix_%04x", i);
- msix_vec = phba->msix_entries[i].vector;
- ret = request_irq(msix_vec, be_isr_msix, 0, desc,
- &phwi_context->be_eq[i]);
- if (ret) {
- shost_printk(KERN_ERR, phba->shost,
- "beiscsi_init_irqs-Failed to"
- "register msix for i = %d\n", i);
- if (!i)
- return ret;
- goto free_msix_irqs;
- }
- }
- msix_vec = phba->msix_entries[i].vector;
- ret = request_irq(msix_vec, be_isr_mcc, 0, "beiscsi_msix_mcc",
- &phwi_context->be_eq[i]);
- if (ret) {
- shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-"
- "Failed to register beiscsi_msix_mcc\n");
- i++;
- goto free_msix_irqs;
- }
- } else {
- ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED,
- "beiscsi", phba);
- if (ret) {
- shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-"
- "Failed to register irq\\n");
- return ret;
- }
- }
- return 0;
- free_msix_irqs:
- for (j = i - 1; j == 0; j++)
- free_irq(msix_vec, &phwi_context->be_eq[j]);
- return ret;
- }
- static void hwi_ring_cq_db(struct beiscsi_hba *phba,
- unsigned int id, unsigned int num_processed,
- unsigned char rearm, unsigned char event)
- {
- u32 val = 0;
- val |= id & DB_CQ_RING_ID_MASK;
- if (rearm)
- val |= 1 << DB_CQ_REARM_SHIFT;
- val |= num_processed << DB_CQ_NUM_POPPED_SHIFT;
- iowrite32(val, phba->db_va + DB_CQ_OFFSET);
- }
- static unsigned int
- beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
- struct beiscsi_hba *phba,
- unsigned short cid,
- struct pdu_base *ppdu,
- unsigned long pdu_len,
- void *pbuffer, unsigned long buf_len)
- {
- struct iscsi_conn *conn = beiscsi_conn->conn;
- struct iscsi_session *session = conn->session;
- struct iscsi_task *task;
- struct beiscsi_io_task *io_task;
- struct iscsi_hdr *login_hdr;
- switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] &
- PDUBASE_OPCODE_MASK) {
- case ISCSI_OP_NOOP_IN:
- pbuffer = NULL;
- buf_len = 0;
- break;
- case ISCSI_OP_ASYNC_EVENT:
- break;
- case ISCSI_OP_REJECT:
- WARN_ON(!pbuffer);
- WARN_ON(!(buf_len == 48));
- SE_DEBUG(DBG_LVL_1, "In ISCSI_OP_REJECT\n");
- break;
- case ISCSI_OP_LOGIN_RSP:
- case ISCSI_OP_TEXT_RSP:
- task = conn->login_task;
- io_task = task->dd_data;
- login_hdr = (struct iscsi_hdr *)ppdu;
- login_hdr->itt = io_task->libiscsi_itt;
- break;
- default:
- shost_printk(KERN_WARNING, phba->shost,
- "Unrecognized opcode 0x%x in async msg\n",
- (ppdu->
- dw[offsetof(struct amap_pdu_base, opcode) / 32]
- & PDUBASE_OPCODE_MASK));
- return 1;
- }
- spin_lock_bh(&session->lock);
- __iscsi_complete_pdu(conn, (struct iscsi_hdr *)ppdu, pbuffer, buf_len);
- spin_unlock_bh(&session->lock);
- return 0;
- }
- static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
- {
- struct sgl_handle *psgl_handle;
- if (phba->io_sgl_hndl_avbl) {
- SE_DEBUG(DBG_LVL_8,
- "In alloc_io_sgl_handle,io_sgl_alloc_index=%d\n",
- phba->io_sgl_alloc_index);
- psgl_handle = phba->io_sgl_hndl_base[phba->
- io_sgl_alloc_index];
- phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL;
- phba->io_sgl_hndl_avbl--;
- if (phba->io_sgl_alloc_index == (phba->params.
- ios_per_ctrl - 1))
- phba->io_sgl_alloc_index = 0;
- else
- phba->io_sgl_alloc_index++;
- } else
- psgl_handle = NULL;
- return psgl_handle;
- }
- static void
- free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
- {
- SE_DEBUG(DBG_LVL_8, "In free_,io_sgl_free_index=%d\n",
- phba->io_sgl_free_index);
- if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) {
- /*
- * this can happen if clean_task is called on a task that
- * failed in xmit_task or alloc_pdu.
- */
- SE_DEBUG(DBG_LVL_8,
- "Double Free in IO SGL io_sgl_free_index=%d,"
- "value there=%p\n", phba->io_sgl_free_index,
- phba->io_sgl_hndl_base[phba->io_sgl_free_index]);
- return;
- }
- phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle;
- phba->io_sgl_hndl_avbl++;
- if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1))
- phba->io_sgl_free_index = 0;
- else
- phba->io_sgl_free_index++;
- }
- /**
- * alloc_wrb_handle - To allocate a wrb handle
- * @phba: The hba pointer
- * @cid: The cid to use for allocation
- *
- * This happens under session_lock until submission to chip
- */
- struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid)
- {
- struct hwi_wrb_context *pwrb_context;
- struct hwi_controller *phwi_ctrlr;
- struct wrb_handle *pwrb_handle, *pwrb_handle_tmp;
- phwi_ctrlr = phba->phwi_ctrlr;
- pwrb_context = &phwi_ctrlr->wrb_context[cid];
- if (pwrb_context->wrb_handles_available >= 2) {
- pwrb_handle = pwrb_context->pwrb_handle_base[
- pwrb_context->alloc_index];
- pwrb_context->wrb_handles_available--;
- if (pwrb_context->alloc_index ==
- (phba->params.wrbs_per_cxn - 1))
- pwrb_context->alloc_index = 0;
- else
- pwrb_context->alloc_index++;
- pwrb_handle_tmp = pwrb_context->pwrb_handle_base[
- pwrb_context->alloc_index];
- pwrb_handle->nxt_wrb_index = pwrb_handle_tmp->wrb_index;
- } else
- pwrb_handle = NULL;
- return pwrb_handle;
- }
- /**
- * free_wrb_handle - To free the wrb handle back to pool
- * @phba: The hba pointer
- * @pwrb_context: The context to free from
- * @pwrb_handle: The wrb_handle to free
- *
- * This happens under session_lock until submission to chip
- */
- static void
- free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context,
- struct wrb_handle *pwrb_handle)
- {
- pwrb_context->pwrb_handle_base[pwrb_context->free_index] = pwrb_handle;
- pwrb_context->wrb_handles_available++;
- if (pwrb_context->free_index == (phba->params.wrbs_per_cxn - 1))
- pwrb_context->free_index = 0;
- else
- pwrb_context->free_index++;
- SE_DEBUG(DBG_LVL_8,
- "FREE WRB: pwrb_handle=%p free_index=0x%x"
- "wrb_handles_available=%d\n",
- pwrb_handle, pwrb_context->free_index,
- pwrb_context->wrb_handles_available);
- }
- static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba)
- {
- struct sgl_handle *psgl_handle;
- if (phba->eh_sgl_hndl_avbl) {
- psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index];
- phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL;
- SE_DEBUG(DBG_LVL_8, "mgmt_sgl_alloc_index=%d=0x%x\n",
- phba->eh_sgl_alloc_index, phba->eh_sgl_alloc_index);
- phba->eh_sgl_hndl_avbl--;
- if (phba->eh_sgl_alloc_index ==
- (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl -
- 1))
- phba->eh_sgl_alloc_index = 0;
- else
- phba->eh_sgl_alloc_index++;
- } else
- psgl_handle = NULL;
- return psgl_handle;
- }
- void
- free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
- {
- SE_DEBUG(DBG_LVL_8, "In free_mgmt_sgl_handle,eh_sgl_free_index=%d\n",
- phba->eh_sgl_free_index);
- if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) {
- /*
- * this can happen if clean_task is called on a task that
- * failed in xmit_task or alloc_pdu.
- */
- SE_DEBUG(DBG_LVL_8,
- "Double Free in eh SGL ,eh_sgl_free_index=%d\n",
- phba->eh_sgl_free_index);
- return;
- }
- phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle;
- phba->eh_sgl_hndl_avbl++;
- if (phba->eh_sgl_free_index ==
- (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1))
- phba->eh_sgl_free_index = 0;
- else
- phba->eh_sgl_free_index++;
- }
- static void
- be_complete_io(struct beiscsi_conn *beiscsi_conn,
- struct iscsi_task *task, struct sol_cqe *psol)
- {
- struct beiscsi_io_task *io_task = task->dd_data;
- struct be_status_bhs *sts_bhs =
- (struct be_status_bhs *)io_task->cmd_bhs;
- struct iscsi_conn *conn = beiscsi_conn->conn;
- unsigned int sense_len;
- unsigned char *sense;
- u32 resid = 0, exp_cmdsn, max_cmdsn;
- u8 rsp, status, flags;
- exp_cmdsn = (psol->
- dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
- & SOL_EXP_CMD_SN_MASK);
- max_cmdsn = ((psol->
- dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
- & SOL_EXP_CMD_SN_MASK) +
- ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
- / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
- rsp = ((psol->dw[offsetof(struct amap_sol_cqe, i_resp) / 32]
- & SOL_RESP_MASK) >> 16);
- status = ((psol->dw[offsetof(struct amap_sol_cqe, i_sts) / 32]
- & SOL_STS_MASK) >> 8);
- flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
- & SOL_FLAGS_MASK) >> 24) | 0x80;
- task->sc->result = (DID_OK << 16) | status;
- if (rsp != ISCSI_STATUS_CMD_COMPLETED) {
- task->sc->result = DID_ERROR << 16;
- goto unmap;
- }
- /* bidi not initially supported */
- if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) {
- resid = (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) /
- 32] & SOL_RES_CNT_MASK);
- if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW))
- task->sc->result = DID_ERROR << 16;
- if (flags & ISCSI_FLAG_CMD_UNDERFLOW) {
- scsi_set_resid(task->sc, resid);
- if (!status && (scsi_bufflen(task->sc) - resid <
- task->sc->underflow))
- task->sc->result = DID_ERROR << 16;
- }
- }
- if (status == SAM_STAT_CHECK_CONDITION) {
- unsigned short *slen = (unsigned short *)sts_bhs->sense_info;
- sense = sts_bhs->sense_info + sizeof(unsigned short);
- sense_len = cpu_to_be16(*slen);
- memcpy(task->sc->sense_buffer, sense,
- min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE));
- }
- if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ) {
- if (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
- & SOL_RES_CNT_MASK)
- conn->rxdata_octets += (psol->
- dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
- & SOL_RES_CNT_MASK);
- }
- unmap:
- scsi_dma_unmap(io_task->scsi_cmnd);
- iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn);
- }
- static void
- be_complete_logout(struct beiscsi_conn *beiscsi_conn,
- struct iscsi_task *task, struct sol_cqe *psol)
- {
- struct iscsi_logout_rsp *hdr;
- struct beiscsi_io_task *io_task = task->dd_data;
- struct iscsi_conn *conn = beiscsi_conn->conn;
- hdr = (struct iscsi_logout_rsp *)task->hdr;
- hdr->opcode = ISCSI_OP_LOGOUT_RSP;
- hdr->t2wait = 5;
- hdr->t2retain = 0;
- hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
- & SOL_FLAGS_MASK) >> 24) | 0x80;
- hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
- 32] & SOL_RESP_MASK);
- hdr->exp_cmdsn = cpu_to_be32(psol->
- dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
- & SOL_EXP_CMD_SN_MASK);
- hdr->max_cmdsn = be32_to_cpu((psol->
- dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
- & SOL_EXP_CMD_SN_MASK) +
- ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
- / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
- hdr->dlength[0] = 0;
- hdr->dlength[1] = 0;
- hdr->dlength[2] = 0;
- hdr->hlength = 0;
- hdr->itt = io_task->libiscsi_itt;
- __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
- }
- static void
- be_complete_tmf(struct beiscsi_conn *beiscsi_conn,
- struct iscsi_task *task, struct sol_cqe *psol)
- {
- struct iscsi_tm_rsp *hdr;
- struct iscsi_conn *conn = beiscsi_conn->conn;
- struct beiscsi_io_task *io_task = task->dd_data;
- hdr = (struct iscsi_tm_rsp *)task->hdr;
- hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP;
- hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
- & SOL_FLAGS_MASK) >> 24) | 0x80;
- hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
- 32] & SOL_RESP_MASK);
- hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
- i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
- hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
- i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
- ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
- / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
- hdr->itt = io_task->libiscsi_itt;
- __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
- }
- static void
- hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
- struct beiscsi_hba *phba, struct sol_cqe *psol)
- {
- struct hwi_wrb_context *pwrb_context;
- struct wrb_handle *pwrb_handle = NULL;
- struct hwi_controller *phwi_ctrlr;
- struct iscsi_task *task;
- struct beiscsi_io_task *io_task;
- struct iscsi_conn *conn = beiscsi_conn->conn;
- struct iscsi_session *session = conn->session;
- phwi_ctrlr = phba->phwi_ctrlr;
- pwrb_context = &phwi_ctrlr->wrb_context[((psol->
- dw[offsetof(struct amap_sol_cqe, cid) / 32] &
- SOL_CID_MASK) >> 6) -
- phba->fw_config.iscsi_cid_start];
- pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
- dw[offsetof(struct amap_sol_cqe, wrb_index) /
- 32] & SOL_WRB_INDEX_MASK) >> 16)];
- task = pwrb_handle->pio_handle;
- io_task = task->dd_data;
- spin_lock(&phba->mgmt_sgl_lock);
- free_mgmt_sgl_handle(phba, io_task->psgl_handle);
- spin_unlock(&phba->mgmt_sgl_lock);
- spin_lock_bh(&session->lock);
- free_wrb_handle(phba, pwrb_context, pwrb_handle);
- spin_unlock_bh(&session->lock);
- }
- static void
- be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn,
- struct iscsi_task *task, struct sol_cqe *psol)
- {
- struct iscsi_nopin *hdr;
- struct iscsi_conn *conn = beiscsi_conn->conn;
- struct beiscsi_io_task *io_task = task->dd_data;
- hdr = (struct iscsi_nopin *)task->hdr;
- hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
- & SOL_FLAGS_MASK) >> 24) | 0x80;
- hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
- i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
- hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
- i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
- ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
- / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
- hdr->opcode = ISCSI_OP_NOOP_IN;
- hdr->itt = io_task->libiscsi_itt;
- __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
- }
- static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
- struct beiscsi_hba *phba, struct sol_cqe *psol)
- {
- struct hwi_wrb_context *pwrb_context;
- struct wrb_handle *pwrb_handle;
- struct iscsi_wrb *pwrb = NULL;
- struct hwi_controller *phwi_ctrlr;
- struct iscsi_task *task;
- unsigned int type;
- struct iscsi_conn *conn = beiscsi_conn->conn;
- struct iscsi_session *session = conn->session;
- phwi_ctrlr = phba->phwi_ctrlr;
- pwrb_context = &phwi_ctrlr->wrb_context[((psol->dw[offsetof
- (struct amap_sol_cqe, cid) / 32]
- & SOL_CID_MASK) >> 6) -
- phba->fw_config.iscsi_cid_start];
- pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
- dw[offsetof(struct amap_sol_cqe, wrb_index) /
- 32] & SOL_WRB_INDEX_MASK) >> 16)];
- task = pwrb_handle->pio_handle;
- pwrb = pwrb_handle->pwrb;
- type = (pwrb->dw[offsetof(struct amap_iscsi_wrb, type) / 32] &
- WRB_TYPE_MASK) >> 28;
- spin_lock_bh(&session->lock);
- switch (type) {
- case HWH_TYPE_IO:
- case HWH_TYPE_IO_RD:
- if ((task->hdr->opcode & ISCSI_OPCODE_MASK) ==
- ISCSI_OP_NOOP_OUT)
- be_complete_nopin_resp(beiscsi_conn, task, psol);
- else
- be_complete_io(beiscsi_conn, task, psol);
- break;
- case HWH_TYPE_LOGOUT:
- if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGOUT)
- be_complete_logout(beiscsi_conn, task, psol);
- else
- be_complete_tmf(beiscsi_conn, task, psol);
- break;
- case HWH_TYPE_LOGIN:
- SE_DEBUG(DBG_LVL_1,
- "\t\t No HWH_TYPE_LOGIN Expected in hwi_complete_cmd"
- "- Solicited path\n");
- break;
- case HWH_TYPE_NOP:
- be_complete_nopin_resp(beiscsi_conn, task, psol);
- break;
- default:
- shost_printk(KERN_WARNING, phba->shost,
- "In hwi_complete_cmd, unknown type = %d"
- "wrb_index 0x%x CID 0x%x\n", type,
- ((psol->dw[offsetof(struct amap_iscsi_wrb,
- type) / 32] & SOL_WRB_INDEX_MASK) >> 16),
- ((psol->dw[offsetof(struct amap_sol_cqe,
- cid) / 32] & SOL_CID_MASK) >> 6));
- break;
- }
- spin_unlock_bh(&session->lock);
- }
- static struct list_head *hwi_get_async_busy_list(struct hwi_async_pdu_context
- *pasync_ctx, unsigned int is_header,
- unsigned int host_write_ptr)
- {
- if (is_header)
- return &pasync_ctx->async_entry[host_write_ptr].
- header_busy_list;
- else
- return &pasync_ctx->async_entry[host_write_ptr].data_busy_list;
- }
- static struct async_pdu_handle *
- hwi_get_async_handle(struct beiscsi_hba *phba,
- struct beiscsi_conn *beiscsi_conn,
- struct hwi_async_pdu_context *pasync_ctx,
- struct i_t_dpdu_cqe *pdpdu_cqe, unsigned int *pcq_index)
- {
- struct be_bus_address phys_addr;
- struct list_head *pbusy_list;
- struct async_pdu_handle *pasync_handle = NULL;
- int buffer_len = 0;
- unsigned char buffer_index = -1;
- unsigned char is_header = 0;
- phys_addr.u.a32.address_lo =
- pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_lo) / 32] -
- ((pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
- & PDUCQE_DPL_MASK) >> 16);
- phys_addr.u.a32.address_hi =
- pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_hi) / 32];
- phys_addr.u.a64.address =
- *((unsigned long long *)(&phys_addr.u.a64.address));
- switch (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, code) / 32]
- & PDUCQE_CODE_MASK) {
- case UNSOL_HDR_NOTIFY:
- is_header = 1;
- pbusy_list = hwi_get_async_busy_list(pasync_ctx, 1,
- (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
- index) / 32] & PDUCQE_INDEX_MASK));
- buffer_len = (unsigned int)(phys_addr.u.a64.address -
- pasync_ctx->async_header.pa_base.u.a64.address);
- buffer_index = buffer_len /
- pasync_ctx->async_header.buffer_size;
- break;
- case UNSOL_DATA_NOTIFY:
- pbusy_list = hwi_get_async_busy_list(pasync_ctx, 0, (pdpdu_cqe->
- dw[offsetof(struct amap_i_t_dpdu_cqe,
- index) / 32] & PDUCQE_INDEX_MASK));
- buffer_len = (unsigned long)(phys_addr.u.a64.address -
- pasync_ctx->async_data.pa_base.u.
- a64.address);
- buffer_index = buffer_len / pasync_ctx->async_data.buffer_size;
- break;
- default:
- pbusy_list = NULL;
- shost_printk(KERN_WARNING, phba->shost,
- "Unexpected code=%d\n",
- pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
- code) / 32] & PDUCQE_CODE_MASK);
- return NULL;
- }
- WARN_ON(!(buffer_index <= pasync_ctx->async_data.num_entries));
- WARN_ON(list_empty(pbusy_list));
- list_for_each_entry(pasync_handle, pbusy_list, link) {
- WARN_ON(pasync_handle->consumed);
- if (pasync_handle->index == buffer_index)
- break;
- }
- WARN_ON(!pasync_handle);
- pasync_handle->cri = (unsigned short)beiscsi_conn->beiscsi_conn_cid -
- phba->fw_config.iscsi_cid_start;
- pasync_handle->is_header = is_header;
- pasync_handle->buffer_len = ((pdpdu_cqe->
- dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
- & PDUCQE_DPL_MASK) >> 16);
- *pcq_index = (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
- index) / 32] & PDUCQE_INDEX_MASK);
- return pasync_handle;
- }
- static unsigned int
- hwi_update_async_writables(struct hwi_async_pdu_context *pasync_ctx,
- unsigned int is_header, unsigned int cq_index)
- {
- struct list_head *pbusy_list;
- struct async_pdu_handle *pasync_handle;
- unsigned int num_entries, writables = 0;
- unsigned int *pep_read_ptr, *pwritables;
- if (is_header) {
- pep_read_ptr = &pasync_ctx->async_header.ep_read_ptr;
- pwritables = &pasync_ctx->async_header.writables;
- num_entries = pasync_ctx->async_header.num_entries;
- } else {
- pep_read_ptr = &pasync_ctx->async_data.ep_read_ptr;
- pwritables = &pasync_ctx->async_data.writables;
- num_entries = pasync_ctx->async_data.num_entries;
- }
- while ((*pep_read_ptr) != cq_index) {
- (*pep_read_ptr)++;
- *pep_read_ptr = (*pep_read_ptr) % num_entries;
- pbusy_list = hwi_get_async_busy_list(pasync_ctx, is_header,
- *pep_read_ptr);
- if (writables == 0)
- WARN_ON(list_empty(pbusy_list));
- if (!list_empty(pbusy_list)) {
- pasync_handle = list_entry(pbusy_list->next,
- struct async_pdu_handle,
- link);
- WARN_ON(!pasync_handle);
- pasync_handle->consumed = 1;
- }
- writables++;
- }
- if (!writables) {
- SE_DEBUG(DBG_LVL_1,
- "Duplicate notification received - index 0x%x!!\n",
- cq_index);
- WARN_ON(1);
- }
- *pwritables = *pwritables + writables;
- return 0;
- }
- static unsigned int hwi_free_async_msg(struct beiscsi_hba *phba,
- unsigned int cri)
- {
- struct hwi_controller *phwi_ctrlr;
- struct hwi_async_pdu_context *pasync_ctx;
- struct async_pdu_handle *pasync_handle, *tmp_handle;
- struct list_head *plist;
- unsigned int i = 0;
- phwi_ctrlr = phba->phwi_ctrlr;
- pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
- plist = &pasync_ctx->async_entry[cri].wait_queue.list;
- list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) {
- list_del(&pasync_handle->link);
- if (i == 0) {
- list_add_tail(&pasync_handle->link,
- &pasync_ctx->async_header.free_list);
- pasync_ctx->async_header.free_entries++;
- i++;
- } else {
- list_add_tail(&pasync_handle->link,
- &pasync_ctx->async_data.free_list);
- pasync_ctx->async_data.free_entries++;
- i++;
- }
- }
- INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wait_queue.list);
- pasync_ctx->async_entry[cri].wait_queue.hdr_received = 0;
- pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
- return 0;
- }
- static struct phys_addr *
- hwi_get_ring_address(struct hwi_async_pdu_context *pasync_ctx,
- unsigned int is_header, unsigned int host_write_ptr)
- {
- struct phys_addr *pasync_sge = NULL;
- if (is_header)
- pasync_sge = pasync_ctx->async_header.ring_base;
- else
- pasync_sge = pasync_ctx->async_data.ring_base;
- return pasync_sge + host_write_ptr;
- }
- static void hwi_post_async_buffers(struct beiscsi_hba *phba,
- unsigned int is_header)
- {
- struct hwi_controller *phwi_ctrlr;
- struct hwi_async_pdu_context *pasync_ctx;
- struct async_pdu_handle *pasync_handle;
- struct list_head *pfree_link, *pbusy_list;
- struct phys_addr *pasync_sge;
- unsigned int ring_id, num_entries;
- unsigned int host_write_num;
- unsigned int writables;
- unsigned int i = 0;
- u32 doorbell = 0;
- phwi_ctrlr = phba->phwi_ctrlr;
- pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
- if (is_header) {
- num_entries = pasync_ctx->async_header.num_entries;
- writables = min(pasync_ctx->async_header.writables,
- pasync_ctx->async_header.free_entries);
- pfree_link = pasync_ctx->async_header.free_list.next;
- host_write_num = pasync_ctx->async_header.host_write_ptr;
- ring_id = phwi_ctrlr->default_pdu_hdr.id;
- } else {
- num_entries = pasync_ctx->async_data.num_entries;
- writables = min(pasync_ctx->async_data.writables,
- pasync_ctx->async_data.free_entries);
- pfree_link = pasync_ctx->async_data.free_list.next;
- host_write_num = pasync_ctx->async_data.host_write_ptr;
- ring_id = phwi_ctrlr->default_pdu_data.id;
- }
- writables = (writables / 8) * 8;
- if (writables) {
- for (i = 0; i < writables; i++) {
- pbusy_list =
- hwi_get_async_busy_list(pasync_ctx, is_header,
- host_write_num);
- pasync_handle =
- list_entry(pfree_link, struct async_pdu_handle,
- link);
- WARN_ON(!pasync_handle);
- pasync_handle->consumed = 0;
- pfree_link = pfree_link->next;
- pasync_sge = hwi_get_ring_address(pasync_ctx,
- is_header, host_write_num);
- pasync_sge->hi = pasync_handle->pa.u.a32.address_lo;
- pasync_sge->lo = pasync_handle->pa.u.a32.address_hi;
- list_move(&pasync_handle->link, pbusy_list);
- host_write_num++;
- host_write_num = host_write_num % num_entries;
- }
- if (is_header) {
- pasync_ctx->async_header.host_write_ptr =
- host_write_num;
- pasync_ctx->async_header.free_entries -= writables;
- pasync_ctx->async_header.writables -= writables;
- pasync_ctx->async_header.busy_entries += writables;
- } else {
- pasync_ctx->async_data.host_write_ptr = host_write_num;
- pasync_ctx->async_data.free_entries -= writables;
- pasync_ctx->async_data.writables -= writables;
- pasync_ctx->async_data.busy_entries += writables;
- }
- doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK;
- doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT;
- doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT;
- doorbell |= (writables & DB_DEF_PDU_CQPROC_MASK)
- << DB_DEF_PDU_CQPROC_SHIFT;
- iowrite32(doorbell, phba->db_va + DB_RXULP0_OFFSET);
- }
- }
- static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba,
- struct beiscsi_conn *beiscsi_conn,
- struct i_t_dpdu_cqe *pdpdu_cqe)
- {
- struct hwi_controller *phwi_ctrlr;
- struct hwi_async_pdu_context *pasync_ctx;
- struct async_pdu_handle *pasync_handle = NULL;
- unsigned int cq_index = -1;
- phwi_ctrlr = phba->phwi_ctrlr;
- pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
- pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
- pdpdu_cqe, &cq_index);
- BUG_ON(pasync_handle->is_header != 0);
- if (pasync_handle->consumed == 0)
- hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
- cq_index);
- hwi_free_async_msg(phba, pasync_handle->cri);
- hwi_post_async_buffers(phba, pasync_handle->is_header);
- }
- static unsigned int
- hwi_fwd_async_msg(struct beiscsi_conn *beiscsi_conn,
- struct beiscsi_hba *phba,
- struct hwi_async_pdu_context *pasync_ctx, unsigned short cri)
- {
- struct list_head *plist;
- struct async_pdu_handle *pasync_handle;
- void *phdr = NULL;
- unsigned int hdr_len = 0, buf_len = 0;
- unsigned int status, index = 0, offset = 0;
- void *pfirst_buffer = NULL;
- unsigned int num_buf = 0;
- plist = &pasync_ctx->async_entry[cri].wait_queue.list;
- list_for_each_entry(pasync_handle, plist, link) {
- if (index == 0) {
- phdr = pasync_handle->pbuffer;
- hdr_len = pasync_handle->buffer_len;
- } else {
- buf_len = pasync_handle->buffer_len;
- if (!num_buf) {
- pfirst_buffer = pasync_handle->pbuffer;
- num_buf++;
- }
- memcpy(pfirst_buffer + offset,
- pasync_handle->pbuffer, buf_len);
- offset = buf_len;
- }
- index++;
- }
- status = beiscsi_process_async_pdu(beiscsi_conn, phba,
- (beiscsi_conn->beiscsi_conn_cid -
- phba->fw_config.iscsi_cid_start),
- phdr, hdr_len, pfirst_buffer,
- buf_len);
- if (status == 0)
- hwi_free_async_msg(phba, cri);
- return 0;
- }
- static unsigned int
- hwi_gather_async_pdu(struct beiscsi_conn *beiscsi_conn,
- struct beiscsi_hba *phba,
- struct async_pdu_handle *pasync_handle)
- {
- struct hwi_async_pdu_context *pasync_ctx;
- struct hwi_controller *phwi_ctrlr;
- unsigned int bytes_needed = 0, status = 0;
- unsigned short cri = pasync_handle->cri;
- struct pdu_base *ppdu;
- phwi_ctrlr = phba->phwi_ctrlr;
- pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
- list_del(&pasync_handle->link);
- if (pasync_handle->is_header) {
- pasync_ctx->async_header.busy_entries--;
- if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
- hwi_free_async_msg(phba, cri);
- BUG();
- }
- pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
- pasync_ctx->async_entry[cri].wait_queue.hdr_received = 1;
- pasync_ctx->async_entry[cri].wait_queue.hdr_len =
- (unsigned short)pasync_handle->buffer_len;
- list_add_tail(&pasync_handle->link,
- &pasync_ctx->async_entry[cri].wait_queue.list);
- ppdu = pasync_handle->pbuffer;
- bytes_needed = ((((ppdu->dw[offsetof(struct amap_pdu_base,
- data_len_hi) / 32] & PDUBASE_DATALENHI_MASK) << 8) &
- 0xFFFF0000) | ((be16_to_cpu((ppdu->
- dw[offsetof(struct amap_pdu_base, data_len_lo) / 32]
- & PDUBASE_DATALENLO_MASK) >> 16)) & 0x0000FFFF));
- if (status == 0) {
- pasync_ctx->async_entry[cri].wait_queue.bytes_needed =
- bytes_needed;
- if (bytes_needed == 0)
- status = hwi_fwd_async_msg(beiscsi_conn, phba,
- pasync_ctx, cri);
- }
- } else {
- pasync_ctx->async_data.busy_entries--;
- if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
- list_add_tail(&pasync_handle->link,
- &pasync_ctx->async_entry[cri].wait_queue.
- list);
- pasync_ctx->async_entry[cri].wait_queue.
- bytes_received +=
- (unsigned short)pasync_handle->buffer_len;
- if (pasync_ctx->async_entry[cri].wait_queue.
- bytes_received >=
- pasync_ctx->async_entry[cri].wait_queue.
- bytes_needed)
- status = hwi_fwd_asy…