/drivers/net/ethernet/intel/igb/igb_main.c
http://github.com/mirrors/linux · C · 9603 lines · 6359 code · 1551 blank · 1693 comment · 1075 complexity · 087bb451361a649de260261d03bce9ee MD5 · raw file
Large files are truncated click here to view the full file
- // SPDX-License-Identifier: GPL-2.0
- /* Copyright(c) 2007 - 2018 Intel Corporation. */
- #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
- #include <linux/module.h>
- #include <linux/types.h>
- #include <linux/init.h>
- #include <linux/bitops.h>
- #include <linux/vmalloc.h>
- #include <linux/pagemap.h>
- #include <linux/netdevice.h>
- #include <linux/ipv6.h>
- #include <linux/slab.h>
- #include <net/checksum.h>
- #include <net/ip6_checksum.h>
- #include <net/pkt_sched.h>
- #include <net/pkt_cls.h>
- #include <linux/net_tstamp.h>
- #include <linux/mii.h>
- #include <linux/ethtool.h>
- #include <linux/if.h>
- #include <linux/if_vlan.h>
- #include <linux/pci.h>
- #include <linux/delay.h>
- #include <linux/interrupt.h>
- #include <linux/ip.h>
- #include <linux/tcp.h>
- #include <linux/sctp.h>
- #include <linux/if_ether.h>
- #include <linux/aer.h>
- #include <linux/prefetch.h>
- #include <linux/pm_runtime.h>
- #include <linux/etherdevice.h>
- #ifdef CONFIG_IGB_DCA
- #include <linux/dca.h>
- #endif
- #include <linux/i2c.h>
- #include "igb.h"
- #define MAJ 5
- #define MIN 6
- #define BUILD 0
- #define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
- __stringify(BUILD) "-k"
- enum queue_mode {
- QUEUE_MODE_STRICT_PRIORITY,
- QUEUE_MODE_STREAM_RESERVATION,
- };
- enum tx_queue_prio {
- TX_QUEUE_PRIO_HIGH,
- TX_QUEUE_PRIO_LOW,
- };
- char igb_driver_name[] = "igb";
- char igb_driver_version[] = DRV_VERSION;
- static const char igb_driver_string[] =
- "Intel(R) Gigabit Ethernet Network Driver";
- static const char igb_copyright[] =
- "Copyright (c) 2007-2014 Intel Corporation.";
- static const struct e1000_info *igb_info_tbl[] = {
- [board_82575] = &e1000_82575_info,
- };
- static const struct pci_device_id igb_pci_tbl[] = {
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_1GBPS) },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_SGMII) },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_I211_COPPER), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_FIBER), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SGMII), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER_FLASHLESS), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES_FLASHLESS), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
- { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
- /* required last entry */
- {0, }
- };
- MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
- static int igb_setup_all_tx_resources(struct igb_adapter *);
- static int igb_setup_all_rx_resources(struct igb_adapter *);
- static void igb_free_all_tx_resources(struct igb_adapter *);
- static void igb_free_all_rx_resources(struct igb_adapter *);
- static void igb_setup_mrqc(struct igb_adapter *);
- static int igb_probe(struct pci_dev *, const struct pci_device_id *);
- static void igb_remove(struct pci_dev *pdev);
- static int igb_sw_init(struct igb_adapter *);
- int igb_open(struct net_device *);
- int igb_close(struct net_device *);
- static void igb_configure(struct igb_adapter *);
- static void igb_configure_tx(struct igb_adapter *);
- static void igb_configure_rx(struct igb_adapter *);
- static void igb_clean_all_tx_rings(struct igb_adapter *);
- static void igb_clean_all_rx_rings(struct igb_adapter *);
- static void igb_clean_tx_ring(struct igb_ring *);
- static void igb_clean_rx_ring(struct igb_ring *);
- static void igb_set_rx_mode(struct net_device *);
- static void igb_update_phy_info(struct timer_list *);
- static void igb_watchdog(struct timer_list *);
- static void igb_watchdog_task(struct work_struct *);
- static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *);
- static void igb_get_stats64(struct net_device *dev,
- struct rtnl_link_stats64 *stats);
- static int igb_change_mtu(struct net_device *, int);
- static int igb_set_mac(struct net_device *, void *);
- static void igb_set_uta(struct igb_adapter *adapter, bool set);
- static irqreturn_t igb_intr(int irq, void *);
- static irqreturn_t igb_intr_msi(int irq, void *);
- static irqreturn_t igb_msix_other(int irq, void *);
- static irqreturn_t igb_msix_ring(int irq, void *);
- #ifdef CONFIG_IGB_DCA
- static void igb_update_dca(struct igb_q_vector *);
- static void igb_setup_dca(struct igb_adapter *);
- #endif /* CONFIG_IGB_DCA */
- static int igb_poll(struct napi_struct *, int);
- static bool igb_clean_tx_irq(struct igb_q_vector *, int);
- static int igb_clean_rx_irq(struct igb_q_vector *, int);
- static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
- static void igb_tx_timeout(struct net_device *, unsigned int txqueue);
- static void igb_reset_task(struct work_struct *);
- static void igb_vlan_mode(struct net_device *netdev,
- netdev_features_t features);
- static int igb_vlan_rx_add_vid(struct net_device *, __be16, u16);
- static int igb_vlan_rx_kill_vid(struct net_device *, __be16, u16);
- static void igb_restore_vlan(struct igb_adapter *);
- static void igb_rar_set_index(struct igb_adapter *, u32);
- static void igb_ping_all_vfs(struct igb_adapter *);
- static void igb_msg_task(struct igb_adapter *);
- static void igb_vmm_control(struct igb_adapter *);
- static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
- static void igb_flush_mac_table(struct igb_adapter *);
- static int igb_available_rars(struct igb_adapter *, u8);
- static void igb_set_default_mac_filter(struct igb_adapter *);
- static int igb_uc_sync(struct net_device *, const unsigned char *);
- static int igb_uc_unsync(struct net_device *, const unsigned char *);
- static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
- static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
- static int igb_ndo_set_vf_vlan(struct net_device *netdev,
- int vf, u16 vlan, u8 qos, __be16 vlan_proto);
- static int igb_ndo_set_vf_bw(struct net_device *, int, int, int);
- static int igb_ndo_set_vf_spoofchk(struct net_device *netdev, int vf,
- bool setting);
- static int igb_ndo_set_vf_trust(struct net_device *netdev, int vf,
- bool setting);
- static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
- struct ifla_vf_info *ivi);
- static void igb_check_vf_rate_limit(struct igb_adapter *);
- static void igb_nfc_filter_exit(struct igb_adapter *adapter);
- static void igb_nfc_filter_restore(struct igb_adapter *adapter);
- #ifdef CONFIG_PCI_IOV
- static int igb_vf_configure(struct igb_adapter *adapter, int vf);
- static int igb_pci_enable_sriov(struct pci_dev *dev, int num_vfs);
- static int igb_disable_sriov(struct pci_dev *dev);
- static int igb_pci_disable_sriov(struct pci_dev *dev);
- #endif
- static int igb_suspend(struct device *);
- static int igb_resume(struct device *);
- static int igb_runtime_suspend(struct device *dev);
- static int igb_runtime_resume(struct device *dev);
- static int igb_runtime_idle(struct device *dev);
- static const struct dev_pm_ops igb_pm_ops = {
- SET_SYSTEM_SLEEP_PM_OPS(igb_suspend, igb_resume)
- SET_RUNTIME_PM_OPS(igb_runtime_suspend, igb_runtime_resume,
- igb_runtime_idle)
- };
- static void igb_shutdown(struct pci_dev *);
- static int igb_pci_sriov_configure(struct pci_dev *dev, int num_vfs);
- #ifdef CONFIG_IGB_DCA
- static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
- static struct notifier_block dca_notifier = {
- .notifier_call = igb_notify_dca,
- .next = NULL,
- .priority = 0
- };
- #endif
- #ifdef CONFIG_PCI_IOV
- static unsigned int max_vfs;
- module_param(max_vfs, uint, 0);
- MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate per physical function");
- #endif /* CONFIG_PCI_IOV */
- static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
- pci_channel_state_t);
- static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
- static void igb_io_resume(struct pci_dev *);
- static const struct pci_error_handlers igb_err_handler = {
- .error_detected = igb_io_error_detected,
- .slot_reset = igb_io_slot_reset,
- .resume = igb_io_resume,
- };
- static void igb_init_dmac(struct igb_adapter *adapter, u32 pba);
- static struct pci_driver igb_driver = {
- .name = igb_driver_name,
- .id_table = igb_pci_tbl,
- .probe = igb_probe,
- .remove = igb_remove,
- #ifdef CONFIG_PM
- .driver.pm = &igb_pm_ops,
- #endif
- .shutdown = igb_shutdown,
- .sriov_configure = igb_pci_sriov_configure,
- .err_handler = &igb_err_handler
- };
- MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
- MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
- MODULE_LICENSE("GPL v2");
- MODULE_VERSION(DRV_VERSION);
- #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
- static int debug = -1;
- module_param(debug, int, 0);
- MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
- struct igb_reg_info {
- u32 ofs;
- char *name;
- };
- static const struct igb_reg_info igb_reg_info_tbl[] = {
- /* General Registers */
- {E1000_CTRL, "CTRL"},
- {E1000_STATUS, "STATUS"},
- {E1000_CTRL_EXT, "CTRL_EXT"},
- /* Interrupt Registers */
- {E1000_ICR, "ICR"},
- /* RX Registers */
- {E1000_RCTL, "RCTL"},
- {E1000_RDLEN(0), "RDLEN"},
- {E1000_RDH(0), "RDH"},
- {E1000_RDT(0), "RDT"},
- {E1000_RXDCTL(0), "RXDCTL"},
- {E1000_RDBAL(0), "RDBAL"},
- {E1000_RDBAH(0), "RDBAH"},
- /* TX Registers */
- {E1000_TCTL, "TCTL"},
- {E1000_TDBAL(0), "TDBAL"},
- {E1000_TDBAH(0), "TDBAH"},
- {E1000_TDLEN(0), "TDLEN"},
- {E1000_TDH(0), "TDH"},
- {E1000_TDT(0), "TDT"},
- {E1000_TXDCTL(0), "TXDCTL"},
- {E1000_TDFH, "TDFH"},
- {E1000_TDFT, "TDFT"},
- {E1000_TDFHS, "TDFHS"},
- {E1000_TDFPC, "TDFPC"},
- /* List Terminator */
- {}
- };
- /* igb_regdump - register printout routine */
- static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
- {
- int n = 0;
- char rname[16];
- u32 regs[8];
- switch (reginfo->ofs) {
- case E1000_RDLEN(0):
- for (n = 0; n < 4; n++)
- regs[n] = rd32(E1000_RDLEN(n));
- break;
- case E1000_RDH(0):
- for (n = 0; n < 4; n++)
- regs[n] = rd32(E1000_RDH(n));
- break;
- case E1000_RDT(0):
- for (n = 0; n < 4; n++)
- regs[n] = rd32(E1000_RDT(n));
- break;
- case E1000_RXDCTL(0):
- for (n = 0; n < 4; n++)
- regs[n] = rd32(E1000_RXDCTL(n));
- break;
- case E1000_RDBAL(0):
- for (n = 0; n < 4; n++)
- regs[n] = rd32(E1000_RDBAL(n));
- break;
- case E1000_RDBAH(0):
- for (n = 0; n < 4; n++)
- regs[n] = rd32(E1000_RDBAH(n));
- break;
- case E1000_TDBAL(0):
- for (n = 0; n < 4; n++)
- regs[n] = rd32(E1000_RDBAL(n));
- break;
- case E1000_TDBAH(0):
- for (n = 0; n < 4; n++)
- regs[n] = rd32(E1000_TDBAH(n));
- break;
- case E1000_TDLEN(0):
- for (n = 0; n < 4; n++)
- regs[n] = rd32(E1000_TDLEN(n));
- break;
- case E1000_TDH(0):
- for (n = 0; n < 4; n++)
- regs[n] = rd32(E1000_TDH(n));
- break;
- case E1000_TDT(0):
- for (n = 0; n < 4; n++)
- regs[n] = rd32(E1000_TDT(n));
- break;
- case E1000_TXDCTL(0):
- for (n = 0; n < 4; n++)
- regs[n] = rd32(E1000_TXDCTL(n));
- break;
- default:
- pr_info("%-15s %08x\n", reginfo->name, rd32(reginfo->ofs));
- return;
- }
- snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
- pr_info("%-15s %08x %08x %08x %08x\n", rname, regs[0], regs[1],
- regs[2], regs[3]);
- }
- /* igb_dump - Print registers, Tx-rings and Rx-rings */
- static void igb_dump(struct igb_adapter *adapter)
- {
- struct net_device *netdev = adapter->netdev;
- struct e1000_hw *hw = &adapter->hw;
- struct igb_reg_info *reginfo;
- struct igb_ring *tx_ring;
- union e1000_adv_tx_desc *tx_desc;
- struct my_u0 { u64 a; u64 b; } *u0;
- struct igb_ring *rx_ring;
- union e1000_adv_rx_desc *rx_desc;
- u32 staterr;
- u16 i, n;
- if (!netif_msg_hw(adapter))
- return;
- /* Print netdevice Info */
- if (netdev) {
- dev_info(&adapter->pdev->dev, "Net device Info\n");
- pr_info("Device Name state trans_start\n");
- pr_info("%-15s %016lX %016lX\n", netdev->name,
- netdev->state, dev_trans_start(netdev));
- }
- /* Print Registers */
- dev_info(&adapter->pdev->dev, "Register Dump\n");
- pr_info(" Register Name Value\n");
- for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
- reginfo->name; reginfo++) {
- igb_regdump(hw, reginfo);
- }
- /* Print TX Ring Summary */
- if (!netdev || !netif_running(netdev))
- goto exit;
- dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
- pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n");
- for (n = 0; n < adapter->num_tx_queues; n++) {
- struct igb_tx_buffer *buffer_info;
- tx_ring = adapter->tx_ring[n];
- buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
- pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n",
- n, tx_ring->next_to_use, tx_ring->next_to_clean,
- (u64)dma_unmap_addr(buffer_info, dma),
- dma_unmap_len(buffer_info, len),
- buffer_info->next_to_watch,
- (u64)buffer_info->time_stamp);
- }
- /* Print TX Rings */
- if (!netif_msg_tx_done(adapter))
- goto rx_ring_summary;
- dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
- /* Transmit Descriptor Formats
- *
- * Advanced Transmit Descriptor
- * +--------------------------------------------------------------+
- * 0 | Buffer Address [63:0] |
- * +--------------------------------------------------------------+
- * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
- * +--------------------------------------------------------------+
- * 63 46 45 40 39 38 36 35 32 31 24 15 0
- */
- for (n = 0; n < adapter->num_tx_queues; n++) {
- tx_ring = adapter->tx_ring[n];
- pr_info("------------------------------------\n");
- pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
- pr_info("------------------------------------\n");
- pr_info("T [desc] [address 63:0 ] [PlPOCIStDDM Ln] [bi->dma ] leng ntw timestamp bi->skb\n");
- for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
- const char *next_desc;
- struct igb_tx_buffer *buffer_info;
- tx_desc = IGB_TX_DESC(tx_ring, i);
- buffer_info = &tx_ring->tx_buffer_info[i];
- u0 = (struct my_u0 *)tx_desc;
- if (i == tx_ring->next_to_use &&
- i == tx_ring->next_to_clean)
- next_desc = " NTC/U";
- else if (i == tx_ring->next_to_use)
- next_desc = " NTU";
- else if (i == tx_ring->next_to_clean)
- next_desc = " NTC";
- else
- next_desc = "";
- pr_info("T [0x%03X] %016llX %016llX %016llX %04X %p %016llX %p%s\n",
- i, le64_to_cpu(u0->a),
- le64_to_cpu(u0->b),
- (u64)dma_unmap_addr(buffer_info, dma),
- dma_unmap_len(buffer_info, len),
- buffer_info->next_to_watch,
- (u64)buffer_info->time_stamp,
- buffer_info->skb, next_desc);
- if (netif_msg_pktdata(adapter) && buffer_info->skb)
- print_hex_dump(KERN_INFO, "",
- DUMP_PREFIX_ADDRESS,
- 16, 1, buffer_info->skb->data,
- dma_unmap_len(buffer_info, len),
- true);
- }
- }
- /* Print RX Rings Summary */
- rx_ring_summary:
- dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
- pr_info("Queue [NTU] [NTC]\n");
- for (n = 0; n < adapter->num_rx_queues; n++) {
- rx_ring = adapter->rx_ring[n];
- pr_info(" %5d %5X %5X\n",
- n, rx_ring->next_to_use, rx_ring->next_to_clean);
- }
- /* Print RX Rings */
- if (!netif_msg_rx_status(adapter))
- goto exit;
- dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
- /* Advanced Receive Descriptor (Read) Format
- * 63 1 0
- * +-----------------------------------------------------+
- * 0 | Packet Buffer Address [63:1] |A0/NSE|
- * +----------------------------------------------+------+
- * 8 | Header Buffer Address [63:1] | DD |
- * +-----------------------------------------------------+
- *
- *
- * Advanced Receive Descriptor (Write-Back) Format
- *
- * 63 48 47 32 31 30 21 20 17 16 4 3 0
- * +------------------------------------------------------+
- * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
- * | Checksum Ident | | | | Type | Type |
- * +------------------------------------------------------+
- * 8 | VLAN Tag | Length | Extended Error | Extended Status |
- * +------------------------------------------------------+
- * 63 48 47 32 31 20 19 0
- */
- for (n = 0; n < adapter->num_rx_queues; n++) {
- rx_ring = adapter->rx_ring[n];
- pr_info("------------------------------------\n");
- pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
- pr_info("------------------------------------\n");
- pr_info("R [desc] [ PktBuf A0] [ HeadBuf DD] [bi->dma ] [bi->skb] <-- Adv Rx Read format\n");
- pr_info("RWB[desc] [PcsmIpSHl PtRs] [vl er S cks ln] ---------------- [bi->skb] <-- Adv Rx Write-Back format\n");
- for (i = 0; i < rx_ring->count; i++) {
- const char *next_desc;
- struct igb_rx_buffer *buffer_info;
- buffer_info = &rx_ring->rx_buffer_info[i];
- rx_desc = IGB_RX_DESC(rx_ring, i);
- u0 = (struct my_u0 *)rx_desc;
- staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
- if (i == rx_ring->next_to_use)
- next_desc = " NTU";
- else if (i == rx_ring->next_to_clean)
- next_desc = " NTC";
- else
- next_desc = "";
- if (staterr & E1000_RXD_STAT_DD) {
- /* Descriptor Done */
- pr_info("%s[0x%03X] %016llX %016llX ---------------- %s\n",
- "RWB", i,
- le64_to_cpu(u0->a),
- le64_to_cpu(u0->b),
- next_desc);
- } else {
- pr_info("%s[0x%03X] %016llX %016llX %016llX %s\n",
- "R ", i,
- le64_to_cpu(u0->a),
- le64_to_cpu(u0->b),
- (u64)buffer_info->dma,
- next_desc);
- if (netif_msg_pktdata(adapter) &&
- buffer_info->dma && buffer_info->page) {
- print_hex_dump(KERN_INFO, "",
- DUMP_PREFIX_ADDRESS,
- 16, 1,
- page_address(buffer_info->page) +
- buffer_info->page_offset,
- igb_rx_bufsz(rx_ring), true);
- }
- }
- }
- }
- exit:
- return;
- }
- /**
- * igb_get_i2c_data - Reads the I2C SDA data bit
- * @hw: pointer to hardware structure
- * @i2cctl: Current value of I2CCTL register
- *
- * Returns the I2C data bit value
- **/
- static int igb_get_i2c_data(void *data)
- {
- struct igb_adapter *adapter = (struct igb_adapter *)data;
- struct e1000_hw *hw = &adapter->hw;
- s32 i2cctl = rd32(E1000_I2CPARAMS);
- return !!(i2cctl & E1000_I2C_DATA_IN);
- }
- /**
- * igb_set_i2c_data - Sets the I2C data bit
- * @data: pointer to hardware structure
- * @state: I2C data value (0 or 1) to set
- *
- * Sets the I2C data bit
- **/
- static void igb_set_i2c_data(void *data, int state)
- {
- struct igb_adapter *adapter = (struct igb_adapter *)data;
- struct e1000_hw *hw = &adapter->hw;
- s32 i2cctl = rd32(E1000_I2CPARAMS);
- if (state)
- i2cctl |= E1000_I2C_DATA_OUT;
- else
- i2cctl &= ~E1000_I2C_DATA_OUT;
- i2cctl &= ~E1000_I2C_DATA_OE_N;
- i2cctl |= E1000_I2C_CLK_OE_N;
- wr32(E1000_I2CPARAMS, i2cctl);
- wrfl();
- }
- /**
- * igb_set_i2c_clk - Sets the I2C SCL clock
- * @data: pointer to hardware structure
- * @state: state to set clock
- *
- * Sets the I2C clock line to state
- **/
- static void igb_set_i2c_clk(void *data, int state)
- {
- struct igb_adapter *adapter = (struct igb_adapter *)data;
- struct e1000_hw *hw = &adapter->hw;
- s32 i2cctl = rd32(E1000_I2CPARAMS);
- if (state) {
- i2cctl |= E1000_I2C_CLK_OUT;
- i2cctl &= ~E1000_I2C_CLK_OE_N;
- } else {
- i2cctl &= ~E1000_I2C_CLK_OUT;
- i2cctl &= ~E1000_I2C_CLK_OE_N;
- }
- wr32(E1000_I2CPARAMS, i2cctl);
- wrfl();
- }
- /**
- * igb_get_i2c_clk - Gets the I2C SCL clock state
- * @data: pointer to hardware structure
- *
- * Gets the I2C clock state
- **/
- static int igb_get_i2c_clk(void *data)
- {
- struct igb_adapter *adapter = (struct igb_adapter *)data;
- struct e1000_hw *hw = &adapter->hw;
- s32 i2cctl = rd32(E1000_I2CPARAMS);
- return !!(i2cctl & E1000_I2C_CLK_IN);
- }
- static const struct i2c_algo_bit_data igb_i2c_algo = {
- .setsda = igb_set_i2c_data,
- .setscl = igb_set_i2c_clk,
- .getsda = igb_get_i2c_data,
- .getscl = igb_get_i2c_clk,
- .udelay = 5,
- .timeout = 20,
- };
- /**
- * igb_get_hw_dev - return device
- * @hw: pointer to hardware structure
- *
- * used by hardware layer to print debugging information
- **/
- struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
- {
- struct igb_adapter *adapter = hw->back;
- return adapter->netdev;
- }
- /**
- * igb_init_module - Driver Registration Routine
- *
- * igb_init_module is the first routine called when the driver is
- * loaded. All it does is register with the PCI subsystem.
- **/
- static int __init igb_init_module(void)
- {
- int ret;
- pr_info("%s - version %s\n",
- igb_driver_string, igb_driver_version);
- pr_info("%s\n", igb_copyright);
- #ifdef CONFIG_IGB_DCA
- dca_register_notify(&dca_notifier);
- #endif
- ret = pci_register_driver(&igb_driver);
- return ret;
- }
- module_init(igb_init_module);
- /**
- * igb_exit_module - Driver Exit Cleanup Routine
- *
- * igb_exit_module is called just before the driver is removed
- * from memory.
- **/
- static void __exit igb_exit_module(void)
- {
- #ifdef CONFIG_IGB_DCA
- dca_unregister_notify(&dca_notifier);
- #endif
- pci_unregister_driver(&igb_driver);
- }
- module_exit(igb_exit_module);
- #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
- /**
- * igb_cache_ring_register - Descriptor ring to register mapping
- * @adapter: board private structure to initialize
- *
- * Once we know the feature-set enabled for the device, we'll cache
- * the register offset the descriptor ring is assigned to.
- **/
- static void igb_cache_ring_register(struct igb_adapter *adapter)
- {
- int i = 0, j = 0;
- u32 rbase_offset = adapter->vfs_allocated_count;
- switch (adapter->hw.mac.type) {
- case e1000_82576:
- /* The queues are allocated for virtualization such that VF 0
- * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
- * In order to avoid collision we start at the first free queue
- * and continue consuming queues in the same sequence
- */
- if (adapter->vfs_allocated_count) {
- for (; i < adapter->rss_queues; i++)
- adapter->rx_ring[i]->reg_idx = rbase_offset +
- Q_IDX_82576(i);
- }
- /* Fall through */
- case e1000_82575:
- case e1000_82580:
- case e1000_i350:
- case e1000_i354:
- case e1000_i210:
- case e1000_i211:
- /* Fall through */
- default:
- for (; i < adapter->num_rx_queues; i++)
- adapter->rx_ring[i]->reg_idx = rbase_offset + i;
- for (; j < adapter->num_tx_queues; j++)
- adapter->tx_ring[j]->reg_idx = rbase_offset + j;
- break;
- }
- }
- u32 igb_rd32(struct e1000_hw *hw, u32 reg)
- {
- struct igb_adapter *igb = container_of(hw, struct igb_adapter, hw);
- u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr);
- u32 value = 0;
- if (E1000_REMOVED(hw_addr))
- return ~value;
- value = readl(&hw_addr[reg]);
- /* reads should not return all F's */
- if (!(~value) && (!reg || !(~readl(hw_addr)))) {
- struct net_device *netdev = igb->netdev;
- hw->hw_addr = NULL;
- netdev_err(netdev, "PCIe link lost\n");
- WARN(pci_device_is_present(igb->pdev),
- "igb: Failed to read reg 0x%x!\n", reg);
- }
- return value;
- }
- /**
- * igb_write_ivar - configure ivar for given MSI-X vector
- * @hw: pointer to the HW structure
- * @msix_vector: vector number we are allocating to a given ring
- * @index: row index of IVAR register to write within IVAR table
- * @offset: column offset of in IVAR, should be multiple of 8
- *
- * This function is intended to handle the writing of the IVAR register
- * for adapters 82576 and newer. The IVAR table consists of 2 columns,
- * each containing an cause allocation for an Rx and Tx ring, and a
- * variable number of rows depending on the number of queues supported.
- **/
- static void igb_write_ivar(struct e1000_hw *hw, int msix_vector,
- int index, int offset)
- {
- u32 ivar = array_rd32(E1000_IVAR0, index);
- /* clear any bits that are currently set */
- ivar &= ~((u32)0xFF << offset);
- /* write vector and valid bit */
- ivar |= (msix_vector | E1000_IVAR_VALID) << offset;
- array_wr32(E1000_IVAR0, index, ivar);
- }
- #define IGB_N0_QUEUE -1
- static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
- {
- struct igb_adapter *adapter = q_vector->adapter;
- struct e1000_hw *hw = &adapter->hw;
- int rx_queue = IGB_N0_QUEUE;
- int tx_queue = IGB_N0_QUEUE;
- u32 msixbm = 0;
- if (q_vector->rx.ring)
- rx_queue = q_vector->rx.ring->reg_idx;
- if (q_vector->tx.ring)
- tx_queue = q_vector->tx.ring->reg_idx;
- switch (hw->mac.type) {
- case e1000_82575:
- /* The 82575 assigns vectors using a bitmask, which matches the
- * bitmask for the EICR/EIMS/EIMC registers. To assign one
- * or more queues to a vector, we write the appropriate bits
- * into the MSIXBM register for that vector.
- */
- if (rx_queue > IGB_N0_QUEUE)
- msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
- if (tx_queue > IGB_N0_QUEUE)
- msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
- if (!(adapter->flags & IGB_FLAG_HAS_MSIX) && msix_vector == 0)
- msixbm |= E1000_EIMS_OTHER;
- array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
- q_vector->eims_value = msixbm;
- break;
- case e1000_82576:
- /* 82576 uses a table that essentially consists of 2 columns
- * with 8 rows. The ordering is column-major so we use the
- * lower 3 bits as the row index, and the 4th bit as the
- * column offset.
- */
- if (rx_queue > IGB_N0_QUEUE)
- igb_write_ivar(hw, msix_vector,
- rx_queue & 0x7,
- (rx_queue & 0x8) << 1);
- if (tx_queue > IGB_N0_QUEUE)
- igb_write_ivar(hw, msix_vector,
- tx_queue & 0x7,
- ((tx_queue & 0x8) << 1) + 8);
- q_vector->eims_value = BIT(msix_vector);
- break;
- case e1000_82580:
- case e1000_i350:
- case e1000_i354:
- case e1000_i210:
- case e1000_i211:
- /* On 82580 and newer adapters the scheme is similar to 82576
- * however instead of ordering column-major we have things
- * ordered row-major. So we traverse the table by using
- * bit 0 as the column offset, and the remaining bits as the
- * row index.
- */
- if (rx_queue > IGB_N0_QUEUE)
- igb_write_ivar(hw, msix_vector,
- rx_queue >> 1,
- (rx_queue & 0x1) << 4);
- if (tx_queue > IGB_N0_QUEUE)
- igb_write_ivar(hw, msix_vector,
- tx_queue >> 1,
- ((tx_queue & 0x1) << 4) + 8);
- q_vector->eims_value = BIT(msix_vector);
- break;
- default:
- BUG();
- break;
- }
- /* add q_vector eims value to global eims_enable_mask */
- adapter->eims_enable_mask |= q_vector->eims_value;
- /* configure q_vector to set itr on first interrupt */
- q_vector->set_itr = 1;
- }
- /**
- * igb_configure_msix - Configure MSI-X hardware
- * @adapter: board private structure to initialize
- *
- * igb_configure_msix sets up the hardware to properly
- * generate MSI-X interrupts.
- **/
- static void igb_configure_msix(struct igb_adapter *adapter)
- {
- u32 tmp;
- int i, vector = 0;
- struct e1000_hw *hw = &adapter->hw;
- adapter->eims_enable_mask = 0;
- /* set vector for other causes, i.e. link changes */
- switch (hw->mac.type) {
- case e1000_82575:
- tmp = rd32(E1000_CTRL_EXT);
- /* enable MSI-X PBA support*/
- tmp |= E1000_CTRL_EXT_PBA_CLR;
- /* Auto-Mask interrupts upon ICR read. */
- tmp |= E1000_CTRL_EXT_EIAME;
- tmp |= E1000_CTRL_EXT_IRCA;
- wr32(E1000_CTRL_EXT, tmp);
- /* enable msix_other interrupt */
- array_wr32(E1000_MSIXBM(0), vector++, E1000_EIMS_OTHER);
- adapter->eims_other = E1000_EIMS_OTHER;
- break;
- case e1000_82576:
- case e1000_82580:
- case e1000_i350:
- case e1000_i354:
- case e1000_i210:
- case e1000_i211:
- /* Turn on MSI-X capability first, or our settings
- * won't stick. And it will take days to debug.
- */
- wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
- E1000_GPIE_PBA | E1000_GPIE_EIAME |
- E1000_GPIE_NSICR);
- /* enable msix_other interrupt */
- adapter->eims_other = BIT(vector);
- tmp = (vector++ | E1000_IVAR_VALID) << 8;
- wr32(E1000_IVAR_MISC, tmp);
- break;
- default:
- /* do nothing, since nothing else supports MSI-X */
- break;
- } /* switch (hw->mac.type) */
- adapter->eims_enable_mask |= adapter->eims_other;
- for (i = 0; i < adapter->num_q_vectors; i++)
- igb_assign_vector(adapter->q_vector[i], vector++);
- wrfl();
- }
- /**
- * igb_request_msix - Initialize MSI-X interrupts
- * @adapter: board private structure to initialize
- *
- * igb_request_msix allocates MSI-X vectors and requests interrupts from the
- * kernel.
- **/
- static int igb_request_msix(struct igb_adapter *adapter)
- {
- struct net_device *netdev = adapter->netdev;
- int i, err = 0, vector = 0, free_vector = 0;
- err = request_irq(adapter->msix_entries[vector].vector,
- igb_msix_other, 0, netdev->name, adapter);
- if (err)
- goto err_out;
- for (i = 0; i < adapter->num_q_vectors; i++) {
- struct igb_q_vector *q_vector = adapter->q_vector[i];
- vector++;
- q_vector->itr_register = adapter->io_addr + E1000_EITR(vector);
- if (q_vector->rx.ring && q_vector->tx.ring)
- sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
- q_vector->rx.ring->queue_index);
- else if (q_vector->tx.ring)
- sprintf(q_vector->name, "%s-tx-%u", netdev->name,
- q_vector->tx.ring->queue_index);
- else if (q_vector->rx.ring)
- sprintf(q_vector->name, "%s-rx-%u", netdev->name,
- q_vector->rx.ring->queue_index);
- else
- sprintf(q_vector->name, "%s-unused", netdev->name);
- err = request_irq(adapter->msix_entries[vector].vector,
- igb_msix_ring, 0, q_vector->name,
- q_vector);
- if (err)
- goto err_free;
- }
- igb_configure_msix(adapter);
- return 0;
- err_free:
- /* free already assigned IRQs */
- free_irq(adapter->msix_entries[free_vector++].vector, adapter);
- vector--;
- for (i = 0; i < vector; i++) {
- free_irq(adapter->msix_entries[free_vector++].vector,
- adapter->q_vector[i]);
- }
- err_out:
- return err;
- }
- /**
- * igb_free_q_vector - Free memory allocated for specific interrupt vector
- * @adapter: board private structure to initialize
- * @v_idx: Index of vector to be freed
- *
- * This function frees the memory allocated to the q_vector.
- **/
- static void igb_free_q_vector(struct igb_adapter *adapter, int v_idx)
- {
- struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
- adapter->q_vector[v_idx] = NULL;
- /* igb_get_stats64() might access the rings on this vector,
- * we must wait a grace period before freeing it.
- */
- if (q_vector)
- kfree_rcu(q_vector, rcu);
- }
- /**
- * igb_reset_q_vector - Reset config for interrupt vector
- * @adapter: board private structure to initialize
- * @v_idx: Index of vector to be reset
- *
- * If NAPI is enabled it will delete any references to the
- * NAPI struct. This is preparation for igb_free_q_vector.
- **/
- static void igb_reset_q_vector(struct igb_adapter *adapter, int v_idx)
- {
- struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
- /* Coming from igb_set_interrupt_capability, the vectors are not yet
- * allocated. So, q_vector is NULL so we should stop here.
- */
- if (!q_vector)
- return;
- if (q_vector->tx.ring)
- adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL;
- if (q_vector->rx.ring)
- adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL;
- netif_napi_del(&q_vector->napi);
- }
- static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
- {
- int v_idx = adapter->num_q_vectors;
- if (adapter->flags & IGB_FLAG_HAS_MSIX)
- pci_disable_msix(adapter->pdev);
- else if (adapter->flags & IGB_FLAG_HAS_MSI)
- pci_disable_msi(adapter->pdev);
- while (v_idx--)
- igb_reset_q_vector(adapter, v_idx);
- }
- /**
- * igb_free_q_vectors - Free memory allocated for interrupt vectors
- * @adapter: board private structure to initialize
- *
- * This function frees the memory allocated to the q_vectors. In addition if
- * NAPI is enabled it will delete any references to the NAPI struct prior
- * to freeing the q_vector.
- **/
- static void igb_free_q_vectors(struct igb_adapter *adapter)
- {
- int v_idx = adapter->num_q_vectors;
- adapter->num_tx_queues = 0;
- adapter->num_rx_queues = 0;
- adapter->num_q_vectors = 0;
- while (v_idx--) {
- igb_reset_q_vector(adapter, v_idx);
- igb_free_q_vector(adapter, v_idx);
- }
- }
- /**
- * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
- * @adapter: board private structure to initialize
- *
- * This function resets the device so that it has 0 Rx queues, Tx queues, and
- * MSI-X interrupts allocated.
- */
- static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
- {
- igb_free_q_vectors(adapter);
- igb_reset_interrupt_capability(adapter);
- }
- /**
- * igb_set_interrupt_capability - set MSI or MSI-X if supported
- * @adapter: board private structure to initialize
- * @msix: boolean value of MSIX capability
- *
- * Attempt to configure interrupts using the best available
- * capabilities of the hardware and kernel.
- **/
- static void igb_set_interrupt_capability(struct igb_adapter *adapter, bool msix)
- {
- int err;
- int numvecs, i;
- if (!msix)
- goto msi_only;
- adapter->flags |= IGB_FLAG_HAS_MSIX;
- /* Number of supported queues. */
- adapter->num_rx_queues = adapter->rss_queues;
- if (adapter->vfs_allocated_count)
- adapter->num_tx_queues = 1;
- else
- adapter->num_tx_queues = adapter->rss_queues;
- /* start with one vector for every Rx queue */
- numvecs = adapter->num_rx_queues;
- /* if Tx handler is separate add 1 for every Tx queue */
- if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
- numvecs += adapter->num_tx_queues;
- /* store the number of vectors reserved for queues */
- adapter->num_q_vectors = numvecs;
- /* add 1 vector for link status interrupts */
- numvecs++;
- for (i = 0; i < numvecs; i++)
- adapter->msix_entries[i].entry = i;
- err = pci_enable_msix_range(adapter->pdev,
- adapter->msix_entries,
- numvecs,
- numvecs);
- if (err > 0)
- return;
- igb_reset_interrupt_capability(adapter);
- /* If we can't do MSI-X, try MSI */
- msi_only:
- adapter->flags &= ~IGB_FLAG_HAS_MSIX;
- #ifdef CONFIG_PCI_IOV
- /* disable SR-IOV for non MSI-X configurations */
- if (adapter->vf_data) {
- struct e1000_hw *hw = &adapter->hw;
- /* disable iov and allow time for transactions to clear */
- pci_disable_sriov(adapter->pdev);
- msleep(500);
- kfree(adapter->vf_mac_list);
- adapter->vf_mac_list = NULL;
- kfree(adapter->vf_data);
- adapter->vf_data = NULL;
- wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
- wrfl();
- msleep(100);
- dev_info(&adapter->pdev->dev, "IOV Disabled\n");
- }
- #endif
- adapter->vfs_allocated_count = 0;
- adapter->rss_queues = 1;
- adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
- adapter->num_rx_queues = 1;
- adapter->num_tx_queues = 1;
- adapter->num_q_vectors = 1;
- if (!pci_enable_msi(adapter->pdev))
- adapter->flags |= IGB_FLAG_HAS_MSI;
- }
- static void igb_add_ring(struct igb_ring *ring,
- struct igb_ring_container *head)
- {
- head->ring = ring;
- head->count++;
- }
- /**
- * igb_alloc_q_vector - Allocate memory for a single interrupt vector
- * @adapter: board private structure to initialize
- * @v_count: q_vectors allocated on adapter, used for ring interleaving
- * @v_idx: index of vector in adapter struct
- * @txr_count: total number of Tx rings to allocate
- * @txr_idx: index of first Tx ring to allocate
- * @rxr_count: total number of Rx rings to allocate
- * @rxr_idx: index of first Rx ring to allocate
- *
- * We allocate one q_vector. If allocation fails we return -ENOMEM.
- **/
- static int igb_alloc_q_vector(struct igb_adapter *adapter,
- int v_count, int v_idx,
- int txr_count, int txr_idx,
- int rxr_count, int rxr_idx)
- {
- struct igb_q_vector *q_vector;
- struct igb_ring *ring;
- int ring_count;
- size_t size;
- /* igb only supports 1 Tx and/or 1 Rx queue per vector */
- if (txr_count > 1 || rxr_count > 1)
- return -ENOMEM;
- ring_count = txr_count + rxr_count;
- size = struct_size(q_vector, ring, ring_count);
- /* allocate q_vector and rings */
- q_vector = adapter->q_vector[v_idx];
- if (!q_vector) {
- q_vector = kzalloc(size, GFP_KERNEL);
- } else if (size > ksize(q_vector)) {
- kfree_rcu(q_vector, rcu);
- q_vector = kzalloc(size, GFP_KERNEL);
- } else {
- memset(q_vector, 0, size);
- }
- if (!q_vector)
- return -ENOMEM;
- /* initialize NAPI */
- netif_napi_add(adapter->netdev, &q_vector->napi,
- igb_poll, 64);
- /* tie q_vector and adapter together */
- adapter->q_vector[v_idx] = q_vector;
- q_vector->adapter = adapter;
- /* initialize work limits */
- q_vector->tx.work_limit = adapter->tx_work_limit;
- /* initialize ITR configuration */
- q_vector->itr_register = adapter->io_addr + E1000_EITR(0);
- q_vector->itr_val = IGB_START_ITR;
- /* initialize pointer to rings */
- ring = q_vector->ring;
- /* intialize ITR */
- if (rxr_count) {
- /* rx or rx/tx vector */
- if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3)
- q_vector->itr_val = adapter->rx_itr_setting;
- } else {
- /* tx only vector */
- if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3)
- q_vector->itr_val = adapter->tx_itr_setting;
- }
- if (txr_count) {
- /* assign generic ring traits */
- ring->dev = &adapter->pdev->dev;
- ring->netdev = adapter->netdev;
- /* configure backlink on ring */
- ring->q_vector = q_vector;
- /* update q_vector Tx values */
- igb_add_ring(ring, &q_vector->tx);
- /* For 82575, context index must be unique per ring. */
- if (adapter->hw.mac.type == e1000_82575)
- set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags);
- /* apply Tx specific ring traits */
- ring->count = adapter->tx_ring_count;
- ring->queue_index = txr_idx;
- ring->cbs_enable = false;
- ring->idleslope = 0;
- ring->sendslope = 0;
- ring->hicredit = 0;
- ring->locredit = 0;
- u64_stats_init(&ring->tx_syncp);
- u64_stats_init(&ring->tx_syncp2);
- /* assign ring to adapter */
- adapter->tx_ring[txr_idx] = ring;
- /* push pointer to next ring */
- ring++;
- }
- if (rxr_count) {
- /* assign generic ring traits */
- ring->dev = &adapter->pdev->dev;
- ring->netdev = adapter->netdev;
- /* configure backlink on ring */
- ring->q_vector = q_vector;
- /* update q_vector Rx values */
- igb_add_ring(ring, &q_vector->rx);
- /* set flag indicating ring supports SCTP checksum offload */
- if (adapter->hw.mac.type >= e1000_82576)
- set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags);
- /* On i350, i354, i210, and i211, loopback VLAN packets
- * have the tag byte-swapped.
- */
- if (adapter->hw.mac.type >= e1000_i350)
- set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags);
- /* apply Rx specific ring traits */
- ring->count = adapter->rx_ring_count;
- ring->queue_index = rxr_idx;
- u64_stats_init(&ring->rx_syncp);
- /* assign ring to adapter */
- adapter->rx_ring[rxr_idx] = ring;
- }
- return 0;
- }
- /**
- * igb_alloc_q_vectors - Allocate memory for interrupt vectors
- * @adapter: board private structure to initialize
- *
- * We allocate one q_vector per queue interrupt. If allocation fails we
- * return -ENOMEM.
- **/
- static int igb_alloc_q_vectors(struct igb_adapter *adapter)
- {
- int q_vectors = adapter->num_q_vectors;
- int rxr_remaining = adapter->num_rx_queues;
- int txr_remaining = adapter->num_tx_queues;
- int rxr_idx = 0, txr_idx = 0, v_idx = 0;
- int err;
- if (q_vectors >= (rxr_remaining + txr_remaining)) {
- for (; rxr_remaining; v_idx++) {
- err = igb_alloc_q_vector(adapter, q_vectors, v_idx,
- 0, 0, 1, rxr_idx);
- if (err)
- goto err_out;
- /* update counts and index */
- rxr_remaining--;
- rxr_idx++;
- }
- }
- for (; v_idx < q_vectors; v_idx++) {
- int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);
- int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);
- err = igb_alloc_q_vector(adapter, q_vectors, v_idx,
- tqpv, txr_idx, rqpv, rxr_idx);
- if (err)
- goto err_out;
- /* update counts and index */
- rxr_remaining -= rqpv;
- txr_remaining -= tqpv;
- rxr_idx++;
- txr_idx++;
- }
- return 0;
- err_out:
- adapter->num_tx_queues = 0;
- adapter->num_rx_queues = 0;
- adapter->num_q_vectors = 0;
- while (v_idx--)
- igb_free_q_vector(adapter, v_idx);
- return -ENOMEM;
- }
- /**
- * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
- * @adapter: board private structure to initialize
- * @msix: boolean value of MSIX capability
- *
- * This function initializes the interrupts and allocates all of the queues.
- **/
- static int igb_init_interrupt_scheme(struct igb_adapter *adapter, bool msix)
- {
- struct pci_dev *pdev = adapter->pdev;
- int err;
- igb_set_interrupt_capability(adapter, msix);
- err = igb_alloc_q_vectors(adapter);
- if (err) {
- dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
- goto err_alloc_q_vectors;
- }
- igb_cache_ring_register(adapter);
- return 0;
- err_alloc_q_vectors:
- igb_reset_interrupt_capability(adapter);
- return err;
- }
- /**
- * igb_request_irq - initialize interrupts
- * @adapter: board private structure to initialize
- *
- * Attempts to configure interrupts using the best available
- * capabilities of the hardware and kernel.
- **/
- static int igb_request_irq(struct igb_adapter *adapter)
- {
- struct net_device *netdev = adapter->netdev;
- struct pci_dev *pdev = adapter->pdev;
- int err = 0;
- if (adapter->flags & IGB_FLAG_HAS_MSIX) {
- err = igb_request_msix(adapter);
- if (!err)
- goto request_done;
- /* fall back to MSI */
- igb_free_all_tx_resources(adapter);
- igb_free_all_rx_resources(adapter);
- igb_clear_interrupt_scheme(adapter);
- err = igb_init_interrupt_scheme(adapter, false);
- if (err)
- goto request_done;
- igb_setup_all_tx_resources(adapter);
- igb_setup_all_rx_resources(adapter);
- igb_configure(adapter);
- }
- igb_assign_vector(adapter->q_vector[0], 0);
- if (adapter->flags & IGB_FLAG_HAS_MSI) {
- err = request_irq(pdev->irq, igb_intr_msi, 0,
- netdev->name, adapter);
- if (!err)
- goto request_done;
- /* fall back to legacy interrupts */
- igb_reset_interrupt_capability(adapter);
- adapter->flags &= ~IGB_FLAG_HAS_MSI;
- }
- err = request_irq(pdev->irq, igb_intr, IRQF_SHARED,
- netdev->name, adapter);
- if (err)
- dev_err(&pdev->dev, "Error %d getting interrupt\n",
- err);
- request_done:
- return err;
- }
- static void igb_free_irq(struct igb_adapter *adapter)
- {
- if (adapter->flags & IGB_FLAG_HAS_MSIX) {
- int vector = 0, i;
- free_irq(adapter->msix_entries[vector++].vector, adapter);
- for (i = 0; i < adapter->num_q_vectors; i++)
- free_irq(adapter->msix_entries[vector++].vector,
- adapter->q_vector[i]);
- } else {
- free_irq(adapter->pdev->irq, adapter);
- }
- }
- /**
- * igb_irq_disable - Mask off interrupt generation on the NIC
- * @adapter: board private structure
- **/
- static void igb_irq_disable(struct igb_adapter *adapter)
- {
- struct e1000_hw *hw = &adapter->hw;
- /* we need to be careful when disabling interrupts. The VFs are also
- * mapped into these registers and so clearing the bits can cause
- * issues on the VF drivers so we only need to clear what we set
- */
- if (adapter->flags & IGB_FLAG_HAS_MSIX) {
- u32 regval = rd32(E1000_EIAM);
- wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
- wr32(E1000_EIMC, adapter->eims_enable_mask);
- regval = rd32(E1000_EIAC);
- wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
- }
- wr32(E1000_IAM, 0);
- wr32(E1000_IMC, ~0);
- wrfl();
- if (adapter->flags & IGB_FLAG_HAS_MSIX) {
- int i;
- for (i = 0; i < adapter->num_q_vectors; i++)
- synchronize_irq(adapter->msix_entries[i].vector);
- } else {
- synchronize_irq(adapter->pdev->irq);
- }
- }
- /**
- * igb_irq_enable - Enable default interrupt generation settings
- * @adapter: board private structure
- **/
- static void igb_irq_enable(struct igb_adapter *adapter)
- {
- struct e1000_hw *hw = &adapter->hw;
- if (adapter->flags & IGB_FLAG_HAS_MSIX) {
- u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA;
- u32 regval = rd32(E1000_EIAC);
- wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
- regval = rd32(E1000_EIAM);
- wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
- wr32(E1000_EIMS, adapter->eims_enable_mask);
- if (adapter->vfs_allocated_count) {
- wr32(E1000_MBVFIMR, 0xFF);
- ims |= E1000_IMS_VMMB;
- }
- wr32(E1000_IMS, ims);
- } else {
- wr32(E1000_IMS, IMS_ENABLE_MASK |
- E1000_IMS_DRSTA);
- wr32(E1000_IAM, IMS_ENABLE_MASK |
- E1000_IMS_DRSTA);
- }
- }
- static void igb_update_mng_vlan(struct igb_adapter *adapter)
- {
- struct e1000_hw *hw = &adapter->hw;
- u16 pf_id = adapter->vfs_allocated_count;
- u16 vid = adapter->hw.mng_cookie.vlan_id;
- u16 old_vid = adapter->mng_vlan_id;
- if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
- /* add VID to filter table */
- igb_vfta_set(hw, vid, pf_id, true, true);
- adapter->mng_vlan_id = vid;
- } else {
- adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
- }
- if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
- (vid != old_vid) &&
- !test_bit(old_vid, adapter->active_vlans)) {
- /* remove VID from filter table */
- igb_vfta_set(hw, vid, pf_id, false, true);
- }
- }
- /**
- * igb_release_hw_control - release control of the h/w to f/w
- * @adapter: address of board private structure
- *
- * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
- * For ASF and Pass Through versions of f/w this means that the
- * driver is no longer loaded.
- **/
- static void igb_release_hw_control(struct igb_adapter *adapter)
- {
- struct e1000_hw *hw = &adapter->hw;
- u32 ctrl_ext;
- /* Let firmware take over control of h/w */
- ctrl_ext = rd32(E1000_CTRL_EXT);
- wr32(E1000_CTRL_EXT,
- ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
- }
- /**
- * igb_get_hw_control - get control of the h/w from f/w
- * @adapter: address of board private structure
- *
- * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
- * For ASF and Pass Through versions of f/w this means that
- * the driver is loaded.
- **/
- static void igb_get_hw_control(struct igb_adapter *adapter)
- {
- struct e1000_hw *hw = &adapter->hw;
- u32 ctrl_ext;
- /* Let firmware know the driver has taken over */
- ctrl_ext = rd32(E1000_CTRL_EXT);
- wr32(E1000_CTRL_EXT,
- ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
- }
- static void enable_fqtss(struct igb_adapter *adapter, bool enable)
- {
- struct net_device *netdev = adapter->netdev;
- struct e1000_hw *hw = &adapter->hw;
- WARN_ON(hw->mac.type != e1000_i210);
- if (enable)
- adapter->flags |= IGB_FLAG_FQTSS;
- else
- adapter->flags &= ~IGB_FLAG_FQTSS;
- if (netif_running(netdev))
- schedule_work(&adapter->reset_task);
- }
- static bool is_fqtss_enabled(struct igb_adapter *adapter)
- {
- return (adapter->flags & IGB_FLAG_FQTSS) ? true : false;
- }
- static void set_tx_desc_fetch_prio(struct e1000_hw *hw, int queue,
- enum tx_queue_prio prio)
- {
- u32 val;
- WARN_ON(hw->mac.type != e1000_i210);
- WARN_ON(queue < 0 || queue > 4);
- val = rd32(E1000_I210_TXDCTL(queue));
- if (prio == TX_QUEUE_PRIO_HIGH)
- val |= E1000_TXDCTL_PRIORITY;
- else
- val &= ~E1000_TXDCTL_PRIORITY;
- wr32(E1000_I210_TXDCTL(queue), val);
- }
- static void set_queue_mode(struct e1000_hw *hw, int queue, enum queue_mode mode)
- {
- u32 val;
- WARN_ON(hw->mac.type != e1000_i210);
- WARN_ON(queue < 0 || queue > 1);
- val = rd32(E1000_I210_TQAVCC(queue));
- if (mode == QUEUE_MODE_STREAM_RESERVATION)
- val |= E1000_TQAVCC_QUEUEMODE;
- else
- val &= ~E1000_TQAVCC_QUEUEMODE;
- wr32(E1000_I210_TQAVCC(queue), val);
- }
- static bool is_any_cbs_enabled(struct igb_adapter *adapter)
- {
- int i;
- for (i = 0; i < adapter->num_tx_queues; i++) {
- if (adapter->tx_ring[i]->cbs_enable)
- return true;
- }
- return false;
- }
- static bool is_any_txtime_enabled(struct igb_adapter *adapter)
- {
- int i;
- for (i = 0; i < adapter->num_tx_queues; i++) {
- if (adapter->tx_ring[i]->launchtime_enable)
- return true;
- }
- return false;
- }
- /**
- * igb_config_tx_modes - Configure "Qav Tx mode" features on igb
- * @adapter: pointer to adapter struct
- * @queue: queue number
- *
- * Configure CBS and Launchtime for a given hardware queue.
- * Parameters are retrieved from the correct Tx ring, so
- * igb_save_cbs_params() and igb_save_txtime_params() should be used
- * for setting those correctly prior to this function being called.
- **/
- static void igb_config_tx_modes(struct igb_adapter *adapter, int queue)
- {
- struct igb_ring *ring = adapter->tx_ring[queue];
- struct net_device *netdev = adapter->netdev;
- struct e1000_hw *hw = &adapter->hw;
- u32 tqavcc, tqavctrl;
- u16 value;
- WARN_ON(hw->mac.type != e1000_i210);
- WARN_ON(queue < 0 || queue > 1);
- /* If any of the Qav features is enabled, configure queues as SR and
- * with HIGH PRIO. If none is, then configure them with LOW PRIO and
- * as SP.
- */
- if (ring->cbs_enable || ring->launchtime_enable) {
- set_tx_desc_fetch_prio(hw, queue, TX_QUEUE_PRIO_HIGH);
- set_queue_mode(hw, queue, QUEUE_MODE_STREAM_RESERVATION);
- } else {
- set_tx_desc_fetch_prio(hw, queue, TX_QUEUE_PRIO_LOW);
- set_queue_mode(hw, queue, QUEUE_MODE_STRICT_PRIORITY);
- }
- /* If CBS is enabled, set DataTranARB and config its parameters. */
- if (ring->cbs_enable || queue == 0) {
- /* i210 does not allow the queue 0 to be in the Strict
- * Priority mode while the Qav mode is enabled, so,
- * instead of disabling strict priority mode, we give
- * queue 0 the maximum of credits possible.
- *
- * See section 8.12.19 of the i210 datasheet, "Note:
- * Queue0 QueueMode must be set to 1b when
- * TransmitMode is set to Qav."
- */
- if (queue == 0 && !ring->cbs_enable) {
- /* max "linkspeed" idleslope in kbps */
- ring->idleslope…