/drivers/net/ethernet/intel/igb/igb_main.c
C | 9603 lines | 6359 code | 1551 blank | 1693 comment | 1075 complexity | 087bb451361a649de260261d03bce9ee MD5 | raw file
Large files files are truncated, but you can click here to view the full file
1// SPDX-License-Identifier: GPL-2.0 2/* Copyright(c) 2007 - 2018 Intel Corporation. */ 3 4#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 5 6#include <linux/module.h> 7#include <linux/types.h> 8#include <linux/init.h> 9#include <linux/bitops.h> 10#include <linux/vmalloc.h> 11#include <linux/pagemap.h> 12#include <linux/netdevice.h> 13#include <linux/ipv6.h> 14#include <linux/slab.h> 15#include <net/checksum.h> 16#include <net/ip6_checksum.h> 17#include <net/pkt_sched.h> 18#include <net/pkt_cls.h> 19#include <linux/net_tstamp.h> 20#include <linux/mii.h> 21#include <linux/ethtool.h> 22#include <linux/if.h> 23#include <linux/if_vlan.h> 24#include <linux/pci.h> 25#include <linux/delay.h> 26#include <linux/interrupt.h> 27#include <linux/ip.h> 28#include <linux/tcp.h> 29#include <linux/sctp.h> 30#include <linux/if_ether.h> 31#include <linux/aer.h> 32#include <linux/prefetch.h> 33#include <linux/pm_runtime.h> 34#include <linux/etherdevice.h> 35#ifdef CONFIG_IGB_DCA 36#include <linux/dca.h> 37#endif 38#include <linux/i2c.h> 39#include "igb.h" 40 41#define MAJ 5 42#define MIN 6 43#define BUILD 0 44#define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \ 45__stringify(BUILD) "-k" 46 47enum queue_mode { 48 QUEUE_MODE_STRICT_PRIORITY, 49 QUEUE_MODE_STREAM_RESERVATION, 50}; 51 52enum tx_queue_prio { 53 TX_QUEUE_PRIO_HIGH, 54 TX_QUEUE_PRIO_LOW, 55}; 56 57char igb_driver_name[] = "igb"; 58char igb_driver_version[] = DRV_VERSION; 59static const char igb_driver_string[] = 60 "Intel(R) Gigabit Ethernet Network Driver"; 61static const char igb_copyright[] = 62 "Copyright (c) 2007-2014 Intel Corporation."; 63 64static const struct e1000_info *igb_info_tbl[] = { 65 [board_82575] = &e1000_82575_info, 66}; 67 68static const struct pci_device_id igb_pci_tbl[] = { 69 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_1GBPS) }, 70 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_SGMII) }, 71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) }, 72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I211_COPPER), board_82575 }, 73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER), board_82575 }, 74 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_FIBER), board_82575 }, 75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES), board_82575 }, 76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SGMII), board_82575 }, 77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER_FLASHLESS), board_82575 }, 78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES_FLASHLESS), board_82575 }, 79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 }, 80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 }, 81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 }, 82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 }, 83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 }, 84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 }, 85 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 }, 86 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 }, 87 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 }, 88 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 }, 89 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 }, 90 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 }, 91 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 }, 92 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 }, 93 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 }, 94 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 }, 95 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 }, 96 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 }, 97 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 }, 98 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 }, 99 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 }, 100 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 }, 101 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 }, 102 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 }, 103 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 }, 104 /* required last entry */ 105 {0, } 106}; 107 108MODULE_DEVICE_TABLE(pci, igb_pci_tbl); 109 110static int igb_setup_all_tx_resources(struct igb_adapter *); 111static int igb_setup_all_rx_resources(struct igb_adapter *); 112static void igb_free_all_tx_resources(struct igb_adapter *); 113static void igb_free_all_rx_resources(struct igb_adapter *); 114static void igb_setup_mrqc(struct igb_adapter *); 115static int igb_probe(struct pci_dev *, const struct pci_device_id *); 116static void igb_remove(struct pci_dev *pdev); 117static int igb_sw_init(struct igb_adapter *); 118int igb_open(struct net_device *); 119int igb_close(struct net_device *); 120static void igb_configure(struct igb_adapter *); 121static void igb_configure_tx(struct igb_adapter *); 122static void igb_configure_rx(struct igb_adapter *); 123static void igb_clean_all_tx_rings(struct igb_adapter *); 124static void igb_clean_all_rx_rings(struct igb_adapter *); 125static void igb_clean_tx_ring(struct igb_ring *); 126static void igb_clean_rx_ring(struct igb_ring *); 127static void igb_set_rx_mode(struct net_device *); 128static void igb_update_phy_info(struct timer_list *); 129static void igb_watchdog(struct timer_list *); 130static void igb_watchdog_task(struct work_struct *); 131static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *); 132static void igb_get_stats64(struct net_device *dev, 133 struct rtnl_link_stats64 *stats); 134static int igb_change_mtu(struct net_device *, int); 135static int igb_set_mac(struct net_device *, void *); 136static void igb_set_uta(struct igb_adapter *adapter, bool set); 137static irqreturn_t igb_intr(int irq, void *); 138static irqreturn_t igb_intr_msi(int irq, void *); 139static irqreturn_t igb_msix_other(int irq, void *); 140static irqreturn_t igb_msix_ring(int irq, void *); 141#ifdef CONFIG_IGB_DCA 142static void igb_update_dca(struct igb_q_vector *); 143static void igb_setup_dca(struct igb_adapter *); 144#endif /* CONFIG_IGB_DCA */ 145static int igb_poll(struct napi_struct *, int); 146static bool igb_clean_tx_irq(struct igb_q_vector *, int); 147static int igb_clean_rx_irq(struct igb_q_vector *, int); 148static int igb_ioctl(struct net_device *, struct ifreq *, int cmd); 149static void igb_tx_timeout(struct net_device *, unsigned int txqueue); 150static void igb_reset_task(struct work_struct *); 151static void igb_vlan_mode(struct net_device *netdev, 152 netdev_features_t features); 153static int igb_vlan_rx_add_vid(struct net_device *, __be16, u16); 154static int igb_vlan_rx_kill_vid(struct net_device *, __be16, u16); 155static void igb_restore_vlan(struct igb_adapter *); 156static void igb_rar_set_index(struct igb_adapter *, u32); 157static void igb_ping_all_vfs(struct igb_adapter *); 158static void igb_msg_task(struct igb_adapter *); 159static void igb_vmm_control(struct igb_adapter *); 160static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *); 161static void igb_flush_mac_table(struct igb_adapter *); 162static int igb_available_rars(struct igb_adapter *, u8); 163static void igb_set_default_mac_filter(struct igb_adapter *); 164static int igb_uc_sync(struct net_device *, const unsigned char *); 165static int igb_uc_unsync(struct net_device *, const unsigned char *); 166static void igb_restore_vf_multicasts(struct igb_adapter *adapter); 167static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac); 168static int igb_ndo_set_vf_vlan(struct net_device *netdev, 169 int vf, u16 vlan, u8 qos, __be16 vlan_proto); 170static int igb_ndo_set_vf_bw(struct net_device *, int, int, int); 171static int igb_ndo_set_vf_spoofchk(struct net_device *netdev, int vf, 172 bool setting); 173static int igb_ndo_set_vf_trust(struct net_device *netdev, int vf, 174 bool setting); 175static int igb_ndo_get_vf_config(struct net_device *netdev, int vf, 176 struct ifla_vf_info *ivi); 177static void igb_check_vf_rate_limit(struct igb_adapter *); 178static void igb_nfc_filter_exit(struct igb_adapter *adapter); 179static void igb_nfc_filter_restore(struct igb_adapter *adapter); 180 181#ifdef CONFIG_PCI_IOV 182static int igb_vf_configure(struct igb_adapter *adapter, int vf); 183static int igb_pci_enable_sriov(struct pci_dev *dev, int num_vfs); 184static int igb_disable_sriov(struct pci_dev *dev); 185static int igb_pci_disable_sriov(struct pci_dev *dev); 186#endif 187 188static int igb_suspend(struct device *); 189static int igb_resume(struct device *); 190static int igb_runtime_suspend(struct device *dev); 191static int igb_runtime_resume(struct device *dev); 192static int igb_runtime_idle(struct device *dev); 193static const struct dev_pm_ops igb_pm_ops = { 194 SET_SYSTEM_SLEEP_PM_OPS(igb_suspend, igb_resume) 195 SET_RUNTIME_PM_OPS(igb_runtime_suspend, igb_runtime_resume, 196 igb_runtime_idle) 197}; 198static void igb_shutdown(struct pci_dev *); 199static int igb_pci_sriov_configure(struct pci_dev *dev, int num_vfs); 200#ifdef CONFIG_IGB_DCA 201static int igb_notify_dca(struct notifier_block *, unsigned long, void *); 202static struct notifier_block dca_notifier = { 203 .notifier_call = igb_notify_dca, 204 .next = NULL, 205 .priority = 0 206}; 207#endif 208#ifdef CONFIG_PCI_IOV 209static unsigned int max_vfs; 210module_param(max_vfs, uint, 0); 211MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate per physical function"); 212#endif /* CONFIG_PCI_IOV */ 213 214static pci_ers_result_t igb_io_error_detected(struct pci_dev *, 215 pci_channel_state_t); 216static pci_ers_result_t igb_io_slot_reset(struct pci_dev *); 217static void igb_io_resume(struct pci_dev *); 218 219static const struct pci_error_handlers igb_err_handler = { 220 .error_detected = igb_io_error_detected, 221 .slot_reset = igb_io_slot_reset, 222 .resume = igb_io_resume, 223}; 224 225static void igb_init_dmac(struct igb_adapter *adapter, u32 pba); 226 227static struct pci_driver igb_driver = { 228 .name = igb_driver_name, 229 .id_table = igb_pci_tbl, 230 .probe = igb_probe, 231 .remove = igb_remove, 232#ifdef CONFIG_PM 233 .driver.pm = &igb_pm_ops, 234#endif 235 .shutdown = igb_shutdown, 236 .sriov_configure = igb_pci_sriov_configure, 237 .err_handler = &igb_err_handler 238}; 239 240MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>"); 241MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver"); 242MODULE_LICENSE("GPL v2"); 243MODULE_VERSION(DRV_VERSION); 244 245#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) 246static int debug = -1; 247module_param(debug, int, 0); 248MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 249 250struct igb_reg_info { 251 u32 ofs; 252 char *name; 253}; 254 255static const struct igb_reg_info igb_reg_info_tbl[] = { 256 257 /* General Registers */ 258 {E1000_CTRL, "CTRL"}, 259 {E1000_STATUS, "STATUS"}, 260 {E1000_CTRL_EXT, "CTRL_EXT"}, 261 262 /* Interrupt Registers */ 263 {E1000_ICR, "ICR"}, 264 265 /* RX Registers */ 266 {E1000_RCTL, "RCTL"}, 267 {E1000_RDLEN(0), "RDLEN"}, 268 {E1000_RDH(0), "RDH"}, 269 {E1000_RDT(0), "RDT"}, 270 {E1000_RXDCTL(0), "RXDCTL"}, 271 {E1000_RDBAL(0), "RDBAL"}, 272 {E1000_RDBAH(0), "RDBAH"}, 273 274 /* TX Registers */ 275 {E1000_TCTL, "TCTL"}, 276 {E1000_TDBAL(0), "TDBAL"}, 277 {E1000_TDBAH(0), "TDBAH"}, 278 {E1000_TDLEN(0), "TDLEN"}, 279 {E1000_TDH(0), "TDH"}, 280 {E1000_TDT(0), "TDT"}, 281 {E1000_TXDCTL(0), "TXDCTL"}, 282 {E1000_TDFH, "TDFH"}, 283 {E1000_TDFT, "TDFT"}, 284 {E1000_TDFHS, "TDFHS"}, 285 {E1000_TDFPC, "TDFPC"}, 286 287 /* List Terminator */ 288 {} 289}; 290 291/* igb_regdump - register printout routine */ 292static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo) 293{ 294 int n = 0; 295 char rname[16]; 296 u32 regs[8]; 297 298 switch (reginfo->ofs) { 299 case E1000_RDLEN(0): 300 for (n = 0; n < 4; n++) 301 regs[n] = rd32(E1000_RDLEN(n)); 302 break; 303 case E1000_RDH(0): 304 for (n = 0; n < 4; n++) 305 regs[n] = rd32(E1000_RDH(n)); 306 break; 307 case E1000_RDT(0): 308 for (n = 0; n < 4; n++) 309 regs[n] = rd32(E1000_RDT(n)); 310 break; 311 case E1000_RXDCTL(0): 312 for (n = 0; n < 4; n++) 313 regs[n] = rd32(E1000_RXDCTL(n)); 314 break; 315 case E1000_RDBAL(0): 316 for (n = 0; n < 4; n++) 317 regs[n] = rd32(E1000_RDBAL(n)); 318 break; 319 case E1000_RDBAH(0): 320 for (n = 0; n < 4; n++) 321 regs[n] = rd32(E1000_RDBAH(n)); 322 break; 323 case E1000_TDBAL(0): 324 for (n = 0; n < 4; n++) 325 regs[n] = rd32(E1000_RDBAL(n)); 326 break; 327 case E1000_TDBAH(0): 328 for (n = 0; n < 4; n++) 329 regs[n] = rd32(E1000_TDBAH(n)); 330 break; 331 case E1000_TDLEN(0): 332 for (n = 0; n < 4; n++) 333 regs[n] = rd32(E1000_TDLEN(n)); 334 break; 335 case E1000_TDH(0): 336 for (n = 0; n < 4; n++) 337 regs[n] = rd32(E1000_TDH(n)); 338 break; 339 case E1000_TDT(0): 340 for (n = 0; n < 4; n++) 341 regs[n] = rd32(E1000_TDT(n)); 342 break; 343 case E1000_TXDCTL(0): 344 for (n = 0; n < 4; n++) 345 regs[n] = rd32(E1000_TXDCTL(n)); 346 break; 347 default: 348 pr_info("%-15s %08x\n", reginfo->name, rd32(reginfo->ofs)); 349 return; 350 } 351 352 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]"); 353 pr_info("%-15s %08x %08x %08x %08x\n", rname, regs[0], regs[1], 354 regs[2], regs[3]); 355} 356 357/* igb_dump - Print registers, Tx-rings and Rx-rings */ 358static void igb_dump(struct igb_adapter *adapter) 359{ 360 struct net_device *netdev = adapter->netdev; 361 struct e1000_hw *hw = &adapter->hw; 362 struct igb_reg_info *reginfo; 363 struct igb_ring *tx_ring; 364 union e1000_adv_tx_desc *tx_desc; 365 struct my_u0 { u64 a; u64 b; } *u0; 366 struct igb_ring *rx_ring; 367 union e1000_adv_rx_desc *rx_desc; 368 u32 staterr; 369 u16 i, n; 370 371 if (!netif_msg_hw(adapter)) 372 return; 373 374 /* Print netdevice Info */ 375 if (netdev) { 376 dev_info(&adapter->pdev->dev, "Net device Info\n"); 377 pr_info("Device Name state trans_start\n"); 378 pr_info("%-15s %016lX %016lX\n", netdev->name, 379 netdev->state, dev_trans_start(netdev)); 380 } 381 382 /* Print Registers */ 383 dev_info(&adapter->pdev->dev, "Register Dump\n"); 384 pr_info(" Register Name Value\n"); 385 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl; 386 reginfo->name; reginfo++) { 387 igb_regdump(hw, reginfo); 388 } 389 390 /* Print TX Ring Summary */ 391 if (!netdev || !netif_running(netdev)) 392 goto exit; 393 394 dev_info(&adapter->pdev->dev, "TX Rings Summary\n"); 395 pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n"); 396 for (n = 0; n < adapter->num_tx_queues; n++) { 397 struct igb_tx_buffer *buffer_info; 398 tx_ring = adapter->tx_ring[n]; 399 buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean]; 400 pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n", 401 n, tx_ring->next_to_use, tx_ring->next_to_clean, 402 (u64)dma_unmap_addr(buffer_info, dma), 403 dma_unmap_len(buffer_info, len), 404 buffer_info->next_to_watch, 405 (u64)buffer_info->time_stamp); 406 } 407 408 /* Print TX Rings */ 409 if (!netif_msg_tx_done(adapter)) 410 goto rx_ring_summary; 411 412 dev_info(&adapter->pdev->dev, "TX Rings Dump\n"); 413 414 /* Transmit Descriptor Formats 415 * 416 * Advanced Transmit Descriptor 417 * +--------------------------------------------------------------+ 418 * 0 | Buffer Address [63:0] | 419 * +--------------------------------------------------------------+ 420 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN | 421 * +--------------------------------------------------------------+ 422 * 63 46 45 40 39 38 36 35 32 31 24 15 0 423 */ 424 425 for (n = 0; n < adapter->num_tx_queues; n++) { 426 tx_ring = adapter->tx_ring[n]; 427 pr_info("------------------------------------\n"); 428 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index); 429 pr_info("------------------------------------\n"); 430 pr_info("T [desc] [address 63:0 ] [PlPOCIStDDM Ln] [bi->dma ] leng ntw timestamp bi->skb\n"); 431 432 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) { 433 const char *next_desc; 434 struct igb_tx_buffer *buffer_info; 435 tx_desc = IGB_TX_DESC(tx_ring, i); 436 buffer_info = &tx_ring->tx_buffer_info[i]; 437 u0 = (struct my_u0 *)tx_desc; 438 if (i == tx_ring->next_to_use && 439 i == tx_ring->next_to_clean) 440 next_desc = " NTC/U"; 441 else if (i == tx_ring->next_to_use) 442 next_desc = " NTU"; 443 else if (i == tx_ring->next_to_clean) 444 next_desc = " NTC"; 445 else 446 next_desc = ""; 447 448 pr_info("T [0x%03X] %016llX %016llX %016llX %04X %p %016llX %p%s\n", 449 i, le64_to_cpu(u0->a), 450 le64_to_cpu(u0->b), 451 (u64)dma_unmap_addr(buffer_info, dma), 452 dma_unmap_len(buffer_info, len), 453 buffer_info->next_to_watch, 454 (u64)buffer_info->time_stamp, 455 buffer_info->skb, next_desc); 456 457 if (netif_msg_pktdata(adapter) && buffer_info->skb) 458 print_hex_dump(KERN_INFO, "", 459 DUMP_PREFIX_ADDRESS, 460 16, 1, buffer_info->skb->data, 461 dma_unmap_len(buffer_info, len), 462 true); 463 } 464 } 465 466 /* Print RX Rings Summary */ 467rx_ring_summary: 468 dev_info(&adapter->pdev->dev, "RX Rings Summary\n"); 469 pr_info("Queue [NTU] [NTC]\n"); 470 for (n = 0; n < adapter->num_rx_queues; n++) { 471 rx_ring = adapter->rx_ring[n]; 472 pr_info(" %5d %5X %5X\n", 473 n, rx_ring->next_to_use, rx_ring->next_to_clean); 474 } 475 476 /* Print RX Rings */ 477 if (!netif_msg_rx_status(adapter)) 478 goto exit; 479 480 dev_info(&adapter->pdev->dev, "RX Rings Dump\n"); 481 482 /* Advanced Receive Descriptor (Read) Format 483 * 63 1 0 484 * +-----------------------------------------------------+ 485 * 0 | Packet Buffer Address [63:1] |A0/NSE| 486 * +----------------------------------------------+------+ 487 * 8 | Header Buffer Address [63:1] | DD | 488 * +-----------------------------------------------------+ 489 * 490 * 491 * Advanced Receive Descriptor (Write-Back) Format 492 * 493 * 63 48 47 32 31 30 21 20 17 16 4 3 0 494 * +------------------------------------------------------+ 495 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS | 496 * | Checksum Ident | | | | Type | Type | 497 * +------------------------------------------------------+ 498 * 8 | VLAN Tag | Length | Extended Error | Extended Status | 499 * +------------------------------------------------------+ 500 * 63 48 47 32 31 20 19 0 501 */ 502 503 for (n = 0; n < adapter->num_rx_queues; n++) { 504 rx_ring = adapter->rx_ring[n]; 505 pr_info("------------------------------------\n"); 506 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index); 507 pr_info("------------------------------------\n"); 508 pr_info("R [desc] [ PktBuf A0] [ HeadBuf DD] [bi->dma ] [bi->skb] <-- Adv Rx Read format\n"); 509 pr_info("RWB[desc] [PcsmIpSHl PtRs] [vl er S cks ln] ---------------- [bi->skb] <-- Adv Rx Write-Back format\n"); 510 511 for (i = 0; i < rx_ring->count; i++) { 512 const char *next_desc; 513 struct igb_rx_buffer *buffer_info; 514 buffer_info = &rx_ring->rx_buffer_info[i]; 515 rx_desc = IGB_RX_DESC(rx_ring, i); 516 u0 = (struct my_u0 *)rx_desc; 517 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 518 519 if (i == rx_ring->next_to_use) 520 next_desc = " NTU"; 521 else if (i == rx_ring->next_to_clean) 522 next_desc = " NTC"; 523 else 524 next_desc = ""; 525 526 if (staterr & E1000_RXD_STAT_DD) { 527 /* Descriptor Done */ 528 pr_info("%s[0x%03X] %016llX %016llX ---------------- %s\n", 529 "RWB", i, 530 le64_to_cpu(u0->a), 531 le64_to_cpu(u0->b), 532 next_desc); 533 } else { 534 pr_info("%s[0x%03X] %016llX %016llX %016llX %s\n", 535 "R ", i, 536 le64_to_cpu(u0->a), 537 le64_to_cpu(u0->b), 538 (u64)buffer_info->dma, 539 next_desc); 540 541 if (netif_msg_pktdata(adapter) && 542 buffer_info->dma && buffer_info->page) { 543 print_hex_dump(KERN_INFO, "", 544 DUMP_PREFIX_ADDRESS, 545 16, 1, 546 page_address(buffer_info->page) + 547 buffer_info->page_offset, 548 igb_rx_bufsz(rx_ring), true); 549 } 550 } 551 } 552 } 553 554exit: 555 return; 556} 557 558/** 559 * igb_get_i2c_data - Reads the I2C SDA data bit 560 * @hw: pointer to hardware structure 561 * @i2cctl: Current value of I2CCTL register 562 * 563 * Returns the I2C data bit value 564 **/ 565static int igb_get_i2c_data(void *data) 566{ 567 struct igb_adapter *adapter = (struct igb_adapter *)data; 568 struct e1000_hw *hw = &adapter->hw; 569 s32 i2cctl = rd32(E1000_I2CPARAMS); 570 571 return !!(i2cctl & E1000_I2C_DATA_IN); 572} 573 574/** 575 * igb_set_i2c_data - Sets the I2C data bit 576 * @data: pointer to hardware structure 577 * @state: I2C data value (0 or 1) to set 578 * 579 * Sets the I2C data bit 580 **/ 581static void igb_set_i2c_data(void *data, int state) 582{ 583 struct igb_adapter *adapter = (struct igb_adapter *)data; 584 struct e1000_hw *hw = &adapter->hw; 585 s32 i2cctl = rd32(E1000_I2CPARAMS); 586 587 if (state) 588 i2cctl |= E1000_I2C_DATA_OUT; 589 else 590 i2cctl &= ~E1000_I2C_DATA_OUT; 591 592 i2cctl &= ~E1000_I2C_DATA_OE_N; 593 i2cctl |= E1000_I2C_CLK_OE_N; 594 wr32(E1000_I2CPARAMS, i2cctl); 595 wrfl(); 596 597} 598 599/** 600 * igb_set_i2c_clk - Sets the I2C SCL clock 601 * @data: pointer to hardware structure 602 * @state: state to set clock 603 * 604 * Sets the I2C clock line to state 605 **/ 606static void igb_set_i2c_clk(void *data, int state) 607{ 608 struct igb_adapter *adapter = (struct igb_adapter *)data; 609 struct e1000_hw *hw = &adapter->hw; 610 s32 i2cctl = rd32(E1000_I2CPARAMS); 611 612 if (state) { 613 i2cctl |= E1000_I2C_CLK_OUT; 614 i2cctl &= ~E1000_I2C_CLK_OE_N; 615 } else { 616 i2cctl &= ~E1000_I2C_CLK_OUT; 617 i2cctl &= ~E1000_I2C_CLK_OE_N; 618 } 619 wr32(E1000_I2CPARAMS, i2cctl); 620 wrfl(); 621} 622 623/** 624 * igb_get_i2c_clk - Gets the I2C SCL clock state 625 * @data: pointer to hardware structure 626 * 627 * Gets the I2C clock state 628 **/ 629static int igb_get_i2c_clk(void *data) 630{ 631 struct igb_adapter *adapter = (struct igb_adapter *)data; 632 struct e1000_hw *hw = &adapter->hw; 633 s32 i2cctl = rd32(E1000_I2CPARAMS); 634 635 return !!(i2cctl & E1000_I2C_CLK_IN); 636} 637 638static const struct i2c_algo_bit_data igb_i2c_algo = { 639 .setsda = igb_set_i2c_data, 640 .setscl = igb_set_i2c_clk, 641 .getsda = igb_get_i2c_data, 642 .getscl = igb_get_i2c_clk, 643 .udelay = 5, 644 .timeout = 20, 645}; 646 647/** 648 * igb_get_hw_dev - return device 649 * @hw: pointer to hardware structure 650 * 651 * used by hardware layer to print debugging information 652 **/ 653struct net_device *igb_get_hw_dev(struct e1000_hw *hw) 654{ 655 struct igb_adapter *adapter = hw->back; 656 return adapter->netdev; 657} 658 659/** 660 * igb_init_module - Driver Registration Routine 661 * 662 * igb_init_module is the first routine called when the driver is 663 * loaded. All it does is register with the PCI subsystem. 664 **/ 665static int __init igb_init_module(void) 666{ 667 int ret; 668 669 pr_info("%s - version %s\n", 670 igb_driver_string, igb_driver_version); 671 pr_info("%s\n", igb_copyright); 672 673#ifdef CONFIG_IGB_DCA 674 dca_register_notify(&dca_notifier); 675#endif 676 ret = pci_register_driver(&igb_driver); 677 return ret; 678} 679 680module_init(igb_init_module); 681 682/** 683 * igb_exit_module - Driver Exit Cleanup Routine 684 * 685 * igb_exit_module is called just before the driver is removed 686 * from memory. 687 **/ 688static void __exit igb_exit_module(void) 689{ 690#ifdef CONFIG_IGB_DCA 691 dca_unregister_notify(&dca_notifier); 692#endif 693 pci_unregister_driver(&igb_driver); 694} 695 696module_exit(igb_exit_module); 697 698#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1)) 699/** 700 * igb_cache_ring_register - Descriptor ring to register mapping 701 * @adapter: board private structure to initialize 702 * 703 * Once we know the feature-set enabled for the device, we'll cache 704 * the register offset the descriptor ring is assigned to. 705 **/ 706static void igb_cache_ring_register(struct igb_adapter *adapter) 707{ 708 int i = 0, j = 0; 709 u32 rbase_offset = adapter->vfs_allocated_count; 710 711 switch (adapter->hw.mac.type) { 712 case e1000_82576: 713 /* The queues are allocated for virtualization such that VF 0 714 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc. 715 * In order to avoid collision we start at the first free queue 716 * and continue consuming queues in the same sequence 717 */ 718 if (adapter->vfs_allocated_count) { 719 for (; i < adapter->rss_queues; i++) 720 adapter->rx_ring[i]->reg_idx = rbase_offset + 721 Q_IDX_82576(i); 722 } 723 /* Fall through */ 724 case e1000_82575: 725 case e1000_82580: 726 case e1000_i350: 727 case e1000_i354: 728 case e1000_i210: 729 case e1000_i211: 730 /* Fall through */ 731 default: 732 for (; i < adapter->num_rx_queues; i++) 733 adapter->rx_ring[i]->reg_idx = rbase_offset + i; 734 for (; j < adapter->num_tx_queues; j++) 735 adapter->tx_ring[j]->reg_idx = rbase_offset + j; 736 break; 737 } 738} 739 740u32 igb_rd32(struct e1000_hw *hw, u32 reg) 741{ 742 struct igb_adapter *igb = container_of(hw, struct igb_adapter, hw); 743 u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr); 744 u32 value = 0; 745 746 if (E1000_REMOVED(hw_addr)) 747 return ~value; 748 749 value = readl(&hw_addr[reg]); 750 751 /* reads should not return all F's */ 752 if (!(~value) && (!reg || !(~readl(hw_addr)))) { 753 struct net_device *netdev = igb->netdev; 754 hw->hw_addr = NULL; 755 netdev_err(netdev, "PCIe link lost\n"); 756 WARN(pci_device_is_present(igb->pdev), 757 "igb: Failed to read reg 0x%x!\n", reg); 758 } 759 760 return value; 761} 762 763/** 764 * igb_write_ivar - configure ivar for given MSI-X vector 765 * @hw: pointer to the HW structure 766 * @msix_vector: vector number we are allocating to a given ring 767 * @index: row index of IVAR register to write within IVAR table 768 * @offset: column offset of in IVAR, should be multiple of 8 769 * 770 * This function is intended to handle the writing of the IVAR register 771 * for adapters 82576 and newer. The IVAR table consists of 2 columns, 772 * each containing an cause allocation for an Rx and Tx ring, and a 773 * variable number of rows depending on the number of queues supported. 774 **/ 775static void igb_write_ivar(struct e1000_hw *hw, int msix_vector, 776 int index, int offset) 777{ 778 u32 ivar = array_rd32(E1000_IVAR0, index); 779 780 /* clear any bits that are currently set */ 781 ivar &= ~((u32)0xFF << offset); 782 783 /* write vector and valid bit */ 784 ivar |= (msix_vector | E1000_IVAR_VALID) << offset; 785 786 array_wr32(E1000_IVAR0, index, ivar); 787} 788 789#define IGB_N0_QUEUE -1 790static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector) 791{ 792 struct igb_adapter *adapter = q_vector->adapter; 793 struct e1000_hw *hw = &adapter->hw; 794 int rx_queue = IGB_N0_QUEUE; 795 int tx_queue = IGB_N0_QUEUE; 796 u32 msixbm = 0; 797 798 if (q_vector->rx.ring) 799 rx_queue = q_vector->rx.ring->reg_idx; 800 if (q_vector->tx.ring) 801 tx_queue = q_vector->tx.ring->reg_idx; 802 803 switch (hw->mac.type) { 804 case e1000_82575: 805 /* The 82575 assigns vectors using a bitmask, which matches the 806 * bitmask for the EICR/EIMS/EIMC registers. To assign one 807 * or more queues to a vector, we write the appropriate bits 808 * into the MSIXBM register for that vector. 809 */ 810 if (rx_queue > IGB_N0_QUEUE) 811 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue; 812 if (tx_queue > IGB_N0_QUEUE) 813 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue; 814 if (!(adapter->flags & IGB_FLAG_HAS_MSIX) && msix_vector == 0) 815 msixbm |= E1000_EIMS_OTHER; 816 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm); 817 q_vector->eims_value = msixbm; 818 break; 819 case e1000_82576: 820 /* 82576 uses a table that essentially consists of 2 columns 821 * with 8 rows. The ordering is column-major so we use the 822 * lower 3 bits as the row index, and the 4th bit as the 823 * column offset. 824 */ 825 if (rx_queue > IGB_N0_QUEUE) 826 igb_write_ivar(hw, msix_vector, 827 rx_queue & 0x7, 828 (rx_queue & 0x8) << 1); 829 if (tx_queue > IGB_N0_QUEUE) 830 igb_write_ivar(hw, msix_vector, 831 tx_queue & 0x7, 832 ((tx_queue & 0x8) << 1) + 8); 833 q_vector->eims_value = BIT(msix_vector); 834 break; 835 case e1000_82580: 836 case e1000_i350: 837 case e1000_i354: 838 case e1000_i210: 839 case e1000_i211: 840 /* On 82580 and newer adapters the scheme is similar to 82576 841 * however instead of ordering column-major we have things 842 * ordered row-major. So we traverse the table by using 843 * bit 0 as the column offset, and the remaining bits as the 844 * row index. 845 */ 846 if (rx_queue > IGB_N0_QUEUE) 847 igb_write_ivar(hw, msix_vector, 848 rx_queue >> 1, 849 (rx_queue & 0x1) << 4); 850 if (tx_queue > IGB_N0_QUEUE) 851 igb_write_ivar(hw, msix_vector, 852 tx_queue >> 1, 853 ((tx_queue & 0x1) << 4) + 8); 854 q_vector->eims_value = BIT(msix_vector); 855 break; 856 default: 857 BUG(); 858 break; 859 } 860 861 /* add q_vector eims value to global eims_enable_mask */ 862 adapter->eims_enable_mask |= q_vector->eims_value; 863 864 /* configure q_vector to set itr on first interrupt */ 865 q_vector->set_itr = 1; 866} 867 868/** 869 * igb_configure_msix - Configure MSI-X hardware 870 * @adapter: board private structure to initialize 871 * 872 * igb_configure_msix sets up the hardware to properly 873 * generate MSI-X interrupts. 874 **/ 875static void igb_configure_msix(struct igb_adapter *adapter) 876{ 877 u32 tmp; 878 int i, vector = 0; 879 struct e1000_hw *hw = &adapter->hw; 880 881 adapter->eims_enable_mask = 0; 882 883 /* set vector for other causes, i.e. link changes */ 884 switch (hw->mac.type) { 885 case e1000_82575: 886 tmp = rd32(E1000_CTRL_EXT); 887 /* enable MSI-X PBA support*/ 888 tmp |= E1000_CTRL_EXT_PBA_CLR; 889 890 /* Auto-Mask interrupts upon ICR read. */ 891 tmp |= E1000_CTRL_EXT_EIAME; 892 tmp |= E1000_CTRL_EXT_IRCA; 893 894 wr32(E1000_CTRL_EXT, tmp); 895 896 /* enable msix_other interrupt */ 897 array_wr32(E1000_MSIXBM(0), vector++, E1000_EIMS_OTHER); 898 adapter->eims_other = E1000_EIMS_OTHER; 899 900 break; 901 902 case e1000_82576: 903 case e1000_82580: 904 case e1000_i350: 905 case e1000_i354: 906 case e1000_i210: 907 case e1000_i211: 908 /* Turn on MSI-X capability first, or our settings 909 * won't stick. And it will take days to debug. 910 */ 911 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE | 912 E1000_GPIE_PBA | E1000_GPIE_EIAME | 913 E1000_GPIE_NSICR); 914 915 /* enable msix_other interrupt */ 916 adapter->eims_other = BIT(vector); 917 tmp = (vector++ | E1000_IVAR_VALID) << 8; 918 919 wr32(E1000_IVAR_MISC, tmp); 920 break; 921 default: 922 /* do nothing, since nothing else supports MSI-X */ 923 break; 924 } /* switch (hw->mac.type) */ 925 926 adapter->eims_enable_mask |= adapter->eims_other; 927 928 for (i = 0; i < adapter->num_q_vectors; i++) 929 igb_assign_vector(adapter->q_vector[i], vector++); 930 931 wrfl(); 932} 933 934/** 935 * igb_request_msix - Initialize MSI-X interrupts 936 * @adapter: board private structure to initialize 937 * 938 * igb_request_msix allocates MSI-X vectors and requests interrupts from the 939 * kernel. 940 **/ 941static int igb_request_msix(struct igb_adapter *adapter) 942{ 943 struct net_device *netdev = adapter->netdev; 944 int i, err = 0, vector = 0, free_vector = 0; 945 946 err = request_irq(adapter->msix_entries[vector].vector, 947 igb_msix_other, 0, netdev->name, adapter); 948 if (err) 949 goto err_out; 950 951 for (i = 0; i < adapter->num_q_vectors; i++) { 952 struct igb_q_vector *q_vector = adapter->q_vector[i]; 953 954 vector++; 955 956 q_vector->itr_register = adapter->io_addr + E1000_EITR(vector); 957 958 if (q_vector->rx.ring && q_vector->tx.ring) 959 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name, 960 q_vector->rx.ring->queue_index); 961 else if (q_vector->tx.ring) 962 sprintf(q_vector->name, "%s-tx-%u", netdev->name, 963 q_vector->tx.ring->queue_index); 964 else if (q_vector->rx.ring) 965 sprintf(q_vector->name, "%s-rx-%u", netdev->name, 966 q_vector->rx.ring->queue_index); 967 else 968 sprintf(q_vector->name, "%s-unused", netdev->name); 969 970 err = request_irq(adapter->msix_entries[vector].vector, 971 igb_msix_ring, 0, q_vector->name, 972 q_vector); 973 if (err) 974 goto err_free; 975 } 976 977 igb_configure_msix(adapter); 978 return 0; 979 980err_free: 981 /* free already assigned IRQs */ 982 free_irq(adapter->msix_entries[free_vector++].vector, adapter); 983 984 vector--; 985 for (i = 0; i < vector; i++) { 986 free_irq(adapter->msix_entries[free_vector++].vector, 987 adapter->q_vector[i]); 988 } 989err_out: 990 return err; 991} 992 993/** 994 * igb_free_q_vector - Free memory allocated for specific interrupt vector 995 * @adapter: board private structure to initialize 996 * @v_idx: Index of vector to be freed 997 * 998 * This function frees the memory allocated to the q_vector. 999 **/ 1000static void igb_free_q_vector(struct igb_adapter *adapter, int v_idx) 1001{ 1002 struct igb_q_vector *q_vector = adapter->q_vector[v_idx]; 1003 1004 adapter->q_vector[v_idx] = NULL; 1005 1006 /* igb_get_stats64() might access the rings on this vector, 1007 * we must wait a grace period before freeing it. 1008 */ 1009 if (q_vector) 1010 kfree_rcu(q_vector, rcu); 1011} 1012 1013/** 1014 * igb_reset_q_vector - Reset config for interrupt vector 1015 * @adapter: board private structure to initialize 1016 * @v_idx: Index of vector to be reset 1017 * 1018 * If NAPI is enabled it will delete any references to the 1019 * NAPI struct. This is preparation for igb_free_q_vector. 1020 **/ 1021static void igb_reset_q_vector(struct igb_adapter *adapter, int v_idx) 1022{ 1023 struct igb_q_vector *q_vector = adapter->q_vector[v_idx]; 1024 1025 /* Coming from igb_set_interrupt_capability, the vectors are not yet 1026 * allocated. So, q_vector is NULL so we should stop here. 1027 */ 1028 if (!q_vector) 1029 return; 1030 1031 if (q_vector->tx.ring) 1032 adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL; 1033 1034 if (q_vector->rx.ring) 1035 adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL; 1036 1037 netif_napi_del(&q_vector->napi); 1038 1039} 1040 1041static void igb_reset_interrupt_capability(struct igb_adapter *adapter) 1042{ 1043 int v_idx = adapter->num_q_vectors; 1044 1045 if (adapter->flags & IGB_FLAG_HAS_MSIX) 1046 pci_disable_msix(adapter->pdev); 1047 else if (adapter->flags & IGB_FLAG_HAS_MSI) 1048 pci_disable_msi(adapter->pdev); 1049 1050 while (v_idx--) 1051 igb_reset_q_vector(adapter, v_idx); 1052} 1053 1054/** 1055 * igb_free_q_vectors - Free memory allocated for interrupt vectors 1056 * @adapter: board private structure to initialize 1057 * 1058 * This function frees the memory allocated to the q_vectors. In addition if 1059 * NAPI is enabled it will delete any references to the NAPI struct prior 1060 * to freeing the q_vector. 1061 **/ 1062static void igb_free_q_vectors(struct igb_adapter *adapter) 1063{ 1064 int v_idx = adapter->num_q_vectors; 1065 1066 adapter->num_tx_queues = 0; 1067 adapter->num_rx_queues = 0; 1068 adapter->num_q_vectors = 0; 1069 1070 while (v_idx--) { 1071 igb_reset_q_vector(adapter, v_idx); 1072 igb_free_q_vector(adapter, v_idx); 1073 } 1074} 1075 1076/** 1077 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts 1078 * @adapter: board private structure to initialize 1079 * 1080 * This function resets the device so that it has 0 Rx queues, Tx queues, and 1081 * MSI-X interrupts allocated. 1082 */ 1083static void igb_clear_interrupt_scheme(struct igb_adapter *adapter) 1084{ 1085 igb_free_q_vectors(adapter); 1086 igb_reset_interrupt_capability(adapter); 1087} 1088 1089/** 1090 * igb_set_interrupt_capability - set MSI or MSI-X if supported 1091 * @adapter: board private structure to initialize 1092 * @msix: boolean value of MSIX capability 1093 * 1094 * Attempt to configure interrupts using the best available 1095 * capabilities of the hardware and kernel. 1096 **/ 1097static void igb_set_interrupt_capability(struct igb_adapter *adapter, bool msix) 1098{ 1099 int err; 1100 int numvecs, i; 1101 1102 if (!msix) 1103 goto msi_only; 1104 adapter->flags |= IGB_FLAG_HAS_MSIX; 1105 1106 /* Number of supported queues. */ 1107 adapter->num_rx_queues = adapter->rss_queues; 1108 if (adapter->vfs_allocated_count) 1109 adapter->num_tx_queues = 1; 1110 else 1111 adapter->num_tx_queues = adapter->rss_queues; 1112 1113 /* start with one vector for every Rx queue */ 1114 numvecs = adapter->num_rx_queues; 1115 1116 /* if Tx handler is separate add 1 for every Tx queue */ 1117 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS)) 1118 numvecs += adapter->num_tx_queues; 1119 1120 /* store the number of vectors reserved for queues */ 1121 adapter->num_q_vectors = numvecs; 1122 1123 /* add 1 vector for link status interrupts */ 1124 numvecs++; 1125 for (i = 0; i < numvecs; i++) 1126 adapter->msix_entries[i].entry = i; 1127 1128 err = pci_enable_msix_range(adapter->pdev, 1129 adapter->msix_entries, 1130 numvecs, 1131 numvecs); 1132 if (err > 0) 1133 return; 1134 1135 igb_reset_interrupt_capability(adapter); 1136 1137 /* If we can't do MSI-X, try MSI */ 1138msi_only: 1139 adapter->flags &= ~IGB_FLAG_HAS_MSIX; 1140#ifdef CONFIG_PCI_IOV 1141 /* disable SR-IOV for non MSI-X configurations */ 1142 if (adapter->vf_data) { 1143 struct e1000_hw *hw = &adapter->hw; 1144 /* disable iov and allow time for transactions to clear */ 1145 pci_disable_sriov(adapter->pdev); 1146 msleep(500); 1147 1148 kfree(adapter->vf_mac_list); 1149 adapter->vf_mac_list = NULL; 1150 kfree(adapter->vf_data); 1151 adapter->vf_data = NULL; 1152 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ); 1153 wrfl(); 1154 msleep(100); 1155 dev_info(&adapter->pdev->dev, "IOV Disabled\n"); 1156 } 1157#endif 1158 adapter->vfs_allocated_count = 0; 1159 adapter->rss_queues = 1; 1160 adapter->flags |= IGB_FLAG_QUEUE_PAIRS; 1161 adapter->num_rx_queues = 1; 1162 adapter->num_tx_queues = 1; 1163 adapter->num_q_vectors = 1; 1164 if (!pci_enable_msi(adapter->pdev)) 1165 adapter->flags |= IGB_FLAG_HAS_MSI; 1166} 1167 1168static void igb_add_ring(struct igb_ring *ring, 1169 struct igb_ring_container *head) 1170{ 1171 head->ring = ring; 1172 head->count++; 1173} 1174 1175/** 1176 * igb_alloc_q_vector - Allocate memory for a single interrupt vector 1177 * @adapter: board private structure to initialize 1178 * @v_count: q_vectors allocated on adapter, used for ring interleaving 1179 * @v_idx: index of vector in adapter struct 1180 * @txr_count: total number of Tx rings to allocate 1181 * @txr_idx: index of first Tx ring to allocate 1182 * @rxr_count: total number of Rx rings to allocate 1183 * @rxr_idx: index of first Rx ring to allocate 1184 * 1185 * We allocate one q_vector. If allocation fails we return -ENOMEM. 1186 **/ 1187static int igb_alloc_q_vector(struct igb_adapter *adapter, 1188 int v_count, int v_idx, 1189 int txr_count, int txr_idx, 1190 int rxr_count, int rxr_idx) 1191{ 1192 struct igb_q_vector *q_vector; 1193 struct igb_ring *ring; 1194 int ring_count; 1195 size_t size; 1196 1197 /* igb only supports 1 Tx and/or 1 Rx queue per vector */ 1198 if (txr_count > 1 || rxr_count > 1) 1199 return -ENOMEM; 1200 1201 ring_count = txr_count + rxr_count; 1202 size = struct_size(q_vector, ring, ring_count); 1203 1204 /* allocate q_vector and rings */ 1205 q_vector = adapter->q_vector[v_idx]; 1206 if (!q_vector) { 1207 q_vector = kzalloc(size, GFP_KERNEL); 1208 } else if (size > ksize(q_vector)) { 1209 kfree_rcu(q_vector, rcu); 1210 q_vector = kzalloc(size, GFP_KERNEL); 1211 } else { 1212 memset(q_vector, 0, size); 1213 } 1214 if (!q_vector) 1215 return -ENOMEM; 1216 1217 /* initialize NAPI */ 1218 netif_napi_add(adapter->netdev, &q_vector->napi, 1219 igb_poll, 64); 1220 1221 /* tie q_vector and adapter together */ 1222 adapter->q_vector[v_idx] = q_vector; 1223 q_vector->adapter = adapter; 1224 1225 /* initialize work limits */ 1226 q_vector->tx.work_limit = adapter->tx_work_limit; 1227 1228 /* initialize ITR configuration */ 1229 q_vector->itr_register = adapter->io_addr + E1000_EITR(0); 1230 q_vector->itr_val = IGB_START_ITR; 1231 1232 /* initialize pointer to rings */ 1233 ring = q_vector->ring; 1234 1235 /* intialize ITR */ 1236 if (rxr_count) { 1237 /* rx or rx/tx vector */ 1238 if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3) 1239 q_vector->itr_val = adapter->rx_itr_setting; 1240 } else { 1241 /* tx only vector */ 1242 if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3) 1243 q_vector->itr_val = adapter->tx_itr_setting; 1244 } 1245 1246 if (txr_count) { 1247 /* assign generic ring traits */ 1248 ring->dev = &adapter->pdev->dev; 1249 ring->netdev = adapter->netdev; 1250 1251 /* configure backlink on ring */ 1252 ring->q_vector = q_vector; 1253 1254 /* update q_vector Tx values */ 1255 igb_add_ring(ring, &q_vector->tx); 1256 1257 /* For 82575, context index must be unique per ring. */ 1258 if (adapter->hw.mac.type == e1000_82575) 1259 set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags); 1260 1261 /* apply Tx specific ring traits */ 1262 ring->count = adapter->tx_ring_count; 1263 ring->queue_index = txr_idx; 1264 1265 ring->cbs_enable = false; 1266 ring->idleslope = 0; 1267 ring->sendslope = 0; 1268 ring->hicredit = 0; 1269 ring->locredit = 0; 1270 1271 u64_stats_init(&ring->tx_syncp); 1272 u64_stats_init(&ring->tx_syncp2); 1273 1274 /* assign ring to adapter */ 1275 adapter->tx_ring[txr_idx] = ring; 1276 1277 /* push pointer to next ring */ 1278 ring++; 1279 } 1280 1281 if (rxr_count) { 1282 /* assign generic ring traits */ 1283 ring->dev = &adapter->pdev->dev; 1284 ring->netdev = adapter->netdev; 1285 1286 /* configure backlink on ring */ 1287 ring->q_vector = q_vector; 1288 1289 /* update q_vector Rx values */ 1290 igb_add_ring(ring, &q_vector->rx); 1291 1292 /* set flag indicating ring supports SCTP checksum offload */ 1293 if (adapter->hw.mac.type >= e1000_82576) 1294 set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags); 1295 1296 /* On i350, i354, i210, and i211, loopback VLAN packets 1297 * have the tag byte-swapped. 1298 */ 1299 if (adapter->hw.mac.type >= e1000_i350) 1300 set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags); 1301 1302 /* apply Rx specific ring traits */ 1303 ring->count = adapter->rx_ring_count; 1304 ring->queue_index = rxr_idx; 1305 1306 u64_stats_init(&ring->rx_syncp); 1307 1308 /* assign ring to adapter */ 1309 adapter->rx_ring[rxr_idx] = ring; 1310 } 1311 1312 return 0; 1313} 1314 1315 1316/** 1317 * igb_alloc_q_vectors - Allocate memory for interrupt vectors 1318 * @adapter: board private structure to initialize 1319 * 1320 * We allocate one q_vector per queue interrupt. If allocation fails we 1321 * return -ENOMEM. 1322 **/ 1323static int igb_alloc_q_vectors(struct igb_adapter *adapter) 1324{ 1325 int q_vectors = adapter->num_q_vectors; 1326 int rxr_remaining = adapter->num_rx_queues; 1327 int txr_remaining = adapter->num_tx_queues; 1328 int rxr_idx = 0, txr_idx = 0, v_idx = 0; 1329 int err; 1330 1331 if (q_vectors >= (rxr_remaining + txr_remaining)) { 1332 for (; rxr_remaining; v_idx++) { 1333 err = igb_alloc_q_vector(adapter, q_vectors, v_idx, 1334 0, 0, 1, rxr_idx); 1335 1336 if (err) 1337 goto err_out; 1338 1339 /* update counts and index */ 1340 rxr_remaining--; 1341 rxr_idx++; 1342 } 1343 } 1344 1345 for (; v_idx < q_vectors; v_idx++) { 1346 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx); 1347 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx); 1348 1349 err = igb_alloc_q_vector(adapter, q_vectors, v_idx, 1350 tqpv, txr_idx, rqpv, rxr_idx); 1351 1352 if (err) 1353 goto err_out; 1354 1355 /* update counts and index */ 1356 rxr_remaining -= rqpv; 1357 txr_remaining -= tqpv; 1358 rxr_idx++; 1359 txr_idx++; 1360 } 1361 1362 return 0; 1363 1364err_out: 1365 adapter->num_tx_queues = 0; 1366 adapter->num_rx_queues = 0; 1367 adapter->num_q_vectors = 0; 1368 1369 while (v_idx--) 1370 igb_free_q_vector(adapter, v_idx); 1371 1372 return -ENOMEM; 1373} 1374 1375/** 1376 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors 1377 * @adapter: board private structure to initialize 1378 * @msix: boolean value of MSIX capability 1379 * 1380 * This function initializes the interrupts and allocates all of the queues. 1381 **/ 1382static int igb_init_interrupt_scheme(struct igb_adapter *adapter, bool msix) 1383{ 1384 struct pci_dev *pdev = adapter->pdev; 1385 int err; 1386 1387 igb_set_interrupt_capability(adapter, msix); 1388 1389 err = igb_alloc_q_vectors(adapter); 1390 if (err) { 1391 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n"); 1392 goto err_alloc_q_vectors; 1393 } 1394 1395 igb_cache_ring_register(adapter); 1396 1397 return 0; 1398 1399err_alloc_q_vectors: 1400 igb_reset_interrupt_capability(adapter); 1401 return err; 1402} 1403 1404/** 1405 * igb_request_irq - initialize interrupts 1406 * @adapter: board private structure to initialize 1407 * 1408 * Attempts to configure interrupts using the best available 1409 * capabilities of the hardware and kernel. 1410 **/ 1411static int igb_request_irq(struct igb_adapter *adapter) 1412{ 1413 struct net_device *netdev = adapter->netdev; 1414 struct pci_dev *pdev = adapter->pdev; 1415 int err = 0; 1416 1417 if (adapter->flags & IGB_FLAG_HAS_MSIX) { 1418 err = igb_request_msix(adapter); 1419 if (!err) 1420 goto request_done; 1421 /* fall back to MSI */ 1422 igb_free_all_tx_resources(adapter); 1423 igb_free_all_rx_resources(adapter); 1424 1425 igb_clear_interrupt_scheme(adapter); 1426 err = igb_init_interrupt_scheme(adapter, false); 1427 if (err) 1428 goto request_done; 1429 1430 igb_setup_all_tx_resources(adapter); 1431 igb_setup_all_rx_resources(adapter); 1432 igb_configure(adapter); 1433 } 1434 1435 igb_assign_vector(adapter->q_vector[0], 0); 1436 1437 if (adapter->flags & IGB_FLAG_HAS_MSI) { 1438 err = request_irq(pdev->irq, igb_intr_msi, 0, 1439 netdev->name, adapter); 1440 if (!err) 1441 goto request_done; 1442 1443 /* fall back to legacy interrupts */ 1444 igb_reset_interrupt_capability(adapter); 1445 adapter->flags &= ~IGB_FLAG_HAS_MSI; 1446 } 1447 1448 err = request_irq(pdev->irq, igb_intr, IRQF_SHARED, 1449 netdev->name, adapter); 1450 1451 if (err) 1452 dev_err(&pdev->dev, "Error %d getting interrupt\n", 1453 err); 1454 1455request_done: 1456 return err; 1457} 1458 1459static void igb_free_irq(struct igb_adapter *adapter) 1460{ 1461 if (adapter->flags & IGB_FLAG_HAS_MSIX) { 1462 int vector = 0, i; 1463 1464 free_irq(adapter->msix_entries[vector++].vector, adapter); 1465 1466 for (i = 0; i < adapter->num_q_vectors; i++) 1467 free_irq(adapter->msix_entries[vector++].vector, 1468 adapter->q_vector[i]); 1469 } else { 1470 free_irq(adapter->pdev->irq, adapter); 1471 } 1472} 1473 1474/** 1475 * igb_irq_disable - Mask off interrupt generation on the NIC 1476 * @adapter: board private structure 1477 **/ 1478static void igb_irq_disable(struct igb_adapter *adapter) 1479{ 1480 struct e1000_hw *hw = &adapter->hw; 1481 1482 /* we need to be careful when disabling interrupts. The VFs are also 1483 * mapped into these registers and so clearing the bits can cause 1484 * issues on the VF drivers so we only need to clear what we set 1485 */ 1486 if (adapter->flags & IGB_FLAG_HAS_MSIX) { 1487 u32 regval = rd32(E1000_EIAM); 1488 1489 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask); 1490 wr32(E1000_EIMC, adapter->eims_enable_mask); 1491 regval = rd32(E1000_EIAC); 1492 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask); 1493 } 1494 1495 wr32(E1000_IAM, 0); 1496 wr32(E1000_IMC, ~0); 1497 wrfl(); 1498 if (adapter->flags & IGB_FLAG_HAS_MSIX) { 1499 int i; 1500 1501 for (i = 0; i < adapter->num_q_vectors; i++) 1502 synchronize_irq(adapter->msix_entries[i].vector); 1503 } else { 1504 synchronize_irq(adapter->pdev->irq); 1505 } 1506} 1507 1508/** 1509 * igb_irq_enable - Enable default interrupt generation settings 1510 * @adapter: board private structure 1511 **/ 1512static void igb_irq_enable(struct igb_adapter *adapter) 1513{ 1514 struct e1000_hw *hw = &adapter->hw; 1515 1516 if (adapter->flags & IGB_FLAG_HAS_MSIX) { 1517 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA; 1518 u32 regval = rd32(E1000_EIAC); 1519 1520 wr32(E1000_EIAC, regval | adapter->eims_enable_mask); 1521 regval = rd32(E1000_EIAM); 1522 wr32(E1000_EIAM, regval | adapter->eims_enable_mask); 1523 wr32(E1000_EIMS, adapter->eims_enable_mask); 1524 if (adapter->vfs_allocated_count) { 1525 wr32(E1000_MBVFIMR, 0xFF); 1526 ims |= E1000_IMS_VMMB; 1527 } 1528 wr32(E1000_IMS, ims); 1529 } else { 1530 wr32(E1000_IMS, IMS_ENABLE_MASK | 1531 E1000_IMS_DRSTA); 1532 wr32(E1000_IAM, IMS_ENABLE_MASK | 1533 E1000_IMS_DRSTA); 1534 } 1535} 1536 1537static void igb_update_mng_vlan(struct igb_adapter *adapter) 1538{ 1539 struct e1000_hw *hw = &adapter->hw; 1540 u16 pf_id = adapter->vfs_allocated_count; 1541 u16 vid = adapter->hw.mng_cookie.vlan_id; 1542 u16 old_vid = adapter->mng_vlan_id; 1543 1544 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) { 1545 /* add VID to filter table */ 1546 igb_vfta_set(hw, vid, pf_id, true, true); 1547 adapter->mng_vlan_id = vid; 1548 } else { 1549 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE; 1550 } 1551 1552 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) && 1553 (vid != old_vid) && 1554 !test_bit(old_vid, adapter->active_vlans)) { 1555 /* remove VID from filter table */ 1556 igb_vfta_set(hw, vid, pf_id, false, true); 1557 } 1558} 1559 1560/** 1561 * igb_release_hw_control - release control of the h/w to f/w 1562 * @adapter: address of board private structure 1563 * 1564 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit. 1565 * For ASF and Pass Through versions of f/w this means that the 1566 * driver is no longer loaded. 1567 **/ 1568static void igb_release_hw_control(struct igb_adapter *adapter) 1569{ 1570 struct e1000_hw *hw = &adapter->hw; 1571 u32 ctrl_ext; 1572 1573 /* Let firmware take over control of h/w */ 1574 ctrl_ext = rd32(E1000_CTRL_EXT); 1575 wr32(E1000_CTRL_EXT, 1576 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 1577} 1578 1579/** 1580 * igb_get_hw_control - get control of the h/w from f/w 1581 * @adapter: address of board private structure 1582 * 1583 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit. 1584 * For ASF and Pass Through versions of f/w this means that 1585 * the driver is loaded. 1586 **/ 1587static void igb_get_hw_control(struct igb_adapter *adapter) 1588{ 1589 struct e1000_hw *hw = &adapter->hw; 1590 u32 ctrl_ext; 1591 1592 /* Let firmware know the driver has taken over */ 1593 ctrl_ext = rd32(E1000_CTRL_EXT); 1594 wr32(E1000_CTRL_EXT, 1595 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 1596} 1597 1598static void enable_fqtss(struct igb_adapter *adapter, bool enable) 1599{ 1600 struct net_device *netdev = adapter->netdev; 1601 struct e1000_hw *hw = &adapter->hw; 1602 1603 WARN_ON(hw->mac.type != e1000_i210); 1604 1605 if (enable) 1606 adapter->flags |= IGB_FLAG_FQTSS; 1607 else 1608 adapter->flags &= ~IGB_FLAG_FQTSS; 1609 1610 if (netif_running(netdev)) 1611 schedule_work(&adapter->reset_task); 1612} 1613 1614static bool is_fqtss_enabled(struct igb_adapter *adapter) 1615{ 1616 return (adapter->flags & IGB_FLAG_FQTSS) ? true : false; 1617} 1618 1619static void set_tx_desc_fetch_prio(struct e1000_hw *hw, int queue, 1620 enum tx_queue_prio prio) 1621{ 1622 u32 val; 1623 1624 WARN_ON(hw->mac.type != e1000_i210); 1625 WARN_ON(queue < 0 || queue > 4); 1626 1627 val = rd32(E1000_I210_TXDCTL(queue)); 1628 1629 if (prio == TX_QUEUE_PRIO_HIGH) 1630 val |= E1000_TXDCTL_PRIORITY; 1631 else 1632 val &= ~E1000_TXDCTL_PRIORITY; 1633 1634 wr32(E1000_I210_TXDCTL(queue), val); 1635} 1636 1637static void set_queue_mode(struct e1000_hw *hw, int queue, enum queue_mode mode) 1638{ 1639 u32 val; 1640 1641 WARN_ON(hw->mac.type != e1000_i210); 1642 WARN_ON(queue < 0 || queue > 1); 1643 1644 val = rd32(E1000_I210_TQAVCC(queue)); 1645 1646 if (mode == QUEUE_MODE_STREAM_RESERVATION) 1647 val |= E1000_TQAVCC_QUEUEMODE; 1648 else 1649 val &= ~E1000_TQAVCC_QUEUEMODE; 1650 1651 wr32(E1000_I210_TQAVCC(queue), val); 1652} 1653 1654static bool is_any_cbs_enabled(struct igb_adapter *adapter) 1655{ 1656 int i; 1657 1658 for (i = 0; i < adapter->num_tx_queues; i++) { 1659 if (adapter->tx_ring[i]->cbs_enable) 1660 return true; 1661 } 1662 1663 return false; 1664} 1665 1666static bool is_any_txtime_enabled(struct igb_adapter *adapter) 1667{ 1668 int i; 1669 1670 for (i = 0; i < adapter->num_tx_queues; i++) { 1671 if (adapter->tx_ring[i]->launchtime_enable) 1672 return true; 1673 } 1674 1675 return false; 1676} 1677 1678/** 1679 * igb_config_tx_modes - Configure "Qav Tx mode" features on igb 1680 * @adapter: pointer to adapter struct 1681 * @queue: queue number 1682 * 1683 * Configure CBS and Launchtime for a given hardware queue. 1684 * Parameters are retrieved from the correct Tx ring, so 1685 * igb_save_cbs_params() and igb_save_txtime_params() should be used 1686 * for setting those correctly prior to this function being called. 1687 **/ 1688static void igb_config_tx_modes(struct igb_adapter *adapter, int queue) 1689{ 1690 struct igb_ring *ring = adapter->tx_ring[queue]; 1691 struct net_device *netdev = adapter->netdev; 1692 struct e1000_hw *hw = &adapter->hw; 1693 u32 tqavcc, tqavctrl; 1694 u16 value; 1695 1696 WARN_ON(hw->mac.type != e1000_i210); 1697 WARN_ON(queue < 0 || queue > 1); 1698 1699 /* If any of the Qav features is enabled, configure queues as SR and 1700 * with HIGH PRIO. If none is, then configure them with LOW PRIO and 1701 * as SP. 1702 */ 1703 if (ring->cbs_enable || ring->launchtime_enable) { 1704 set_tx_desc_fetch_prio(hw, queue, TX_QUEUE_PRIO_HIGH); 1705 set_queue_mode(hw, queue, QUEUE_MODE_STREAM_RESERVATION); 1706 } else { 1707 set_tx_desc_fetch_prio(hw, queue, TX_QUEUE_PRIO_LOW); 1708 set_queue_mode(hw, queue, QUEUE_MODE_STRICT_PRIORITY); 1709 } 1710 1711 /* If CBS is enabled, set DataTranARB and config its parameters. */ 1712 if (ring->cbs_enable || queue == 0) { 1713 /* i210 does not allow the queue 0 to be in the Strict 1714 * Priority mode while the Qav mode is enabled, so, 1715 * instead of disabling strict priority mode, we give 1716 * queue 0 the maximum of credits possible. 1717 * 1718 * See section 8.12.19 of the i210 datasheet, "Note: 1719 * Queue0 QueueMode must be set to 1b when 1720 * TransmitMode is set to Qav." 1721 */ 1722 if (queue == 0 && !ring->cbs_enable) { 1723 /* max "linkspeed" idleslope in kbps */ 1724 ring->idleslope…
Large files files are truncated, but you can click here to view the full file