PageRenderTime 85ms CodeModel.GetById 11ms RepoModel.GetById 0ms app.codeStats 1ms

/drivers/net/ethernet/intel/igb/igb_main.c

http://github.com/mirrors/linux
C | 9603 lines | 6359 code | 1551 blank | 1693 comment | 1075 complexity | 087bb451361a649de260261d03bce9ee MD5 | raw file
Possible License(s): AGPL-1.0, GPL-2.0, LGPL-2.0
  1. // SPDX-License-Identifier: GPL-2.0
  2. /* Copyright(c) 2007 - 2018 Intel Corporation. */
  3. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  4. #include <linux/module.h>
  5. #include <linux/types.h>
  6. #include <linux/init.h>
  7. #include <linux/bitops.h>
  8. #include <linux/vmalloc.h>
  9. #include <linux/pagemap.h>
  10. #include <linux/netdevice.h>
  11. #include <linux/ipv6.h>
  12. #include <linux/slab.h>
  13. #include <net/checksum.h>
  14. #include <net/ip6_checksum.h>
  15. #include <net/pkt_sched.h>
  16. #include <net/pkt_cls.h>
  17. #include <linux/net_tstamp.h>
  18. #include <linux/mii.h>
  19. #include <linux/ethtool.h>
  20. #include <linux/if.h>
  21. #include <linux/if_vlan.h>
  22. #include <linux/pci.h>
  23. #include <linux/delay.h>
  24. #include <linux/interrupt.h>
  25. #include <linux/ip.h>
  26. #include <linux/tcp.h>
  27. #include <linux/sctp.h>
  28. #include <linux/if_ether.h>
  29. #include <linux/aer.h>
  30. #include <linux/prefetch.h>
  31. #include <linux/pm_runtime.h>
  32. #include <linux/etherdevice.h>
  33. #ifdef CONFIG_IGB_DCA
  34. #include <linux/dca.h>
  35. #endif
  36. #include <linux/i2c.h>
  37. #include "igb.h"
  38. #define MAJ 5
  39. #define MIN 6
  40. #define BUILD 0
  41. #define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
  42. __stringify(BUILD) "-k"
  43. enum queue_mode {
  44. QUEUE_MODE_STRICT_PRIORITY,
  45. QUEUE_MODE_STREAM_RESERVATION,
  46. };
  47. enum tx_queue_prio {
  48. TX_QUEUE_PRIO_HIGH,
  49. TX_QUEUE_PRIO_LOW,
  50. };
  51. char igb_driver_name[] = "igb";
  52. char igb_driver_version[] = DRV_VERSION;
  53. static const char igb_driver_string[] =
  54. "Intel(R) Gigabit Ethernet Network Driver";
  55. static const char igb_copyright[] =
  56. "Copyright (c) 2007-2014 Intel Corporation.";
  57. static const struct e1000_info *igb_info_tbl[] = {
  58. [board_82575] = &e1000_82575_info,
  59. };
  60. static const struct pci_device_id igb_pci_tbl[] = {
  61. { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_1GBPS) },
  62. { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_SGMII) },
  63. { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) },
  64. { PCI_VDEVICE(INTEL, E1000_DEV_ID_I211_COPPER), board_82575 },
  65. { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER), board_82575 },
  66. { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_FIBER), board_82575 },
  67. { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES), board_82575 },
  68. { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SGMII), board_82575 },
  69. { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER_FLASHLESS), board_82575 },
  70. { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES_FLASHLESS), board_82575 },
  71. { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
  72. { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
  73. { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
  74. { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
  75. { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
  76. { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
  77. { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 },
  78. { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
  79. { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
  80. { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
  81. { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
  82. { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
  83. { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 },
  84. { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 },
  85. { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
  86. { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
  87. { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
  88. { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
  89. { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
  90. { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
  91. { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
  92. { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
  93. { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
  94. { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
  95. { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
  96. /* required last entry */
  97. {0, }
  98. };
  99. MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
  100. static int igb_setup_all_tx_resources(struct igb_adapter *);
  101. static int igb_setup_all_rx_resources(struct igb_adapter *);
  102. static void igb_free_all_tx_resources(struct igb_adapter *);
  103. static void igb_free_all_rx_resources(struct igb_adapter *);
  104. static void igb_setup_mrqc(struct igb_adapter *);
  105. static int igb_probe(struct pci_dev *, const struct pci_device_id *);
  106. static void igb_remove(struct pci_dev *pdev);
  107. static int igb_sw_init(struct igb_adapter *);
  108. int igb_open(struct net_device *);
  109. int igb_close(struct net_device *);
  110. static void igb_configure(struct igb_adapter *);
  111. static void igb_configure_tx(struct igb_adapter *);
  112. static void igb_configure_rx(struct igb_adapter *);
  113. static void igb_clean_all_tx_rings(struct igb_adapter *);
  114. static void igb_clean_all_rx_rings(struct igb_adapter *);
  115. static void igb_clean_tx_ring(struct igb_ring *);
  116. static void igb_clean_rx_ring(struct igb_ring *);
  117. static void igb_set_rx_mode(struct net_device *);
  118. static void igb_update_phy_info(struct timer_list *);
  119. static void igb_watchdog(struct timer_list *);
  120. static void igb_watchdog_task(struct work_struct *);
  121. static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *);
  122. static void igb_get_stats64(struct net_device *dev,
  123. struct rtnl_link_stats64 *stats);
  124. static int igb_change_mtu(struct net_device *, int);
  125. static int igb_set_mac(struct net_device *, void *);
  126. static void igb_set_uta(struct igb_adapter *adapter, bool set);
  127. static irqreturn_t igb_intr(int irq, void *);
  128. static irqreturn_t igb_intr_msi(int irq, void *);
  129. static irqreturn_t igb_msix_other(int irq, void *);
  130. static irqreturn_t igb_msix_ring(int irq, void *);
  131. #ifdef CONFIG_IGB_DCA
  132. static void igb_update_dca(struct igb_q_vector *);
  133. static void igb_setup_dca(struct igb_adapter *);
  134. #endif /* CONFIG_IGB_DCA */
  135. static int igb_poll(struct napi_struct *, int);
  136. static bool igb_clean_tx_irq(struct igb_q_vector *, int);
  137. static int igb_clean_rx_irq(struct igb_q_vector *, int);
  138. static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
  139. static void igb_tx_timeout(struct net_device *, unsigned int txqueue);
  140. static void igb_reset_task(struct work_struct *);
  141. static void igb_vlan_mode(struct net_device *netdev,
  142. netdev_features_t features);
  143. static int igb_vlan_rx_add_vid(struct net_device *, __be16, u16);
  144. static int igb_vlan_rx_kill_vid(struct net_device *, __be16, u16);
  145. static void igb_restore_vlan(struct igb_adapter *);
  146. static void igb_rar_set_index(struct igb_adapter *, u32);
  147. static void igb_ping_all_vfs(struct igb_adapter *);
  148. static void igb_msg_task(struct igb_adapter *);
  149. static void igb_vmm_control(struct igb_adapter *);
  150. static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
  151. static void igb_flush_mac_table(struct igb_adapter *);
  152. static int igb_available_rars(struct igb_adapter *, u8);
  153. static void igb_set_default_mac_filter(struct igb_adapter *);
  154. static int igb_uc_sync(struct net_device *, const unsigned char *);
  155. static int igb_uc_unsync(struct net_device *, const unsigned char *);
  156. static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
  157. static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
  158. static int igb_ndo_set_vf_vlan(struct net_device *netdev,
  159. int vf, u16 vlan, u8 qos, __be16 vlan_proto);
  160. static int igb_ndo_set_vf_bw(struct net_device *, int, int, int);
  161. static int igb_ndo_set_vf_spoofchk(struct net_device *netdev, int vf,
  162. bool setting);
  163. static int igb_ndo_set_vf_trust(struct net_device *netdev, int vf,
  164. bool setting);
  165. static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
  166. struct ifla_vf_info *ivi);
  167. static void igb_check_vf_rate_limit(struct igb_adapter *);
  168. static void igb_nfc_filter_exit(struct igb_adapter *adapter);
  169. static void igb_nfc_filter_restore(struct igb_adapter *adapter);
  170. #ifdef CONFIG_PCI_IOV
  171. static int igb_vf_configure(struct igb_adapter *adapter, int vf);
  172. static int igb_pci_enable_sriov(struct pci_dev *dev, int num_vfs);
  173. static int igb_disable_sriov(struct pci_dev *dev);
  174. static int igb_pci_disable_sriov(struct pci_dev *dev);
  175. #endif
  176. static int igb_suspend(struct device *);
  177. static int igb_resume(struct device *);
  178. static int igb_runtime_suspend(struct device *dev);
  179. static int igb_runtime_resume(struct device *dev);
  180. static int igb_runtime_idle(struct device *dev);
  181. static const struct dev_pm_ops igb_pm_ops = {
  182. SET_SYSTEM_SLEEP_PM_OPS(igb_suspend, igb_resume)
  183. SET_RUNTIME_PM_OPS(igb_runtime_suspend, igb_runtime_resume,
  184. igb_runtime_idle)
  185. };
  186. static void igb_shutdown(struct pci_dev *);
  187. static int igb_pci_sriov_configure(struct pci_dev *dev, int num_vfs);
  188. #ifdef CONFIG_IGB_DCA
  189. static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
  190. static struct notifier_block dca_notifier = {
  191. .notifier_call = igb_notify_dca,
  192. .next = NULL,
  193. .priority = 0
  194. };
  195. #endif
  196. #ifdef CONFIG_PCI_IOV
  197. static unsigned int max_vfs;
  198. module_param(max_vfs, uint, 0);
  199. MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate per physical function");
  200. #endif /* CONFIG_PCI_IOV */
  201. static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
  202. pci_channel_state_t);
  203. static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
  204. static void igb_io_resume(struct pci_dev *);
  205. static const struct pci_error_handlers igb_err_handler = {
  206. .error_detected = igb_io_error_detected,
  207. .slot_reset = igb_io_slot_reset,
  208. .resume = igb_io_resume,
  209. };
  210. static void igb_init_dmac(struct igb_adapter *adapter, u32 pba);
  211. static struct pci_driver igb_driver = {
  212. .name = igb_driver_name,
  213. .id_table = igb_pci_tbl,
  214. .probe = igb_probe,
  215. .remove = igb_remove,
  216. #ifdef CONFIG_PM
  217. .driver.pm = &igb_pm_ops,
  218. #endif
  219. .shutdown = igb_shutdown,
  220. .sriov_configure = igb_pci_sriov_configure,
  221. .err_handler = &igb_err_handler
  222. };
  223. MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
  224. MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
  225. MODULE_LICENSE("GPL v2");
  226. MODULE_VERSION(DRV_VERSION);
  227. #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
  228. static int debug = -1;
  229. module_param(debug, int, 0);
  230. MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  231. struct igb_reg_info {
  232. u32 ofs;
  233. char *name;
  234. };
  235. static const struct igb_reg_info igb_reg_info_tbl[] = {
  236. /* General Registers */
  237. {E1000_CTRL, "CTRL"},
  238. {E1000_STATUS, "STATUS"},
  239. {E1000_CTRL_EXT, "CTRL_EXT"},
  240. /* Interrupt Registers */
  241. {E1000_ICR, "ICR"},
  242. /* RX Registers */
  243. {E1000_RCTL, "RCTL"},
  244. {E1000_RDLEN(0), "RDLEN"},
  245. {E1000_RDH(0), "RDH"},
  246. {E1000_RDT(0), "RDT"},
  247. {E1000_RXDCTL(0), "RXDCTL"},
  248. {E1000_RDBAL(0), "RDBAL"},
  249. {E1000_RDBAH(0), "RDBAH"},
  250. /* TX Registers */
  251. {E1000_TCTL, "TCTL"},
  252. {E1000_TDBAL(0), "TDBAL"},
  253. {E1000_TDBAH(0), "TDBAH"},
  254. {E1000_TDLEN(0), "TDLEN"},
  255. {E1000_TDH(0), "TDH"},
  256. {E1000_TDT(0), "TDT"},
  257. {E1000_TXDCTL(0), "TXDCTL"},
  258. {E1000_TDFH, "TDFH"},
  259. {E1000_TDFT, "TDFT"},
  260. {E1000_TDFHS, "TDFHS"},
  261. {E1000_TDFPC, "TDFPC"},
  262. /* List Terminator */
  263. {}
  264. };
  265. /* igb_regdump - register printout routine */
  266. static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
  267. {
  268. int n = 0;
  269. char rname[16];
  270. u32 regs[8];
  271. switch (reginfo->ofs) {
  272. case E1000_RDLEN(0):
  273. for (n = 0; n < 4; n++)
  274. regs[n] = rd32(E1000_RDLEN(n));
  275. break;
  276. case E1000_RDH(0):
  277. for (n = 0; n < 4; n++)
  278. regs[n] = rd32(E1000_RDH(n));
  279. break;
  280. case E1000_RDT(0):
  281. for (n = 0; n < 4; n++)
  282. regs[n] = rd32(E1000_RDT(n));
  283. break;
  284. case E1000_RXDCTL(0):
  285. for (n = 0; n < 4; n++)
  286. regs[n] = rd32(E1000_RXDCTL(n));
  287. break;
  288. case E1000_RDBAL(0):
  289. for (n = 0; n < 4; n++)
  290. regs[n] = rd32(E1000_RDBAL(n));
  291. break;
  292. case E1000_RDBAH(0):
  293. for (n = 0; n < 4; n++)
  294. regs[n] = rd32(E1000_RDBAH(n));
  295. break;
  296. case E1000_TDBAL(0):
  297. for (n = 0; n < 4; n++)
  298. regs[n] = rd32(E1000_RDBAL(n));
  299. break;
  300. case E1000_TDBAH(0):
  301. for (n = 0; n < 4; n++)
  302. regs[n] = rd32(E1000_TDBAH(n));
  303. break;
  304. case E1000_TDLEN(0):
  305. for (n = 0; n < 4; n++)
  306. regs[n] = rd32(E1000_TDLEN(n));
  307. break;
  308. case E1000_TDH(0):
  309. for (n = 0; n < 4; n++)
  310. regs[n] = rd32(E1000_TDH(n));
  311. break;
  312. case E1000_TDT(0):
  313. for (n = 0; n < 4; n++)
  314. regs[n] = rd32(E1000_TDT(n));
  315. break;
  316. case E1000_TXDCTL(0):
  317. for (n = 0; n < 4; n++)
  318. regs[n] = rd32(E1000_TXDCTL(n));
  319. break;
  320. default:
  321. pr_info("%-15s %08x\n", reginfo->name, rd32(reginfo->ofs));
  322. return;
  323. }
  324. snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
  325. pr_info("%-15s %08x %08x %08x %08x\n", rname, regs[0], regs[1],
  326. regs[2], regs[3]);
  327. }
  328. /* igb_dump - Print registers, Tx-rings and Rx-rings */
  329. static void igb_dump(struct igb_adapter *adapter)
  330. {
  331. struct net_device *netdev = adapter->netdev;
  332. struct e1000_hw *hw = &adapter->hw;
  333. struct igb_reg_info *reginfo;
  334. struct igb_ring *tx_ring;
  335. union e1000_adv_tx_desc *tx_desc;
  336. struct my_u0 { u64 a; u64 b; } *u0;
  337. struct igb_ring *rx_ring;
  338. union e1000_adv_rx_desc *rx_desc;
  339. u32 staterr;
  340. u16 i, n;
  341. if (!netif_msg_hw(adapter))
  342. return;
  343. /* Print netdevice Info */
  344. if (netdev) {
  345. dev_info(&adapter->pdev->dev, "Net device Info\n");
  346. pr_info("Device Name state trans_start\n");
  347. pr_info("%-15s %016lX %016lX\n", netdev->name,
  348. netdev->state, dev_trans_start(netdev));
  349. }
  350. /* Print Registers */
  351. dev_info(&adapter->pdev->dev, "Register Dump\n");
  352. pr_info(" Register Name Value\n");
  353. for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
  354. reginfo->name; reginfo++) {
  355. igb_regdump(hw, reginfo);
  356. }
  357. /* Print TX Ring Summary */
  358. if (!netdev || !netif_running(netdev))
  359. goto exit;
  360. dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
  361. pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n");
  362. for (n = 0; n < adapter->num_tx_queues; n++) {
  363. struct igb_tx_buffer *buffer_info;
  364. tx_ring = adapter->tx_ring[n];
  365. buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
  366. pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n",
  367. n, tx_ring->next_to_use, tx_ring->next_to_clean,
  368. (u64)dma_unmap_addr(buffer_info, dma),
  369. dma_unmap_len(buffer_info, len),
  370. buffer_info->next_to_watch,
  371. (u64)buffer_info->time_stamp);
  372. }
  373. /* Print TX Rings */
  374. if (!netif_msg_tx_done(adapter))
  375. goto rx_ring_summary;
  376. dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
  377. /* Transmit Descriptor Formats
  378. *
  379. * Advanced Transmit Descriptor
  380. * +--------------------------------------------------------------+
  381. * 0 | Buffer Address [63:0] |
  382. * +--------------------------------------------------------------+
  383. * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
  384. * +--------------------------------------------------------------+
  385. * 63 46 45 40 39 38 36 35 32 31 24 15 0
  386. */
  387. for (n = 0; n < adapter->num_tx_queues; n++) {
  388. tx_ring = adapter->tx_ring[n];
  389. pr_info("------------------------------------\n");
  390. pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
  391. pr_info("------------------------------------\n");
  392. pr_info("T [desc] [address 63:0 ] [PlPOCIStDDM Ln] [bi->dma ] leng ntw timestamp bi->skb\n");
  393. for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
  394. const char *next_desc;
  395. struct igb_tx_buffer *buffer_info;
  396. tx_desc = IGB_TX_DESC(tx_ring, i);
  397. buffer_info = &tx_ring->tx_buffer_info[i];
  398. u0 = (struct my_u0 *)tx_desc;
  399. if (i == tx_ring->next_to_use &&
  400. i == tx_ring->next_to_clean)
  401. next_desc = " NTC/U";
  402. else if (i == tx_ring->next_to_use)
  403. next_desc = " NTU";
  404. else if (i == tx_ring->next_to_clean)
  405. next_desc = " NTC";
  406. else
  407. next_desc = "";
  408. pr_info("T [0x%03X] %016llX %016llX %016llX %04X %p %016llX %p%s\n",
  409. i, le64_to_cpu(u0->a),
  410. le64_to_cpu(u0->b),
  411. (u64)dma_unmap_addr(buffer_info, dma),
  412. dma_unmap_len(buffer_info, len),
  413. buffer_info->next_to_watch,
  414. (u64)buffer_info->time_stamp,
  415. buffer_info->skb, next_desc);
  416. if (netif_msg_pktdata(adapter) && buffer_info->skb)
  417. print_hex_dump(KERN_INFO, "",
  418. DUMP_PREFIX_ADDRESS,
  419. 16, 1, buffer_info->skb->data,
  420. dma_unmap_len(buffer_info, len),
  421. true);
  422. }
  423. }
  424. /* Print RX Rings Summary */
  425. rx_ring_summary:
  426. dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
  427. pr_info("Queue [NTU] [NTC]\n");
  428. for (n = 0; n < adapter->num_rx_queues; n++) {
  429. rx_ring = adapter->rx_ring[n];
  430. pr_info(" %5d %5X %5X\n",
  431. n, rx_ring->next_to_use, rx_ring->next_to_clean);
  432. }
  433. /* Print RX Rings */
  434. if (!netif_msg_rx_status(adapter))
  435. goto exit;
  436. dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
  437. /* Advanced Receive Descriptor (Read) Format
  438. * 63 1 0
  439. * +-----------------------------------------------------+
  440. * 0 | Packet Buffer Address [63:1] |A0/NSE|
  441. * +----------------------------------------------+------+
  442. * 8 | Header Buffer Address [63:1] | DD |
  443. * +-----------------------------------------------------+
  444. *
  445. *
  446. * Advanced Receive Descriptor (Write-Back) Format
  447. *
  448. * 63 48 47 32 31 30 21 20 17 16 4 3 0
  449. * +------------------------------------------------------+
  450. * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
  451. * | Checksum Ident | | | | Type | Type |
  452. * +------------------------------------------------------+
  453. * 8 | VLAN Tag | Length | Extended Error | Extended Status |
  454. * +------------------------------------------------------+
  455. * 63 48 47 32 31 20 19 0
  456. */
  457. for (n = 0; n < adapter->num_rx_queues; n++) {
  458. rx_ring = adapter->rx_ring[n];
  459. pr_info("------------------------------------\n");
  460. pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
  461. pr_info("------------------------------------\n");
  462. pr_info("R [desc] [ PktBuf A0] [ HeadBuf DD] [bi->dma ] [bi->skb] <-- Adv Rx Read format\n");
  463. pr_info("RWB[desc] [PcsmIpSHl PtRs] [vl er S cks ln] ---------------- [bi->skb] <-- Adv Rx Write-Back format\n");
  464. for (i = 0; i < rx_ring->count; i++) {
  465. const char *next_desc;
  466. struct igb_rx_buffer *buffer_info;
  467. buffer_info = &rx_ring->rx_buffer_info[i];
  468. rx_desc = IGB_RX_DESC(rx_ring, i);
  469. u0 = (struct my_u0 *)rx_desc;
  470. staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
  471. if (i == rx_ring->next_to_use)
  472. next_desc = " NTU";
  473. else if (i == rx_ring->next_to_clean)
  474. next_desc = " NTC";
  475. else
  476. next_desc = "";
  477. if (staterr & E1000_RXD_STAT_DD) {
  478. /* Descriptor Done */
  479. pr_info("%s[0x%03X] %016llX %016llX ---------------- %s\n",
  480. "RWB", i,
  481. le64_to_cpu(u0->a),
  482. le64_to_cpu(u0->b),
  483. next_desc);
  484. } else {
  485. pr_info("%s[0x%03X] %016llX %016llX %016llX %s\n",
  486. "R ", i,
  487. le64_to_cpu(u0->a),
  488. le64_to_cpu(u0->b),
  489. (u64)buffer_info->dma,
  490. next_desc);
  491. if (netif_msg_pktdata(adapter) &&
  492. buffer_info->dma && buffer_info->page) {
  493. print_hex_dump(KERN_INFO, "",
  494. DUMP_PREFIX_ADDRESS,
  495. 16, 1,
  496. page_address(buffer_info->page) +
  497. buffer_info->page_offset,
  498. igb_rx_bufsz(rx_ring), true);
  499. }
  500. }
  501. }
  502. }
  503. exit:
  504. return;
  505. }
  506. /**
  507. * igb_get_i2c_data - Reads the I2C SDA data bit
  508. * @hw: pointer to hardware structure
  509. * @i2cctl: Current value of I2CCTL register
  510. *
  511. * Returns the I2C data bit value
  512. **/
  513. static int igb_get_i2c_data(void *data)
  514. {
  515. struct igb_adapter *adapter = (struct igb_adapter *)data;
  516. struct e1000_hw *hw = &adapter->hw;
  517. s32 i2cctl = rd32(E1000_I2CPARAMS);
  518. return !!(i2cctl & E1000_I2C_DATA_IN);
  519. }
  520. /**
  521. * igb_set_i2c_data - Sets the I2C data bit
  522. * @data: pointer to hardware structure
  523. * @state: I2C data value (0 or 1) to set
  524. *
  525. * Sets the I2C data bit
  526. **/
  527. static void igb_set_i2c_data(void *data, int state)
  528. {
  529. struct igb_adapter *adapter = (struct igb_adapter *)data;
  530. struct e1000_hw *hw = &adapter->hw;
  531. s32 i2cctl = rd32(E1000_I2CPARAMS);
  532. if (state)
  533. i2cctl |= E1000_I2C_DATA_OUT;
  534. else
  535. i2cctl &= ~E1000_I2C_DATA_OUT;
  536. i2cctl &= ~E1000_I2C_DATA_OE_N;
  537. i2cctl |= E1000_I2C_CLK_OE_N;
  538. wr32(E1000_I2CPARAMS, i2cctl);
  539. wrfl();
  540. }
  541. /**
  542. * igb_set_i2c_clk - Sets the I2C SCL clock
  543. * @data: pointer to hardware structure
  544. * @state: state to set clock
  545. *
  546. * Sets the I2C clock line to state
  547. **/
  548. static void igb_set_i2c_clk(void *data, int state)
  549. {
  550. struct igb_adapter *adapter = (struct igb_adapter *)data;
  551. struct e1000_hw *hw = &adapter->hw;
  552. s32 i2cctl = rd32(E1000_I2CPARAMS);
  553. if (state) {
  554. i2cctl |= E1000_I2C_CLK_OUT;
  555. i2cctl &= ~E1000_I2C_CLK_OE_N;
  556. } else {
  557. i2cctl &= ~E1000_I2C_CLK_OUT;
  558. i2cctl &= ~E1000_I2C_CLK_OE_N;
  559. }
  560. wr32(E1000_I2CPARAMS, i2cctl);
  561. wrfl();
  562. }
  563. /**
  564. * igb_get_i2c_clk - Gets the I2C SCL clock state
  565. * @data: pointer to hardware structure
  566. *
  567. * Gets the I2C clock state
  568. **/
  569. static int igb_get_i2c_clk(void *data)
  570. {
  571. struct igb_adapter *adapter = (struct igb_adapter *)data;
  572. struct e1000_hw *hw = &adapter->hw;
  573. s32 i2cctl = rd32(E1000_I2CPARAMS);
  574. return !!(i2cctl & E1000_I2C_CLK_IN);
  575. }
  576. static const struct i2c_algo_bit_data igb_i2c_algo = {
  577. .setsda = igb_set_i2c_data,
  578. .setscl = igb_set_i2c_clk,
  579. .getsda = igb_get_i2c_data,
  580. .getscl = igb_get_i2c_clk,
  581. .udelay = 5,
  582. .timeout = 20,
  583. };
  584. /**
  585. * igb_get_hw_dev - return device
  586. * @hw: pointer to hardware structure
  587. *
  588. * used by hardware layer to print debugging information
  589. **/
  590. struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
  591. {
  592. struct igb_adapter *adapter = hw->back;
  593. return adapter->netdev;
  594. }
  595. /**
  596. * igb_init_module - Driver Registration Routine
  597. *
  598. * igb_init_module is the first routine called when the driver is
  599. * loaded. All it does is register with the PCI subsystem.
  600. **/
  601. static int __init igb_init_module(void)
  602. {
  603. int ret;
  604. pr_info("%s - version %s\n",
  605. igb_driver_string, igb_driver_version);
  606. pr_info("%s\n", igb_copyright);
  607. #ifdef CONFIG_IGB_DCA
  608. dca_register_notify(&dca_notifier);
  609. #endif
  610. ret = pci_register_driver(&igb_driver);
  611. return ret;
  612. }
  613. module_init(igb_init_module);
  614. /**
  615. * igb_exit_module - Driver Exit Cleanup Routine
  616. *
  617. * igb_exit_module is called just before the driver is removed
  618. * from memory.
  619. **/
  620. static void __exit igb_exit_module(void)
  621. {
  622. #ifdef CONFIG_IGB_DCA
  623. dca_unregister_notify(&dca_notifier);
  624. #endif
  625. pci_unregister_driver(&igb_driver);
  626. }
  627. module_exit(igb_exit_module);
  628. #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
  629. /**
  630. * igb_cache_ring_register - Descriptor ring to register mapping
  631. * @adapter: board private structure to initialize
  632. *
  633. * Once we know the feature-set enabled for the device, we'll cache
  634. * the register offset the descriptor ring is assigned to.
  635. **/
  636. static void igb_cache_ring_register(struct igb_adapter *adapter)
  637. {
  638. int i = 0, j = 0;
  639. u32 rbase_offset = adapter->vfs_allocated_count;
  640. switch (adapter->hw.mac.type) {
  641. case e1000_82576:
  642. /* The queues are allocated for virtualization such that VF 0
  643. * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
  644. * In order to avoid collision we start at the first free queue
  645. * and continue consuming queues in the same sequence
  646. */
  647. if (adapter->vfs_allocated_count) {
  648. for (; i < adapter->rss_queues; i++)
  649. adapter->rx_ring[i]->reg_idx = rbase_offset +
  650. Q_IDX_82576(i);
  651. }
  652. /* Fall through */
  653. case e1000_82575:
  654. case e1000_82580:
  655. case e1000_i350:
  656. case e1000_i354:
  657. case e1000_i210:
  658. case e1000_i211:
  659. /* Fall through */
  660. default:
  661. for (; i < adapter->num_rx_queues; i++)
  662. adapter->rx_ring[i]->reg_idx = rbase_offset + i;
  663. for (; j < adapter->num_tx_queues; j++)
  664. adapter->tx_ring[j]->reg_idx = rbase_offset + j;
  665. break;
  666. }
  667. }
  668. u32 igb_rd32(struct e1000_hw *hw, u32 reg)
  669. {
  670. struct igb_adapter *igb = container_of(hw, struct igb_adapter, hw);
  671. u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr);
  672. u32 value = 0;
  673. if (E1000_REMOVED(hw_addr))
  674. return ~value;
  675. value = readl(&hw_addr[reg]);
  676. /* reads should not return all F's */
  677. if (!(~value) && (!reg || !(~readl(hw_addr)))) {
  678. struct net_device *netdev = igb->netdev;
  679. hw->hw_addr = NULL;
  680. netdev_err(netdev, "PCIe link lost\n");
  681. WARN(pci_device_is_present(igb->pdev),
  682. "igb: Failed to read reg 0x%x!\n", reg);
  683. }
  684. return value;
  685. }
  686. /**
  687. * igb_write_ivar - configure ivar for given MSI-X vector
  688. * @hw: pointer to the HW structure
  689. * @msix_vector: vector number we are allocating to a given ring
  690. * @index: row index of IVAR register to write within IVAR table
  691. * @offset: column offset of in IVAR, should be multiple of 8
  692. *
  693. * This function is intended to handle the writing of the IVAR register
  694. * for adapters 82576 and newer. The IVAR table consists of 2 columns,
  695. * each containing an cause allocation for an Rx and Tx ring, and a
  696. * variable number of rows depending on the number of queues supported.
  697. **/
  698. static void igb_write_ivar(struct e1000_hw *hw, int msix_vector,
  699. int index, int offset)
  700. {
  701. u32 ivar = array_rd32(E1000_IVAR0, index);
  702. /* clear any bits that are currently set */
  703. ivar &= ~((u32)0xFF << offset);
  704. /* write vector and valid bit */
  705. ivar |= (msix_vector | E1000_IVAR_VALID) << offset;
  706. array_wr32(E1000_IVAR0, index, ivar);
  707. }
  708. #define IGB_N0_QUEUE -1
  709. static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
  710. {
  711. struct igb_adapter *adapter = q_vector->adapter;
  712. struct e1000_hw *hw = &adapter->hw;
  713. int rx_queue = IGB_N0_QUEUE;
  714. int tx_queue = IGB_N0_QUEUE;
  715. u32 msixbm = 0;
  716. if (q_vector->rx.ring)
  717. rx_queue = q_vector->rx.ring->reg_idx;
  718. if (q_vector->tx.ring)
  719. tx_queue = q_vector->tx.ring->reg_idx;
  720. switch (hw->mac.type) {
  721. case e1000_82575:
  722. /* The 82575 assigns vectors using a bitmask, which matches the
  723. * bitmask for the EICR/EIMS/EIMC registers. To assign one
  724. * or more queues to a vector, we write the appropriate bits
  725. * into the MSIXBM register for that vector.
  726. */
  727. if (rx_queue > IGB_N0_QUEUE)
  728. msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
  729. if (tx_queue > IGB_N0_QUEUE)
  730. msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
  731. if (!(adapter->flags & IGB_FLAG_HAS_MSIX) && msix_vector == 0)
  732. msixbm |= E1000_EIMS_OTHER;
  733. array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
  734. q_vector->eims_value = msixbm;
  735. break;
  736. case e1000_82576:
  737. /* 82576 uses a table that essentially consists of 2 columns
  738. * with 8 rows. The ordering is column-major so we use the
  739. * lower 3 bits as the row index, and the 4th bit as the
  740. * column offset.
  741. */
  742. if (rx_queue > IGB_N0_QUEUE)
  743. igb_write_ivar(hw, msix_vector,
  744. rx_queue & 0x7,
  745. (rx_queue & 0x8) << 1);
  746. if (tx_queue > IGB_N0_QUEUE)
  747. igb_write_ivar(hw, msix_vector,
  748. tx_queue & 0x7,
  749. ((tx_queue & 0x8) << 1) + 8);
  750. q_vector->eims_value = BIT(msix_vector);
  751. break;
  752. case e1000_82580:
  753. case e1000_i350:
  754. case e1000_i354:
  755. case e1000_i210:
  756. case e1000_i211:
  757. /* On 82580 and newer adapters the scheme is similar to 82576
  758. * however instead of ordering column-major we have things
  759. * ordered row-major. So we traverse the table by using
  760. * bit 0 as the column offset, and the remaining bits as the
  761. * row index.
  762. */
  763. if (rx_queue > IGB_N0_QUEUE)
  764. igb_write_ivar(hw, msix_vector,
  765. rx_queue >> 1,
  766. (rx_queue & 0x1) << 4);
  767. if (tx_queue > IGB_N0_QUEUE)
  768. igb_write_ivar(hw, msix_vector,
  769. tx_queue >> 1,
  770. ((tx_queue & 0x1) << 4) + 8);
  771. q_vector->eims_value = BIT(msix_vector);
  772. break;
  773. default:
  774. BUG();
  775. break;
  776. }
  777. /* add q_vector eims value to global eims_enable_mask */
  778. adapter->eims_enable_mask |= q_vector->eims_value;
  779. /* configure q_vector to set itr on first interrupt */
  780. q_vector->set_itr = 1;
  781. }
  782. /**
  783. * igb_configure_msix - Configure MSI-X hardware
  784. * @adapter: board private structure to initialize
  785. *
  786. * igb_configure_msix sets up the hardware to properly
  787. * generate MSI-X interrupts.
  788. **/
  789. static void igb_configure_msix(struct igb_adapter *adapter)
  790. {
  791. u32 tmp;
  792. int i, vector = 0;
  793. struct e1000_hw *hw = &adapter->hw;
  794. adapter->eims_enable_mask = 0;
  795. /* set vector for other causes, i.e. link changes */
  796. switch (hw->mac.type) {
  797. case e1000_82575:
  798. tmp = rd32(E1000_CTRL_EXT);
  799. /* enable MSI-X PBA support*/
  800. tmp |= E1000_CTRL_EXT_PBA_CLR;
  801. /* Auto-Mask interrupts upon ICR read. */
  802. tmp |= E1000_CTRL_EXT_EIAME;
  803. tmp |= E1000_CTRL_EXT_IRCA;
  804. wr32(E1000_CTRL_EXT, tmp);
  805. /* enable msix_other interrupt */
  806. array_wr32(E1000_MSIXBM(0), vector++, E1000_EIMS_OTHER);
  807. adapter->eims_other = E1000_EIMS_OTHER;
  808. break;
  809. case e1000_82576:
  810. case e1000_82580:
  811. case e1000_i350:
  812. case e1000_i354:
  813. case e1000_i210:
  814. case e1000_i211:
  815. /* Turn on MSI-X capability first, or our settings
  816. * won't stick. And it will take days to debug.
  817. */
  818. wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
  819. E1000_GPIE_PBA | E1000_GPIE_EIAME |
  820. E1000_GPIE_NSICR);
  821. /* enable msix_other interrupt */
  822. adapter->eims_other = BIT(vector);
  823. tmp = (vector++ | E1000_IVAR_VALID) << 8;
  824. wr32(E1000_IVAR_MISC, tmp);
  825. break;
  826. default:
  827. /* do nothing, since nothing else supports MSI-X */
  828. break;
  829. } /* switch (hw->mac.type) */
  830. adapter->eims_enable_mask |= adapter->eims_other;
  831. for (i = 0; i < adapter->num_q_vectors; i++)
  832. igb_assign_vector(adapter->q_vector[i], vector++);
  833. wrfl();
  834. }
  835. /**
  836. * igb_request_msix - Initialize MSI-X interrupts
  837. * @adapter: board private structure to initialize
  838. *
  839. * igb_request_msix allocates MSI-X vectors and requests interrupts from the
  840. * kernel.
  841. **/
  842. static int igb_request_msix(struct igb_adapter *adapter)
  843. {
  844. struct net_device *netdev = adapter->netdev;
  845. int i, err = 0, vector = 0, free_vector = 0;
  846. err = request_irq(adapter->msix_entries[vector].vector,
  847. igb_msix_other, 0, netdev->name, adapter);
  848. if (err)
  849. goto err_out;
  850. for (i = 0; i < adapter->num_q_vectors; i++) {
  851. struct igb_q_vector *q_vector = adapter->q_vector[i];
  852. vector++;
  853. q_vector->itr_register = adapter->io_addr + E1000_EITR(vector);
  854. if (q_vector->rx.ring && q_vector->tx.ring)
  855. sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
  856. q_vector->rx.ring->queue_index);
  857. else if (q_vector->tx.ring)
  858. sprintf(q_vector->name, "%s-tx-%u", netdev->name,
  859. q_vector->tx.ring->queue_index);
  860. else if (q_vector->rx.ring)
  861. sprintf(q_vector->name, "%s-rx-%u", netdev->name,
  862. q_vector->rx.ring->queue_index);
  863. else
  864. sprintf(q_vector->name, "%s-unused", netdev->name);
  865. err = request_irq(adapter->msix_entries[vector].vector,
  866. igb_msix_ring, 0, q_vector->name,
  867. q_vector);
  868. if (err)
  869. goto err_free;
  870. }
  871. igb_configure_msix(adapter);
  872. return 0;
  873. err_free:
  874. /* free already assigned IRQs */
  875. free_irq(adapter->msix_entries[free_vector++].vector, adapter);
  876. vector--;
  877. for (i = 0; i < vector; i++) {
  878. free_irq(adapter->msix_entries[free_vector++].vector,
  879. adapter->q_vector[i]);
  880. }
  881. err_out:
  882. return err;
  883. }
  884. /**
  885. * igb_free_q_vector - Free memory allocated for specific interrupt vector
  886. * @adapter: board private structure to initialize
  887. * @v_idx: Index of vector to be freed
  888. *
  889. * This function frees the memory allocated to the q_vector.
  890. **/
  891. static void igb_free_q_vector(struct igb_adapter *adapter, int v_idx)
  892. {
  893. struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
  894. adapter->q_vector[v_idx] = NULL;
  895. /* igb_get_stats64() might access the rings on this vector,
  896. * we must wait a grace period before freeing it.
  897. */
  898. if (q_vector)
  899. kfree_rcu(q_vector, rcu);
  900. }
  901. /**
  902. * igb_reset_q_vector - Reset config for interrupt vector
  903. * @adapter: board private structure to initialize
  904. * @v_idx: Index of vector to be reset
  905. *
  906. * If NAPI is enabled it will delete any references to the
  907. * NAPI struct. This is preparation for igb_free_q_vector.
  908. **/
  909. static void igb_reset_q_vector(struct igb_adapter *adapter, int v_idx)
  910. {
  911. struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
  912. /* Coming from igb_set_interrupt_capability, the vectors are not yet
  913. * allocated. So, q_vector is NULL so we should stop here.
  914. */
  915. if (!q_vector)
  916. return;
  917. if (q_vector->tx.ring)
  918. adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL;
  919. if (q_vector->rx.ring)
  920. adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL;
  921. netif_napi_del(&q_vector->napi);
  922. }
  923. static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
  924. {
  925. int v_idx = adapter->num_q_vectors;
  926. if (adapter->flags & IGB_FLAG_HAS_MSIX)
  927. pci_disable_msix(adapter->pdev);
  928. else if (adapter->flags & IGB_FLAG_HAS_MSI)
  929. pci_disable_msi(adapter->pdev);
  930. while (v_idx--)
  931. igb_reset_q_vector(adapter, v_idx);
  932. }
  933. /**
  934. * igb_free_q_vectors - Free memory allocated for interrupt vectors
  935. * @adapter: board private structure to initialize
  936. *
  937. * This function frees the memory allocated to the q_vectors. In addition if
  938. * NAPI is enabled it will delete any references to the NAPI struct prior
  939. * to freeing the q_vector.
  940. **/
  941. static void igb_free_q_vectors(struct igb_adapter *adapter)
  942. {
  943. int v_idx = adapter->num_q_vectors;
  944. adapter->num_tx_queues = 0;
  945. adapter->num_rx_queues = 0;
  946. adapter->num_q_vectors = 0;
  947. while (v_idx--) {
  948. igb_reset_q_vector(adapter, v_idx);
  949. igb_free_q_vector(adapter, v_idx);
  950. }
  951. }
  952. /**
  953. * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
  954. * @adapter: board private structure to initialize
  955. *
  956. * This function resets the device so that it has 0 Rx queues, Tx queues, and
  957. * MSI-X interrupts allocated.
  958. */
  959. static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
  960. {
  961. igb_free_q_vectors(adapter);
  962. igb_reset_interrupt_capability(adapter);
  963. }
  964. /**
  965. * igb_set_interrupt_capability - set MSI or MSI-X if supported
  966. * @adapter: board private structure to initialize
  967. * @msix: boolean value of MSIX capability
  968. *
  969. * Attempt to configure interrupts using the best available
  970. * capabilities of the hardware and kernel.
  971. **/
  972. static void igb_set_interrupt_capability(struct igb_adapter *adapter, bool msix)
  973. {
  974. int err;
  975. int numvecs, i;
  976. if (!msix)
  977. goto msi_only;
  978. adapter->flags |= IGB_FLAG_HAS_MSIX;
  979. /* Number of supported queues. */
  980. adapter->num_rx_queues = adapter->rss_queues;
  981. if (adapter->vfs_allocated_count)
  982. adapter->num_tx_queues = 1;
  983. else
  984. adapter->num_tx_queues = adapter->rss_queues;
  985. /* start with one vector for every Rx queue */
  986. numvecs = adapter->num_rx_queues;
  987. /* if Tx handler is separate add 1 for every Tx queue */
  988. if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
  989. numvecs += adapter->num_tx_queues;
  990. /* store the number of vectors reserved for queues */
  991. adapter->num_q_vectors = numvecs;
  992. /* add 1 vector for link status interrupts */
  993. numvecs++;
  994. for (i = 0; i < numvecs; i++)
  995. adapter->msix_entries[i].entry = i;
  996. err = pci_enable_msix_range(adapter->pdev,
  997. adapter->msix_entries,
  998. numvecs,
  999. numvecs);
  1000. if (err > 0)
  1001. return;
  1002. igb_reset_interrupt_capability(adapter);
  1003. /* If we can't do MSI-X, try MSI */
  1004. msi_only:
  1005. adapter->flags &= ~IGB_FLAG_HAS_MSIX;
  1006. #ifdef CONFIG_PCI_IOV
  1007. /* disable SR-IOV for non MSI-X configurations */
  1008. if (adapter->vf_data) {
  1009. struct e1000_hw *hw = &adapter->hw;
  1010. /* disable iov and allow time for transactions to clear */
  1011. pci_disable_sriov(adapter->pdev);
  1012. msleep(500);
  1013. kfree(adapter->vf_mac_list);
  1014. adapter->vf_mac_list = NULL;
  1015. kfree(adapter->vf_data);
  1016. adapter->vf_data = NULL;
  1017. wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
  1018. wrfl();
  1019. msleep(100);
  1020. dev_info(&adapter->pdev->dev, "IOV Disabled\n");
  1021. }
  1022. #endif
  1023. adapter->vfs_allocated_count = 0;
  1024. adapter->rss_queues = 1;
  1025. adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
  1026. adapter->num_rx_queues = 1;
  1027. adapter->num_tx_queues = 1;
  1028. adapter->num_q_vectors = 1;
  1029. if (!pci_enable_msi(adapter->pdev))
  1030. adapter->flags |= IGB_FLAG_HAS_MSI;
  1031. }
  1032. static void igb_add_ring(struct igb_ring *ring,
  1033. struct igb_ring_container *head)
  1034. {
  1035. head->ring = ring;
  1036. head->count++;
  1037. }
  1038. /**
  1039. * igb_alloc_q_vector - Allocate memory for a single interrupt vector
  1040. * @adapter: board private structure to initialize
  1041. * @v_count: q_vectors allocated on adapter, used for ring interleaving
  1042. * @v_idx: index of vector in adapter struct
  1043. * @txr_count: total number of Tx rings to allocate
  1044. * @txr_idx: index of first Tx ring to allocate
  1045. * @rxr_count: total number of Rx rings to allocate
  1046. * @rxr_idx: index of first Rx ring to allocate
  1047. *
  1048. * We allocate one q_vector. If allocation fails we return -ENOMEM.
  1049. **/
  1050. static int igb_alloc_q_vector(struct igb_adapter *adapter,
  1051. int v_count, int v_idx,
  1052. int txr_count, int txr_idx,
  1053. int rxr_count, int rxr_idx)
  1054. {
  1055. struct igb_q_vector *q_vector;
  1056. struct igb_ring *ring;
  1057. int ring_count;
  1058. size_t size;
  1059. /* igb only supports 1 Tx and/or 1 Rx queue per vector */
  1060. if (txr_count > 1 || rxr_count > 1)
  1061. return -ENOMEM;
  1062. ring_count = txr_count + rxr_count;
  1063. size = struct_size(q_vector, ring, ring_count);
  1064. /* allocate q_vector and rings */
  1065. q_vector = adapter->q_vector[v_idx];
  1066. if (!q_vector) {
  1067. q_vector = kzalloc(size, GFP_KERNEL);
  1068. } else if (size > ksize(q_vector)) {
  1069. kfree_rcu(q_vector, rcu);
  1070. q_vector = kzalloc(size, GFP_KERNEL);
  1071. } else {
  1072. memset(q_vector, 0, size);
  1073. }
  1074. if (!q_vector)
  1075. return -ENOMEM;
  1076. /* initialize NAPI */
  1077. netif_napi_add(adapter->netdev, &q_vector->napi,
  1078. igb_poll, 64);
  1079. /* tie q_vector and adapter together */
  1080. adapter->q_vector[v_idx] = q_vector;
  1081. q_vector->adapter = adapter;
  1082. /* initialize work limits */
  1083. q_vector->tx.work_limit = adapter->tx_work_limit;
  1084. /* initialize ITR configuration */
  1085. q_vector->itr_register = adapter->io_addr + E1000_EITR(0);
  1086. q_vector->itr_val = IGB_START_ITR;
  1087. /* initialize pointer to rings */
  1088. ring = q_vector->ring;
  1089. /* intialize ITR */
  1090. if (rxr_count) {
  1091. /* rx or rx/tx vector */
  1092. if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3)
  1093. q_vector->itr_val = adapter->rx_itr_setting;
  1094. } else {
  1095. /* tx only vector */
  1096. if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3)
  1097. q_vector->itr_val = adapter->tx_itr_setting;
  1098. }
  1099. if (txr_count) {
  1100. /* assign generic ring traits */
  1101. ring->dev = &adapter->pdev->dev;
  1102. ring->netdev = adapter->netdev;
  1103. /* configure backlink on ring */
  1104. ring->q_vector = q_vector;
  1105. /* update q_vector Tx values */
  1106. igb_add_ring(ring, &q_vector->tx);
  1107. /* For 82575, context index must be unique per ring. */
  1108. if (adapter->hw.mac.type == e1000_82575)
  1109. set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags);
  1110. /* apply Tx specific ring traits */
  1111. ring->count = adapter->tx_ring_count;
  1112. ring->queue_index = txr_idx;
  1113. ring->cbs_enable = false;
  1114. ring->idleslope = 0;
  1115. ring->sendslope = 0;
  1116. ring->hicredit = 0;
  1117. ring->locredit = 0;
  1118. u64_stats_init(&ring->tx_syncp);
  1119. u64_stats_init(&ring->tx_syncp2);
  1120. /* assign ring to adapter */
  1121. adapter->tx_ring[txr_idx] = ring;
  1122. /* push pointer to next ring */
  1123. ring++;
  1124. }
  1125. if (rxr_count) {
  1126. /* assign generic ring traits */
  1127. ring->dev = &adapter->pdev->dev;
  1128. ring->netdev = adapter->netdev;
  1129. /* configure backlink on ring */
  1130. ring->q_vector = q_vector;
  1131. /* update q_vector Rx values */
  1132. igb_add_ring(ring, &q_vector->rx);
  1133. /* set flag indicating ring supports SCTP checksum offload */
  1134. if (adapter->hw.mac.type >= e1000_82576)
  1135. set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags);
  1136. /* On i350, i354, i210, and i211, loopback VLAN packets
  1137. * have the tag byte-swapped.
  1138. */
  1139. if (adapter->hw.mac.type >= e1000_i350)
  1140. set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags);
  1141. /* apply Rx specific ring traits */
  1142. ring->count = adapter->rx_ring_count;
  1143. ring->queue_index = rxr_idx;
  1144. u64_stats_init(&ring->rx_syncp);
  1145. /* assign ring to adapter */
  1146. adapter->rx_ring[rxr_idx] = ring;
  1147. }
  1148. return 0;
  1149. }
  1150. /**
  1151. * igb_alloc_q_vectors - Allocate memory for interrupt vectors
  1152. * @adapter: board private structure to initialize
  1153. *
  1154. * We allocate one q_vector per queue interrupt. If allocation fails we
  1155. * return -ENOMEM.
  1156. **/
  1157. static int igb_alloc_q_vectors(struct igb_adapter *adapter)
  1158. {
  1159. int q_vectors = adapter->num_q_vectors;
  1160. int rxr_remaining = adapter->num_rx_queues;
  1161. int txr_remaining = adapter->num_tx_queues;
  1162. int rxr_idx = 0, txr_idx = 0, v_idx = 0;
  1163. int err;
  1164. if (q_vectors >= (rxr_remaining + txr_remaining)) {
  1165. for (; rxr_remaining; v_idx++) {
  1166. err = igb_alloc_q_vector(adapter, q_vectors, v_idx,
  1167. 0, 0, 1, rxr_idx);
  1168. if (err)
  1169. goto err_out;
  1170. /* update counts and index */
  1171. rxr_remaining--;
  1172. rxr_idx++;
  1173. }
  1174. }
  1175. for (; v_idx < q_vectors; v_idx++) {
  1176. int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);
  1177. int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);
  1178. err = igb_alloc_q_vector(adapter, q_vectors, v_idx,
  1179. tqpv, txr_idx, rqpv, rxr_idx);
  1180. if (err)
  1181. goto err_out;
  1182. /* update counts and index */
  1183. rxr_remaining -= rqpv;
  1184. txr_remaining -= tqpv;
  1185. rxr_idx++;
  1186. txr_idx++;
  1187. }
  1188. return 0;
  1189. err_out:
  1190. adapter->num_tx_queues = 0;
  1191. adapter->num_rx_queues = 0;
  1192. adapter->num_q_vectors = 0;
  1193. while (v_idx--)
  1194. igb_free_q_vector(adapter, v_idx);
  1195. return -ENOMEM;
  1196. }
  1197. /**
  1198. * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
  1199. * @adapter: board private structure to initialize
  1200. * @msix: boolean value of MSIX capability
  1201. *
  1202. * This function initializes the interrupts and allocates all of the queues.
  1203. **/
  1204. static int igb_init_interrupt_scheme(struct igb_adapter *adapter, bool msix)
  1205. {
  1206. struct pci_dev *pdev = adapter->pdev;
  1207. int err;
  1208. igb_set_interrupt_capability(adapter, msix);
  1209. err = igb_alloc_q_vectors(adapter);
  1210. if (err) {
  1211. dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
  1212. goto err_alloc_q_vectors;
  1213. }
  1214. igb_cache_ring_register(adapter);
  1215. return 0;
  1216. err_alloc_q_vectors:
  1217. igb_reset_interrupt_capability(adapter);
  1218. return err;
  1219. }
  1220. /**
  1221. * igb_request_irq - initialize interrupts
  1222. * @adapter: board private structure to initialize
  1223. *
  1224. * Attempts to configure interrupts using the best available
  1225. * capabilities of the hardware and kernel.
  1226. **/
  1227. static int igb_request_irq(struct igb_adapter *adapter)
  1228. {
  1229. struct net_device *netdev = adapter->netdev;
  1230. struct pci_dev *pdev = adapter->pdev;
  1231. int err = 0;
  1232. if (adapter->flags & IGB_FLAG_HAS_MSIX) {
  1233. err = igb_request_msix(adapter);
  1234. if (!err)
  1235. goto request_done;
  1236. /* fall back to MSI */
  1237. igb_free_all_tx_resources(adapter);
  1238. igb_free_all_rx_resources(adapter);
  1239. igb_clear_interrupt_scheme(adapter);
  1240. err = igb_init_interrupt_scheme(adapter, false);
  1241. if (err)
  1242. goto request_done;
  1243. igb_setup_all_tx_resources(adapter);
  1244. igb_setup_all_rx_resources(adapter);
  1245. igb_configure(adapter);
  1246. }
  1247. igb_assign_vector(adapter->q_vector[0], 0);
  1248. if (adapter->flags & IGB_FLAG_HAS_MSI) {
  1249. err = request_irq(pdev->irq, igb_intr_msi, 0,
  1250. netdev->name, adapter);
  1251. if (!err)
  1252. goto request_done;
  1253. /* fall back to legacy interrupts */
  1254. igb_reset_interrupt_capability(adapter);
  1255. adapter->flags &= ~IGB_FLAG_HAS_MSI;
  1256. }
  1257. err = request_irq(pdev->irq, igb_intr, IRQF_SHARED,
  1258. netdev->name, adapter);
  1259. if (err)
  1260. dev_err(&pdev->dev, "Error %d getting interrupt\n",
  1261. err);
  1262. request_done:
  1263. return err;
  1264. }
  1265. static void igb_free_irq(struct igb_adapter *adapter)
  1266. {
  1267. if (adapter->flags & IGB_FLAG_HAS_MSIX) {
  1268. int vector = 0, i;
  1269. free_irq(adapter->msix_entries[vector++].vector, adapter);
  1270. for (i = 0; i < adapter->num_q_vectors; i++)
  1271. free_irq(adapter->msix_entries[vector++].vector,
  1272. adapter->q_vector[i]);
  1273. } else {
  1274. free_irq(adapter->pdev->irq, adapter);
  1275. }
  1276. }
  1277. /**
  1278. * igb_irq_disable - Mask off interrupt generation on the NIC
  1279. * @adapter: board private structure
  1280. **/
  1281. static void igb_irq_disable(struct igb_adapter *adapter)
  1282. {
  1283. struct e1000_hw *hw = &adapter->hw;
  1284. /* we need to be careful when disabling interrupts. The VFs are also
  1285. * mapped into these registers and so clearing the bits can cause
  1286. * issues on the VF drivers so we only need to clear what we set
  1287. */
  1288. if (adapter->flags & IGB_FLAG_HAS_MSIX) {
  1289. u32 regval = rd32(E1000_EIAM);
  1290. wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
  1291. wr32(E1000_EIMC, adapter->eims_enable_mask);
  1292. regval = rd32(E1000_EIAC);
  1293. wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
  1294. }
  1295. wr32(E1000_IAM, 0);
  1296. wr32(E1000_IMC, ~0);
  1297. wrfl();
  1298. if (adapter->flags & IGB_FLAG_HAS_MSIX) {
  1299. int i;
  1300. for (i = 0; i < adapter->num_q_vectors; i++)
  1301. synchronize_irq(adapter->msix_entries[i].vector);
  1302. } else {
  1303. synchronize_irq(adapter->pdev->irq);
  1304. }
  1305. }
  1306. /**
  1307. * igb_irq_enable - Enable default interrupt generation settings
  1308. * @adapter: board private structure
  1309. **/
  1310. static void igb_irq_enable(struct igb_adapter *adapter)
  1311. {
  1312. struct e1000_hw *hw = &adapter->hw;
  1313. if (adapter->flags & IGB_FLAG_HAS_MSIX) {
  1314. u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA;
  1315. u32 regval = rd32(E1000_EIAC);
  1316. wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
  1317. regval = rd32(E1000_EIAM);
  1318. wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
  1319. wr32(E1000_EIMS, adapter->eims_enable_mask);
  1320. if (adapter->vfs_allocated_count) {
  1321. wr32(E1000_MBVFIMR, 0xFF);
  1322. ims |= E1000_IMS_VMMB;
  1323. }
  1324. wr32(E1000_IMS, ims);
  1325. } else {
  1326. wr32(E1000_IMS, IMS_ENABLE_MASK |
  1327. E1000_IMS_DRSTA);
  1328. wr32(E1000_IAM, IMS_ENABLE_MASK |
  1329. E1000_IMS_DRSTA);
  1330. }
  1331. }
  1332. static void igb_update_mng_vlan(struct igb_adapter *adapter)
  1333. {
  1334. struct e1000_hw *hw = &adapter->hw;
  1335. u16 pf_id = adapter->vfs_allocated_count;
  1336. u16 vid = adapter->hw.mng_cookie.vlan_id;
  1337. u16 old_vid = adapter->mng_vlan_id;
  1338. if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
  1339. /* add VID to filter table */
  1340. igb_vfta_set(hw, vid, pf_id, true, true);
  1341. adapter->mng_vlan_id = vid;
  1342. } else {
  1343. adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
  1344. }
  1345. if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
  1346. (vid != old_vid) &&
  1347. !test_bit(old_vid, adapter->active_vlans)) {
  1348. /* remove VID from filter table */
  1349. igb_vfta_set(hw, vid, pf_id, false, true);
  1350. }
  1351. }
  1352. /**
  1353. * igb_release_hw_control - release control of the h/w to f/w
  1354. * @adapter: address of board private structure
  1355. *
  1356. * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
  1357. * For ASF and Pass Through versions of f/w this means that the
  1358. * driver is no longer loaded.
  1359. **/
  1360. static void igb_release_hw_control(struct igb_adapter *adapter)
  1361. {
  1362. struct e1000_hw *hw = &adapter->hw;
  1363. u32 ctrl_ext;
  1364. /* Let firmware take over control of h/w */
  1365. ctrl_ext = rd32(E1000_CTRL_EXT);
  1366. wr32(E1000_CTRL_EXT,
  1367. ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
  1368. }
  1369. /**
  1370. * igb_get_hw_control - get control of the h/w from f/w
  1371. * @adapter: address of board private structure
  1372. *
  1373. * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
  1374. * For ASF and Pass Through versions of f/w this means that
  1375. * the driver is loaded.
  1376. **/
  1377. static void igb_get_hw_control(struct igb_adapter *adapter)
  1378. {
  1379. struct e1000_hw *hw = &adapter->hw;
  1380. u32 ctrl_ext;
  1381. /* Let firmware know the driver has taken over */
  1382. ctrl_ext = rd32(E1000_CTRL_EXT);
  1383. wr32(E1000_CTRL_EXT,
  1384. ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
  1385. }
  1386. static void enable_fqtss(struct igb_adapter *adapter, bool enable)
  1387. {
  1388. struct net_device *netdev = adapter->netdev;
  1389. struct e1000_hw *hw = &adapter->hw;
  1390. WARN_ON(hw->mac.type != e1000_i210);
  1391. if (enable)
  1392. adapter->flags |= IGB_FLAG_FQTSS;
  1393. else
  1394. adapter->flags &= ~IGB_FLAG_FQTSS;
  1395. if (netif_running(netdev))
  1396. schedule_work(&adapter->reset_task);
  1397. }
  1398. static bool is_fqtss_enabled(struct igb_adapter *adapter)
  1399. {
  1400. return (adapter->flags & IGB_FLAG_FQTSS) ? true : false;
  1401. }
  1402. static void set_tx_desc_fetch_prio(struct e1000_hw *hw, int queue,
  1403. enum tx_queue_prio prio)
  1404. {
  1405. u32 val;
  1406. WARN_ON(hw->mac.type != e1000_i210);
  1407. WARN_ON(queue < 0 || queue > 4);
  1408. val = rd32(E1000_I210_TXDCTL(queue));
  1409. if (prio == TX_QUEUE_PRIO_HIGH)
  1410. val |= E1000_TXDCTL_PRIORITY;
  1411. else
  1412. val &= ~E1000_TXDCTL_PRIORITY;
  1413. wr32(E1000_I210_TXDCTL(queue), val);
  1414. }
  1415. static void set_queue_mode(struct e1000_hw *hw, int queue, enum queue_mode mode)
  1416. {
  1417. u32 val;
  1418. WARN_ON(hw->mac.type != e1000_i210);
  1419. WARN_ON(queue < 0 || queue > 1);
  1420. val = rd32(E1000_I210_TQAVCC(queue));
  1421. if (mode == QUEUE_MODE_STREAM_RESERVATION)
  1422. val |= E1000_TQAVCC_QUEUEMODE;
  1423. else
  1424. val &= ~E1000_TQAVCC_QUEUEMODE;
  1425. wr32(E1000_I210_TQAVCC(queue), val);
  1426. }
  1427. static bool is_any_cbs_enabled(struct igb_adapter *adapter)
  1428. {
  1429. int i;
  1430. for (i = 0; i < adapter->num_tx_queues; i++) {
  1431. if (adapter->tx_ring[i]->cbs_enable)
  1432. return true;
  1433. }
  1434. return false;
  1435. }
  1436. static bool is_any_txtime_enabled(struct igb_adapter *adapter)
  1437. {
  1438. int i;
  1439. for (i = 0; i < adapter->num_tx_queues; i++) {
  1440. if (adapter->tx_ring[i]->launchtime_enable)
  1441. return true;
  1442. }
  1443. return false;
  1444. }
  1445. /**
  1446. * igb_config_tx_modes - Configure "Qav Tx mode" features on igb
  1447. * @adapter: pointer to adapter struct
  1448. * @queue: queue number
  1449. *
  1450. * Configure CBS and Launchtime for a given hardware queue.
  1451. * Parameters are retrieved from the correct Tx ring, so
  1452. * igb_save_cbs_params() and igb_save_txtime_params() should be used
  1453. * for setting those correctly prior to this function being called.
  1454. **/
  1455. static void igb_config_tx_modes(struct igb_adapter *adapter, int queue)
  1456. {
  1457. struct igb_ring *ring = adapter->tx_ring[queue];
  1458. struct net_device *netdev = adapter->netdev;
  1459. struct e1000_hw *hw = &adapter->hw;
  1460. u32 tqavcc, tqavctrl;
  1461. u16 value;
  1462. WARN_ON(hw->mac.type != e1000_i210);
  1463. WARN_ON(queue < 0 || queue > 1);
  1464. /* If any of the Qav features is enabled, configure queues as SR and
  1465. * with HIGH PRIO. If none is, then configure them with LOW PRIO and
  1466. * as SP.
  1467. */
  1468. if (ring->cbs_enable || ring->launchtime_enable) {
  1469. set_tx_desc_fetch_prio(hw, queue, TX_QUEUE_PRIO_HIGH);
  1470. set_queue_mode(hw, queue, QUEUE_MODE_STREAM_RESERVATION);
  1471. } else {
  1472. set_tx_desc_fetch_prio(hw, queue, TX_QUEUE_PRIO_LOW);
  1473. set_queue_mode(hw, queue, QUEUE_MODE_STRICT_PRIORITY);
  1474. }
  1475. /* If CBS is enabled, set DataTranARB and config its parameters. */
  1476. if (ring->cbs_enable || queue == 0) {
  1477. /* i210 does not allow the queue 0 to be in the Strict
  1478. * Priority mode while the Qav mode is enabled, so,
  1479. * instead of disabling strict priority mode, we give
  1480. * queue 0 the maximum of credits possible.
  1481. *
  1482. * See section 8.12.19 of the i210 datasheet, "Note:
  1483. * Queue0 QueueMode must be set to 1b when
  1484. * TransmitMode is set to Qav."
  1485. */
  1486. if (queue == 0 && !ring->cbs_enable) {
  1487. /* max "linkspeed" idleslope in kbps */
  1488. ring->idleslope = 1000000;
  1489. ring->hicredit = ETH_FRAME_LEN;
  1490. }
  1491. /* Always set data transfer arbitration to credit-based
  1492. * shaper algorithm on TQAVCTRL if CBS is enabled for any of
  1493. * the queues.
  1494. */
  1495. tqavctrl = rd32(E1000_I210_TQAVCTRL);
  1496. tqavctrl |= E1000_TQAVCTRL_DATATRANARB;
  1497. wr32(E1000_I210_TQAVCTRL, tqavctrl);
  1498. /* According to i210 datasheet section 7.2.7.7, we should set
  1499. * the 'idleSlope' field from TQAVCC register following the
  1500. * equation:
  1501. *
  1502. * For 100 Mbps link speed:
  1503. *
  1504. * value = BW * 0x7735 * 0.2 (E1)
  1505. *
  1506. * For 1000Mbps link speed:
  1507. *
  1508. * value = BW * 0x7735 * 2 (E2)
  1509. *
  1510. * E1 and E2 can be merged into one equation as shown below.
  1511. * Note that 'link-speed' is in Mbps.
  1512. *
  1513. * value = BW * 0x7735 * 2 * link-speed
  1514. * -------------- (E3)
  1515. * 1000
  1516. *
  1517. * 'BW' is the percentage bandwidth out of full link speed
  1518. * which can be found with the following equation. Note that
  1519. * idleSlope here is the parameter from this function which
  1520. * is in kbps.
  1521. *
  1522. * BW = idleSlope
  1523. * ----------------- (E4)
  1524. * link-speed * 1000
  1525. *
  1526. * That said, we can come up with a generic equation to
  1527. * calculate the value we should set it TQAVCC register by
  1528. * replacing 'BW' in E3 by E4. The resulting equation is:
  1529. *
  1530. * value = idleSlope * 0x7735 * 2 * link-speed
  1531. * ----------------- -------------- (E5)
  1532. * link-speed * 1000 1000
  1533. *
  1534. * 'link-speed' is present in both sides of the fraction so
  1535. * it is canceled out. The final equation is the following:
  1536. *
  1537. * value = idleSlope * 61034
  1538. * ----------------- (E6)
  1539. * 1000000
  1540. *
  1541. * NOTE: For i210, given the above, we can see that idleslope
  1542. * is represented in 16.38431 kbps units by the value at
  1543. * the TQAVCC register (1Gbps / 61034), which reduces
  1544. * the granularity for idleslope increments.
  1545. * For instance, if you want to configure a 2576kbps
  1546. * idleslope, the value to be written on the register
  1547. * would have to be 157.23. If rounded down, you end
  1548. * up with less bandwidth available than originally
  1549. * required (~2572 kbps). If rounded up, you end up
  1550. * with a higher bandwidth (~2589 kbps). Below the
  1551. * approach we take is to always round up the
  1552. * calculated value, so the resulting bandwidth might
  1553. * be slightly higher for some configurations.
  1554. */
  1555. value = DIV_ROUND_UP_ULL(ring->idleslope * 61034ULL, 1000000);
  1556. tqavcc = rd32(E1000_I210_TQAVCC(queue));
  1557. tqavcc &= ~E1000_TQAVCC_IDLESLOPE_MASK;
  1558. tqavcc |= value;
  1559. wr32(E1000_I210_TQAVCC(queue), tqavcc);
  1560. wr32(E1000_I210_TQAVHC(queue),
  1561. 0x80000000 + ring->hicredit * 0x7735);
  1562. } else {
  1563. /* Set idleSlope to zero. */
  1564. tqavcc = rd32(E1000_I210_TQAVCC(queue));
  1565. tqavcc &= ~E1000_TQAVCC_IDLESLOPE_MASK;
  1566. wr32(E1000_I210_TQAVCC(queue), tqavcc);
  1567. /* Set hiCredit to zero. */
  1568. wr32(E1000_I210_TQAVHC(queue), 0);
  1569. /* If CBS is not enabled for any queues anymore, then return to
  1570. * the default state of Data Transmission Arbitration on
  1571. * TQAVCTRL.
  1572. */
  1573. if (!is_any_cbs_enabled(adapter)) {
  1574. tqavctrl = rd32(E1000_I210_TQAVCTRL);
  1575. tqavctrl &= ~E1000_TQAVCTRL_DATATRANARB;
  1576. wr32(E1000_I210_TQAVCTRL, tqavctrl);
  1577. }
  1578. }
  1579. /* If LaunchTime is enabled, set DataTranTIM. */
  1580. if (ring->launchtime_enable) {
  1581. /* Always set DataTranTIM on TQAVCTRL if LaunchTime is enabled
  1582. * for any of the SR queues, and configure fetchtime delta.
  1583. * XXX NOTE:
  1584. * - LaunchTime will be enabled for all SR queues.
  1585. * - A fixed offset can be added relative to the launch
  1586. * time of all packets if configured at reg LAUNCH_OS0.
  1587. * We are keeping it as 0 for now (default value).
  1588. */
  1589. tqavctrl = rd32(E1000_I210_TQAVCTRL);
  1590. tqavctrl |= E1000_TQAVCTRL_DATATRANTIM |
  1591. E1000_TQAVCTRL_FETCHTIME_DELTA;
  1592. wr32(E1000_I210_TQAVCTRL, tqavctrl);
  1593. } else {
  1594. /* If Launchtime is not enabled for any SR queues anymore,
  1595. * then clear DataTranTIM on TQAVCTRL and clear fetchtime delta,
  1596. * effectively disabling Launchtime.
  1597. */
  1598. if (!is_any_txtime_enabled(adapter)) {
  1599. tqavctrl = rd32(E1000_I210_TQAVCTRL);
  1600. tqavctrl &= ~E1000_TQAVCTRL_DATATRANTIM;
  1601. tqavctrl &= ~E1000_TQAVCTRL_FETCHTIME_DELTA;
  1602. wr32(E1000_I210_TQAVCTRL, tqavctrl);
  1603. }
  1604. }
  1605. /* XXX: In i210 controller the sendSlope and loCredit parameters from
  1606. * CBS are not configurable by software so we don't do any 'controller
  1607. * configuration' in respect to these parameters.
  1608. */
  1609. netdev_dbg(netdev, "Qav Tx mode: cbs %s, launchtime %s, queue %d idleslope %d sendslope %d hiCredit %d locredit %d\n",
  1610. ring->cbs_enable ? "enabled" : "disabled",
  1611. ring->launchtime_enable ? "enabled" : "disabled",
  1612. queue,
  1613. ring->idleslope, ring->sendslope,
  1614. ring->hicredit, ring->locredit);
  1615. }
  1616. static int igb_save_txtime_params(struct igb_adapter *adapter, int queue,
  1617. bool enable)
  1618. {
  1619. struct igb_ring *ring;
  1620. if (queue < 0 || queue > adapter->num_tx_queues)
  1621. return -EINVAL;
  1622. ring = adapter->tx_ring[queue];
  1623. ring->launchtime_enable = enable;
  1624. return 0;
  1625. }
  1626. static int igb_save_cbs_params(struct igb_adapter *adapter, int queue,
  1627. bool enable, int idleslope, int sendslope,
  1628. int hicredit, int locredit)
  1629. {
  1630. struct igb_ring *ring;
  1631. if (queue < 0 || queue > adapter->num_tx_queues)
  1632. return -EINVAL;
  1633. ring = adapter->tx_ring[queue];
  1634. ring->cbs_enable = enable;
  1635. ring->idleslope = idleslope;
  1636. ring->sendslope = sendslope;
  1637. ring->hicredit = hicredit;
  1638. ring->locredit = locredit;
  1639. return 0;
  1640. }
  1641. /**
  1642. * igb_setup_tx_mode - Switch to/from Qav Tx mode when applicable
  1643. * @adapter: pointer to adapter struct
  1644. *
  1645. * Configure TQAVCTRL register switching the controller's Tx mode
  1646. * if FQTSS mode is enabled or disabled. Additionally, will issue
  1647. * a call to igb_config_tx_modes() per queue so any previously saved
  1648. * Tx parameters are applied.
  1649. **/
  1650. static void igb_setup_tx_mode(struct igb_adapter *adapter)
  1651. {
  1652. struct net_device *netdev = adapter->netdev;
  1653. struct e1000_hw *hw = &adapter->hw;
  1654. u32 val;
  1655. /* Only i210 controller supports changing the transmission mode. */
  1656. if (hw->mac.type != e1000_i210)
  1657. return;
  1658. if (is_fqtss_enabled(adapter)) {
  1659. int i, max_queue;
  1660. /* Configure TQAVCTRL register: set transmit mode to 'Qav',
  1661. * set data fetch arbitration to 'round robin', set SP_WAIT_SR
  1662. * so SP queues wait for SR ones.
  1663. */
  1664. val = rd32(E1000_I210_TQAVCTRL);
  1665. val |= E1000_TQAVCTRL_XMIT_MODE | E1000_TQAVCTRL_SP_WAIT_SR;
  1666. val &= ~E1000_TQAVCTRL_DATAFETCHARB;
  1667. wr32(E1000_I210_TQAVCTRL, val);
  1668. /* Configure Tx and Rx packet buffers sizes as described in
  1669. * i210 datasheet section 7.2.7.7.
  1670. */
  1671. val = rd32(E1000_TXPBS);
  1672. val &= ~I210_TXPBSIZE_MASK;
  1673. val |= I210_TXPBSIZE_PB0_8KB | I210_TXPBSIZE_PB1_8KB |
  1674. I210_TXPBSIZE_PB2_4KB | I210_TXPBSIZE_PB3_4KB;
  1675. wr32(E1000_TXPBS, val);
  1676. val = rd32(E1000_RXPBS);
  1677. val &= ~I210_RXPBSIZE_MASK;
  1678. val |= I210_RXPBSIZE_PB_30KB;
  1679. wr32(E1000_RXPBS, val);
  1680. /* Section 8.12.9 states that MAX_TPKT_SIZE from DTXMXPKTSZ
  1681. * register should not exceed the buffer size programmed in
  1682. * TXPBS. The smallest buffer size programmed in TXPBS is 4kB
  1683. * so according to the datasheet we should set MAX_TPKT_SIZE to
  1684. * 4kB / 64.
  1685. *
  1686. * However, when we do so, no frame from queue 2 and 3 are
  1687. * transmitted. It seems the MAX_TPKT_SIZE should not be great
  1688. * or _equal_ to the buffer size programmed in TXPBS. For this
  1689. * reason, we set set MAX_ TPKT_SIZE to (4kB - 1) / 64.
  1690. */
  1691. val = (4096 - 1) / 64;
  1692. wr32(E1000_I210_DTXMXPKTSZ, val);
  1693. /* Since FQTSS mode is enabled, apply any CBS configuration
  1694. * previously set. If no previous CBS configuration has been
  1695. * done, then the initial configuration is applied, which means
  1696. * CBS is disabled.
  1697. */
  1698. max_queue = (adapter->num_tx_queues < I210_SR_QUEUES_NUM) ?
  1699. adapter->num_tx_queues : I210_SR_QUEUES_NUM;
  1700. for (i = 0; i < max_queue; i++) {
  1701. igb_config_tx_modes(adapter, i);
  1702. }
  1703. } else {
  1704. wr32(E1000_RXPBS, I210_RXPBSIZE_DEFAULT);
  1705. wr32(E1000_TXPBS, I210_TXPBSIZE_DEFAULT);
  1706. wr32(E1000_I210_DTXMXPKTSZ, I210_DTXMXPKTSZ_DEFAULT);
  1707. val = rd32(E1000_I210_TQAVCTRL);
  1708. /* According to Section 8.12.21, the other flags we've set when
  1709. * enabling FQTSS are not relevant when disabling FQTSS so we
  1710. * don't set they here.
  1711. */
  1712. val &= ~E1000_TQAVCTRL_XMIT_MODE;
  1713. wr32(E1000_I210_TQAVCTRL, val);
  1714. }
  1715. netdev_dbg(netdev, "FQTSS %s\n", (is_fqtss_enabled(adapter)) ?
  1716. "enabled" : "disabled");
  1717. }
  1718. /**
  1719. * igb_configure - configure the hardware for RX and TX
  1720. * @adapter: private board structure
  1721. **/
  1722. static void igb_configure(struct igb_adapter *adapter)
  1723. {
  1724. struct net_device *netdev = adapter->netdev;
  1725. int i;
  1726. igb_get_hw_control(adapter);
  1727. igb_set_rx_mode(netdev);
  1728. igb_setup_tx_mode(adapter);
  1729. igb_restore_vlan(adapter);
  1730. igb_setup_tctl(adapter);
  1731. igb_setup_mrqc(adapter);
  1732. igb_setup_rctl(adapter);
  1733. igb_nfc_filter_restore(adapter);
  1734. igb_configure_tx(adapter);
  1735. igb_configure_rx(adapter);
  1736. igb_rx_fifo_flush_82575(&adapter->hw);
  1737. /* call igb_desc_unused which always leaves
  1738. * at least 1 descriptor unused to make sure
  1739. * next_to_use != next_to_clean
  1740. */
  1741. for (i = 0; i < adapter->num_rx_queues; i++) {
  1742. struct igb_ring *ring = adapter->rx_ring[i];
  1743. igb_alloc_rx_buffers(ring, igb_desc_unused(ring));
  1744. }
  1745. }
  1746. /**
  1747. * igb_power_up_link - Power up the phy/serdes link
  1748. * @adapter: address of board private structure
  1749. **/
  1750. void igb_power_up_link(struct igb_adapter *adapter)
  1751. {
  1752. igb_reset_phy(&adapter->hw);
  1753. if (adapter->hw.phy.media_type == e1000_media_type_copper)
  1754. igb_power_up_phy_copper(&adapter->hw);
  1755. else
  1756. igb_power_up_serdes_link_82575(&adapter->hw);
  1757. igb_setup_link(&adapter->hw);
  1758. }
  1759. /**
  1760. * igb_power_down_link - Power down the phy/serdes link
  1761. * @adapter: address of board private structure
  1762. */
  1763. static void igb_power_down_link(struct igb_adapter *adapter)
  1764. {
  1765. if (adapter->hw.phy.media_type == e1000_media_type_copper)
  1766. igb_power_down_phy_copper_82575(&adapter->hw);
  1767. else
  1768. igb_shutdown_serdes_link_82575(&adapter->hw);
  1769. }
  1770. /**
  1771. * Detect and switch function for Media Auto Sense
  1772. * @adapter: address of the board private structure
  1773. **/
  1774. static void igb_check_swap_media(struct igb_adapter *adapter)
  1775. {
  1776. struct e1000_hw *hw = &adapter->hw;
  1777. u32 ctrl_ext, connsw;
  1778. bool swap_now = false;
  1779. ctrl_ext = rd32(E1000_CTRL_EXT);
  1780. connsw = rd32(E1000_CONNSW);
  1781. /* need to live swap if current media is copper and we have fiber/serdes
  1782. * to go to.
  1783. */
  1784. if ((hw->phy.media_type == e1000_media_type_copper) &&
  1785. (!(connsw & E1000_CONNSW_AUTOSENSE_EN))) {
  1786. swap_now = true;
  1787. } else if ((hw->phy.media_type != e1000_media_type_copper) &&
  1788. !(connsw & E1000_CONNSW_SERDESD)) {
  1789. /* copper signal takes time to appear */
  1790. if (adapter->copper_tries < 4) {
  1791. adapter->copper_tries++;
  1792. connsw |= E1000_CONNSW_AUTOSENSE_CONF;
  1793. wr32(E1000_CONNSW, connsw);
  1794. return;
  1795. } else {
  1796. adapter->copper_tries = 0;
  1797. if ((connsw & E1000_CONNSW_PHYSD) &&
  1798. (!(connsw & E1000_CONNSW_PHY_PDN))) {
  1799. swap_now = true;
  1800. connsw &= ~E1000_CONNSW_AUTOSENSE_CONF;
  1801. wr32(E1000_CONNSW, connsw);
  1802. }
  1803. }
  1804. }
  1805. if (!swap_now)
  1806. return;
  1807. switch (hw->phy.media_type) {
  1808. case e1000_media_type_copper:
  1809. netdev_info(adapter->netdev,
  1810. "MAS: changing media to fiber/serdes\n");
  1811. ctrl_ext |=
  1812. E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES;
  1813. adapter->flags |= IGB_FLAG_MEDIA_RESET;
  1814. adapter->copper_tries = 0;
  1815. break;
  1816. case e1000_media_type_internal_serdes:
  1817. case e1000_media_type_fiber:
  1818. netdev_info(adapter->netdev,
  1819. "MAS: changing media to copper\n");
  1820. ctrl_ext &=
  1821. ~E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES;
  1822. adapter->flags |= IGB_FLAG_MEDIA_RESET;
  1823. break;
  1824. default:
  1825. /* shouldn't get here during regular operation */
  1826. netdev_err(adapter->netdev,
  1827. "AMS: Invalid media type found, returning\n");
  1828. break;
  1829. }
  1830. wr32(E1000_CTRL_EXT, ctrl_ext);
  1831. }
  1832. /**
  1833. * igb_up - Open the interface and prepare it to handle traffic
  1834. * @adapter: board private structure
  1835. **/
  1836. int igb_up(struct igb_adapter *adapter)
  1837. {
  1838. struct e1000_hw *hw = &adapter->hw;
  1839. int i;
  1840. /* hardware has been reset, we need to reload some things */
  1841. igb_configure(adapter);
  1842. clear_bit(__IGB_DOWN, &adapter->state);
  1843. for (i = 0; i < adapter->num_q_vectors; i++)
  1844. napi_enable(&(adapter->q_vector[i]->napi));
  1845. if (adapter->flags & IGB_FLAG_HAS_MSIX)
  1846. igb_configure_msix(adapter);
  1847. else
  1848. igb_assign_vector(adapter->q_vector[0], 0);
  1849. /* Clear any pending interrupts. */
  1850. rd32(E1000_TSICR);
  1851. rd32(E1000_ICR);
  1852. igb_irq_enable(adapter);
  1853. /* notify VFs that reset has been completed */
  1854. if (adapter->vfs_allocated_count) {
  1855. u32 reg_data = rd32(E1000_CTRL_EXT);
  1856. reg_data |= E1000_CTRL_EXT_PFRSTD;
  1857. wr32(E1000_CTRL_EXT, reg_data);
  1858. }
  1859. netif_tx_start_all_queues(adapter->netdev);
  1860. /* start the watchdog. */
  1861. hw->mac.get_link_status = 1;
  1862. schedule_work(&adapter->watchdog_task);
  1863. if ((adapter->flags & IGB_FLAG_EEE) &&
  1864. (!hw->dev_spec._82575.eee_disable))
  1865. adapter->eee_advert = MDIO_EEE_100TX | MDIO_EEE_1000T;
  1866. return 0;
  1867. }
  1868. void igb_down(struct igb_adapter *adapter)
  1869. {
  1870. struct net_device *netdev = adapter->netdev;
  1871. struct e1000_hw *hw = &adapter->hw;
  1872. u32 tctl, rctl;
  1873. int i;
  1874. /* signal that we're down so the interrupt handler does not
  1875. * reschedule our watchdog timer
  1876. */
  1877. set_bit(__IGB_DOWN, &adapter->state);
  1878. /* disable receives in the hardware */
  1879. rctl = rd32(E1000_RCTL);
  1880. wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
  1881. /* flush and sleep below */
  1882. igb_nfc_filter_exit(adapter);
  1883. netif_carrier_off(netdev);
  1884. netif_tx_stop_all_queues(netdev);
  1885. /* disable transmits in the hardware */
  1886. tctl = rd32(E1000_TCTL);
  1887. tctl &= ~E1000_TCTL_EN;
  1888. wr32(E1000_TCTL, tctl);
  1889. /* flush both disables and wait for them to finish */
  1890. wrfl();
  1891. usleep_range(10000, 11000);
  1892. igb_irq_disable(adapter);
  1893. adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE;
  1894. for (i = 0; i < adapter->num_q_vectors; i++) {
  1895. if (adapter->q_vector[i]) {
  1896. napi_synchronize(&adapter->q_vector[i]->napi);
  1897. napi_disable(&adapter->q_vector[i]->napi);
  1898. }
  1899. }
  1900. del_timer_sync(&adapter->watchdog_timer);
  1901. del_timer_sync(&adapter->phy_info_timer);
  1902. /* record the stats before reset*/
  1903. spin_lock(&adapter->stats64_lock);
  1904. igb_update_stats(adapter);
  1905. spin_unlock(&adapter->stats64_lock);
  1906. adapter->link_speed = 0;
  1907. adapter->link_duplex = 0;
  1908. if (!pci_channel_offline(adapter->pdev))
  1909. igb_reset(adapter);
  1910. /* clear VLAN promisc flag so VFTA will be updated if necessary */
  1911. adapter->flags &= ~IGB_FLAG_VLAN_PROMISC;
  1912. igb_clean_all_tx_rings(adapter);
  1913. igb_clean_all_rx_rings(adapter);
  1914. #ifdef CONFIG_IGB_DCA
  1915. /* since we reset the hardware DCA settings were cleared */
  1916. igb_setup_dca(adapter);
  1917. #endif
  1918. }
  1919. void igb_reinit_locked(struct igb_adapter *adapter)
  1920. {
  1921. WARN_ON(in_interrupt());
  1922. while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
  1923. usleep_range(1000, 2000);
  1924. igb_down(adapter);
  1925. igb_up(adapter);
  1926. clear_bit(__IGB_RESETTING, &adapter->state);
  1927. }
  1928. /** igb_enable_mas - Media Autosense re-enable after swap
  1929. *
  1930. * @adapter: adapter struct
  1931. **/
  1932. static void igb_enable_mas(struct igb_adapter *adapter)
  1933. {
  1934. struct e1000_hw *hw = &adapter->hw;
  1935. u32 connsw = rd32(E1000_CONNSW);
  1936. /* configure for SerDes media detect */
  1937. if ((hw->phy.media_type == e1000_media_type_copper) &&
  1938. (!(connsw & E1000_CONNSW_SERDESD))) {
  1939. connsw |= E1000_CONNSW_ENRGSRC;
  1940. connsw |= E1000_CONNSW_AUTOSENSE_EN;
  1941. wr32(E1000_CONNSW, connsw);
  1942. wrfl();
  1943. }
  1944. }
  1945. void igb_reset(struct igb_adapter *adapter)
  1946. {
  1947. struct pci_dev *pdev = adapter->pdev;
  1948. struct e1000_hw *hw = &adapter->hw;
  1949. struct e1000_mac_info *mac = &hw->mac;
  1950. struct e1000_fc_info *fc = &hw->fc;
  1951. u32 pba, hwm;
  1952. /* Repartition Pba for greater than 9k mtu
  1953. * To take effect CTRL.RST is required.
  1954. */
  1955. switch (mac->type) {
  1956. case e1000_i350:
  1957. case e1000_i354:
  1958. case e1000_82580:
  1959. pba = rd32(E1000_RXPBS);
  1960. pba = igb_rxpbs_adjust_82580(pba);
  1961. break;
  1962. case e1000_82576:
  1963. pba = rd32(E1000_RXPBS);
  1964. pba &= E1000_RXPBS_SIZE_MASK_82576;
  1965. break;
  1966. case e1000_82575:
  1967. case e1000_i210:
  1968. case e1000_i211:
  1969. default:
  1970. pba = E1000_PBA_34K;
  1971. break;
  1972. }
  1973. if (mac->type == e1000_82575) {
  1974. u32 min_rx_space, min_tx_space, needed_tx_space;
  1975. /* write Rx PBA so that hardware can report correct Tx PBA */
  1976. wr32(E1000_PBA, pba);
  1977. /* To maintain wire speed transmits, the Tx FIFO should be
  1978. * large enough to accommodate two full transmit packets,
  1979. * rounded up to the next 1KB and expressed in KB. Likewise,
  1980. * the Rx FIFO should be large enough to accommodate at least
  1981. * one full receive packet and is similarly rounded up and
  1982. * expressed in KB.
  1983. */
  1984. min_rx_space = DIV_ROUND_UP(MAX_JUMBO_FRAME_SIZE, 1024);
  1985. /* The Tx FIFO also stores 16 bytes of information about the Tx
  1986. * but don't include Ethernet FCS because hardware appends it.
  1987. * We only need to round down to the nearest 512 byte block
  1988. * count since the value we care about is 2 frames, not 1.
  1989. */
  1990. min_tx_space = adapter->max_frame_size;
  1991. min_tx_space += sizeof(union e1000_adv_tx_desc) - ETH_FCS_LEN;
  1992. min_tx_space = DIV_ROUND_UP(min_tx_space, 512);
  1993. /* upper 16 bits has Tx packet buffer allocation size in KB */
  1994. needed_tx_space = min_tx_space - (rd32(E1000_PBA) >> 16);
  1995. /* If current Tx allocation is less than the min Tx FIFO size,
  1996. * and the min Tx FIFO size is less than the current Rx FIFO
  1997. * allocation, take space away from current Rx allocation.
  1998. */
  1999. if (needed_tx_space < pba) {
  2000. pba -= needed_tx_space;
  2001. /* if short on Rx space, Rx wins and must trump Tx
  2002. * adjustment
  2003. */
  2004. if (pba < min_rx_space)
  2005. pba = min_rx_space;
  2006. }
  2007. /* adjust PBA for jumbo frames */
  2008. wr32(E1000_PBA, pba);
  2009. }
  2010. /* flow control settings
  2011. * The high water mark must be low enough to fit one full frame
  2012. * after transmitting the pause frame. As such we must have enough
  2013. * space to allow for us to complete our current transmit and then
  2014. * receive the frame that is in progress from the link partner.
  2015. * Set it to:
  2016. * - the full Rx FIFO size minus one full Tx plus one full Rx frame
  2017. */
  2018. hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE);
  2019. fc->high_water = hwm & 0xFFFFFFF0; /* 16-byte granularity */
  2020. fc->low_water = fc->high_water - 16;
  2021. fc->pause_time = 0xFFFF;
  2022. fc->send_xon = 1;
  2023. fc->current_mode = fc->requested_mode;
  2024. /* disable receive for all VFs and wait one second */
  2025. if (adapter->vfs_allocated_count) {
  2026. int i;
  2027. for (i = 0 ; i < adapter->vfs_allocated_count; i++)
  2028. adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
  2029. /* ping all the active vfs to let them know we are going down */
  2030. igb_ping_all_vfs(adapter);
  2031. /* disable transmits and receives */
  2032. wr32(E1000_VFRE, 0);
  2033. wr32(E1000_VFTE, 0);
  2034. }
  2035. /* Allow time for pending master requests to run */
  2036. hw->mac.ops.reset_hw(hw);
  2037. wr32(E1000_WUC, 0);
  2038. if (adapter->flags & IGB_FLAG_MEDIA_RESET) {
  2039. /* need to resetup here after media swap */
  2040. adapter->ei.get_invariants(hw);
  2041. adapter->flags &= ~IGB_FLAG_MEDIA_RESET;
  2042. }
  2043. if ((mac->type == e1000_82575 || mac->type == e1000_i350) &&
  2044. (adapter->flags & IGB_FLAG_MAS_ENABLE)) {
  2045. igb_enable_mas(adapter);
  2046. }
  2047. if (hw->mac.ops.init_hw(hw))
  2048. dev_err(&pdev->dev, "Hardware Error\n");
  2049. /* RAR registers were cleared during init_hw, clear mac table */
  2050. igb_flush_mac_table(adapter);
  2051. __dev_uc_unsync(adapter->netdev, NULL);
  2052. /* Recover default RAR entry */
  2053. igb_set_default_mac_filter(adapter);
  2054. /* Flow control settings reset on hardware reset, so guarantee flow
  2055. * control is off when forcing speed.
  2056. */
  2057. if (!hw->mac.autoneg)
  2058. igb_force_mac_fc(hw);
  2059. igb_init_dmac(adapter, pba);
  2060. #ifdef CONFIG_IGB_HWMON
  2061. /* Re-initialize the thermal sensor on i350 devices. */
  2062. if (!test_bit(__IGB_DOWN, &adapter->state)) {
  2063. if (mac->type == e1000_i350 && hw->bus.func == 0) {
  2064. /* If present, re-initialize the external thermal sensor
  2065. * interface.
  2066. */
  2067. if (adapter->ets)
  2068. mac->ops.init_thermal_sensor_thresh(hw);
  2069. }
  2070. }
  2071. #endif
  2072. /* Re-establish EEE setting */
  2073. if (hw->phy.media_type == e1000_media_type_copper) {
  2074. switch (mac->type) {
  2075. case e1000_i350:
  2076. case e1000_i210:
  2077. case e1000_i211:
  2078. igb_set_eee_i350(hw, true, true);
  2079. break;
  2080. case e1000_i354:
  2081. igb_set_eee_i354(hw, true, true);
  2082. break;
  2083. default:
  2084. break;
  2085. }
  2086. }
  2087. if (!netif_running(adapter->netdev))
  2088. igb_power_down_link(adapter);
  2089. igb_update_mng_vlan(adapter);
  2090. /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
  2091. wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
  2092. /* Re-enable PTP, where applicable. */
  2093. if (adapter->ptp_flags & IGB_PTP_ENABLED)
  2094. igb_ptp_reset(adapter);
  2095. igb_get_phy_info(hw);
  2096. }
  2097. static netdev_features_t igb_fix_features(struct net_device *netdev,
  2098. netdev_features_t features)
  2099. {
  2100. /* Since there is no support for separate Rx/Tx vlan accel
  2101. * enable/disable make sure Tx flag is always in same state as Rx.
  2102. */
  2103. if (features & NETIF_F_HW_VLAN_CTAG_RX)
  2104. features |= NETIF_F_HW_VLAN_CTAG_TX;
  2105. else
  2106. features &= ~NETIF_F_HW_VLAN_CTAG_TX;
  2107. return features;
  2108. }
  2109. static int igb_set_features(struct net_device *netdev,
  2110. netdev_features_t features)
  2111. {
  2112. netdev_features_t changed = netdev->features ^ features;
  2113. struct igb_adapter *adapter = netdev_priv(netdev);
  2114. if (changed & NETIF_F_HW_VLAN_CTAG_RX)
  2115. igb_vlan_mode(netdev, features);
  2116. if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE)))
  2117. return 0;
  2118. if (!(features & NETIF_F_NTUPLE)) {
  2119. struct hlist_node *node2;
  2120. struct igb_nfc_filter *rule;
  2121. spin_lock(&adapter->nfc_lock);
  2122. hlist_for_each_entry_safe(rule, node2,
  2123. &adapter->nfc_filter_list, nfc_node) {
  2124. igb_erase_filter(adapter, rule);
  2125. hlist_del(&rule->nfc_node);
  2126. kfree(rule);
  2127. }
  2128. spin_unlock(&adapter->nfc_lock);
  2129. adapter->nfc_filter_count = 0;
  2130. }
  2131. netdev->features = features;
  2132. if (netif_running(netdev))
  2133. igb_reinit_locked(adapter);
  2134. else
  2135. igb_reset(adapter);
  2136. return 1;
  2137. }
  2138. static int igb_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
  2139. struct net_device *dev,
  2140. const unsigned char *addr, u16 vid,
  2141. u16 flags,
  2142. struct netlink_ext_ack *extack)
  2143. {
  2144. /* guarantee we can provide a unique filter for the unicast address */
  2145. if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) {
  2146. struct igb_adapter *adapter = netdev_priv(dev);
  2147. int vfn = adapter->vfs_allocated_count;
  2148. if (netdev_uc_count(dev) >= igb_available_rars(adapter, vfn))
  2149. return -ENOMEM;
  2150. }
  2151. return ndo_dflt_fdb_add(ndm, tb, dev, addr, vid, flags);
  2152. }
  2153. #define IGB_MAX_MAC_HDR_LEN 127
  2154. #define IGB_MAX_NETWORK_HDR_LEN 511
  2155. static netdev_features_t
  2156. igb_features_check(struct sk_buff *skb, struct net_device *dev,
  2157. netdev_features_t features)
  2158. {
  2159. unsigned int network_hdr_len, mac_hdr_len;
  2160. /* Make certain the headers can be described by a context descriptor */
  2161. mac_hdr_len = skb_network_header(skb) - skb->data;
  2162. if (unlikely(mac_hdr_len > IGB_MAX_MAC_HDR_LEN))
  2163. return features & ~(NETIF_F_HW_CSUM |
  2164. NETIF_F_SCTP_CRC |
  2165. NETIF_F_GSO_UDP_L4 |
  2166. NETIF_F_HW_VLAN_CTAG_TX |
  2167. NETIF_F_TSO |
  2168. NETIF_F_TSO6);
  2169. network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb);
  2170. if (unlikely(network_hdr_len > IGB_MAX_NETWORK_HDR_LEN))
  2171. return features & ~(NETIF_F_HW_CSUM |
  2172. NETIF_F_SCTP_CRC |
  2173. NETIF_F_GSO_UDP_L4 |
  2174. NETIF_F_TSO |
  2175. NETIF_F_TSO6);
  2176. /* We can only support IPV4 TSO in tunnels if we can mangle the
  2177. * inner IP ID field, so strip TSO if MANGLEID is not supported.
  2178. */
  2179. if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID))
  2180. features &= ~NETIF_F_TSO;
  2181. return features;
  2182. }
  2183. static void igb_offload_apply(struct igb_adapter *adapter, s32 queue)
  2184. {
  2185. if (!is_fqtss_enabled(adapter)) {
  2186. enable_fqtss(adapter, true);
  2187. return;
  2188. }
  2189. igb_config_tx_modes(adapter, queue);
  2190. if (!is_any_cbs_enabled(adapter) && !is_any_txtime_enabled(adapter))
  2191. enable_fqtss(adapter, false);
  2192. }
  2193. static int igb_offload_cbs(struct igb_adapter *adapter,
  2194. struct tc_cbs_qopt_offload *qopt)
  2195. {
  2196. struct e1000_hw *hw = &adapter->hw;
  2197. int err;
  2198. /* CBS offloading is only supported by i210 controller. */
  2199. if (hw->mac.type != e1000_i210)
  2200. return -EOPNOTSUPP;
  2201. /* CBS offloading is only supported by queue 0 and queue 1. */
  2202. if (qopt->queue < 0 || qopt->queue > 1)
  2203. return -EINVAL;
  2204. err = igb_save_cbs_params(adapter, qopt->queue, qopt->enable,
  2205. qopt->idleslope, qopt->sendslope,
  2206. qopt->hicredit, qopt->locredit);
  2207. if (err)
  2208. return err;
  2209. igb_offload_apply(adapter, qopt->queue);
  2210. return 0;
  2211. }
  2212. #define ETHER_TYPE_FULL_MASK ((__force __be16)~0)
  2213. #define VLAN_PRIO_FULL_MASK (0x07)
  2214. static int igb_parse_cls_flower(struct igb_adapter *adapter,
  2215. struct flow_cls_offload *f,
  2216. int traffic_class,
  2217. struct igb_nfc_filter *input)
  2218. {
  2219. struct flow_rule *rule = flow_cls_offload_flow_rule(f);
  2220. struct flow_dissector *dissector = rule->match.dissector;
  2221. struct netlink_ext_ack *extack = f->common.extack;
  2222. if (dissector->used_keys &
  2223. ~(BIT(FLOW_DISSECTOR_KEY_BASIC) |
  2224. BIT(FLOW_DISSECTOR_KEY_CONTROL) |
  2225. BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
  2226. BIT(FLOW_DISSECTOR_KEY_VLAN))) {
  2227. NL_SET_ERR_MSG_MOD(extack,
  2228. "Unsupported key used, only BASIC, CONTROL, ETH_ADDRS and VLAN are supported");
  2229. return -EOPNOTSUPP;
  2230. }
  2231. if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
  2232. struct flow_match_eth_addrs match;
  2233. flow_rule_match_eth_addrs(rule, &match);
  2234. if (!is_zero_ether_addr(match.mask->dst)) {
  2235. if (!is_broadcast_ether_addr(match.mask->dst)) {
  2236. NL_SET_ERR_MSG_MOD(extack, "Only full masks are supported for destination MAC address");
  2237. return -EINVAL;
  2238. }
  2239. input->filter.match_flags |=
  2240. IGB_FILTER_FLAG_DST_MAC_ADDR;
  2241. ether_addr_copy(input->filter.dst_addr, match.key->dst);
  2242. }
  2243. if (!is_zero_ether_addr(match.mask->src)) {
  2244. if (!is_broadcast_ether_addr(match.mask->src)) {
  2245. NL_SET_ERR_MSG_MOD(extack, "Only full masks are supported for source MAC address");
  2246. return -EINVAL;
  2247. }
  2248. input->filter.match_flags |=
  2249. IGB_FILTER_FLAG_SRC_MAC_ADDR;
  2250. ether_addr_copy(input->filter.src_addr, match.key->src);
  2251. }
  2252. }
  2253. if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
  2254. struct flow_match_basic match;
  2255. flow_rule_match_basic(rule, &match);
  2256. if (match.mask->n_proto) {
  2257. if (match.mask->n_proto != ETHER_TYPE_FULL_MASK) {
  2258. NL_SET_ERR_MSG_MOD(extack, "Only full mask is supported for EtherType filter");
  2259. return -EINVAL;
  2260. }
  2261. input->filter.match_flags |= IGB_FILTER_FLAG_ETHER_TYPE;
  2262. input->filter.etype = match.key->n_proto;
  2263. }
  2264. }
  2265. if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
  2266. struct flow_match_vlan match;
  2267. flow_rule_match_vlan(rule, &match);
  2268. if (match.mask->vlan_priority) {
  2269. if (match.mask->vlan_priority != VLAN_PRIO_FULL_MASK) {
  2270. NL_SET_ERR_MSG_MOD(extack, "Only full mask is supported for VLAN priority");
  2271. return -EINVAL;
  2272. }
  2273. input->filter.match_flags |= IGB_FILTER_FLAG_VLAN_TCI;
  2274. input->filter.vlan_tci = match.key->vlan_priority;
  2275. }
  2276. }
  2277. input->action = traffic_class;
  2278. input->cookie = f->cookie;
  2279. return 0;
  2280. }
  2281. static int igb_configure_clsflower(struct igb_adapter *adapter,
  2282. struct flow_cls_offload *cls_flower)
  2283. {
  2284. struct netlink_ext_ack *extack = cls_flower->common.extack;
  2285. struct igb_nfc_filter *filter, *f;
  2286. int err, tc;
  2287. tc = tc_classid_to_hwtc(adapter->netdev, cls_flower->classid);
  2288. if (tc < 0) {
  2289. NL_SET_ERR_MSG_MOD(extack, "Invalid traffic class");
  2290. return -EINVAL;
  2291. }
  2292. filter = kzalloc(sizeof(*filter), GFP_KERNEL);
  2293. if (!filter)
  2294. return -ENOMEM;
  2295. err = igb_parse_cls_flower(adapter, cls_flower, tc, filter);
  2296. if (err < 0)
  2297. goto err_parse;
  2298. spin_lock(&adapter->nfc_lock);
  2299. hlist_for_each_entry(f, &adapter->nfc_filter_list, nfc_node) {
  2300. if (!memcmp(&f->filter, &filter->filter, sizeof(f->filter))) {
  2301. err = -EEXIST;
  2302. NL_SET_ERR_MSG_MOD(extack,
  2303. "This filter is already set in ethtool");
  2304. goto err_locked;
  2305. }
  2306. }
  2307. hlist_for_each_entry(f, &adapter->cls_flower_list, nfc_node) {
  2308. if (!memcmp(&f->filter, &filter->filter, sizeof(f->filter))) {
  2309. err = -EEXIST;
  2310. NL_SET_ERR_MSG_MOD(extack,
  2311. "This filter is already set in cls_flower");
  2312. goto err_locked;
  2313. }
  2314. }
  2315. err = igb_add_filter(adapter, filter);
  2316. if (err < 0) {
  2317. NL_SET_ERR_MSG_MOD(extack, "Could not add filter to the adapter");
  2318. goto err_locked;
  2319. }
  2320. hlist_add_head(&filter->nfc_node, &adapter->cls_flower_list);
  2321. spin_unlock(&adapter->nfc_lock);
  2322. return 0;
  2323. err_locked:
  2324. spin_unlock(&adapter->nfc_lock);
  2325. err_parse:
  2326. kfree(filter);
  2327. return err;
  2328. }
  2329. static int igb_delete_clsflower(struct igb_adapter *adapter,
  2330. struct flow_cls_offload *cls_flower)
  2331. {
  2332. struct igb_nfc_filter *filter;
  2333. int err;
  2334. spin_lock(&adapter->nfc_lock);
  2335. hlist_for_each_entry(filter, &adapter->cls_flower_list, nfc_node)
  2336. if (filter->cookie == cls_flower->cookie)
  2337. break;
  2338. if (!filter) {
  2339. err = -ENOENT;
  2340. goto out;
  2341. }
  2342. err = igb_erase_filter(adapter, filter);
  2343. if (err < 0)
  2344. goto out;
  2345. hlist_del(&filter->nfc_node);
  2346. kfree(filter);
  2347. out:
  2348. spin_unlock(&adapter->nfc_lock);
  2349. return err;
  2350. }
  2351. static int igb_setup_tc_cls_flower(struct igb_adapter *adapter,
  2352. struct flow_cls_offload *cls_flower)
  2353. {
  2354. switch (cls_flower->command) {
  2355. case FLOW_CLS_REPLACE:
  2356. return igb_configure_clsflower(adapter, cls_flower);
  2357. case FLOW_CLS_DESTROY:
  2358. return igb_delete_clsflower(adapter, cls_flower);
  2359. case FLOW_CLS_STATS:
  2360. return -EOPNOTSUPP;
  2361. default:
  2362. return -EOPNOTSUPP;
  2363. }
  2364. }
  2365. static int igb_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
  2366. void *cb_priv)
  2367. {
  2368. struct igb_adapter *adapter = cb_priv;
  2369. if (!tc_cls_can_offload_and_chain0(adapter->netdev, type_data))
  2370. return -EOPNOTSUPP;
  2371. switch (type) {
  2372. case TC_SETUP_CLSFLOWER:
  2373. return igb_setup_tc_cls_flower(adapter, type_data);
  2374. default:
  2375. return -EOPNOTSUPP;
  2376. }
  2377. }
  2378. static int igb_offload_txtime(struct igb_adapter *adapter,
  2379. struct tc_etf_qopt_offload *qopt)
  2380. {
  2381. struct e1000_hw *hw = &adapter->hw;
  2382. int err;
  2383. /* Launchtime offloading is only supported by i210 controller. */
  2384. if (hw->mac.type != e1000_i210)
  2385. return -EOPNOTSUPP;
  2386. /* Launchtime offloading is only supported by queues 0 and 1. */
  2387. if (qopt->queue < 0 || qopt->queue > 1)
  2388. return -EINVAL;
  2389. err = igb_save_txtime_params(adapter, qopt->queue, qopt->enable);
  2390. if (err)
  2391. return err;
  2392. igb_offload_apply(adapter, qopt->queue);
  2393. return 0;
  2394. }
  2395. static LIST_HEAD(igb_block_cb_list);
  2396. static int igb_setup_tc(struct net_device *dev, enum tc_setup_type type,
  2397. void *type_data)
  2398. {
  2399. struct igb_adapter *adapter = netdev_priv(dev);
  2400. switch (type) {
  2401. case TC_SETUP_QDISC_CBS:
  2402. return igb_offload_cbs(adapter, type_data);
  2403. case TC_SETUP_BLOCK:
  2404. return flow_block_cb_setup_simple(type_data,
  2405. &igb_block_cb_list,
  2406. igb_setup_tc_block_cb,
  2407. adapter, adapter, true);
  2408. case TC_SETUP_QDISC_ETF:
  2409. return igb_offload_txtime(adapter, type_data);
  2410. default:
  2411. return -EOPNOTSUPP;
  2412. }
  2413. }
  2414. static const struct net_device_ops igb_netdev_ops = {
  2415. .ndo_open = igb_open,
  2416. .ndo_stop = igb_close,
  2417. .ndo_start_xmit = igb_xmit_frame,
  2418. .ndo_get_stats64 = igb_get_stats64,
  2419. .ndo_set_rx_mode = igb_set_rx_mode,
  2420. .ndo_set_mac_address = igb_set_mac,
  2421. .ndo_change_mtu = igb_change_mtu,
  2422. .ndo_do_ioctl = igb_ioctl,
  2423. .ndo_tx_timeout = igb_tx_timeout,
  2424. .ndo_validate_addr = eth_validate_addr,
  2425. .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
  2426. .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
  2427. .ndo_set_vf_mac = igb_ndo_set_vf_mac,
  2428. .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
  2429. .ndo_set_vf_rate = igb_ndo_set_vf_bw,
  2430. .ndo_set_vf_spoofchk = igb_ndo_set_vf_spoofchk,
  2431. .ndo_set_vf_trust = igb_ndo_set_vf_trust,
  2432. .ndo_get_vf_config = igb_ndo_get_vf_config,
  2433. .ndo_fix_features = igb_fix_features,
  2434. .ndo_set_features = igb_set_features,
  2435. .ndo_fdb_add = igb_ndo_fdb_add,
  2436. .ndo_features_check = igb_features_check,
  2437. .ndo_setup_tc = igb_setup_tc,
  2438. };
  2439. /**
  2440. * igb_set_fw_version - Configure version string for ethtool
  2441. * @adapter: adapter struct
  2442. **/
  2443. void igb_set_fw_version(struct igb_adapter *adapter)
  2444. {
  2445. struct e1000_hw *hw = &adapter->hw;
  2446. struct e1000_fw_version fw;
  2447. igb_get_fw_version(hw, &fw);
  2448. switch (hw->mac.type) {
  2449. case e1000_i210:
  2450. case e1000_i211:
  2451. if (!(igb_get_flash_presence_i210(hw))) {
  2452. snprintf(adapter->fw_version,
  2453. sizeof(adapter->fw_version),
  2454. "%2d.%2d-%d",
  2455. fw.invm_major, fw.invm_minor,
  2456. fw.invm_img_type);
  2457. break;
  2458. }
  2459. /* fall through */
  2460. default:
  2461. /* if option is rom valid, display its version too */
  2462. if (fw.or_valid) {
  2463. snprintf(adapter->fw_version,
  2464. sizeof(adapter->fw_version),
  2465. "%d.%d, 0x%08x, %d.%d.%d",
  2466. fw.eep_major, fw.eep_minor, fw.etrack_id,
  2467. fw.or_major, fw.or_build, fw.or_patch);
  2468. /* no option rom */
  2469. } else if (fw.etrack_id != 0X0000) {
  2470. snprintf(adapter->fw_version,
  2471. sizeof(adapter->fw_version),
  2472. "%d.%d, 0x%08x",
  2473. fw.eep_major, fw.eep_minor, fw.etrack_id);
  2474. } else {
  2475. snprintf(adapter->fw_version,
  2476. sizeof(adapter->fw_version),
  2477. "%d.%d.%d",
  2478. fw.eep_major, fw.eep_minor, fw.eep_build);
  2479. }
  2480. break;
  2481. }
  2482. }
  2483. /**
  2484. * igb_init_mas - init Media Autosense feature if enabled in the NVM
  2485. *
  2486. * @adapter: adapter struct
  2487. **/
  2488. static void igb_init_mas(struct igb_adapter *adapter)
  2489. {
  2490. struct e1000_hw *hw = &adapter->hw;
  2491. u16 eeprom_data;
  2492. hw->nvm.ops.read(hw, NVM_COMPAT, 1, &eeprom_data);
  2493. switch (hw->bus.func) {
  2494. case E1000_FUNC_0:
  2495. if (eeprom_data & IGB_MAS_ENABLE_0) {
  2496. adapter->flags |= IGB_FLAG_MAS_ENABLE;
  2497. netdev_info(adapter->netdev,
  2498. "MAS: Enabling Media Autosense for port %d\n",
  2499. hw->bus.func);
  2500. }
  2501. break;
  2502. case E1000_FUNC_1:
  2503. if (eeprom_data & IGB_MAS_ENABLE_1) {
  2504. adapter->flags |= IGB_FLAG_MAS_ENABLE;
  2505. netdev_info(adapter->netdev,
  2506. "MAS: Enabling Media Autosense for port %d\n",
  2507. hw->bus.func);
  2508. }
  2509. break;
  2510. case E1000_FUNC_2:
  2511. if (eeprom_data & IGB_MAS_ENABLE_2) {
  2512. adapter->flags |= IGB_FLAG_MAS_ENABLE;
  2513. netdev_info(adapter->netdev,
  2514. "MAS: Enabling Media Autosense for port %d\n",
  2515. hw->bus.func);
  2516. }
  2517. break;
  2518. case E1000_FUNC_3:
  2519. if (eeprom_data & IGB_MAS_ENABLE_3) {
  2520. adapter->flags |= IGB_FLAG_MAS_ENABLE;
  2521. netdev_info(adapter->netdev,
  2522. "MAS: Enabling Media Autosense for port %d\n",
  2523. hw->bus.func);
  2524. }
  2525. break;
  2526. default:
  2527. /* Shouldn't get here */
  2528. netdev_err(adapter->netdev,
  2529. "MAS: Invalid port configuration, returning\n");
  2530. break;
  2531. }
  2532. }
  2533. /**
  2534. * igb_init_i2c - Init I2C interface
  2535. * @adapter: pointer to adapter structure
  2536. **/
  2537. static s32 igb_init_i2c(struct igb_adapter *adapter)
  2538. {
  2539. s32 status = 0;
  2540. /* I2C interface supported on i350 devices */
  2541. if (adapter->hw.mac.type != e1000_i350)
  2542. return 0;
  2543. /* Initialize the i2c bus which is controlled by the registers.
  2544. * This bus will use the i2c_algo_bit structue that implements
  2545. * the protocol through toggling of the 4 bits in the register.
  2546. */
  2547. adapter->i2c_adap.owner = THIS_MODULE;
  2548. adapter->i2c_algo = igb_i2c_algo;
  2549. adapter->i2c_algo.data = adapter;
  2550. adapter->i2c_adap.algo_data = &adapter->i2c_algo;
  2551. adapter->i2c_adap.dev.parent = &adapter->pdev->dev;
  2552. strlcpy(adapter->i2c_adap.name, "igb BB",
  2553. sizeof(adapter->i2c_adap.name));
  2554. status = i2c_bit_add_bus(&adapter->i2c_adap);
  2555. return status;
  2556. }
  2557. /**
  2558. * igb_probe - Device Initialization Routine
  2559. * @pdev: PCI device information struct
  2560. * @ent: entry in igb_pci_tbl
  2561. *
  2562. * Returns 0 on success, negative on failure
  2563. *
  2564. * igb_probe initializes an adapter identified by a pci_dev structure.
  2565. * The OS initialization, configuring of the adapter private structure,
  2566. * and a hardware reset occur.
  2567. **/
  2568. static int igb_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
  2569. {
  2570. struct net_device *netdev;
  2571. struct igb_adapter *adapter;
  2572. struct e1000_hw *hw;
  2573. u16 eeprom_data = 0;
  2574. s32 ret_val;
  2575. static int global_quad_port_a; /* global quad port a indication */
  2576. const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
  2577. int err, pci_using_dac;
  2578. u8 part_str[E1000_PBANUM_LENGTH];
  2579. /* Catch broken hardware that put the wrong VF device ID in
  2580. * the PCIe SR-IOV capability.
  2581. */
  2582. if (pdev->is_virtfn) {
  2583. WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
  2584. pci_name(pdev), pdev->vendor, pdev->device);
  2585. return -EINVAL;
  2586. }
  2587. err = pci_enable_device_mem(pdev);
  2588. if (err)
  2589. return err;
  2590. pci_using_dac = 0;
  2591. err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
  2592. if (!err) {
  2593. pci_using_dac = 1;
  2594. } else {
  2595. err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
  2596. if (err) {
  2597. dev_err(&pdev->dev,
  2598. "No usable DMA configuration, aborting\n");
  2599. goto err_dma;
  2600. }
  2601. }
  2602. err = pci_request_mem_regions(pdev, igb_driver_name);
  2603. if (err)
  2604. goto err_pci_reg;
  2605. pci_enable_pcie_error_reporting(pdev);
  2606. pci_set_master(pdev);
  2607. pci_save_state(pdev);
  2608. err = -ENOMEM;
  2609. netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
  2610. IGB_MAX_TX_QUEUES);
  2611. if (!netdev)
  2612. goto err_alloc_etherdev;
  2613. SET_NETDEV_DEV(netdev, &pdev->dev);
  2614. pci_set_drvdata(pdev, netdev);
  2615. adapter = netdev_priv(netdev);
  2616. adapter->netdev = netdev;
  2617. adapter->pdev = pdev;
  2618. hw = &adapter->hw;
  2619. hw->back = adapter;
  2620. adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
  2621. err = -EIO;
  2622. adapter->io_addr = pci_iomap(pdev, 0, 0);
  2623. if (!adapter->io_addr)
  2624. goto err_ioremap;
  2625. /* hw->hw_addr can be altered, we'll use adapter->io_addr for unmap */
  2626. hw->hw_addr = adapter->io_addr;
  2627. netdev->netdev_ops = &igb_netdev_ops;
  2628. igb_set_ethtool_ops(netdev);
  2629. netdev->watchdog_timeo = 5 * HZ;
  2630. strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
  2631. netdev->mem_start = pci_resource_start(pdev, 0);
  2632. netdev->mem_end = pci_resource_end(pdev, 0);
  2633. /* PCI config space info */
  2634. hw->vendor_id = pdev->vendor;
  2635. hw->device_id = pdev->device;
  2636. hw->revision_id = pdev->revision;
  2637. hw->subsystem_vendor_id = pdev->subsystem_vendor;
  2638. hw->subsystem_device_id = pdev->subsystem_device;
  2639. /* Copy the default MAC, PHY and NVM function pointers */
  2640. memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
  2641. memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
  2642. memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
  2643. /* Initialize skew-specific constants */
  2644. err = ei->get_invariants(hw);
  2645. if (err)
  2646. goto err_sw_init;
  2647. /* setup the private structure */
  2648. err = igb_sw_init(adapter);
  2649. if (err)
  2650. goto err_sw_init;
  2651. igb_get_bus_info_pcie(hw);
  2652. hw->phy.autoneg_wait_to_complete = false;
  2653. /* Copper options */
  2654. if (hw->phy.media_type == e1000_media_type_copper) {
  2655. hw->phy.mdix = AUTO_ALL_MODES;
  2656. hw->phy.disable_polarity_correction = false;
  2657. hw->phy.ms_type = e1000_ms_hw_default;
  2658. }
  2659. if (igb_check_reset_block(hw))
  2660. dev_info(&pdev->dev,
  2661. "PHY reset is blocked due to SOL/IDER session.\n");
  2662. /* features is initialized to 0 in allocation, it might have bits
  2663. * set by igb_sw_init so we should use an or instead of an
  2664. * assignment.
  2665. */
  2666. netdev->features |= NETIF_F_SG |
  2667. NETIF_F_TSO |
  2668. NETIF_F_TSO6 |
  2669. NETIF_F_RXHASH |
  2670. NETIF_F_RXCSUM |
  2671. NETIF_F_HW_CSUM;
  2672. if (hw->mac.type >= e1000_82576)
  2673. netdev->features |= NETIF_F_SCTP_CRC | NETIF_F_GSO_UDP_L4;
  2674. if (hw->mac.type >= e1000_i350)
  2675. netdev->features |= NETIF_F_HW_TC;
  2676. #define IGB_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
  2677. NETIF_F_GSO_GRE_CSUM | \
  2678. NETIF_F_GSO_IPXIP4 | \
  2679. NETIF_F_GSO_IPXIP6 | \
  2680. NETIF_F_GSO_UDP_TUNNEL | \
  2681. NETIF_F_GSO_UDP_TUNNEL_CSUM)
  2682. netdev->gso_partial_features = IGB_GSO_PARTIAL_FEATURES;
  2683. netdev->features |= NETIF_F_GSO_PARTIAL | IGB_GSO_PARTIAL_FEATURES;
  2684. /* copy netdev features into list of user selectable features */
  2685. netdev->hw_features |= netdev->features |
  2686. NETIF_F_HW_VLAN_CTAG_RX |
  2687. NETIF_F_HW_VLAN_CTAG_TX |
  2688. NETIF_F_RXALL;
  2689. if (hw->mac.type >= e1000_i350)
  2690. netdev->hw_features |= NETIF_F_NTUPLE;
  2691. if (pci_using_dac)
  2692. netdev->features |= NETIF_F_HIGHDMA;
  2693. netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID;
  2694. netdev->mpls_features |= NETIF_F_HW_CSUM;
  2695. netdev->hw_enc_features |= netdev->vlan_features;
  2696. /* set this bit last since it cannot be part of vlan_features */
  2697. netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER |
  2698. NETIF_F_HW_VLAN_CTAG_RX |
  2699. NETIF_F_HW_VLAN_CTAG_TX;
  2700. netdev->priv_flags |= IFF_SUPP_NOFCS;
  2701. netdev->priv_flags |= IFF_UNICAST_FLT;
  2702. /* MTU range: 68 - 9216 */
  2703. netdev->min_mtu = ETH_MIN_MTU;
  2704. netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE;
  2705. adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
  2706. /* before reading the NVM, reset the controller to put the device in a
  2707. * known good starting state
  2708. */
  2709. hw->mac.ops.reset_hw(hw);
  2710. /* make sure the NVM is good , i211/i210 parts can have special NVM
  2711. * that doesn't contain a checksum
  2712. */
  2713. switch (hw->mac.type) {
  2714. case e1000_i210:
  2715. case e1000_i211:
  2716. if (igb_get_flash_presence_i210(hw)) {
  2717. if (hw->nvm.ops.validate(hw) < 0) {
  2718. dev_err(&pdev->dev,
  2719. "The NVM Checksum Is Not Valid\n");
  2720. err = -EIO;
  2721. goto err_eeprom;
  2722. }
  2723. }
  2724. break;
  2725. default:
  2726. if (hw->nvm.ops.validate(hw) < 0) {
  2727. dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
  2728. err = -EIO;
  2729. goto err_eeprom;
  2730. }
  2731. break;
  2732. }
  2733. if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) {
  2734. /* copy the MAC address out of the NVM */
  2735. if (hw->mac.ops.read_mac_addr(hw))
  2736. dev_err(&pdev->dev, "NVM Read Error\n");
  2737. }
  2738. memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
  2739. if (!is_valid_ether_addr(netdev->dev_addr)) {
  2740. dev_err(&pdev->dev, "Invalid MAC Address\n");
  2741. err = -EIO;
  2742. goto err_eeprom;
  2743. }
  2744. igb_set_default_mac_filter(adapter);
  2745. /* get firmware version for ethtool -i */
  2746. igb_set_fw_version(adapter);
  2747. /* configure RXPBSIZE and TXPBSIZE */
  2748. if (hw->mac.type == e1000_i210) {
  2749. wr32(E1000_RXPBS, I210_RXPBSIZE_DEFAULT);
  2750. wr32(E1000_TXPBS, I210_TXPBSIZE_DEFAULT);
  2751. }
  2752. timer_setup(&adapter->watchdog_timer, igb_watchdog, 0);
  2753. timer_setup(&adapter->phy_info_timer, igb_update_phy_info, 0);
  2754. INIT_WORK(&adapter->reset_task, igb_reset_task);
  2755. INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
  2756. /* Initialize link properties that are user-changeable */
  2757. adapter->fc_autoneg = true;
  2758. hw->mac.autoneg = true;
  2759. hw->phy.autoneg_advertised = 0x2f;
  2760. hw->fc.requested_mode = e1000_fc_default;
  2761. hw->fc.current_mode = e1000_fc_default;
  2762. igb_validate_mdi_setting(hw);
  2763. /* By default, support wake on port A */
  2764. if (hw->bus.func == 0)
  2765. adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
  2766. /* Check the NVM for wake support on non-port A ports */
  2767. if (hw->mac.type >= e1000_82580)
  2768. hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
  2769. NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
  2770. &eeprom_data);
  2771. else if (hw->bus.func == 1)
  2772. hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
  2773. if (eeprom_data & IGB_EEPROM_APME)
  2774. adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
  2775. /* now that we have the eeprom settings, apply the special cases where
  2776. * the eeprom may be wrong or the board simply won't support wake on
  2777. * lan on a particular port
  2778. */
  2779. switch (pdev->device) {
  2780. case E1000_DEV_ID_82575GB_QUAD_COPPER:
  2781. adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
  2782. break;
  2783. case E1000_DEV_ID_82575EB_FIBER_SERDES:
  2784. case E1000_DEV_ID_82576_FIBER:
  2785. case E1000_DEV_ID_82576_SERDES:
  2786. /* Wake events only supported on port A for dual fiber
  2787. * regardless of eeprom setting
  2788. */
  2789. if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
  2790. adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
  2791. break;
  2792. case E1000_DEV_ID_82576_QUAD_COPPER:
  2793. case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
  2794. /* if quad port adapter, disable WoL on all but port A */
  2795. if (global_quad_port_a != 0)
  2796. adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
  2797. else
  2798. adapter->flags |= IGB_FLAG_QUAD_PORT_A;
  2799. /* Reset for multiple quad port adapters */
  2800. if (++global_quad_port_a == 4)
  2801. global_quad_port_a = 0;
  2802. break;
  2803. default:
  2804. /* If the device can't wake, don't set software support */
  2805. if (!device_can_wakeup(&adapter->pdev->dev))
  2806. adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
  2807. }
  2808. /* initialize the wol settings based on the eeprom settings */
  2809. if (adapter->flags & IGB_FLAG_WOL_SUPPORTED)
  2810. adapter->wol |= E1000_WUFC_MAG;
  2811. /* Some vendors want WoL disabled by default, but still supported */
  2812. if ((hw->mac.type == e1000_i350) &&
  2813. (pdev->subsystem_vendor == PCI_VENDOR_ID_HP)) {
  2814. adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
  2815. adapter->wol = 0;
  2816. }
  2817. /* Some vendors want the ability to Use the EEPROM setting as
  2818. * enable/disable only, and not for capability
  2819. */
  2820. if (((hw->mac.type == e1000_i350) ||
  2821. (hw->mac.type == e1000_i354)) &&
  2822. (pdev->subsystem_vendor == PCI_VENDOR_ID_DELL)) {
  2823. adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
  2824. adapter->wol = 0;
  2825. }
  2826. if (hw->mac.type == e1000_i350) {
  2827. if (((pdev->subsystem_device == 0x5001) ||
  2828. (pdev->subsystem_device == 0x5002)) &&
  2829. (hw->bus.func == 0)) {
  2830. adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
  2831. adapter->wol = 0;
  2832. }
  2833. if (pdev->subsystem_device == 0x1F52)
  2834. adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
  2835. }
  2836. device_set_wakeup_enable(&adapter->pdev->dev,
  2837. adapter->flags & IGB_FLAG_WOL_SUPPORTED);
  2838. /* reset the hardware with the new settings */
  2839. igb_reset(adapter);
  2840. /* Init the I2C interface */
  2841. err = igb_init_i2c(adapter);
  2842. if (err) {
  2843. dev_err(&pdev->dev, "failed to init i2c interface\n");
  2844. goto err_eeprom;
  2845. }
  2846. /* let the f/w know that the h/w is now under the control of the
  2847. * driver.
  2848. */
  2849. igb_get_hw_control(adapter);
  2850. strcpy(netdev->name, "eth%d");
  2851. err = register_netdev(netdev);
  2852. if (err)
  2853. goto err_register;
  2854. /* carrier off reporting is important to ethtool even BEFORE open */
  2855. netif_carrier_off(netdev);
  2856. #ifdef CONFIG_IGB_DCA
  2857. if (dca_add_requester(&pdev->dev) == 0) {
  2858. adapter->flags |= IGB_FLAG_DCA_ENABLED;
  2859. dev_info(&pdev->dev, "DCA enabled\n");
  2860. igb_setup_dca(adapter);
  2861. }
  2862. #endif
  2863. #ifdef CONFIG_IGB_HWMON
  2864. /* Initialize the thermal sensor on i350 devices. */
  2865. if (hw->mac.type == e1000_i350 && hw->bus.func == 0) {
  2866. u16 ets_word;
  2867. /* Read the NVM to determine if this i350 device supports an
  2868. * external thermal sensor.
  2869. */
  2870. hw->nvm.ops.read(hw, NVM_ETS_CFG, 1, &ets_word);
  2871. if (ets_word != 0x0000 && ets_word != 0xFFFF)
  2872. adapter->ets = true;
  2873. else
  2874. adapter->ets = false;
  2875. if (igb_sysfs_init(adapter))
  2876. dev_err(&pdev->dev,
  2877. "failed to allocate sysfs resources\n");
  2878. } else {
  2879. adapter->ets = false;
  2880. }
  2881. #endif
  2882. /* Check if Media Autosense is enabled */
  2883. adapter->ei = *ei;
  2884. if (hw->dev_spec._82575.mas_capable)
  2885. igb_init_mas(adapter);
  2886. /* do hw tstamp init after resetting */
  2887. igb_ptp_init(adapter);
  2888. dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
  2889. /* print bus type/speed/width info, not applicable to i354 */
  2890. if (hw->mac.type != e1000_i354) {
  2891. dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
  2892. netdev->name,
  2893. ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
  2894. (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
  2895. "unknown"),
  2896. ((hw->bus.width == e1000_bus_width_pcie_x4) ?
  2897. "Width x4" :
  2898. (hw->bus.width == e1000_bus_width_pcie_x2) ?
  2899. "Width x2" :
  2900. (hw->bus.width == e1000_bus_width_pcie_x1) ?
  2901. "Width x1" : "unknown"), netdev->dev_addr);
  2902. }
  2903. if ((hw->mac.type >= e1000_i210 ||
  2904. igb_get_flash_presence_i210(hw))) {
  2905. ret_val = igb_read_part_string(hw, part_str,
  2906. E1000_PBANUM_LENGTH);
  2907. } else {
  2908. ret_val = -E1000_ERR_INVM_VALUE_NOT_FOUND;
  2909. }
  2910. if (ret_val)
  2911. strcpy(part_str, "Unknown");
  2912. dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str);
  2913. dev_info(&pdev->dev,
  2914. "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
  2915. (adapter->flags & IGB_FLAG_HAS_MSIX) ? "MSI-X" :
  2916. (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
  2917. adapter->num_rx_queues, adapter->num_tx_queues);
  2918. if (hw->phy.media_type == e1000_media_type_copper) {
  2919. switch (hw->mac.type) {
  2920. case e1000_i350:
  2921. case e1000_i210:
  2922. case e1000_i211:
  2923. /* Enable EEE for internal copper PHY devices */
  2924. err = igb_set_eee_i350(hw, true, true);
  2925. if ((!err) &&
  2926. (!hw->dev_spec._82575.eee_disable)) {
  2927. adapter->eee_advert =
  2928. MDIO_EEE_100TX | MDIO_EEE_1000T;
  2929. adapter->flags |= IGB_FLAG_EEE;
  2930. }
  2931. break;
  2932. case e1000_i354:
  2933. if ((rd32(E1000_CTRL_EXT) &
  2934. E1000_CTRL_EXT_LINK_MODE_SGMII)) {
  2935. err = igb_set_eee_i354(hw, true, true);
  2936. if ((!err) &&
  2937. (!hw->dev_spec._82575.eee_disable)) {
  2938. adapter->eee_advert =
  2939. MDIO_EEE_100TX | MDIO_EEE_1000T;
  2940. adapter->flags |= IGB_FLAG_EEE;
  2941. }
  2942. }
  2943. break;
  2944. default:
  2945. break;
  2946. }
  2947. }
  2948. dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_NEVER_SKIP);
  2949. pm_runtime_put_noidle(&pdev->dev);
  2950. return 0;
  2951. err_register:
  2952. igb_release_hw_control(adapter);
  2953. memset(&adapter->i2c_adap, 0, sizeof(adapter->i2c_adap));
  2954. err_eeprom:
  2955. if (!igb_check_reset_block(hw))
  2956. igb_reset_phy(hw);
  2957. if (hw->flash_address)
  2958. iounmap(hw->flash_address);
  2959. err_sw_init:
  2960. kfree(adapter->mac_table);
  2961. kfree(adapter->shadow_vfta);
  2962. igb_clear_interrupt_scheme(adapter);
  2963. #ifdef CONFIG_PCI_IOV
  2964. igb_disable_sriov(pdev);
  2965. #endif
  2966. pci_iounmap(pdev, adapter->io_addr);
  2967. err_ioremap:
  2968. free_netdev(netdev);
  2969. err_alloc_etherdev:
  2970. pci_release_mem_regions(pdev);
  2971. err_pci_reg:
  2972. err_dma:
  2973. pci_disable_device(pdev);
  2974. return err;
  2975. }
  2976. #ifdef CONFIG_PCI_IOV
  2977. static int igb_disable_sriov(struct pci_dev *pdev)
  2978. {
  2979. struct net_device *netdev = pci_get_drvdata(pdev);
  2980. struct igb_adapter *adapter = netdev_priv(netdev);
  2981. struct e1000_hw *hw = &adapter->hw;
  2982. /* reclaim resources allocated to VFs */
  2983. if (adapter->vf_data) {
  2984. /* disable iov and allow time for transactions to clear */
  2985. if (pci_vfs_assigned(pdev)) {
  2986. dev_warn(&pdev->dev,
  2987. "Cannot deallocate SR-IOV virtual functions while they are assigned - VFs will not be deallocated\n");
  2988. return -EPERM;
  2989. } else {
  2990. pci_disable_sriov(pdev);
  2991. msleep(500);
  2992. }
  2993. kfree(adapter->vf_mac_list);
  2994. adapter->vf_mac_list = NULL;
  2995. kfree(adapter->vf_data);
  2996. adapter->vf_data = NULL;
  2997. adapter->vfs_allocated_count = 0;
  2998. wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
  2999. wrfl();
  3000. msleep(100);
  3001. dev_info(&pdev->dev, "IOV Disabled\n");
  3002. /* Re-enable DMA Coalescing flag since IOV is turned off */
  3003. adapter->flags |= IGB_FLAG_DMAC;
  3004. }
  3005. return 0;
  3006. }
  3007. static int igb_enable_sriov(struct pci_dev *pdev, int num_vfs)
  3008. {
  3009. struct net_device *netdev = pci_get_drvdata(pdev);
  3010. struct igb_adapter *adapter = netdev_priv(netdev);
  3011. int old_vfs = pci_num_vf(pdev);
  3012. struct vf_mac_filter *mac_list;
  3013. int err = 0;
  3014. int num_vf_mac_filters, i;
  3015. if (!(adapter->flags & IGB_FLAG_HAS_MSIX) || num_vfs > 7) {
  3016. err = -EPERM;
  3017. goto out;
  3018. }
  3019. if (!num_vfs)
  3020. goto out;
  3021. if (old_vfs) {
  3022. dev_info(&pdev->dev, "%d pre-allocated VFs found - override max_vfs setting of %d\n",
  3023. old_vfs, max_vfs);
  3024. adapter->vfs_allocated_count = old_vfs;
  3025. } else
  3026. adapter->vfs_allocated_count = num_vfs;
  3027. adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
  3028. sizeof(struct vf_data_storage), GFP_KERNEL);
  3029. /* if allocation failed then we do not support SR-IOV */
  3030. if (!adapter->vf_data) {
  3031. adapter->vfs_allocated_count = 0;
  3032. err = -ENOMEM;
  3033. goto out;
  3034. }
  3035. /* Due to the limited number of RAR entries calculate potential
  3036. * number of MAC filters available for the VFs. Reserve entries
  3037. * for PF default MAC, PF MAC filters and at least one RAR entry
  3038. * for each VF for VF MAC.
  3039. */
  3040. num_vf_mac_filters = adapter->hw.mac.rar_entry_count -
  3041. (1 + IGB_PF_MAC_FILTERS_RESERVED +
  3042. adapter->vfs_allocated_count);
  3043. adapter->vf_mac_list = kcalloc(num_vf_mac_filters,
  3044. sizeof(struct vf_mac_filter),
  3045. GFP_KERNEL);
  3046. mac_list = adapter->vf_mac_list;
  3047. INIT_LIST_HEAD(&adapter->vf_macs.l);
  3048. if (adapter->vf_mac_list) {
  3049. /* Initialize list of VF MAC filters */
  3050. for (i = 0; i < num_vf_mac_filters; i++) {
  3051. mac_list->vf = -1;
  3052. mac_list->free = true;
  3053. list_add(&mac_list->l, &adapter->vf_macs.l);
  3054. mac_list++;
  3055. }
  3056. } else {
  3057. /* If we could not allocate memory for the VF MAC filters
  3058. * we can continue without this feature but warn user.
  3059. */
  3060. dev_err(&pdev->dev,
  3061. "Unable to allocate memory for VF MAC filter list\n");
  3062. }
  3063. /* only call pci_enable_sriov() if no VFs are allocated already */
  3064. if (!old_vfs) {
  3065. err = pci_enable_sriov(pdev, adapter->vfs_allocated_count);
  3066. if (err)
  3067. goto err_out;
  3068. }
  3069. dev_info(&pdev->dev, "%d VFs allocated\n",
  3070. adapter->vfs_allocated_count);
  3071. for (i = 0; i < adapter->vfs_allocated_count; i++)
  3072. igb_vf_configure(adapter, i);
  3073. /* DMA Coalescing is not supported in IOV mode. */
  3074. adapter->flags &= ~IGB_FLAG_DMAC;
  3075. goto out;
  3076. err_out:
  3077. kfree(adapter->vf_mac_list);
  3078. adapter->vf_mac_list = NULL;
  3079. kfree(adapter->vf_data);
  3080. adapter->vf_data = NULL;
  3081. adapter->vfs_allocated_count = 0;
  3082. out:
  3083. return err;
  3084. }
  3085. #endif
  3086. /**
  3087. * igb_remove_i2c - Cleanup I2C interface
  3088. * @adapter: pointer to adapter structure
  3089. **/
  3090. static void igb_remove_i2c(struct igb_adapter *adapter)
  3091. {
  3092. /* free the adapter bus structure */
  3093. i2c_del_adapter(&adapter->i2c_adap);
  3094. }
  3095. /**
  3096. * igb_remove - Device Removal Routine
  3097. * @pdev: PCI device information struct
  3098. *
  3099. * igb_remove is called by the PCI subsystem to alert the driver
  3100. * that it should release a PCI device. The could be caused by a
  3101. * Hot-Plug event, or because the driver is going to be removed from
  3102. * memory.
  3103. **/
  3104. static void igb_remove(struct pci_dev *pdev)
  3105. {
  3106. struct net_device *netdev = pci_get_drvdata(pdev);
  3107. struct igb_adapter *adapter = netdev_priv(netdev);
  3108. struct e1000_hw *hw = &adapter->hw;
  3109. pm_runtime_get_noresume(&pdev->dev);
  3110. #ifdef CONFIG_IGB_HWMON
  3111. igb_sysfs_exit(adapter);
  3112. #endif
  3113. igb_remove_i2c(adapter);
  3114. igb_ptp_stop(adapter);
  3115. /* The watchdog timer may be rescheduled, so explicitly
  3116. * disable watchdog from being rescheduled.
  3117. */
  3118. set_bit(__IGB_DOWN, &adapter->state);
  3119. del_timer_sync(&adapter->watchdog_timer);
  3120. del_timer_sync(&adapter->phy_info_timer);
  3121. cancel_work_sync(&adapter->reset_task);
  3122. cancel_work_sync(&adapter->watchdog_task);
  3123. #ifdef CONFIG_IGB_DCA
  3124. if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
  3125. dev_info(&pdev->dev, "DCA disabled\n");
  3126. dca_remove_requester(&pdev->dev);
  3127. adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
  3128. wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
  3129. }
  3130. #endif
  3131. /* Release control of h/w to f/w. If f/w is AMT enabled, this
  3132. * would have already happened in close and is redundant.
  3133. */
  3134. igb_release_hw_control(adapter);
  3135. #ifdef CONFIG_PCI_IOV
  3136. igb_disable_sriov(pdev);
  3137. #endif
  3138. unregister_netdev(netdev);
  3139. igb_clear_interrupt_scheme(adapter);
  3140. pci_iounmap(pdev, adapter->io_addr);
  3141. if (hw->flash_address)
  3142. iounmap(hw->flash_address);
  3143. pci_release_mem_regions(pdev);
  3144. kfree(adapter->mac_table);
  3145. kfree(adapter->shadow_vfta);
  3146. free_netdev(netdev);
  3147. pci_disable_pcie_error_reporting(pdev);
  3148. pci_disable_device(pdev);
  3149. }
  3150. /**
  3151. * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
  3152. * @adapter: board private structure to initialize
  3153. *
  3154. * This function initializes the vf specific data storage and then attempts to
  3155. * allocate the VFs. The reason for ordering it this way is because it is much
  3156. * mor expensive time wise to disable SR-IOV than it is to allocate and free
  3157. * the memory for the VFs.
  3158. **/
  3159. static void igb_probe_vfs(struct igb_adapter *adapter)
  3160. {
  3161. #ifdef CONFIG_PCI_IOV
  3162. struct pci_dev *pdev = adapter->pdev;
  3163. struct e1000_hw *hw = &adapter->hw;
  3164. /* Virtualization features not supported on i210 family. */
  3165. if ((hw->mac.type == e1000_i210) || (hw->mac.type == e1000_i211))
  3166. return;
  3167. /* Of the below we really only want the effect of getting
  3168. * IGB_FLAG_HAS_MSIX set (if available), without which
  3169. * igb_enable_sriov() has no effect.
  3170. */
  3171. igb_set_interrupt_capability(adapter, true);
  3172. igb_reset_interrupt_capability(adapter);
  3173. pci_sriov_set_totalvfs(pdev, 7);
  3174. igb_enable_sriov(pdev, max_vfs);
  3175. #endif /* CONFIG_PCI_IOV */
  3176. }
  3177. unsigned int igb_get_max_rss_queues(struct igb_adapter *adapter)
  3178. {
  3179. struct e1000_hw *hw = &adapter->hw;
  3180. unsigned int max_rss_queues;
  3181. /* Determine the maximum number of RSS queues supported. */
  3182. switch (hw->mac.type) {
  3183. case e1000_i211:
  3184. max_rss_queues = IGB_MAX_RX_QUEUES_I211;
  3185. break;
  3186. case e1000_82575:
  3187. case e1000_i210:
  3188. max_rss_queues = IGB_MAX_RX_QUEUES_82575;
  3189. break;
  3190. case e1000_i350:
  3191. /* I350 cannot do RSS and SR-IOV at the same time */
  3192. if (!!adapter->vfs_allocated_count) {
  3193. max_rss_queues = 1;
  3194. break;
  3195. }
  3196. /* fall through */
  3197. case e1000_82576:
  3198. if (!!adapter->vfs_allocated_count) {
  3199. max_rss_queues = 2;
  3200. break;
  3201. }
  3202. /* fall through */
  3203. case e1000_82580:
  3204. case e1000_i354:
  3205. default:
  3206. max_rss_queues = IGB_MAX_RX_QUEUES;
  3207. break;
  3208. }
  3209. return max_rss_queues;
  3210. }
  3211. static void igb_init_queue_configuration(struct igb_adapter *adapter)
  3212. {
  3213. u32 max_rss_queues;
  3214. max_rss_queues = igb_get_max_rss_queues(adapter);
  3215. adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus());
  3216. igb_set_flag_queue_pairs(adapter, max_rss_queues);
  3217. }
  3218. void igb_set_flag_queue_pairs(struct igb_adapter *adapter,
  3219. const u32 max_rss_queues)
  3220. {
  3221. struct e1000_hw *hw = &adapter->hw;
  3222. /* Determine if we need to pair queues. */
  3223. switch (hw->mac.type) {
  3224. case e1000_82575:
  3225. case e1000_i211:
  3226. /* Device supports enough interrupts without queue pairing. */
  3227. break;
  3228. case e1000_82576:
  3229. case e1000_82580:
  3230. case e1000_i350:
  3231. case e1000_i354:
  3232. case e1000_i210:
  3233. default:
  3234. /* If rss_queues > half of max_rss_queues, pair the queues in
  3235. * order to conserve interrupts due to limited supply.
  3236. */
  3237. if (adapter->rss_queues > (max_rss_queues / 2))
  3238. adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
  3239. else
  3240. adapter->flags &= ~IGB_FLAG_QUEUE_PAIRS;
  3241. break;
  3242. }
  3243. }
  3244. /**
  3245. * igb_sw_init - Initialize general software structures (struct igb_adapter)
  3246. * @adapter: board private structure to initialize
  3247. *
  3248. * igb_sw_init initializes the Adapter private data structure.
  3249. * Fields are initialized based on PCI device information and
  3250. * OS network device settings (MTU size).
  3251. **/
  3252. static int igb_sw_init(struct igb_adapter *adapter)
  3253. {
  3254. struct e1000_hw *hw = &adapter->hw;
  3255. struct net_device *netdev = adapter->netdev;
  3256. struct pci_dev *pdev = adapter->pdev;
  3257. pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
  3258. /* set default ring sizes */
  3259. adapter->tx_ring_count = IGB_DEFAULT_TXD;
  3260. adapter->rx_ring_count = IGB_DEFAULT_RXD;
  3261. /* set default ITR values */
  3262. adapter->rx_itr_setting = IGB_DEFAULT_ITR;
  3263. adapter->tx_itr_setting = IGB_DEFAULT_ITR;
  3264. /* set default work limits */
  3265. adapter->tx_work_limit = IGB_DEFAULT_TX_WORK;
  3266. adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
  3267. VLAN_HLEN;
  3268. adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
  3269. spin_lock_init(&adapter->nfc_lock);
  3270. spin_lock_init(&adapter->stats64_lock);
  3271. #ifdef CONFIG_PCI_IOV
  3272. switch (hw->mac.type) {
  3273. case e1000_82576:
  3274. case e1000_i350:
  3275. if (max_vfs > 7) {
  3276. dev_warn(&pdev->dev,
  3277. "Maximum of 7 VFs per PF, using max\n");
  3278. max_vfs = adapter->vfs_allocated_count = 7;
  3279. } else
  3280. adapter->vfs_allocated_count = max_vfs;
  3281. if (adapter->vfs_allocated_count)
  3282. dev_warn(&pdev->dev,
  3283. "Enabling SR-IOV VFs using the module parameter is deprecated - please use the pci sysfs interface.\n");
  3284. break;
  3285. default:
  3286. break;
  3287. }
  3288. #endif /* CONFIG_PCI_IOV */
  3289. /* Assume MSI-X interrupts, will be checked during IRQ allocation */
  3290. adapter->flags |= IGB_FLAG_HAS_MSIX;
  3291. adapter->mac_table = kcalloc(hw->mac.rar_entry_count,
  3292. sizeof(struct igb_mac_addr),
  3293. GFP_KERNEL);
  3294. if (!adapter->mac_table)
  3295. return -ENOMEM;
  3296. igb_probe_vfs(adapter);
  3297. igb_init_queue_configuration(adapter);
  3298. /* Setup and initialize a copy of the hw vlan table array */
  3299. adapter->shadow_vfta = kcalloc(E1000_VLAN_FILTER_TBL_SIZE, sizeof(u32),
  3300. GFP_KERNEL);
  3301. if (!adapter->shadow_vfta)
  3302. return -ENOMEM;
  3303. /* This call may decrease the number of queues */
  3304. if (igb_init_interrupt_scheme(adapter, true)) {
  3305. dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
  3306. return -ENOMEM;
  3307. }
  3308. /* Explicitly disable IRQ since the NIC can be in any state. */
  3309. igb_irq_disable(adapter);
  3310. if (hw->mac.type >= e1000_i350)
  3311. adapter->flags &= ~IGB_FLAG_DMAC;
  3312. set_bit(__IGB_DOWN, &adapter->state);
  3313. return 0;
  3314. }
  3315. /**
  3316. * igb_open - Called when a network interface is made active
  3317. * @netdev: network interface device structure
  3318. *
  3319. * Returns 0 on success, negative value on failure
  3320. *
  3321. * The open entry point is called when a network interface is made
  3322. * active by the system (IFF_UP). At this point all resources needed
  3323. * for transmit and receive operations are allocated, the interrupt
  3324. * handler is registered with the OS, the watchdog timer is started,
  3325. * and the stack is notified that the interface is ready.
  3326. **/
  3327. static int __igb_open(struct net_device *netdev, bool resuming)
  3328. {
  3329. struct igb_adapter *adapter = netdev_priv(netdev);
  3330. struct e1000_hw *hw = &adapter->hw;
  3331. struct pci_dev *pdev = adapter->pdev;
  3332. int err;
  3333. int i;
  3334. /* disallow open during test */
  3335. if (test_bit(__IGB_TESTING, &adapter->state)) {
  3336. WARN_ON(resuming);
  3337. return -EBUSY;
  3338. }
  3339. if (!resuming)
  3340. pm_runtime_get_sync(&pdev->dev);
  3341. netif_carrier_off(netdev);
  3342. /* allocate transmit descriptors */
  3343. err = igb_setup_all_tx_resources(adapter);
  3344. if (err)
  3345. goto err_setup_tx;
  3346. /* allocate receive descriptors */
  3347. err = igb_setup_all_rx_resources(adapter);
  3348. if (err)
  3349. goto err_setup_rx;
  3350. igb_power_up_link(adapter);
  3351. /* before we allocate an interrupt, we must be ready to handle it.
  3352. * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
  3353. * as soon as we call pci_request_irq, so we have to setup our
  3354. * clean_rx handler before we do so.
  3355. */
  3356. igb_configure(adapter);
  3357. err = igb_request_irq(adapter);
  3358. if (err)
  3359. goto err_req_irq;
  3360. /* Notify the stack of the actual queue counts. */
  3361. err = netif_set_real_num_tx_queues(adapter->netdev,
  3362. adapter->num_tx_queues);
  3363. if (err)
  3364. goto err_set_queues;
  3365. err = netif_set_real_num_rx_queues(adapter->netdev,
  3366. adapter->num_rx_queues);
  3367. if (err)
  3368. goto err_set_queues;
  3369. /* From here on the code is the same as igb_up() */
  3370. clear_bit(__IGB_DOWN, &adapter->state);
  3371. for (i = 0; i < adapter->num_q_vectors; i++)
  3372. napi_enable(&(adapter->q_vector[i]->napi));
  3373. /* Clear any pending interrupts. */
  3374. rd32(E1000_TSICR);
  3375. rd32(E1000_ICR);
  3376. igb_irq_enable(adapter);
  3377. /* notify VFs that reset has been completed */
  3378. if (adapter->vfs_allocated_count) {
  3379. u32 reg_data = rd32(E1000_CTRL_EXT);
  3380. reg_data |= E1000_CTRL_EXT_PFRSTD;
  3381. wr32(E1000_CTRL_EXT, reg_data);
  3382. }
  3383. netif_tx_start_all_queues(netdev);
  3384. if (!resuming)
  3385. pm_runtime_put(&pdev->dev);
  3386. /* start the watchdog. */
  3387. hw->mac.get_link_status = 1;
  3388. schedule_work(&adapter->watchdog_task);
  3389. return 0;
  3390. err_set_queues:
  3391. igb_free_irq(adapter);
  3392. err_req_irq:
  3393. igb_release_hw_control(adapter);
  3394. igb_power_down_link(adapter);
  3395. igb_free_all_rx_resources(adapter);
  3396. err_setup_rx:
  3397. igb_free_all_tx_resources(adapter);
  3398. err_setup_tx:
  3399. igb_reset(adapter);
  3400. if (!resuming)
  3401. pm_runtime_put(&pdev->dev);
  3402. return err;
  3403. }
  3404. int igb_open(struct net_device *netdev)
  3405. {
  3406. return __igb_open(netdev, false);
  3407. }
  3408. /**
  3409. * igb_close - Disables a network interface
  3410. * @netdev: network interface device structure
  3411. *
  3412. * Returns 0, this is not allowed to fail
  3413. *
  3414. * The close entry point is called when an interface is de-activated
  3415. * by the OS. The hardware is still under the driver's control, but
  3416. * needs to be disabled. A global MAC reset is issued to stop the
  3417. * hardware, and all transmit and receive resources are freed.
  3418. **/
  3419. static int __igb_close(struct net_device *netdev, bool suspending)
  3420. {
  3421. struct igb_adapter *adapter = netdev_priv(netdev);
  3422. struct pci_dev *pdev = adapter->pdev;
  3423. WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
  3424. if (!suspending)
  3425. pm_runtime_get_sync(&pdev->dev);
  3426. igb_down(adapter);
  3427. igb_free_irq(adapter);
  3428. igb_free_all_tx_resources(adapter);
  3429. igb_free_all_rx_resources(adapter);
  3430. if (!suspending)
  3431. pm_runtime_put_sync(&pdev->dev);
  3432. return 0;
  3433. }
  3434. int igb_close(struct net_device *netdev)
  3435. {
  3436. if (netif_device_present(netdev) || netdev->dismantle)
  3437. return __igb_close(netdev, false);
  3438. return 0;
  3439. }
  3440. /**
  3441. * igb_setup_tx_resources - allocate Tx resources (Descriptors)
  3442. * @tx_ring: tx descriptor ring (for a specific queue) to setup
  3443. *
  3444. * Return 0 on success, negative on failure
  3445. **/
  3446. int igb_setup_tx_resources(struct igb_ring *tx_ring)
  3447. {
  3448. struct device *dev = tx_ring->dev;
  3449. int size;
  3450. size = sizeof(struct igb_tx_buffer) * tx_ring->count;
  3451. tx_ring->tx_buffer_info = vmalloc(size);
  3452. if (!tx_ring->tx_buffer_info)
  3453. goto err;
  3454. /* round up to nearest 4K */
  3455. tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
  3456. tx_ring->size = ALIGN(tx_ring->size, 4096);
  3457. tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
  3458. &tx_ring->dma, GFP_KERNEL);
  3459. if (!tx_ring->desc)
  3460. goto err;
  3461. tx_ring->next_to_use = 0;
  3462. tx_ring->next_to_clean = 0;
  3463. return 0;
  3464. err:
  3465. vfree(tx_ring->tx_buffer_info);
  3466. tx_ring->tx_buffer_info = NULL;
  3467. dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");
  3468. return -ENOMEM;
  3469. }
  3470. /**
  3471. * igb_setup_all_tx_resources - wrapper to allocate Tx resources
  3472. * (Descriptors) for all queues
  3473. * @adapter: board private structure
  3474. *
  3475. * Return 0 on success, negative on failure
  3476. **/
  3477. static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
  3478. {
  3479. struct pci_dev *pdev = adapter->pdev;
  3480. int i, err = 0;
  3481. for (i = 0; i < adapter->num_tx_queues; i++) {
  3482. err = igb_setup_tx_resources(adapter->tx_ring[i]);
  3483. if (err) {
  3484. dev_err(&pdev->dev,
  3485. "Allocation for Tx Queue %u failed\n", i);
  3486. for (i--; i >= 0; i--)
  3487. igb_free_tx_resources(adapter->tx_ring[i]);
  3488. break;
  3489. }
  3490. }
  3491. return err;
  3492. }
  3493. /**
  3494. * igb_setup_tctl - configure the transmit control registers
  3495. * @adapter: Board private structure
  3496. **/
  3497. void igb_setup_tctl(struct igb_adapter *adapter)
  3498. {
  3499. struct e1000_hw *hw = &adapter->hw;
  3500. u32 tctl;
  3501. /* disable queue 0 which is enabled by default on 82575 and 82576 */
  3502. wr32(E1000_TXDCTL(0), 0);
  3503. /* Program the Transmit Control Register */
  3504. tctl = rd32(E1000_TCTL);
  3505. tctl &= ~E1000_TCTL_CT;
  3506. tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
  3507. (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
  3508. igb_config_collision_dist(hw);
  3509. /* Enable transmits */
  3510. tctl |= E1000_TCTL_EN;
  3511. wr32(E1000_TCTL, tctl);
  3512. }
  3513. /**
  3514. * igb_configure_tx_ring - Configure transmit ring after Reset
  3515. * @adapter: board private structure
  3516. * @ring: tx ring to configure
  3517. *
  3518. * Configure a transmit ring after a reset.
  3519. **/
  3520. void igb_configure_tx_ring(struct igb_adapter *adapter,
  3521. struct igb_ring *ring)
  3522. {
  3523. struct e1000_hw *hw = &adapter->hw;
  3524. u32 txdctl = 0;
  3525. u64 tdba = ring->dma;
  3526. int reg_idx = ring->reg_idx;
  3527. wr32(E1000_TDLEN(reg_idx),
  3528. ring->count * sizeof(union e1000_adv_tx_desc));
  3529. wr32(E1000_TDBAL(reg_idx),
  3530. tdba & 0x00000000ffffffffULL);
  3531. wr32(E1000_TDBAH(reg_idx), tdba >> 32);
  3532. ring->tail = adapter->io_addr + E1000_TDT(reg_idx);
  3533. wr32(E1000_TDH(reg_idx), 0);
  3534. writel(0, ring->tail);
  3535. txdctl |= IGB_TX_PTHRESH;
  3536. txdctl |= IGB_TX_HTHRESH << 8;
  3537. txdctl |= IGB_TX_WTHRESH << 16;
  3538. /* reinitialize tx_buffer_info */
  3539. memset(ring->tx_buffer_info, 0,
  3540. sizeof(struct igb_tx_buffer) * ring->count);
  3541. txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
  3542. wr32(E1000_TXDCTL(reg_idx), txdctl);
  3543. }
  3544. /**
  3545. * igb_configure_tx - Configure transmit Unit after Reset
  3546. * @adapter: board private structure
  3547. *
  3548. * Configure the Tx unit of the MAC after a reset.
  3549. **/
  3550. static void igb_configure_tx(struct igb_adapter *adapter)
  3551. {
  3552. struct e1000_hw *hw = &adapter->hw;
  3553. int i;
  3554. /* disable the queues */
  3555. for (i = 0; i < adapter->num_tx_queues; i++)
  3556. wr32(E1000_TXDCTL(adapter->tx_ring[i]->reg_idx), 0);
  3557. wrfl();
  3558. usleep_range(10000, 20000);
  3559. for (i = 0; i < adapter->num_tx_queues; i++)
  3560. igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
  3561. }
  3562. /**
  3563. * igb_setup_rx_resources - allocate Rx resources (Descriptors)
  3564. * @rx_ring: Rx descriptor ring (for a specific queue) to setup
  3565. *
  3566. * Returns 0 on success, negative on failure
  3567. **/
  3568. int igb_setup_rx_resources(struct igb_ring *rx_ring)
  3569. {
  3570. struct device *dev = rx_ring->dev;
  3571. int size;
  3572. size = sizeof(struct igb_rx_buffer) * rx_ring->count;
  3573. rx_ring->rx_buffer_info = vmalloc(size);
  3574. if (!rx_ring->rx_buffer_info)
  3575. goto err;
  3576. /* Round up to nearest 4K */
  3577. rx_ring->size = rx_ring->count * sizeof(union e1000_adv_rx_desc);
  3578. rx_ring->size = ALIGN(rx_ring->size, 4096);
  3579. rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
  3580. &rx_ring->dma, GFP_KERNEL);
  3581. if (!rx_ring->desc)
  3582. goto err;
  3583. rx_ring->next_to_alloc = 0;
  3584. rx_ring->next_to_clean = 0;
  3585. rx_ring->next_to_use = 0;
  3586. return 0;
  3587. err:
  3588. vfree(rx_ring->rx_buffer_info);
  3589. rx_ring->rx_buffer_info = NULL;
  3590. dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");
  3591. return -ENOMEM;
  3592. }
  3593. /**
  3594. * igb_setup_all_rx_resources - wrapper to allocate Rx resources
  3595. * (Descriptors) for all queues
  3596. * @adapter: board private structure
  3597. *
  3598. * Return 0 on success, negative on failure
  3599. **/
  3600. static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
  3601. {
  3602. struct pci_dev *pdev = adapter->pdev;
  3603. int i, err = 0;
  3604. for (i = 0; i < adapter->num_rx_queues; i++) {
  3605. err = igb_setup_rx_resources(adapter->rx_ring[i]);
  3606. if (err) {
  3607. dev_err(&pdev->dev,
  3608. "Allocation for Rx Queue %u failed\n", i);
  3609. for (i--; i >= 0; i--)
  3610. igb_free_rx_resources(adapter->rx_ring[i]);
  3611. break;
  3612. }
  3613. }
  3614. return err;
  3615. }
  3616. /**
  3617. * igb_setup_mrqc - configure the multiple receive queue control registers
  3618. * @adapter: Board private structure
  3619. **/
  3620. static void igb_setup_mrqc(struct igb_adapter *adapter)
  3621. {
  3622. struct e1000_hw *hw = &adapter->hw;
  3623. u32 mrqc, rxcsum;
  3624. u32 j, num_rx_queues;
  3625. u32 rss_key[10];
  3626. netdev_rss_key_fill(rss_key, sizeof(rss_key));
  3627. for (j = 0; j < 10; j++)
  3628. wr32(E1000_RSSRK(j), rss_key[j]);
  3629. num_rx_queues = adapter->rss_queues;
  3630. switch (hw->mac.type) {
  3631. case e1000_82576:
  3632. /* 82576 supports 2 RSS queues for SR-IOV */
  3633. if (adapter->vfs_allocated_count)
  3634. num_rx_queues = 2;
  3635. break;
  3636. default:
  3637. break;
  3638. }
  3639. if (adapter->rss_indir_tbl_init != num_rx_queues) {
  3640. for (j = 0; j < IGB_RETA_SIZE; j++)
  3641. adapter->rss_indir_tbl[j] =
  3642. (j * num_rx_queues) / IGB_RETA_SIZE;
  3643. adapter->rss_indir_tbl_init = num_rx_queues;
  3644. }
  3645. igb_write_rss_indir_tbl(adapter);
  3646. /* Disable raw packet checksumming so that RSS hash is placed in
  3647. * descriptor on writeback. No need to enable TCP/UDP/IP checksum
  3648. * offloads as they are enabled by default
  3649. */
  3650. rxcsum = rd32(E1000_RXCSUM);
  3651. rxcsum |= E1000_RXCSUM_PCSD;
  3652. if (adapter->hw.mac.type >= e1000_82576)
  3653. /* Enable Receive Checksum Offload for SCTP */
  3654. rxcsum |= E1000_RXCSUM_CRCOFL;
  3655. /* Don't need to set TUOFL or IPOFL, they default to 1 */
  3656. wr32(E1000_RXCSUM, rxcsum);
  3657. /* Generate RSS hash based on packet types, TCP/UDP
  3658. * port numbers and/or IPv4/v6 src and dst addresses
  3659. */
  3660. mrqc = E1000_MRQC_RSS_FIELD_IPV4 |
  3661. E1000_MRQC_RSS_FIELD_IPV4_TCP |
  3662. E1000_MRQC_RSS_FIELD_IPV6 |
  3663. E1000_MRQC_RSS_FIELD_IPV6_TCP |
  3664. E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
  3665. if (adapter->flags & IGB_FLAG_RSS_FIELD_IPV4_UDP)
  3666. mrqc |= E1000_MRQC_RSS_FIELD_IPV4_UDP;
  3667. if (adapter->flags & IGB_FLAG_RSS_FIELD_IPV6_UDP)
  3668. mrqc |= E1000_MRQC_RSS_FIELD_IPV6_UDP;
  3669. /* If VMDq is enabled then we set the appropriate mode for that, else
  3670. * we default to RSS so that an RSS hash is calculated per packet even
  3671. * if we are only using one queue
  3672. */
  3673. if (adapter->vfs_allocated_count) {
  3674. if (hw->mac.type > e1000_82575) {
  3675. /* Set the default pool for the PF's first queue */
  3676. u32 vtctl = rd32(E1000_VT_CTL);
  3677. vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
  3678. E1000_VT_CTL_DISABLE_DEF_POOL);
  3679. vtctl |= adapter->vfs_allocated_count <<
  3680. E1000_VT_CTL_DEFAULT_POOL_SHIFT;
  3681. wr32(E1000_VT_CTL, vtctl);
  3682. }
  3683. if (adapter->rss_queues > 1)
  3684. mrqc |= E1000_MRQC_ENABLE_VMDQ_RSS_MQ;
  3685. else
  3686. mrqc |= E1000_MRQC_ENABLE_VMDQ;
  3687. } else {
  3688. if (hw->mac.type != e1000_i211)
  3689. mrqc |= E1000_MRQC_ENABLE_RSS_MQ;
  3690. }
  3691. igb_vmm_control(adapter);
  3692. wr32(E1000_MRQC, mrqc);
  3693. }
  3694. /**
  3695. * igb_setup_rctl - configure the receive control registers
  3696. * @adapter: Board private structure
  3697. **/
  3698. void igb_setup_rctl(struct igb_adapter *adapter)
  3699. {
  3700. struct e1000_hw *hw = &adapter->hw;
  3701. u32 rctl;
  3702. rctl = rd32(E1000_RCTL);
  3703. rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
  3704. rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
  3705. rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
  3706. (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
  3707. /* enable stripping of CRC. It's unlikely this will break BMC
  3708. * redirection as it did with e1000. Newer features require
  3709. * that the HW strips the CRC.
  3710. */
  3711. rctl |= E1000_RCTL_SECRC;
  3712. /* disable store bad packets and clear size bits. */
  3713. rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
  3714. /* enable LPE to allow for reception of jumbo frames */
  3715. rctl |= E1000_RCTL_LPE;
  3716. /* disable queue 0 to prevent tail write w/o re-config */
  3717. wr32(E1000_RXDCTL(0), 0);
  3718. /* Attention!!! For SR-IOV PF driver operations you must enable
  3719. * queue drop for all VF and PF queues to prevent head of line blocking
  3720. * if an un-trusted VF does not provide descriptors to hardware.
  3721. */
  3722. if (adapter->vfs_allocated_count) {
  3723. /* set all queue drop enable bits */
  3724. wr32(E1000_QDE, ALL_QUEUES);
  3725. }
  3726. /* This is useful for sniffing bad packets. */
  3727. if (adapter->netdev->features & NETIF_F_RXALL) {
  3728. /* UPE and MPE will be handled by normal PROMISC logic
  3729. * in e1000e_set_rx_mode
  3730. */
  3731. rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
  3732. E1000_RCTL_BAM | /* RX All Bcast Pkts */
  3733. E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
  3734. rctl &= ~(E1000_RCTL_DPF | /* Allow filtered pause */
  3735. E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
  3736. /* Do not mess with E1000_CTRL_VME, it affects transmit as well,
  3737. * and that breaks VLANs.
  3738. */
  3739. }
  3740. wr32(E1000_RCTL, rctl);
  3741. }
  3742. static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
  3743. int vfn)
  3744. {
  3745. struct e1000_hw *hw = &adapter->hw;
  3746. u32 vmolr;
  3747. if (size > MAX_JUMBO_FRAME_SIZE)
  3748. size = MAX_JUMBO_FRAME_SIZE;
  3749. vmolr = rd32(E1000_VMOLR(vfn));
  3750. vmolr &= ~E1000_VMOLR_RLPML_MASK;
  3751. vmolr |= size | E1000_VMOLR_LPE;
  3752. wr32(E1000_VMOLR(vfn), vmolr);
  3753. return 0;
  3754. }
  3755. static inline void igb_set_vf_vlan_strip(struct igb_adapter *adapter,
  3756. int vfn, bool enable)
  3757. {
  3758. struct e1000_hw *hw = &adapter->hw;
  3759. u32 val, reg;
  3760. if (hw->mac.type < e1000_82576)
  3761. return;
  3762. if (hw->mac.type == e1000_i350)
  3763. reg = E1000_DVMOLR(vfn);
  3764. else
  3765. reg = E1000_VMOLR(vfn);
  3766. val = rd32(reg);
  3767. if (enable)
  3768. val |= E1000_VMOLR_STRVLAN;
  3769. else
  3770. val &= ~(E1000_VMOLR_STRVLAN);
  3771. wr32(reg, val);
  3772. }
  3773. static inline void igb_set_vmolr(struct igb_adapter *adapter,
  3774. int vfn, bool aupe)
  3775. {
  3776. struct e1000_hw *hw = &adapter->hw;
  3777. u32 vmolr;
  3778. /* This register exists only on 82576 and newer so if we are older then
  3779. * we should exit and do nothing
  3780. */
  3781. if (hw->mac.type < e1000_82576)
  3782. return;
  3783. vmolr = rd32(E1000_VMOLR(vfn));
  3784. if (aupe)
  3785. vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
  3786. else
  3787. vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
  3788. /* clear all bits that might not be set */
  3789. vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
  3790. if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
  3791. vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
  3792. /* for VMDq only allow the VFs and pool 0 to accept broadcast and
  3793. * multicast packets
  3794. */
  3795. if (vfn <= adapter->vfs_allocated_count)
  3796. vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
  3797. wr32(E1000_VMOLR(vfn), vmolr);
  3798. }
  3799. /**
  3800. * igb_setup_srrctl - configure the split and replication receive control
  3801. * registers
  3802. * @adapter: Board private structure
  3803. * @ring: receive ring to be configured
  3804. **/
  3805. void igb_setup_srrctl(struct igb_adapter *adapter, struct igb_ring *ring)
  3806. {
  3807. struct e1000_hw *hw = &adapter->hw;
  3808. int reg_idx = ring->reg_idx;
  3809. u32 srrctl = 0;
  3810. srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
  3811. if (ring_uses_large_buffer(ring))
  3812. srrctl |= IGB_RXBUFFER_3072 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
  3813. else
  3814. srrctl |= IGB_RXBUFFER_2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
  3815. srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
  3816. if (hw->mac.type >= e1000_82580)
  3817. srrctl |= E1000_SRRCTL_TIMESTAMP;
  3818. /* Only set Drop Enable if VFs allocated, or we are supporting multiple
  3819. * queues and rx flow control is disabled
  3820. */
  3821. if (adapter->vfs_allocated_count ||
  3822. (!(hw->fc.current_mode & e1000_fc_rx_pause) &&
  3823. adapter->num_rx_queues > 1))
  3824. srrctl |= E1000_SRRCTL_DROP_EN;
  3825. wr32(E1000_SRRCTL(reg_idx), srrctl);
  3826. }
  3827. /**
  3828. * igb_configure_rx_ring - Configure a receive ring after Reset
  3829. * @adapter: board private structure
  3830. * @ring: receive ring to be configured
  3831. *
  3832. * Configure the Rx unit of the MAC after a reset.
  3833. **/
  3834. void igb_configure_rx_ring(struct igb_adapter *adapter,
  3835. struct igb_ring *ring)
  3836. {
  3837. struct e1000_hw *hw = &adapter->hw;
  3838. union e1000_adv_rx_desc *rx_desc;
  3839. u64 rdba = ring->dma;
  3840. int reg_idx = ring->reg_idx;
  3841. u32 rxdctl = 0;
  3842. /* disable the queue */
  3843. wr32(E1000_RXDCTL(reg_idx), 0);
  3844. /* Set DMA base address registers */
  3845. wr32(E1000_RDBAL(reg_idx),
  3846. rdba & 0x00000000ffffffffULL);
  3847. wr32(E1000_RDBAH(reg_idx), rdba >> 32);
  3848. wr32(E1000_RDLEN(reg_idx),
  3849. ring->count * sizeof(union e1000_adv_rx_desc));
  3850. /* initialize head and tail */
  3851. ring->tail = adapter->io_addr + E1000_RDT(reg_idx);
  3852. wr32(E1000_RDH(reg_idx), 0);
  3853. writel(0, ring->tail);
  3854. /* set descriptor configuration */
  3855. igb_setup_srrctl(adapter, ring);
  3856. /* set filtering for VMDQ pools */
  3857. igb_set_vmolr(adapter, reg_idx & 0x7, true);
  3858. rxdctl |= IGB_RX_PTHRESH;
  3859. rxdctl |= IGB_RX_HTHRESH << 8;
  3860. rxdctl |= IGB_RX_WTHRESH << 16;
  3861. /* initialize rx_buffer_info */
  3862. memset(ring->rx_buffer_info, 0,
  3863. sizeof(struct igb_rx_buffer) * ring->count);
  3864. /* initialize Rx descriptor 0 */
  3865. rx_desc = IGB_RX_DESC(ring, 0);
  3866. rx_desc->wb.upper.length = 0;
  3867. /* enable receive descriptor fetching */
  3868. rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
  3869. wr32(E1000_RXDCTL(reg_idx), rxdctl);
  3870. }
  3871. static void igb_set_rx_buffer_len(struct igb_adapter *adapter,
  3872. struct igb_ring *rx_ring)
  3873. {
  3874. /* set build_skb and buffer size flags */
  3875. clear_ring_build_skb_enabled(rx_ring);
  3876. clear_ring_uses_large_buffer(rx_ring);
  3877. if (adapter->flags & IGB_FLAG_RX_LEGACY)
  3878. return;
  3879. set_ring_build_skb_enabled(rx_ring);
  3880. #if (PAGE_SIZE < 8192)
  3881. if (adapter->max_frame_size <= IGB_MAX_FRAME_BUILD_SKB)
  3882. return;
  3883. set_ring_uses_large_buffer(rx_ring);
  3884. #endif
  3885. }
  3886. /**
  3887. * igb_configure_rx - Configure receive Unit after Reset
  3888. * @adapter: board private structure
  3889. *
  3890. * Configure the Rx unit of the MAC after a reset.
  3891. **/
  3892. static void igb_configure_rx(struct igb_adapter *adapter)
  3893. {
  3894. int i;
  3895. /* set the correct pool for the PF default MAC address in entry 0 */
  3896. igb_set_default_mac_filter(adapter);
  3897. /* Setup the HW Rx Head and Tail Descriptor Pointers and
  3898. * the Base and Length of the Rx Descriptor Ring
  3899. */
  3900. for (i = 0; i < adapter->num_rx_queues; i++) {
  3901. struct igb_ring *rx_ring = adapter->rx_ring[i];
  3902. igb_set_rx_buffer_len(adapter, rx_ring);
  3903. igb_configure_rx_ring(adapter, rx_ring);
  3904. }
  3905. }
  3906. /**
  3907. * igb_free_tx_resources - Free Tx Resources per Queue
  3908. * @tx_ring: Tx descriptor ring for a specific queue
  3909. *
  3910. * Free all transmit software resources
  3911. **/
  3912. void igb_free_tx_resources(struct igb_ring *tx_ring)
  3913. {
  3914. igb_clean_tx_ring(tx_ring);
  3915. vfree(tx_ring->tx_buffer_info);
  3916. tx_ring->tx_buffer_info = NULL;
  3917. /* if not set, then don't free */
  3918. if (!tx_ring->desc)
  3919. return;
  3920. dma_free_coherent(tx_ring->dev, tx_ring->size,
  3921. tx_ring->desc, tx_ring->dma);
  3922. tx_ring->desc = NULL;
  3923. }
  3924. /**
  3925. * igb_free_all_tx_resources - Free Tx Resources for All Queues
  3926. * @adapter: board private structure
  3927. *
  3928. * Free all transmit software resources
  3929. **/
  3930. static void igb_free_all_tx_resources(struct igb_adapter *adapter)
  3931. {
  3932. int i;
  3933. for (i = 0; i < adapter->num_tx_queues; i++)
  3934. if (adapter->tx_ring[i])
  3935. igb_free_tx_resources(adapter->tx_ring[i]);
  3936. }
  3937. /**
  3938. * igb_clean_tx_ring - Free Tx Buffers
  3939. * @tx_ring: ring to be cleaned
  3940. **/
  3941. static void igb_clean_tx_ring(struct igb_ring *tx_ring)
  3942. {
  3943. u16 i = tx_ring->next_to_clean;
  3944. struct igb_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i];
  3945. while (i != tx_ring->next_to_use) {
  3946. union e1000_adv_tx_desc *eop_desc, *tx_desc;
  3947. /* Free all the Tx ring sk_buffs */
  3948. dev_kfree_skb_any(tx_buffer->skb);
  3949. /* unmap skb header data */
  3950. dma_unmap_single(tx_ring->dev,
  3951. dma_unmap_addr(tx_buffer, dma),
  3952. dma_unmap_len(tx_buffer, len),
  3953. DMA_TO_DEVICE);
  3954. /* check for eop_desc to determine the end of the packet */
  3955. eop_desc = tx_buffer->next_to_watch;
  3956. tx_desc = IGB_TX_DESC(tx_ring, i);
  3957. /* unmap remaining buffers */
  3958. while (tx_desc != eop_desc) {
  3959. tx_buffer++;
  3960. tx_desc++;
  3961. i++;
  3962. if (unlikely(i == tx_ring->count)) {
  3963. i = 0;
  3964. tx_buffer = tx_ring->tx_buffer_info;
  3965. tx_desc = IGB_TX_DESC(tx_ring, 0);
  3966. }
  3967. /* unmap any remaining paged data */
  3968. if (dma_unmap_len(tx_buffer, len))
  3969. dma_unmap_page(tx_ring->dev,
  3970. dma_unmap_addr(tx_buffer, dma),
  3971. dma_unmap_len(tx_buffer, len),
  3972. DMA_TO_DEVICE);
  3973. }
  3974. /* move us one more past the eop_desc for start of next pkt */
  3975. tx_buffer++;
  3976. i++;
  3977. if (unlikely(i == tx_ring->count)) {
  3978. i = 0;
  3979. tx_buffer = tx_ring->tx_buffer_info;
  3980. }
  3981. }
  3982. /* reset BQL for queue */
  3983. netdev_tx_reset_queue(txring_txq(tx_ring));
  3984. /* reset next_to_use and next_to_clean */
  3985. tx_ring->next_to_use = 0;
  3986. tx_ring->next_to_clean = 0;
  3987. }
  3988. /**
  3989. * igb_clean_all_tx_rings - Free Tx Buffers for all queues
  3990. * @adapter: board private structure
  3991. **/
  3992. static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
  3993. {
  3994. int i;
  3995. for (i = 0; i < adapter->num_tx_queues; i++)
  3996. if (adapter->tx_ring[i])
  3997. igb_clean_tx_ring(adapter->tx_ring[i]);
  3998. }
  3999. /**
  4000. * igb_free_rx_resources - Free Rx Resources
  4001. * @rx_ring: ring to clean the resources from
  4002. *
  4003. * Free all receive software resources
  4004. **/
  4005. void igb_free_rx_resources(struct igb_ring *rx_ring)
  4006. {
  4007. igb_clean_rx_ring(rx_ring);
  4008. vfree(rx_ring->rx_buffer_info);
  4009. rx_ring->rx_buffer_info = NULL;
  4010. /* if not set, then don't free */
  4011. if (!rx_ring->desc)
  4012. return;
  4013. dma_free_coherent(rx_ring->dev, rx_ring->size,
  4014. rx_ring->desc, rx_ring->dma);
  4015. rx_ring->desc = NULL;
  4016. }
  4017. /**
  4018. * igb_free_all_rx_resources - Free Rx Resources for All Queues
  4019. * @adapter: board private structure
  4020. *
  4021. * Free all receive software resources
  4022. **/
  4023. static void igb_free_all_rx_resources(struct igb_adapter *adapter)
  4024. {
  4025. int i;
  4026. for (i = 0; i < adapter->num_rx_queues; i++)
  4027. if (adapter->rx_ring[i])
  4028. igb_free_rx_resources(adapter->rx_ring[i]);
  4029. }
  4030. /**
  4031. * igb_clean_rx_ring - Free Rx Buffers per Queue
  4032. * @rx_ring: ring to free buffers from
  4033. **/
  4034. static void igb_clean_rx_ring(struct igb_ring *rx_ring)
  4035. {
  4036. u16 i = rx_ring->next_to_clean;
  4037. dev_kfree_skb(rx_ring->skb);
  4038. rx_ring->skb = NULL;
  4039. /* Free all the Rx ring sk_buffs */
  4040. while (i != rx_ring->next_to_alloc) {
  4041. struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
  4042. /* Invalidate cache lines that may have been written to by
  4043. * device so that we avoid corrupting memory.
  4044. */
  4045. dma_sync_single_range_for_cpu(rx_ring->dev,
  4046. buffer_info->dma,
  4047. buffer_info->page_offset,
  4048. igb_rx_bufsz(rx_ring),
  4049. DMA_FROM_DEVICE);
  4050. /* free resources associated with mapping */
  4051. dma_unmap_page_attrs(rx_ring->dev,
  4052. buffer_info->dma,
  4053. igb_rx_pg_size(rx_ring),
  4054. DMA_FROM_DEVICE,
  4055. IGB_RX_DMA_ATTR);
  4056. __page_frag_cache_drain(buffer_info->page,
  4057. buffer_info->pagecnt_bias);
  4058. i++;
  4059. if (i == rx_ring->count)
  4060. i = 0;
  4061. }
  4062. rx_ring->next_to_alloc = 0;
  4063. rx_ring->next_to_clean = 0;
  4064. rx_ring->next_to_use = 0;
  4065. }
  4066. /**
  4067. * igb_clean_all_rx_rings - Free Rx Buffers for all queues
  4068. * @adapter: board private structure
  4069. **/
  4070. static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
  4071. {
  4072. int i;
  4073. for (i = 0; i < adapter->num_rx_queues; i++)
  4074. if (adapter->rx_ring[i])
  4075. igb_clean_rx_ring(adapter->rx_ring[i]);
  4076. }
  4077. /**
  4078. * igb_set_mac - Change the Ethernet Address of the NIC
  4079. * @netdev: network interface device structure
  4080. * @p: pointer to an address structure
  4081. *
  4082. * Returns 0 on success, negative on failure
  4083. **/
  4084. static int igb_set_mac(struct net_device *netdev, void *p)
  4085. {
  4086. struct igb_adapter *adapter = netdev_priv(netdev);
  4087. struct e1000_hw *hw = &adapter->hw;
  4088. struct sockaddr *addr = p;
  4089. if (!is_valid_ether_addr(addr->sa_data))
  4090. return -EADDRNOTAVAIL;
  4091. memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
  4092. memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
  4093. /* set the correct pool for the new PF MAC address in entry 0 */
  4094. igb_set_default_mac_filter(adapter);
  4095. return 0;
  4096. }
  4097. /**
  4098. * igb_write_mc_addr_list - write multicast addresses to MTA
  4099. * @netdev: network interface device structure
  4100. *
  4101. * Writes multicast address list to the MTA hash table.
  4102. * Returns: -ENOMEM on failure
  4103. * 0 on no addresses written
  4104. * X on writing X addresses to MTA
  4105. **/
  4106. static int igb_write_mc_addr_list(struct net_device *netdev)
  4107. {
  4108. struct igb_adapter *adapter = netdev_priv(netdev);
  4109. struct e1000_hw *hw = &adapter->hw;
  4110. struct netdev_hw_addr *ha;
  4111. u8 *mta_list;
  4112. int i;
  4113. if (netdev_mc_empty(netdev)) {
  4114. /* nothing to program, so clear mc list */
  4115. igb_update_mc_addr_list(hw, NULL, 0);
  4116. igb_restore_vf_multicasts(adapter);
  4117. return 0;
  4118. }
  4119. mta_list = kcalloc(netdev_mc_count(netdev), 6, GFP_ATOMIC);
  4120. if (!mta_list)
  4121. return -ENOMEM;
  4122. /* The shared function expects a packed array of only addresses. */
  4123. i = 0;
  4124. netdev_for_each_mc_addr(ha, netdev)
  4125. memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
  4126. igb_update_mc_addr_list(hw, mta_list, i);
  4127. kfree(mta_list);
  4128. return netdev_mc_count(netdev);
  4129. }
  4130. static int igb_vlan_promisc_enable(struct igb_adapter *adapter)
  4131. {
  4132. struct e1000_hw *hw = &adapter->hw;
  4133. u32 i, pf_id;
  4134. switch (hw->mac.type) {
  4135. case e1000_i210:
  4136. case e1000_i211:
  4137. case e1000_i350:
  4138. /* VLAN filtering needed for VLAN prio filter */
  4139. if (adapter->netdev->features & NETIF_F_NTUPLE)
  4140. break;
  4141. /* fall through */
  4142. case e1000_82576:
  4143. case e1000_82580:
  4144. case e1000_i354:
  4145. /* VLAN filtering needed for pool filtering */
  4146. if (adapter->vfs_allocated_count)
  4147. break;
  4148. /* fall through */
  4149. default:
  4150. return 1;
  4151. }
  4152. /* We are already in VLAN promisc, nothing to do */
  4153. if (adapter->flags & IGB_FLAG_VLAN_PROMISC)
  4154. return 0;
  4155. if (!adapter->vfs_allocated_count)
  4156. goto set_vfta;
  4157. /* Add PF to all active pools */
  4158. pf_id = adapter->vfs_allocated_count + E1000_VLVF_POOLSEL_SHIFT;
  4159. for (i = E1000_VLVF_ARRAY_SIZE; --i;) {
  4160. u32 vlvf = rd32(E1000_VLVF(i));
  4161. vlvf |= BIT(pf_id);
  4162. wr32(E1000_VLVF(i), vlvf);
  4163. }
  4164. set_vfta:
  4165. /* Set all bits in the VLAN filter table array */
  4166. for (i = E1000_VLAN_FILTER_TBL_SIZE; i--;)
  4167. hw->mac.ops.write_vfta(hw, i, ~0U);
  4168. /* Set flag so we don't redo unnecessary work */
  4169. adapter->flags |= IGB_FLAG_VLAN_PROMISC;
  4170. return 0;
  4171. }
  4172. #define VFTA_BLOCK_SIZE 8
  4173. static void igb_scrub_vfta(struct igb_adapter *adapter, u32 vfta_offset)
  4174. {
  4175. struct e1000_hw *hw = &adapter->hw;
  4176. u32 vfta[VFTA_BLOCK_SIZE] = { 0 };
  4177. u32 vid_start = vfta_offset * 32;
  4178. u32 vid_end = vid_start + (VFTA_BLOCK_SIZE * 32);
  4179. u32 i, vid, word, bits, pf_id;
  4180. /* guarantee that we don't scrub out management VLAN */
  4181. vid = adapter->mng_vlan_id;
  4182. if (vid >= vid_start && vid < vid_end)
  4183. vfta[(vid - vid_start) / 32] |= BIT(vid % 32);
  4184. if (!adapter->vfs_allocated_count)
  4185. goto set_vfta;
  4186. pf_id = adapter->vfs_allocated_count + E1000_VLVF_POOLSEL_SHIFT;
  4187. for (i = E1000_VLVF_ARRAY_SIZE; --i;) {
  4188. u32 vlvf = rd32(E1000_VLVF(i));
  4189. /* pull VLAN ID from VLVF */
  4190. vid = vlvf & VLAN_VID_MASK;
  4191. /* only concern ourselves with a certain range */
  4192. if (vid < vid_start || vid >= vid_end)
  4193. continue;
  4194. if (vlvf & E1000_VLVF_VLANID_ENABLE) {
  4195. /* record VLAN ID in VFTA */
  4196. vfta[(vid - vid_start) / 32] |= BIT(vid % 32);
  4197. /* if PF is part of this then continue */
  4198. if (test_bit(vid, adapter->active_vlans))
  4199. continue;
  4200. }
  4201. /* remove PF from the pool */
  4202. bits = ~BIT(pf_id);
  4203. bits &= rd32(E1000_VLVF(i));
  4204. wr32(E1000_VLVF(i), bits);
  4205. }
  4206. set_vfta:
  4207. /* extract values from active_vlans and write back to VFTA */
  4208. for (i = VFTA_BLOCK_SIZE; i--;) {
  4209. vid = (vfta_offset + i) * 32;
  4210. word = vid / BITS_PER_LONG;
  4211. bits = vid % BITS_PER_LONG;
  4212. vfta[i] |= adapter->active_vlans[word] >> bits;
  4213. hw->mac.ops.write_vfta(hw, vfta_offset + i, vfta[i]);
  4214. }
  4215. }
  4216. static void igb_vlan_promisc_disable(struct igb_adapter *adapter)
  4217. {
  4218. u32 i;
  4219. /* We are not in VLAN promisc, nothing to do */
  4220. if (!(adapter->flags & IGB_FLAG_VLAN_PROMISC))
  4221. return;
  4222. /* Set flag so we don't redo unnecessary work */
  4223. adapter->flags &= ~IGB_FLAG_VLAN_PROMISC;
  4224. for (i = 0; i < E1000_VLAN_FILTER_TBL_SIZE; i += VFTA_BLOCK_SIZE)
  4225. igb_scrub_vfta(adapter, i);
  4226. }
  4227. /**
  4228. * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
  4229. * @netdev: network interface device structure
  4230. *
  4231. * The set_rx_mode entry point is called whenever the unicast or multicast
  4232. * address lists or the network interface flags are updated. This routine is
  4233. * responsible for configuring the hardware for proper unicast, multicast,
  4234. * promiscuous mode, and all-multi behavior.
  4235. **/
  4236. static void igb_set_rx_mode(struct net_device *netdev)
  4237. {
  4238. struct igb_adapter *adapter = netdev_priv(netdev);
  4239. struct e1000_hw *hw = &adapter->hw;
  4240. unsigned int vfn = adapter->vfs_allocated_count;
  4241. u32 rctl = 0, vmolr = 0, rlpml = MAX_JUMBO_FRAME_SIZE;
  4242. int count;
  4243. /* Check for Promiscuous and All Multicast modes */
  4244. if (netdev->flags & IFF_PROMISC) {
  4245. rctl |= E1000_RCTL_UPE | E1000_RCTL_MPE;
  4246. vmolr |= E1000_VMOLR_MPME;
  4247. /* enable use of UTA filter to force packets to default pool */
  4248. if (hw->mac.type == e1000_82576)
  4249. vmolr |= E1000_VMOLR_ROPE;
  4250. } else {
  4251. if (netdev->flags & IFF_ALLMULTI) {
  4252. rctl |= E1000_RCTL_MPE;
  4253. vmolr |= E1000_VMOLR_MPME;
  4254. } else {
  4255. /* Write addresses to the MTA, if the attempt fails
  4256. * then we should just turn on promiscuous mode so
  4257. * that we can at least receive multicast traffic
  4258. */
  4259. count = igb_write_mc_addr_list(netdev);
  4260. if (count < 0) {
  4261. rctl |= E1000_RCTL_MPE;
  4262. vmolr |= E1000_VMOLR_MPME;
  4263. } else if (count) {
  4264. vmolr |= E1000_VMOLR_ROMPE;
  4265. }
  4266. }
  4267. }
  4268. /* Write addresses to available RAR registers, if there is not
  4269. * sufficient space to store all the addresses then enable
  4270. * unicast promiscuous mode
  4271. */
  4272. if (__dev_uc_sync(netdev, igb_uc_sync, igb_uc_unsync)) {
  4273. rctl |= E1000_RCTL_UPE;
  4274. vmolr |= E1000_VMOLR_ROPE;
  4275. }
  4276. /* enable VLAN filtering by default */
  4277. rctl |= E1000_RCTL_VFE;
  4278. /* disable VLAN filtering for modes that require it */
  4279. if ((netdev->flags & IFF_PROMISC) ||
  4280. (netdev->features & NETIF_F_RXALL)) {
  4281. /* if we fail to set all rules then just clear VFE */
  4282. if (igb_vlan_promisc_enable(adapter))
  4283. rctl &= ~E1000_RCTL_VFE;
  4284. } else {
  4285. igb_vlan_promisc_disable(adapter);
  4286. }
  4287. /* update state of unicast, multicast, and VLAN filtering modes */
  4288. rctl |= rd32(E1000_RCTL) & ~(E1000_RCTL_UPE | E1000_RCTL_MPE |
  4289. E1000_RCTL_VFE);
  4290. wr32(E1000_RCTL, rctl);
  4291. #if (PAGE_SIZE < 8192)
  4292. if (!adapter->vfs_allocated_count) {
  4293. if (adapter->max_frame_size <= IGB_MAX_FRAME_BUILD_SKB)
  4294. rlpml = IGB_MAX_FRAME_BUILD_SKB;
  4295. }
  4296. #endif
  4297. wr32(E1000_RLPML, rlpml);
  4298. /* In order to support SR-IOV and eventually VMDq it is necessary to set
  4299. * the VMOLR to enable the appropriate modes. Without this workaround
  4300. * we will have issues with VLAN tag stripping not being done for frames
  4301. * that are only arriving because we are the default pool
  4302. */
  4303. if ((hw->mac.type < e1000_82576) || (hw->mac.type > e1000_i350))
  4304. return;
  4305. /* set UTA to appropriate mode */
  4306. igb_set_uta(adapter, !!(vmolr & E1000_VMOLR_ROPE));
  4307. vmolr |= rd32(E1000_VMOLR(vfn)) &
  4308. ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
  4309. /* enable Rx jumbo frames, restrict as needed to support build_skb */
  4310. vmolr &= ~E1000_VMOLR_RLPML_MASK;
  4311. #if (PAGE_SIZE < 8192)
  4312. if (adapter->max_frame_size <= IGB_MAX_FRAME_BUILD_SKB)
  4313. vmolr |= IGB_MAX_FRAME_BUILD_SKB;
  4314. else
  4315. #endif
  4316. vmolr |= MAX_JUMBO_FRAME_SIZE;
  4317. vmolr |= E1000_VMOLR_LPE;
  4318. wr32(E1000_VMOLR(vfn), vmolr);
  4319. igb_restore_vf_multicasts(adapter);
  4320. }
  4321. static void igb_check_wvbr(struct igb_adapter *adapter)
  4322. {
  4323. struct e1000_hw *hw = &adapter->hw;
  4324. u32 wvbr = 0;
  4325. switch (hw->mac.type) {
  4326. case e1000_82576:
  4327. case e1000_i350:
  4328. wvbr = rd32(E1000_WVBR);
  4329. if (!wvbr)
  4330. return;
  4331. break;
  4332. default:
  4333. break;
  4334. }
  4335. adapter->wvbr |= wvbr;
  4336. }
  4337. #define IGB_STAGGERED_QUEUE_OFFSET 8
  4338. static void igb_spoof_check(struct igb_adapter *adapter)
  4339. {
  4340. int j;
  4341. if (!adapter->wvbr)
  4342. return;
  4343. for (j = 0; j < adapter->vfs_allocated_count; j++) {
  4344. if (adapter->wvbr & BIT(j) ||
  4345. adapter->wvbr & BIT(j + IGB_STAGGERED_QUEUE_OFFSET)) {
  4346. dev_warn(&adapter->pdev->dev,
  4347. "Spoof event(s) detected on VF %d\n", j);
  4348. adapter->wvbr &=
  4349. ~(BIT(j) |
  4350. BIT(j + IGB_STAGGERED_QUEUE_OFFSET));
  4351. }
  4352. }
  4353. }
  4354. /* Need to wait a few seconds after link up to get diagnostic information from
  4355. * the phy
  4356. */
  4357. static void igb_update_phy_info(struct timer_list *t)
  4358. {
  4359. struct igb_adapter *adapter = from_timer(adapter, t, phy_info_timer);
  4360. igb_get_phy_info(&adapter->hw);
  4361. }
  4362. /**
  4363. * igb_has_link - check shared code for link and determine up/down
  4364. * @adapter: pointer to driver private info
  4365. **/
  4366. bool igb_has_link(struct igb_adapter *adapter)
  4367. {
  4368. struct e1000_hw *hw = &adapter->hw;
  4369. bool link_active = false;
  4370. /* get_link_status is set on LSC (link status) interrupt or
  4371. * rx sequence error interrupt. get_link_status will stay
  4372. * false until the e1000_check_for_link establishes link
  4373. * for copper adapters ONLY
  4374. */
  4375. switch (hw->phy.media_type) {
  4376. case e1000_media_type_copper:
  4377. if (!hw->mac.get_link_status)
  4378. return true;
  4379. /* fall through */
  4380. case e1000_media_type_internal_serdes:
  4381. hw->mac.ops.check_for_link(hw);
  4382. link_active = !hw->mac.get_link_status;
  4383. break;
  4384. default:
  4385. case e1000_media_type_unknown:
  4386. break;
  4387. }
  4388. if (((hw->mac.type == e1000_i210) ||
  4389. (hw->mac.type == e1000_i211)) &&
  4390. (hw->phy.id == I210_I_PHY_ID)) {
  4391. if (!netif_carrier_ok(adapter->netdev)) {
  4392. adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE;
  4393. } else if (!(adapter->flags & IGB_FLAG_NEED_LINK_UPDATE)) {
  4394. adapter->flags |= IGB_FLAG_NEED_LINK_UPDATE;
  4395. adapter->link_check_timeout = jiffies;
  4396. }
  4397. }
  4398. return link_active;
  4399. }
  4400. static bool igb_thermal_sensor_event(struct e1000_hw *hw, u32 event)
  4401. {
  4402. bool ret = false;
  4403. u32 ctrl_ext, thstat;
  4404. /* check for thermal sensor event on i350 copper only */
  4405. if (hw->mac.type == e1000_i350) {
  4406. thstat = rd32(E1000_THSTAT);
  4407. ctrl_ext = rd32(E1000_CTRL_EXT);
  4408. if ((hw->phy.media_type == e1000_media_type_copper) &&
  4409. !(ctrl_ext & E1000_CTRL_EXT_LINK_MODE_SGMII))
  4410. ret = !!(thstat & event);
  4411. }
  4412. return ret;
  4413. }
  4414. /**
  4415. * igb_check_lvmmc - check for malformed packets received
  4416. * and indicated in LVMMC register
  4417. * @adapter: pointer to adapter
  4418. **/
  4419. static void igb_check_lvmmc(struct igb_adapter *adapter)
  4420. {
  4421. struct e1000_hw *hw = &adapter->hw;
  4422. u32 lvmmc;
  4423. lvmmc = rd32(E1000_LVMMC);
  4424. if (lvmmc) {
  4425. if (unlikely(net_ratelimit())) {
  4426. netdev_warn(adapter->netdev,
  4427. "malformed Tx packet detected and dropped, LVMMC:0x%08x\n",
  4428. lvmmc);
  4429. }
  4430. }
  4431. }
  4432. /**
  4433. * igb_watchdog - Timer Call-back
  4434. * @data: pointer to adapter cast into an unsigned long
  4435. **/
  4436. static void igb_watchdog(struct timer_list *t)
  4437. {
  4438. struct igb_adapter *adapter = from_timer(adapter, t, watchdog_timer);
  4439. /* Do the rest outside of interrupt context */
  4440. schedule_work(&adapter->watchdog_task);
  4441. }
  4442. static void igb_watchdog_task(struct work_struct *work)
  4443. {
  4444. struct igb_adapter *adapter = container_of(work,
  4445. struct igb_adapter,
  4446. watchdog_task);
  4447. struct e1000_hw *hw = &adapter->hw;
  4448. struct e1000_phy_info *phy = &hw->phy;
  4449. struct net_device *netdev = adapter->netdev;
  4450. u32 link;
  4451. int i;
  4452. u32 connsw;
  4453. u16 phy_data, retry_count = 20;
  4454. link = igb_has_link(adapter);
  4455. if (adapter->flags & IGB_FLAG_NEED_LINK_UPDATE) {
  4456. if (time_after(jiffies, (adapter->link_check_timeout + HZ)))
  4457. adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE;
  4458. else
  4459. link = false;
  4460. }
  4461. /* Force link down if we have fiber to swap to */
  4462. if (adapter->flags & IGB_FLAG_MAS_ENABLE) {
  4463. if (hw->phy.media_type == e1000_media_type_copper) {
  4464. connsw = rd32(E1000_CONNSW);
  4465. if (!(connsw & E1000_CONNSW_AUTOSENSE_EN))
  4466. link = 0;
  4467. }
  4468. }
  4469. if (link) {
  4470. /* Perform a reset if the media type changed. */
  4471. if (hw->dev_spec._82575.media_changed) {
  4472. hw->dev_spec._82575.media_changed = false;
  4473. adapter->flags |= IGB_FLAG_MEDIA_RESET;
  4474. igb_reset(adapter);
  4475. }
  4476. /* Cancel scheduled suspend requests. */
  4477. pm_runtime_resume(netdev->dev.parent);
  4478. if (!netif_carrier_ok(netdev)) {
  4479. u32 ctrl;
  4480. hw->mac.ops.get_speed_and_duplex(hw,
  4481. &adapter->link_speed,
  4482. &adapter->link_duplex);
  4483. ctrl = rd32(E1000_CTRL);
  4484. /* Links status message must follow this format */
  4485. netdev_info(netdev,
  4486. "igb: %s NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n",
  4487. netdev->name,
  4488. adapter->link_speed,
  4489. adapter->link_duplex == FULL_DUPLEX ?
  4490. "Full" : "Half",
  4491. (ctrl & E1000_CTRL_TFCE) &&
  4492. (ctrl & E1000_CTRL_RFCE) ? "RX/TX" :
  4493. (ctrl & E1000_CTRL_RFCE) ? "RX" :
  4494. (ctrl & E1000_CTRL_TFCE) ? "TX" : "None");
  4495. /* disable EEE if enabled */
  4496. if ((adapter->flags & IGB_FLAG_EEE) &&
  4497. (adapter->link_duplex == HALF_DUPLEX)) {
  4498. dev_info(&adapter->pdev->dev,
  4499. "EEE Disabled: unsupported at half duplex. Re-enable using ethtool when at full duplex.\n");
  4500. adapter->hw.dev_spec._82575.eee_disable = true;
  4501. adapter->flags &= ~IGB_FLAG_EEE;
  4502. }
  4503. /* check if SmartSpeed worked */
  4504. igb_check_downshift(hw);
  4505. if (phy->speed_downgraded)
  4506. netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n");
  4507. /* check for thermal sensor event */
  4508. if (igb_thermal_sensor_event(hw,
  4509. E1000_THSTAT_LINK_THROTTLE))
  4510. netdev_info(netdev, "The network adapter link speed was downshifted because it overheated\n");
  4511. /* adjust timeout factor according to speed/duplex */
  4512. adapter->tx_timeout_factor = 1;
  4513. switch (adapter->link_speed) {
  4514. case SPEED_10:
  4515. adapter->tx_timeout_factor = 14;
  4516. break;
  4517. case SPEED_100:
  4518. /* maybe add some timeout factor ? */
  4519. break;
  4520. }
  4521. if (adapter->link_speed != SPEED_1000)
  4522. goto no_wait;
  4523. /* wait for Remote receiver status OK */
  4524. retry_read_status:
  4525. if (!igb_read_phy_reg(hw, PHY_1000T_STATUS,
  4526. &phy_data)) {
  4527. if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) &&
  4528. retry_count) {
  4529. msleep(100);
  4530. retry_count--;
  4531. goto retry_read_status;
  4532. } else if (!retry_count) {
  4533. dev_err(&adapter->pdev->dev, "exceed max 2 second\n");
  4534. }
  4535. } else {
  4536. dev_err(&adapter->pdev->dev, "read 1000Base-T Status Reg\n");
  4537. }
  4538. no_wait:
  4539. netif_carrier_on(netdev);
  4540. igb_ping_all_vfs(adapter);
  4541. igb_check_vf_rate_limit(adapter);
  4542. /* link state has changed, schedule phy info update */
  4543. if (!test_bit(__IGB_DOWN, &adapter->state))
  4544. mod_timer(&adapter->phy_info_timer,
  4545. round_jiffies(jiffies + 2 * HZ));
  4546. }
  4547. } else {
  4548. if (netif_carrier_ok(netdev)) {
  4549. adapter->link_speed = 0;
  4550. adapter->link_duplex = 0;
  4551. /* check for thermal sensor event */
  4552. if (igb_thermal_sensor_event(hw,
  4553. E1000_THSTAT_PWR_DOWN)) {
  4554. netdev_err(netdev, "The network adapter was stopped because it overheated\n");
  4555. }
  4556. /* Links status message must follow this format */
  4557. netdev_info(netdev, "igb: %s NIC Link is Down\n",
  4558. netdev->name);
  4559. netif_carrier_off(netdev);
  4560. igb_ping_all_vfs(adapter);
  4561. /* link state has changed, schedule phy info update */
  4562. if (!test_bit(__IGB_DOWN, &adapter->state))
  4563. mod_timer(&adapter->phy_info_timer,
  4564. round_jiffies(jiffies + 2 * HZ));
  4565. /* link is down, time to check for alternate media */
  4566. if (adapter->flags & IGB_FLAG_MAS_ENABLE) {
  4567. igb_check_swap_media(adapter);
  4568. if (adapter->flags & IGB_FLAG_MEDIA_RESET) {
  4569. schedule_work(&adapter->reset_task);
  4570. /* return immediately */
  4571. return;
  4572. }
  4573. }
  4574. pm_schedule_suspend(netdev->dev.parent,
  4575. MSEC_PER_SEC * 5);
  4576. /* also check for alternate media here */
  4577. } else if (!netif_carrier_ok(netdev) &&
  4578. (adapter->flags & IGB_FLAG_MAS_ENABLE)) {
  4579. igb_check_swap_media(adapter);
  4580. if (adapter->flags & IGB_FLAG_MEDIA_RESET) {
  4581. schedule_work(&adapter->reset_task);
  4582. /* return immediately */
  4583. return;
  4584. }
  4585. }
  4586. }
  4587. spin_lock(&adapter->stats64_lock);
  4588. igb_update_stats(adapter);
  4589. spin_unlock(&adapter->stats64_lock);
  4590. for (i = 0; i < adapter->num_tx_queues; i++) {
  4591. struct igb_ring *tx_ring = adapter->tx_ring[i];
  4592. if (!netif_carrier_ok(netdev)) {
  4593. /* We've lost link, so the controller stops DMA,
  4594. * but we've got queued Tx work that's never going
  4595. * to get done, so reset controller to flush Tx.
  4596. * (Do the reset outside of interrupt context).
  4597. */
  4598. if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
  4599. adapter->tx_timeout_count++;
  4600. schedule_work(&adapter->reset_task);
  4601. /* return immediately since reset is imminent */
  4602. return;
  4603. }
  4604. }
  4605. /* Force detection of hung controller every watchdog period */
  4606. set_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
  4607. }
  4608. /* Cause software interrupt to ensure Rx ring is cleaned */
  4609. if (adapter->flags & IGB_FLAG_HAS_MSIX) {
  4610. u32 eics = 0;
  4611. for (i = 0; i < adapter->num_q_vectors; i++)
  4612. eics |= adapter->q_vector[i]->eims_value;
  4613. wr32(E1000_EICS, eics);
  4614. } else {
  4615. wr32(E1000_ICS, E1000_ICS_RXDMT0);
  4616. }
  4617. igb_spoof_check(adapter);
  4618. igb_ptp_rx_hang(adapter);
  4619. igb_ptp_tx_hang(adapter);
  4620. /* Check LVMMC register on i350/i354 only */
  4621. if ((adapter->hw.mac.type == e1000_i350) ||
  4622. (adapter->hw.mac.type == e1000_i354))
  4623. igb_check_lvmmc(adapter);
  4624. /* Reset the timer */
  4625. if (!test_bit(__IGB_DOWN, &adapter->state)) {
  4626. if (adapter->flags & IGB_FLAG_NEED_LINK_UPDATE)
  4627. mod_timer(&adapter->watchdog_timer,
  4628. round_jiffies(jiffies + HZ));
  4629. else
  4630. mod_timer(&adapter->watchdog_timer,
  4631. round_jiffies(jiffies + 2 * HZ));
  4632. }
  4633. }
  4634. enum latency_range {
  4635. lowest_latency = 0,
  4636. low_latency = 1,
  4637. bulk_latency = 2,
  4638. latency_invalid = 255
  4639. };
  4640. /**
  4641. * igb_update_ring_itr - update the dynamic ITR value based on packet size
  4642. * @q_vector: pointer to q_vector
  4643. *
  4644. * Stores a new ITR value based on strictly on packet size. This
  4645. * algorithm is less sophisticated than that used in igb_update_itr,
  4646. * due to the difficulty of synchronizing statistics across multiple
  4647. * receive rings. The divisors and thresholds used by this function
  4648. * were determined based on theoretical maximum wire speed and testing
  4649. * data, in order to minimize response time while increasing bulk
  4650. * throughput.
  4651. * This functionality is controlled by ethtool's coalescing settings.
  4652. * NOTE: This function is called only when operating in a multiqueue
  4653. * receive environment.
  4654. **/
  4655. static void igb_update_ring_itr(struct igb_q_vector *q_vector)
  4656. {
  4657. int new_val = q_vector->itr_val;
  4658. int avg_wire_size = 0;
  4659. struct igb_adapter *adapter = q_vector->adapter;
  4660. unsigned int packets;
  4661. /* For non-gigabit speeds, just fix the interrupt rate at 4000
  4662. * ints/sec - ITR timer value of 120 ticks.
  4663. */
  4664. if (adapter->link_speed != SPEED_1000) {
  4665. new_val = IGB_4K_ITR;
  4666. goto set_itr_val;
  4667. }
  4668. packets = q_vector->rx.total_packets;
  4669. if (packets)
  4670. avg_wire_size = q_vector->rx.total_bytes / packets;
  4671. packets = q_vector->tx.total_packets;
  4672. if (packets)
  4673. avg_wire_size = max_t(u32, avg_wire_size,
  4674. q_vector->tx.total_bytes / packets);
  4675. /* if avg_wire_size isn't set no work was done */
  4676. if (!avg_wire_size)
  4677. goto clear_counts;
  4678. /* Add 24 bytes to size to account for CRC, preamble, and gap */
  4679. avg_wire_size += 24;
  4680. /* Don't starve jumbo frames */
  4681. avg_wire_size = min(avg_wire_size, 3000);
  4682. /* Give a little boost to mid-size frames */
  4683. if ((avg_wire_size > 300) && (avg_wire_size < 1200))
  4684. new_val = avg_wire_size / 3;
  4685. else
  4686. new_val = avg_wire_size / 2;
  4687. /* conservative mode (itr 3) eliminates the lowest_latency setting */
  4688. if (new_val < IGB_20K_ITR &&
  4689. ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
  4690. (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
  4691. new_val = IGB_20K_ITR;
  4692. set_itr_val:
  4693. if (new_val != q_vector->itr_val) {
  4694. q_vector->itr_val = new_val;
  4695. q_vector->set_itr = 1;
  4696. }
  4697. clear_counts:
  4698. q_vector->rx.total_bytes = 0;
  4699. q_vector->rx.total_packets = 0;
  4700. q_vector->tx.total_bytes = 0;
  4701. q_vector->tx.total_packets = 0;
  4702. }
  4703. /**
  4704. * igb_update_itr - update the dynamic ITR value based on statistics
  4705. * @q_vector: pointer to q_vector
  4706. * @ring_container: ring info to update the itr for
  4707. *
  4708. * Stores a new ITR value based on packets and byte
  4709. * counts during the last interrupt. The advantage of per interrupt
  4710. * computation is faster updates and more accurate ITR for the current
  4711. * traffic pattern. Constants in this function were computed
  4712. * based on theoretical maximum wire speed and thresholds were set based
  4713. * on testing data as well as attempting to minimize response time
  4714. * while increasing bulk throughput.
  4715. * This functionality is controlled by ethtool's coalescing settings.
  4716. * NOTE: These calculations are only valid when operating in a single-
  4717. * queue environment.
  4718. **/
  4719. static void igb_update_itr(struct igb_q_vector *q_vector,
  4720. struct igb_ring_container *ring_container)
  4721. {
  4722. unsigned int packets = ring_container->total_packets;
  4723. unsigned int bytes = ring_container->total_bytes;
  4724. u8 itrval = ring_container->itr;
  4725. /* no packets, exit with status unchanged */
  4726. if (packets == 0)
  4727. return;
  4728. switch (itrval) {
  4729. case lowest_latency:
  4730. /* handle TSO and jumbo frames */
  4731. if (bytes/packets > 8000)
  4732. itrval = bulk_latency;
  4733. else if ((packets < 5) && (bytes > 512))
  4734. itrval = low_latency;
  4735. break;
  4736. case low_latency: /* 50 usec aka 20000 ints/s */
  4737. if (bytes > 10000) {
  4738. /* this if handles the TSO accounting */
  4739. if (bytes/packets > 8000)
  4740. itrval = bulk_latency;
  4741. else if ((packets < 10) || ((bytes/packets) > 1200))
  4742. itrval = bulk_latency;
  4743. else if ((packets > 35))
  4744. itrval = lowest_latency;
  4745. } else if (bytes/packets > 2000) {
  4746. itrval = bulk_latency;
  4747. } else if (packets <= 2 && bytes < 512) {
  4748. itrval = lowest_latency;
  4749. }
  4750. break;
  4751. case bulk_latency: /* 250 usec aka 4000 ints/s */
  4752. if (bytes > 25000) {
  4753. if (packets > 35)
  4754. itrval = low_latency;
  4755. } else if (bytes < 1500) {
  4756. itrval = low_latency;
  4757. }
  4758. break;
  4759. }
  4760. /* clear work counters since we have the values we need */
  4761. ring_container->total_bytes = 0;
  4762. ring_container->total_packets = 0;
  4763. /* write updated itr to ring container */
  4764. ring_container->itr = itrval;
  4765. }
  4766. static void igb_set_itr(struct igb_q_vector *q_vector)
  4767. {
  4768. struct igb_adapter *adapter = q_vector->adapter;
  4769. u32 new_itr = q_vector->itr_val;
  4770. u8 current_itr = 0;
  4771. /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
  4772. if (adapter->link_speed != SPEED_1000) {
  4773. current_itr = 0;
  4774. new_itr = IGB_4K_ITR;
  4775. goto set_itr_now;
  4776. }
  4777. igb_update_itr(q_vector, &q_vector->tx);
  4778. igb_update_itr(q_vector, &q_vector->rx);
  4779. current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
  4780. /* conservative mode (itr 3) eliminates the lowest_latency setting */
  4781. if (current_itr == lowest_latency &&
  4782. ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
  4783. (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
  4784. current_itr = low_latency;
  4785. switch (current_itr) {
  4786. /* counts and packets in update_itr are dependent on these numbers */
  4787. case lowest_latency:
  4788. new_itr = IGB_70K_ITR; /* 70,000 ints/sec */
  4789. break;
  4790. case low_latency:
  4791. new_itr = IGB_20K_ITR; /* 20,000 ints/sec */
  4792. break;
  4793. case bulk_latency:
  4794. new_itr = IGB_4K_ITR; /* 4,000 ints/sec */
  4795. break;
  4796. default:
  4797. break;
  4798. }
  4799. set_itr_now:
  4800. if (new_itr != q_vector->itr_val) {
  4801. /* this attempts to bias the interrupt rate towards Bulk
  4802. * by adding intermediate steps when interrupt rate is
  4803. * increasing
  4804. */
  4805. new_itr = new_itr > q_vector->itr_val ?
  4806. max((new_itr * q_vector->itr_val) /
  4807. (new_itr + (q_vector->itr_val >> 2)),
  4808. new_itr) : new_itr;
  4809. /* Don't write the value here; it resets the adapter's
  4810. * internal timer, and causes us to delay far longer than
  4811. * we should between interrupts. Instead, we write the ITR
  4812. * value at the beginning of the next interrupt so the timing
  4813. * ends up being correct.
  4814. */
  4815. q_vector->itr_val = new_itr;
  4816. q_vector->set_itr = 1;
  4817. }
  4818. }
  4819. static void igb_tx_ctxtdesc(struct igb_ring *tx_ring,
  4820. struct igb_tx_buffer *first,
  4821. u32 vlan_macip_lens, u32 type_tucmd,
  4822. u32 mss_l4len_idx)
  4823. {
  4824. struct e1000_adv_tx_context_desc *context_desc;
  4825. u16 i = tx_ring->next_to_use;
  4826. struct timespec64 ts;
  4827. context_desc = IGB_TX_CTXTDESC(tx_ring, i);
  4828. i++;
  4829. tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
  4830. /* set bits to identify this as an advanced context descriptor */
  4831. type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
  4832. /* For 82575, context index must be unique per ring. */
  4833. if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
  4834. mss_l4len_idx |= tx_ring->reg_idx << 4;
  4835. context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
  4836. context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
  4837. context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
  4838. /* We assume there is always a valid tx time available. Invalid times
  4839. * should have been handled by the upper layers.
  4840. */
  4841. if (tx_ring->launchtime_enable) {
  4842. ts = ktime_to_timespec64(first->skb->tstamp);
  4843. first->skb->tstamp = ktime_set(0, 0);
  4844. context_desc->seqnum_seed = cpu_to_le32(ts.tv_nsec / 32);
  4845. } else {
  4846. context_desc->seqnum_seed = 0;
  4847. }
  4848. }
  4849. static int igb_tso(struct igb_ring *tx_ring,
  4850. struct igb_tx_buffer *first,
  4851. u8 *hdr_len)
  4852. {
  4853. u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
  4854. struct sk_buff *skb = first->skb;
  4855. union {
  4856. struct iphdr *v4;
  4857. struct ipv6hdr *v6;
  4858. unsigned char *hdr;
  4859. } ip;
  4860. union {
  4861. struct tcphdr *tcp;
  4862. struct udphdr *udp;
  4863. unsigned char *hdr;
  4864. } l4;
  4865. u32 paylen, l4_offset;
  4866. int err;
  4867. if (skb->ip_summed != CHECKSUM_PARTIAL)
  4868. return 0;
  4869. if (!skb_is_gso(skb))
  4870. return 0;
  4871. err = skb_cow_head(skb, 0);
  4872. if (err < 0)
  4873. return err;
  4874. ip.hdr = skb_network_header(skb);
  4875. l4.hdr = skb_checksum_start(skb);
  4876. /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
  4877. type_tucmd = (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) ?
  4878. E1000_ADVTXD_TUCMD_L4T_UDP : E1000_ADVTXD_TUCMD_L4T_TCP;
  4879. /* initialize outer IP header fields */
  4880. if (ip.v4->version == 4) {
  4881. unsigned char *csum_start = skb_checksum_start(skb);
  4882. unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4);
  4883. /* IP header will have to cancel out any data that
  4884. * is not a part of the outer IP header
  4885. */
  4886. ip.v4->check = csum_fold(csum_partial(trans_start,
  4887. csum_start - trans_start,
  4888. 0));
  4889. type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
  4890. ip.v4->tot_len = 0;
  4891. first->tx_flags |= IGB_TX_FLAGS_TSO |
  4892. IGB_TX_FLAGS_CSUM |
  4893. IGB_TX_FLAGS_IPV4;
  4894. } else {
  4895. ip.v6->payload_len = 0;
  4896. first->tx_flags |= IGB_TX_FLAGS_TSO |
  4897. IGB_TX_FLAGS_CSUM;
  4898. }
  4899. /* determine offset of inner transport header */
  4900. l4_offset = l4.hdr - skb->data;
  4901. /* remove payload length from inner checksum */
  4902. paylen = skb->len - l4_offset;
  4903. if (type_tucmd & E1000_ADVTXD_TUCMD_L4T_TCP) {
  4904. /* compute length of segmentation header */
  4905. *hdr_len = (l4.tcp->doff * 4) + l4_offset;
  4906. csum_replace_by_diff(&l4.tcp->check,
  4907. (__force __wsum)htonl(paylen));
  4908. } else {
  4909. /* compute length of segmentation header */
  4910. *hdr_len = sizeof(*l4.udp) + l4_offset;
  4911. csum_replace_by_diff(&l4.udp->check,
  4912. (__force __wsum)htonl(paylen));
  4913. }
  4914. /* update gso size and bytecount with header size */
  4915. first->gso_segs = skb_shinfo(skb)->gso_segs;
  4916. first->bytecount += (first->gso_segs - 1) * *hdr_len;
  4917. /* MSS L4LEN IDX */
  4918. mss_l4len_idx = (*hdr_len - l4_offset) << E1000_ADVTXD_L4LEN_SHIFT;
  4919. mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
  4920. /* VLAN MACLEN IPLEN */
  4921. vlan_macip_lens = l4.hdr - ip.hdr;
  4922. vlan_macip_lens |= (ip.hdr - skb->data) << E1000_ADVTXD_MACLEN_SHIFT;
  4923. vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
  4924. igb_tx_ctxtdesc(tx_ring, first, vlan_macip_lens,
  4925. type_tucmd, mss_l4len_idx);
  4926. return 1;
  4927. }
  4928. static inline bool igb_ipv6_csum_is_sctp(struct sk_buff *skb)
  4929. {
  4930. unsigned int offset = 0;
  4931. ipv6_find_hdr(skb, &offset, IPPROTO_SCTP, NULL, NULL);
  4932. return offset == skb_checksum_start_offset(skb);
  4933. }
  4934. static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first)
  4935. {
  4936. struct sk_buff *skb = first->skb;
  4937. u32 vlan_macip_lens = 0;
  4938. u32 type_tucmd = 0;
  4939. if (skb->ip_summed != CHECKSUM_PARTIAL) {
  4940. csum_failed:
  4941. if (!(first->tx_flags & IGB_TX_FLAGS_VLAN) &&
  4942. !tx_ring->launchtime_enable)
  4943. return;
  4944. goto no_csum;
  4945. }
  4946. switch (skb->csum_offset) {
  4947. case offsetof(struct tcphdr, check):
  4948. type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
  4949. /* fall through */
  4950. case offsetof(struct udphdr, check):
  4951. break;
  4952. case offsetof(struct sctphdr, checksum):
  4953. /* validate that this is actually an SCTP request */
  4954. if (((first->protocol == htons(ETH_P_IP)) &&
  4955. (ip_hdr(skb)->protocol == IPPROTO_SCTP)) ||
  4956. ((first->protocol == htons(ETH_P_IPV6)) &&
  4957. igb_ipv6_csum_is_sctp(skb))) {
  4958. type_tucmd = E1000_ADVTXD_TUCMD_L4T_SCTP;
  4959. break;
  4960. }
  4961. /* fall through */
  4962. default:
  4963. skb_checksum_help(skb);
  4964. goto csum_failed;
  4965. }
  4966. /* update TX checksum flag */
  4967. first->tx_flags |= IGB_TX_FLAGS_CSUM;
  4968. vlan_macip_lens = skb_checksum_start_offset(skb) -
  4969. skb_network_offset(skb);
  4970. no_csum:
  4971. vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
  4972. vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
  4973. igb_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, type_tucmd, 0);
  4974. }
  4975. #define IGB_SET_FLAG(_input, _flag, _result) \
  4976. ((_flag <= _result) ? \
  4977. ((u32)(_input & _flag) * (_result / _flag)) : \
  4978. ((u32)(_input & _flag) / (_flag / _result)))
  4979. static u32 igb_tx_cmd_type(struct sk_buff *skb, u32 tx_flags)
  4980. {
  4981. /* set type for advanced descriptor with frame checksum insertion */
  4982. u32 cmd_type = E1000_ADVTXD_DTYP_DATA |
  4983. E1000_ADVTXD_DCMD_DEXT |
  4984. E1000_ADVTXD_DCMD_IFCS;
  4985. /* set HW vlan bit if vlan is present */
  4986. cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_VLAN,
  4987. (E1000_ADVTXD_DCMD_VLE));
  4988. /* set segmentation bits for TSO */
  4989. cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_TSO,
  4990. (E1000_ADVTXD_DCMD_TSE));
  4991. /* set timestamp bit if present */
  4992. cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_TSTAMP,
  4993. (E1000_ADVTXD_MAC_TSTAMP));
  4994. /* insert frame checksum */
  4995. cmd_type ^= IGB_SET_FLAG(skb->no_fcs, 1, E1000_ADVTXD_DCMD_IFCS);
  4996. return cmd_type;
  4997. }
  4998. static void igb_tx_olinfo_status(struct igb_ring *tx_ring,
  4999. union e1000_adv_tx_desc *tx_desc,
  5000. u32 tx_flags, unsigned int paylen)
  5001. {
  5002. u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT;
  5003. /* 82575 requires a unique index per ring */
  5004. if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
  5005. olinfo_status |= tx_ring->reg_idx << 4;
  5006. /* insert L4 checksum */
  5007. olinfo_status |= IGB_SET_FLAG(tx_flags,
  5008. IGB_TX_FLAGS_CSUM,
  5009. (E1000_TXD_POPTS_TXSM << 8));
  5010. /* insert IPv4 checksum */
  5011. olinfo_status |= IGB_SET_FLAG(tx_flags,
  5012. IGB_TX_FLAGS_IPV4,
  5013. (E1000_TXD_POPTS_IXSM << 8));
  5014. tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
  5015. }
  5016. static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
  5017. {
  5018. struct net_device *netdev = tx_ring->netdev;
  5019. netif_stop_subqueue(netdev, tx_ring->queue_index);
  5020. /* Herbert's original patch had:
  5021. * smp_mb__after_netif_stop_queue();
  5022. * but since that doesn't exist yet, just open code it.
  5023. */
  5024. smp_mb();
  5025. /* We need to check again in a case another CPU has just
  5026. * made room available.
  5027. */
  5028. if (igb_desc_unused(tx_ring) < size)
  5029. return -EBUSY;
  5030. /* A reprieve! */
  5031. netif_wake_subqueue(netdev, tx_ring->queue_index);
  5032. u64_stats_update_begin(&tx_ring->tx_syncp2);
  5033. tx_ring->tx_stats.restart_queue2++;
  5034. u64_stats_update_end(&tx_ring->tx_syncp2);
  5035. return 0;
  5036. }
  5037. static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
  5038. {
  5039. if (igb_desc_unused(tx_ring) >= size)
  5040. return 0;
  5041. return __igb_maybe_stop_tx(tx_ring, size);
  5042. }
  5043. static int igb_tx_map(struct igb_ring *tx_ring,
  5044. struct igb_tx_buffer *first,
  5045. const u8 hdr_len)
  5046. {
  5047. struct sk_buff *skb = first->skb;
  5048. struct igb_tx_buffer *tx_buffer;
  5049. union e1000_adv_tx_desc *tx_desc;
  5050. skb_frag_t *frag;
  5051. dma_addr_t dma;
  5052. unsigned int data_len, size;
  5053. u32 tx_flags = first->tx_flags;
  5054. u32 cmd_type = igb_tx_cmd_type(skb, tx_flags);
  5055. u16 i = tx_ring->next_to_use;
  5056. tx_desc = IGB_TX_DESC(tx_ring, i);
  5057. igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len);
  5058. size = skb_headlen(skb);
  5059. data_len = skb->data_len;
  5060. dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
  5061. tx_buffer = first;
  5062. for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
  5063. if (dma_mapping_error(tx_ring->dev, dma))
  5064. goto dma_error;
  5065. /* record length, and DMA address */
  5066. dma_unmap_len_set(tx_buffer, len, size);
  5067. dma_unmap_addr_set(tx_buffer, dma, dma);
  5068. tx_desc->read.buffer_addr = cpu_to_le64(dma);
  5069. while (unlikely(size > IGB_MAX_DATA_PER_TXD)) {
  5070. tx_desc->read.cmd_type_len =
  5071. cpu_to_le32(cmd_type ^ IGB_MAX_DATA_PER_TXD);
  5072. i++;
  5073. tx_desc++;
  5074. if (i == tx_ring->count) {
  5075. tx_desc = IGB_TX_DESC(tx_ring, 0);
  5076. i = 0;
  5077. }
  5078. tx_desc->read.olinfo_status = 0;
  5079. dma += IGB_MAX_DATA_PER_TXD;
  5080. size -= IGB_MAX_DATA_PER_TXD;
  5081. tx_desc->read.buffer_addr = cpu_to_le64(dma);
  5082. }
  5083. if (likely(!data_len))
  5084. break;
  5085. tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
  5086. i++;
  5087. tx_desc++;
  5088. if (i == tx_ring->count) {
  5089. tx_desc = IGB_TX_DESC(tx_ring, 0);
  5090. i = 0;
  5091. }
  5092. tx_desc->read.olinfo_status = 0;
  5093. size = skb_frag_size(frag);
  5094. data_len -= size;
  5095. dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
  5096. size, DMA_TO_DEVICE);
  5097. tx_buffer = &tx_ring->tx_buffer_info[i];
  5098. }
  5099. /* write last descriptor with RS and EOP bits */
  5100. cmd_type |= size | IGB_TXD_DCMD;
  5101. tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
  5102. netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
  5103. /* set the timestamp */
  5104. first->time_stamp = jiffies;
  5105. skb_tx_timestamp(skb);
  5106. /* Force memory writes to complete before letting h/w know there
  5107. * are new descriptors to fetch. (Only applicable for weak-ordered
  5108. * memory model archs, such as IA-64).
  5109. *
  5110. * We also need this memory barrier to make certain all of the
  5111. * status bits have been updated before next_to_watch is written.
  5112. */
  5113. dma_wmb();
  5114. /* set next_to_watch value indicating a packet is present */
  5115. first->next_to_watch = tx_desc;
  5116. i++;
  5117. if (i == tx_ring->count)
  5118. i = 0;
  5119. tx_ring->next_to_use = i;
  5120. /* Make sure there is space in the ring for the next send. */
  5121. igb_maybe_stop_tx(tx_ring, DESC_NEEDED);
  5122. if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) {
  5123. writel(i, tx_ring->tail);
  5124. }
  5125. return 0;
  5126. dma_error:
  5127. dev_err(tx_ring->dev, "TX DMA map failed\n");
  5128. tx_buffer = &tx_ring->tx_buffer_info[i];
  5129. /* clear dma mappings for failed tx_buffer_info map */
  5130. while (tx_buffer != first) {
  5131. if (dma_unmap_len(tx_buffer, len))
  5132. dma_unmap_page(tx_ring->dev,
  5133. dma_unmap_addr(tx_buffer, dma),
  5134. dma_unmap_len(tx_buffer, len),
  5135. DMA_TO_DEVICE);
  5136. dma_unmap_len_set(tx_buffer, len, 0);
  5137. if (i-- == 0)
  5138. i += tx_ring->count;
  5139. tx_buffer = &tx_ring->tx_buffer_info[i];
  5140. }
  5141. if (dma_unmap_len(tx_buffer, len))
  5142. dma_unmap_single(tx_ring->dev,
  5143. dma_unmap_addr(tx_buffer, dma),
  5144. dma_unmap_len(tx_buffer, len),
  5145. DMA_TO_DEVICE);
  5146. dma_unmap_len_set(tx_buffer, len, 0);
  5147. dev_kfree_skb_any(tx_buffer->skb);
  5148. tx_buffer->skb = NULL;
  5149. tx_ring->next_to_use = i;
  5150. return -1;
  5151. }
  5152. netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
  5153. struct igb_ring *tx_ring)
  5154. {
  5155. struct igb_tx_buffer *first;
  5156. int tso;
  5157. u32 tx_flags = 0;
  5158. unsigned short f;
  5159. u16 count = TXD_USE_COUNT(skb_headlen(skb));
  5160. __be16 protocol = vlan_get_protocol(skb);
  5161. u8 hdr_len = 0;
  5162. /* need: 1 descriptor per page * PAGE_SIZE/IGB_MAX_DATA_PER_TXD,
  5163. * + 1 desc for skb_headlen/IGB_MAX_DATA_PER_TXD,
  5164. * + 2 desc gap to keep tail from touching head,
  5165. * + 1 desc for context descriptor,
  5166. * otherwise try next time
  5167. */
  5168. for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
  5169. count += TXD_USE_COUNT(skb_frag_size(
  5170. &skb_shinfo(skb)->frags[f]));
  5171. if (igb_maybe_stop_tx(tx_ring, count + 3)) {
  5172. /* this is a hard error */
  5173. return NETDEV_TX_BUSY;
  5174. }
  5175. /* record the location of the first descriptor for this packet */
  5176. first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
  5177. first->skb = skb;
  5178. first->bytecount = skb->len;
  5179. first->gso_segs = 1;
  5180. if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
  5181. struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
  5182. if (adapter->tstamp_config.tx_type == HWTSTAMP_TX_ON &&
  5183. !test_and_set_bit_lock(__IGB_PTP_TX_IN_PROGRESS,
  5184. &adapter->state)) {
  5185. skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
  5186. tx_flags |= IGB_TX_FLAGS_TSTAMP;
  5187. adapter->ptp_tx_skb = skb_get(skb);
  5188. adapter->ptp_tx_start = jiffies;
  5189. if (adapter->hw.mac.type == e1000_82576)
  5190. schedule_work(&adapter->ptp_tx_work);
  5191. } else {
  5192. adapter->tx_hwtstamp_skipped++;
  5193. }
  5194. }
  5195. if (skb_vlan_tag_present(skb)) {
  5196. tx_flags |= IGB_TX_FLAGS_VLAN;
  5197. tx_flags |= (skb_vlan_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
  5198. }
  5199. /* record initial flags and protocol */
  5200. first->tx_flags = tx_flags;
  5201. first->protocol = protocol;
  5202. tso = igb_tso(tx_ring, first, &hdr_len);
  5203. if (tso < 0)
  5204. goto out_drop;
  5205. else if (!tso)
  5206. igb_tx_csum(tx_ring, first);
  5207. if (igb_tx_map(tx_ring, first, hdr_len))
  5208. goto cleanup_tx_tstamp;
  5209. return NETDEV_TX_OK;
  5210. out_drop:
  5211. dev_kfree_skb_any(first->skb);
  5212. first->skb = NULL;
  5213. cleanup_tx_tstamp:
  5214. if (unlikely(tx_flags & IGB_TX_FLAGS_TSTAMP)) {
  5215. struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
  5216. dev_kfree_skb_any(adapter->ptp_tx_skb);
  5217. adapter->ptp_tx_skb = NULL;
  5218. if (adapter->hw.mac.type == e1000_82576)
  5219. cancel_work_sync(&adapter->ptp_tx_work);
  5220. clear_bit_unlock(__IGB_PTP_TX_IN_PROGRESS, &adapter->state);
  5221. }
  5222. return NETDEV_TX_OK;
  5223. }
  5224. static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter,
  5225. struct sk_buff *skb)
  5226. {
  5227. unsigned int r_idx = skb->queue_mapping;
  5228. if (r_idx >= adapter->num_tx_queues)
  5229. r_idx = r_idx % adapter->num_tx_queues;
  5230. return adapter->tx_ring[r_idx];
  5231. }
  5232. static netdev_tx_t igb_xmit_frame(struct sk_buff *skb,
  5233. struct net_device *netdev)
  5234. {
  5235. struct igb_adapter *adapter = netdev_priv(netdev);
  5236. /* The minimum packet size with TCTL.PSP set is 17 so pad the skb
  5237. * in order to meet this minimum size requirement.
  5238. */
  5239. if (skb_put_padto(skb, 17))
  5240. return NETDEV_TX_OK;
  5241. return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb));
  5242. }
  5243. /**
  5244. * igb_tx_timeout - Respond to a Tx Hang
  5245. * @netdev: network interface device structure
  5246. **/
  5247. static void igb_tx_timeout(struct net_device *netdev, unsigned int txqueue)
  5248. {
  5249. struct igb_adapter *adapter = netdev_priv(netdev);
  5250. struct e1000_hw *hw = &adapter->hw;
  5251. /* Do the reset outside of interrupt context */
  5252. adapter->tx_timeout_count++;
  5253. if (hw->mac.type >= e1000_82580)
  5254. hw->dev_spec._82575.global_device_reset = true;
  5255. schedule_work(&adapter->reset_task);
  5256. wr32(E1000_EICS,
  5257. (adapter->eims_enable_mask & ~adapter->eims_other));
  5258. }
  5259. static void igb_reset_task(struct work_struct *work)
  5260. {
  5261. struct igb_adapter *adapter;
  5262. adapter = container_of(work, struct igb_adapter, reset_task);
  5263. igb_dump(adapter);
  5264. netdev_err(adapter->netdev, "Reset adapter\n");
  5265. igb_reinit_locked(adapter);
  5266. }
  5267. /**
  5268. * igb_get_stats64 - Get System Network Statistics
  5269. * @netdev: network interface device structure
  5270. * @stats: rtnl_link_stats64 pointer
  5271. **/
  5272. static void igb_get_stats64(struct net_device *netdev,
  5273. struct rtnl_link_stats64 *stats)
  5274. {
  5275. struct igb_adapter *adapter = netdev_priv(netdev);
  5276. spin_lock(&adapter->stats64_lock);
  5277. igb_update_stats(adapter);
  5278. memcpy(stats, &adapter->stats64, sizeof(*stats));
  5279. spin_unlock(&adapter->stats64_lock);
  5280. }
  5281. /**
  5282. * igb_change_mtu - Change the Maximum Transfer Unit
  5283. * @netdev: network interface device structure
  5284. * @new_mtu: new value for maximum frame size
  5285. *
  5286. * Returns 0 on success, negative on failure
  5287. **/
  5288. static int igb_change_mtu(struct net_device *netdev, int new_mtu)
  5289. {
  5290. struct igb_adapter *adapter = netdev_priv(netdev);
  5291. int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
  5292. /* adjust max frame to be at least the size of a standard frame */
  5293. if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN))
  5294. max_frame = ETH_FRAME_LEN + ETH_FCS_LEN;
  5295. while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
  5296. usleep_range(1000, 2000);
  5297. /* igb_down has a dependency on max_frame_size */
  5298. adapter->max_frame_size = max_frame;
  5299. if (netif_running(netdev))
  5300. igb_down(adapter);
  5301. netdev_dbg(netdev, "changing MTU from %d to %d\n",
  5302. netdev->mtu, new_mtu);
  5303. netdev->mtu = new_mtu;
  5304. if (netif_running(netdev))
  5305. igb_up(adapter);
  5306. else
  5307. igb_reset(adapter);
  5308. clear_bit(__IGB_RESETTING, &adapter->state);
  5309. return 0;
  5310. }
  5311. /**
  5312. * igb_update_stats - Update the board statistics counters
  5313. * @adapter: board private structure
  5314. **/
  5315. void igb_update_stats(struct igb_adapter *adapter)
  5316. {
  5317. struct rtnl_link_stats64 *net_stats = &adapter->stats64;
  5318. struct e1000_hw *hw = &adapter->hw;
  5319. struct pci_dev *pdev = adapter->pdev;
  5320. u32 reg, mpc;
  5321. int i;
  5322. u64 bytes, packets;
  5323. unsigned int start;
  5324. u64 _bytes, _packets;
  5325. /* Prevent stats update while adapter is being reset, or if the pci
  5326. * connection is down.
  5327. */
  5328. if (adapter->link_speed == 0)
  5329. return;
  5330. if (pci_channel_offline(pdev))
  5331. return;
  5332. bytes = 0;
  5333. packets = 0;
  5334. rcu_read_lock();
  5335. for (i = 0; i < adapter->num_rx_queues; i++) {
  5336. struct igb_ring *ring = adapter->rx_ring[i];
  5337. u32 rqdpc = rd32(E1000_RQDPC(i));
  5338. if (hw->mac.type >= e1000_i210)
  5339. wr32(E1000_RQDPC(i), 0);
  5340. if (rqdpc) {
  5341. ring->rx_stats.drops += rqdpc;
  5342. net_stats->rx_fifo_errors += rqdpc;
  5343. }
  5344. do {
  5345. start = u64_stats_fetch_begin_irq(&ring->rx_syncp);
  5346. _bytes = ring->rx_stats.bytes;
  5347. _packets = ring->rx_stats.packets;
  5348. } while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start));
  5349. bytes += _bytes;
  5350. packets += _packets;
  5351. }
  5352. net_stats->rx_bytes = bytes;
  5353. net_stats->rx_packets = packets;
  5354. bytes = 0;
  5355. packets = 0;
  5356. for (i = 0; i < adapter->num_tx_queues; i++) {
  5357. struct igb_ring *ring = adapter->tx_ring[i];
  5358. do {
  5359. start = u64_stats_fetch_begin_irq(&ring->tx_syncp);
  5360. _bytes = ring->tx_stats.bytes;
  5361. _packets = ring->tx_stats.packets;
  5362. } while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start));
  5363. bytes += _bytes;
  5364. packets += _packets;
  5365. }
  5366. net_stats->tx_bytes = bytes;
  5367. net_stats->tx_packets = packets;
  5368. rcu_read_unlock();
  5369. /* read stats registers */
  5370. adapter->stats.crcerrs += rd32(E1000_CRCERRS);
  5371. adapter->stats.gprc += rd32(E1000_GPRC);
  5372. adapter->stats.gorc += rd32(E1000_GORCL);
  5373. rd32(E1000_GORCH); /* clear GORCL */
  5374. adapter->stats.bprc += rd32(E1000_BPRC);
  5375. adapter->stats.mprc += rd32(E1000_MPRC);
  5376. adapter->stats.roc += rd32(E1000_ROC);
  5377. adapter->stats.prc64 += rd32(E1000_PRC64);
  5378. adapter->stats.prc127 += rd32(E1000_PRC127);
  5379. adapter->stats.prc255 += rd32(E1000_PRC255);
  5380. adapter->stats.prc511 += rd32(E1000_PRC511);
  5381. adapter->stats.prc1023 += rd32(E1000_PRC1023);
  5382. adapter->stats.prc1522 += rd32(E1000_PRC1522);
  5383. adapter->stats.symerrs += rd32(E1000_SYMERRS);
  5384. adapter->stats.sec += rd32(E1000_SEC);
  5385. mpc = rd32(E1000_MPC);
  5386. adapter->stats.mpc += mpc;
  5387. net_stats->rx_fifo_errors += mpc;
  5388. adapter->stats.scc += rd32(E1000_SCC);
  5389. adapter->stats.ecol += rd32(E1000_ECOL);
  5390. adapter->stats.mcc += rd32(E1000_MCC);
  5391. adapter->stats.latecol += rd32(E1000_LATECOL);
  5392. adapter->stats.dc += rd32(E1000_DC);
  5393. adapter->stats.rlec += rd32(E1000_RLEC);
  5394. adapter->stats.xonrxc += rd32(E1000_XONRXC);
  5395. adapter->stats.xontxc += rd32(E1000_XONTXC);
  5396. adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
  5397. adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
  5398. adapter->stats.fcruc += rd32(E1000_FCRUC);
  5399. adapter->stats.gptc += rd32(E1000_GPTC);
  5400. adapter->stats.gotc += rd32(E1000_GOTCL);
  5401. rd32(E1000_GOTCH); /* clear GOTCL */
  5402. adapter->stats.rnbc += rd32(E1000_RNBC);
  5403. adapter->stats.ruc += rd32(E1000_RUC);
  5404. adapter->stats.rfc += rd32(E1000_RFC);
  5405. adapter->stats.rjc += rd32(E1000_RJC);
  5406. adapter->stats.tor += rd32(E1000_TORH);
  5407. adapter->stats.tot += rd32(E1000_TOTH);
  5408. adapter->stats.tpr += rd32(E1000_TPR);
  5409. adapter->stats.ptc64 += rd32(E1000_PTC64);
  5410. adapter->stats.ptc127 += rd32(E1000_PTC127);
  5411. adapter->stats.ptc255 += rd32(E1000_PTC255);
  5412. adapter->stats.ptc511 += rd32(E1000_PTC511);
  5413. adapter->stats.ptc1023 += rd32(E1000_PTC1023);
  5414. adapter->stats.ptc1522 += rd32(E1000_PTC1522);
  5415. adapter->stats.mptc += rd32(E1000_MPTC);
  5416. adapter->stats.bptc += rd32(E1000_BPTC);
  5417. adapter->stats.tpt += rd32(E1000_TPT);
  5418. adapter->stats.colc += rd32(E1000_COLC);
  5419. adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
  5420. /* read internal phy specific stats */
  5421. reg = rd32(E1000_CTRL_EXT);
  5422. if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
  5423. adapter->stats.rxerrc += rd32(E1000_RXERRC);
  5424. /* this stat has invalid values on i210/i211 */
  5425. if ((hw->mac.type != e1000_i210) &&
  5426. (hw->mac.type != e1000_i211))
  5427. adapter->stats.tncrs += rd32(E1000_TNCRS);
  5428. }
  5429. adapter->stats.tsctc += rd32(E1000_TSCTC);
  5430. adapter->stats.tsctfc += rd32(E1000_TSCTFC);
  5431. adapter->stats.iac += rd32(E1000_IAC);
  5432. adapter->stats.icrxoc += rd32(E1000_ICRXOC);
  5433. adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
  5434. adapter->stats.icrxatc += rd32(E1000_ICRXATC);
  5435. adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
  5436. adapter->stats.ictxatc += rd32(E1000_ICTXATC);
  5437. adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
  5438. adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
  5439. adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
  5440. /* Fill out the OS statistics structure */
  5441. net_stats->multicast = adapter->stats.mprc;
  5442. net_stats->collisions = adapter->stats.colc;
  5443. /* Rx Errors */
  5444. /* RLEC on some newer hardware can be incorrect so build
  5445. * our own version based on RUC and ROC
  5446. */
  5447. net_stats->rx_errors = adapter->stats.rxerrc +
  5448. adapter->stats.crcerrs + adapter->stats.algnerrc +
  5449. adapter->stats.ruc + adapter->stats.roc +
  5450. adapter->stats.cexterr;
  5451. net_stats->rx_length_errors = adapter->stats.ruc +
  5452. adapter->stats.roc;
  5453. net_stats->rx_crc_errors = adapter->stats.crcerrs;
  5454. net_stats->rx_frame_errors = adapter->stats.algnerrc;
  5455. net_stats->rx_missed_errors = adapter->stats.mpc;
  5456. /* Tx Errors */
  5457. net_stats->tx_errors = adapter->stats.ecol +
  5458. adapter->stats.latecol;
  5459. net_stats->tx_aborted_errors = adapter->stats.ecol;
  5460. net_stats->tx_window_errors = adapter->stats.latecol;
  5461. net_stats->tx_carrier_errors = adapter->stats.tncrs;
  5462. /* Tx Dropped needs to be maintained elsewhere */
  5463. /* Management Stats */
  5464. adapter->stats.mgptc += rd32(E1000_MGTPTC);
  5465. adapter->stats.mgprc += rd32(E1000_MGTPRC);
  5466. adapter->stats.mgpdc += rd32(E1000_MGTPDC);
  5467. /* OS2BMC Stats */
  5468. reg = rd32(E1000_MANC);
  5469. if (reg & E1000_MANC_EN_BMC2OS) {
  5470. adapter->stats.o2bgptc += rd32(E1000_O2BGPTC);
  5471. adapter->stats.o2bspc += rd32(E1000_O2BSPC);
  5472. adapter->stats.b2ospc += rd32(E1000_B2OSPC);
  5473. adapter->stats.b2ogprc += rd32(E1000_B2OGPRC);
  5474. }
  5475. }
  5476. static void igb_tsync_interrupt(struct igb_adapter *adapter)
  5477. {
  5478. struct e1000_hw *hw = &adapter->hw;
  5479. struct ptp_clock_event event;
  5480. struct timespec64 ts;
  5481. u32 ack = 0, tsauxc, sec, nsec, tsicr = rd32(E1000_TSICR);
  5482. if (tsicr & TSINTR_SYS_WRAP) {
  5483. event.type = PTP_CLOCK_PPS;
  5484. if (adapter->ptp_caps.pps)
  5485. ptp_clock_event(adapter->ptp_clock, &event);
  5486. ack |= TSINTR_SYS_WRAP;
  5487. }
  5488. if (tsicr & E1000_TSICR_TXTS) {
  5489. /* retrieve hardware timestamp */
  5490. schedule_work(&adapter->ptp_tx_work);
  5491. ack |= E1000_TSICR_TXTS;
  5492. }
  5493. if (tsicr & TSINTR_TT0) {
  5494. spin_lock(&adapter->tmreg_lock);
  5495. ts = timespec64_add(adapter->perout[0].start,
  5496. adapter->perout[0].period);
  5497. /* u32 conversion of tv_sec is safe until y2106 */
  5498. wr32(E1000_TRGTTIML0, ts.tv_nsec);
  5499. wr32(E1000_TRGTTIMH0, (u32)ts.tv_sec);
  5500. tsauxc = rd32(E1000_TSAUXC);
  5501. tsauxc |= TSAUXC_EN_TT0;
  5502. wr32(E1000_TSAUXC, tsauxc);
  5503. adapter->perout[0].start = ts;
  5504. spin_unlock(&adapter->tmreg_lock);
  5505. ack |= TSINTR_TT0;
  5506. }
  5507. if (tsicr & TSINTR_TT1) {
  5508. spin_lock(&adapter->tmreg_lock);
  5509. ts = timespec64_add(adapter->perout[1].start,
  5510. adapter->perout[1].period);
  5511. wr32(E1000_TRGTTIML1, ts.tv_nsec);
  5512. wr32(E1000_TRGTTIMH1, (u32)ts.tv_sec);
  5513. tsauxc = rd32(E1000_TSAUXC);
  5514. tsauxc |= TSAUXC_EN_TT1;
  5515. wr32(E1000_TSAUXC, tsauxc);
  5516. adapter->perout[1].start = ts;
  5517. spin_unlock(&adapter->tmreg_lock);
  5518. ack |= TSINTR_TT1;
  5519. }
  5520. if (tsicr & TSINTR_AUTT0) {
  5521. nsec = rd32(E1000_AUXSTMPL0);
  5522. sec = rd32(E1000_AUXSTMPH0);
  5523. event.type = PTP_CLOCK_EXTTS;
  5524. event.index = 0;
  5525. event.timestamp = sec * 1000000000ULL + nsec;
  5526. ptp_clock_event(adapter->ptp_clock, &event);
  5527. ack |= TSINTR_AUTT0;
  5528. }
  5529. if (tsicr & TSINTR_AUTT1) {
  5530. nsec = rd32(E1000_AUXSTMPL1);
  5531. sec = rd32(E1000_AUXSTMPH1);
  5532. event.type = PTP_CLOCK_EXTTS;
  5533. event.index = 1;
  5534. event.timestamp = sec * 1000000000ULL + nsec;
  5535. ptp_clock_event(adapter->ptp_clock, &event);
  5536. ack |= TSINTR_AUTT1;
  5537. }
  5538. /* acknowledge the interrupts */
  5539. wr32(E1000_TSICR, ack);
  5540. }
  5541. static irqreturn_t igb_msix_other(int irq, void *data)
  5542. {
  5543. struct igb_adapter *adapter = data;
  5544. struct e1000_hw *hw = &adapter->hw;
  5545. u32 icr = rd32(E1000_ICR);
  5546. /* reading ICR causes bit 31 of EICR to be cleared */
  5547. if (icr & E1000_ICR_DRSTA)
  5548. schedule_work(&adapter->reset_task);
  5549. if (icr & E1000_ICR_DOUTSYNC) {
  5550. /* HW is reporting DMA is out of sync */
  5551. adapter->stats.doosync++;
  5552. /* The DMA Out of Sync is also indication of a spoof event
  5553. * in IOV mode. Check the Wrong VM Behavior register to
  5554. * see if it is really a spoof event.
  5555. */
  5556. igb_check_wvbr(adapter);
  5557. }
  5558. /* Check for a mailbox event */
  5559. if (icr & E1000_ICR_VMMB)
  5560. igb_msg_task(adapter);
  5561. if (icr & E1000_ICR_LSC) {
  5562. hw->mac.get_link_status = 1;
  5563. /* guard against interrupt when we're going down */
  5564. if (!test_bit(__IGB_DOWN, &adapter->state))
  5565. mod_timer(&adapter->watchdog_timer, jiffies + 1);
  5566. }
  5567. if (icr & E1000_ICR_TS)
  5568. igb_tsync_interrupt(adapter);
  5569. wr32(E1000_EIMS, adapter->eims_other);
  5570. return IRQ_HANDLED;
  5571. }
  5572. static void igb_write_itr(struct igb_q_vector *q_vector)
  5573. {
  5574. struct igb_adapter *adapter = q_vector->adapter;
  5575. u32 itr_val = q_vector->itr_val & 0x7FFC;
  5576. if (!q_vector->set_itr)
  5577. return;
  5578. if (!itr_val)
  5579. itr_val = 0x4;
  5580. if (adapter->hw.mac.type == e1000_82575)
  5581. itr_val |= itr_val << 16;
  5582. else
  5583. itr_val |= E1000_EITR_CNT_IGNR;
  5584. writel(itr_val, q_vector->itr_register);
  5585. q_vector->set_itr = 0;
  5586. }
  5587. static irqreturn_t igb_msix_ring(int irq, void *data)
  5588. {
  5589. struct igb_q_vector *q_vector = data;
  5590. /* Write the ITR value calculated from the previous interrupt. */
  5591. igb_write_itr(q_vector);
  5592. napi_schedule(&q_vector->napi);
  5593. return IRQ_HANDLED;
  5594. }
  5595. #ifdef CONFIG_IGB_DCA
  5596. static void igb_update_tx_dca(struct igb_adapter *adapter,
  5597. struct igb_ring *tx_ring,
  5598. int cpu)
  5599. {
  5600. struct e1000_hw *hw = &adapter->hw;
  5601. u32 txctrl = dca3_get_tag(tx_ring->dev, cpu);
  5602. if (hw->mac.type != e1000_82575)
  5603. txctrl <<= E1000_DCA_TXCTRL_CPUID_SHIFT;
  5604. /* We can enable relaxed ordering for reads, but not writes when
  5605. * DCA is enabled. This is due to a known issue in some chipsets
  5606. * which will cause the DCA tag to be cleared.
  5607. */
  5608. txctrl |= E1000_DCA_TXCTRL_DESC_RRO_EN |
  5609. E1000_DCA_TXCTRL_DATA_RRO_EN |
  5610. E1000_DCA_TXCTRL_DESC_DCA_EN;
  5611. wr32(E1000_DCA_TXCTRL(tx_ring->reg_idx), txctrl);
  5612. }
  5613. static void igb_update_rx_dca(struct igb_adapter *adapter,
  5614. struct igb_ring *rx_ring,
  5615. int cpu)
  5616. {
  5617. struct e1000_hw *hw = &adapter->hw;
  5618. u32 rxctrl = dca3_get_tag(&adapter->pdev->dev, cpu);
  5619. if (hw->mac.type != e1000_82575)
  5620. rxctrl <<= E1000_DCA_RXCTRL_CPUID_SHIFT;
  5621. /* We can enable relaxed ordering for reads, but not writes when
  5622. * DCA is enabled. This is due to a known issue in some chipsets
  5623. * which will cause the DCA tag to be cleared.
  5624. */
  5625. rxctrl |= E1000_DCA_RXCTRL_DESC_RRO_EN |
  5626. E1000_DCA_RXCTRL_DESC_DCA_EN;
  5627. wr32(E1000_DCA_RXCTRL(rx_ring->reg_idx), rxctrl);
  5628. }
  5629. static void igb_update_dca(struct igb_q_vector *q_vector)
  5630. {
  5631. struct igb_adapter *adapter = q_vector->adapter;
  5632. int cpu = get_cpu();
  5633. if (q_vector->cpu == cpu)
  5634. goto out_no_update;
  5635. if (q_vector->tx.ring)
  5636. igb_update_tx_dca(adapter, q_vector->tx.ring, cpu);
  5637. if (q_vector->rx.ring)
  5638. igb_update_rx_dca(adapter, q_vector->rx.ring, cpu);
  5639. q_vector->cpu = cpu;
  5640. out_no_update:
  5641. put_cpu();
  5642. }
  5643. static void igb_setup_dca(struct igb_adapter *adapter)
  5644. {
  5645. struct e1000_hw *hw = &adapter->hw;
  5646. int i;
  5647. if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
  5648. return;
  5649. /* Always use CB2 mode, difference is masked in the CB driver. */
  5650. wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
  5651. for (i = 0; i < adapter->num_q_vectors; i++) {
  5652. adapter->q_vector[i]->cpu = -1;
  5653. igb_update_dca(adapter->q_vector[i]);
  5654. }
  5655. }
  5656. static int __igb_notify_dca(struct device *dev, void *data)
  5657. {
  5658. struct net_device *netdev = dev_get_drvdata(dev);
  5659. struct igb_adapter *adapter = netdev_priv(netdev);
  5660. struct pci_dev *pdev = adapter->pdev;
  5661. struct e1000_hw *hw = &adapter->hw;
  5662. unsigned long event = *(unsigned long *)data;
  5663. switch (event) {
  5664. case DCA_PROVIDER_ADD:
  5665. /* if already enabled, don't do it again */
  5666. if (adapter->flags & IGB_FLAG_DCA_ENABLED)
  5667. break;
  5668. if (dca_add_requester(dev) == 0) {
  5669. adapter->flags |= IGB_FLAG_DCA_ENABLED;
  5670. dev_info(&pdev->dev, "DCA enabled\n");
  5671. igb_setup_dca(adapter);
  5672. break;
  5673. }
  5674. /* Fall Through - since DCA is disabled. */
  5675. case DCA_PROVIDER_REMOVE:
  5676. if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
  5677. /* without this a class_device is left
  5678. * hanging around in the sysfs model
  5679. */
  5680. dca_remove_requester(dev);
  5681. dev_info(&pdev->dev, "DCA disabled\n");
  5682. adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
  5683. wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
  5684. }
  5685. break;
  5686. }
  5687. return 0;
  5688. }
  5689. static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
  5690. void *p)
  5691. {
  5692. int ret_val;
  5693. ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
  5694. __igb_notify_dca);
  5695. return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
  5696. }
  5697. #endif /* CONFIG_IGB_DCA */
  5698. #ifdef CONFIG_PCI_IOV
  5699. static int igb_vf_configure(struct igb_adapter *adapter, int vf)
  5700. {
  5701. unsigned char mac_addr[ETH_ALEN];
  5702. eth_zero_addr(mac_addr);
  5703. igb_set_vf_mac(adapter, vf, mac_addr);
  5704. /* By default spoof check is enabled for all VFs */
  5705. adapter->vf_data[vf].spoofchk_enabled = true;
  5706. /* By default VFs are not trusted */
  5707. adapter->vf_data[vf].trusted = false;
  5708. return 0;
  5709. }
  5710. #endif
  5711. static void igb_ping_all_vfs(struct igb_adapter *adapter)
  5712. {
  5713. struct e1000_hw *hw = &adapter->hw;
  5714. u32 ping;
  5715. int i;
  5716. for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
  5717. ping = E1000_PF_CONTROL_MSG;
  5718. if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
  5719. ping |= E1000_VT_MSGTYPE_CTS;
  5720. igb_write_mbx(hw, &ping, 1, i);
  5721. }
  5722. }
  5723. static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
  5724. {
  5725. struct e1000_hw *hw = &adapter->hw;
  5726. u32 vmolr = rd32(E1000_VMOLR(vf));
  5727. struct vf_data_storage *vf_data = &adapter->vf_data[vf];
  5728. vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
  5729. IGB_VF_FLAG_MULTI_PROMISC);
  5730. vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
  5731. if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
  5732. vmolr |= E1000_VMOLR_MPME;
  5733. vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
  5734. *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
  5735. } else {
  5736. /* if we have hashes and we are clearing a multicast promisc
  5737. * flag we need to write the hashes to the MTA as this step
  5738. * was previously skipped
  5739. */
  5740. if (vf_data->num_vf_mc_hashes > 30) {
  5741. vmolr |= E1000_VMOLR_MPME;
  5742. } else if (vf_data->num_vf_mc_hashes) {
  5743. int j;
  5744. vmolr |= E1000_VMOLR_ROMPE;
  5745. for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
  5746. igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
  5747. }
  5748. }
  5749. wr32(E1000_VMOLR(vf), vmolr);
  5750. /* there are flags left unprocessed, likely not supported */
  5751. if (*msgbuf & E1000_VT_MSGINFO_MASK)
  5752. return -EINVAL;
  5753. return 0;
  5754. }
  5755. static int igb_set_vf_multicasts(struct igb_adapter *adapter,
  5756. u32 *msgbuf, u32 vf)
  5757. {
  5758. int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
  5759. u16 *hash_list = (u16 *)&msgbuf[1];
  5760. struct vf_data_storage *vf_data = &adapter->vf_data[vf];
  5761. int i;
  5762. /* salt away the number of multicast addresses assigned
  5763. * to this VF for later use to restore when the PF multi cast
  5764. * list changes
  5765. */
  5766. vf_data->num_vf_mc_hashes = n;
  5767. /* only up to 30 hash values supported */
  5768. if (n > 30)
  5769. n = 30;
  5770. /* store the hashes for later use */
  5771. for (i = 0; i < n; i++)
  5772. vf_data->vf_mc_hashes[i] = hash_list[i];
  5773. /* Flush and reset the mta with the new values */
  5774. igb_set_rx_mode(adapter->netdev);
  5775. return 0;
  5776. }
  5777. static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
  5778. {
  5779. struct e1000_hw *hw = &adapter->hw;
  5780. struct vf_data_storage *vf_data;
  5781. int i, j;
  5782. for (i = 0; i < adapter->vfs_allocated_count; i++) {
  5783. u32 vmolr = rd32(E1000_VMOLR(i));
  5784. vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
  5785. vf_data = &adapter->vf_data[i];
  5786. if ((vf_data->num_vf_mc_hashes > 30) ||
  5787. (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
  5788. vmolr |= E1000_VMOLR_MPME;
  5789. } else if (vf_data->num_vf_mc_hashes) {
  5790. vmolr |= E1000_VMOLR_ROMPE;
  5791. for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
  5792. igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
  5793. }
  5794. wr32(E1000_VMOLR(i), vmolr);
  5795. }
  5796. }
  5797. static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
  5798. {
  5799. struct e1000_hw *hw = &adapter->hw;
  5800. u32 pool_mask, vlvf_mask, i;
  5801. /* create mask for VF and other pools */
  5802. pool_mask = E1000_VLVF_POOLSEL_MASK;
  5803. vlvf_mask = BIT(E1000_VLVF_POOLSEL_SHIFT + vf);
  5804. /* drop PF from pool bits */
  5805. pool_mask &= ~BIT(E1000_VLVF_POOLSEL_SHIFT +
  5806. adapter->vfs_allocated_count);
  5807. /* Find the vlan filter for this id */
  5808. for (i = E1000_VLVF_ARRAY_SIZE; i--;) {
  5809. u32 vlvf = rd32(E1000_VLVF(i));
  5810. u32 vfta_mask, vid, vfta;
  5811. /* remove the vf from the pool */
  5812. if (!(vlvf & vlvf_mask))
  5813. continue;
  5814. /* clear out bit from VLVF */
  5815. vlvf ^= vlvf_mask;
  5816. /* if other pools are present, just remove ourselves */
  5817. if (vlvf & pool_mask)
  5818. goto update_vlvfb;
  5819. /* if PF is present, leave VFTA */
  5820. if (vlvf & E1000_VLVF_POOLSEL_MASK)
  5821. goto update_vlvf;
  5822. vid = vlvf & E1000_VLVF_VLANID_MASK;
  5823. vfta_mask = BIT(vid % 32);
  5824. /* clear bit from VFTA */
  5825. vfta = adapter->shadow_vfta[vid / 32];
  5826. if (vfta & vfta_mask)
  5827. hw->mac.ops.write_vfta(hw, vid / 32, vfta ^ vfta_mask);
  5828. update_vlvf:
  5829. /* clear pool selection enable */
  5830. if (adapter->flags & IGB_FLAG_VLAN_PROMISC)
  5831. vlvf &= E1000_VLVF_POOLSEL_MASK;
  5832. else
  5833. vlvf = 0;
  5834. update_vlvfb:
  5835. /* clear pool bits */
  5836. wr32(E1000_VLVF(i), vlvf);
  5837. }
  5838. }
  5839. static int igb_find_vlvf_entry(struct e1000_hw *hw, u32 vlan)
  5840. {
  5841. u32 vlvf;
  5842. int idx;
  5843. /* short cut the special case */
  5844. if (vlan == 0)
  5845. return 0;
  5846. /* Search for the VLAN id in the VLVF entries */
  5847. for (idx = E1000_VLVF_ARRAY_SIZE; --idx;) {
  5848. vlvf = rd32(E1000_VLVF(idx));
  5849. if ((vlvf & VLAN_VID_MASK) == vlan)
  5850. break;
  5851. }
  5852. return idx;
  5853. }
  5854. static void igb_update_pf_vlvf(struct igb_adapter *adapter, u32 vid)
  5855. {
  5856. struct e1000_hw *hw = &adapter->hw;
  5857. u32 bits, pf_id;
  5858. int idx;
  5859. idx = igb_find_vlvf_entry(hw, vid);
  5860. if (!idx)
  5861. return;
  5862. /* See if any other pools are set for this VLAN filter
  5863. * entry other than the PF.
  5864. */
  5865. pf_id = adapter->vfs_allocated_count + E1000_VLVF_POOLSEL_SHIFT;
  5866. bits = ~BIT(pf_id) & E1000_VLVF_POOLSEL_MASK;
  5867. bits &= rd32(E1000_VLVF(idx));
  5868. /* Disable the filter so this falls into the default pool. */
  5869. if (!bits) {
  5870. if (adapter->flags & IGB_FLAG_VLAN_PROMISC)
  5871. wr32(E1000_VLVF(idx), BIT(pf_id));
  5872. else
  5873. wr32(E1000_VLVF(idx), 0);
  5874. }
  5875. }
  5876. static s32 igb_set_vf_vlan(struct igb_adapter *adapter, u32 vid,
  5877. bool add, u32 vf)
  5878. {
  5879. int pf_id = adapter->vfs_allocated_count;
  5880. struct e1000_hw *hw = &adapter->hw;
  5881. int err;
  5882. /* If VLAN overlaps with one the PF is currently monitoring make
  5883. * sure that we are able to allocate a VLVF entry. This may be
  5884. * redundant but it guarantees PF will maintain visibility to
  5885. * the VLAN.
  5886. */
  5887. if (add && test_bit(vid, adapter->active_vlans)) {
  5888. err = igb_vfta_set(hw, vid, pf_id, true, false);
  5889. if (err)
  5890. return err;
  5891. }
  5892. err = igb_vfta_set(hw, vid, vf, add, false);
  5893. if (add && !err)
  5894. return err;
  5895. /* If we failed to add the VF VLAN or we are removing the VF VLAN
  5896. * we may need to drop the PF pool bit in order to allow us to free
  5897. * up the VLVF resources.
  5898. */
  5899. if (test_bit(vid, adapter->active_vlans) ||
  5900. (adapter->flags & IGB_FLAG_VLAN_PROMISC))
  5901. igb_update_pf_vlvf(adapter, vid);
  5902. return err;
  5903. }
  5904. static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
  5905. {
  5906. struct e1000_hw *hw = &adapter->hw;
  5907. if (vid)
  5908. wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
  5909. else
  5910. wr32(E1000_VMVIR(vf), 0);
  5911. }
  5912. static int igb_enable_port_vlan(struct igb_adapter *adapter, int vf,
  5913. u16 vlan, u8 qos)
  5914. {
  5915. int err;
  5916. err = igb_set_vf_vlan(adapter, vlan, true, vf);
  5917. if (err)
  5918. return err;
  5919. igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
  5920. igb_set_vmolr(adapter, vf, !vlan);
  5921. /* revoke access to previous VLAN */
  5922. if (vlan != adapter->vf_data[vf].pf_vlan)
  5923. igb_set_vf_vlan(adapter, adapter->vf_data[vf].pf_vlan,
  5924. false, vf);
  5925. adapter->vf_data[vf].pf_vlan = vlan;
  5926. adapter->vf_data[vf].pf_qos = qos;
  5927. igb_set_vf_vlan_strip(adapter, vf, true);
  5928. dev_info(&adapter->pdev->dev,
  5929. "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
  5930. if (test_bit(__IGB_DOWN, &adapter->state)) {
  5931. dev_warn(&adapter->pdev->dev,
  5932. "The VF VLAN has been set, but the PF device is not up.\n");
  5933. dev_warn(&adapter->pdev->dev,
  5934. "Bring the PF device up before attempting to use the VF device.\n");
  5935. }
  5936. return err;
  5937. }
  5938. static int igb_disable_port_vlan(struct igb_adapter *adapter, int vf)
  5939. {
  5940. /* Restore tagless access via VLAN 0 */
  5941. igb_set_vf_vlan(adapter, 0, true, vf);
  5942. igb_set_vmvir(adapter, 0, vf);
  5943. igb_set_vmolr(adapter, vf, true);
  5944. /* Remove any PF assigned VLAN */
  5945. if (adapter->vf_data[vf].pf_vlan)
  5946. igb_set_vf_vlan(adapter, adapter->vf_data[vf].pf_vlan,
  5947. false, vf);
  5948. adapter->vf_data[vf].pf_vlan = 0;
  5949. adapter->vf_data[vf].pf_qos = 0;
  5950. igb_set_vf_vlan_strip(adapter, vf, false);
  5951. return 0;
  5952. }
  5953. static int igb_ndo_set_vf_vlan(struct net_device *netdev, int vf,
  5954. u16 vlan, u8 qos, __be16 vlan_proto)
  5955. {
  5956. struct igb_adapter *adapter = netdev_priv(netdev);
  5957. if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
  5958. return -EINVAL;
  5959. if (vlan_proto != htons(ETH_P_8021Q))
  5960. return -EPROTONOSUPPORT;
  5961. return (vlan || qos) ? igb_enable_port_vlan(adapter, vf, vlan, qos) :
  5962. igb_disable_port_vlan(adapter, vf);
  5963. }
  5964. static int igb_set_vf_vlan_msg(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
  5965. {
  5966. int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
  5967. int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
  5968. int ret;
  5969. if (adapter->vf_data[vf].pf_vlan)
  5970. return -1;
  5971. /* VLAN 0 is a special case, don't allow it to be removed */
  5972. if (!vid && !add)
  5973. return 0;
  5974. ret = igb_set_vf_vlan(adapter, vid, !!add, vf);
  5975. if (!ret)
  5976. igb_set_vf_vlan_strip(adapter, vf, !!vid);
  5977. return ret;
  5978. }
  5979. static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
  5980. {
  5981. struct vf_data_storage *vf_data = &adapter->vf_data[vf];
  5982. /* clear flags - except flag that indicates PF has set the MAC */
  5983. vf_data->flags &= IGB_VF_FLAG_PF_SET_MAC;
  5984. vf_data->last_nack = jiffies;
  5985. /* reset vlans for device */
  5986. igb_clear_vf_vfta(adapter, vf);
  5987. igb_set_vf_vlan(adapter, vf_data->pf_vlan, true, vf);
  5988. igb_set_vmvir(adapter, vf_data->pf_vlan |
  5989. (vf_data->pf_qos << VLAN_PRIO_SHIFT), vf);
  5990. igb_set_vmolr(adapter, vf, !vf_data->pf_vlan);
  5991. igb_set_vf_vlan_strip(adapter, vf, !!(vf_data->pf_vlan));
  5992. /* reset multicast table array for vf */
  5993. adapter->vf_data[vf].num_vf_mc_hashes = 0;
  5994. /* Flush and reset the mta with the new values */
  5995. igb_set_rx_mode(adapter->netdev);
  5996. }
  5997. static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
  5998. {
  5999. unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
  6000. /* clear mac address as we were hotplug removed/added */
  6001. if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
  6002. eth_zero_addr(vf_mac);
  6003. /* process remaining reset events */
  6004. igb_vf_reset(adapter, vf);
  6005. }
  6006. static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
  6007. {
  6008. struct e1000_hw *hw = &adapter->hw;
  6009. unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
  6010. u32 reg, msgbuf[3];
  6011. u8 *addr = (u8 *)(&msgbuf[1]);
  6012. /* process all the same items cleared in a function level reset */
  6013. igb_vf_reset(adapter, vf);
  6014. /* set vf mac address */
  6015. igb_set_vf_mac(adapter, vf, vf_mac);
  6016. /* enable transmit and receive for vf */
  6017. reg = rd32(E1000_VFTE);
  6018. wr32(E1000_VFTE, reg | BIT(vf));
  6019. reg = rd32(E1000_VFRE);
  6020. wr32(E1000_VFRE, reg | BIT(vf));
  6021. adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
  6022. /* reply to reset with ack and vf mac address */
  6023. if (!is_zero_ether_addr(vf_mac)) {
  6024. msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
  6025. memcpy(addr, vf_mac, ETH_ALEN);
  6026. } else {
  6027. msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_NACK;
  6028. }
  6029. igb_write_mbx(hw, msgbuf, 3, vf);
  6030. }
  6031. static void igb_flush_mac_table(struct igb_adapter *adapter)
  6032. {
  6033. struct e1000_hw *hw = &adapter->hw;
  6034. int i;
  6035. for (i = 0; i < hw->mac.rar_entry_count; i++) {
  6036. adapter->mac_table[i].state &= ~IGB_MAC_STATE_IN_USE;
  6037. memset(adapter->mac_table[i].addr, 0, ETH_ALEN);
  6038. adapter->mac_table[i].queue = 0;
  6039. igb_rar_set_index(adapter, i);
  6040. }
  6041. }
  6042. static int igb_available_rars(struct igb_adapter *adapter, u8 queue)
  6043. {
  6044. struct e1000_hw *hw = &adapter->hw;
  6045. /* do not count rar entries reserved for VFs MAC addresses */
  6046. int rar_entries = hw->mac.rar_entry_count -
  6047. adapter->vfs_allocated_count;
  6048. int i, count = 0;
  6049. for (i = 0; i < rar_entries; i++) {
  6050. /* do not count default entries */
  6051. if (adapter->mac_table[i].state & IGB_MAC_STATE_DEFAULT)
  6052. continue;
  6053. /* do not count "in use" entries for different queues */
  6054. if ((adapter->mac_table[i].state & IGB_MAC_STATE_IN_USE) &&
  6055. (adapter->mac_table[i].queue != queue))
  6056. continue;
  6057. count++;
  6058. }
  6059. return count;
  6060. }
  6061. /* Set default MAC address for the PF in the first RAR entry */
  6062. static void igb_set_default_mac_filter(struct igb_adapter *adapter)
  6063. {
  6064. struct igb_mac_addr *mac_table = &adapter->mac_table[0];
  6065. ether_addr_copy(mac_table->addr, adapter->hw.mac.addr);
  6066. mac_table->queue = adapter->vfs_allocated_count;
  6067. mac_table->state = IGB_MAC_STATE_DEFAULT | IGB_MAC_STATE_IN_USE;
  6068. igb_rar_set_index(adapter, 0);
  6069. }
  6070. /* If the filter to be added and an already existing filter express
  6071. * the same address and address type, it should be possible to only
  6072. * override the other configurations, for example the queue to steer
  6073. * traffic.
  6074. */
  6075. static bool igb_mac_entry_can_be_used(const struct igb_mac_addr *entry,
  6076. const u8 *addr, const u8 flags)
  6077. {
  6078. if (!(entry->state & IGB_MAC_STATE_IN_USE))
  6079. return true;
  6080. if ((entry->state & IGB_MAC_STATE_SRC_ADDR) !=
  6081. (flags & IGB_MAC_STATE_SRC_ADDR))
  6082. return false;
  6083. if (!ether_addr_equal(addr, entry->addr))
  6084. return false;
  6085. return true;
  6086. }
  6087. /* Add a MAC filter for 'addr' directing matching traffic to 'queue',
  6088. * 'flags' is used to indicate what kind of match is made, match is by
  6089. * default for the destination address, if matching by source address
  6090. * is desired the flag IGB_MAC_STATE_SRC_ADDR can be used.
  6091. */
  6092. static int igb_add_mac_filter_flags(struct igb_adapter *adapter,
  6093. const u8 *addr, const u8 queue,
  6094. const u8 flags)
  6095. {
  6096. struct e1000_hw *hw = &adapter->hw;
  6097. int rar_entries = hw->mac.rar_entry_count -
  6098. adapter->vfs_allocated_count;
  6099. int i;
  6100. if (is_zero_ether_addr(addr))
  6101. return -EINVAL;
  6102. /* Search for the first empty entry in the MAC table.
  6103. * Do not touch entries at the end of the table reserved for the VF MAC
  6104. * addresses.
  6105. */
  6106. for (i = 0; i < rar_entries; i++) {
  6107. if (!igb_mac_entry_can_be_used(&adapter->mac_table[i],
  6108. addr, flags))
  6109. continue;
  6110. ether_addr_copy(adapter->mac_table[i].addr, addr);
  6111. adapter->mac_table[i].queue = queue;
  6112. adapter->mac_table[i].state |= IGB_MAC_STATE_IN_USE | flags;
  6113. igb_rar_set_index(adapter, i);
  6114. return i;
  6115. }
  6116. return -ENOSPC;
  6117. }
  6118. static int igb_add_mac_filter(struct igb_adapter *adapter, const u8 *addr,
  6119. const u8 queue)
  6120. {
  6121. return igb_add_mac_filter_flags(adapter, addr, queue, 0);
  6122. }
  6123. /* Remove a MAC filter for 'addr' directing matching traffic to
  6124. * 'queue', 'flags' is used to indicate what kind of match need to be
  6125. * removed, match is by default for the destination address, if
  6126. * matching by source address is to be removed the flag
  6127. * IGB_MAC_STATE_SRC_ADDR can be used.
  6128. */
  6129. static int igb_del_mac_filter_flags(struct igb_adapter *adapter,
  6130. const u8 *addr, const u8 queue,
  6131. const u8 flags)
  6132. {
  6133. struct e1000_hw *hw = &adapter->hw;
  6134. int rar_entries = hw->mac.rar_entry_count -
  6135. adapter->vfs_allocated_count;
  6136. int i;
  6137. if (is_zero_ether_addr(addr))
  6138. return -EINVAL;
  6139. /* Search for matching entry in the MAC table based on given address
  6140. * and queue. Do not touch entries at the end of the table reserved
  6141. * for the VF MAC addresses.
  6142. */
  6143. for (i = 0; i < rar_entries; i++) {
  6144. if (!(adapter->mac_table[i].state & IGB_MAC_STATE_IN_USE))
  6145. continue;
  6146. if ((adapter->mac_table[i].state & flags) != flags)
  6147. continue;
  6148. if (adapter->mac_table[i].queue != queue)
  6149. continue;
  6150. if (!ether_addr_equal(adapter->mac_table[i].addr, addr))
  6151. continue;
  6152. /* When a filter for the default address is "deleted",
  6153. * we return it to its initial configuration
  6154. */
  6155. if (adapter->mac_table[i].state & IGB_MAC_STATE_DEFAULT) {
  6156. adapter->mac_table[i].state =
  6157. IGB_MAC_STATE_DEFAULT | IGB_MAC_STATE_IN_USE;
  6158. adapter->mac_table[i].queue =
  6159. adapter->vfs_allocated_count;
  6160. } else {
  6161. adapter->mac_table[i].state = 0;
  6162. adapter->mac_table[i].queue = 0;
  6163. memset(adapter->mac_table[i].addr, 0, ETH_ALEN);
  6164. }
  6165. igb_rar_set_index(adapter, i);
  6166. return 0;
  6167. }
  6168. return -ENOENT;
  6169. }
  6170. static int igb_del_mac_filter(struct igb_adapter *adapter, const u8 *addr,
  6171. const u8 queue)
  6172. {
  6173. return igb_del_mac_filter_flags(adapter, addr, queue, 0);
  6174. }
  6175. int igb_add_mac_steering_filter(struct igb_adapter *adapter,
  6176. const u8 *addr, u8 queue, u8 flags)
  6177. {
  6178. struct e1000_hw *hw = &adapter->hw;
  6179. /* In theory, this should be supported on 82575 as well, but
  6180. * that part wasn't easily accessible during development.
  6181. */
  6182. if (hw->mac.type != e1000_i210)
  6183. return -EOPNOTSUPP;
  6184. return igb_add_mac_filter_flags(adapter, addr, queue,
  6185. IGB_MAC_STATE_QUEUE_STEERING | flags);
  6186. }
  6187. int igb_del_mac_steering_filter(struct igb_adapter *adapter,
  6188. const u8 *addr, u8 queue, u8 flags)
  6189. {
  6190. return igb_del_mac_filter_flags(adapter, addr, queue,
  6191. IGB_MAC_STATE_QUEUE_STEERING | flags);
  6192. }
  6193. static int igb_uc_sync(struct net_device *netdev, const unsigned char *addr)
  6194. {
  6195. struct igb_adapter *adapter = netdev_priv(netdev);
  6196. int ret;
  6197. ret = igb_add_mac_filter(adapter, addr, adapter->vfs_allocated_count);
  6198. return min_t(int, ret, 0);
  6199. }
  6200. static int igb_uc_unsync(struct net_device *netdev, const unsigned char *addr)
  6201. {
  6202. struct igb_adapter *adapter = netdev_priv(netdev);
  6203. igb_del_mac_filter(adapter, addr, adapter->vfs_allocated_count);
  6204. return 0;
  6205. }
  6206. static int igb_set_vf_mac_filter(struct igb_adapter *adapter, const int vf,
  6207. const u32 info, const u8 *addr)
  6208. {
  6209. struct pci_dev *pdev = adapter->pdev;
  6210. struct vf_data_storage *vf_data = &adapter->vf_data[vf];
  6211. struct list_head *pos;
  6212. struct vf_mac_filter *entry = NULL;
  6213. int ret = 0;
  6214. switch (info) {
  6215. case E1000_VF_MAC_FILTER_CLR:
  6216. /* remove all unicast MAC filters related to the current VF */
  6217. list_for_each(pos, &adapter->vf_macs.l) {
  6218. entry = list_entry(pos, struct vf_mac_filter, l);
  6219. if (entry->vf == vf) {
  6220. entry->vf = -1;
  6221. entry->free = true;
  6222. igb_del_mac_filter(adapter, entry->vf_mac, vf);
  6223. }
  6224. }
  6225. break;
  6226. case E1000_VF_MAC_FILTER_ADD:
  6227. if ((vf_data->flags & IGB_VF_FLAG_PF_SET_MAC) &&
  6228. !vf_data->trusted) {
  6229. dev_warn(&pdev->dev,
  6230. "VF %d requested MAC filter but is administratively denied\n",
  6231. vf);
  6232. return -EINVAL;
  6233. }
  6234. if (!is_valid_ether_addr(addr)) {
  6235. dev_warn(&pdev->dev,
  6236. "VF %d attempted to set invalid MAC filter\n",
  6237. vf);
  6238. return -EINVAL;
  6239. }
  6240. /* try to find empty slot in the list */
  6241. list_for_each(pos, &adapter->vf_macs.l) {
  6242. entry = list_entry(pos, struct vf_mac_filter, l);
  6243. if (entry->free)
  6244. break;
  6245. }
  6246. if (entry && entry->free) {
  6247. entry->free = false;
  6248. entry->vf = vf;
  6249. ether_addr_copy(entry->vf_mac, addr);
  6250. ret = igb_add_mac_filter(adapter, addr, vf);
  6251. ret = min_t(int, ret, 0);
  6252. } else {
  6253. ret = -ENOSPC;
  6254. }
  6255. if (ret == -ENOSPC)
  6256. dev_warn(&pdev->dev,
  6257. "VF %d has requested MAC filter but there is no space for it\n",
  6258. vf);
  6259. break;
  6260. default:
  6261. ret = -EINVAL;
  6262. break;
  6263. }
  6264. return ret;
  6265. }
  6266. static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
  6267. {
  6268. struct pci_dev *pdev = adapter->pdev;
  6269. struct vf_data_storage *vf_data = &adapter->vf_data[vf];
  6270. u32 info = msg[0] & E1000_VT_MSGINFO_MASK;
  6271. /* The VF MAC Address is stored in a packed array of bytes
  6272. * starting at the second 32 bit word of the msg array
  6273. */
  6274. unsigned char *addr = (unsigned char *)&msg[1];
  6275. int ret = 0;
  6276. if (!info) {
  6277. if ((vf_data->flags & IGB_VF_FLAG_PF_SET_MAC) &&
  6278. !vf_data->trusted) {
  6279. dev_warn(&pdev->dev,
  6280. "VF %d attempted to override administratively set MAC address\nReload the VF driver to resume operations\n",
  6281. vf);
  6282. return -EINVAL;
  6283. }
  6284. if (!is_valid_ether_addr(addr)) {
  6285. dev_warn(&pdev->dev,
  6286. "VF %d attempted to set invalid MAC\n",
  6287. vf);
  6288. return -EINVAL;
  6289. }
  6290. ret = igb_set_vf_mac(adapter, vf, addr);
  6291. } else {
  6292. ret = igb_set_vf_mac_filter(adapter, vf, info, addr);
  6293. }
  6294. return ret;
  6295. }
  6296. static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
  6297. {
  6298. struct e1000_hw *hw = &adapter->hw;
  6299. struct vf_data_storage *vf_data = &adapter->vf_data[vf];
  6300. u32 msg = E1000_VT_MSGTYPE_NACK;
  6301. /* if device isn't clear to send it shouldn't be reading either */
  6302. if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
  6303. time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
  6304. igb_write_mbx(hw, &msg, 1, vf);
  6305. vf_data->last_nack = jiffies;
  6306. }
  6307. }
  6308. static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
  6309. {
  6310. struct pci_dev *pdev = adapter->pdev;
  6311. u32 msgbuf[E1000_VFMAILBOX_SIZE];
  6312. struct e1000_hw *hw = &adapter->hw;
  6313. struct vf_data_storage *vf_data = &adapter->vf_data[vf];
  6314. s32 retval;
  6315. retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf, false);
  6316. if (retval) {
  6317. /* if receive failed revoke VF CTS stats and restart init */
  6318. dev_err(&pdev->dev, "Error receiving message from VF\n");
  6319. vf_data->flags &= ~IGB_VF_FLAG_CTS;
  6320. if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
  6321. goto unlock;
  6322. goto out;
  6323. }
  6324. /* this is a message we already processed, do nothing */
  6325. if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
  6326. goto unlock;
  6327. /* until the vf completes a reset it should not be
  6328. * allowed to start any configuration.
  6329. */
  6330. if (msgbuf[0] == E1000_VF_RESET) {
  6331. /* unlocks mailbox */
  6332. igb_vf_reset_msg(adapter, vf);
  6333. return;
  6334. }
  6335. if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
  6336. if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
  6337. goto unlock;
  6338. retval = -1;
  6339. goto out;
  6340. }
  6341. switch ((msgbuf[0] & 0xFFFF)) {
  6342. case E1000_VF_SET_MAC_ADDR:
  6343. retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
  6344. break;
  6345. case E1000_VF_SET_PROMISC:
  6346. retval = igb_set_vf_promisc(adapter, msgbuf, vf);
  6347. break;
  6348. case E1000_VF_SET_MULTICAST:
  6349. retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
  6350. break;
  6351. case E1000_VF_SET_LPE:
  6352. retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
  6353. break;
  6354. case E1000_VF_SET_VLAN:
  6355. retval = -1;
  6356. if (vf_data->pf_vlan)
  6357. dev_warn(&pdev->dev,
  6358. "VF %d attempted to override administratively set VLAN tag\nReload the VF driver to resume operations\n",
  6359. vf);
  6360. else
  6361. retval = igb_set_vf_vlan_msg(adapter, msgbuf, vf);
  6362. break;
  6363. default:
  6364. dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
  6365. retval = -1;
  6366. break;
  6367. }
  6368. msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
  6369. out:
  6370. /* notify the VF of the results of what it sent us */
  6371. if (retval)
  6372. msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
  6373. else
  6374. msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
  6375. /* unlocks mailbox */
  6376. igb_write_mbx(hw, msgbuf, 1, vf);
  6377. return;
  6378. unlock:
  6379. igb_unlock_mbx(hw, vf);
  6380. }
  6381. static void igb_msg_task(struct igb_adapter *adapter)
  6382. {
  6383. struct e1000_hw *hw = &adapter->hw;
  6384. u32 vf;
  6385. for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
  6386. /* process any reset requests */
  6387. if (!igb_check_for_rst(hw, vf))
  6388. igb_vf_reset_event(adapter, vf);
  6389. /* process any messages pending */
  6390. if (!igb_check_for_msg(hw, vf))
  6391. igb_rcv_msg_from_vf(adapter, vf);
  6392. /* process any acks */
  6393. if (!igb_check_for_ack(hw, vf))
  6394. igb_rcv_ack_from_vf(adapter, vf);
  6395. }
  6396. }
  6397. /**
  6398. * igb_set_uta - Set unicast filter table address
  6399. * @adapter: board private structure
  6400. * @set: boolean indicating if we are setting or clearing bits
  6401. *
  6402. * The unicast table address is a register array of 32-bit registers.
  6403. * The table is meant to be used in a way similar to how the MTA is used
  6404. * however due to certain limitations in the hardware it is necessary to
  6405. * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
  6406. * enable bit to allow vlan tag stripping when promiscuous mode is enabled
  6407. **/
  6408. static void igb_set_uta(struct igb_adapter *adapter, bool set)
  6409. {
  6410. struct e1000_hw *hw = &adapter->hw;
  6411. u32 uta = set ? ~0 : 0;
  6412. int i;
  6413. /* we only need to do this if VMDq is enabled */
  6414. if (!adapter->vfs_allocated_count)
  6415. return;
  6416. for (i = hw->mac.uta_reg_count; i--;)
  6417. array_wr32(E1000_UTA, i, uta);
  6418. }
  6419. /**
  6420. * igb_intr_msi - Interrupt Handler
  6421. * @irq: interrupt number
  6422. * @data: pointer to a network interface device structure
  6423. **/
  6424. static irqreturn_t igb_intr_msi(int irq, void *data)
  6425. {
  6426. struct igb_adapter *adapter = data;
  6427. struct igb_q_vector *q_vector = adapter->q_vector[0];
  6428. struct e1000_hw *hw = &adapter->hw;
  6429. /* read ICR disables interrupts using IAM */
  6430. u32 icr = rd32(E1000_ICR);
  6431. igb_write_itr(q_vector);
  6432. if (icr & E1000_ICR_DRSTA)
  6433. schedule_work(&adapter->reset_task);
  6434. if (icr & E1000_ICR_DOUTSYNC) {
  6435. /* HW is reporting DMA is out of sync */
  6436. adapter->stats.doosync++;
  6437. }
  6438. if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
  6439. hw->mac.get_link_status = 1;
  6440. if (!test_bit(__IGB_DOWN, &adapter->state))
  6441. mod_timer(&adapter->watchdog_timer, jiffies + 1);
  6442. }
  6443. if (icr & E1000_ICR_TS)
  6444. igb_tsync_interrupt(adapter);
  6445. napi_schedule(&q_vector->napi);
  6446. return IRQ_HANDLED;
  6447. }
  6448. /**
  6449. * igb_intr - Legacy Interrupt Handler
  6450. * @irq: interrupt number
  6451. * @data: pointer to a network interface device structure
  6452. **/
  6453. static irqreturn_t igb_intr(int irq, void *data)
  6454. {
  6455. struct igb_adapter *adapter = data;
  6456. struct igb_q_vector *q_vector = adapter->q_vector[0];
  6457. struct e1000_hw *hw = &adapter->hw;
  6458. /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
  6459. * need for the IMC write
  6460. */
  6461. u32 icr = rd32(E1000_ICR);
  6462. /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
  6463. * not set, then the adapter didn't send an interrupt
  6464. */
  6465. if (!(icr & E1000_ICR_INT_ASSERTED))
  6466. return IRQ_NONE;
  6467. igb_write_itr(q_vector);
  6468. if (icr & E1000_ICR_DRSTA)
  6469. schedule_work(&adapter->reset_task);
  6470. if (icr & E1000_ICR_DOUTSYNC) {
  6471. /* HW is reporting DMA is out of sync */
  6472. adapter->stats.doosync++;
  6473. }
  6474. if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
  6475. hw->mac.get_link_status = 1;
  6476. /* guard against interrupt when we're going down */
  6477. if (!test_bit(__IGB_DOWN, &adapter->state))
  6478. mod_timer(&adapter->watchdog_timer, jiffies + 1);
  6479. }
  6480. if (icr & E1000_ICR_TS)
  6481. igb_tsync_interrupt(adapter);
  6482. napi_schedule(&q_vector->napi);
  6483. return IRQ_HANDLED;
  6484. }
  6485. static void igb_ring_irq_enable(struct igb_q_vector *q_vector)
  6486. {
  6487. struct igb_adapter *adapter = q_vector->adapter;
  6488. struct e1000_hw *hw = &adapter->hw;
  6489. if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
  6490. (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
  6491. if ((adapter->num_q_vectors == 1) && !adapter->vf_data)
  6492. igb_set_itr(q_vector);
  6493. else
  6494. igb_update_ring_itr(q_vector);
  6495. }
  6496. if (!test_bit(__IGB_DOWN, &adapter->state)) {
  6497. if (adapter->flags & IGB_FLAG_HAS_MSIX)
  6498. wr32(E1000_EIMS, q_vector->eims_value);
  6499. else
  6500. igb_irq_enable(adapter);
  6501. }
  6502. }
  6503. /**
  6504. * igb_poll - NAPI Rx polling callback
  6505. * @napi: napi polling structure
  6506. * @budget: count of how many packets we should handle
  6507. **/
  6508. static int igb_poll(struct napi_struct *napi, int budget)
  6509. {
  6510. struct igb_q_vector *q_vector = container_of(napi,
  6511. struct igb_q_vector,
  6512. napi);
  6513. bool clean_complete = true;
  6514. int work_done = 0;
  6515. #ifdef CONFIG_IGB_DCA
  6516. if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
  6517. igb_update_dca(q_vector);
  6518. #endif
  6519. if (q_vector->tx.ring)
  6520. clean_complete = igb_clean_tx_irq(q_vector, budget);
  6521. if (q_vector->rx.ring) {
  6522. int cleaned = igb_clean_rx_irq(q_vector, budget);
  6523. work_done += cleaned;
  6524. if (cleaned >= budget)
  6525. clean_complete = false;
  6526. }
  6527. /* If all work not completed, return budget and keep polling */
  6528. if (!clean_complete)
  6529. return budget;
  6530. /* Exit the polling mode, but don't re-enable interrupts if stack might
  6531. * poll us due to busy-polling
  6532. */
  6533. if (likely(napi_complete_done(napi, work_done)))
  6534. igb_ring_irq_enable(q_vector);
  6535. return min(work_done, budget - 1);
  6536. }
  6537. /**
  6538. * igb_clean_tx_irq - Reclaim resources after transmit completes
  6539. * @q_vector: pointer to q_vector containing needed info
  6540. * @napi_budget: Used to determine if we are in netpoll
  6541. *
  6542. * returns true if ring is completely cleaned
  6543. **/
  6544. static bool igb_clean_tx_irq(struct igb_q_vector *q_vector, int napi_budget)
  6545. {
  6546. struct igb_adapter *adapter = q_vector->adapter;
  6547. struct igb_ring *tx_ring = q_vector->tx.ring;
  6548. struct igb_tx_buffer *tx_buffer;
  6549. union e1000_adv_tx_desc *tx_desc;
  6550. unsigned int total_bytes = 0, total_packets = 0;
  6551. unsigned int budget = q_vector->tx.work_limit;
  6552. unsigned int i = tx_ring->next_to_clean;
  6553. if (test_bit(__IGB_DOWN, &adapter->state))
  6554. return true;
  6555. tx_buffer = &tx_ring->tx_buffer_info[i];
  6556. tx_desc = IGB_TX_DESC(tx_ring, i);
  6557. i -= tx_ring->count;
  6558. do {
  6559. union e1000_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
  6560. /* if next_to_watch is not set then there is no work pending */
  6561. if (!eop_desc)
  6562. break;
  6563. /* prevent any other reads prior to eop_desc */
  6564. smp_rmb();
  6565. /* if DD is not set pending work has not been completed */
  6566. if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
  6567. break;
  6568. /* clear next_to_watch to prevent false hangs */
  6569. tx_buffer->next_to_watch = NULL;
  6570. /* update the statistics for this packet */
  6571. total_bytes += tx_buffer->bytecount;
  6572. total_packets += tx_buffer->gso_segs;
  6573. /* free the skb */
  6574. napi_consume_skb(tx_buffer->skb, napi_budget);
  6575. /* unmap skb header data */
  6576. dma_unmap_single(tx_ring->dev,
  6577. dma_unmap_addr(tx_buffer, dma),
  6578. dma_unmap_len(tx_buffer, len),
  6579. DMA_TO_DEVICE);
  6580. /* clear tx_buffer data */
  6581. dma_unmap_len_set(tx_buffer, len, 0);
  6582. /* clear last DMA location and unmap remaining buffers */
  6583. while (tx_desc != eop_desc) {
  6584. tx_buffer++;
  6585. tx_desc++;
  6586. i++;
  6587. if (unlikely(!i)) {
  6588. i -= tx_ring->count;
  6589. tx_buffer = tx_ring->tx_buffer_info;
  6590. tx_desc = IGB_TX_DESC(tx_ring, 0);
  6591. }
  6592. /* unmap any remaining paged data */
  6593. if (dma_unmap_len(tx_buffer, len)) {
  6594. dma_unmap_page(tx_ring->dev,
  6595. dma_unmap_addr(tx_buffer, dma),
  6596. dma_unmap_len(tx_buffer, len),
  6597. DMA_TO_DEVICE);
  6598. dma_unmap_len_set(tx_buffer, len, 0);
  6599. }
  6600. }
  6601. /* move us one more past the eop_desc for start of next pkt */
  6602. tx_buffer++;
  6603. tx_desc++;
  6604. i++;
  6605. if (unlikely(!i)) {
  6606. i -= tx_ring->count;
  6607. tx_buffer = tx_ring->tx_buffer_info;
  6608. tx_desc = IGB_TX_DESC(tx_ring, 0);
  6609. }
  6610. /* issue prefetch for next Tx descriptor */
  6611. prefetch(tx_desc);
  6612. /* update budget accounting */
  6613. budget--;
  6614. } while (likely(budget));
  6615. netdev_tx_completed_queue(txring_txq(tx_ring),
  6616. total_packets, total_bytes);
  6617. i += tx_ring->count;
  6618. tx_ring->next_to_clean = i;
  6619. u64_stats_update_begin(&tx_ring->tx_syncp);
  6620. tx_ring->tx_stats.bytes += total_bytes;
  6621. tx_ring->tx_stats.packets += total_packets;
  6622. u64_stats_update_end(&tx_ring->tx_syncp);
  6623. q_vector->tx.total_bytes += total_bytes;
  6624. q_vector->tx.total_packets += total_packets;
  6625. if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
  6626. struct e1000_hw *hw = &adapter->hw;
  6627. /* Detect a transmit hang in hardware, this serializes the
  6628. * check with the clearing of time_stamp and movement of i
  6629. */
  6630. clear_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
  6631. if (tx_buffer->next_to_watch &&
  6632. time_after(jiffies, tx_buffer->time_stamp +
  6633. (adapter->tx_timeout_factor * HZ)) &&
  6634. !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
  6635. /* detected Tx unit hang */
  6636. dev_err(tx_ring->dev,
  6637. "Detected Tx Unit Hang\n"
  6638. " Tx Queue <%d>\n"
  6639. " TDH <%x>\n"
  6640. " TDT <%x>\n"
  6641. " next_to_use <%x>\n"
  6642. " next_to_clean <%x>\n"
  6643. "buffer_info[next_to_clean]\n"
  6644. " time_stamp <%lx>\n"
  6645. " next_to_watch <%p>\n"
  6646. " jiffies <%lx>\n"
  6647. " desc.status <%x>\n",
  6648. tx_ring->queue_index,
  6649. rd32(E1000_TDH(tx_ring->reg_idx)),
  6650. readl(tx_ring->tail),
  6651. tx_ring->next_to_use,
  6652. tx_ring->next_to_clean,
  6653. tx_buffer->time_stamp,
  6654. tx_buffer->next_to_watch,
  6655. jiffies,
  6656. tx_buffer->next_to_watch->wb.status);
  6657. netif_stop_subqueue(tx_ring->netdev,
  6658. tx_ring->queue_index);
  6659. /* we are about to reset, no point in enabling stuff */
  6660. return true;
  6661. }
  6662. }
  6663. #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
  6664. if (unlikely(total_packets &&
  6665. netif_carrier_ok(tx_ring->netdev) &&
  6666. igb_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) {
  6667. /* Make sure that anybody stopping the queue after this
  6668. * sees the new next_to_clean.
  6669. */
  6670. smp_mb();
  6671. if (__netif_subqueue_stopped(tx_ring->netdev,
  6672. tx_ring->queue_index) &&
  6673. !(test_bit(__IGB_DOWN, &adapter->state))) {
  6674. netif_wake_subqueue(tx_ring->netdev,
  6675. tx_ring->queue_index);
  6676. u64_stats_update_begin(&tx_ring->tx_syncp);
  6677. tx_ring->tx_stats.restart_queue++;
  6678. u64_stats_update_end(&tx_ring->tx_syncp);
  6679. }
  6680. }
  6681. return !!budget;
  6682. }
  6683. /**
  6684. * igb_reuse_rx_page - page flip buffer and store it back on the ring
  6685. * @rx_ring: rx descriptor ring to store buffers on
  6686. * @old_buff: donor buffer to have page reused
  6687. *
  6688. * Synchronizes page for reuse by the adapter
  6689. **/
  6690. static void igb_reuse_rx_page(struct igb_ring *rx_ring,
  6691. struct igb_rx_buffer *old_buff)
  6692. {
  6693. struct igb_rx_buffer *new_buff;
  6694. u16 nta = rx_ring->next_to_alloc;
  6695. new_buff = &rx_ring->rx_buffer_info[nta];
  6696. /* update, and store next to alloc */
  6697. nta++;
  6698. rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
  6699. /* Transfer page from old buffer to new buffer.
  6700. * Move each member individually to avoid possible store
  6701. * forwarding stalls.
  6702. */
  6703. new_buff->dma = old_buff->dma;
  6704. new_buff->page = old_buff->page;
  6705. new_buff->page_offset = old_buff->page_offset;
  6706. new_buff->pagecnt_bias = old_buff->pagecnt_bias;
  6707. }
  6708. static inline bool igb_page_is_reserved(struct page *page)
  6709. {
  6710. return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page);
  6711. }
  6712. static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer)
  6713. {
  6714. unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
  6715. struct page *page = rx_buffer->page;
  6716. /* avoid re-using remote pages */
  6717. if (unlikely(igb_page_is_reserved(page)))
  6718. return false;
  6719. #if (PAGE_SIZE < 8192)
  6720. /* if we are only owner of page we can reuse it */
  6721. if (unlikely((page_ref_count(page) - pagecnt_bias) > 1))
  6722. return false;
  6723. #else
  6724. #define IGB_LAST_OFFSET \
  6725. (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGB_RXBUFFER_2048)
  6726. if (rx_buffer->page_offset > IGB_LAST_OFFSET)
  6727. return false;
  6728. #endif
  6729. /* If we have drained the page fragment pool we need to update
  6730. * the pagecnt_bias and page count so that we fully restock the
  6731. * number of references the driver holds.
  6732. */
  6733. if (unlikely(!pagecnt_bias)) {
  6734. page_ref_add(page, USHRT_MAX);
  6735. rx_buffer->pagecnt_bias = USHRT_MAX;
  6736. }
  6737. return true;
  6738. }
  6739. /**
  6740. * igb_add_rx_frag - Add contents of Rx buffer to sk_buff
  6741. * @rx_ring: rx descriptor ring to transact packets on
  6742. * @rx_buffer: buffer containing page to add
  6743. * @skb: sk_buff to place the data into
  6744. * @size: size of buffer to be added
  6745. *
  6746. * This function will add the data contained in rx_buffer->page to the skb.
  6747. **/
  6748. static void igb_add_rx_frag(struct igb_ring *rx_ring,
  6749. struct igb_rx_buffer *rx_buffer,
  6750. struct sk_buff *skb,
  6751. unsigned int size)
  6752. {
  6753. #if (PAGE_SIZE < 8192)
  6754. unsigned int truesize = igb_rx_pg_size(rx_ring) / 2;
  6755. #else
  6756. unsigned int truesize = ring_uses_build_skb(rx_ring) ?
  6757. SKB_DATA_ALIGN(IGB_SKB_PAD + size) :
  6758. SKB_DATA_ALIGN(size);
  6759. #endif
  6760. skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
  6761. rx_buffer->page_offset, size, truesize);
  6762. #if (PAGE_SIZE < 8192)
  6763. rx_buffer->page_offset ^= truesize;
  6764. #else
  6765. rx_buffer->page_offset += truesize;
  6766. #endif
  6767. }
  6768. static struct sk_buff *igb_construct_skb(struct igb_ring *rx_ring,
  6769. struct igb_rx_buffer *rx_buffer,
  6770. union e1000_adv_rx_desc *rx_desc,
  6771. unsigned int size)
  6772. {
  6773. void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
  6774. #if (PAGE_SIZE < 8192)
  6775. unsigned int truesize = igb_rx_pg_size(rx_ring) / 2;
  6776. #else
  6777. unsigned int truesize = SKB_DATA_ALIGN(size);
  6778. #endif
  6779. unsigned int headlen;
  6780. struct sk_buff *skb;
  6781. /* prefetch first cache line of first page */
  6782. prefetch(va);
  6783. #if L1_CACHE_BYTES < 128
  6784. prefetch(va + L1_CACHE_BYTES);
  6785. #endif
  6786. /* allocate a skb to store the frags */
  6787. skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGB_RX_HDR_LEN);
  6788. if (unlikely(!skb))
  6789. return NULL;
  6790. if (unlikely(igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP))) {
  6791. igb_ptp_rx_pktstamp(rx_ring->q_vector, va, skb);
  6792. va += IGB_TS_HDR_LEN;
  6793. size -= IGB_TS_HDR_LEN;
  6794. }
  6795. /* Determine available headroom for copy */
  6796. headlen = size;
  6797. if (headlen > IGB_RX_HDR_LEN)
  6798. headlen = eth_get_headlen(skb->dev, va, IGB_RX_HDR_LEN);
  6799. /* align pull length to size of long to optimize memcpy performance */
  6800. memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long)));
  6801. /* update all of the pointers */
  6802. size -= headlen;
  6803. if (size) {
  6804. skb_add_rx_frag(skb, 0, rx_buffer->page,
  6805. (va + headlen) - page_address(rx_buffer->page),
  6806. size, truesize);
  6807. #if (PAGE_SIZE < 8192)
  6808. rx_buffer->page_offset ^= truesize;
  6809. #else
  6810. rx_buffer->page_offset += truesize;
  6811. #endif
  6812. } else {
  6813. rx_buffer->pagecnt_bias++;
  6814. }
  6815. return skb;
  6816. }
  6817. static struct sk_buff *igb_build_skb(struct igb_ring *rx_ring,
  6818. struct igb_rx_buffer *rx_buffer,
  6819. union e1000_adv_rx_desc *rx_desc,
  6820. unsigned int size)
  6821. {
  6822. void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
  6823. #if (PAGE_SIZE < 8192)
  6824. unsigned int truesize = igb_rx_pg_size(rx_ring) / 2;
  6825. #else
  6826. unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
  6827. SKB_DATA_ALIGN(IGB_SKB_PAD + size);
  6828. #endif
  6829. struct sk_buff *skb;
  6830. /* prefetch first cache line of first page */
  6831. prefetch(va);
  6832. #if L1_CACHE_BYTES < 128
  6833. prefetch(va + L1_CACHE_BYTES);
  6834. #endif
  6835. /* build an skb around the page buffer */
  6836. skb = build_skb(va - IGB_SKB_PAD, truesize);
  6837. if (unlikely(!skb))
  6838. return NULL;
  6839. /* update pointers within the skb to store the data */
  6840. skb_reserve(skb, IGB_SKB_PAD);
  6841. __skb_put(skb, size);
  6842. /* pull timestamp out of packet data */
  6843. if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) {
  6844. igb_ptp_rx_pktstamp(rx_ring->q_vector, skb->data, skb);
  6845. __skb_pull(skb, IGB_TS_HDR_LEN);
  6846. }
  6847. /* update buffer offset */
  6848. #if (PAGE_SIZE < 8192)
  6849. rx_buffer->page_offset ^= truesize;
  6850. #else
  6851. rx_buffer->page_offset += truesize;
  6852. #endif
  6853. return skb;
  6854. }
  6855. static inline void igb_rx_checksum(struct igb_ring *ring,
  6856. union e1000_adv_rx_desc *rx_desc,
  6857. struct sk_buff *skb)
  6858. {
  6859. skb_checksum_none_assert(skb);
  6860. /* Ignore Checksum bit is set */
  6861. if (igb_test_staterr(rx_desc, E1000_RXD_STAT_IXSM))
  6862. return;
  6863. /* Rx checksum disabled via ethtool */
  6864. if (!(ring->netdev->features & NETIF_F_RXCSUM))
  6865. return;
  6866. /* TCP/UDP checksum error bit is set */
  6867. if (igb_test_staterr(rx_desc,
  6868. E1000_RXDEXT_STATERR_TCPE |
  6869. E1000_RXDEXT_STATERR_IPE)) {
  6870. /* work around errata with sctp packets where the TCPE aka
  6871. * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
  6872. * packets, (aka let the stack check the crc32c)
  6873. */
  6874. if (!((skb->len == 60) &&
  6875. test_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) {
  6876. u64_stats_update_begin(&ring->rx_syncp);
  6877. ring->rx_stats.csum_err++;
  6878. u64_stats_update_end(&ring->rx_syncp);
  6879. }
  6880. /* let the stack verify checksum errors */
  6881. return;
  6882. }
  6883. /* It must be a TCP or UDP packet with a valid checksum */
  6884. if (igb_test_staterr(rx_desc, E1000_RXD_STAT_TCPCS |
  6885. E1000_RXD_STAT_UDPCS))
  6886. skb->ip_summed = CHECKSUM_UNNECESSARY;
  6887. dev_dbg(ring->dev, "cksum success: bits %08X\n",
  6888. le32_to_cpu(rx_desc->wb.upper.status_error));
  6889. }
  6890. static inline void igb_rx_hash(struct igb_ring *ring,
  6891. union e1000_adv_rx_desc *rx_desc,
  6892. struct sk_buff *skb)
  6893. {
  6894. if (ring->netdev->features & NETIF_F_RXHASH)
  6895. skb_set_hash(skb,
  6896. le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
  6897. PKT_HASH_TYPE_L3);
  6898. }
  6899. /**
  6900. * igb_is_non_eop - process handling of non-EOP buffers
  6901. * @rx_ring: Rx ring being processed
  6902. * @rx_desc: Rx descriptor for current buffer
  6903. * @skb: current socket buffer containing buffer in progress
  6904. *
  6905. * This function updates next to clean. If the buffer is an EOP buffer
  6906. * this function exits returning false, otherwise it will place the
  6907. * sk_buff in the next buffer to be chained and return true indicating
  6908. * that this is in fact a non-EOP buffer.
  6909. **/
  6910. static bool igb_is_non_eop(struct igb_ring *rx_ring,
  6911. union e1000_adv_rx_desc *rx_desc)
  6912. {
  6913. u32 ntc = rx_ring->next_to_clean + 1;
  6914. /* fetch, update, and store next to clean */
  6915. ntc = (ntc < rx_ring->count) ? ntc : 0;
  6916. rx_ring->next_to_clean = ntc;
  6917. prefetch(IGB_RX_DESC(rx_ring, ntc));
  6918. if (likely(igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP)))
  6919. return false;
  6920. return true;
  6921. }
  6922. /**
  6923. * igb_cleanup_headers - Correct corrupted or empty headers
  6924. * @rx_ring: rx descriptor ring packet is being transacted on
  6925. * @rx_desc: pointer to the EOP Rx descriptor
  6926. * @skb: pointer to current skb being fixed
  6927. *
  6928. * Address the case where we are pulling data in on pages only
  6929. * and as such no data is present in the skb header.
  6930. *
  6931. * In addition if skb is not at least 60 bytes we need to pad it so that
  6932. * it is large enough to qualify as a valid Ethernet frame.
  6933. *
  6934. * Returns true if an error was encountered and skb was freed.
  6935. **/
  6936. static bool igb_cleanup_headers(struct igb_ring *rx_ring,
  6937. union e1000_adv_rx_desc *rx_desc,
  6938. struct sk_buff *skb)
  6939. {
  6940. if (unlikely((igb_test_staterr(rx_desc,
  6941. E1000_RXDEXT_ERR_FRAME_ERR_MASK)))) {
  6942. struct net_device *netdev = rx_ring->netdev;
  6943. if (!(netdev->features & NETIF_F_RXALL)) {
  6944. dev_kfree_skb_any(skb);
  6945. return true;
  6946. }
  6947. }
  6948. /* if eth_skb_pad returns an error the skb was freed */
  6949. if (eth_skb_pad(skb))
  6950. return true;
  6951. return false;
  6952. }
  6953. /**
  6954. * igb_process_skb_fields - Populate skb header fields from Rx descriptor
  6955. * @rx_ring: rx descriptor ring packet is being transacted on
  6956. * @rx_desc: pointer to the EOP Rx descriptor
  6957. * @skb: pointer to current skb being populated
  6958. *
  6959. * This function checks the ring, descriptor, and packet information in
  6960. * order to populate the hash, checksum, VLAN, timestamp, protocol, and
  6961. * other fields within the skb.
  6962. **/
  6963. static void igb_process_skb_fields(struct igb_ring *rx_ring,
  6964. union e1000_adv_rx_desc *rx_desc,
  6965. struct sk_buff *skb)
  6966. {
  6967. struct net_device *dev = rx_ring->netdev;
  6968. igb_rx_hash(rx_ring, rx_desc, skb);
  6969. igb_rx_checksum(rx_ring, rx_desc, skb);
  6970. if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TS) &&
  6971. !igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP))
  6972. igb_ptp_rx_rgtstamp(rx_ring->q_vector, skb);
  6973. if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
  6974. igb_test_staterr(rx_desc, E1000_RXD_STAT_VP)) {
  6975. u16 vid;
  6976. if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) &&
  6977. test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &rx_ring->flags))
  6978. vid = be16_to_cpu(rx_desc->wb.upper.vlan);
  6979. else
  6980. vid = le16_to_cpu(rx_desc->wb.upper.vlan);
  6981. __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
  6982. }
  6983. skb_record_rx_queue(skb, rx_ring->queue_index);
  6984. skb->protocol = eth_type_trans(skb, rx_ring->netdev);
  6985. }
  6986. static struct igb_rx_buffer *igb_get_rx_buffer(struct igb_ring *rx_ring,
  6987. const unsigned int size)
  6988. {
  6989. struct igb_rx_buffer *rx_buffer;
  6990. rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
  6991. prefetchw(rx_buffer->page);
  6992. /* we are reusing so sync this buffer for CPU use */
  6993. dma_sync_single_range_for_cpu(rx_ring->dev,
  6994. rx_buffer->dma,
  6995. rx_buffer->page_offset,
  6996. size,
  6997. DMA_FROM_DEVICE);
  6998. rx_buffer->pagecnt_bias--;
  6999. return rx_buffer;
  7000. }
  7001. static void igb_put_rx_buffer(struct igb_ring *rx_ring,
  7002. struct igb_rx_buffer *rx_buffer)
  7003. {
  7004. if (igb_can_reuse_rx_page(rx_buffer)) {
  7005. /* hand second half of page back to the ring */
  7006. igb_reuse_rx_page(rx_ring, rx_buffer);
  7007. } else {
  7008. /* We are not reusing the buffer so unmap it and free
  7009. * any references we are holding to it
  7010. */
  7011. dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
  7012. igb_rx_pg_size(rx_ring), DMA_FROM_DEVICE,
  7013. IGB_RX_DMA_ATTR);
  7014. __page_frag_cache_drain(rx_buffer->page,
  7015. rx_buffer->pagecnt_bias);
  7016. }
  7017. /* clear contents of rx_buffer */
  7018. rx_buffer->page = NULL;
  7019. }
  7020. static int igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget)
  7021. {
  7022. struct igb_ring *rx_ring = q_vector->rx.ring;
  7023. struct sk_buff *skb = rx_ring->skb;
  7024. unsigned int total_bytes = 0, total_packets = 0;
  7025. u16 cleaned_count = igb_desc_unused(rx_ring);
  7026. while (likely(total_packets < budget)) {
  7027. union e1000_adv_rx_desc *rx_desc;
  7028. struct igb_rx_buffer *rx_buffer;
  7029. unsigned int size;
  7030. /* return some buffers to hardware, one at a time is too slow */
  7031. if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
  7032. igb_alloc_rx_buffers(rx_ring, cleaned_count);
  7033. cleaned_count = 0;
  7034. }
  7035. rx_desc = IGB_RX_DESC(rx_ring, rx_ring->next_to_clean);
  7036. size = le16_to_cpu(rx_desc->wb.upper.length);
  7037. if (!size)
  7038. break;
  7039. /* This memory barrier is needed to keep us from reading
  7040. * any other fields out of the rx_desc until we know the
  7041. * descriptor has been written back
  7042. */
  7043. dma_rmb();
  7044. rx_buffer = igb_get_rx_buffer(rx_ring, size);
  7045. /* retrieve a buffer from the ring */
  7046. if (skb)
  7047. igb_add_rx_frag(rx_ring, rx_buffer, skb, size);
  7048. else if (ring_uses_build_skb(rx_ring))
  7049. skb = igb_build_skb(rx_ring, rx_buffer, rx_desc, size);
  7050. else
  7051. skb = igb_construct_skb(rx_ring, rx_buffer,
  7052. rx_desc, size);
  7053. /* exit if we failed to retrieve a buffer */
  7054. if (!skb) {
  7055. rx_ring->rx_stats.alloc_failed++;
  7056. rx_buffer->pagecnt_bias++;
  7057. break;
  7058. }
  7059. igb_put_rx_buffer(rx_ring, rx_buffer);
  7060. cleaned_count++;
  7061. /* fetch next buffer in frame if non-eop */
  7062. if (igb_is_non_eop(rx_ring, rx_desc))
  7063. continue;
  7064. /* verify the packet layout is correct */
  7065. if (igb_cleanup_headers(rx_ring, rx_desc, skb)) {
  7066. skb = NULL;
  7067. continue;
  7068. }
  7069. /* probably a little skewed due to removing CRC */
  7070. total_bytes += skb->len;
  7071. /* populate checksum, timestamp, VLAN, and protocol */
  7072. igb_process_skb_fields(rx_ring, rx_desc, skb);
  7073. napi_gro_receive(&q_vector->napi, skb);
  7074. /* reset skb pointer */
  7075. skb = NULL;
  7076. /* update budget accounting */
  7077. total_packets++;
  7078. }
  7079. /* place incomplete frames back on ring for completion */
  7080. rx_ring->skb = skb;
  7081. u64_stats_update_begin(&rx_ring->rx_syncp);
  7082. rx_ring->rx_stats.packets += total_packets;
  7083. rx_ring->rx_stats.bytes += total_bytes;
  7084. u64_stats_update_end(&rx_ring->rx_syncp);
  7085. q_vector->rx.total_packets += total_packets;
  7086. q_vector->rx.total_bytes += total_bytes;
  7087. if (cleaned_count)
  7088. igb_alloc_rx_buffers(rx_ring, cleaned_count);
  7089. return total_packets;
  7090. }
  7091. static inline unsigned int igb_rx_offset(struct igb_ring *rx_ring)
  7092. {
  7093. return ring_uses_build_skb(rx_ring) ? IGB_SKB_PAD : 0;
  7094. }
  7095. static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
  7096. struct igb_rx_buffer *bi)
  7097. {
  7098. struct page *page = bi->page;
  7099. dma_addr_t dma;
  7100. /* since we are recycling buffers we should seldom need to alloc */
  7101. if (likely(page))
  7102. return true;
  7103. /* alloc new page for storage */
  7104. page = dev_alloc_pages(igb_rx_pg_order(rx_ring));
  7105. if (unlikely(!page)) {
  7106. rx_ring->rx_stats.alloc_failed++;
  7107. return false;
  7108. }
  7109. /* map page for use */
  7110. dma = dma_map_page_attrs(rx_ring->dev, page, 0,
  7111. igb_rx_pg_size(rx_ring),
  7112. DMA_FROM_DEVICE,
  7113. IGB_RX_DMA_ATTR);
  7114. /* if mapping failed free memory back to system since
  7115. * there isn't much point in holding memory we can't use
  7116. */
  7117. if (dma_mapping_error(rx_ring->dev, dma)) {
  7118. __free_pages(page, igb_rx_pg_order(rx_ring));
  7119. rx_ring->rx_stats.alloc_failed++;
  7120. return false;
  7121. }
  7122. bi->dma = dma;
  7123. bi->page = page;
  7124. bi->page_offset = igb_rx_offset(rx_ring);
  7125. bi->pagecnt_bias = 1;
  7126. return true;
  7127. }
  7128. /**
  7129. * igb_alloc_rx_buffers - Replace used receive buffers; packet split
  7130. * @adapter: address of board private structure
  7131. **/
  7132. void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
  7133. {
  7134. union e1000_adv_rx_desc *rx_desc;
  7135. struct igb_rx_buffer *bi;
  7136. u16 i = rx_ring->next_to_use;
  7137. u16 bufsz;
  7138. /* nothing to do */
  7139. if (!cleaned_count)
  7140. return;
  7141. rx_desc = IGB_RX_DESC(rx_ring, i);
  7142. bi = &rx_ring->rx_buffer_info[i];
  7143. i -= rx_ring->count;
  7144. bufsz = igb_rx_bufsz(rx_ring);
  7145. do {
  7146. if (!igb_alloc_mapped_page(rx_ring, bi))
  7147. break;
  7148. /* sync the buffer for use by the device */
  7149. dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
  7150. bi->page_offset, bufsz,
  7151. DMA_FROM_DEVICE);
  7152. /* Refresh the desc even if buffer_addrs didn't change
  7153. * because each write-back erases this info.
  7154. */
  7155. rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
  7156. rx_desc++;
  7157. bi++;
  7158. i++;
  7159. if (unlikely(!i)) {
  7160. rx_desc = IGB_RX_DESC(rx_ring, 0);
  7161. bi = rx_ring->rx_buffer_info;
  7162. i -= rx_ring->count;
  7163. }
  7164. /* clear the length for the next_to_use descriptor */
  7165. rx_desc->wb.upper.length = 0;
  7166. cleaned_count--;
  7167. } while (cleaned_count);
  7168. i += rx_ring->count;
  7169. if (rx_ring->next_to_use != i) {
  7170. /* record the next descriptor to use */
  7171. rx_ring->next_to_use = i;
  7172. /* update next to alloc since we have filled the ring */
  7173. rx_ring->next_to_alloc = i;
  7174. /* Force memory writes to complete before letting h/w
  7175. * know there are new descriptors to fetch. (Only
  7176. * applicable for weak-ordered memory model archs,
  7177. * such as IA-64).
  7178. */
  7179. dma_wmb();
  7180. writel(i, rx_ring->tail);
  7181. }
  7182. }
  7183. /**
  7184. * igb_mii_ioctl -
  7185. * @netdev:
  7186. * @ifreq:
  7187. * @cmd:
  7188. **/
  7189. static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
  7190. {
  7191. struct igb_adapter *adapter = netdev_priv(netdev);
  7192. struct mii_ioctl_data *data = if_mii(ifr);
  7193. if (adapter->hw.phy.media_type != e1000_media_type_copper)
  7194. return -EOPNOTSUPP;
  7195. switch (cmd) {
  7196. case SIOCGMIIPHY:
  7197. data->phy_id = adapter->hw.phy.addr;
  7198. break;
  7199. case SIOCGMIIREG:
  7200. if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
  7201. &data->val_out))
  7202. return -EIO;
  7203. break;
  7204. case SIOCSMIIREG:
  7205. default:
  7206. return -EOPNOTSUPP;
  7207. }
  7208. return 0;
  7209. }
  7210. /**
  7211. * igb_ioctl -
  7212. * @netdev:
  7213. * @ifreq:
  7214. * @cmd:
  7215. **/
  7216. static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
  7217. {
  7218. switch (cmd) {
  7219. case SIOCGMIIPHY:
  7220. case SIOCGMIIREG:
  7221. case SIOCSMIIREG:
  7222. return igb_mii_ioctl(netdev, ifr, cmd);
  7223. case SIOCGHWTSTAMP:
  7224. return igb_ptp_get_ts_config(netdev, ifr);
  7225. case SIOCSHWTSTAMP:
  7226. return igb_ptp_set_ts_config(netdev, ifr);
  7227. default:
  7228. return -EOPNOTSUPP;
  7229. }
  7230. }
  7231. void igb_read_pci_cfg(struct e1000_hw *hw, u32 reg, u16 *value)
  7232. {
  7233. struct igb_adapter *adapter = hw->back;
  7234. pci_read_config_word(adapter->pdev, reg, value);
  7235. }
  7236. void igb_write_pci_cfg(struct e1000_hw *hw, u32 reg, u16 *value)
  7237. {
  7238. struct igb_adapter *adapter = hw->back;
  7239. pci_write_config_word(adapter->pdev, reg, *value);
  7240. }
  7241. s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
  7242. {
  7243. struct igb_adapter *adapter = hw->back;
  7244. if (pcie_capability_read_word(adapter->pdev, reg, value))
  7245. return -E1000_ERR_CONFIG;
  7246. return 0;
  7247. }
  7248. s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
  7249. {
  7250. struct igb_adapter *adapter = hw->back;
  7251. if (pcie_capability_write_word(adapter->pdev, reg, *value))
  7252. return -E1000_ERR_CONFIG;
  7253. return 0;
  7254. }
  7255. static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features)
  7256. {
  7257. struct igb_adapter *adapter = netdev_priv(netdev);
  7258. struct e1000_hw *hw = &adapter->hw;
  7259. u32 ctrl, rctl;
  7260. bool enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX);
  7261. if (enable) {
  7262. /* enable VLAN tag insert/strip */
  7263. ctrl = rd32(E1000_CTRL);
  7264. ctrl |= E1000_CTRL_VME;
  7265. wr32(E1000_CTRL, ctrl);
  7266. /* Disable CFI check */
  7267. rctl = rd32(E1000_RCTL);
  7268. rctl &= ~E1000_RCTL_CFIEN;
  7269. wr32(E1000_RCTL, rctl);
  7270. } else {
  7271. /* disable VLAN tag insert/strip */
  7272. ctrl = rd32(E1000_CTRL);
  7273. ctrl &= ~E1000_CTRL_VME;
  7274. wr32(E1000_CTRL, ctrl);
  7275. }
  7276. igb_set_vf_vlan_strip(adapter, adapter->vfs_allocated_count, enable);
  7277. }
  7278. static int igb_vlan_rx_add_vid(struct net_device *netdev,
  7279. __be16 proto, u16 vid)
  7280. {
  7281. struct igb_adapter *adapter = netdev_priv(netdev);
  7282. struct e1000_hw *hw = &adapter->hw;
  7283. int pf_id = adapter->vfs_allocated_count;
  7284. /* add the filter since PF can receive vlans w/o entry in vlvf */
  7285. if (!vid || !(adapter->flags & IGB_FLAG_VLAN_PROMISC))
  7286. igb_vfta_set(hw, vid, pf_id, true, !!vid);
  7287. set_bit(vid, adapter->active_vlans);
  7288. return 0;
  7289. }
  7290. static int igb_vlan_rx_kill_vid(struct net_device *netdev,
  7291. __be16 proto, u16 vid)
  7292. {
  7293. struct igb_adapter *adapter = netdev_priv(netdev);
  7294. int pf_id = adapter->vfs_allocated_count;
  7295. struct e1000_hw *hw = &adapter->hw;
  7296. /* remove VID from filter table */
  7297. if (vid && !(adapter->flags & IGB_FLAG_VLAN_PROMISC))
  7298. igb_vfta_set(hw, vid, pf_id, false, true);
  7299. clear_bit(vid, adapter->active_vlans);
  7300. return 0;
  7301. }
  7302. static void igb_restore_vlan(struct igb_adapter *adapter)
  7303. {
  7304. u16 vid = 1;
  7305. igb_vlan_mode(adapter->netdev, adapter->netdev->features);
  7306. igb_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), 0);
  7307. for_each_set_bit_from(vid, adapter->active_vlans, VLAN_N_VID)
  7308. igb_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);
  7309. }
  7310. int igb_set_spd_dplx(struct igb_adapter *adapter, u32 spd, u8 dplx)
  7311. {
  7312. struct pci_dev *pdev = adapter->pdev;
  7313. struct e1000_mac_info *mac = &adapter->hw.mac;
  7314. mac->autoneg = 0;
  7315. /* Make sure dplx is at most 1 bit and lsb of speed is not set
  7316. * for the switch() below to work
  7317. */
  7318. if ((spd & 1) || (dplx & ~1))
  7319. goto err_inval;
  7320. /* Fiber NIC's only allow 1000 gbps Full duplex
  7321. * and 100Mbps Full duplex for 100baseFx sfp
  7322. */
  7323. if (adapter->hw.phy.media_type == e1000_media_type_internal_serdes) {
  7324. switch (spd + dplx) {
  7325. case SPEED_10 + DUPLEX_HALF:
  7326. case SPEED_10 + DUPLEX_FULL:
  7327. case SPEED_100 + DUPLEX_HALF:
  7328. goto err_inval;
  7329. default:
  7330. break;
  7331. }
  7332. }
  7333. switch (spd + dplx) {
  7334. case SPEED_10 + DUPLEX_HALF:
  7335. mac->forced_speed_duplex = ADVERTISE_10_HALF;
  7336. break;
  7337. case SPEED_10 + DUPLEX_FULL:
  7338. mac->forced_speed_duplex = ADVERTISE_10_FULL;
  7339. break;
  7340. case SPEED_100 + DUPLEX_HALF:
  7341. mac->forced_speed_duplex = ADVERTISE_100_HALF;
  7342. break;
  7343. case SPEED_100 + DUPLEX_FULL:
  7344. mac->forced_speed_duplex = ADVERTISE_100_FULL;
  7345. break;
  7346. case SPEED_1000 + DUPLEX_FULL:
  7347. mac->autoneg = 1;
  7348. adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
  7349. break;
  7350. case SPEED_1000 + DUPLEX_HALF: /* not supported */
  7351. default:
  7352. goto err_inval;
  7353. }
  7354. /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
  7355. adapter->hw.phy.mdix = AUTO_ALL_MODES;
  7356. return 0;
  7357. err_inval:
  7358. dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
  7359. return -EINVAL;
  7360. }
  7361. static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake,
  7362. bool runtime)
  7363. {
  7364. struct net_device *netdev = pci_get_drvdata(pdev);
  7365. struct igb_adapter *adapter = netdev_priv(netdev);
  7366. struct e1000_hw *hw = &adapter->hw;
  7367. u32 ctrl, rctl, status;
  7368. u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
  7369. bool wake;
  7370. rtnl_lock();
  7371. netif_device_detach(netdev);
  7372. if (netif_running(netdev))
  7373. __igb_close(netdev, true);
  7374. igb_ptp_suspend(adapter);
  7375. igb_clear_interrupt_scheme(adapter);
  7376. rtnl_unlock();
  7377. status = rd32(E1000_STATUS);
  7378. if (status & E1000_STATUS_LU)
  7379. wufc &= ~E1000_WUFC_LNKC;
  7380. if (wufc) {
  7381. igb_setup_rctl(adapter);
  7382. igb_set_rx_mode(netdev);
  7383. /* turn on all-multi mode if wake on multicast is enabled */
  7384. if (wufc & E1000_WUFC_MC) {
  7385. rctl = rd32(E1000_RCTL);
  7386. rctl |= E1000_RCTL_MPE;
  7387. wr32(E1000_RCTL, rctl);
  7388. }
  7389. ctrl = rd32(E1000_CTRL);
  7390. ctrl |= E1000_CTRL_ADVD3WUC;
  7391. wr32(E1000_CTRL, ctrl);
  7392. /* Allow time for pending master requests to run */
  7393. igb_disable_pcie_master(hw);
  7394. wr32(E1000_WUC, E1000_WUC_PME_EN);
  7395. wr32(E1000_WUFC, wufc);
  7396. } else {
  7397. wr32(E1000_WUC, 0);
  7398. wr32(E1000_WUFC, 0);
  7399. }
  7400. wake = wufc || adapter->en_mng_pt;
  7401. if (!wake)
  7402. igb_power_down_link(adapter);
  7403. else
  7404. igb_power_up_link(adapter);
  7405. if (enable_wake)
  7406. *enable_wake = wake;
  7407. /* Release control of h/w to f/w. If f/w is AMT enabled, this
  7408. * would have already happened in close and is redundant.
  7409. */
  7410. igb_release_hw_control(adapter);
  7411. pci_disable_device(pdev);
  7412. return 0;
  7413. }
  7414. static void igb_deliver_wake_packet(struct net_device *netdev)
  7415. {
  7416. struct igb_adapter *adapter = netdev_priv(netdev);
  7417. struct e1000_hw *hw = &adapter->hw;
  7418. struct sk_buff *skb;
  7419. u32 wupl;
  7420. wupl = rd32(E1000_WUPL) & E1000_WUPL_MASK;
  7421. /* WUPM stores only the first 128 bytes of the wake packet.
  7422. * Read the packet only if we have the whole thing.
  7423. */
  7424. if ((wupl == 0) || (wupl > E1000_WUPM_BYTES))
  7425. return;
  7426. skb = netdev_alloc_skb_ip_align(netdev, E1000_WUPM_BYTES);
  7427. if (!skb)
  7428. return;
  7429. skb_put(skb, wupl);
  7430. /* Ensure reads are 32-bit aligned */
  7431. wupl = roundup(wupl, 4);
  7432. memcpy_fromio(skb->data, hw->hw_addr + E1000_WUPM_REG(0), wupl);
  7433. skb->protocol = eth_type_trans(skb, netdev);
  7434. netif_rx(skb);
  7435. }
  7436. static int __maybe_unused igb_suspend(struct device *dev)
  7437. {
  7438. return __igb_shutdown(to_pci_dev(dev), NULL, 0);
  7439. }
  7440. static int __maybe_unused igb_resume(struct device *dev)
  7441. {
  7442. struct pci_dev *pdev = to_pci_dev(dev);
  7443. struct net_device *netdev = pci_get_drvdata(pdev);
  7444. struct igb_adapter *adapter = netdev_priv(netdev);
  7445. struct e1000_hw *hw = &adapter->hw;
  7446. u32 err, val;
  7447. pci_set_power_state(pdev, PCI_D0);
  7448. pci_restore_state(pdev);
  7449. pci_save_state(pdev);
  7450. if (!pci_device_is_present(pdev))
  7451. return -ENODEV;
  7452. err = pci_enable_device_mem(pdev);
  7453. if (err) {
  7454. dev_err(&pdev->dev,
  7455. "igb: Cannot enable PCI device from suspend\n");
  7456. return err;
  7457. }
  7458. pci_set_master(pdev);
  7459. pci_enable_wake(pdev, PCI_D3hot, 0);
  7460. pci_enable_wake(pdev, PCI_D3cold, 0);
  7461. if (igb_init_interrupt_scheme(adapter, true)) {
  7462. dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
  7463. return -ENOMEM;
  7464. }
  7465. igb_reset(adapter);
  7466. /* let the f/w know that the h/w is now under the control of the
  7467. * driver.
  7468. */
  7469. igb_get_hw_control(adapter);
  7470. val = rd32(E1000_WUS);
  7471. if (val & WAKE_PKT_WUS)
  7472. igb_deliver_wake_packet(netdev);
  7473. wr32(E1000_WUS, ~0);
  7474. rtnl_lock();
  7475. if (!err && netif_running(netdev))
  7476. err = __igb_open(netdev, true);
  7477. if (!err)
  7478. netif_device_attach(netdev);
  7479. rtnl_unlock();
  7480. return err;
  7481. }
  7482. static int __maybe_unused igb_runtime_idle(struct device *dev)
  7483. {
  7484. struct net_device *netdev = dev_get_drvdata(dev);
  7485. struct igb_adapter *adapter = netdev_priv(netdev);
  7486. if (!igb_has_link(adapter))
  7487. pm_schedule_suspend(dev, MSEC_PER_SEC * 5);
  7488. return -EBUSY;
  7489. }
  7490. static int __maybe_unused igb_runtime_suspend(struct device *dev)
  7491. {
  7492. return __igb_shutdown(to_pci_dev(dev), NULL, 1);
  7493. }
  7494. static int __maybe_unused igb_runtime_resume(struct device *dev)
  7495. {
  7496. return igb_resume(dev);
  7497. }
  7498. static void igb_shutdown(struct pci_dev *pdev)
  7499. {
  7500. bool wake;
  7501. __igb_shutdown(pdev, &wake, 0);
  7502. if (system_state == SYSTEM_POWER_OFF) {
  7503. pci_wake_from_d3(pdev, wake);
  7504. pci_set_power_state(pdev, PCI_D3hot);
  7505. }
  7506. }
  7507. #ifdef CONFIG_PCI_IOV
  7508. static int igb_sriov_reinit(struct pci_dev *dev)
  7509. {
  7510. struct net_device *netdev = pci_get_drvdata(dev);
  7511. struct igb_adapter *adapter = netdev_priv(netdev);
  7512. struct pci_dev *pdev = adapter->pdev;
  7513. rtnl_lock();
  7514. if (netif_running(netdev))
  7515. igb_close(netdev);
  7516. else
  7517. igb_reset(adapter);
  7518. igb_clear_interrupt_scheme(adapter);
  7519. igb_init_queue_configuration(adapter);
  7520. if (igb_init_interrupt_scheme(adapter, true)) {
  7521. rtnl_unlock();
  7522. dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
  7523. return -ENOMEM;
  7524. }
  7525. if (netif_running(netdev))
  7526. igb_open(netdev);
  7527. rtnl_unlock();
  7528. return 0;
  7529. }
  7530. static int igb_pci_disable_sriov(struct pci_dev *dev)
  7531. {
  7532. int err = igb_disable_sriov(dev);
  7533. if (!err)
  7534. err = igb_sriov_reinit(dev);
  7535. return err;
  7536. }
  7537. static int igb_pci_enable_sriov(struct pci_dev *dev, int num_vfs)
  7538. {
  7539. int err = igb_enable_sriov(dev, num_vfs);
  7540. if (err)
  7541. goto out;
  7542. err = igb_sriov_reinit(dev);
  7543. if (!err)
  7544. return num_vfs;
  7545. out:
  7546. return err;
  7547. }
  7548. #endif
  7549. static int igb_pci_sriov_configure(struct pci_dev *dev, int num_vfs)
  7550. {
  7551. #ifdef CONFIG_PCI_IOV
  7552. if (num_vfs == 0)
  7553. return igb_pci_disable_sriov(dev);
  7554. else
  7555. return igb_pci_enable_sriov(dev, num_vfs);
  7556. #endif
  7557. return 0;
  7558. }
  7559. /**
  7560. * igb_io_error_detected - called when PCI error is detected
  7561. * @pdev: Pointer to PCI device
  7562. * @state: The current pci connection state
  7563. *
  7564. * This function is called after a PCI bus error affecting
  7565. * this device has been detected.
  7566. **/
  7567. static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
  7568. pci_channel_state_t state)
  7569. {
  7570. struct net_device *netdev = pci_get_drvdata(pdev);
  7571. struct igb_adapter *adapter = netdev_priv(netdev);
  7572. netif_device_detach(netdev);
  7573. if (state == pci_channel_io_perm_failure)
  7574. return PCI_ERS_RESULT_DISCONNECT;
  7575. if (netif_running(netdev))
  7576. igb_down(adapter);
  7577. pci_disable_device(pdev);
  7578. /* Request a slot slot reset. */
  7579. return PCI_ERS_RESULT_NEED_RESET;
  7580. }
  7581. /**
  7582. * igb_io_slot_reset - called after the pci bus has been reset.
  7583. * @pdev: Pointer to PCI device
  7584. *
  7585. * Restart the card from scratch, as if from a cold-boot. Implementation
  7586. * resembles the first-half of the igb_resume routine.
  7587. **/
  7588. static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
  7589. {
  7590. struct net_device *netdev = pci_get_drvdata(pdev);
  7591. struct igb_adapter *adapter = netdev_priv(netdev);
  7592. struct e1000_hw *hw = &adapter->hw;
  7593. pci_ers_result_t result;
  7594. if (pci_enable_device_mem(pdev)) {
  7595. dev_err(&pdev->dev,
  7596. "Cannot re-enable PCI device after reset.\n");
  7597. result = PCI_ERS_RESULT_DISCONNECT;
  7598. } else {
  7599. pci_set_master(pdev);
  7600. pci_restore_state(pdev);
  7601. pci_save_state(pdev);
  7602. pci_enable_wake(pdev, PCI_D3hot, 0);
  7603. pci_enable_wake(pdev, PCI_D3cold, 0);
  7604. /* In case of PCI error, adapter lose its HW address
  7605. * so we should re-assign it here.
  7606. */
  7607. hw->hw_addr = adapter->io_addr;
  7608. igb_reset(adapter);
  7609. wr32(E1000_WUS, ~0);
  7610. result = PCI_ERS_RESULT_RECOVERED;
  7611. }
  7612. return result;
  7613. }
  7614. /**
  7615. * igb_io_resume - called when traffic can start flowing again.
  7616. * @pdev: Pointer to PCI device
  7617. *
  7618. * This callback is called when the error recovery driver tells us that
  7619. * its OK to resume normal operation. Implementation resembles the
  7620. * second-half of the igb_resume routine.
  7621. */
  7622. static void igb_io_resume(struct pci_dev *pdev)
  7623. {
  7624. struct net_device *netdev = pci_get_drvdata(pdev);
  7625. struct igb_adapter *adapter = netdev_priv(netdev);
  7626. if (netif_running(netdev)) {
  7627. if (igb_up(adapter)) {
  7628. dev_err(&pdev->dev, "igb_up failed after reset\n");
  7629. return;
  7630. }
  7631. }
  7632. netif_device_attach(netdev);
  7633. /* let the f/w know that the h/w is now under the control of the
  7634. * driver.
  7635. */
  7636. igb_get_hw_control(adapter);
  7637. }
  7638. /**
  7639. * igb_rar_set_index - Sync RAL[index] and RAH[index] registers with MAC table
  7640. * @adapter: Pointer to adapter structure
  7641. * @index: Index of the RAR entry which need to be synced with MAC table
  7642. **/
  7643. static void igb_rar_set_index(struct igb_adapter *adapter, u32 index)
  7644. {
  7645. struct e1000_hw *hw = &adapter->hw;
  7646. u32 rar_low, rar_high;
  7647. u8 *addr = adapter->mac_table[index].addr;
  7648. /* HW expects these to be in network order when they are plugged
  7649. * into the registers which are little endian. In order to guarantee
  7650. * that ordering we need to do an leXX_to_cpup here in order to be
  7651. * ready for the byteswap that occurs with writel
  7652. */
  7653. rar_low = le32_to_cpup((__le32 *)(addr));
  7654. rar_high = le16_to_cpup((__le16 *)(addr + 4));
  7655. /* Indicate to hardware the Address is Valid. */
  7656. if (adapter->mac_table[index].state & IGB_MAC_STATE_IN_USE) {
  7657. if (is_valid_ether_addr(addr))
  7658. rar_high |= E1000_RAH_AV;
  7659. if (adapter->mac_table[index].state & IGB_MAC_STATE_SRC_ADDR)
  7660. rar_high |= E1000_RAH_ASEL_SRC_ADDR;
  7661. switch (hw->mac.type) {
  7662. case e1000_82575:
  7663. case e1000_i210:
  7664. if (adapter->mac_table[index].state &
  7665. IGB_MAC_STATE_QUEUE_STEERING)
  7666. rar_high |= E1000_RAH_QSEL_ENABLE;
  7667. rar_high |= E1000_RAH_POOL_1 *
  7668. adapter->mac_table[index].queue;
  7669. break;
  7670. default:
  7671. rar_high |= E1000_RAH_POOL_1 <<
  7672. adapter->mac_table[index].queue;
  7673. break;
  7674. }
  7675. }
  7676. wr32(E1000_RAL(index), rar_low);
  7677. wrfl();
  7678. wr32(E1000_RAH(index), rar_high);
  7679. wrfl();
  7680. }
  7681. static int igb_set_vf_mac(struct igb_adapter *adapter,
  7682. int vf, unsigned char *mac_addr)
  7683. {
  7684. struct e1000_hw *hw = &adapter->hw;
  7685. /* VF MAC addresses start at end of receive addresses and moves
  7686. * towards the first, as a result a collision should not be possible
  7687. */
  7688. int rar_entry = hw->mac.rar_entry_count - (vf + 1);
  7689. unsigned char *vf_mac_addr = adapter->vf_data[vf].vf_mac_addresses;
  7690. ether_addr_copy(vf_mac_addr, mac_addr);
  7691. ether_addr_copy(adapter->mac_table[rar_entry].addr, mac_addr);
  7692. adapter->mac_table[rar_entry].queue = vf;
  7693. adapter->mac_table[rar_entry].state |= IGB_MAC_STATE_IN_USE;
  7694. igb_rar_set_index(adapter, rar_entry);
  7695. return 0;
  7696. }
  7697. static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
  7698. {
  7699. struct igb_adapter *adapter = netdev_priv(netdev);
  7700. if (vf >= adapter->vfs_allocated_count)
  7701. return -EINVAL;
  7702. /* Setting the VF MAC to 0 reverts the IGB_VF_FLAG_PF_SET_MAC
  7703. * flag and allows to overwrite the MAC via VF netdev. This
  7704. * is necessary to allow libvirt a way to restore the original
  7705. * MAC after unbinding vfio-pci and reloading igbvf after shutting
  7706. * down a VM.
  7707. */
  7708. if (is_zero_ether_addr(mac)) {
  7709. adapter->vf_data[vf].flags &= ~IGB_VF_FLAG_PF_SET_MAC;
  7710. dev_info(&adapter->pdev->dev,
  7711. "remove administratively set MAC on VF %d\n",
  7712. vf);
  7713. } else if (is_valid_ether_addr(mac)) {
  7714. adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
  7715. dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n",
  7716. mac, vf);
  7717. dev_info(&adapter->pdev->dev,
  7718. "Reload the VF driver to make this change effective.");
  7719. /* Generate additional warning if PF is down */
  7720. if (test_bit(__IGB_DOWN, &adapter->state)) {
  7721. dev_warn(&adapter->pdev->dev,
  7722. "The VF MAC address has been set, but the PF device is not up.\n");
  7723. dev_warn(&adapter->pdev->dev,
  7724. "Bring the PF device up before attempting to use the VF device.\n");
  7725. }
  7726. } else {
  7727. return -EINVAL;
  7728. }
  7729. return igb_set_vf_mac(adapter, vf, mac);
  7730. }
  7731. static int igb_link_mbps(int internal_link_speed)
  7732. {
  7733. switch (internal_link_speed) {
  7734. case SPEED_100:
  7735. return 100;
  7736. case SPEED_1000:
  7737. return 1000;
  7738. default:
  7739. return 0;
  7740. }
  7741. }
  7742. static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
  7743. int link_speed)
  7744. {
  7745. int rf_dec, rf_int;
  7746. u32 bcnrc_val;
  7747. if (tx_rate != 0) {
  7748. /* Calculate the rate factor values to set */
  7749. rf_int = link_speed / tx_rate;
  7750. rf_dec = (link_speed - (rf_int * tx_rate));
  7751. rf_dec = (rf_dec * BIT(E1000_RTTBCNRC_RF_INT_SHIFT)) /
  7752. tx_rate;
  7753. bcnrc_val = E1000_RTTBCNRC_RS_ENA;
  7754. bcnrc_val |= ((rf_int << E1000_RTTBCNRC_RF_INT_SHIFT) &
  7755. E1000_RTTBCNRC_RF_INT_MASK);
  7756. bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
  7757. } else {
  7758. bcnrc_val = 0;
  7759. }
  7760. wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */
  7761. /* Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
  7762. * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported.
  7763. */
  7764. wr32(E1000_RTTBCNRM, 0x14);
  7765. wr32(E1000_RTTBCNRC, bcnrc_val);
  7766. }
  7767. static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
  7768. {
  7769. int actual_link_speed, i;
  7770. bool reset_rate = false;
  7771. /* VF TX rate limit was not set or not supported */
  7772. if ((adapter->vf_rate_link_speed == 0) ||
  7773. (adapter->hw.mac.type != e1000_82576))
  7774. return;
  7775. actual_link_speed = igb_link_mbps(adapter->link_speed);
  7776. if (actual_link_speed != adapter->vf_rate_link_speed) {
  7777. reset_rate = true;
  7778. adapter->vf_rate_link_speed = 0;
  7779. dev_info(&adapter->pdev->dev,
  7780. "Link speed has been changed. VF Transmit rate is disabled\n");
  7781. }
  7782. for (i = 0; i < adapter->vfs_allocated_count; i++) {
  7783. if (reset_rate)
  7784. adapter->vf_data[i].tx_rate = 0;
  7785. igb_set_vf_rate_limit(&adapter->hw, i,
  7786. adapter->vf_data[i].tx_rate,
  7787. actual_link_speed);
  7788. }
  7789. }
  7790. static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf,
  7791. int min_tx_rate, int max_tx_rate)
  7792. {
  7793. struct igb_adapter *adapter = netdev_priv(netdev);
  7794. struct e1000_hw *hw = &adapter->hw;
  7795. int actual_link_speed;
  7796. if (hw->mac.type != e1000_82576)
  7797. return -EOPNOTSUPP;
  7798. if (min_tx_rate)
  7799. return -EINVAL;
  7800. actual_link_speed = igb_link_mbps(adapter->link_speed);
  7801. if ((vf >= adapter->vfs_allocated_count) ||
  7802. (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) ||
  7803. (max_tx_rate < 0) ||
  7804. (max_tx_rate > actual_link_speed))
  7805. return -EINVAL;
  7806. adapter->vf_rate_link_speed = actual_link_speed;
  7807. adapter->vf_data[vf].tx_rate = (u16)max_tx_rate;
  7808. igb_set_vf_rate_limit(hw, vf, max_tx_rate, actual_link_speed);
  7809. return 0;
  7810. }
  7811. static int igb_ndo_set_vf_spoofchk(struct net_device *netdev, int vf,
  7812. bool setting)
  7813. {
  7814. struct igb_adapter *adapter = netdev_priv(netdev);
  7815. struct e1000_hw *hw = &adapter->hw;
  7816. u32 reg_val, reg_offset;
  7817. if (!adapter->vfs_allocated_count)
  7818. return -EOPNOTSUPP;
  7819. if (vf >= adapter->vfs_allocated_count)
  7820. return -EINVAL;
  7821. reg_offset = (hw->mac.type == e1000_82576) ? E1000_DTXSWC : E1000_TXSWC;
  7822. reg_val = rd32(reg_offset);
  7823. if (setting)
  7824. reg_val |= (BIT(vf) |
  7825. BIT(vf + E1000_DTXSWC_VLAN_SPOOF_SHIFT));
  7826. else
  7827. reg_val &= ~(BIT(vf) |
  7828. BIT(vf + E1000_DTXSWC_VLAN_SPOOF_SHIFT));
  7829. wr32(reg_offset, reg_val);
  7830. adapter->vf_data[vf].spoofchk_enabled = setting;
  7831. return 0;
  7832. }
  7833. static int igb_ndo_set_vf_trust(struct net_device *netdev, int vf, bool setting)
  7834. {
  7835. struct igb_adapter *adapter = netdev_priv(netdev);
  7836. if (vf >= adapter->vfs_allocated_count)
  7837. return -EINVAL;
  7838. if (adapter->vf_data[vf].trusted == setting)
  7839. return 0;
  7840. adapter->vf_data[vf].trusted = setting;
  7841. dev_info(&adapter->pdev->dev, "VF %u is %strusted\n",
  7842. vf, setting ? "" : "not ");
  7843. return 0;
  7844. }
  7845. static int igb_ndo_get_vf_config(struct net_device *netdev,
  7846. int vf, struct ifla_vf_info *ivi)
  7847. {
  7848. struct igb_adapter *adapter = netdev_priv(netdev);
  7849. if (vf >= adapter->vfs_allocated_count)
  7850. return -EINVAL;
  7851. ivi->vf = vf;
  7852. memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
  7853. ivi->max_tx_rate = adapter->vf_data[vf].tx_rate;
  7854. ivi->min_tx_rate = 0;
  7855. ivi->vlan = adapter->vf_data[vf].pf_vlan;
  7856. ivi->qos = adapter->vf_data[vf].pf_qos;
  7857. ivi->spoofchk = adapter->vf_data[vf].spoofchk_enabled;
  7858. ivi->trusted = adapter->vf_data[vf].trusted;
  7859. return 0;
  7860. }
  7861. static void igb_vmm_control(struct igb_adapter *adapter)
  7862. {
  7863. struct e1000_hw *hw = &adapter->hw;
  7864. u32 reg;
  7865. switch (hw->mac.type) {
  7866. case e1000_82575:
  7867. case e1000_i210:
  7868. case e1000_i211:
  7869. case e1000_i354:
  7870. default:
  7871. /* replication is not supported for 82575 */
  7872. return;
  7873. case e1000_82576:
  7874. /* notify HW that the MAC is adding vlan tags */
  7875. reg = rd32(E1000_DTXCTL);
  7876. reg |= E1000_DTXCTL_VLAN_ADDED;
  7877. wr32(E1000_DTXCTL, reg);
  7878. /* Fall through */
  7879. case e1000_82580:
  7880. /* enable replication vlan tag stripping */
  7881. reg = rd32(E1000_RPLOLR);
  7882. reg |= E1000_RPLOLR_STRVLAN;
  7883. wr32(E1000_RPLOLR, reg);
  7884. /* Fall through */
  7885. case e1000_i350:
  7886. /* none of the above registers are supported by i350 */
  7887. break;
  7888. }
  7889. if (adapter->vfs_allocated_count) {
  7890. igb_vmdq_set_loopback_pf(hw, true);
  7891. igb_vmdq_set_replication_pf(hw, true);
  7892. igb_vmdq_set_anti_spoofing_pf(hw, true,
  7893. adapter->vfs_allocated_count);
  7894. } else {
  7895. igb_vmdq_set_loopback_pf(hw, false);
  7896. igb_vmdq_set_replication_pf(hw, false);
  7897. }
  7898. }
  7899. static void igb_init_dmac(struct igb_adapter *adapter, u32 pba)
  7900. {
  7901. struct e1000_hw *hw = &adapter->hw;
  7902. u32 dmac_thr;
  7903. u16 hwm;
  7904. if (hw->mac.type > e1000_82580) {
  7905. if (adapter->flags & IGB_FLAG_DMAC) {
  7906. u32 reg;
  7907. /* force threshold to 0. */
  7908. wr32(E1000_DMCTXTH, 0);
  7909. /* DMA Coalescing high water mark needs to be greater
  7910. * than the Rx threshold. Set hwm to PBA - max frame
  7911. * size in 16B units, capping it at PBA - 6KB.
  7912. */
  7913. hwm = 64 * (pba - 6);
  7914. reg = rd32(E1000_FCRTC);
  7915. reg &= ~E1000_FCRTC_RTH_COAL_MASK;
  7916. reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
  7917. & E1000_FCRTC_RTH_COAL_MASK);
  7918. wr32(E1000_FCRTC, reg);
  7919. /* Set the DMA Coalescing Rx threshold to PBA - 2 * max
  7920. * frame size, capping it at PBA - 10KB.
  7921. */
  7922. dmac_thr = pba - 10;
  7923. reg = rd32(E1000_DMACR);
  7924. reg &= ~E1000_DMACR_DMACTHR_MASK;
  7925. reg |= ((dmac_thr << E1000_DMACR_DMACTHR_SHIFT)
  7926. & E1000_DMACR_DMACTHR_MASK);
  7927. /* transition to L0x or L1 if available..*/
  7928. reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
  7929. /* watchdog timer= +-1000 usec in 32usec intervals */
  7930. reg |= (1000 >> 5);
  7931. /* Disable BMC-to-OS Watchdog Enable */
  7932. if (hw->mac.type != e1000_i354)
  7933. reg &= ~E1000_DMACR_DC_BMC2OSW_EN;
  7934. wr32(E1000_DMACR, reg);
  7935. /* no lower threshold to disable
  7936. * coalescing(smart fifb)-UTRESH=0
  7937. */
  7938. wr32(E1000_DMCRTRH, 0);
  7939. reg = (IGB_DMCTLX_DCFLUSH_DIS | 0x4);
  7940. wr32(E1000_DMCTLX, reg);
  7941. /* free space in tx packet buffer to wake from
  7942. * DMA coal
  7943. */
  7944. wr32(E1000_DMCTXTH, (IGB_MIN_TXPBSIZE -
  7945. (IGB_TX_BUF_4096 + adapter->max_frame_size)) >> 6);
  7946. /* make low power state decision controlled
  7947. * by DMA coal
  7948. */
  7949. reg = rd32(E1000_PCIEMISC);
  7950. reg &= ~E1000_PCIEMISC_LX_DECISION;
  7951. wr32(E1000_PCIEMISC, reg);
  7952. } /* endif adapter->dmac is not disabled */
  7953. } else if (hw->mac.type == e1000_82580) {
  7954. u32 reg = rd32(E1000_PCIEMISC);
  7955. wr32(E1000_PCIEMISC, reg & ~E1000_PCIEMISC_LX_DECISION);
  7956. wr32(E1000_DMACR, 0);
  7957. }
  7958. }
  7959. /**
  7960. * igb_read_i2c_byte - Reads 8 bit word over I2C
  7961. * @hw: pointer to hardware structure
  7962. * @byte_offset: byte offset to read
  7963. * @dev_addr: device address
  7964. * @data: value read
  7965. *
  7966. * Performs byte read operation over I2C interface at
  7967. * a specified device address.
  7968. **/
  7969. s32 igb_read_i2c_byte(struct e1000_hw *hw, u8 byte_offset,
  7970. u8 dev_addr, u8 *data)
  7971. {
  7972. struct igb_adapter *adapter = container_of(hw, struct igb_adapter, hw);
  7973. struct i2c_client *this_client = adapter->i2c_client;
  7974. s32 status;
  7975. u16 swfw_mask = 0;
  7976. if (!this_client)
  7977. return E1000_ERR_I2C;
  7978. swfw_mask = E1000_SWFW_PHY0_SM;
  7979. if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask))
  7980. return E1000_ERR_SWFW_SYNC;
  7981. status = i2c_smbus_read_byte_data(this_client, byte_offset);
  7982. hw->mac.ops.release_swfw_sync(hw, swfw_mask);
  7983. if (status < 0)
  7984. return E1000_ERR_I2C;
  7985. else {
  7986. *data = status;
  7987. return 0;
  7988. }
  7989. }
  7990. /**
  7991. * igb_write_i2c_byte - Writes 8 bit word over I2C
  7992. * @hw: pointer to hardware structure
  7993. * @byte_offset: byte offset to write
  7994. * @dev_addr: device address
  7995. * @data: value to write
  7996. *
  7997. * Performs byte write operation over I2C interface at
  7998. * a specified device address.
  7999. **/
  8000. s32 igb_write_i2c_byte(struct e1000_hw *hw, u8 byte_offset,
  8001. u8 dev_addr, u8 data)
  8002. {
  8003. struct igb_adapter *adapter = container_of(hw, struct igb_adapter, hw);
  8004. struct i2c_client *this_client = adapter->i2c_client;
  8005. s32 status;
  8006. u16 swfw_mask = E1000_SWFW_PHY0_SM;
  8007. if (!this_client)
  8008. return E1000_ERR_I2C;
  8009. if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask))
  8010. return E1000_ERR_SWFW_SYNC;
  8011. status = i2c_smbus_write_byte_data(this_client, byte_offset, data);
  8012. hw->mac.ops.release_swfw_sync(hw, swfw_mask);
  8013. if (status)
  8014. return E1000_ERR_I2C;
  8015. else
  8016. return 0;
  8017. }
  8018. int igb_reinit_queues(struct igb_adapter *adapter)
  8019. {
  8020. struct net_device *netdev = adapter->netdev;
  8021. struct pci_dev *pdev = adapter->pdev;
  8022. int err = 0;
  8023. if (netif_running(netdev))
  8024. igb_close(netdev);
  8025. igb_reset_interrupt_capability(adapter);
  8026. if (igb_init_interrupt_scheme(adapter, true)) {
  8027. dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
  8028. return -ENOMEM;
  8029. }
  8030. if (netif_running(netdev))
  8031. err = igb_open(netdev);
  8032. return err;
  8033. }
  8034. static void igb_nfc_filter_exit(struct igb_adapter *adapter)
  8035. {
  8036. struct igb_nfc_filter *rule;
  8037. spin_lock(&adapter->nfc_lock);
  8038. hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node)
  8039. igb_erase_filter(adapter, rule);
  8040. hlist_for_each_entry(rule, &adapter->cls_flower_list, nfc_node)
  8041. igb_erase_filter(adapter, rule);
  8042. spin_unlock(&adapter->nfc_lock);
  8043. }
  8044. static void igb_nfc_filter_restore(struct igb_adapter *adapter)
  8045. {
  8046. struct igb_nfc_filter *rule;
  8047. spin_lock(&adapter->nfc_lock);
  8048. hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node)
  8049. igb_add_filter(adapter, rule);
  8050. spin_unlock(&adapter->nfc_lock);
  8051. }
  8052. /* igb_main.c */