PageRenderTime 61ms CodeModel.GetById 18ms RepoModel.GetById 0ms app.codeStats 1ms

/drivers/infiniband/hw/ipath/ipath_driver.c

https://bitbucket.org/abioy/linux
C | 2816 lines | 1898 code | 306 blank | 612 comment | 328 complexity | ca44f78cee3afdf9b7593b762b99e426 MD5 | raw file
Possible License(s): CC-BY-SA-3.0, GPL-2.0, LGPL-2.0, AGPL-1.0
  1. /*
  2. * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
  3. * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
  4. *
  5. * This software is available to you under a choice of one of two
  6. * licenses. You may choose to be licensed under the terms of the GNU
  7. * General Public License (GPL) Version 2, available from the file
  8. * COPYING in the main directory of this source tree, or the
  9. * OpenIB.org BSD license below:
  10. *
  11. * Redistribution and use in source and binary forms, with or
  12. * without modification, are permitted provided that the following
  13. * conditions are met:
  14. *
  15. * - Redistributions of source code must retain the above
  16. * copyright notice, this list of conditions and the following
  17. * disclaimer.
  18. *
  19. * - Redistributions in binary form must reproduce the above
  20. * copyright notice, this list of conditions and the following
  21. * disclaimer in the documentation and/or other materials
  22. * provided with the distribution.
  23. *
  24. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31. * SOFTWARE.
  32. */
  33. #include <linux/sched.h>
  34. #include <linux/spinlock.h>
  35. #include <linux/idr.h>
  36. #include <linux/pci.h>
  37. #include <linux/io.h>
  38. #include <linux/delay.h>
  39. #include <linux/netdevice.h>
  40. #include <linux/vmalloc.h>
  41. #include <linux/bitmap.h>
  42. #include <linux/slab.h>
  43. #include "ipath_kernel.h"
  44. #include "ipath_verbs.h"
  45. static void ipath_update_pio_bufs(struct ipath_devdata *);
  46. const char *ipath_get_unit_name(int unit)
  47. {
  48. static char iname[16];
  49. snprintf(iname, sizeof iname, "infinipath%u", unit);
  50. return iname;
  51. }
  52. #define DRIVER_LOAD_MSG "QLogic " IPATH_DRV_NAME " loaded: "
  53. #define PFX IPATH_DRV_NAME ": "
  54. /*
  55. * The size has to be longer than this string, so we can append
  56. * board/chip information to it in the init code.
  57. */
  58. const char ib_ipath_version[] = IPATH_IDSTR "\n";
  59. static struct idr unit_table;
  60. DEFINE_SPINLOCK(ipath_devs_lock);
  61. LIST_HEAD(ipath_dev_list);
  62. wait_queue_head_t ipath_state_wait;
  63. unsigned ipath_debug = __IPATH_INFO;
  64. module_param_named(debug, ipath_debug, uint, S_IWUSR | S_IRUGO);
  65. MODULE_PARM_DESC(debug, "mask for debug prints");
  66. EXPORT_SYMBOL_GPL(ipath_debug);
  67. unsigned ipath_mtu4096 = 1; /* max 4KB IB mtu by default, if supported */
  68. module_param_named(mtu4096, ipath_mtu4096, uint, S_IRUGO);
  69. MODULE_PARM_DESC(mtu4096, "enable MTU of 4096 bytes, if supported");
  70. static unsigned ipath_hol_timeout_ms = 13000;
  71. module_param_named(hol_timeout_ms, ipath_hol_timeout_ms, uint, S_IRUGO);
  72. MODULE_PARM_DESC(hol_timeout_ms,
  73. "duration of user app suspension after link failure");
  74. unsigned ipath_linkrecovery = 1;
  75. module_param_named(linkrecovery, ipath_linkrecovery, uint, S_IWUSR | S_IRUGO);
  76. MODULE_PARM_DESC(linkrecovery, "enable workaround for link recovery issue");
  77. MODULE_LICENSE("GPL");
  78. MODULE_AUTHOR("QLogic <support@qlogic.com>");
  79. MODULE_DESCRIPTION("QLogic InfiniPath driver");
  80. /*
  81. * Table to translate the LINKTRAININGSTATE portion of
  82. * IBCStatus to a human-readable form.
  83. */
  84. const char *ipath_ibcstatus_str[] = {
  85. "Disabled",
  86. "LinkUp",
  87. "PollActive",
  88. "PollQuiet",
  89. "SleepDelay",
  90. "SleepQuiet",
  91. "LState6", /* unused */
  92. "LState7", /* unused */
  93. "CfgDebounce",
  94. "CfgRcvfCfg",
  95. "CfgWaitRmt",
  96. "CfgIdle",
  97. "RecovRetrain",
  98. "CfgTxRevLane", /* unused before IBA7220 */
  99. "RecovWaitRmt",
  100. "RecovIdle",
  101. /* below were added for IBA7220 */
  102. "CfgEnhanced",
  103. "CfgTest",
  104. "CfgWaitRmtTest",
  105. "CfgWaitCfgEnhanced",
  106. "SendTS_T",
  107. "SendTstIdles",
  108. "RcvTS_T",
  109. "SendTst_TS1s",
  110. "LTState18", "LTState19", "LTState1A", "LTState1B",
  111. "LTState1C", "LTState1D", "LTState1E", "LTState1F"
  112. };
  113. static void __devexit ipath_remove_one(struct pci_dev *);
  114. static int __devinit ipath_init_one(struct pci_dev *,
  115. const struct pci_device_id *);
  116. /* Only needed for registration, nothing else needs this info */
  117. #define PCI_VENDOR_ID_PATHSCALE 0x1fc1
  118. #define PCI_VENDOR_ID_QLOGIC 0x1077
  119. #define PCI_DEVICE_ID_INFINIPATH_HT 0xd
  120. #define PCI_DEVICE_ID_INFINIPATH_PE800 0x10
  121. #define PCI_DEVICE_ID_INFINIPATH_7220 0x7220
  122. /* Number of seconds before our card status check... */
  123. #define STATUS_TIMEOUT 60
  124. static const struct pci_device_id ipath_pci_tbl[] = {
  125. { PCI_DEVICE(PCI_VENDOR_ID_PATHSCALE, PCI_DEVICE_ID_INFINIPATH_HT) },
  126. { PCI_DEVICE(PCI_VENDOR_ID_PATHSCALE, PCI_DEVICE_ID_INFINIPATH_PE800) },
  127. { PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, PCI_DEVICE_ID_INFINIPATH_7220) },
  128. { 0, }
  129. };
  130. MODULE_DEVICE_TABLE(pci, ipath_pci_tbl);
  131. static struct pci_driver ipath_driver = {
  132. .name = IPATH_DRV_NAME,
  133. .probe = ipath_init_one,
  134. .remove = __devexit_p(ipath_remove_one),
  135. .id_table = ipath_pci_tbl,
  136. .driver = {
  137. .groups = ipath_driver_attr_groups,
  138. },
  139. };
  140. static inline void read_bars(struct ipath_devdata *dd, struct pci_dev *dev,
  141. u32 *bar0, u32 *bar1)
  142. {
  143. int ret;
  144. ret = pci_read_config_dword(dev, PCI_BASE_ADDRESS_0, bar0);
  145. if (ret)
  146. ipath_dev_err(dd, "failed to read bar0 before enable: "
  147. "error %d\n", -ret);
  148. ret = pci_read_config_dword(dev, PCI_BASE_ADDRESS_1, bar1);
  149. if (ret)
  150. ipath_dev_err(dd, "failed to read bar1 before enable: "
  151. "error %d\n", -ret);
  152. ipath_dbg("Read bar0 %x bar1 %x\n", *bar0, *bar1);
  153. }
  154. static void ipath_free_devdata(struct pci_dev *pdev,
  155. struct ipath_devdata *dd)
  156. {
  157. unsigned long flags;
  158. pci_set_drvdata(pdev, NULL);
  159. if (dd->ipath_unit != -1) {
  160. spin_lock_irqsave(&ipath_devs_lock, flags);
  161. idr_remove(&unit_table, dd->ipath_unit);
  162. list_del(&dd->ipath_list);
  163. spin_unlock_irqrestore(&ipath_devs_lock, flags);
  164. }
  165. vfree(dd);
  166. }
  167. static struct ipath_devdata *ipath_alloc_devdata(struct pci_dev *pdev)
  168. {
  169. unsigned long flags;
  170. struct ipath_devdata *dd;
  171. int ret;
  172. if (!idr_pre_get(&unit_table, GFP_KERNEL)) {
  173. dd = ERR_PTR(-ENOMEM);
  174. goto bail;
  175. }
  176. dd = vmalloc(sizeof(*dd));
  177. if (!dd) {
  178. dd = ERR_PTR(-ENOMEM);
  179. goto bail;
  180. }
  181. memset(dd, 0, sizeof(*dd));
  182. dd->ipath_unit = -1;
  183. spin_lock_irqsave(&ipath_devs_lock, flags);
  184. ret = idr_get_new(&unit_table, dd, &dd->ipath_unit);
  185. if (ret < 0) {
  186. printk(KERN_ERR IPATH_DRV_NAME
  187. ": Could not allocate unit ID: error %d\n", -ret);
  188. ipath_free_devdata(pdev, dd);
  189. dd = ERR_PTR(ret);
  190. goto bail_unlock;
  191. }
  192. dd->pcidev = pdev;
  193. pci_set_drvdata(pdev, dd);
  194. list_add(&dd->ipath_list, &ipath_dev_list);
  195. bail_unlock:
  196. spin_unlock_irqrestore(&ipath_devs_lock, flags);
  197. bail:
  198. return dd;
  199. }
  200. static inline struct ipath_devdata *__ipath_lookup(int unit)
  201. {
  202. return idr_find(&unit_table, unit);
  203. }
  204. struct ipath_devdata *ipath_lookup(int unit)
  205. {
  206. struct ipath_devdata *dd;
  207. unsigned long flags;
  208. spin_lock_irqsave(&ipath_devs_lock, flags);
  209. dd = __ipath_lookup(unit);
  210. spin_unlock_irqrestore(&ipath_devs_lock, flags);
  211. return dd;
  212. }
  213. int ipath_count_units(int *npresentp, int *nupp, int *maxportsp)
  214. {
  215. int nunits, npresent, nup;
  216. struct ipath_devdata *dd;
  217. unsigned long flags;
  218. int maxports;
  219. nunits = npresent = nup = maxports = 0;
  220. spin_lock_irqsave(&ipath_devs_lock, flags);
  221. list_for_each_entry(dd, &ipath_dev_list, ipath_list) {
  222. nunits++;
  223. if ((dd->ipath_flags & IPATH_PRESENT) && dd->ipath_kregbase)
  224. npresent++;
  225. if (dd->ipath_lid &&
  226. !(dd->ipath_flags & (IPATH_DISABLED | IPATH_LINKDOWN
  227. | IPATH_LINKUNK)))
  228. nup++;
  229. if (dd->ipath_cfgports > maxports)
  230. maxports = dd->ipath_cfgports;
  231. }
  232. spin_unlock_irqrestore(&ipath_devs_lock, flags);
  233. if (npresentp)
  234. *npresentp = npresent;
  235. if (nupp)
  236. *nupp = nup;
  237. if (maxportsp)
  238. *maxportsp = maxports;
  239. return nunits;
  240. }
  241. /*
  242. * These next two routines are placeholders in case we don't have per-arch
  243. * code for controlling write combining. If explicit control of write
  244. * combining is not available, performance will probably be awful.
  245. */
  246. int __attribute__((weak)) ipath_enable_wc(struct ipath_devdata *dd)
  247. {
  248. return -EOPNOTSUPP;
  249. }
  250. void __attribute__((weak)) ipath_disable_wc(struct ipath_devdata *dd)
  251. {
  252. }
  253. /*
  254. * Perform a PIO buffer bandwidth write test, to verify proper system
  255. * configuration. Even when all the setup calls work, occasionally
  256. * BIOS or other issues can prevent write combining from working, or
  257. * can cause other bandwidth problems to the chip.
  258. *
  259. * This test simply writes the same buffer over and over again, and
  260. * measures close to the peak bandwidth to the chip (not testing
  261. * data bandwidth to the wire). On chips that use an address-based
  262. * trigger to send packets to the wire, this is easy. On chips that
  263. * use a count to trigger, we want to make sure that the packet doesn't
  264. * go out on the wire, or trigger flow control checks.
  265. */
  266. static void ipath_verify_pioperf(struct ipath_devdata *dd)
  267. {
  268. u32 pbnum, cnt, lcnt;
  269. u32 __iomem *piobuf;
  270. u32 *addr;
  271. u64 msecs, emsecs;
  272. piobuf = ipath_getpiobuf(dd, 0, &pbnum);
  273. if (!piobuf) {
  274. dev_info(&dd->pcidev->dev,
  275. "No PIObufs for checking perf, skipping\n");
  276. return;
  277. }
  278. /*
  279. * Enough to give us a reasonable test, less than piobuf size, and
  280. * likely multiple of store buffer length.
  281. */
  282. cnt = 1024;
  283. addr = vmalloc(cnt);
  284. if (!addr) {
  285. dev_info(&dd->pcidev->dev,
  286. "Couldn't get memory for checking PIO perf,"
  287. " skipping\n");
  288. goto done;
  289. }
  290. preempt_disable(); /* we want reasonably accurate elapsed time */
  291. msecs = 1 + jiffies_to_msecs(jiffies);
  292. for (lcnt = 0; lcnt < 10000U; lcnt++) {
  293. /* wait until we cross msec boundary */
  294. if (jiffies_to_msecs(jiffies) >= msecs)
  295. break;
  296. udelay(1);
  297. }
  298. ipath_disable_armlaunch(dd);
  299. /*
  300. * length 0, no dwords actually sent, and mark as VL15
  301. * on chips where that may matter (due to IB flowcontrol)
  302. */
  303. if ((dd->ipath_flags & IPATH_HAS_PBC_CNT))
  304. writeq(1UL << 63, piobuf);
  305. else
  306. writeq(0, piobuf);
  307. ipath_flush_wc();
  308. /*
  309. * this is only roughly accurate, since even with preempt we
  310. * still take interrupts that could take a while. Running for
  311. * >= 5 msec seems to get us "close enough" to accurate values
  312. */
  313. msecs = jiffies_to_msecs(jiffies);
  314. for (emsecs = lcnt = 0; emsecs <= 5UL; lcnt++) {
  315. __iowrite32_copy(piobuf + 64, addr, cnt >> 2);
  316. emsecs = jiffies_to_msecs(jiffies) - msecs;
  317. }
  318. /* 1 GiB/sec, slightly over IB SDR line rate */
  319. if (lcnt < (emsecs * 1024U))
  320. ipath_dev_err(dd,
  321. "Performance problem: bandwidth to PIO buffers is "
  322. "only %u MiB/sec\n",
  323. lcnt / (u32) emsecs);
  324. else
  325. ipath_dbg("PIO buffer bandwidth %u MiB/sec is OK\n",
  326. lcnt / (u32) emsecs);
  327. preempt_enable();
  328. vfree(addr);
  329. done:
  330. /* disarm piobuf, so it's available again */
  331. ipath_disarm_piobufs(dd, pbnum, 1);
  332. ipath_enable_armlaunch(dd);
  333. }
  334. static int __devinit ipath_init_one(struct pci_dev *pdev,
  335. const struct pci_device_id *ent)
  336. {
  337. int ret, len, j;
  338. struct ipath_devdata *dd;
  339. unsigned long long addr;
  340. u32 bar0 = 0, bar1 = 0;
  341. u8 rev;
  342. dd = ipath_alloc_devdata(pdev);
  343. if (IS_ERR(dd)) {
  344. ret = PTR_ERR(dd);
  345. printk(KERN_ERR IPATH_DRV_NAME
  346. ": Could not allocate devdata: error %d\n", -ret);
  347. goto bail;
  348. }
  349. ipath_cdbg(VERBOSE, "initializing unit #%u\n", dd->ipath_unit);
  350. ret = pci_enable_device(pdev);
  351. if (ret) {
  352. /* This can happen iff:
  353. *
  354. * We did a chip reset, and then failed to reprogram the
  355. * BAR, or the chip reset due to an internal error. We then
  356. * unloaded the driver and reloaded it.
  357. *
  358. * Both reset cases set the BAR back to initial state. For
  359. * the latter case, the AER sticky error bit at offset 0x718
  360. * should be set, but the Linux kernel doesn't yet know
  361. * about that, it appears. If the original BAR was retained
  362. * in the kernel data structures, this may be OK.
  363. */
  364. ipath_dev_err(dd, "enable unit %d failed: error %d\n",
  365. dd->ipath_unit, -ret);
  366. goto bail_devdata;
  367. }
  368. addr = pci_resource_start(pdev, 0);
  369. len = pci_resource_len(pdev, 0);
  370. ipath_cdbg(VERBOSE, "regbase (0) %llx len %d irq %d, vend %x/%x "
  371. "driver_data %lx\n", addr, len, pdev->irq, ent->vendor,
  372. ent->device, ent->driver_data);
  373. read_bars(dd, pdev, &bar0, &bar1);
  374. if (!bar1 && !(bar0 & ~0xf)) {
  375. if (addr) {
  376. dev_info(&pdev->dev, "BAR is 0 (probable RESET), "
  377. "rewriting as %llx\n", addr);
  378. ret = pci_write_config_dword(
  379. pdev, PCI_BASE_ADDRESS_0, addr);
  380. if (ret) {
  381. ipath_dev_err(dd, "rewrite of BAR0 "
  382. "failed: err %d\n", -ret);
  383. goto bail_disable;
  384. }
  385. ret = pci_write_config_dword(
  386. pdev, PCI_BASE_ADDRESS_1, addr >> 32);
  387. if (ret) {
  388. ipath_dev_err(dd, "rewrite of BAR1 "
  389. "failed: err %d\n", -ret);
  390. goto bail_disable;
  391. }
  392. } else {
  393. ipath_dev_err(dd, "BAR is 0 (probable RESET), "
  394. "not usable until reboot\n");
  395. ret = -ENODEV;
  396. goto bail_disable;
  397. }
  398. }
  399. ret = pci_request_regions(pdev, IPATH_DRV_NAME);
  400. if (ret) {
  401. dev_info(&pdev->dev, "pci_request_regions unit %u fails: "
  402. "err %d\n", dd->ipath_unit, -ret);
  403. goto bail_disable;
  404. }
  405. ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
  406. if (ret) {
  407. /*
  408. * if the 64 bit setup fails, try 32 bit. Some systems
  409. * do not setup 64 bit maps on systems with 2GB or less
  410. * memory installed.
  411. */
  412. ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
  413. if (ret) {
  414. dev_info(&pdev->dev,
  415. "Unable to set DMA mask for unit %u: %d\n",
  416. dd->ipath_unit, ret);
  417. goto bail_regions;
  418. }
  419. else {
  420. ipath_dbg("No 64bit DMA mask, used 32 bit mask\n");
  421. ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
  422. if (ret)
  423. dev_info(&pdev->dev,
  424. "Unable to set DMA consistent mask "
  425. "for unit %u: %d\n",
  426. dd->ipath_unit, ret);
  427. }
  428. }
  429. else {
  430. ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
  431. if (ret)
  432. dev_info(&pdev->dev,
  433. "Unable to set DMA consistent mask "
  434. "for unit %u: %d\n",
  435. dd->ipath_unit, ret);
  436. }
  437. pci_set_master(pdev);
  438. /*
  439. * Save BARs to rewrite after device reset. Save all 64 bits of
  440. * BAR, just in case.
  441. */
  442. dd->ipath_pcibar0 = addr;
  443. dd->ipath_pcibar1 = addr >> 32;
  444. dd->ipath_deviceid = ent->device; /* save for later use */
  445. dd->ipath_vendorid = ent->vendor;
  446. /* setup the chip-specific functions, as early as possible. */
  447. switch (ent->device) {
  448. case PCI_DEVICE_ID_INFINIPATH_HT:
  449. #ifdef CONFIG_HT_IRQ
  450. ipath_init_iba6110_funcs(dd);
  451. break;
  452. #else
  453. ipath_dev_err(dd, "QLogic HT device 0x%x cannot work if "
  454. "CONFIG_HT_IRQ is not enabled\n", ent->device);
  455. return -ENODEV;
  456. #endif
  457. case PCI_DEVICE_ID_INFINIPATH_PE800:
  458. #ifdef CONFIG_PCI_MSI
  459. ipath_init_iba6120_funcs(dd);
  460. break;
  461. #else
  462. ipath_dev_err(dd, "QLogic PCIE device 0x%x cannot work if "
  463. "CONFIG_PCI_MSI is not enabled\n", ent->device);
  464. return -ENODEV;
  465. #endif
  466. case PCI_DEVICE_ID_INFINIPATH_7220:
  467. #ifndef CONFIG_PCI_MSI
  468. ipath_dbg("CONFIG_PCI_MSI is not enabled, "
  469. "using INTx for unit %u\n", dd->ipath_unit);
  470. #endif
  471. ipath_init_iba7220_funcs(dd);
  472. break;
  473. default:
  474. ipath_dev_err(dd, "Found unknown QLogic deviceid 0x%x, "
  475. "failing\n", ent->device);
  476. return -ENODEV;
  477. }
  478. for (j = 0; j < 6; j++) {
  479. if (!pdev->resource[j].start)
  480. continue;
  481. ipath_cdbg(VERBOSE, "BAR %d start %llx, end %llx, len %llx\n",
  482. j, (unsigned long long)pdev->resource[j].start,
  483. (unsigned long long)pdev->resource[j].end,
  484. (unsigned long long)pci_resource_len(pdev, j));
  485. }
  486. if (!addr) {
  487. ipath_dev_err(dd, "No valid address in BAR 0!\n");
  488. ret = -ENODEV;
  489. goto bail_regions;
  490. }
  491. ret = pci_read_config_byte(pdev, PCI_REVISION_ID, &rev);
  492. if (ret) {
  493. ipath_dev_err(dd, "Failed to read PCI revision ID unit "
  494. "%u: err %d\n", dd->ipath_unit, -ret);
  495. goto bail_regions; /* shouldn't ever happen */
  496. }
  497. dd->ipath_pcirev = rev;
  498. #if defined(__powerpc__)
  499. /* There isn't a generic way to specify writethrough mappings */
  500. dd->ipath_kregbase = __ioremap(addr, len,
  501. (_PAGE_NO_CACHE|_PAGE_WRITETHRU));
  502. #else
  503. dd->ipath_kregbase = ioremap_nocache(addr, len);
  504. #endif
  505. if (!dd->ipath_kregbase) {
  506. ipath_dbg("Unable to map io addr %llx to kvirt, failing\n",
  507. addr);
  508. ret = -ENOMEM;
  509. goto bail_iounmap;
  510. }
  511. dd->ipath_kregend = (u64 __iomem *)
  512. ((void __iomem *)dd->ipath_kregbase + len);
  513. dd->ipath_physaddr = addr; /* used for io_remap, etc. */
  514. /* for user mmap */
  515. ipath_cdbg(VERBOSE, "mapped io addr %llx to kregbase %p\n",
  516. addr, dd->ipath_kregbase);
  517. if (dd->ipath_f_bus(dd, pdev))
  518. ipath_dev_err(dd, "Failed to setup config space; "
  519. "continuing anyway\n");
  520. /*
  521. * set up our interrupt handler; IRQF_SHARED probably not needed,
  522. * since MSI interrupts shouldn't be shared but won't hurt for now.
  523. * check 0 irq after we return from chip-specific bus setup, since
  524. * that can affect this due to setup
  525. */
  526. if (!dd->ipath_irq)
  527. ipath_dev_err(dd, "irq is 0, BIOS error? Interrupts won't "
  528. "work\n");
  529. else {
  530. ret = request_irq(dd->ipath_irq, ipath_intr, IRQF_SHARED,
  531. IPATH_DRV_NAME, dd);
  532. if (ret) {
  533. ipath_dev_err(dd, "Couldn't setup irq handler, "
  534. "irq=%d: %d\n", dd->ipath_irq, ret);
  535. goto bail_iounmap;
  536. }
  537. }
  538. ret = ipath_init_chip(dd, 0); /* do the chip-specific init */
  539. if (ret)
  540. goto bail_irqsetup;
  541. ret = ipath_enable_wc(dd);
  542. if (ret) {
  543. ipath_dev_err(dd, "Write combining not enabled "
  544. "(err %d): performance may be poor\n",
  545. -ret);
  546. ret = 0;
  547. }
  548. ipath_verify_pioperf(dd);
  549. ipath_device_create_group(&pdev->dev, dd);
  550. ipathfs_add_device(dd);
  551. ipath_user_add(dd);
  552. ipath_diag_add(dd);
  553. ipath_register_ib_device(dd);
  554. goto bail;
  555. bail_irqsetup:
  556. if (pdev->irq)
  557. free_irq(pdev->irq, dd);
  558. bail_iounmap:
  559. iounmap((volatile void __iomem *) dd->ipath_kregbase);
  560. bail_regions:
  561. pci_release_regions(pdev);
  562. bail_disable:
  563. pci_disable_device(pdev);
  564. bail_devdata:
  565. ipath_free_devdata(pdev, dd);
  566. bail:
  567. return ret;
  568. }
  569. static void __devexit cleanup_device(struct ipath_devdata *dd)
  570. {
  571. int port;
  572. struct ipath_portdata **tmp;
  573. unsigned long flags;
  574. if (*dd->ipath_statusp & IPATH_STATUS_CHIP_PRESENT) {
  575. /* can't do anything more with chip; needs re-init */
  576. *dd->ipath_statusp &= ~IPATH_STATUS_CHIP_PRESENT;
  577. if (dd->ipath_kregbase) {
  578. /*
  579. * if we haven't already cleaned up before these are
  580. * to ensure any register reads/writes "fail" until
  581. * re-init
  582. */
  583. dd->ipath_kregbase = NULL;
  584. dd->ipath_uregbase = 0;
  585. dd->ipath_sregbase = 0;
  586. dd->ipath_cregbase = 0;
  587. dd->ipath_kregsize = 0;
  588. }
  589. ipath_disable_wc(dd);
  590. }
  591. if (dd->ipath_spectriggerhit)
  592. dev_info(&dd->pcidev->dev, "%lu special trigger hits\n",
  593. dd->ipath_spectriggerhit);
  594. if (dd->ipath_pioavailregs_dma) {
  595. dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE,
  596. (void *) dd->ipath_pioavailregs_dma,
  597. dd->ipath_pioavailregs_phys);
  598. dd->ipath_pioavailregs_dma = NULL;
  599. }
  600. if (dd->ipath_dummy_hdrq) {
  601. dma_free_coherent(&dd->pcidev->dev,
  602. dd->ipath_pd[0]->port_rcvhdrq_size,
  603. dd->ipath_dummy_hdrq, dd->ipath_dummy_hdrq_phys);
  604. dd->ipath_dummy_hdrq = NULL;
  605. }
  606. if (dd->ipath_pageshadow) {
  607. struct page **tmpp = dd->ipath_pageshadow;
  608. dma_addr_t *tmpd = dd->ipath_physshadow;
  609. int i, cnt = 0;
  610. ipath_cdbg(VERBOSE, "Unlocking any expTID pages still "
  611. "locked\n");
  612. for (port = 0; port < dd->ipath_cfgports; port++) {
  613. int port_tidbase = port * dd->ipath_rcvtidcnt;
  614. int maxtid = port_tidbase + dd->ipath_rcvtidcnt;
  615. for (i = port_tidbase; i < maxtid; i++) {
  616. if (!tmpp[i])
  617. continue;
  618. pci_unmap_page(dd->pcidev, tmpd[i],
  619. PAGE_SIZE, PCI_DMA_FROMDEVICE);
  620. ipath_release_user_pages(&tmpp[i], 1);
  621. tmpp[i] = NULL;
  622. cnt++;
  623. }
  624. }
  625. if (cnt) {
  626. ipath_stats.sps_pageunlocks += cnt;
  627. ipath_cdbg(VERBOSE, "There were still %u expTID "
  628. "entries locked\n", cnt);
  629. }
  630. if (ipath_stats.sps_pagelocks ||
  631. ipath_stats.sps_pageunlocks)
  632. ipath_cdbg(VERBOSE, "%llu pages locked, %llu "
  633. "unlocked via ipath_m{un}lock\n",
  634. (unsigned long long)
  635. ipath_stats.sps_pagelocks,
  636. (unsigned long long)
  637. ipath_stats.sps_pageunlocks);
  638. ipath_cdbg(VERBOSE, "Free shadow page tid array at %p\n",
  639. dd->ipath_pageshadow);
  640. tmpp = dd->ipath_pageshadow;
  641. dd->ipath_pageshadow = NULL;
  642. vfree(tmpp);
  643. dd->ipath_egrtidbase = NULL;
  644. }
  645. /*
  646. * free any resources still in use (usually just kernel ports)
  647. * at unload; we do for portcnt, because that's what we allocate.
  648. * We acquire lock to be really paranoid that ipath_pd isn't being
  649. * accessed from some interrupt-related code (that should not happen,
  650. * but best to be sure).
  651. */
  652. spin_lock_irqsave(&dd->ipath_uctxt_lock, flags);
  653. tmp = dd->ipath_pd;
  654. dd->ipath_pd = NULL;
  655. spin_unlock_irqrestore(&dd->ipath_uctxt_lock, flags);
  656. for (port = 0; port < dd->ipath_portcnt; port++) {
  657. struct ipath_portdata *pd = tmp[port];
  658. tmp[port] = NULL; /* debugging paranoia */
  659. ipath_free_pddata(dd, pd);
  660. }
  661. kfree(tmp);
  662. }
  663. static void __devexit ipath_remove_one(struct pci_dev *pdev)
  664. {
  665. struct ipath_devdata *dd = pci_get_drvdata(pdev);
  666. ipath_cdbg(VERBOSE, "removing, pdev=%p, dd=%p\n", pdev, dd);
  667. /*
  668. * disable the IB link early, to be sure no new packets arrive, which
  669. * complicates the shutdown process
  670. */
  671. ipath_shutdown_device(dd);
  672. flush_scheduled_work();
  673. if (dd->verbs_dev)
  674. ipath_unregister_ib_device(dd->verbs_dev);
  675. ipath_diag_remove(dd);
  676. ipath_user_remove(dd);
  677. ipathfs_remove_device(dd);
  678. ipath_device_remove_group(&pdev->dev, dd);
  679. ipath_cdbg(VERBOSE, "Releasing pci memory regions, dd %p, "
  680. "unit %u\n", dd, (u32) dd->ipath_unit);
  681. cleanup_device(dd);
  682. /*
  683. * turn off rcv, send, and interrupts for all ports, all drivers
  684. * should also hard reset the chip here?
  685. * free up port 0 (kernel) rcvhdr, egr bufs, and eventually tid bufs
  686. * for all versions of the driver, if they were allocated
  687. */
  688. if (dd->ipath_irq) {
  689. ipath_cdbg(VERBOSE, "unit %u free irq %d\n",
  690. dd->ipath_unit, dd->ipath_irq);
  691. dd->ipath_f_free_irq(dd);
  692. } else
  693. ipath_dbg("irq is 0, not doing free_irq "
  694. "for unit %u\n", dd->ipath_unit);
  695. /*
  696. * we check for NULL here, because it's outside
  697. * the kregbase check, and we need to call it
  698. * after the free_irq. Thus it's possible that
  699. * the function pointers were never initialized.
  700. */
  701. if (dd->ipath_f_cleanup)
  702. /* clean up chip-specific stuff */
  703. dd->ipath_f_cleanup(dd);
  704. ipath_cdbg(VERBOSE, "Unmapping kregbase %p\n", dd->ipath_kregbase);
  705. iounmap((volatile void __iomem *) dd->ipath_kregbase);
  706. pci_release_regions(pdev);
  707. ipath_cdbg(VERBOSE, "calling pci_disable_device\n");
  708. pci_disable_device(pdev);
  709. ipath_free_devdata(pdev, dd);
  710. }
  711. /* general driver use */
  712. DEFINE_MUTEX(ipath_mutex);
  713. static DEFINE_SPINLOCK(ipath_pioavail_lock);
  714. /**
  715. * ipath_disarm_piobufs - cancel a range of PIO buffers
  716. * @dd: the infinipath device
  717. * @first: the first PIO buffer to cancel
  718. * @cnt: the number of PIO buffers to cancel
  719. *
  720. * cancel a range of PIO buffers, used when they might be armed, but
  721. * not triggered. Used at init to ensure buffer state, and also user
  722. * process close, in case it died while writing to a PIO buffer
  723. * Also after errors.
  724. */
  725. void ipath_disarm_piobufs(struct ipath_devdata *dd, unsigned first,
  726. unsigned cnt)
  727. {
  728. unsigned i, last = first + cnt;
  729. unsigned long flags;
  730. ipath_cdbg(PKT, "disarm %u PIObufs first=%u\n", cnt, first);
  731. for (i = first; i < last; i++) {
  732. spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
  733. /*
  734. * The disarm-related bits are write-only, so it
  735. * is ok to OR them in with our copy of sendctrl
  736. * while we hold the lock.
  737. */
  738. ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
  739. dd->ipath_sendctrl | INFINIPATH_S_DISARM |
  740. (i << INFINIPATH_S_DISARMPIOBUF_SHIFT));
  741. /* can't disarm bufs back-to-back per iba7220 spec */
  742. ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
  743. spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);
  744. }
  745. /* on some older chips, update may not happen after cancel */
  746. ipath_force_pio_avail_update(dd);
  747. }
  748. /**
  749. * ipath_wait_linkstate - wait for an IB link state change to occur
  750. * @dd: the infinipath device
  751. * @state: the state to wait for
  752. * @msecs: the number of milliseconds to wait
  753. *
  754. * wait up to msecs milliseconds for IB link state change to occur for
  755. * now, take the easy polling route. Currently used only by
  756. * ipath_set_linkstate. Returns 0 if state reached, otherwise
  757. * -ETIMEDOUT state can have multiple states set, for any of several
  758. * transitions.
  759. */
  760. int ipath_wait_linkstate(struct ipath_devdata *dd, u32 state, int msecs)
  761. {
  762. dd->ipath_state_wanted = state;
  763. wait_event_interruptible_timeout(ipath_state_wait,
  764. (dd->ipath_flags & state),
  765. msecs_to_jiffies(msecs));
  766. dd->ipath_state_wanted = 0;
  767. if (!(dd->ipath_flags & state)) {
  768. u64 val;
  769. ipath_cdbg(VERBOSE, "Didn't reach linkstate %s within %u"
  770. " ms\n",
  771. /* test INIT ahead of DOWN, both can be set */
  772. (state & IPATH_LINKINIT) ? "INIT" :
  773. ((state & IPATH_LINKDOWN) ? "DOWN" :
  774. ((state & IPATH_LINKARMED) ? "ARM" : "ACTIVE")),
  775. msecs);
  776. val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_ibcstatus);
  777. ipath_cdbg(VERBOSE, "ibcc=%llx ibcstatus=%llx (%s)\n",
  778. (unsigned long long) ipath_read_kreg64(
  779. dd, dd->ipath_kregs->kr_ibcctrl),
  780. (unsigned long long) val,
  781. ipath_ibcstatus_str[val & dd->ibcs_lts_mask]);
  782. }
  783. return (dd->ipath_flags & state) ? 0 : -ETIMEDOUT;
  784. }
  785. static void decode_sdma_errs(struct ipath_devdata *dd, ipath_err_t err,
  786. char *buf, size_t blen)
  787. {
  788. static const struct {
  789. ipath_err_t err;
  790. const char *msg;
  791. } errs[] = {
  792. { INFINIPATH_E_SDMAGENMISMATCH, "SDmaGenMismatch" },
  793. { INFINIPATH_E_SDMAOUTOFBOUND, "SDmaOutOfBound" },
  794. { INFINIPATH_E_SDMATAILOUTOFBOUND, "SDmaTailOutOfBound" },
  795. { INFINIPATH_E_SDMABASE, "SDmaBase" },
  796. { INFINIPATH_E_SDMA1STDESC, "SDma1stDesc" },
  797. { INFINIPATH_E_SDMARPYTAG, "SDmaRpyTag" },
  798. { INFINIPATH_E_SDMADWEN, "SDmaDwEn" },
  799. { INFINIPATH_E_SDMAMISSINGDW, "SDmaMissingDw" },
  800. { INFINIPATH_E_SDMAUNEXPDATA, "SDmaUnexpData" },
  801. { INFINIPATH_E_SDMADESCADDRMISALIGN, "SDmaDescAddrMisalign" },
  802. { INFINIPATH_E_SENDBUFMISUSE, "SendBufMisuse" },
  803. { INFINIPATH_E_SDMADISABLED, "SDmaDisabled" },
  804. };
  805. int i;
  806. int expected;
  807. size_t bidx = 0;
  808. for (i = 0; i < ARRAY_SIZE(errs); i++) {
  809. expected = (errs[i].err != INFINIPATH_E_SDMADISABLED) ? 0 :
  810. test_bit(IPATH_SDMA_ABORTING, &dd->ipath_sdma_status);
  811. if ((err & errs[i].err) && !expected)
  812. bidx += snprintf(buf + bidx, blen - bidx,
  813. "%s ", errs[i].msg);
  814. }
  815. }
  816. /*
  817. * Decode the error status into strings, deciding whether to always
  818. * print * it or not depending on "normal packet errors" vs everything
  819. * else. Return 1 if "real" errors, otherwise 0 if only packet
  820. * errors, so caller can decide what to print with the string.
  821. */
  822. int ipath_decode_err(struct ipath_devdata *dd, char *buf, size_t blen,
  823. ipath_err_t err)
  824. {
  825. int iserr = 1;
  826. *buf = '\0';
  827. if (err & INFINIPATH_E_PKTERRS) {
  828. if (!(err & ~INFINIPATH_E_PKTERRS))
  829. iserr = 0; // if only packet errors.
  830. if (ipath_debug & __IPATH_ERRPKTDBG) {
  831. if (err & INFINIPATH_E_REBP)
  832. strlcat(buf, "EBP ", blen);
  833. if (err & INFINIPATH_E_RVCRC)
  834. strlcat(buf, "VCRC ", blen);
  835. if (err & INFINIPATH_E_RICRC) {
  836. strlcat(buf, "CRC ", blen);
  837. // clear for check below, so only once
  838. err &= INFINIPATH_E_RICRC;
  839. }
  840. if (err & INFINIPATH_E_RSHORTPKTLEN)
  841. strlcat(buf, "rshortpktlen ", blen);
  842. if (err & INFINIPATH_E_SDROPPEDDATAPKT)
  843. strlcat(buf, "sdroppeddatapkt ", blen);
  844. if (err & INFINIPATH_E_SPKTLEN)
  845. strlcat(buf, "spktlen ", blen);
  846. }
  847. if ((err & INFINIPATH_E_RICRC) &&
  848. !(err&(INFINIPATH_E_RVCRC|INFINIPATH_E_REBP)))
  849. strlcat(buf, "CRC ", blen);
  850. if (!iserr)
  851. goto done;
  852. }
  853. if (err & INFINIPATH_E_RHDRLEN)
  854. strlcat(buf, "rhdrlen ", blen);
  855. if (err & INFINIPATH_E_RBADTID)
  856. strlcat(buf, "rbadtid ", blen);
  857. if (err & INFINIPATH_E_RBADVERSION)
  858. strlcat(buf, "rbadversion ", blen);
  859. if (err & INFINIPATH_E_RHDR)
  860. strlcat(buf, "rhdr ", blen);
  861. if (err & INFINIPATH_E_SENDSPECIALTRIGGER)
  862. strlcat(buf, "sendspecialtrigger ", blen);
  863. if (err & INFINIPATH_E_RLONGPKTLEN)
  864. strlcat(buf, "rlongpktlen ", blen);
  865. if (err & INFINIPATH_E_RMAXPKTLEN)
  866. strlcat(buf, "rmaxpktlen ", blen);
  867. if (err & INFINIPATH_E_RMINPKTLEN)
  868. strlcat(buf, "rminpktlen ", blen);
  869. if (err & INFINIPATH_E_SMINPKTLEN)
  870. strlcat(buf, "sminpktlen ", blen);
  871. if (err & INFINIPATH_E_RFORMATERR)
  872. strlcat(buf, "rformaterr ", blen);
  873. if (err & INFINIPATH_E_RUNSUPVL)
  874. strlcat(buf, "runsupvl ", blen);
  875. if (err & INFINIPATH_E_RUNEXPCHAR)
  876. strlcat(buf, "runexpchar ", blen);
  877. if (err & INFINIPATH_E_RIBFLOW)
  878. strlcat(buf, "ribflow ", blen);
  879. if (err & INFINIPATH_E_SUNDERRUN)
  880. strlcat(buf, "sunderrun ", blen);
  881. if (err & INFINIPATH_E_SPIOARMLAUNCH)
  882. strlcat(buf, "spioarmlaunch ", blen);
  883. if (err & INFINIPATH_E_SUNEXPERRPKTNUM)
  884. strlcat(buf, "sunexperrpktnum ", blen);
  885. if (err & INFINIPATH_E_SDROPPEDSMPPKT)
  886. strlcat(buf, "sdroppedsmppkt ", blen);
  887. if (err & INFINIPATH_E_SMAXPKTLEN)
  888. strlcat(buf, "smaxpktlen ", blen);
  889. if (err & INFINIPATH_E_SUNSUPVL)
  890. strlcat(buf, "sunsupVL ", blen);
  891. if (err & INFINIPATH_E_INVALIDADDR)
  892. strlcat(buf, "invalidaddr ", blen);
  893. if (err & INFINIPATH_E_RRCVEGRFULL)
  894. strlcat(buf, "rcvegrfull ", blen);
  895. if (err & INFINIPATH_E_RRCVHDRFULL)
  896. strlcat(buf, "rcvhdrfull ", blen);
  897. if (err & INFINIPATH_E_IBSTATUSCHANGED)
  898. strlcat(buf, "ibcstatuschg ", blen);
  899. if (err & INFINIPATH_E_RIBLOSTLINK)
  900. strlcat(buf, "riblostlink ", blen);
  901. if (err & INFINIPATH_E_HARDWARE)
  902. strlcat(buf, "hardware ", blen);
  903. if (err & INFINIPATH_E_RESET)
  904. strlcat(buf, "reset ", blen);
  905. if (err & INFINIPATH_E_SDMAERRS)
  906. decode_sdma_errs(dd, err, buf, blen);
  907. if (err & INFINIPATH_E_INVALIDEEPCMD)
  908. strlcat(buf, "invalideepromcmd ", blen);
  909. done:
  910. return iserr;
  911. }
  912. /**
  913. * get_rhf_errstring - decode RHF errors
  914. * @err: the err number
  915. * @msg: the output buffer
  916. * @len: the length of the output buffer
  917. *
  918. * only used one place now, may want more later
  919. */
  920. static void get_rhf_errstring(u32 err, char *msg, size_t len)
  921. {
  922. /* if no errors, and so don't need to check what's first */
  923. *msg = '\0';
  924. if (err & INFINIPATH_RHF_H_ICRCERR)
  925. strlcat(msg, "icrcerr ", len);
  926. if (err & INFINIPATH_RHF_H_VCRCERR)
  927. strlcat(msg, "vcrcerr ", len);
  928. if (err & INFINIPATH_RHF_H_PARITYERR)
  929. strlcat(msg, "parityerr ", len);
  930. if (err & INFINIPATH_RHF_H_LENERR)
  931. strlcat(msg, "lenerr ", len);
  932. if (err & INFINIPATH_RHF_H_MTUERR)
  933. strlcat(msg, "mtuerr ", len);
  934. if (err & INFINIPATH_RHF_H_IHDRERR)
  935. /* infinipath hdr checksum error */
  936. strlcat(msg, "ipathhdrerr ", len);
  937. if (err & INFINIPATH_RHF_H_TIDERR)
  938. strlcat(msg, "tiderr ", len);
  939. if (err & INFINIPATH_RHF_H_MKERR)
  940. /* bad port, offset, etc. */
  941. strlcat(msg, "invalid ipathhdr ", len);
  942. if (err & INFINIPATH_RHF_H_IBERR)
  943. strlcat(msg, "iberr ", len);
  944. if (err & INFINIPATH_RHF_L_SWA)
  945. strlcat(msg, "swA ", len);
  946. if (err & INFINIPATH_RHF_L_SWB)
  947. strlcat(msg, "swB ", len);
  948. }
  949. /**
  950. * ipath_get_egrbuf - get an eager buffer
  951. * @dd: the infinipath device
  952. * @bufnum: the eager buffer to get
  953. *
  954. * must only be called if ipath_pd[port] is known to be allocated
  955. */
  956. static inline void *ipath_get_egrbuf(struct ipath_devdata *dd, u32 bufnum)
  957. {
  958. return dd->ipath_port0_skbinfo ?
  959. (void *) dd->ipath_port0_skbinfo[bufnum].skb->data : NULL;
  960. }
  961. /**
  962. * ipath_alloc_skb - allocate an skb and buffer with possible constraints
  963. * @dd: the infinipath device
  964. * @gfp_mask: the sk_buff SFP mask
  965. */
  966. struct sk_buff *ipath_alloc_skb(struct ipath_devdata *dd,
  967. gfp_t gfp_mask)
  968. {
  969. struct sk_buff *skb;
  970. u32 len;
  971. /*
  972. * Only fully supported way to handle this is to allocate lots
  973. * extra, align as needed, and then do skb_reserve(). That wastes
  974. * a lot of memory... I'll have to hack this into infinipath_copy
  975. * also.
  976. */
  977. /*
  978. * We need 2 extra bytes for ipath_ether data sent in the
  979. * key header. In order to keep everything dword aligned,
  980. * we'll reserve 4 bytes.
  981. */
  982. len = dd->ipath_ibmaxlen + 4;
  983. if (dd->ipath_flags & IPATH_4BYTE_TID) {
  984. /* We need a 2KB multiple alignment, and there is no way
  985. * to do it except to allocate extra and then skb_reserve
  986. * enough to bring it up to the right alignment.
  987. */
  988. len += 2047;
  989. }
  990. skb = __dev_alloc_skb(len, gfp_mask);
  991. if (!skb) {
  992. ipath_dev_err(dd, "Failed to allocate skbuff, length %u\n",
  993. len);
  994. goto bail;
  995. }
  996. skb_reserve(skb, 4);
  997. if (dd->ipath_flags & IPATH_4BYTE_TID) {
  998. u32 una = (unsigned long)skb->data & 2047;
  999. if (una)
  1000. skb_reserve(skb, 2048 - una);
  1001. }
  1002. bail:
  1003. return skb;
  1004. }
  1005. static void ipath_rcv_hdrerr(struct ipath_devdata *dd,
  1006. u32 eflags,
  1007. u32 l,
  1008. u32 etail,
  1009. __le32 *rhf_addr,
  1010. struct ipath_message_header *hdr)
  1011. {
  1012. char emsg[128];
  1013. get_rhf_errstring(eflags, emsg, sizeof emsg);
  1014. ipath_cdbg(PKT, "RHFerrs %x hdrqtail=%x typ=%u "
  1015. "tlen=%x opcode=%x egridx=%x: %s\n",
  1016. eflags, l,
  1017. ipath_hdrget_rcv_type(rhf_addr),
  1018. ipath_hdrget_length_in_bytes(rhf_addr),
  1019. be32_to_cpu(hdr->bth[0]) >> 24,
  1020. etail, emsg);
  1021. /* Count local link integrity errors. */
  1022. if (eflags & (INFINIPATH_RHF_H_ICRCERR | INFINIPATH_RHF_H_VCRCERR)) {
  1023. u8 n = (dd->ipath_ibcctrl >>
  1024. INFINIPATH_IBCC_PHYERRTHRESHOLD_SHIFT) &
  1025. INFINIPATH_IBCC_PHYERRTHRESHOLD_MASK;
  1026. if (++dd->ipath_lli_counter > n) {
  1027. dd->ipath_lli_counter = 0;
  1028. dd->ipath_lli_errors++;
  1029. }
  1030. }
  1031. }
  1032. /*
  1033. * ipath_kreceive - receive a packet
  1034. * @pd: the infinipath port
  1035. *
  1036. * called from interrupt handler for errors or receive interrupt
  1037. */
  1038. void ipath_kreceive(struct ipath_portdata *pd)
  1039. {
  1040. struct ipath_devdata *dd = pd->port_dd;
  1041. __le32 *rhf_addr;
  1042. void *ebuf;
  1043. const u32 rsize = dd->ipath_rcvhdrentsize; /* words */
  1044. const u32 maxcnt = dd->ipath_rcvhdrcnt * rsize; /* words */
  1045. u32 etail = -1, l, hdrqtail;
  1046. struct ipath_message_header *hdr;
  1047. u32 eflags, i, etype, tlen, pkttot = 0, updegr = 0, reloop = 0;
  1048. static u64 totcalls; /* stats, may eventually remove */
  1049. int last;
  1050. l = pd->port_head;
  1051. rhf_addr = (__le32 *) pd->port_rcvhdrq + l + dd->ipath_rhf_offset;
  1052. if (dd->ipath_flags & IPATH_NODMA_RTAIL) {
  1053. u32 seq = ipath_hdrget_seq(rhf_addr);
  1054. if (seq != pd->port_seq_cnt)
  1055. goto bail;
  1056. hdrqtail = 0;
  1057. } else {
  1058. hdrqtail = ipath_get_rcvhdrtail(pd);
  1059. if (l == hdrqtail)
  1060. goto bail;
  1061. smp_rmb();
  1062. }
  1063. reloop:
  1064. for (last = 0, i = 1; !last; i += !last) {
  1065. hdr = dd->ipath_f_get_msgheader(dd, rhf_addr);
  1066. eflags = ipath_hdrget_err_flags(rhf_addr);
  1067. etype = ipath_hdrget_rcv_type(rhf_addr);
  1068. /* total length */
  1069. tlen = ipath_hdrget_length_in_bytes(rhf_addr);
  1070. ebuf = NULL;
  1071. if ((dd->ipath_flags & IPATH_NODMA_RTAIL) ?
  1072. ipath_hdrget_use_egr_buf(rhf_addr) :
  1073. (etype != RCVHQ_RCV_TYPE_EXPECTED)) {
  1074. /*
  1075. * It turns out that the chip uses an eager buffer
  1076. * for all non-expected packets, whether it "needs"
  1077. * one or not. So always get the index, but don't
  1078. * set ebuf (so we try to copy data) unless the
  1079. * length requires it.
  1080. */
  1081. etail = ipath_hdrget_index(rhf_addr);
  1082. updegr = 1;
  1083. if (tlen > sizeof(*hdr) ||
  1084. etype == RCVHQ_RCV_TYPE_NON_KD)
  1085. ebuf = ipath_get_egrbuf(dd, etail);
  1086. }
  1087. /*
  1088. * both tiderr and ipathhdrerr are set for all plain IB
  1089. * packets; only ipathhdrerr should be set.
  1090. */
  1091. if (etype != RCVHQ_RCV_TYPE_NON_KD &&
  1092. etype != RCVHQ_RCV_TYPE_ERROR &&
  1093. ipath_hdrget_ipath_ver(hdr->iph.ver_port_tid_offset) !=
  1094. IPS_PROTO_VERSION)
  1095. ipath_cdbg(PKT, "Bad InfiniPath protocol version "
  1096. "%x\n", etype);
  1097. if (unlikely(eflags))
  1098. ipath_rcv_hdrerr(dd, eflags, l, etail, rhf_addr, hdr);
  1099. else if (etype == RCVHQ_RCV_TYPE_NON_KD) {
  1100. ipath_ib_rcv(dd->verbs_dev, (u32 *)hdr, ebuf, tlen);
  1101. if (dd->ipath_lli_counter)
  1102. dd->ipath_lli_counter--;
  1103. } else if (etype == RCVHQ_RCV_TYPE_EAGER) {
  1104. u8 opcode = be32_to_cpu(hdr->bth[0]) >> 24;
  1105. u32 qp = be32_to_cpu(hdr->bth[1]) & 0xffffff;
  1106. ipath_cdbg(PKT, "typ %x, opcode %x (eager, "
  1107. "qp=%x), len %x; ignored\n",
  1108. etype, opcode, qp, tlen);
  1109. }
  1110. else if (etype == RCVHQ_RCV_TYPE_EXPECTED)
  1111. ipath_dbg("Bug: Expected TID, opcode %x; ignored\n",
  1112. be32_to_cpu(hdr->bth[0]) >> 24);
  1113. else {
  1114. /*
  1115. * error packet, type of error unknown.
  1116. * Probably type 3, but we don't know, so don't
  1117. * even try to print the opcode, etc.
  1118. * Usually caused by a "bad packet", that has no
  1119. * BTH, when the LRH says it should.
  1120. */
  1121. ipath_cdbg(ERRPKT, "Error Pkt, but no eflags! egrbuf"
  1122. " %x, len %x hdrq+%x rhf: %Lx\n",
  1123. etail, tlen, l, (unsigned long long)
  1124. le64_to_cpu(*(__le64 *) rhf_addr));
  1125. if (ipath_debug & __IPATH_ERRPKTDBG) {
  1126. u32 j, *d, dw = rsize-2;
  1127. if (rsize > (tlen>>2))
  1128. dw = tlen>>2;
  1129. d = (u32 *)hdr;
  1130. printk(KERN_DEBUG "EPkt rcvhdr(%x dw):\n",
  1131. dw);
  1132. for (j = 0; j < dw; j++)
  1133. printk(KERN_DEBUG "%8x%s", d[j],
  1134. (j%8) == 7 ? "\n" : " ");
  1135. printk(KERN_DEBUG ".\n");
  1136. }
  1137. }
  1138. l += rsize;
  1139. if (l >= maxcnt)
  1140. l = 0;
  1141. rhf_addr = (__le32 *) pd->port_rcvhdrq +
  1142. l + dd->ipath_rhf_offset;
  1143. if (dd->ipath_flags & IPATH_NODMA_RTAIL) {
  1144. u32 seq = ipath_hdrget_seq(rhf_addr);
  1145. if (++pd->port_seq_cnt > 13)
  1146. pd->port_seq_cnt = 1;
  1147. if (seq != pd->port_seq_cnt)
  1148. last = 1;
  1149. } else if (l == hdrqtail)
  1150. last = 1;
  1151. /*
  1152. * update head regs on last packet, and every 16 packets.
  1153. * Reduce bus traffic, while still trying to prevent
  1154. * rcvhdrq overflows, for when the queue is nearly full
  1155. */
  1156. if (last || !(i & 0xf)) {
  1157. u64 lval = l;
  1158. /* request IBA6120 and 7220 interrupt only on last */
  1159. if (last)
  1160. lval |= dd->ipath_rhdrhead_intr_off;
  1161. ipath_write_ureg(dd, ur_rcvhdrhead, lval,
  1162. pd->port_port);
  1163. if (updegr) {
  1164. ipath_write_ureg(dd, ur_rcvegrindexhead,
  1165. etail, pd->port_port);
  1166. updegr = 0;
  1167. }
  1168. }
  1169. }
  1170. if (!dd->ipath_rhdrhead_intr_off && !reloop &&
  1171. !(dd->ipath_flags & IPATH_NODMA_RTAIL)) {
  1172. /* IBA6110 workaround; we can have a race clearing chip
  1173. * interrupt with another interrupt about to be delivered,
  1174. * and can clear it before it is delivered on the GPIO
  1175. * workaround. By doing the extra check here for the
  1176. * in-memory tail register updating while we were doing
  1177. * earlier packets, we "almost" guarantee we have covered
  1178. * that case.
  1179. */
  1180. u32 hqtail = ipath_get_rcvhdrtail(pd);
  1181. if (hqtail != hdrqtail) {
  1182. hdrqtail = hqtail;
  1183. reloop = 1; /* loop 1 extra time at most */
  1184. goto reloop;
  1185. }
  1186. }
  1187. pkttot += i;
  1188. pd->port_head = l;
  1189. if (pkttot > ipath_stats.sps_maxpkts_call)
  1190. ipath_stats.sps_maxpkts_call = pkttot;
  1191. ipath_stats.sps_port0pkts += pkttot;
  1192. ipath_stats.sps_avgpkts_call =
  1193. ipath_stats.sps_port0pkts / ++totcalls;
  1194. bail:;
  1195. }
  1196. /**
  1197. * ipath_update_pio_bufs - update shadow copy of the PIO availability map
  1198. * @dd: the infinipath device
  1199. *
  1200. * called whenever our local copy indicates we have run out of send buffers
  1201. * NOTE: This can be called from interrupt context by some code
  1202. * and from non-interrupt context by ipath_getpiobuf().
  1203. */
  1204. static void ipath_update_pio_bufs(struct ipath_devdata *dd)
  1205. {
  1206. unsigned long flags;
  1207. int i;
  1208. const unsigned piobregs = (unsigned)dd->ipath_pioavregs;
  1209. /* If the generation (check) bits have changed, then we update the
  1210. * busy bit for the corresponding PIO buffer. This algorithm will
  1211. * modify positions to the value they already have in some cases
  1212. * (i.e., no change), but it's faster than changing only the bits
  1213. * that have changed.
  1214. *
  1215. * We would like to do this atomicly, to avoid spinlocks in the
  1216. * critical send path, but that's not really possible, given the
  1217. * type of changes, and that this routine could be called on
  1218. * multiple cpu's simultaneously, so we lock in this routine only,
  1219. * to avoid conflicting updates; all we change is the shadow, and
  1220. * it's a single 64 bit memory location, so by definition the update
  1221. * is atomic in terms of what other cpu's can see in testing the
  1222. * bits. The spin_lock overhead isn't too bad, since it only
  1223. * happens when all buffers are in use, so only cpu overhead, not
  1224. * latency or bandwidth is affected.
  1225. */
  1226. if (!dd->ipath_pioavailregs_dma) {
  1227. ipath_dbg("Update shadow pioavail, but regs_dma NULL!\n");
  1228. return;
  1229. }
  1230. if (ipath_debug & __IPATH_VERBDBG) {
  1231. /* only if packet debug and verbose */
  1232. volatile __le64 *dma = dd->ipath_pioavailregs_dma;
  1233. unsigned long *shadow = dd->ipath_pioavailshadow;
  1234. ipath_cdbg(PKT, "Refill avail, dma0=%llx shad0=%lx, "
  1235. "d1=%llx s1=%lx, d2=%llx s2=%lx, d3=%llx "
  1236. "s3=%lx\n",
  1237. (unsigned long long) le64_to_cpu(dma[0]),
  1238. shadow[0],
  1239. (unsigned long long) le64_to_cpu(dma[1]),
  1240. shadow[1],
  1241. (unsigned long long) le64_to_cpu(dma[2]),
  1242. shadow[2],
  1243. (unsigned long long) le64_to_cpu(dma[3]),
  1244. shadow[3]);
  1245. if (piobregs > 4)
  1246. ipath_cdbg(
  1247. PKT, "2nd group, dma4=%llx shad4=%lx, "
  1248. "d5=%llx s5=%lx, d6=%llx s6=%lx, "
  1249. "d7=%llx s7=%lx\n",
  1250. (unsigned long long) le64_to_cpu(dma[4]),
  1251. shadow[4],
  1252. (unsigned long long) le64_to_cpu(dma[5]),
  1253. shadow[5],
  1254. (unsigned long long) le64_to_cpu(dma[6]),
  1255. shadow[6],
  1256. (unsigned long long) le64_to_cpu(dma[7]),
  1257. shadow[7]);
  1258. }
  1259. spin_lock_irqsave(&ipath_pioavail_lock, flags);
  1260. for (i = 0; i < piobregs; i++) {
  1261. u64 pchbusy, pchg, piov, pnew;
  1262. /*
  1263. * Chip Errata: bug 6641; even and odd qwords>3 are swapped
  1264. */
  1265. if (i > 3 && (dd->ipath_flags & IPATH_SWAP_PIOBUFS))
  1266. piov = le64_to_cpu(dd->ipath_pioavailregs_dma[i ^ 1]);
  1267. else
  1268. piov = le64_to_cpu(dd->ipath_pioavailregs_dma[i]);
  1269. pchg = dd->ipath_pioavailkernel[i] &
  1270. ~(dd->ipath_pioavailshadow[i] ^ piov);
  1271. pchbusy = pchg << INFINIPATH_SENDPIOAVAIL_BUSY_SHIFT;
  1272. if (pchg && (pchbusy & dd->ipath_pioavailshadow[i])) {
  1273. pnew = dd->ipath_pioavailshadow[i] & ~pchbusy;
  1274. pnew |= piov & pchbusy;
  1275. dd->ipath_pioavailshadow[i] = pnew;
  1276. }
  1277. }
  1278. spin_unlock_irqrestore(&ipath_pioavail_lock, flags);
  1279. }
  1280. /*
  1281. * used to force update of pioavailshadow if we can't get a pio buffer.
  1282. * Needed primarily due to exitting freeze mode after recovering
  1283. * from errors. Done lazily, because it's safer (known to not
  1284. * be writing pio buffers).
  1285. */
  1286. static void ipath_reset_availshadow(struct ipath_devdata *dd)
  1287. {
  1288. int i, im;
  1289. unsigned long flags;
  1290. spin_lock_irqsave(&ipath_pioavail_lock, flags);
  1291. for (i = 0; i < dd->ipath_pioavregs; i++) {
  1292. u64 val, oldval;
  1293. /* deal with 6110 chip bug on high register #s */
  1294. im = (i > 3 && (dd->ipath_flags & IPATH_SWAP_PIOBUFS)) ?
  1295. i ^ 1 : i;
  1296. val = le64_to_cpu(dd->ipath_pioavailregs_dma[im]);
  1297. /*
  1298. * busy out the buffers not in the kernel avail list,
  1299. * without changing the generation bits.
  1300. */
  1301. oldval = dd->ipath_pioavailshadow[i];
  1302. dd->ipath_pioavailshadow[i] = val |
  1303. ((~dd->ipath_pioavailkernel[i] <<
  1304. INFINIPATH_SENDPIOAVAIL_BUSY_SHIFT) &
  1305. 0xaaaaaaaaaaaaaaaaULL); /* All BUSY bits in qword */
  1306. if (oldval != dd->ipath_pioavailshadow[i])
  1307. ipath_dbg("shadow[%d] was %Lx, now %lx\n",
  1308. i, (unsigned long long) oldval,
  1309. dd->ipath_pioavailshadow[i]);
  1310. }
  1311. spin_unlock_irqrestore(&ipath_pioavail_lock, flags);
  1312. }
  1313. /**
  1314. * ipath_setrcvhdrsize - set the receive header size
  1315. * @dd: the infinipath device
  1316. * @rhdrsize: the receive header size
  1317. *
  1318. * called from user init code, and also layered driver init
  1319. */
  1320. int ipath_setrcvhdrsize(struct ipath_devdata *dd, unsigned rhdrsize)
  1321. {
  1322. int ret = 0;
  1323. if (dd->ipath_flags & IPATH_RCVHDRSZ_SET) {
  1324. if (dd->ipath_rcvhdrsize != rhdrsize) {
  1325. dev_info(&dd->pcidev->dev,
  1326. "Error: can't set protocol header "
  1327. "size %u, already %u\n",
  1328. rhdrsize, dd->ipath_rcvhdrsize);
  1329. ret = -EAGAIN;
  1330. } else
  1331. ipath_cdbg(VERBOSE, "Reuse same protocol header "
  1332. "size %u\n", dd->ipath_rcvhdrsize);
  1333. } else if (rhdrsize > (dd->ipath_rcvhdrentsize -
  1334. (sizeof(u64) / sizeof(u32)))) {
  1335. ipath_dbg("Error: can't set protocol header size %u "
  1336. "(> max %u)\n", rhdrsize,
  1337. dd->ipath_rcvhdrentsize -
  1338. (u32) (sizeof(u64) / sizeof(u32)));
  1339. ret = -EOVERFLOW;
  1340. } else {
  1341. dd->ipath_flags |= IPATH_RCVHDRSZ_SET;
  1342. dd->ipath_rcvhdrsize = rhdrsize;
  1343. ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvhdrsize,
  1344. dd->ipath_rcvhdrsize);
  1345. ipath_cdbg(VERBOSE, "Set protocol header size to %u\n",
  1346. dd->ipath_rcvhdrsize);
  1347. }
  1348. return ret;
  1349. }
  1350. /*
  1351. * debugging code and stats updates if no pio buffers available.
  1352. */
  1353. static noinline void no_pio_bufs(struct ipath_devdata *dd)
  1354. {
  1355. unsigned long *shadow = dd->ipath_pioavailshadow;
  1356. __le64 *dma = (__le64 *)dd->ipath_pioavailregs_dma;
  1357. dd->ipath_upd_pio_shadow = 1;
  1358. /*
  1359. * not atomic, but if we lose a stat count in a while, that's OK
  1360. */
  1361. ipath_stats.sps_nopiobufs++;
  1362. if (!(++dd->ipath_consec_nopiobuf % 100000)) {
  1363. ipath_force_pio_avail_update(dd); /* at start */
  1364. ipath_dbg("%u tries no piobufavail ts%lx; dmacopy: "
  1365. "%llx %llx %llx %llx\n"
  1366. "ipath shadow: %lx %lx %lx %lx\n",
  1367. dd->ipath_consec_nopiobuf,
  1368. (unsigned long)get_cycles(),
  1369. (unsigned long long) le64_to_cpu(dma[0]),
  1370. (unsigned long long) le64_to_cpu(dma[1]),
  1371. (unsigned long long) le64_to_cpu(dma[2]),
  1372. (unsigned long long) le64_to_cpu(dma[3]),
  1373. shadow[0], shadow[1], shadow[2], shadow[3]);
  1374. /*
  1375. * 4 buffers per byte, 4 registers above, cover rest
  1376. * below
  1377. */
  1378. if ((dd->ipath_piobcnt2k + dd->ipath_piobcnt4k) >
  1379. (sizeof(shadow[0]) * 4 * 4))
  1380. ipath_dbg("2nd group: dmacopy: "
  1381. "%llx %llx %llx %llx\n"
  1382. "ipath shadow: %lx %lx %lx %lx\n",
  1383. (unsigned long long)le64_to_cpu(dma[4]),
  1384. (unsigned long long)le64_to_cpu(dma[5]),
  1385. (unsigned long long)le64_to_cpu(dma[6]),
  1386. (unsigned long long)le64_to_cpu(dma[7]),
  1387. shadow[4], shadow[5], shadow[6], shadow[7]);
  1388. /* at end, so update likely happened */
  1389. ipath_reset_availshadow(dd);
  1390. }
  1391. }
  1392. /*
  1393. * common code for normal driver pio buffer allocation, and reserved
  1394. * allocation.
  1395. *
  1396. * do appropriate marking as busy, etc.
  1397. * returns buffer number if one found (>=0), negative number is error.
  1398. */
  1399. static u32 __iomem *ipath_getpiobuf_range(struct ipath_devdata *dd,
  1400. u32 *pbufnum, u32 first, u32 last, u32 firsti)
  1401. {
  1402. int i, j, updated = 0;
  1403. unsigned piobcnt;
  1404. unsigned long flags;
  1405. unsigned long *shadow = dd->ipath_pioavailshadow;
  1406. u32 __iomem *buf;
  1407. piobcnt = last - first;
  1408. if (dd->ipath_upd_pio_shadow) {
  1409. /*
  1410. * Minor optimization. If we had no buffers on last call,
  1411. * start out by doing the update; continue and do scan even
  1412. * if no buffers were updated, to be paranoid
  1413. */
  1414. ipath_update_pio_bufs(dd);
  1415. updated++;
  1416. i = first;
  1417. } else
  1418. i = firsti;
  1419. rescan:
  1420. /*
  1421. * while test_and_set_bit() is atomic, we do that and then the
  1422. * change_bit(), and the pair is not. See if this is the cause
  1423. * of the remaining armlaunch errors.
  1424. */
  1425. spin_lock_irqsave(&ipath_pioavail_lock, flags);
  1426. for (j = 0; j < piobcnt; j++, i++) {
  1427. if (i >= last)
  1428. i = first;
  1429. if (__test_and_set_bit((2 * i) + 1, shadow))
  1430. continue;
  1431. /* flip generation bit */
  1432. __change_bit(2 * i, shadow);
  1433. break;
  1434. }
  1435. spin_unlock_irqrestore(&ipath_pioavail_lock, flags);
  1436. if (j == piobcnt) {
  1437. if (!updated) {
  1438. /*
  1439. * first time through; shadow exhausted, but may be
  1440. * buffers available, try an update and then rescan.
  1441. */
  1442. ipath_update_pio_bufs(dd);
  1443. updated++;
  1444. i = first;
  1445. goto rescan;
  1446. } else if (updated == 1 && piobcnt <=
  1447. ((dd->ipath_sendctrl
  1448. >> INFINIPATH_S_UPDTHRESH_SHIFT) &
  1449. INFINIPATH_S_UPDTHRESH_MASK)) {
  1450. /*
  1451. * for chips supporting and using the update
  1452. * threshold we need to force an update of the
  1453. * in-memory copy if the count is less than the
  1454. * thershold, then check one more time.
  1455. */
  1456. ipath_force_pio_avail_update(dd);
  1457. ipath_update_pio_bufs(dd);
  1458. updated++;
  1459. i = first;
  1460. goto rescan;
  1461. }
  1462. no_pio_bufs(dd);
  1463. buf = NULL;
  1464. } else {
  1465. if (i < dd->ipath_piobcnt2k)
  1466. buf = (u32 __iomem *) (dd->ipath_pio2kbase +
  1467. i * dd->ipath_palign);
  1468. else
  1469. buf = (u32 __iomem *)
  1470. (dd->ipath_pio4kbase +
  1471. (i - dd->ipath_piobcnt2k) * dd->ipath_4kalign);
  1472. if (pbufnum)
  1473. *pbufnum = i;
  1474. }
  1475. return buf;
  1476. }
  1477. /**
  1478. * ipath_getpiobuf - find an available pio buffer
  1479. * @dd: the infinipath device
  1480. * @plen: the size of the PIO buffer needed in 32-bit words
  1481. * @pbufnum: the buffer number is placed here
  1482. */
  1483. u32 __iomem *ipath_getpiobuf(struct ipath_devdata *dd, u32 plen, u32 *pbufnum)
  1484. {
  1485. u32 __iomem *buf;
  1486. u32 pnum, nbufs;
  1487. u32 first, lasti;
  1488. if (plen + 1 >= IPATH_SMALLBUF_DWORDS) {
  1489. first = dd->ipath_piobcnt2k;
  1490. lasti = dd->ipath_lastpioindexl;
  1491. } else {
  1492. first = 0;
  1493. lasti = dd->ipath_lastpioindex;
  1494. }
  1495. nbufs = dd->ipath_piobcnt2k + dd->ipath_piobcnt4k;
  1496. buf = ipath_getpiobuf_range(dd, &pnum, first, nbufs, lasti);
  1497. if (buf) {
  1498. /*
  1499. * Set next starting place. It's just an optimization,
  1500. * it doesn't matter who wins on this, so no locking
  1501. */
  1502. if (plen + 1 >= IPATH_SMALLBUF_DWORDS)
  1503. dd->ipath_lastpioindexl = pnum + 1;
  1504. else
  1505. dd->ipath_lastpioindex = pnum + 1;
  1506. if (dd->ipath_upd_pio_shadow)
  1507. dd->ipath_upd_pio_shadow = 0;
  1508. if (dd->ipath_consec_nopiobuf)
  1509. dd->ipath_consec_nopiobuf = 0;
  1510. ipath_cdbg(VERBOSE, "Return piobuf%u %uk @ %p\n",
  1511. pnum, (pnum < dd->ipath_piobcnt2k) ? 2 : 4, buf);
  1512. if (pbufnum)
  1513. *pbufnum = pnum;
  1514. }
  1515. return buf;
  1516. }
  1517. /**
  1518. * ipath_chg_pioavailkernel - change which send buffers are available for kernel
  1519. * @dd: the infinipath device
  1520. * @start: the starting send buffer number
  1521. * @len: the number of send buffers
  1522. * @avail: true if the buffers are available for kernel use, false otherwise
  1523. */
  1524. void ipath_chg_pioavailkernel(struct ipath_devdata *dd, unsigned start,
  1525. unsigned len, int avail)
  1526. {
  1527. unsigned long flags;
  1528. unsigned end, cnt = 0;
  1529. /* There are two bits per send buffer (busy and generation) */
  1530. start *= 2;
  1531. end = start + len * 2;
  1532. spin_lock_irqsave(&ipath_pioavail_lock, flags);
  1533. /* Set or clear the busy bit in the shadow. */
  1534. while (start < end) {
  1535. if (avail) {
  1536. unsigned long dma;
  1537. int i, im;
  1538. /*
  1539. * the BUSY bit will never be set, because we disarm
  1540. * the user buffers before we hand them back to the
  1541. * kernel. We do have to make sure the generation
  1542. * bit is set correctly in shadow, since it could
  1543. * have changed many times while allocated to user.
  1544. * We can't use the bitmap functions on the full
  1545. * dma array because it is always little-endian, so
  1546. * we have to flip to host-order first.
  1547. * BITS_PER_LONG is slightly wrong, since it's
  1548. * always 64 bits per register in chip...
  1549. * We only work on 64 bit kernels, so that's OK.
  1550. */
  1551. /* deal with 6110 chip bug on high register #s */
  1552. i = start / BITS_PER_LONG;
  1553. im = (i > 3 && (dd->ipath_flags & IPATH_SWAP_PIOBUFS)) ?
  1554. i ^ 1 : i;
  1555. __clear_bit(INFINIPATH_SENDPIOAVAIL_BUSY_SHIFT
  1556. + start, dd->ipath_pioavailshadow);
  1557. dma = (unsigned long) le64_to_cpu(
  1558. dd->ipath_pioavailregs_dma[im]);
  1559. if (test_bit((INFINIPATH_SENDPIOAVAIL_CHECK_SHIFT
  1560. + start) % BITS_PER_LONG, &dma))
  1561. __set_bit(INFINIPATH_SENDPIOAVAIL_CHECK_SHIFT
  1562. + start, dd->ipath_pioavailshadow);
  1563. else
  1564. __clear_bit(INFINIPATH_SENDPIOAVAIL_CHECK_SHIFT
  1565. + start, dd->ipath_pioavailshadow);
  1566. __set_bit(start, dd->ipath_pioavailkernel);
  1567. } else {
  1568. __set_bit(start + INFINIPATH_SENDPIOAVAIL_BUSY_SHIFT,
  1569. dd->ipath_pioavailshadow);
  1570. __clear_bit(start, dd->ipath_pioavailkernel);
  1571. }
  1572. start += 2;
  1573. }
  1574. if (dd->ipath_pioupd_thresh) {
  1575. end = 2 * (dd->ipath_piobcnt2k + dd->ipath_piobcnt4k);
  1576. cnt = bitmap_weight(dd->ipath_pioavailkernel, end);
  1577. }
  1578. spin_unlock_irqrestore(&ipath_pioavail_lock, flags);
  1579. /*
  1580. * When moving buffers from kernel to user, if number assigned to
  1581. * the user is less than the pio update threshold, and threshold
  1582. * is supported (cnt was computed > 0), drop the update threshold
  1583. * so we update at least once per allocated number of buffers.
  1584. * In any case, if the kernel buffers are less than the threshold,
  1585. * drop the threshold. We don't bother increasing it, having once
  1586. * decreased it, since it would typically just cycle back and forth.
  1587. * If we don't decrease below buffers in use, we can wait a long
  1588. * time for an update, until some other context uses PIO buffers.
  1589. */
  1590. if (!avail && len < cnt)
  1591. cnt = len;
  1592. if (cnt < dd->ipath_pioupd_thresh) {
  1593. dd->ipath_pioupd_thresh = cnt;
  1594. ipath_dbg("Decreased pio update threshold to %u\n",
  1595. dd->ipath_pioupd_thresh);
  1596. spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
  1597. dd->ipath_sendctrl &= ~(INFINIPATH_S_UPDTHRESH_MASK
  1598. << INFINIPATH_S_UPDTHRESH_SHIFT);
  1599. dd->ipath_sendctrl |= dd->ipath_pioupd_thresh
  1600. << INFINIPATH_S_UPDTHRESH_SHIFT;
  1601. ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
  1602. dd->ipath_sendctrl);
  1603. spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);
  1604. }
  1605. }
  1606. /**
  1607. * ipath_create_rcvhdrq - create a receive header queue
  1608. * @dd: the infinipath device
  1609. * @pd: the port data
  1610. *
  1611. * this must be contiguous memory (from an i/o perspective), and must be
  1612. * DMA'able (which means for some systems, it will go through an IOMMU,
  1613. * or be forced into a low address range).
  1614. */
  1615. int ipath_create_rcvhdrq(struct ipath_devdata *dd,
  1616. struct ipath_portdata *pd)
  1617. {
  1618. int ret = 0;
  1619. if (!pd->port_rcvhdrq) {
  1620. dma_addr_t phys_hdrqtail;
  1621. gfp_t gfp_flags = GFP_USER | __GFP_COMP;
  1622. int amt = ALIGN(dd->ipath_rcvhdrcnt * dd->ipath_rcvhdrentsize *
  1623. sizeof(u32), PAGE_SIZE);
  1624. pd->port_rcvhdrq = dma_alloc_coherent(
  1625. &dd->pcidev->dev, amt, &pd->port_rcvhdrq_phys,
  1626. gfp_flags);
  1627. if (!pd->port_rcvhdrq) {
  1628. ipath_dev_err(dd, "attempt to allocate %d bytes "
  1629. "for port %u rcvhdrq failed\n",
  1630. amt, pd->port_port);
  1631. ret = -ENOMEM;
  1632. goto bail;
  1633. }
  1634. if (!(dd->ipath_flags & IPATH_NODMA_RTAIL)) {
  1635. pd->port_rcvhdrtail_kvaddr = dma_alloc_coherent(
  1636. &dd->pcidev->dev, PAGE_SIZE, &phys_hdrqtail,
  1637. GFP_KERNEL);
  1638. if (!pd->port_rcvhdrtail_kvaddr) {
  1639. ipath_dev_err(dd, "attempt to allocate 1 page "
  1640. "for port %u rcvhdrqtailaddr "
  1641. "failed\n", pd->port_port);
  1642. ret = -ENOMEM;
  1643. dma_free_coherent(&dd->pcidev->dev, amt,
  1644. pd->port_rcvhdrq,
  1645. pd->port_rcvhdrq_phys);
  1646. pd->port_rcvhdrq = NULL;
  1647. goto bail;
  1648. }
  1649. pd->port_rcvhdrqtailaddr_phys = phys_hdrqtail;
  1650. ipath_cdbg(VERBOSE, "port %d hdrtailaddr, %llx "
  1651. "physical\n", pd->port_port,
  1652. (unsigned long long) phys_hdrqtail);
  1653. }
  1654. pd->port_rcvhdrq_size = amt;
  1655. ipath_cdbg(VERBOSE, "%d pages at %p (phys %lx) size=%lu "
  1656. "for port %u rcvhdr Q\n",
  1657. amt >> PAGE_SHIFT, pd->port_rcvhdrq,
  1658. (unsigned long) pd->port_rcvhdrq_phys,
  1659. (unsigned long) pd->port_rcvhdrq_size,
  1660. pd->port_port);
  1661. }
  1662. else
  1663. ipath_cdbg(VERBOSE, "reuse port %d rcvhdrq @%p %llx phys; "
  1664. "hdrtailaddr@%p %llx physical\n",
  1665. pd->port_port, pd->port_rcvhdrq,
  1666. (unsigned long long) pd->port_rcvhdrq_phys,
  1667. pd->port_rcvhdrtail_kvaddr, (unsigned long long)
  1668. pd->port_rcvhdrqtailaddr_phys);
  1669. /* clear for security and sanity on each use */
  1670. memset(pd->port_rcvhdrq, 0, pd->port_rcvhdrq_size);
  1671. if (pd->port_rcvhdrtail_kvaddr)
  1672. memset(pd->port_rcvhdrtail_kvaddr, 0, PAGE_SIZE);
  1673. /*
  1674. * tell chip each time we init it, even if we are re-using previous
  1675. * memory (we zero the register at process close)
  1676. */
  1677. ipath_write_kreg_port(dd, dd->ipath_kregs->kr_rcvhdrtailaddr,
  1678. pd->port_port, pd->port_rcvhdrqtailaddr_phys);
  1679. ipath_write_kreg_port(dd, dd->ipath_kregs->kr_rcvhdraddr,
  1680. pd->port_port, pd->port_rcvhdrq_phys);
  1681. bail:
  1682. return ret;
  1683. }
  1684. /*
  1685. * Flush all sends that might be in the ready to send state, as well as any
  1686. * that are in the process of being sent. Used whenever we need to be
  1687. * sure the send side is idle. Cleans up all buffer state by canceling
  1688. * all pio buffers, and issuing an abort, which cleans up anything in the
  1689. * launch fifo. The cancel is superfluous on some chip versions, but
  1690. * it's safer to always do it.
  1691. * PIOAvail bits are updated by the chip as if normal send had happened.
  1692. */
  1693. void ipath_cancel_sends(struct ipath_devdata *dd, int restore_sendctrl)
  1694. {
  1695. unsigned long flags;
  1696. if (dd->ipath_flags & IPATH_IB_AUTONEG_INPROG) {
  1697. ipath_cdbg(VERBOSE, "Ignore while in autonegotiation\n");
  1698. goto bail;
  1699. }
  1700. /*
  1701. * If we have SDMA, and it's not disabled, we have to kick off the
  1702. * abort state machine, provided we aren't already aborting.
  1703. * If we are in the process of aborting SDMA (!DISABLED, but ABORTING),
  1704. * we skip the rest of this routine. It is already "in progress"
  1705. */
  1706. if (dd->ipath_flags & IPATH_HAS_SEND_DMA) {
  1707. int skip_cancel;
  1708. unsigned long *statp = &dd->ipath_sdma_status;
  1709. spin_lock_irqsave(&dd->ipath_sdma_lock, flags);
  1710. skip_cancel =
  1711. test_and_set_bit(IPATH_SDMA_ABORTING, statp)
  1712. && !test_bit(IPATH_SDMA_DISABLED, statp);
  1713. spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
  1714. if (skip_cancel)
  1715. goto bail;
  1716. }
  1717. ipath_dbg("Cancelling all in-progress send buffers\n");
  1718. /* skip armlaunch errs for a while */
  1719. dd->ipath_lastcancel = jiffies + HZ / 2;
  1720. /*
  1721. * The abort bit is auto-clearing. We also don't want pioavail
  1722. * update happening during this, and we don't want any other
  1723. * sends going out, so turn those off for the duration. We read
  1724. * the scratch register to be sure that cancels and the abort
  1725. * have taken effect in the chip. Otherwise two parts are same
  1726. * as ipath_force_pio_avail_update()
  1727. */
  1728. spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
  1729. dd->ipath_sendctrl &= ~(INFINIPATH_S_PIOBUFAVAILUPD
  1730. | INFINIPATH_S_PIOENABLE);
  1731. ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
  1732. dd->ipath_sendctrl | INFINIPATH_S_ABORT);
  1733. ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
  1734. spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);
  1735. /* disarm all send buffers */
  1736. ipath_disarm_piobufs(dd, 0,
  1737. dd->ipath_piobcnt2k + dd->ipath_piobcnt4k);
  1738. if (dd->ipath_flags & IPATH_HAS_SEND_DMA)
  1739. set_bit(IPATH_SDMA_DISARMED, &dd->ipath_sdma_status);
  1740. if (restore_sendctrl) {
  1741. /* else done by caller later if needed */
  1742. spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
  1743. dd->ipath_sendctrl |= INFINIPATH_S_PIOBUFAVAILUPD |
  1744. INFINIPATH_S_PIOENABLE;
  1745. ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
  1746. dd->ipath_sendctrl);
  1747. /* and again, be sure all have hit the chip */
  1748. ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
  1749. spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);
  1750. }
  1751. if ((dd->ipath_flags & IPATH_HAS_SEND_DMA) &&
  1752. !test_bit(IPATH_SDMA_DISABLED, &dd->ipath_sdma_status) &&
  1753. test_bit(IPATH_SDMA_RUNNING, &dd->ipath_sdma_status)) {
  1754. spin_lock_irqsave(&dd->ipath_sdma_lock, flags);
  1755. /* only wait so long for intr */
  1756. dd->ipath_sdma_abort_intr_timeout = jiffies + HZ;
  1757. dd->ipath_sdma_reset_wait = 200;
  1758. if (!test_bit(IPATH_SDMA_SHUTDOWN, &dd->ipath_sdma_status))
  1759. tasklet_hi_schedule(&dd->ipath_sdma_abort_task);
  1760. spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
  1761. }
  1762. bail:;
  1763. }
  1764. /*
  1765. * Force an update of in-memory copy of the pioavail registers, when
  1766. * needed for any of a variety of reasons. We read the scratch register
  1767. * to make it highly likely that the update will have happened by the
  1768. * time we return. If already off (as in cancel_sends above), this
  1769. * routine is a nop, on the assumption that the caller will "do the
  1770. * right thing".
  1771. */
  1772. void ipath_force_pio_avail_update(struct ipath_devdata *dd)
  1773. {
  1774. unsigned long flags;
  1775. spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
  1776. if (dd->ipath_sendctrl & INFINIPATH_S_PIOBUFAVAILUPD) {
  1777. ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
  1778. dd->ipath_sendctrl & ~INFINIPATH_S_PIOBUFAVAILUPD);
  1779. ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
  1780. ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
  1781. dd->ipath_sendctrl);
  1782. ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
  1783. }
  1784. spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);
  1785. }
  1786. static void ipath_set_ib_lstate(struct ipath_devdata *dd, int linkcmd,
  1787. int linitcmd)
  1788. {
  1789. u64 mod_wd;
  1790. static const char *what[4] = {
  1791. [0] = "NOP",
  1792. [INFINIPATH_IBCC_LINKCMD_DOWN] = "DOWN",
  1793. [INFINIPATH_IBCC_LINKCMD_ARMED] = "ARMED",
  1794. [INFINIPATH_IBCC_LINKCMD_ACTIVE] = "ACTIVE"
  1795. };
  1796. if (linitcmd == INFINIPATH_IBCC_LINKINITCMD_DISABLE) {
  1797. /*
  1798. * If we are told to disable, note that so link-recovery
  1799. * code does not attempt to bring us back up.
  1800. */
  1801. preempt_disable();
  1802. dd->ipath_flags |= IPATH_IB_LINK_DISABLED;
  1803. preempt_enable();
  1804. } else if (linitcmd) {
  1805. /*
  1806. * Any other linkinitcmd will lead to LINKDOWN and then
  1807. * to INIT (if all is well), so clear flag to let
  1808. * link-recovery code attempt to bring us back up.
  1809. */
  1810. preempt_disable();
  1811. dd->ipath_flags &= ~IPATH_IB_LINK_DISABLED;
  1812. preempt_enable();
  1813. }
  1814. mod_wd = (linkcmd << dd->ibcc_lc_shift) |
  1815. (linitcmd << INFINIPATH_IBCC_LINKINITCMD_SHIFT);
  1816. ipath_cdbg(VERBOSE,
  1817. "Moving unit %u to %s (initcmd=0x%x), current ltstate is %s\n",
  1818. dd->ipath_unit, what[linkcmd], linitcmd,
  1819. ipath_ibcstatus_str[ipath_ib_linktrstate(dd,
  1820. ipath_read_kreg64(dd, dd->ipath_kregs->kr_ibcstatus))]);
  1821. ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcctrl,
  1822. dd->ipath_ibcctrl | mod_wd);
  1823. /* read from chip so write is flushed */
  1824. (void) ipath_read_kreg64(dd, dd->ipath_kregs->kr_ibcstatus);
  1825. }
  1826. int ipath_set_linkstate(struct ipath_devdata *dd, u8 newstate)
  1827. {
  1828. u32 lstate;
  1829. int ret;
  1830. switch (newstate) {
  1831. case IPATH_IB_LINKDOWN_ONLY:
  1832. ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_DOWN, 0);
  1833. /* don't wait */
  1834. ret = 0;
  1835. goto bail;
  1836. case IPATH_IB_LINKDOWN:
  1837. ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_DOWN,
  1838. INFINIPATH_IBCC_LINKINITCMD_POLL);
  1839. /* don't wait */
  1840. ret = 0;
  1841. goto bail;
  1842. case IPATH_IB_LINKDOWN_SLEEP:
  1843. ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_DOWN,
  1844. INFINIPATH_IBCC_LINKINITCMD_SLEEP);
  1845. /* don't wait */
  1846. ret = 0;
  1847. goto bail;
  1848. case IPATH_IB_LINKDOWN_DISABLE:
  1849. ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_DOWN,
  1850. INFINIPATH_IBCC_LINKINITCMD_DISABLE);
  1851. /* don't wait */
  1852. ret = 0;
  1853. goto bail;
  1854. case IPATH_IB_LINKARM:
  1855. if (dd->ipath_flags & IPATH_LINKARMED) {
  1856. ret = 0;
  1857. goto bail;
  1858. }
  1859. if (!(dd->ipath_flags &
  1860. (IPATH_LINKINIT | IPATH_LINKACTIVE))) {
  1861. ret = -EINVAL;
  1862. goto bail;
  1863. }
  1864. ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_ARMED, 0);
  1865. /*
  1866. * Since the port can transition to ACTIVE by receiving
  1867. * a non VL 15 packet, wait for either state.
  1868. */
  1869. lstate = IPATH_LINKARMED | IPATH_LINKACTIVE;
  1870. break;
  1871. case IPATH_IB_LINKACTIVE:
  1872. if (dd->ipath_flags & IPATH_LINKACTIVE) {
  1873. ret = 0;
  1874. goto bail;
  1875. }
  1876. if (!(dd->ipath_flags & IPATH_LINKARMED)) {
  1877. ret = -EINVAL;
  1878. goto bail;
  1879. }
  1880. ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_ACTIVE, 0);
  1881. lstate = IPATH_LINKACTIVE;
  1882. break;
  1883. case IPATH_IB_LINK_LOOPBACK:
  1884. dev_info(&dd->pcidev->dev, "Enabling IB local loopback\n");
  1885. dd->ipath_ibcctrl |= INFINIPATH_IBCC_LOOPBACK;
  1886. ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcctrl,
  1887. dd->ipath_ibcctrl);
  1888. /* turn heartbeat off, as it causes loopback to fail */
  1889. dd->ipath_f_set_ib_cfg(dd, IPATH_IB_CFG_HRTBT,
  1890. IPATH_IB_HRTBT_OFF);
  1891. /* don't wait */
  1892. ret = 0;
  1893. goto bail;
  1894. case IPATH_IB_LINK_EXTERNAL:
  1895. dev_info(&dd->pcidev->dev,
  1896. "Disabling IB local loopback (normal)\n");
  1897. dd->ipath_f_set_ib_cfg(dd, IPATH_IB_CFG_HRTBT,
  1898. IPATH_IB_HRTBT_ON);
  1899. dd->ipath_ibcctrl &= ~INFINIPATH_IBCC_LOOPBACK;
  1900. ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcctrl,
  1901. dd->ipath_ibcctrl);
  1902. /* don't wait */
  1903. ret = 0;
  1904. goto bail;
  1905. /*
  1906. * Heartbeat can be explicitly enabled by the user via
  1907. * "hrtbt_enable" "file", and if disabled, trying to enable here
  1908. * will have no effect. Implicit changes (heartbeat off when
  1909. * loopback on, and vice versa) are included to ease testing.
  1910. */
  1911. case IPATH_IB_LINK_HRTBT:
  1912. ret = dd->ipath_f_set_ib_cfg(dd, IPATH_IB_CFG_HRTBT,
  1913. IPATH_IB_HRTBT_ON);
  1914. goto bail;
  1915. case IPATH_IB_LINK_NO_HRTBT:
  1916. ret = dd->ipath_f_set_ib_cfg(dd, IPATH_IB_CFG_HRTBT,
  1917. IPATH_IB_HRTBT_OFF);
  1918. goto bail;
  1919. default:
  1920. ipath_dbg("Invalid linkstate 0x%x requested\n", newstate);
  1921. ret = -EINVAL;
  1922. goto bail;
  1923. }
  1924. ret = ipath_wait_linkstate(dd, lstate, 2000);
  1925. bail:
  1926. return ret;
  1927. }
  1928. /**
  1929. * ipath_set_mtu - set the MTU
  1930. * @dd: the infinipath device
  1931. * @arg: the new MTU
  1932. *
  1933. * we can handle "any" incoming size, the issue here is whether we
  1934. * need to restrict our outgoing size. For now, we don't do any
  1935. * sanity checking on this, and we don't deal with what happens to
  1936. * programs that are already running when the size changes.
  1937. * NOTE: changing the MTU will usually cause the IBC to go back to
  1938. * link INIT state...
  1939. */
  1940. int ipath_set_mtu(struct ipath_devdata *dd, u16 arg)
  1941. {
  1942. u32 piosize;
  1943. int changed = 0;
  1944. int ret;
  1945. /*
  1946. * mtu is IB data payload max. It's the largest power of 2 less
  1947. * than piosize (or even larger, since it only really controls the
  1948. * largest we can receive; we can send the max of the mtu and
  1949. * piosize). We check that it's one of the valid IB sizes.
  1950. */
  1951. if (arg != 256 && arg != 512 && arg != 1024 && arg != 2048 &&
  1952. (arg != 4096 || !ipath_mtu4096)) {
  1953. ipath_dbg("Trying to set invalid mtu %u, failing\n", arg);
  1954. ret = -EINVAL;
  1955. goto bail;
  1956. }
  1957. if (dd->ipath_ibmtu == arg) {
  1958. ret = 0; /* same as current */
  1959. goto bail;
  1960. }
  1961. piosize = dd->ipath_ibmaxlen;
  1962. dd->ipath_ibmtu = arg;
  1963. if (arg >= (piosize - IPATH_PIO_MAXIBHDR)) {
  1964. /* Only if it's not the initial value (or reset to it) */
  1965. if (piosize != dd->ipath_init_ibmaxlen) {
  1966. if (arg > piosize && arg <= dd->ipath_init_ibmaxlen)
  1967. piosize = dd->ipath_init_ibmaxlen;
  1968. dd->ipath_ibmaxlen = piosize;
  1969. changed = 1;
  1970. }
  1971. } else if ((arg + IPATH_PIO_MAXIBHDR) != dd->ipath_ibmaxlen) {
  1972. piosize = arg + IPATH_PIO_MAXIBHDR;
  1973. ipath_cdbg(VERBOSE, "ibmaxlen was 0x%x, setting to 0x%x "
  1974. "(mtu 0x%x)\n", dd->ipath_ibmaxlen, piosize,
  1975. arg);
  1976. dd->ipath_ibmaxlen = piosize;
  1977. changed = 1;
  1978. }
  1979. if (changed) {
  1980. u64 ibc = dd->ipath_ibcctrl, ibdw;
  1981. /*
  1982. * update our housekeeping variables, and set IBC max
  1983. * size, same as init code; max IBC is max we allow in
  1984. * buffer, less the qword pbc, plus 1 for ICRC, in dwords
  1985. */
  1986. dd->ipath_ibmaxlen = piosize - 2 * sizeof(u32);
  1987. ibdw = (dd->ipath_ibmaxlen >> 2) + 1;
  1988. ibc &= ~(INFINIPATH_IBCC_MAXPKTLEN_MASK <<
  1989. dd->ibcc_mpl_shift);
  1990. ibc |= ibdw << dd->ibcc_mpl_shift;
  1991. dd->ipath_ibcctrl = ibc;
  1992. ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcctrl,
  1993. dd->ipath_ibcctrl);
  1994. dd->ipath_f_tidtemplate(dd);
  1995. }
  1996. ret = 0;
  1997. bail:
  1998. return ret;
  1999. }
  2000. int ipath_set_lid(struct ipath_devdata *dd, u32 lid, u8 lmc)
  2001. {
  2002. dd->ipath_lid = lid;
  2003. dd->ipath_lmc = lmc;
  2004. dd->ipath_f_set_ib_cfg(dd, IPATH_IB_CFG_LIDLMC, lid |
  2005. (~((1U << lmc) - 1)) << 16);
  2006. dev_info(&dd->pcidev->dev, "We got a lid: 0x%x\n", lid);
  2007. return 0;
  2008. }
  2009. /**
  2010. * ipath_write_kreg_port - write a device's per-port 64-bit kernel register
  2011. * @dd: the infinipath device
  2012. * @regno: the register number to write
  2013. * @port: the port containing the register
  2014. * @value: the value to write
  2015. *
  2016. * Registers that vary with the chip implementation constants (port)
  2017. * use this routine.
  2018. */
  2019. void ipath_write_kreg_port(const struct ipath_devdata *dd, ipath_kreg regno,
  2020. unsigned port, u64 value)
  2021. {
  2022. u16 where;
  2023. if (port < dd->ipath_portcnt &&
  2024. (regno == dd->ipath_kregs->kr_rcvhdraddr ||
  2025. regno == dd->ipath_kregs->kr_rcvhdrtailaddr))
  2026. where = regno + port;
  2027. else
  2028. where = -1;
  2029. ipath_write_kreg(dd, where, value);
  2030. }
  2031. /*
  2032. * Following deal with the "obviously simple" task of overriding the state
  2033. * of the LEDS, which normally indicate link physical and logical status.
  2034. * The complications arise in dealing with different hardware mappings
  2035. * and the board-dependent routine being called from interrupts.
  2036. * and then there's the requirement to _flash_ them.
  2037. */
  2038. #define LED_OVER_FREQ_SHIFT 8
  2039. #define LED_OVER_FREQ_MASK (0xFF<<LED_OVER_FREQ_SHIFT)
  2040. /* Below is "non-zero" to force override, but both actual LEDs are off */
  2041. #define LED_OVER_BOTH_OFF (8)
  2042. static void ipath_run_led_override(unsigned long opaque)
  2043. {
  2044. struct ipath_devdata *dd = (struct ipath_devdata *)opaque;
  2045. int timeoff;
  2046. int pidx;
  2047. u64 lstate, ltstate, val;
  2048. if (!(dd->ipath_flags & IPATH_INITTED))
  2049. return;
  2050. pidx = dd->ipath_led_override_phase++ & 1;
  2051. dd->ipath_led_override = dd->ipath_led_override_vals[pidx];
  2052. timeoff = dd->ipath_led_override_timeoff;
  2053. /*
  2054. * below potentially restores the LED values per current status,
  2055. * should also possibly setup the traffic-blink register,
  2056. * but leave that to per-chip functions.
  2057. */
  2058. val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_ibcstatus);
  2059. ltstate = ipath_ib_linktrstate(dd, val);
  2060. lstate = ipath_ib_linkstate(dd, val);
  2061. dd->ipath_f_setextled(dd, lstate, ltstate);
  2062. mod_timer(&dd->ipath_led_override_timer, jiffies + timeoff);
  2063. }
  2064. void ipath_set_led_override(struct ipath_devdata *dd, unsigned int val)
  2065. {
  2066. int timeoff, freq;
  2067. if (!(dd->ipath_flags & IPATH_INITTED))
  2068. return;
  2069. /* First check if we are blinking. If not, use 1HZ polling */
  2070. timeoff = HZ;
  2071. freq = (val & LED_OVER_FREQ_MASK) >> LED_OVER_FREQ_SHIFT;
  2072. if (freq) {
  2073. /* For blink, set each phase from one nybble of val */
  2074. dd->ipath_led_override_vals[0] = val & 0xF;
  2075. dd->ipath_led_override_vals[1] = (val >> 4) & 0xF;
  2076. timeoff = (HZ << 4)/freq;
  2077. } else {
  2078. /* Non-blink set both phases the same. */
  2079. dd->ipath_led_override_vals[0] = val & 0xF;
  2080. dd->ipath_led_override_vals[1] = val & 0xF;
  2081. }
  2082. dd->ipath_led_override_timeoff = timeoff;
  2083. /*
  2084. * If the timer has not already been started, do so. Use a "quick"
  2085. * timeout so the function will be called soon, to look at our request.
  2086. */
  2087. if (atomic_inc_return(&dd->ipath_led_override_timer_active) == 1) {
  2088. /* Need to start timer */
  2089. init_timer(&dd->ipath_led_override_timer);
  2090. dd->ipath_led_override_timer.function =
  2091. ipath_run_led_override;
  2092. dd->ipath_led_override_timer.data = (unsigned long) dd;
  2093. dd->ipath_led_override_timer.expires = jiffies + 1;
  2094. add_timer(&dd->ipath_led_override_timer);
  2095. } else
  2096. atomic_dec(&dd->ipath_led_override_timer_active);
  2097. }
  2098. /**
  2099. * ipath_shutdown_device - shut down a device
  2100. * @dd: the infinipath device
  2101. *
  2102. * This is called to make the device quiet when we are about to
  2103. * unload the driver, and also when the device is administratively
  2104. * disabled. It does not free any data structures.
  2105. * Everything it does has to be setup again by ipath_init_chip(dd,1)
  2106. */
  2107. void ipath_shutdown_device(struct ipath_devdata *dd)
  2108. {
  2109. unsigned long flags;
  2110. ipath_dbg("Shutting down the device\n");
  2111. ipath_hol_up(dd); /* make sure user processes aren't suspended */
  2112. dd->ipath_flags |= IPATH_LINKUNK;
  2113. dd->ipath_flags &= ~(IPATH_INITTED | IPATH_LINKDOWN |
  2114. IPATH_LINKINIT | IPATH_LINKARMED |
  2115. IPATH_LINKACTIVE);
  2116. *dd->ipath_statusp &= ~(IPATH_STATUS_IB_CONF |
  2117. IPATH_STATUS_IB_READY);
  2118. /* mask interrupts, but not errors */
  2119. ipath_write_kreg(dd, dd->ipath_kregs->kr_intmask, 0ULL);
  2120. dd->ipath_rcvctrl = 0;
  2121. ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvctrl,
  2122. dd->ipath_rcvctrl);
  2123. if (dd->ipath_flags & IPATH_HAS_SEND_DMA)
  2124. teardown_sdma(dd);
  2125. /*
  2126. * gracefully stop all sends allowing any in progress to trickle out
  2127. * first.
  2128. */
  2129. spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
  2130. dd->ipath_sendctrl = 0;
  2131. ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl, dd->ipath_sendctrl);
  2132. /* flush it */
  2133. ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
  2134. spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);
  2135. /*
  2136. * enough for anything that's going to trickle out to have actually
  2137. * done so.
  2138. */
  2139. udelay(5);
  2140. dd->ipath_f_setextled(dd, 0, 0); /* make sure LEDs are off */
  2141. ipath_set_ib_lstate(dd, 0, INFINIPATH_IBCC_LINKINITCMD_DISABLE);
  2142. ipath_cancel_sends(dd, 0);
  2143. /*
  2144. * we are shutting down, so tell components that care. We don't do
  2145. * this on just a link state change, much like ethernet, a cable
  2146. * unplug, etc. doesn't change driver state
  2147. */
  2148. signal_ib_event(dd, IB_EVENT_PORT_ERR);
  2149. /* disable IBC */
  2150. dd->ipath_control &= ~INFINIPATH_C_LINKENABLE;
  2151. ipath_write_kreg(dd, dd->ipath_kregs->kr_control,
  2152. dd->ipath_control | INFINIPATH_C_FREEZEMODE);
  2153. /*
  2154. * clear SerdesEnable and turn the leds off; do this here because
  2155. * we are unloading, so don't count on interrupts to move along
  2156. * Turn the LEDs off explictly for the same reason.
  2157. */
  2158. dd->ipath_f_quiet_serdes(dd);
  2159. /* stop all the timers that might still be running */
  2160. del_timer_sync(&dd->ipath_hol_timer);
  2161. if (dd->ipath_stats_timer_active) {
  2162. del_timer_sync(&dd->ipath_stats_timer);
  2163. dd->ipath_stats_timer_active = 0;
  2164. }
  2165. if (dd->ipath_intrchk_timer.data) {
  2166. del_timer_sync(&dd->ipath_intrchk_timer);
  2167. dd->ipath_intrchk_timer.data = 0;
  2168. }
  2169. if (atomic_read(&dd->ipath_led_override_timer_active)) {
  2170. del_timer_sync(&dd->ipath_led_override_timer);
  2171. atomic_set(&dd->ipath_led_override_timer_active, 0);
  2172. }
  2173. /*
  2174. * clear all interrupts and errors, so that the next time the driver
  2175. * is loaded or device is enabled, we know that whatever is set
  2176. * happened while we were unloaded
  2177. */
  2178. ipath_write_kreg(dd, dd->ipath_kregs->kr_hwerrclear,
  2179. ~0ULL & ~INFINIPATH_HWE_MEMBISTFAILED);
  2180. ipath_write_kreg(dd, dd->ipath_kregs->kr_errorclear, -1LL);
  2181. ipath_write_kreg(dd, dd->ipath_kregs->kr_intclear, -1LL);
  2182. ipath_cdbg(VERBOSE, "Flush time and errors to EEPROM\n");
  2183. ipath_update_eeprom_log(dd);
  2184. }
  2185. /**
  2186. * ipath_free_pddata - free a port's allocated data
  2187. * @dd: the infinipath device
  2188. * @pd: the portdata structure
  2189. *
  2190. * free up any allocated data for a port
  2191. * This should not touch anything that would affect a simultaneous
  2192. * re-allocation of port data, because it is called after ipath_mutex
  2193. * is released (and can be called from reinit as well).
  2194. * It should never change any chip state, or global driver state.
  2195. * (The only exception to global state is freeing the port0 port0_skbs.)
  2196. */
  2197. void ipath_free_pddata(struct ipath_devdata *dd, struct ipath_portdata *pd)
  2198. {
  2199. if (!pd)
  2200. return;
  2201. if (pd->port_rcvhdrq) {
  2202. ipath_cdbg(VERBOSE, "free closed port %d rcvhdrq @ %p "
  2203. "(size=%lu)\n", pd->port_port, pd->port_rcvhdrq,
  2204. (unsigned long) pd->port_rcvhdrq_size);
  2205. dma_free_coherent(&dd->pcidev->dev, pd->port_rcvhdrq_size,
  2206. pd->port_rcvhdrq, pd->port_rcvhdrq_phys);
  2207. pd->port_rcvhdrq = NULL;
  2208. if (pd->port_rcvhdrtail_kvaddr) {
  2209. dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE,
  2210. pd->port_rcvhdrtail_kvaddr,
  2211. pd->port_rcvhdrqtailaddr_phys);
  2212. pd->port_rcvhdrtail_kvaddr = NULL;
  2213. }
  2214. }
  2215. if (pd->port_port && pd->port_rcvegrbuf) {
  2216. unsigned e;
  2217. for (e = 0; e < pd->port_rcvegrbuf_chunks; e++) {
  2218. void *base = pd->port_rcvegrbuf[e];
  2219. size_t size = pd->port_rcvegrbuf_size;
  2220. ipath_cdbg(VERBOSE, "egrbuf free(%p, %lu), "
  2221. "chunk %u/%u\n", base,
  2222. (unsigned long) size,
  2223. e, pd->port_rcvegrbuf_chunks);
  2224. dma_free_coherent(&dd->pcidev->dev, size,
  2225. base, pd->port_rcvegrbuf_phys[e]);
  2226. }
  2227. kfree(pd->port_rcvegrbuf);
  2228. pd->port_rcvegrbuf = NULL;
  2229. kfree(pd->port_rcvegrbuf_phys);
  2230. pd->port_rcvegrbuf_phys = NULL;
  2231. pd->port_rcvegrbuf_chunks = 0;
  2232. } else if (pd->port_port == 0 && dd->ipath_port0_skbinfo) {
  2233. unsigned e;
  2234. struct ipath_skbinfo *skbinfo = dd->ipath_port0_skbinfo;
  2235. dd->ipath_port0_skbinfo = NULL;
  2236. ipath_cdbg(VERBOSE, "free closed port %d "
  2237. "ipath_port0_skbinfo @ %p\n", pd->port_port,
  2238. skbinfo);
  2239. for (e = 0; e < dd->ipath_p0_rcvegrcnt; e++)
  2240. if (skbinfo[e].skb) {
  2241. pci_unmap_single(dd->pcidev, skbinfo[e].phys,
  2242. dd->ipath_ibmaxlen,
  2243. PCI_DMA_FROMDEVICE);
  2244. dev_kfree_skb(skbinfo[e].skb);
  2245. }
  2246. vfree(skbinfo);
  2247. }
  2248. kfree(pd->port_tid_pg_list);
  2249. vfree(pd->subport_uregbase);
  2250. vfree(pd->subport_rcvegrbuf);
  2251. vfree(pd->subport_rcvhdr_base);
  2252. kfree(pd);
  2253. }
  2254. static int __init infinipath_init(void)
  2255. {
  2256. int ret;
  2257. if (ipath_debug & __IPATH_DBG)
  2258. printk(KERN_INFO DRIVER_LOAD_MSG "%s", ib_ipath_version);
  2259. /*
  2260. * These must be called before the driver is registered with
  2261. * the PCI subsystem.
  2262. */
  2263. idr_init(&unit_table);
  2264. if (!idr_pre_get(&unit_table, GFP_KERNEL)) {
  2265. printk(KERN_ERR IPATH_DRV_NAME ": idr_pre_get() failed\n");
  2266. ret = -ENOMEM;
  2267. goto bail;
  2268. }
  2269. ret = pci_register_driver(&ipath_driver);
  2270. if (ret < 0) {
  2271. printk(KERN_ERR IPATH_DRV_NAME
  2272. ": Unable to register driver: error %d\n", -ret);
  2273. goto bail_unit;
  2274. }
  2275. ret = ipath_init_ipathfs();
  2276. if (ret < 0) {
  2277. printk(KERN_ERR IPATH_DRV_NAME ": Unable to create "
  2278. "ipathfs: error %d\n", -ret);
  2279. goto bail_pci;
  2280. }
  2281. goto bail;
  2282. bail_pci:
  2283. pci_unregister_driver(&ipath_driver);
  2284. bail_unit:
  2285. idr_destroy(&unit_table);
  2286. bail:
  2287. return ret;
  2288. }
  2289. static void __exit infinipath_cleanup(void)
  2290. {
  2291. ipath_exit_ipathfs();
  2292. ipath_cdbg(VERBOSE, "Unregistering pci driver\n");
  2293. pci_unregister_driver(&ipath_driver);
  2294. idr_destroy(&unit_table);
  2295. }
  2296. /**
  2297. * ipath_reset_device - reset the chip if possible
  2298. * @unit: the device to reset
  2299. *
  2300. * Whether or not reset is successful, we attempt to re-initialize the chip
  2301. * (that is, much like a driver unload/reload). We clear the INITTED flag
  2302. * so that the various entry points will fail until we reinitialize. For
  2303. * now, we only allow this if no user ports are open that use chip resources
  2304. */
  2305. int ipath_reset_device(int unit)
  2306. {
  2307. int ret, i;
  2308. struct ipath_devdata *dd = ipath_lookup(unit);
  2309. unsigned long flags;
  2310. if (!dd) {
  2311. ret = -ENODEV;
  2312. goto bail;
  2313. }
  2314. if (atomic_read(&dd->ipath_led_override_timer_active)) {
  2315. /* Need to stop LED timer, _then_ shut off LEDs */
  2316. del_timer_sync(&dd->ipath_led_override_timer);
  2317. atomic_set(&dd->ipath_led_override_timer_active, 0);
  2318. }
  2319. /* Shut off LEDs after we are sure timer is not running */
  2320. dd->ipath_led_override = LED_OVER_BOTH_OFF;
  2321. dd->ipath_f_setextled(dd, 0, 0);
  2322. dev_info(&dd->pcidev->dev, "Reset on unit %u requested\n", unit);
  2323. if (!dd->ipath_kregbase || !(dd->ipath_flags & IPATH_PRESENT)) {
  2324. dev_info(&dd->pcidev->dev, "Invalid unit number %u or "
  2325. "not initialized or not present\n", unit);
  2326. ret = -ENXIO;
  2327. goto bail;
  2328. }
  2329. spin_lock_irqsave(&dd->ipath_uctxt_lock, flags);
  2330. if (dd->ipath_pd)
  2331. for (i = 1; i < dd->ipath_cfgports; i++) {
  2332. if (!dd->ipath_pd[i] || !dd->ipath_pd[i]->port_cnt)
  2333. continue;
  2334. spin_unlock_irqrestore(&dd->ipath_uctxt_lock, flags);
  2335. ipath_dbg("unit %u port %d is in use "
  2336. "(PID %u cmd %s), can't reset\n",
  2337. unit, i,
  2338. pid_nr(dd->ipath_pd[i]->port_pid),
  2339. dd->ipath_pd[i]->port_comm);
  2340. ret = -EBUSY;
  2341. goto bail;
  2342. }
  2343. spin_unlock_irqrestore(&dd->ipath_uctxt_lock, flags);
  2344. if (dd->ipath_flags & IPATH_HAS_SEND_DMA)
  2345. teardown_sdma(dd);
  2346. dd->ipath_flags &= ~IPATH_INITTED;
  2347. ipath_write_kreg(dd, dd->ipath_kregs->kr_intmask, 0ULL);
  2348. ret = dd->ipath_f_reset(dd);
  2349. if (ret == 1) {
  2350. ipath_dbg("Reinitializing unit %u after reset attempt\n",
  2351. unit);
  2352. ret = ipath_init_chip(dd, 1);
  2353. } else
  2354. ret = -EAGAIN;
  2355. if (ret)
  2356. ipath_dev_err(dd, "Reinitialize unit %u after "
  2357. "reset failed with %d\n", unit, ret);
  2358. else
  2359. dev_info(&dd->pcidev->dev, "Reinitialized unit %u after "
  2360. "resetting\n", unit);
  2361. bail:
  2362. return ret;
  2363. }
  2364. /*
  2365. * send a signal to all the processes that have the driver open
  2366. * through the normal interfaces (i.e., everything other than diags
  2367. * interface). Returns number of signalled processes.
  2368. */
  2369. static int ipath_signal_procs(struct ipath_devdata *dd, int sig)
  2370. {
  2371. int i, sub, any = 0;
  2372. struct pid *pid;
  2373. unsigned long flags;
  2374. if (!dd->ipath_pd)
  2375. return 0;
  2376. spin_lock_irqsave(&dd->ipath_uctxt_lock, flags);
  2377. for (i = 1; i < dd->ipath_cfgports; i++) {
  2378. if (!dd->ipath_pd[i] || !dd->ipath_pd[i]->port_cnt)
  2379. continue;
  2380. pid = dd->ipath_pd[i]->port_pid;
  2381. if (!pid)
  2382. continue;
  2383. dev_info(&dd->pcidev->dev, "context %d in use "
  2384. "(PID %u), sending signal %d\n",
  2385. i, pid_nr(pid), sig);
  2386. kill_pid(pid, sig, 1);
  2387. any++;
  2388. for (sub = 0; sub < INFINIPATH_MAX_SUBPORT; sub++) {
  2389. pid = dd->ipath_pd[i]->port_subpid[sub];
  2390. if (!pid)
  2391. continue;
  2392. dev_info(&dd->pcidev->dev, "sub-context "
  2393. "%d:%d in use (PID %u), sending "
  2394. "signal %d\n", i, sub, pid_nr(pid), sig);
  2395. kill_pid(pid, sig, 1);
  2396. any++;
  2397. }
  2398. }
  2399. spin_unlock_irqrestore(&dd->ipath_uctxt_lock, flags);
  2400. return any;
  2401. }
  2402. static void ipath_hol_signal_down(struct ipath_devdata *dd)
  2403. {
  2404. if (ipath_signal_procs(dd, SIGSTOP))
  2405. ipath_dbg("Stopped some processes\n");
  2406. ipath_cancel_sends(dd, 1);
  2407. }
  2408. static void ipath_hol_signal_up(struct ipath_devdata *dd)
  2409. {
  2410. if (ipath_signal_procs(dd, SIGCONT))
  2411. ipath_dbg("Continued some processes\n");
  2412. }
  2413. /*
  2414. * link is down, stop any users processes, and flush pending sends
  2415. * to prevent HoL blocking, then start the HoL timer that
  2416. * periodically continues, then stop procs, so they can detect
  2417. * link down if they want, and do something about it.
  2418. * Timer may already be running, so use mod_timer, not add_timer.
  2419. */
  2420. void ipath_hol_down(struct ipath_devdata *dd)
  2421. {
  2422. dd->ipath_hol_state = IPATH_HOL_DOWN;
  2423. ipath_hol_signal_down(dd);
  2424. dd->ipath_hol_next = IPATH_HOL_DOWNCONT;
  2425. dd->ipath_hol_timer.expires = jiffies +
  2426. msecs_to_jiffies(ipath_hol_timeout_ms);
  2427. mod_timer(&dd->ipath_hol_timer, dd->ipath_hol_timer.expires);
  2428. }
  2429. /*
  2430. * link is up, continue any user processes, and ensure timer
  2431. * is a nop, if running. Let timer keep running, if set; it
  2432. * will nop when it sees the link is up
  2433. */
  2434. void ipath_hol_up(struct ipath_devdata *dd)
  2435. {
  2436. ipath_hol_signal_up(dd);
  2437. dd->ipath_hol_state = IPATH_HOL_UP;
  2438. }
  2439. /*
  2440. * toggle the running/not running state of user proceses
  2441. * to prevent HoL blocking on chip resources, but still allow
  2442. * user processes to do link down special case handling.
  2443. * Should only be called via the timer
  2444. */
  2445. void ipath_hol_event(unsigned long opaque)
  2446. {
  2447. struct ipath_devdata *dd = (struct ipath_devdata *)opaque;
  2448. if (dd->ipath_hol_next == IPATH_HOL_DOWNSTOP
  2449. && dd->ipath_hol_state != IPATH_HOL_UP) {
  2450. dd->ipath_hol_next = IPATH_HOL_DOWNCONT;
  2451. ipath_dbg("Stopping processes\n");
  2452. ipath_hol_signal_down(dd);
  2453. } else { /* may do "extra" if also in ipath_hol_up() */
  2454. dd->ipath_hol_next = IPATH_HOL_DOWNSTOP;
  2455. ipath_dbg("Continuing processes\n");
  2456. ipath_hol_signal_up(dd);
  2457. }
  2458. if (dd->ipath_hol_state == IPATH_HOL_UP)
  2459. ipath_dbg("link's up, don't resched timer\n");
  2460. else {
  2461. dd->ipath_hol_timer.expires = jiffies +
  2462. msecs_to_jiffies(ipath_hol_timeout_ms);
  2463. mod_timer(&dd->ipath_hol_timer,
  2464. dd->ipath_hol_timer.expires);
  2465. }
  2466. }
  2467. int ipath_set_rx_pol_inv(struct ipath_devdata *dd, u8 new_pol_inv)
  2468. {
  2469. u64 val;
  2470. if (new_pol_inv > INFINIPATH_XGXS_RX_POL_MASK)
  2471. return -1;
  2472. if (dd->ipath_rx_pol_inv != new_pol_inv) {
  2473. dd->ipath_rx_pol_inv = new_pol_inv;
  2474. val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_xgxsconfig);
  2475. val &= ~(INFINIPATH_XGXS_RX_POL_MASK <<
  2476. INFINIPATH_XGXS_RX_POL_SHIFT);
  2477. val |= ((u64)dd->ipath_rx_pol_inv) <<
  2478. INFINIPATH_XGXS_RX_POL_SHIFT;
  2479. ipath_write_kreg(dd, dd->ipath_kregs->kr_xgxsconfig, val);
  2480. }
  2481. return 0;
  2482. }
  2483. /*
  2484. * Disable and enable the armlaunch error. Used for PIO bandwidth testing on
  2485. * the 7220, which is count-based, rather than trigger-based. Safe for the
  2486. * driver check, since it's at init. Not completely safe when used for
  2487. * user-mode checking, since some error checking can be lost, but not
  2488. * particularly risky, and only has problematic side-effects in the face of
  2489. * very buggy user code. There is no reference counting, but that's also
  2490. * fine, given the intended use.
  2491. */
  2492. void ipath_enable_armlaunch(struct ipath_devdata *dd)
  2493. {
  2494. dd->ipath_lasterror &= ~INFINIPATH_E_SPIOARMLAUNCH;
  2495. ipath_write_kreg(dd, dd->ipath_kregs->kr_errorclear,
  2496. INFINIPATH_E_SPIOARMLAUNCH);
  2497. dd->ipath_errormask |= INFINIPATH_E_SPIOARMLAUNCH;
  2498. ipath_write_kreg(dd, dd->ipath_kregs->kr_errormask,
  2499. dd->ipath_errormask);
  2500. }
  2501. void ipath_disable_armlaunch(struct ipath_devdata *dd)
  2502. {
  2503. /* so don't re-enable if already set */
  2504. dd->ipath_maskederrs &= ~INFINIPATH_E_SPIOARMLAUNCH;
  2505. dd->ipath_errormask &= ~INFINIPATH_E_SPIOARMLAUNCH;
  2506. ipath_write_kreg(dd, dd->ipath_kregs->kr_errormask,
  2507. dd->ipath_errormask);
  2508. }
  2509. module_init(infinipath_init);
  2510. module_exit(infinipath_cleanup);