/drivers/infiniband/hw/ipath/ipath_file_ops.c

https://bitbucket.org/cresqo/cm7-p500-kernel · C · 2617 lines · 1894 code · 278 blank · 445 comment · 306 complexity · f6751c75abf34c137f8b85179b5485de MD5 · raw file

Large files are truncated click here to view the full file

  1. /*
  2. * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
  3. * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
  4. *
  5. * This software is available to you under a choice of one of two
  6. * licenses. You may choose to be licensed under the terms of the GNU
  7. * General Public License (GPL) Version 2, available from the file
  8. * COPYING in the main directory of this source tree, or the
  9. * OpenIB.org BSD license below:
  10. *
  11. * Redistribution and use in source and binary forms, with or
  12. * without modification, are permitted provided that the following
  13. * conditions are met:
  14. *
  15. * - Redistributions of source code must retain the above
  16. * copyright notice, this list of conditions and the following
  17. * disclaimer.
  18. *
  19. * - Redistributions in binary form must reproduce the above
  20. * copyright notice, this list of conditions and the following
  21. * disclaimer in the documentation and/or other materials
  22. * provided with the distribution.
  23. *
  24. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31. * SOFTWARE.
  32. */
  33. #include <linux/pci.h>
  34. #include <linux/poll.h>
  35. #include <linux/cdev.h>
  36. #include <linux/swap.h>
  37. #include <linux/vmalloc.h>
  38. #include <linux/slab.h>
  39. #include <linux/highmem.h>
  40. #include <linux/io.h>
  41. #include <linux/jiffies.h>
  42. #include <linux/smp_lock.h>
  43. #include <asm/pgtable.h>
  44. #include "ipath_kernel.h"
  45. #include "ipath_common.h"
  46. #include "ipath_user_sdma.h"
  47. static int ipath_open(struct inode *, struct file *);
  48. static int ipath_close(struct inode *, struct file *);
  49. static ssize_t ipath_write(struct file *, const char __user *, size_t,
  50. loff_t *);
  51. static ssize_t ipath_writev(struct kiocb *, const struct iovec *,
  52. unsigned long , loff_t);
  53. static unsigned int ipath_poll(struct file *, struct poll_table_struct *);
  54. static int ipath_mmap(struct file *, struct vm_area_struct *);
  55. static const struct file_operations ipath_file_ops = {
  56. .owner = THIS_MODULE,
  57. .write = ipath_write,
  58. .aio_write = ipath_writev,
  59. .open = ipath_open,
  60. .release = ipath_close,
  61. .poll = ipath_poll,
  62. .mmap = ipath_mmap
  63. };
  64. /*
  65. * Convert kernel virtual addresses to physical addresses so they don't
  66. * potentially conflict with the chip addresses used as mmap offsets.
  67. * It doesn't really matter what mmap offset we use as long as we can
  68. * interpret it correctly.
  69. */
  70. static u64 cvt_kvaddr(void *p)
  71. {
  72. struct page *page;
  73. u64 paddr = 0;
  74. page = vmalloc_to_page(p);
  75. if (page)
  76. paddr = page_to_pfn(page) << PAGE_SHIFT;
  77. return paddr;
  78. }
  79. static int ipath_get_base_info(struct file *fp,
  80. void __user *ubase, size_t ubase_size)
  81. {
  82. struct ipath_portdata *pd = port_fp(fp);
  83. int ret = 0;
  84. struct ipath_base_info *kinfo = NULL;
  85. struct ipath_devdata *dd = pd->port_dd;
  86. unsigned subport_cnt;
  87. int shared, master;
  88. size_t sz;
  89. subport_cnt = pd->port_subport_cnt;
  90. if (!subport_cnt) {
  91. shared = 0;
  92. master = 0;
  93. subport_cnt = 1;
  94. } else {
  95. shared = 1;
  96. master = !subport_fp(fp);
  97. }
  98. sz = sizeof(*kinfo);
  99. /* If port sharing is not requested, allow the old size structure */
  100. if (!shared)
  101. sz -= 7 * sizeof(u64);
  102. if (ubase_size < sz) {
  103. ipath_cdbg(PROC,
  104. "Base size %zu, need %zu (version mismatch?)\n",
  105. ubase_size, sz);
  106. ret = -EINVAL;
  107. goto bail;
  108. }
  109. kinfo = kzalloc(sizeof(*kinfo), GFP_KERNEL);
  110. if (kinfo == NULL) {
  111. ret = -ENOMEM;
  112. goto bail;
  113. }
  114. ret = dd->ipath_f_get_base_info(pd, kinfo);
  115. if (ret < 0)
  116. goto bail;
  117. kinfo->spi_rcvhdr_cnt = dd->ipath_rcvhdrcnt;
  118. kinfo->spi_rcvhdrent_size = dd->ipath_rcvhdrentsize;
  119. kinfo->spi_tidegrcnt = dd->ipath_rcvegrcnt;
  120. kinfo->spi_rcv_egrbufsize = dd->ipath_rcvegrbufsize;
  121. /*
  122. * have to mmap whole thing
  123. */
  124. kinfo->spi_rcv_egrbuftotlen =
  125. pd->port_rcvegrbuf_chunks * pd->port_rcvegrbuf_size;
  126. kinfo->spi_rcv_egrperchunk = pd->port_rcvegrbufs_perchunk;
  127. kinfo->spi_rcv_egrchunksize = kinfo->spi_rcv_egrbuftotlen /
  128. pd->port_rcvegrbuf_chunks;
  129. kinfo->spi_tidcnt = dd->ipath_rcvtidcnt / subport_cnt;
  130. if (master)
  131. kinfo->spi_tidcnt += dd->ipath_rcvtidcnt % subport_cnt;
  132. /*
  133. * for this use, may be ipath_cfgports summed over all chips that
  134. * are are configured and present
  135. */
  136. kinfo->spi_nports = dd->ipath_cfgports;
  137. /* unit (chip/board) our port is on */
  138. kinfo->spi_unit = dd->ipath_unit;
  139. /* for now, only a single page */
  140. kinfo->spi_tid_maxsize = PAGE_SIZE;
  141. /*
  142. * Doing this per port, and based on the skip value, etc. This has
  143. * to be the actual buffer size, since the protocol code treats it
  144. * as an array.
  145. *
  146. * These have to be set to user addresses in the user code via mmap.
  147. * These values are used on return to user code for the mmap target
  148. * addresses only. For 32 bit, same 44 bit address problem, so use
  149. * the physical address, not virtual. Before 2.6.11, using the
  150. * page_address() macro worked, but in 2.6.11, even that returns the
  151. * full 64 bit address (upper bits all 1's). So far, using the
  152. * physical addresses (or chip offsets, for chip mapping) works, but
  153. * no doubt some future kernel release will change that, and we'll be
  154. * on to yet another method of dealing with this.
  155. */
  156. kinfo->spi_rcvhdr_base = (u64) pd->port_rcvhdrq_phys;
  157. kinfo->spi_rcvhdr_tailaddr = (u64) pd->port_rcvhdrqtailaddr_phys;
  158. kinfo->spi_rcv_egrbufs = (u64) pd->port_rcvegr_phys;
  159. kinfo->spi_pioavailaddr = (u64) dd->ipath_pioavailregs_phys;
  160. kinfo->spi_status = (u64) kinfo->spi_pioavailaddr +
  161. (void *) dd->ipath_statusp -
  162. (void *) dd->ipath_pioavailregs_dma;
  163. if (!shared) {
  164. kinfo->spi_piocnt = pd->port_piocnt;
  165. kinfo->spi_piobufbase = (u64) pd->port_piobufs;
  166. kinfo->__spi_uregbase = (u64) dd->ipath_uregbase +
  167. dd->ipath_ureg_align * pd->port_port;
  168. } else if (master) {
  169. kinfo->spi_piocnt = (pd->port_piocnt / subport_cnt) +
  170. (pd->port_piocnt % subport_cnt);
  171. /* Master's PIO buffers are after all the slave's */
  172. kinfo->spi_piobufbase = (u64) pd->port_piobufs +
  173. dd->ipath_palign *
  174. (pd->port_piocnt - kinfo->spi_piocnt);
  175. } else {
  176. unsigned slave = subport_fp(fp) - 1;
  177. kinfo->spi_piocnt = pd->port_piocnt / subport_cnt;
  178. kinfo->spi_piobufbase = (u64) pd->port_piobufs +
  179. dd->ipath_palign * kinfo->spi_piocnt * slave;
  180. }
  181. if (shared) {
  182. kinfo->spi_port_uregbase = (u64) dd->ipath_uregbase +
  183. dd->ipath_ureg_align * pd->port_port;
  184. kinfo->spi_port_rcvegrbuf = kinfo->spi_rcv_egrbufs;
  185. kinfo->spi_port_rcvhdr_base = kinfo->spi_rcvhdr_base;
  186. kinfo->spi_port_rcvhdr_tailaddr = kinfo->spi_rcvhdr_tailaddr;
  187. kinfo->__spi_uregbase = cvt_kvaddr(pd->subport_uregbase +
  188. PAGE_SIZE * subport_fp(fp));
  189. kinfo->spi_rcvhdr_base = cvt_kvaddr(pd->subport_rcvhdr_base +
  190. pd->port_rcvhdrq_size * subport_fp(fp));
  191. kinfo->spi_rcvhdr_tailaddr = 0;
  192. kinfo->spi_rcv_egrbufs = cvt_kvaddr(pd->subport_rcvegrbuf +
  193. pd->port_rcvegrbuf_chunks * pd->port_rcvegrbuf_size *
  194. subport_fp(fp));
  195. kinfo->spi_subport_uregbase =
  196. cvt_kvaddr(pd->subport_uregbase);
  197. kinfo->spi_subport_rcvegrbuf =
  198. cvt_kvaddr(pd->subport_rcvegrbuf);
  199. kinfo->spi_subport_rcvhdr_base =
  200. cvt_kvaddr(pd->subport_rcvhdr_base);
  201. ipath_cdbg(PROC, "port %u flags %x %llx %llx %llx\n",
  202. kinfo->spi_port, kinfo->spi_runtime_flags,
  203. (unsigned long long) kinfo->spi_subport_uregbase,
  204. (unsigned long long) kinfo->spi_subport_rcvegrbuf,
  205. (unsigned long long) kinfo->spi_subport_rcvhdr_base);
  206. }
  207. /*
  208. * All user buffers are 2KB buffers. If we ever support
  209. * giving 4KB buffers to user processes, this will need some
  210. * work.
  211. */
  212. kinfo->spi_pioindex = (kinfo->spi_piobufbase -
  213. (dd->ipath_piobufbase & 0xffffffff)) / dd->ipath_palign;
  214. kinfo->spi_pioalign = dd->ipath_palign;
  215. kinfo->spi_qpair = IPATH_KD_QP;
  216. /*
  217. * user mode PIO buffers are always 2KB, even when 4KB can
  218. * be received, and sent via the kernel; this is ibmaxlen
  219. * for 2K MTU.
  220. */
  221. kinfo->spi_piosize = dd->ipath_piosize2k - 2 * sizeof(u32);
  222. kinfo->spi_mtu = dd->ipath_ibmaxlen; /* maxlen, not ibmtu */
  223. kinfo->spi_port = pd->port_port;
  224. kinfo->spi_subport = subport_fp(fp);
  225. kinfo->spi_sw_version = IPATH_KERN_SWVERSION;
  226. kinfo->spi_hw_version = dd->ipath_revision;
  227. if (master) {
  228. kinfo->spi_runtime_flags |= IPATH_RUNTIME_MASTER;
  229. }
  230. sz = (ubase_size < sizeof(*kinfo)) ? ubase_size : sizeof(*kinfo);
  231. if (copy_to_user(ubase, kinfo, sz))
  232. ret = -EFAULT;
  233. bail:
  234. kfree(kinfo);
  235. return ret;
  236. }
  237. /**
  238. * ipath_tid_update - update a port TID
  239. * @pd: the port
  240. * @fp: the ipath device file
  241. * @ti: the TID information
  242. *
  243. * The new implementation as of Oct 2004 is that the driver assigns
  244. * the tid and returns it to the caller. To make it easier to
  245. * catch bugs, and to reduce search time, we keep a cursor for
  246. * each port, walking the shadow tid array to find one that's not
  247. * in use.
  248. *
  249. * For now, if we can't allocate the full list, we fail, although
  250. * in the long run, we'll allocate as many as we can, and the
  251. * caller will deal with that by trying the remaining pages later.
  252. * That means that when we fail, we have to mark the tids as not in
  253. * use again, in our shadow copy.
  254. *
  255. * It's up to the caller to free the tids when they are done.
  256. * We'll unlock the pages as they free them.
  257. *
  258. * Also, right now we are locking one page at a time, but since
  259. * the intended use of this routine is for a single group of
  260. * virtually contiguous pages, that should change to improve
  261. * performance.
  262. */
  263. static int ipath_tid_update(struct ipath_portdata *pd, struct file *fp,
  264. const struct ipath_tid_info *ti)
  265. {
  266. int ret = 0, ntids;
  267. u32 tid, porttid, cnt, i, tidcnt, tidoff;
  268. u16 *tidlist;
  269. struct ipath_devdata *dd = pd->port_dd;
  270. u64 physaddr;
  271. unsigned long vaddr;
  272. u64 __iomem *tidbase;
  273. unsigned long tidmap[8];
  274. struct page **pagep = NULL;
  275. unsigned subport = subport_fp(fp);
  276. if (!dd->ipath_pageshadow) {
  277. ret = -ENOMEM;
  278. goto done;
  279. }
  280. cnt = ti->tidcnt;
  281. if (!cnt) {
  282. ipath_dbg("After copyin, tidcnt 0, tidlist %llx\n",
  283. (unsigned long long) ti->tidlist);
  284. /*
  285. * Should we treat as success? likely a bug
  286. */
  287. ret = -EFAULT;
  288. goto done;
  289. }
  290. porttid = pd->port_port * dd->ipath_rcvtidcnt;
  291. if (!pd->port_subport_cnt) {
  292. tidcnt = dd->ipath_rcvtidcnt;
  293. tid = pd->port_tidcursor;
  294. tidoff = 0;
  295. } else if (!subport) {
  296. tidcnt = (dd->ipath_rcvtidcnt / pd->port_subport_cnt) +
  297. (dd->ipath_rcvtidcnt % pd->port_subport_cnt);
  298. tidoff = dd->ipath_rcvtidcnt - tidcnt;
  299. porttid += tidoff;
  300. tid = tidcursor_fp(fp);
  301. } else {
  302. tidcnt = dd->ipath_rcvtidcnt / pd->port_subport_cnt;
  303. tidoff = tidcnt * (subport - 1);
  304. porttid += tidoff;
  305. tid = tidcursor_fp(fp);
  306. }
  307. if (cnt > tidcnt) {
  308. /* make sure it all fits in port_tid_pg_list */
  309. dev_info(&dd->pcidev->dev, "Process tried to allocate %u "
  310. "TIDs, only trying max (%u)\n", cnt, tidcnt);
  311. cnt = tidcnt;
  312. }
  313. pagep = &((struct page **) pd->port_tid_pg_list)[tidoff];
  314. tidlist = &((u16 *) &pagep[dd->ipath_rcvtidcnt])[tidoff];
  315. memset(tidmap, 0, sizeof(tidmap));
  316. /* before decrement; chip actual # */
  317. ntids = tidcnt;
  318. tidbase = (u64 __iomem *) (((char __iomem *) dd->ipath_kregbase) +
  319. dd->ipath_rcvtidbase +
  320. porttid * sizeof(*tidbase));
  321. ipath_cdbg(VERBOSE, "Port%u %u tids, cursor %u, tidbase %p\n",
  322. pd->port_port, cnt, tid, tidbase);
  323. /* virtual address of first page in transfer */
  324. vaddr = ti->tidvaddr;
  325. if (!access_ok(VERIFY_WRITE, (void __user *) vaddr,
  326. cnt * PAGE_SIZE)) {
  327. ipath_dbg("Fail vaddr %p, %u pages, !access_ok\n",
  328. (void *)vaddr, cnt);
  329. ret = -EFAULT;
  330. goto done;
  331. }
  332. ret = ipath_get_user_pages(vaddr, cnt, pagep);
  333. if (ret) {
  334. if (ret == -EBUSY) {
  335. ipath_dbg("Failed to lock addr %p, %u pages "
  336. "(already locked)\n",
  337. (void *) vaddr, cnt);
  338. /*
  339. * for now, continue, and see what happens but with
  340. * the new implementation, this should never happen,
  341. * unless perhaps the user has mpin'ed the pages
  342. * themselves (something we need to test)
  343. */
  344. ret = 0;
  345. } else {
  346. dev_info(&dd->pcidev->dev,
  347. "Failed to lock addr %p, %u pages: "
  348. "errno %d\n", (void *) vaddr, cnt, -ret);
  349. goto done;
  350. }
  351. }
  352. for (i = 0; i < cnt; i++, vaddr += PAGE_SIZE) {
  353. for (; ntids--; tid++) {
  354. if (tid == tidcnt)
  355. tid = 0;
  356. if (!dd->ipath_pageshadow[porttid + tid])
  357. break;
  358. }
  359. if (ntids < 0) {
  360. /*
  361. * oops, wrapped all the way through their TIDs,
  362. * and didn't have enough free; see comments at
  363. * start of routine
  364. */
  365. ipath_dbg("Not enough free TIDs for %u pages "
  366. "(index %d), failing\n", cnt, i);
  367. i--; /* last tidlist[i] not filled in */
  368. ret = -ENOMEM;
  369. break;
  370. }
  371. tidlist[i] = tid + tidoff;
  372. ipath_cdbg(VERBOSE, "Updating idx %u to TID %u, "
  373. "vaddr %lx\n", i, tid + tidoff, vaddr);
  374. /* we "know" system pages and TID pages are same size */
  375. dd->ipath_pageshadow[porttid + tid] = pagep[i];
  376. dd->ipath_physshadow[porttid + tid] = ipath_map_page(
  377. dd->pcidev, pagep[i], 0, PAGE_SIZE,
  378. PCI_DMA_FROMDEVICE);
  379. /*
  380. * don't need atomic or it's overhead
  381. */
  382. __set_bit(tid, tidmap);
  383. physaddr = dd->ipath_physshadow[porttid + tid];
  384. ipath_stats.sps_pagelocks++;
  385. ipath_cdbg(VERBOSE,
  386. "TID %u, vaddr %lx, physaddr %llx pgp %p\n",
  387. tid, vaddr, (unsigned long long) physaddr,
  388. pagep[i]);
  389. dd->ipath_f_put_tid(dd, &tidbase[tid], RCVHQ_RCV_TYPE_EXPECTED,
  390. physaddr);
  391. /*
  392. * don't check this tid in ipath_portshadow, since we
  393. * just filled it in; start with the next one.
  394. */
  395. tid++;
  396. }
  397. if (ret) {
  398. u32 limit;
  399. cleanup:
  400. /* jump here if copy out of updated info failed... */
  401. ipath_dbg("After failure (ret=%d), undo %d of %d entries\n",
  402. -ret, i, cnt);
  403. /* same code that's in ipath_free_tid() */
  404. limit = sizeof(tidmap) * BITS_PER_BYTE;
  405. if (limit > tidcnt)
  406. /* just in case size changes in future */
  407. limit = tidcnt;
  408. tid = find_first_bit((const unsigned long *)tidmap, limit);
  409. for (; tid < limit; tid++) {
  410. if (!test_bit(tid, tidmap))
  411. continue;
  412. if (dd->ipath_pageshadow[porttid + tid]) {
  413. ipath_cdbg(VERBOSE, "Freeing TID %u\n",
  414. tid);
  415. dd->ipath_f_put_tid(dd, &tidbase[tid],
  416. RCVHQ_RCV_TYPE_EXPECTED,
  417. dd->ipath_tidinvalid);
  418. pci_unmap_page(dd->pcidev,
  419. dd->ipath_physshadow[porttid + tid],
  420. PAGE_SIZE, PCI_DMA_FROMDEVICE);
  421. dd->ipath_pageshadow[porttid + tid] = NULL;
  422. ipath_stats.sps_pageunlocks++;
  423. }
  424. }
  425. ipath_release_user_pages(pagep, cnt);
  426. } else {
  427. /*
  428. * Copy the updated array, with ipath_tid's filled in, back
  429. * to user. Since we did the copy in already, this "should
  430. * never fail" If it does, we have to clean up...
  431. */
  432. if (copy_to_user((void __user *)
  433. (unsigned long) ti->tidlist,
  434. tidlist, cnt * sizeof(*tidlist))) {
  435. ret = -EFAULT;
  436. goto cleanup;
  437. }
  438. if (copy_to_user((void __user *) (unsigned long) ti->tidmap,
  439. tidmap, sizeof tidmap)) {
  440. ret = -EFAULT;
  441. goto cleanup;
  442. }
  443. if (tid == tidcnt)
  444. tid = 0;
  445. if (!pd->port_subport_cnt)
  446. pd->port_tidcursor = tid;
  447. else
  448. tidcursor_fp(fp) = tid;
  449. }
  450. done:
  451. if (ret)
  452. ipath_dbg("Failed to map %u TID pages, failing with %d\n",
  453. ti->tidcnt, -ret);
  454. return ret;
  455. }
  456. /**
  457. * ipath_tid_free - free a port TID
  458. * @pd: the port
  459. * @subport: the subport
  460. * @ti: the TID info
  461. *
  462. * right now we are unlocking one page at a time, but since
  463. * the intended use of this routine is for a single group of
  464. * virtually contiguous pages, that should change to improve
  465. * performance. We check that the TID is in range for this port
  466. * but otherwise don't check validity; if user has an error and
  467. * frees the wrong tid, it's only their own data that can thereby
  468. * be corrupted. We do check that the TID was in use, for sanity
  469. * We always use our idea of the saved address, not the address that
  470. * they pass in to us.
  471. */
  472. static int ipath_tid_free(struct ipath_portdata *pd, unsigned subport,
  473. const struct ipath_tid_info *ti)
  474. {
  475. int ret = 0;
  476. u32 tid, porttid, cnt, limit, tidcnt;
  477. struct ipath_devdata *dd = pd->port_dd;
  478. u64 __iomem *tidbase;
  479. unsigned long tidmap[8];
  480. if (!dd->ipath_pageshadow) {
  481. ret = -ENOMEM;
  482. goto done;
  483. }
  484. if (copy_from_user(tidmap, (void __user *)(unsigned long)ti->tidmap,
  485. sizeof tidmap)) {
  486. ret = -EFAULT;
  487. goto done;
  488. }
  489. porttid = pd->port_port * dd->ipath_rcvtidcnt;
  490. if (!pd->port_subport_cnt)
  491. tidcnt = dd->ipath_rcvtidcnt;
  492. else if (!subport) {
  493. tidcnt = (dd->ipath_rcvtidcnt / pd->port_subport_cnt) +
  494. (dd->ipath_rcvtidcnt % pd->port_subport_cnt);
  495. porttid += dd->ipath_rcvtidcnt - tidcnt;
  496. } else {
  497. tidcnt = dd->ipath_rcvtidcnt / pd->port_subport_cnt;
  498. porttid += tidcnt * (subport - 1);
  499. }
  500. tidbase = (u64 __iomem *) ((char __iomem *)(dd->ipath_kregbase) +
  501. dd->ipath_rcvtidbase +
  502. porttid * sizeof(*tidbase));
  503. limit = sizeof(tidmap) * BITS_PER_BYTE;
  504. if (limit > tidcnt)
  505. /* just in case size changes in future */
  506. limit = tidcnt;
  507. tid = find_first_bit(tidmap, limit);
  508. ipath_cdbg(VERBOSE, "Port%u free %u tids; first bit (max=%d) "
  509. "set is %d, porttid %u\n", pd->port_port, ti->tidcnt,
  510. limit, tid, porttid);
  511. for (cnt = 0; tid < limit; tid++) {
  512. /*
  513. * small optimization; if we detect a run of 3 or so without
  514. * any set, use find_first_bit again. That's mainly to
  515. * accelerate the case where we wrapped, so we have some at
  516. * the beginning, and some at the end, and a big gap
  517. * in the middle.
  518. */
  519. if (!test_bit(tid, tidmap))
  520. continue;
  521. cnt++;
  522. if (dd->ipath_pageshadow[porttid + tid]) {
  523. struct page *p;
  524. p = dd->ipath_pageshadow[porttid + tid];
  525. dd->ipath_pageshadow[porttid + tid] = NULL;
  526. ipath_cdbg(VERBOSE, "PID %u freeing TID %u\n",
  527. pid_nr(pd->port_pid), tid);
  528. dd->ipath_f_put_tid(dd, &tidbase[tid],
  529. RCVHQ_RCV_TYPE_EXPECTED,
  530. dd->ipath_tidinvalid);
  531. pci_unmap_page(dd->pcidev,
  532. dd->ipath_physshadow[porttid + tid],
  533. PAGE_SIZE, PCI_DMA_FROMDEVICE);
  534. ipath_release_user_pages(&p, 1);
  535. ipath_stats.sps_pageunlocks++;
  536. } else
  537. ipath_dbg("Unused tid %u, ignoring\n", tid);
  538. }
  539. if (cnt != ti->tidcnt)
  540. ipath_dbg("passed in tidcnt %d, only %d bits set in map\n",
  541. ti->tidcnt, cnt);
  542. done:
  543. if (ret)
  544. ipath_dbg("Failed to unmap %u TID pages, failing with %d\n",
  545. ti->tidcnt, -ret);
  546. return ret;
  547. }
  548. /**
  549. * ipath_set_part_key - set a partition key
  550. * @pd: the port
  551. * @key: the key
  552. *
  553. * We can have up to 4 active at a time (other than the default, which is
  554. * always allowed). This is somewhat tricky, since multiple ports may set
  555. * the same key, so we reference count them, and clean up at exit. All 4
  556. * partition keys are packed into a single infinipath register. It's an
  557. * error for a process to set the same pkey multiple times. We provide no
  558. * mechanism to de-allocate a pkey at this time, we may eventually need to
  559. * do that. I've used the atomic operations, and no locking, and only make
  560. * a single pass through what's available. This should be more than
  561. * adequate for some time. I'll think about spinlocks or the like if and as
  562. * it's necessary.
  563. */
  564. static int ipath_set_part_key(struct ipath_portdata *pd, u16 key)
  565. {
  566. struct ipath_devdata *dd = pd->port_dd;
  567. int i, any = 0, pidx = -1;
  568. u16 lkey = key & 0x7FFF;
  569. int ret;
  570. if (lkey == (IPATH_DEFAULT_P_KEY & 0x7FFF)) {
  571. /* nothing to do; this key always valid */
  572. ret = 0;
  573. goto bail;
  574. }
  575. ipath_cdbg(VERBOSE, "p%u try to set pkey %hx, current keys "
  576. "%hx:%x %hx:%x %hx:%x %hx:%x\n",
  577. pd->port_port, key, dd->ipath_pkeys[0],
  578. atomic_read(&dd->ipath_pkeyrefs[0]), dd->ipath_pkeys[1],
  579. atomic_read(&dd->ipath_pkeyrefs[1]), dd->ipath_pkeys[2],
  580. atomic_read(&dd->ipath_pkeyrefs[2]), dd->ipath_pkeys[3],
  581. atomic_read(&dd->ipath_pkeyrefs[3]));
  582. if (!lkey) {
  583. ipath_cdbg(PROC, "p%u tries to set key 0, not allowed\n",
  584. pd->port_port);
  585. ret = -EINVAL;
  586. goto bail;
  587. }
  588. /*
  589. * Set the full membership bit, because it has to be
  590. * set in the register or the packet, and it seems
  591. * cleaner to set in the register than to force all
  592. * callers to set it. (see bug 4331)
  593. */
  594. key |= 0x8000;
  595. for (i = 0; i < ARRAY_SIZE(pd->port_pkeys); i++) {
  596. if (!pd->port_pkeys[i] && pidx == -1)
  597. pidx = i;
  598. if (pd->port_pkeys[i] == key) {
  599. ipath_cdbg(VERBOSE, "p%u tries to set same pkey "
  600. "(%x) more than once\n",
  601. pd->port_port, key);
  602. ret = -EEXIST;
  603. goto bail;
  604. }
  605. }
  606. if (pidx == -1) {
  607. ipath_dbg("All pkeys for port %u already in use, "
  608. "can't set %x\n", pd->port_port, key);
  609. ret = -EBUSY;
  610. goto bail;
  611. }
  612. for (any = i = 0; i < ARRAY_SIZE(dd->ipath_pkeys); i++) {
  613. if (!dd->ipath_pkeys[i]) {
  614. any++;
  615. continue;
  616. }
  617. if (dd->ipath_pkeys[i] == key) {
  618. atomic_t *pkrefs = &dd->ipath_pkeyrefs[i];
  619. if (atomic_inc_return(pkrefs) > 1) {
  620. pd->port_pkeys[pidx] = key;
  621. ipath_cdbg(VERBOSE, "p%u set key %x "
  622. "matches #%d, count now %d\n",
  623. pd->port_port, key, i,
  624. atomic_read(pkrefs));
  625. ret = 0;
  626. goto bail;
  627. } else {
  628. /*
  629. * lost race, decrement count, catch below
  630. */
  631. atomic_dec(pkrefs);
  632. ipath_cdbg(VERBOSE, "Lost race, count was "
  633. "0, after dec, it's %d\n",
  634. atomic_read(pkrefs));
  635. any++;
  636. }
  637. }
  638. if ((dd->ipath_pkeys[i] & 0x7FFF) == lkey) {
  639. /*
  640. * It makes no sense to have both the limited and
  641. * full membership PKEY set at the same time since
  642. * the unlimited one will disable the limited one.
  643. */
  644. ret = -EEXIST;
  645. goto bail;
  646. }
  647. }
  648. if (!any) {
  649. ipath_dbg("port %u, all pkeys already in use, "
  650. "can't set %x\n", pd->port_port, key);
  651. ret = -EBUSY;
  652. goto bail;
  653. }
  654. for (any = i = 0; i < ARRAY_SIZE(dd->ipath_pkeys); i++) {
  655. if (!dd->ipath_pkeys[i] &&
  656. atomic_inc_return(&dd->ipath_pkeyrefs[i]) == 1) {
  657. u64 pkey;
  658. /* for ipathstats, etc. */
  659. ipath_stats.sps_pkeys[i] = lkey;
  660. pd->port_pkeys[pidx] = dd->ipath_pkeys[i] = key;
  661. pkey =
  662. (u64) dd->ipath_pkeys[0] |
  663. ((u64) dd->ipath_pkeys[1] << 16) |
  664. ((u64) dd->ipath_pkeys[2] << 32) |
  665. ((u64) dd->ipath_pkeys[3] << 48);
  666. ipath_cdbg(PROC, "p%u set key %x in #%d, "
  667. "portidx %d, new pkey reg %llx\n",
  668. pd->port_port, key, i, pidx,
  669. (unsigned long long) pkey);
  670. ipath_write_kreg(
  671. dd, dd->ipath_kregs->kr_partitionkey, pkey);
  672. ret = 0;
  673. goto bail;
  674. }
  675. }
  676. ipath_dbg("port %u, all pkeys already in use 2nd pass, "
  677. "can't set %x\n", pd->port_port, key);
  678. ret = -EBUSY;
  679. bail:
  680. return ret;
  681. }
  682. /**
  683. * ipath_manage_rcvq - manage a port's receive queue
  684. * @pd: the port
  685. * @subport: the subport
  686. * @start_stop: action to carry out
  687. *
  688. * start_stop == 0 disables receive on the port, for use in queue
  689. * overflow conditions. start_stop==1 re-enables, to be used to
  690. * re-init the software copy of the head register
  691. */
  692. static int ipath_manage_rcvq(struct ipath_portdata *pd, unsigned subport,
  693. int start_stop)
  694. {
  695. struct ipath_devdata *dd = pd->port_dd;
  696. ipath_cdbg(PROC, "%sabling rcv for unit %u port %u:%u\n",
  697. start_stop ? "en" : "dis", dd->ipath_unit,
  698. pd->port_port, subport);
  699. if (subport)
  700. goto bail;
  701. /* atomically clear receive enable port. */
  702. if (start_stop) {
  703. /*
  704. * On enable, force in-memory copy of the tail register to
  705. * 0, so that protocol code doesn't have to worry about
  706. * whether or not the chip has yet updated the in-memory
  707. * copy or not on return from the system call. The chip
  708. * always resets it's tail register back to 0 on a
  709. * transition from disabled to enabled. This could cause a
  710. * problem if software was broken, and did the enable w/o
  711. * the disable, but eventually the in-memory copy will be
  712. * updated and correct itself, even in the face of software
  713. * bugs.
  714. */
  715. if (pd->port_rcvhdrtail_kvaddr)
  716. ipath_clear_rcvhdrtail(pd);
  717. set_bit(dd->ipath_r_portenable_shift + pd->port_port,
  718. &dd->ipath_rcvctrl);
  719. } else
  720. clear_bit(dd->ipath_r_portenable_shift + pd->port_port,
  721. &dd->ipath_rcvctrl);
  722. ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvctrl,
  723. dd->ipath_rcvctrl);
  724. /* now be sure chip saw it before we return */
  725. ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
  726. if (start_stop) {
  727. /*
  728. * And try to be sure that tail reg update has happened too.
  729. * This should in theory interlock with the RXE changes to
  730. * the tail register. Don't assign it to the tail register
  731. * in memory copy, since we could overwrite an update by the
  732. * chip if we did.
  733. */
  734. ipath_read_ureg32(dd, ur_rcvhdrtail, pd->port_port);
  735. }
  736. /* always; new head should be equal to new tail; see above */
  737. bail:
  738. return 0;
  739. }
  740. static void ipath_clean_part_key(struct ipath_portdata *pd,
  741. struct ipath_devdata *dd)
  742. {
  743. int i, j, pchanged = 0;
  744. u64 oldpkey;
  745. /* for debugging only */
  746. oldpkey = (u64) dd->ipath_pkeys[0] |
  747. ((u64) dd->ipath_pkeys[1] << 16) |
  748. ((u64) dd->ipath_pkeys[2] << 32) |
  749. ((u64) dd->ipath_pkeys[3] << 48);
  750. for (i = 0; i < ARRAY_SIZE(pd->port_pkeys); i++) {
  751. if (!pd->port_pkeys[i])
  752. continue;
  753. ipath_cdbg(VERBOSE, "look for key[%d] %hx in pkeys\n", i,
  754. pd->port_pkeys[i]);
  755. for (j = 0; j < ARRAY_SIZE(dd->ipath_pkeys); j++) {
  756. /* check for match independent of the global bit */
  757. if ((dd->ipath_pkeys[j] & 0x7fff) !=
  758. (pd->port_pkeys[i] & 0x7fff))
  759. continue;
  760. if (atomic_dec_and_test(&dd->ipath_pkeyrefs[j])) {
  761. ipath_cdbg(VERBOSE, "p%u clear key "
  762. "%x matches #%d\n",
  763. pd->port_port,
  764. pd->port_pkeys[i], j);
  765. ipath_stats.sps_pkeys[j] =
  766. dd->ipath_pkeys[j] = 0;
  767. pchanged++;
  768. }
  769. else ipath_cdbg(
  770. VERBOSE, "p%u key %x matches #%d, "
  771. "but ref still %d\n", pd->port_port,
  772. pd->port_pkeys[i], j,
  773. atomic_read(&dd->ipath_pkeyrefs[j]));
  774. break;
  775. }
  776. pd->port_pkeys[i] = 0;
  777. }
  778. if (pchanged) {
  779. u64 pkey = (u64) dd->ipath_pkeys[0] |
  780. ((u64) dd->ipath_pkeys[1] << 16) |
  781. ((u64) dd->ipath_pkeys[2] << 32) |
  782. ((u64) dd->ipath_pkeys[3] << 48);
  783. ipath_cdbg(VERBOSE, "p%u old pkey reg %llx, "
  784. "new pkey reg %llx\n", pd->port_port,
  785. (unsigned long long) oldpkey,
  786. (unsigned long long) pkey);
  787. ipath_write_kreg(dd, dd->ipath_kregs->kr_partitionkey,
  788. pkey);
  789. }
  790. }
  791. /*
  792. * Initialize the port data with the receive buffer sizes
  793. * so this can be done while the master port is locked.
  794. * Otherwise, there is a race with a slave opening the port
  795. * and seeing these fields uninitialized.
  796. */
  797. static void init_user_egr_sizes(struct ipath_portdata *pd)
  798. {
  799. struct ipath_devdata *dd = pd->port_dd;
  800. unsigned egrperchunk, egrcnt, size;
  801. /*
  802. * to avoid wasting a lot of memory, we allocate 32KB chunks of
  803. * physically contiguous memory, advance through it until used up
  804. * and then allocate more. Of course, we need memory to store those
  805. * extra pointers, now. Started out with 256KB, but under heavy
  806. * memory pressure (creating large files and then copying them over
  807. * NFS while doing lots of MPI jobs), we hit some allocation
  808. * failures, even though we can sleep... (2.6.10) Still get
  809. * failures at 64K. 32K is the lowest we can go without wasting
  810. * additional memory.
  811. */
  812. size = 0x8000;
  813. egrperchunk = size / dd->ipath_rcvegrbufsize;
  814. egrcnt = dd->ipath_rcvegrcnt;
  815. pd->port_rcvegrbuf_chunks = (egrcnt + egrperchunk - 1) / egrperchunk;
  816. pd->port_rcvegrbufs_perchunk = egrperchunk;
  817. pd->port_rcvegrbuf_size = size;
  818. }
  819. /**
  820. * ipath_create_user_egr - allocate eager TID buffers
  821. * @pd: the port to allocate TID buffers for
  822. *
  823. * This routine is now quite different for user and kernel, because
  824. * the kernel uses skb's, for the accelerated network performance
  825. * This is the user port version
  826. *
  827. * Allocate the eager TID buffers and program them into infinipath
  828. * They are no longer completely contiguous, we do multiple allocation
  829. * calls.
  830. */
  831. static int ipath_create_user_egr(struct ipath_portdata *pd)
  832. {
  833. struct ipath_devdata *dd = pd->port_dd;
  834. unsigned e, egrcnt, egrperchunk, chunk, egrsize, egroff;
  835. size_t size;
  836. int ret;
  837. gfp_t gfp_flags;
  838. /*
  839. * GFP_USER, but without GFP_FS, so buffer cache can be
  840. * coalesced (we hope); otherwise, even at order 4,
  841. * heavy filesystem activity makes these fail, and we can
  842. * use compound pages.
  843. */
  844. gfp_flags = __GFP_WAIT | __GFP_IO | __GFP_COMP;
  845. egrcnt = dd->ipath_rcvegrcnt;
  846. /* TID number offset for this port */
  847. egroff = (pd->port_port - 1) * egrcnt + dd->ipath_p0_rcvegrcnt;
  848. egrsize = dd->ipath_rcvegrbufsize;
  849. ipath_cdbg(VERBOSE, "Allocating %d egr buffers, at egrtid "
  850. "offset %x, egrsize %u\n", egrcnt, egroff, egrsize);
  851. chunk = pd->port_rcvegrbuf_chunks;
  852. egrperchunk = pd->port_rcvegrbufs_perchunk;
  853. size = pd->port_rcvegrbuf_size;
  854. pd->port_rcvegrbuf = kmalloc(chunk * sizeof(pd->port_rcvegrbuf[0]),
  855. GFP_KERNEL);
  856. if (!pd->port_rcvegrbuf) {
  857. ret = -ENOMEM;
  858. goto bail;
  859. }
  860. pd->port_rcvegrbuf_phys =
  861. kmalloc(chunk * sizeof(pd->port_rcvegrbuf_phys[0]),
  862. GFP_KERNEL);
  863. if (!pd->port_rcvegrbuf_phys) {
  864. ret = -ENOMEM;
  865. goto bail_rcvegrbuf;
  866. }
  867. for (e = 0; e < pd->port_rcvegrbuf_chunks; e++) {
  868. pd->port_rcvegrbuf[e] = dma_alloc_coherent(
  869. &dd->pcidev->dev, size, &pd->port_rcvegrbuf_phys[e],
  870. gfp_flags);
  871. if (!pd->port_rcvegrbuf[e]) {
  872. ret = -ENOMEM;
  873. goto bail_rcvegrbuf_phys;
  874. }
  875. }
  876. pd->port_rcvegr_phys = pd->port_rcvegrbuf_phys[0];
  877. for (e = chunk = 0; chunk < pd->port_rcvegrbuf_chunks; chunk++) {
  878. dma_addr_t pa = pd->port_rcvegrbuf_phys[chunk];
  879. unsigned i;
  880. for (i = 0; e < egrcnt && i < egrperchunk; e++, i++) {
  881. dd->ipath_f_put_tid(dd, e + egroff +
  882. (u64 __iomem *)
  883. ((char __iomem *)
  884. dd->ipath_kregbase +
  885. dd->ipath_rcvegrbase),
  886. RCVHQ_RCV_TYPE_EAGER, pa);
  887. pa += egrsize;
  888. }
  889. cond_resched(); /* don't hog the cpu */
  890. }
  891. ret = 0;
  892. goto bail;
  893. bail_rcvegrbuf_phys:
  894. for (e = 0; e < pd->port_rcvegrbuf_chunks &&
  895. pd->port_rcvegrbuf[e]; e++) {
  896. dma_free_coherent(&dd->pcidev->dev, size,
  897. pd->port_rcvegrbuf[e],
  898. pd->port_rcvegrbuf_phys[e]);
  899. }
  900. kfree(pd->port_rcvegrbuf_phys);
  901. pd->port_rcvegrbuf_phys = NULL;
  902. bail_rcvegrbuf:
  903. kfree(pd->port_rcvegrbuf);
  904. pd->port_rcvegrbuf = NULL;
  905. bail:
  906. return ret;
  907. }
  908. /* common code for the mappings on dma_alloc_coherent mem */
  909. static int ipath_mmap_mem(struct vm_area_struct *vma,
  910. struct ipath_portdata *pd, unsigned len, int write_ok,
  911. void *kvaddr, char *what)
  912. {
  913. struct ipath_devdata *dd = pd->port_dd;
  914. unsigned long pfn;
  915. int ret;
  916. if ((vma->vm_end - vma->vm_start) > len) {
  917. dev_info(&dd->pcidev->dev,
  918. "FAIL on %s: len %lx > %x\n", what,
  919. vma->vm_end - vma->vm_start, len);
  920. ret = -EFAULT;
  921. goto bail;
  922. }
  923. if (!write_ok) {
  924. if (vma->vm_flags & VM_WRITE) {
  925. dev_info(&dd->pcidev->dev,
  926. "%s must be mapped readonly\n", what);
  927. ret = -EPERM;
  928. goto bail;
  929. }
  930. /* don't allow them to later change with mprotect */
  931. vma->vm_flags &= ~VM_MAYWRITE;
  932. }
  933. pfn = virt_to_phys(kvaddr) >> PAGE_SHIFT;
  934. ret = remap_pfn_range(vma, vma->vm_start, pfn,
  935. len, vma->vm_page_prot);
  936. if (ret)
  937. dev_info(&dd->pcidev->dev, "%s port%u mmap of %lx, %x "
  938. "bytes r%c failed: %d\n", what, pd->port_port,
  939. pfn, len, write_ok?'w':'o', ret);
  940. else
  941. ipath_cdbg(VERBOSE, "%s port%u mmaped %lx, %x bytes "
  942. "r%c\n", what, pd->port_port, pfn, len,
  943. write_ok?'w':'o');
  944. bail:
  945. return ret;
  946. }
  947. static int mmap_ureg(struct vm_area_struct *vma, struct ipath_devdata *dd,
  948. u64 ureg)
  949. {
  950. unsigned long phys;
  951. int ret;
  952. /*
  953. * This is real hardware, so use io_remap. This is the mechanism
  954. * for the user process to update the head registers for their port
  955. * in the chip.
  956. */
  957. if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) {
  958. dev_info(&dd->pcidev->dev, "FAIL mmap userreg: reqlen "
  959. "%lx > PAGE\n", vma->vm_end - vma->vm_start);
  960. ret = -EFAULT;
  961. } else {
  962. phys = dd->ipath_physaddr + ureg;
  963. vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
  964. vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND;
  965. ret = io_remap_pfn_range(vma, vma->vm_start,
  966. phys >> PAGE_SHIFT,
  967. vma->vm_end - vma->vm_start,
  968. vma->vm_page_prot);
  969. }
  970. return ret;
  971. }
  972. static int mmap_piobufs(struct vm_area_struct *vma,
  973. struct ipath_devdata *dd,
  974. struct ipath_portdata *pd,
  975. unsigned piobufs, unsigned piocnt)
  976. {
  977. unsigned long phys;
  978. int ret;
  979. /*
  980. * When we map the PIO buffers in the chip, we want to map them as
  981. * writeonly, no read possible. This prevents access to previous
  982. * process data, and catches users who might try to read the i/o
  983. * space due to a bug.
  984. */
  985. if ((vma->vm_end - vma->vm_start) > (piocnt * dd->ipath_palign)) {
  986. dev_info(&dd->pcidev->dev, "FAIL mmap piobufs: "
  987. "reqlen %lx > PAGE\n",
  988. vma->vm_end - vma->vm_start);
  989. ret = -EINVAL;
  990. goto bail;
  991. }
  992. phys = dd->ipath_physaddr + piobufs;
  993. #if defined(__powerpc__)
  994. /* There isn't a generic way to specify writethrough mappings */
  995. pgprot_val(vma->vm_page_prot) |= _PAGE_NO_CACHE;
  996. pgprot_val(vma->vm_page_prot) |= _PAGE_WRITETHRU;
  997. pgprot_val(vma->vm_page_prot) &= ~_PAGE_GUARDED;
  998. #endif
  999. /*
  1000. * don't allow them to later change to readable with mprotect (for when
  1001. * not initially mapped readable, as is normally the case)
  1002. */
  1003. vma->vm_flags &= ~VM_MAYREAD;
  1004. vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND;
  1005. ret = io_remap_pfn_range(vma, vma->vm_start, phys >> PAGE_SHIFT,
  1006. vma->vm_end - vma->vm_start,
  1007. vma->vm_page_prot);
  1008. bail:
  1009. return ret;
  1010. }
  1011. static int mmap_rcvegrbufs(struct vm_area_struct *vma,
  1012. struct ipath_portdata *pd)
  1013. {
  1014. struct ipath_devdata *dd = pd->port_dd;
  1015. unsigned long start, size;
  1016. size_t total_size, i;
  1017. unsigned long pfn;
  1018. int ret;
  1019. size = pd->port_rcvegrbuf_size;
  1020. total_size = pd->port_rcvegrbuf_chunks * size;
  1021. if ((vma->vm_end - vma->vm_start) > total_size) {
  1022. dev_info(&dd->pcidev->dev, "FAIL on egr bufs: "
  1023. "reqlen %lx > actual %lx\n",
  1024. vma->vm_end - vma->vm_start,
  1025. (unsigned long) total_size);
  1026. ret = -EINVAL;
  1027. goto bail;
  1028. }
  1029. if (vma->vm_flags & VM_WRITE) {
  1030. dev_info(&dd->pcidev->dev, "Can't map eager buffers as "
  1031. "writable (flags=%lx)\n", vma->vm_flags);
  1032. ret = -EPERM;
  1033. goto bail;
  1034. }
  1035. /* don't allow them to later change to writeable with mprotect */
  1036. vma->vm_flags &= ~VM_MAYWRITE;
  1037. start = vma->vm_start;
  1038. for (i = 0; i < pd->port_rcvegrbuf_chunks; i++, start += size) {
  1039. pfn = virt_to_phys(pd->port_rcvegrbuf[i]) >> PAGE_SHIFT;
  1040. ret = remap_pfn_range(vma, start, pfn, size,
  1041. vma->vm_page_prot);
  1042. if (ret < 0)
  1043. goto bail;
  1044. }
  1045. ret = 0;
  1046. bail:
  1047. return ret;
  1048. }
  1049. /*
  1050. * ipath_file_vma_fault - handle a VMA page fault.
  1051. */
  1052. static int ipath_file_vma_fault(struct vm_area_struct *vma,
  1053. struct vm_fault *vmf)
  1054. {
  1055. struct page *page;
  1056. page = vmalloc_to_page((void *)(vmf->pgoff << PAGE_SHIFT));
  1057. if (!page)
  1058. return VM_FAULT_SIGBUS;
  1059. get_page(page);
  1060. vmf->page = page;
  1061. return 0;
  1062. }
  1063. static const struct vm_operations_struct ipath_file_vm_ops = {
  1064. .fault = ipath_file_vma_fault,
  1065. };
  1066. static int mmap_kvaddr(struct vm_area_struct *vma, u64 pgaddr,
  1067. struct ipath_portdata *pd, unsigned subport)
  1068. {
  1069. unsigned long len;
  1070. struct ipath_devdata *dd;
  1071. void *addr;
  1072. size_t size;
  1073. int ret = 0;
  1074. /* If the port is not shared, all addresses should be physical */
  1075. if (!pd->port_subport_cnt)
  1076. goto bail;
  1077. dd = pd->port_dd;
  1078. size = pd->port_rcvegrbuf_chunks * pd->port_rcvegrbuf_size;
  1079. /*
  1080. * Each process has all the subport uregbase, rcvhdrq, and
  1081. * rcvegrbufs mmapped - as an array for all the processes,
  1082. * and also separately for this process.
  1083. */
  1084. if (pgaddr == cvt_kvaddr(pd->subport_uregbase)) {
  1085. addr = pd->subport_uregbase;
  1086. size = PAGE_SIZE * pd->port_subport_cnt;
  1087. } else if (pgaddr == cvt_kvaddr(pd->subport_rcvhdr_base)) {
  1088. addr = pd->subport_rcvhdr_base;
  1089. size = pd->port_rcvhdrq_size * pd->port_subport_cnt;
  1090. } else if (pgaddr == cvt_kvaddr(pd->subport_rcvegrbuf)) {
  1091. addr = pd->subport_rcvegrbuf;
  1092. size *= pd->port_subport_cnt;
  1093. } else if (pgaddr == cvt_kvaddr(pd->subport_uregbase +
  1094. PAGE_SIZE * subport)) {
  1095. addr = pd->subport_uregbase + PAGE_SIZE * subport;
  1096. size = PAGE_SIZE;
  1097. } else if (pgaddr == cvt_kvaddr(pd->subport_rcvhdr_base +
  1098. pd->port_rcvhdrq_size * subport)) {
  1099. addr = pd->subport_rcvhdr_base +
  1100. pd->port_rcvhdrq_size * subport;
  1101. size = pd->port_rcvhdrq_size;
  1102. } else if (pgaddr == cvt_kvaddr(pd->subport_rcvegrbuf +
  1103. size * subport)) {
  1104. addr = pd->subport_rcvegrbuf + size * subport;
  1105. /* rcvegrbufs are read-only on the slave */
  1106. if (vma->vm_flags & VM_WRITE) {
  1107. dev_info(&dd->pcidev->dev,
  1108. "Can't map eager buffers as "
  1109. "writable (flags=%lx)\n", vma->vm_flags);
  1110. ret = -EPERM;
  1111. goto bail;
  1112. }
  1113. /*
  1114. * Don't allow permission to later change to writeable
  1115. * with mprotect.
  1116. */
  1117. vma->vm_flags &= ~VM_MAYWRITE;
  1118. } else {
  1119. goto bail;
  1120. }
  1121. len = vma->vm_end - vma->vm_start;
  1122. if (len > size) {
  1123. ipath_cdbg(MM, "FAIL: reqlen %lx > %zx\n", len, size);
  1124. ret = -EINVAL;
  1125. goto bail;
  1126. }
  1127. vma->vm_pgoff = (unsigned long) addr >> PAGE_SHIFT;
  1128. vma->vm_ops = &ipath_file_vm_ops;
  1129. vma->vm_flags |= VM_RESERVED | VM_DONTEXPAND;
  1130. ret = 1;
  1131. bail:
  1132. return ret;
  1133. }
  1134. /**
  1135. * ipath_mmap - mmap various structures into user space
  1136. * @fp: the file pointer
  1137. * @vma: the VM area
  1138. *
  1139. * We use this to have a shared buffer between the kernel and the user code
  1140. * for the rcvhdr queue, egr buffers, and the per-port user regs and pio
  1141. * buffers in the chip. We have the open and close entries so we can bump
  1142. * the ref count and keep the driver from being unloaded while still mapped.
  1143. */
  1144. static int ipath_mmap(struct file *fp, struct vm_area_struct *vma)
  1145. {
  1146. struct ipath_portdata *pd;
  1147. struct ipath_devdata *dd;
  1148. u64 pgaddr, ureg;
  1149. unsigned piobufs, piocnt;
  1150. int ret;
  1151. pd = port_fp(fp);
  1152. if (!pd) {
  1153. ret = -EINVAL;
  1154. goto bail;
  1155. }
  1156. dd = pd->port_dd;
  1157. /*
  1158. * This is the ipath_do_user_init() code, mapping the shared buffers
  1159. * into the user process. The address referred to by vm_pgoff is the
  1160. * file offset passed via mmap(). For shared ports, this is the
  1161. * kernel vmalloc() address of the pages to share with the master.
  1162. * For non-shared or master ports, this is a physical address.
  1163. * We only do one mmap for each space mapped.
  1164. */
  1165. pgaddr = vma->vm_pgoff << PAGE_SHIFT;
  1166. /*
  1167. * Check for 0 in case one of the allocations failed, but user
  1168. * called mmap anyway.
  1169. */
  1170. if (!pgaddr) {
  1171. ret = -EINVAL;
  1172. goto bail;
  1173. }
  1174. ipath_cdbg(MM, "pgaddr %llx vm_start=%lx len %lx port %u:%u:%u\n",
  1175. (unsigned long long) pgaddr, vma->vm_start,
  1176. vma->vm_end - vma->vm_start, dd->ipath_unit,
  1177. pd->port_port, subport_fp(fp));
  1178. /*
  1179. * Physical addresses must fit in 40 bits for our hardware.
  1180. * Check for kernel virtual addresses first, anything else must
  1181. * match a HW or memory address.
  1182. */
  1183. ret = mmap_kvaddr(vma, pgaddr, pd, subport_fp(fp));
  1184. if (ret) {
  1185. if (ret > 0)
  1186. ret = 0;
  1187. goto bail;
  1188. }
  1189. ureg = dd->ipath_uregbase + dd->ipath_ureg_align * pd->port_port;
  1190. if (!pd->port_subport_cnt) {
  1191. /* port is not shared */
  1192. piocnt = pd->port_piocnt;
  1193. piobufs = pd->port_piobufs;
  1194. } else if (!subport_fp(fp)) {
  1195. /* caller is the master */
  1196. piocnt = (pd->port_piocnt / pd->port_subport_cnt) +
  1197. (pd->port_piocnt % pd->port_subport_cnt);
  1198. piobufs = pd->port_piobufs +
  1199. dd->ipath_palign * (pd->port_piocnt - piocnt);
  1200. } else {
  1201. unsigned slave = subport_fp(fp) - 1;
  1202. /* caller is a slave */
  1203. piocnt = pd->port_piocnt / pd->port_subport_cnt;
  1204. piobufs = pd->port_piobufs + dd->ipath_palign * piocnt * slave;
  1205. }
  1206. if (pgaddr == ureg)
  1207. ret = mmap_ureg(vma, dd, ureg);
  1208. else if (pgaddr == piobufs)
  1209. ret = mmap_piobufs(vma, dd, pd, piobufs, piocnt);
  1210. else if (pgaddr == dd->ipath_pioavailregs_phys)
  1211. /* in-memory copy of pioavail registers */
  1212. ret = ipath_mmap_mem(vma, pd, PAGE_SIZE, 0,
  1213. (void *) dd->ipath_pioavailregs_dma,
  1214. "pioavail registers");
  1215. else if (pgaddr == pd->port_rcvegr_phys)
  1216. ret = mmap_rcvegrbufs(vma, pd);
  1217. else if (pgaddr == (u64) pd->port_rcvhdrq_phys)
  1218. /*
  1219. * The rcvhdrq itself; readonly except on HT (so have
  1220. * to allow writable mapping), multiple pages, contiguous
  1221. * from an i/o perspective.
  1222. */
  1223. ret = ipath_mmap_mem(vma, pd, pd->port_rcvhdrq_size, 1,
  1224. pd->port_rcvhdrq,
  1225. "rcvhdrq");
  1226. else if (pgaddr == (u64) pd->port_rcvhdrqtailaddr_phys)
  1227. /* in-memory copy of rcvhdrq tail register */
  1228. ret = ipath_mmap_mem(vma, pd, PAGE_SIZE, 0,
  1229. pd->port_rcvhdrtail_kvaddr,
  1230. "rcvhdrq tail");
  1231. else
  1232. ret = -EINVAL;
  1233. vma->vm_private_data = NULL;
  1234. if (ret < 0)
  1235. dev_info(&dd->pcidev->dev,
  1236. "Failure %d on off %llx len %lx\n",
  1237. -ret, (unsigned long long)pgaddr,
  1238. vma->vm_end - vma->vm_start);
  1239. bail:
  1240. return ret;
  1241. }
  1242. static unsigned ipath_poll_hdrqfull(struct ipath_portdata *pd)
  1243. {
  1244. unsigned pollflag = 0;
  1245. if ((pd->poll_type & IPATH_POLL_TYPE_OVERFLOW) &&
  1246. pd->port_hdrqfull != pd->port_hdrqfull_poll) {
  1247. pollflag |= POLLIN | POLLRDNORM;
  1248. pd->port_hdrqfull_poll = pd->port_hdrqfull;
  1249. }
  1250. return pollflag;
  1251. }
  1252. static unsigned int ipath_poll_urgent(struct ipath_portdata *pd,
  1253. struct file *fp,
  1254. struct poll_table_struct *pt)
  1255. {
  1256. unsigned pollflag = 0;
  1257. struct ipath_devdata *dd;
  1258. dd = pd->port_dd;
  1259. /* variable access in ipath_poll_hdrqfull() needs this */
  1260. rmb();
  1261. pollflag = ipath_poll_hdrqfull(pd);
  1262. if (pd->port_urgent != pd->port_urgent_poll) {
  1263. pollflag |= POLLIN | POLLRDNORM;
  1264. pd->port_urgent_poll = pd->port_urgent;
  1265. }
  1266. if (!pollflag) {
  1267. /* this saves a spin_lock/unlock in interrupt handler... */
  1268. set_bit(IPATH_PORT_WAITING_URG, &pd->port_flag);
  1269. /* flush waiting flag so don't miss an event... */
  1270. wmb();
  1271. poll_wait(fp, &pd->port_wait, pt);
  1272. }
  1273. return pollflag;
  1274. }
  1275. static unsigned int ipath_poll_next(struct ipath_portdata *pd,
  1276. struct file *fp,
  1277. struct poll_table_struct *pt)
  1278. {
  1279. u32 head;
  1280. u32 tail;
  1281. unsigned pollflag = 0;
  1282. struct ipath_devdata *dd;
  1283. dd = pd->port_dd;
  1284. /* variable access in ipath_poll_hdrqfull() needs this */
  1285. rmb();
  1286. pollflag = ipath_poll_hdrqfull(pd);
  1287. head = ipath_read_ureg32(dd, ur_rcvhdrhead, pd->port_port);
  1288. if (pd->port_rcvhdrtail_kvaddr)
  1289. tail = ipath_get_rcvhdrtail(pd);
  1290. else
  1291. tail = ipath_read_ureg32(dd, ur_rcvhdrtail, pd->port_port);
  1292. if (head != tail)
  1293. pollflag |= POLLIN | POLLRDNORM;
  1294. else {
  1295. /* this saves a spin_lock/unlock in interrupt handler */
  1296. set_bit(IPATH_PORT_WAITING_RCV, &pd->port_flag);
  1297. /* flush waiting flag so we don't miss an event */
  1298. wmb();
  1299. set_bit(pd->port_port + dd->ipath_r_intravail_shift,
  1300. &dd->ipath_rcvctrl);
  1301. ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvctrl,
  1302. dd->ipath_rcvctrl);
  1303. if (dd->ipath_rhdrhead_intr_off) /* arm rcv interrupt */
  1304. ipath_write_ureg(dd, ur_rcvhdrhead,
  1305. dd->ipath_rhdrhead_intr_off | head,
  1306. pd->port_port);
  1307. poll_wait(fp, &pd->port_wait, pt);
  1308. }
  1309. return pollflag;
  1310. }
  1311. static unsigned int ipath_poll(struct file *fp,
  1312. struct poll_table_struct *pt)
  1313. {
  1314. struct ipath_portdata *pd;
  1315. unsigned pollflag;
  1316. pd = port_fp(fp);
  1317. if (!pd)
  1318. pollflag = 0;
  1319. else if (pd->poll_type & IPATH_POLL_TYPE_URGENT)
  1320. pollflag = ipath_poll_urgent(pd, fp, pt);
  1321. else
  1322. pollflag = ipath_poll_next(pd, fp, pt);
  1323. return pollflag;
  1324. }
  1325. static int ipath_supports_subports(int user_swmajor, int user_swminor)
  1326. {
  1327. /* no subport implementation prior to software version 1.3 */
  1328. return (user_swmajor > 1) || (user_swminor >= 3);
  1329. }
  1330. static int ipath_compatible_subports(int user_swmajor, int user_swminor)
  1331. {
  1332. /* this code is written long-hand for clarity */
  1333. if (IPATH_USER_SWMAJOR != user_swmajor) {
  1334. /* no promise of compatibility if major mismatch */
  1335. return 0;
  1336. }
  1337. if (IPATH_USER_SWMAJOR == 1) {
  1338. switch (IPATH_USER_SWMINOR) {
  1339. case 0:
  1340. case 1:
  1341. case 2:
  1342. /* no subport implementation so cannot be compatible */
  1343. return 0;
  1344. case 3:
  1345. /* 3 is only compatible with itself */
  1346. return user_swminor == 3;
  1347. default:
  1348. /* >= 4 are compatible (or are expected to be) */
  1349. return user_swminor >= 4;
  1350. }
  1351. }
  1352. /* make no promises yet for future major versions */
  1353. return 0;
  1354. }
  1355. static int init_subports(struct ipath_devdata *dd,
  1356. struct ipath_portdata *pd,
  1357. const struct ipath_user_info *uinfo)
  1358. {
  1359. int ret = 0;
  1360. unsigned num_subports;
  1361. size_t size;
  1362. /*
  1363. * If the user is requesting zero subports,
  1364. * skip the subport allocation.
  1365. */
  1366. if (uinfo->spu_subport_cnt <= 0)
  1367. goto bail;
  1368. /* Self-consistency check for ipath_compatible_subports() */
  1369. if (ipath_supports_subports(IPATH_USER_SWMAJOR, IPATH_USER_SWMINOR) &&
  1370. !ipath_compatible_subports(IPATH_USER_SWMAJOR,
  1371. IPATH_USER_SWMINOR)) {
  1372. dev_info(&dd->pcidev->dev,
  1373. "Inconsistent ipath_compatible_subports()\n");
  1374. goto bail;
  1375. }
  1376. /* Check for subport compatibility */
  1377. if (!ipath_compatible_subports(uinfo->spu_userversion >> 16,
  1378. uinfo->spu_userversion & 0xffff)) {
  1379. dev_info(&dd->pcidev->dev,
  1380. "Mismatched user version (%d.%d) and driver "
  1381. "version (%d.%d) while port sharing. Ensure "
  1382. "that driver and library are from the same "
  1383. "release.\n",
  1384. (int) (uinfo->spu_userversion >> 16),
  1385. (int) (uinfo->spu_userversion & 0xffff),
  1386. IPATH_USER_SWMAJOR,
  1387. IPATH_USER_SWMINOR);
  1388. goto bail;
  1389. }
  1390. if (uinfo->spu_subport_cnt > INFINIPATH_MAX_SUBPORT) {
  1391. ret = -EINVAL;
  1392. goto bail;
  1393. }
  1394. num_subports = uinfo->spu_subport_cnt;
  1395. pd->subport_uregbase = vmalloc(PAGE_SIZE * num_subports);
  1396. if (!pd->subport_uregbase) {
  1397. ret = -ENOMEM;
  1398. goto bail;
  1399. }
  1400. /* Note: pd->port_rcvhdrq_size isn't initialized yet. */
  1401. size = ALIGN(dd->ipath_rcvhdrcnt * dd->ipath_rcvhdrentsize *
  1402. sizeof(u32), PAGE_SIZE) * num_subports;
  1403. pd->subport_rcvhdr_base = vmalloc(size);
  1404. if (!pd->subport_rcvhdr_base) {
  1405. ret = -ENOMEM;
  1406. goto bail_ureg;
  1407. }
  1408. pd->subport_rcvegrbuf = vmalloc(pd->port_rcvegrbuf_chunks *
  1409. pd->port_rcvegrbuf_size *
  1410. num_subports);
  1411. if (!pd->subport_rcvegrbuf) {
  1412. ret = -ENOMEM;
  1413. goto bail_rhdr;
  1414. }
  1415. pd->port_subport_cnt = uinfo->spu_subport_cnt;
  1416. pd->port_subport_id = uinfo->spu_subport_id;
  1417. pd->active_slaves = 1;
  1418. set_bit(IPATH_PORT_MASTER_UNINIT, &pd->port_flag);
  1419. memset(pd->subport_uregbase, 0, PAGE_SIZE * num_subports);
  1420. memset(pd->subport_rcvhdr_base, 0, size);
  1421. memset(pd->subport_rcvegrbuf, 0, pd->port_rcvegrbuf_chunks *
  1422. pd->port_rcvegrbuf_size *
  1423. num_subports);
  1424. goto bail;
  1425. bail_rhdr:
  1426. vfree(pd->subport_rcvhdr_base);
  1427. bail_ureg:
  1428. vfree(pd->subport_uregbase);
  1429. pd->subport_uregbase = NULL;
  1430. bail:
  1431. return ret;
  1432. }
  1433. static int try_alloc_port(struct ipath_devdata *dd, int port,
  1434. struct file *fp,
  1435. const struct ipath_user_info *uinfo)
  1436. {
  1437. struct ipath_portdata *pd;
  1438. int ret;
  1439. if (!(pd = dd->ipath_pd[port])) {
  1440. void *ptmp;
  1441. pd = kzalloc(sizeof(struct ipath_portdata), GFP_KERNEL);
  1442. /*
  1443. * Allocate memory for use in ipath_tid_update() just once
  1444. * at open, not per call. Reduces cost of expected send
  1445. * setup.
  1446. */
  1447. ptmp = kmalloc(dd->ipath_rcvtidcnt * sizeof(u16) +
  1448. dd->ipath_rcvtidcnt * sizeof(struct page **),
  1449. GFP_KERNEL);
  1450. if (!pd || !ptmp) {
  1451. ipath_dev_err(dd, "Unable to allocate portdata "
  1452. "memory, failing open\n");
  1453. ret = -ENOMEM;
  1454. kfree(pd);
  1455. kfree(ptmp);
  1456. goto bail;
  1457. }
  1458. dd->ipath_pd[port] = pd;
  1459. dd->ipath_pd[port]->port_port = port;
  1460. dd->ipath_pd[port]->port_dd = dd;
  1461. dd->ipath_pd[port]->port_tid_pg_list = ptmp;
  1462. init_waitqueue_head(&dd->ipath_pd[port]->port_wait);
  1463. }
  1464. if (!pd->port_cnt) {
  1465. pd->userversion = uinfo->spu_userversion;
  1466. init_user_egr_sizes(pd);
  1467. if ((ret = init_subports(dd, pd, uinfo)) != 0)
  1468. goto bail;
  1469. ipath_cdbg(PROC, "%s[%u] opened unit:port %u:%u\n",
  1470. current->comm, current->pid, dd->ipath_unit,
  1471. port);
  1472. pd->port_cnt = 1;
  1473. port_fp(fp) = pd;
  1474. pd->port_pid = get_pid(task_pid(current));
  1475. strlcpy(pd->port_comm, current->comm, sizeof(pd->port_comm));
  1476. ipath_stats.sps_ports++;
  1477. ret = 0;
  1478. } else
  1479. ret = -EBUSY;
  1480. bail:
  1481. return ret;
  1482. }
  1483. static inline int usable(struct ipath_devdata *dd)
  1484. {
  1485. return dd &&
  1486. (dd->ipath_flags & IPATH_PRESENT) &&
  1487. dd->ipath_kregbase &&
  1488. dd->ipath_lid &&
  1489. !(dd->ipath_flags & (IPATH_LINKDOWN | IPATH_DISABLED
  1490. | IPATH_LINKUNK));
  1491. }
  1492. static int find_free_port(int unit, struct file *fp,
  1493. const struct ipath_user_info *uinfo)
  1494. {
  1495. struct ipath_devdata *dd = ipath_lookup(unit);
  1496. int ret, i;
  1497. if (!dd) {
  1498. ret = -ENODEV;
  1499. goto bail;
  1500. }
  1501. if (!usable(dd)) {
  1502. ret = -ENETDOWN;
  1503. goto bail;
  1504. }
  1505. for (i = 1; i < dd->ipath_cfgports; i++) {
  1506. ret = try_alloc_port(dd, i, fp, uinfo);
  1507. if (ret != -EBUSY)
  1508. goto bail;
  1509. }
  1510. ret = -EBUSY;
  1511. bail:
  1512. return ret;
  1513. }
  1514. static int find_best_unit(struct file *fp,
  1515. const struct ipath_user_info *uinfo)
  1516. {
  1517. int ret = 0, i, prefunit = -1, devmax;
  1518. int maxofallports, npresent, nup;
  1519. int ndev;
  1520. devmax = ipath_count_units(&npresent, &nup, &maxofallports);
  1521. /*
  1522. * This code is present to allow a knowledgeable person to
  1523. * specify the layout of processes to processors before opening
  1524. * this driver, and then we'll assign the process to the "closest"
  1525. * InfiniPath chip to that processor (we assume reasonable connectivity,
  1526. * for now). This code assumes that if affinity has been set
  1527. * before this point, that at most one cpu is set; for now this
  1528. * is reasonable. I check for both cpumask_empty() and cpumask_full(),
  1529. * in case some kernel variant sets none of the bits when no
  1530. * affinity is set. 2.6.11 and 12 kernels have all present
  1531. * cpus set. Some day we'll have to fix it up further to handle
  1532. * a cpu subset. This algorithm fails for two HT chips connected
  1533. * in…