PageRenderTime 51ms CodeModel.GetById 11ms RepoModel.GetById 1ms app.codeStats 0ms

/drivers/infiniband/hw/ipath/ipath_file_ops.c

https://bitbucket.org/cresqo/cm7-p500-kernel
C | 2617 lines | 1894 code | 278 blank | 445 comment | 306 complexity | f6751c75abf34c137f8b85179b5485de MD5 | raw file
Possible License(s): LGPL-2.0, AGPL-1.0, GPL-2.0
  1. /*
  2. * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
  3. * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
  4. *
  5. * This software is available to you under a choice of one of two
  6. * licenses. You may choose to be licensed under the terms of the GNU
  7. * General Public License (GPL) Version 2, available from the file
  8. * COPYING in the main directory of this source tree, or the
  9. * OpenIB.org BSD license below:
  10. *
  11. * Redistribution and use in source and binary forms, with or
  12. * without modification, are permitted provided that the following
  13. * conditions are met:
  14. *
  15. * - Redistributions of source code must retain the above
  16. * copyright notice, this list of conditions and the following
  17. * disclaimer.
  18. *
  19. * - Redistributions in binary form must reproduce the above
  20. * copyright notice, this list of conditions and the following
  21. * disclaimer in the documentation and/or other materials
  22. * provided with the distribution.
  23. *
  24. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31. * SOFTWARE.
  32. */
  33. #include <linux/pci.h>
  34. #include <linux/poll.h>
  35. #include <linux/cdev.h>
  36. #include <linux/swap.h>
  37. #include <linux/vmalloc.h>
  38. #include <linux/slab.h>
  39. #include <linux/highmem.h>
  40. #include <linux/io.h>
  41. #include <linux/jiffies.h>
  42. #include <linux/smp_lock.h>
  43. #include <asm/pgtable.h>
  44. #include "ipath_kernel.h"
  45. #include "ipath_common.h"
  46. #include "ipath_user_sdma.h"
  47. static int ipath_open(struct inode *, struct file *);
  48. static int ipath_close(struct inode *, struct file *);
  49. static ssize_t ipath_write(struct file *, const char __user *, size_t,
  50. loff_t *);
  51. static ssize_t ipath_writev(struct kiocb *, const struct iovec *,
  52. unsigned long , loff_t);
  53. static unsigned int ipath_poll(struct file *, struct poll_table_struct *);
  54. static int ipath_mmap(struct file *, struct vm_area_struct *);
  55. static const struct file_operations ipath_file_ops = {
  56. .owner = THIS_MODULE,
  57. .write = ipath_write,
  58. .aio_write = ipath_writev,
  59. .open = ipath_open,
  60. .release = ipath_close,
  61. .poll = ipath_poll,
  62. .mmap = ipath_mmap
  63. };
  64. /*
  65. * Convert kernel virtual addresses to physical addresses so they don't
  66. * potentially conflict with the chip addresses used as mmap offsets.
  67. * It doesn't really matter what mmap offset we use as long as we can
  68. * interpret it correctly.
  69. */
  70. static u64 cvt_kvaddr(void *p)
  71. {
  72. struct page *page;
  73. u64 paddr = 0;
  74. page = vmalloc_to_page(p);
  75. if (page)
  76. paddr = page_to_pfn(page) << PAGE_SHIFT;
  77. return paddr;
  78. }
  79. static int ipath_get_base_info(struct file *fp,
  80. void __user *ubase, size_t ubase_size)
  81. {
  82. struct ipath_portdata *pd = port_fp(fp);
  83. int ret = 0;
  84. struct ipath_base_info *kinfo = NULL;
  85. struct ipath_devdata *dd = pd->port_dd;
  86. unsigned subport_cnt;
  87. int shared, master;
  88. size_t sz;
  89. subport_cnt = pd->port_subport_cnt;
  90. if (!subport_cnt) {
  91. shared = 0;
  92. master = 0;
  93. subport_cnt = 1;
  94. } else {
  95. shared = 1;
  96. master = !subport_fp(fp);
  97. }
  98. sz = sizeof(*kinfo);
  99. /* If port sharing is not requested, allow the old size structure */
  100. if (!shared)
  101. sz -= 7 * sizeof(u64);
  102. if (ubase_size < sz) {
  103. ipath_cdbg(PROC,
  104. "Base size %zu, need %zu (version mismatch?)\n",
  105. ubase_size, sz);
  106. ret = -EINVAL;
  107. goto bail;
  108. }
  109. kinfo = kzalloc(sizeof(*kinfo), GFP_KERNEL);
  110. if (kinfo == NULL) {
  111. ret = -ENOMEM;
  112. goto bail;
  113. }
  114. ret = dd->ipath_f_get_base_info(pd, kinfo);
  115. if (ret < 0)
  116. goto bail;
  117. kinfo->spi_rcvhdr_cnt = dd->ipath_rcvhdrcnt;
  118. kinfo->spi_rcvhdrent_size = dd->ipath_rcvhdrentsize;
  119. kinfo->spi_tidegrcnt = dd->ipath_rcvegrcnt;
  120. kinfo->spi_rcv_egrbufsize = dd->ipath_rcvegrbufsize;
  121. /*
  122. * have to mmap whole thing
  123. */
  124. kinfo->spi_rcv_egrbuftotlen =
  125. pd->port_rcvegrbuf_chunks * pd->port_rcvegrbuf_size;
  126. kinfo->spi_rcv_egrperchunk = pd->port_rcvegrbufs_perchunk;
  127. kinfo->spi_rcv_egrchunksize = kinfo->spi_rcv_egrbuftotlen /
  128. pd->port_rcvegrbuf_chunks;
  129. kinfo->spi_tidcnt = dd->ipath_rcvtidcnt / subport_cnt;
  130. if (master)
  131. kinfo->spi_tidcnt += dd->ipath_rcvtidcnt % subport_cnt;
  132. /*
  133. * for this use, may be ipath_cfgports summed over all chips that
  134. * are are configured and present
  135. */
  136. kinfo->spi_nports = dd->ipath_cfgports;
  137. /* unit (chip/board) our port is on */
  138. kinfo->spi_unit = dd->ipath_unit;
  139. /* for now, only a single page */
  140. kinfo->spi_tid_maxsize = PAGE_SIZE;
  141. /*
  142. * Doing this per port, and based on the skip value, etc. This has
  143. * to be the actual buffer size, since the protocol code treats it
  144. * as an array.
  145. *
  146. * These have to be set to user addresses in the user code via mmap.
  147. * These values are used on return to user code for the mmap target
  148. * addresses only. For 32 bit, same 44 bit address problem, so use
  149. * the physical address, not virtual. Before 2.6.11, using the
  150. * page_address() macro worked, but in 2.6.11, even that returns the
  151. * full 64 bit address (upper bits all 1's). So far, using the
  152. * physical addresses (or chip offsets, for chip mapping) works, but
  153. * no doubt some future kernel release will change that, and we'll be
  154. * on to yet another method of dealing with this.
  155. */
  156. kinfo->spi_rcvhdr_base = (u64) pd->port_rcvhdrq_phys;
  157. kinfo->spi_rcvhdr_tailaddr = (u64) pd->port_rcvhdrqtailaddr_phys;
  158. kinfo->spi_rcv_egrbufs = (u64) pd->port_rcvegr_phys;
  159. kinfo->spi_pioavailaddr = (u64) dd->ipath_pioavailregs_phys;
  160. kinfo->spi_status = (u64) kinfo->spi_pioavailaddr +
  161. (void *) dd->ipath_statusp -
  162. (void *) dd->ipath_pioavailregs_dma;
  163. if (!shared) {
  164. kinfo->spi_piocnt = pd->port_piocnt;
  165. kinfo->spi_piobufbase = (u64) pd->port_piobufs;
  166. kinfo->__spi_uregbase = (u64) dd->ipath_uregbase +
  167. dd->ipath_ureg_align * pd->port_port;
  168. } else if (master) {
  169. kinfo->spi_piocnt = (pd->port_piocnt / subport_cnt) +
  170. (pd->port_piocnt % subport_cnt);
  171. /* Master's PIO buffers are after all the slave's */
  172. kinfo->spi_piobufbase = (u64) pd->port_piobufs +
  173. dd->ipath_palign *
  174. (pd->port_piocnt - kinfo->spi_piocnt);
  175. } else {
  176. unsigned slave = subport_fp(fp) - 1;
  177. kinfo->spi_piocnt = pd->port_piocnt / subport_cnt;
  178. kinfo->spi_piobufbase = (u64) pd->port_piobufs +
  179. dd->ipath_palign * kinfo->spi_piocnt * slave;
  180. }
  181. if (shared) {
  182. kinfo->spi_port_uregbase = (u64) dd->ipath_uregbase +
  183. dd->ipath_ureg_align * pd->port_port;
  184. kinfo->spi_port_rcvegrbuf = kinfo->spi_rcv_egrbufs;
  185. kinfo->spi_port_rcvhdr_base = kinfo->spi_rcvhdr_base;
  186. kinfo->spi_port_rcvhdr_tailaddr = kinfo->spi_rcvhdr_tailaddr;
  187. kinfo->__spi_uregbase = cvt_kvaddr(pd->subport_uregbase +
  188. PAGE_SIZE * subport_fp(fp));
  189. kinfo->spi_rcvhdr_base = cvt_kvaddr(pd->subport_rcvhdr_base +
  190. pd->port_rcvhdrq_size * subport_fp(fp));
  191. kinfo->spi_rcvhdr_tailaddr = 0;
  192. kinfo->spi_rcv_egrbufs = cvt_kvaddr(pd->subport_rcvegrbuf +
  193. pd->port_rcvegrbuf_chunks * pd->port_rcvegrbuf_size *
  194. subport_fp(fp));
  195. kinfo->spi_subport_uregbase =
  196. cvt_kvaddr(pd->subport_uregbase);
  197. kinfo->spi_subport_rcvegrbuf =
  198. cvt_kvaddr(pd->subport_rcvegrbuf);
  199. kinfo->spi_subport_rcvhdr_base =
  200. cvt_kvaddr(pd->subport_rcvhdr_base);
  201. ipath_cdbg(PROC, "port %u flags %x %llx %llx %llx\n",
  202. kinfo->spi_port, kinfo->spi_runtime_flags,
  203. (unsigned long long) kinfo->spi_subport_uregbase,
  204. (unsigned long long) kinfo->spi_subport_rcvegrbuf,
  205. (unsigned long long) kinfo->spi_subport_rcvhdr_base);
  206. }
  207. /*
  208. * All user buffers are 2KB buffers. If we ever support
  209. * giving 4KB buffers to user processes, this will need some
  210. * work.
  211. */
  212. kinfo->spi_pioindex = (kinfo->spi_piobufbase -
  213. (dd->ipath_piobufbase & 0xffffffff)) / dd->ipath_palign;
  214. kinfo->spi_pioalign = dd->ipath_palign;
  215. kinfo->spi_qpair = IPATH_KD_QP;
  216. /*
  217. * user mode PIO buffers are always 2KB, even when 4KB can
  218. * be received, and sent via the kernel; this is ibmaxlen
  219. * for 2K MTU.
  220. */
  221. kinfo->spi_piosize = dd->ipath_piosize2k - 2 * sizeof(u32);
  222. kinfo->spi_mtu = dd->ipath_ibmaxlen; /* maxlen, not ibmtu */
  223. kinfo->spi_port = pd->port_port;
  224. kinfo->spi_subport = subport_fp(fp);
  225. kinfo->spi_sw_version = IPATH_KERN_SWVERSION;
  226. kinfo->spi_hw_version = dd->ipath_revision;
  227. if (master) {
  228. kinfo->spi_runtime_flags |= IPATH_RUNTIME_MASTER;
  229. }
  230. sz = (ubase_size < sizeof(*kinfo)) ? ubase_size : sizeof(*kinfo);
  231. if (copy_to_user(ubase, kinfo, sz))
  232. ret = -EFAULT;
  233. bail:
  234. kfree(kinfo);
  235. return ret;
  236. }
  237. /**
  238. * ipath_tid_update - update a port TID
  239. * @pd: the port
  240. * @fp: the ipath device file
  241. * @ti: the TID information
  242. *
  243. * The new implementation as of Oct 2004 is that the driver assigns
  244. * the tid and returns it to the caller. To make it easier to
  245. * catch bugs, and to reduce search time, we keep a cursor for
  246. * each port, walking the shadow tid array to find one that's not
  247. * in use.
  248. *
  249. * For now, if we can't allocate the full list, we fail, although
  250. * in the long run, we'll allocate as many as we can, and the
  251. * caller will deal with that by trying the remaining pages later.
  252. * That means that when we fail, we have to mark the tids as not in
  253. * use again, in our shadow copy.
  254. *
  255. * It's up to the caller to free the tids when they are done.
  256. * We'll unlock the pages as they free them.
  257. *
  258. * Also, right now we are locking one page at a time, but since
  259. * the intended use of this routine is for a single group of
  260. * virtually contiguous pages, that should change to improve
  261. * performance.
  262. */
  263. static int ipath_tid_update(struct ipath_portdata *pd, struct file *fp,
  264. const struct ipath_tid_info *ti)
  265. {
  266. int ret = 0, ntids;
  267. u32 tid, porttid, cnt, i, tidcnt, tidoff;
  268. u16 *tidlist;
  269. struct ipath_devdata *dd = pd->port_dd;
  270. u64 physaddr;
  271. unsigned long vaddr;
  272. u64 __iomem *tidbase;
  273. unsigned long tidmap[8];
  274. struct page **pagep = NULL;
  275. unsigned subport = subport_fp(fp);
  276. if (!dd->ipath_pageshadow) {
  277. ret = -ENOMEM;
  278. goto done;
  279. }
  280. cnt = ti->tidcnt;
  281. if (!cnt) {
  282. ipath_dbg("After copyin, tidcnt 0, tidlist %llx\n",
  283. (unsigned long long) ti->tidlist);
  284. /*
  285. * Should we treat as success? likely a bug
  286. */
  287. ret = -EFAULT;
  288. goto done;
  289. }
  290. porttid = pd->port_port * dd->ipath_rcvtidcnt;
  291. if (!pd->port_subport_cnt) {
  292. tidcnt = dd->ipath_rcvtidcnt;
  293. tid = pd->port_tidcursor;
  294. tidoff = 0;
  295. } else if (!subport) {
  296. tidcnt = (dd->ipath_rcvtidcnt / pd->port_subport_cnt) +
  297. (dd->ipath_rcvtidcnt % pd->port_subport_cnt);
  298. tidoff = dd->ipath_rcvtidcnt - tidcnt;
  299. porttid += tidoff;
  300. tid = tidcursor_fp(fp);
  301. } else {
  302. tidcnt = dd->ipath_rcvtidcnt / pd->port_subport_cnt;
  303. tidoff = tidcnt * (subport - 1);
  304. porttid += tidoff;
  305. tid = tidcursor_fp(fp);
  306. }
  307. if (cnt > tidcnt) {
  308. /* make sure it all fits in port_tid_pg_list */
  309. dev_info(&dd->pcidev->dev, "Process tried to allocate %u "
  310. "TIDs, only trying max (%u)\n", cnt, tidcnt);
  311. cnt = tidcnt;
  312. }
  313. pagep = &((struct page **) pd->port_tid_pg_list)[tidoff];
  314. tidlist = &((u16 *) &pagep[dd->ipath_rcvtidcnt])[tidoff];
  315. memset(tidmap, 0, sizeof(tidmap));
  316. /* before decrement; chip actual # */
  317. ntids = tidcnt;
  318. tidbase = (u64 __iomem *) (((char __iomem *) dd->ipath_kregbase) +
  319. dd->ipath_rcvtidbase +
  320. porttid * sizeof(*tidbase));
  321. ipath_cdbg(VERBOSE, "Port%u %u tids, cursor %u, tidbase %p\n",
  322. pd->port_port, cnt, tid, tidbase);
  323. /* virtual address of first page in transfer */
  324. vaddr = ti->tidvaddr;
  325. if (!access_ok(VERIFY_WRITE, (void __user *) vaddr,
  326. cnt * PAGE_SIZE)) {
  327. ipath_dbg("Fail vaddr %p, %u pages, !access_ok\n",
  328. (void *)vaddr, cnt);
  329. ret = -EFAULT;
  330. goto done;
  331. }
  332. ret = ipath_get_user_pages(vaddr, cnt, pagep);
  333. if (ret) {
  334. if (ret == -EBUSY) {
  335. ipath_dbg("Failed to lock addr %p, %u pages "
  336. "(already locked)\n",
  337. (void *) vaddr, cnt);
  338. /*
  339. * for now, continue, and see what happens but with
  340. * the new implementation, this should never happen,
  341. * unless perhaps the user has mpin'ed the pages
  342. * themselves (something we need to test)
  343. */
  344. ret = 0;
  345. } else {
  346. dev_info(&dd->pcidev->dev,
  347. "Failed to lock addr %p, %u pages: "
  348. "errno %d\n", (void *) vaddr, cnt, -ret);
  349. goto done;
  350. }
  351. }
  352. for (i = 0; i < cnt; i++, vaddr += PAGE_SIZE) {
  353. for (; ntids--; tid++) {
  354. if (tid == tidcnt)
  355. tid = 0;
  356. if (!dd->ipath_pageshadow[porttid + tid])
  357. break;
  358. }
  359. if (ntids < 0) {
  360. /*
  361. * oops, wrapped all the way through their TIDs,
  362. * and didn't have enough free; see comments at
  363. * start of routine
  364. */
  365. ipath_dbg("Not enough free TIDs for %u pages "
  366. "(index %d), failing\n", cnt, i);
  367. i--; /* last tidlist[i] not filled in */
  368. ret = -ENOMEM;
  369. break;
  370. }
  371. tidlist[i] = tid + tidoff;
  372. ipath_cdbg(VERBOSE, "Updating idx %u to TID %u, "
  373. "vaddr %lx\n", i, tid + tidoff, vaddr);
  374. /* we "know" system pages and TID pages are same size */
  375. dd->ipath_pageshadow[porttid + tid] = pagep[i];
  376. dd->ipath_physshadow[porttid + tid] = ipath_map_page(
  377. dd->pcidev, pagep[i], 0, PAGE_SIZE,
  378. PCI_DMA_FROMDEVICE);
  379. /*
  380. * don't need atomic or it's overhead
  381. */
  382. __set_bit(tid, tidmap);
  383. physaddr = dd->ipath_physshadow[porttid + tid];
  384. ipath_stats.sps_pagelocks++;
  385. ipath_cdbg(VERBOSE,
  386. "TID %u, vaddr %lx, physaddr %llx pgp %p\n",
  387. tid, vaddr, (unsigned long long) physaddr,
  388. pagep[i]);
  389. dd->ipath_f_put_tid(dd, &tidbase[tid], RCVHQ_RCV_TYPE_EXPECTED,
  390. physaddr);
  391. /*
  392. * don't check this tid in ipath_portshadow, since we
  393. * just filled it in; start with the next one.
  394. */
  395. tid++;
  396. }
  397. if (ret) {
  398. u32 limit;
  399. cleanup:
  400. /* jump here if copy out of updated info failed... */
  401. ipath_dbg("After failure (ret=%d), undo %d of %d entries\n",
  402. -ret, i, cnt);
  403. /* same code that's in ipath_free_tid() */
  404. limit = sizeof(tidmap) * BITS_PER_BYTE;
  405. if (limit > tidcnt)
  406. /* just in case size changes in future */
  407. limit = tidcnt;
  408. tid = find_first_bit((const unsigned long *)tidmap, limit);
  409. for (; tid < limit; tid++) {
  410. if (!test_bit(tid, tidmap))
  411. continue;
  412. if (dd->ipath_pageshadow[porttid + tid]) {
  413. ipath_cdbg(VERBOSE, "Freeing TID %u\n",
  414. tid);
  415. dd->ipath_f_put_tid(dd, &tidbase[tid],
  416. RCVHQ_RCV_TYPE_EXPECTED,
  417. dd->ipath_tidinvalid);
  418. pci_unmap_page(dd->pcidev,
  419. dd->ipath_physshadow[porttid + tid],
  420. PAGE_SIZE, PCI_DMA_FROMDEVICE);
  421. dd->ipath_pageshadow[porttid + tid] = NULL;
  422. ipath_stats.sps_pageunlocks++;
  423. }
  424. }
  425. ipath_release_user_pages(pagep, cnt);
  426. } else {
  427. /*
  428. * Copy the updated array, with ipath_tid's filled in, back
  429. * to user. Since we did the copy in already, this "should
  430. * never fail" If it does, we have to clean up...
  431. */
  432. if (copy_to_user((void __user *)
  433. (unsigned long) ti->tidlist,
  434. tidlist, cnt * sizeof(*tidlist))) {
  435. ret = -EFAULT;
  436. goto cleanup;
  437. }
  438. if (copy_to_user((void __user *) (unsigned long) ti->tidmap,
  439. tidmap, sizeof tidmap)) {
  440. ret = -EFAULT;
  441. goto cleanup;
  442. }
  443. if (tid == tidcnt)
  444. tid = 0;
  445. if (!pd->port_subport_cnt)
  446. pd->port_tidcursor = tid;
  447. else
  448. tidcursor_fp(fp) = tid;
  449. }
  450. done:
  451. if (ret)
  452. ipath_dbg("Failed to map %u TID pages, failing with %d\n",
  453. ti->tidcnt, -ret);
  454. return ret;
  455. }
  456. /**
  457. * ipath_tid_free - free a port TID
  458. * @pd: the port
  459. * @subport: the subport
  460. * @ti: the TID info
  461. *
  462. * right now we are unlocking one page at a time, but since
  463. * the intended use of this routine is for a single group of
  464. * virtually contiguous pages, that should change to improve
  465. * performance. We check that the TID is in range for this port
  466. * but otherwise don't check validity; if user has an error and
  467. * frees the wrong tid, it's only their own data that can thereby
  468. * be corrupted. We do check that the TID was in use, for sanity
  469. * We always use our idea of the saved address, not the address that
  470. * they pass in to us.
  471. */
  472. static int ipath_tid_free(struct ipath_portdata *pd, unsigned subport,
  473. const struct ipath_tid_info *ti)
  474. {
  475. int ret = 0;
  476. u32 tid, porttid, cnt, limit, tidcnt;
  477. struct ipath_devdata *dd = pd->port_dd;
  478. u64 __iomem *tidbase;
  479. unsigned long tidmap[8];
  480. if (!dd->ipath_pageshadow) {
  481. ret = -ENOMEM;
  482. goto done;
  483. }
  484. if (copy_from_user(tidmap, (void __user *)(unsigned long)ti->tidmap,
  485. sizeof tidmap)) {
  486. ret = -EFAULT;
  487. goto done;
  488. }
  489. porttid = pd->port_port * dd->ipath_rcvtidcnt;
  490. if (!pd->port_subport_cnt)
  491. tidcnt = dd->ipath_rcvtidcnt;
  492. else if (!subport) {
  493. tidcnt = (dd->ipath_rcvtidcnt / pd->port_subport_cnt) +
  494. (dd->ipath_rcvtidcnt % pd->port_subport_cnt);
  495. porttid += dd->ipath_rcvtidcnt - tidcnt;
  496. } else {
  497. tidcnt = dd->ipath_rcvtidcnt / pd->port_subport_cnt;
  498. porttid += tidcnt * (subport - 1);
  499. }
  500. tidbase = (u64 __iomem *) ((char __iomem *)(dd->ipath_kregbase) +
  501. dd->ipath_rcvtidbase +
  502. porttid * sizeof(*tidbase));
  503. limit = sizeof(tidmap) * BITS_PER_BYTE;
  504. if (limit > tidcnt)
  505. /* just in case size changes in future */
  506. limit = tidcnt;
  507. tid = find_first_bit(tidmap, limit);
  508. ipath_cdbg(VERBOSE, "Port%u free %u tids; first bit (max=%d) "
  509. "set is %d, porttid %u\n", pd->port_port, ti->tidcnt,
  510. limit, tid, porttid);
  511. for (cnt = 0; tid < limit; tid++) {
  512. /*
  513. * small optimization; if we detect a run of 3 or so without
  514. * any set, use find_first_bit again. That's mainly to
  515. * accelerate the case where we wrapped, so we have some at
  516. * the beginning, and some at the end, and a big gap
  517. * in the middle.
  518. */
  519. if (!test_bit(tid, tidmap))
  520. continue;
  521. cnt++;
  522. if (dd->ipath_pageshadow[porttid + tid]) {
  523. struct page *p;
  524. p = dd->ipath_pageshadow[porttid + tid];
  525. dd->ipath_pageshadow[porttid + tid] = NULL;
  526. ipath_cdbg(VERBOSE, "PID %u freeing TID %u\n",
  527. pid_nr(pd->port_pid), tid);
  528. dd->ipath_f_put_tid(dd, &tidbase[tid],
  529. RCVHQ_RCV_TYPE_EXPECTED,
  530. dd->ipath_tidinvalid);
  531. pci_unmap_page(dd->pcidev,
  532. dd->ipath_physshadow[porttid + tid],
  533. PAGE_SIZE, PCI_DMA_FROMDEVICE);
  534. ipath_release_user_pages(&p, 1);
  535. ipath_stats.sps_pageunlocks++;
  536. } else
  537. ipath_dbg("Unused tid %u, ignoring\n", tid);
  538. }
  539. if (cnt != ti->tidcnt)
  540. ipath_dbg("passed in tidcnt %d, only %d bits set in map\n",
  541. ti->tidcnt, cnt);
  542. done:
  543. if (ret)
  544. ipath_dbg("Failed to unmap %u TID pages, failing with %d\n",
  545. ti->tidcnt, -ret);
  546. return ret;
  547. }
  548. /**
  549. * ipath_set_part_key - set a partition key
  550. * @pd: the port
  551. * @key: the key
  552. *
  553. * We can have up to 4 active at a time (other than the default, which is
  554. * always allowed). This is somewhat tricky, since multiple ports may set
  555. * the same key, so we reference count them, and clean up at exit. All 4
  556. * partition keys are packed into a single infinipath register. It's an
  557. * error for a process to set the same pkey multiple times. We provide no
  558. * mechanism to de-allocate a pkey at this time, we may eventually need to
  559. * do that. I've used the atomic operations, and no locking, and only make
  560. * a single pass through what's available. This should be more than
  561. * adequate for some time. I'll think about spinlocks or the like if and as
  562. * it's necessary.
  563. */
  564. static int ipath_set_part_key(struct ipath_portdata *pd, u16 key)
  565. {
  566. struct ipath_devdata *dd = pd->port_dd;
  567. int i, any = 0, pidx = -1;
  568. u16 lkey = key & 0x7FFF;
  569. int ret;
  570. if (lkey == (IPATH_DEFAULT_P_KEY & 0x7FFF)) {
  571. /* nothing to do; this key always valid */
  572. ret = 0;
  573. goto bail;
  574. }
  575. ipath_cdbg(VERBOSE, "p%u try to set pkey %hx, current keys "
  576. "%hx:%x %hx:%x %hx:%x %hx:%x\n",
  577. pd->port_port, key, dd->ipath_pkeys[0],
  578. atomic_read(&dd->ipath_pkeyrefs[0]), dd->ipath_pkeys[1],
  579. atomic_read(&dd->ipath_pkeyrefs[1]), dd->ipath_pkeys[2],
  580. atomic_read(&dd->ipath_pkeyrefs[2]), dd->ipath_pkeys[3],
  581. atomic_read(&dd->ipath_pkeyrefs[3]));
  582. if (!lkey) {
  583. ipath_cdbg(PROC, "p%u tries to set key 0, not allowed\n",
  584. pd->port_port);
  585. ret = -EINVAL;
  586. goto bail;
  587. }
  588. /*
  589. * Set the full membership bit, because it has to be
  590. * set in the register or the packet, and it seems
  591. * cleaner to set in the register than to force all
  592. * callers to set it. (see bug 4331)
  593. */
  594. key |= 0x8000;
  595. for (i = 0; i < ARRAY_SIZE(pd->port_pkeys); i++) {
  596. if (!pd->port_pkeys[i] && pidx == -1)
  597. pidx = i;
  598. if (pd->port_pkeys[i] == key) {
  599. ipath_cdbg(VERBOSE, "p%u tries to set same pkey "
  600. "(%x) more than once\n",
  601. pd->port_port, key);
  602. ret = -EEXIST;
  603. goto bail;
  604. }
  605. }
  606. if (pidx == -1) {
  607. ipath_dbg("All pkeys for port %u already in use, "
  608. "can't set %x\n", pd->port_port, key);
  609. ret = -EBUSY;
  610. goto bail;
  611. }
  612. for (any = i = 0; i < ARRAY_SIZE(dd->ipath_pkeys); i++) {
  613. if (!dd->ipath_pkeys[i]) {
  614. any++;
  615. continue;
  616. }
  617. if (dd->ipath_pkeys[i] == key) {
  618. atomic_t *pkrefs = &dd->ipath_pkeyrefs[i];
  619. if (atomic_inc_return(pkrefs) > 1) {
  620. pd->port_pkeys[pidx] = key;
  621. ipath_cdbg(VERBOSE, "p%u set key %x "
  622. "matches #%d, count now %d\n",
  623. pd->port_port, key, i,
  624. atomic_read(pkrefs));
  625. ret = 0;
  626. goto bail;
  627. } else {
  628. /*
  629. * lost race, decrement count, catch below
  630. */
  631. atomic_dec(pkrefs);
  632. ipath_cdbg(VERBOSE, "Lost race, count was "
  633. "0, after dec, it's %d\n",
  634. atomic_read(pkrefs));
  635. any++;
  636. }
  637. }
  638. if ((dd->ipath_pkeys[i] & 0x7FFF) == lkey) {
  639. /*
  640. * It makes no sense to have both the limited and
  641. * full membership PKEY set at the same time since
  642. * the unlimited one will disable the limited one.
  643. */
  644. ret = -EEXIST;
  645. goto bail;
  646. }
  647. }
  648. if (!any) {
  649. ipath_dbg("port %u, all pkeys already in use, "
  650. "can't set %x\n", pd->port_port, key);
  651. ret = -EBUSY;
  652. goto bail;
  653. }
  654. for (any = i = 0; i < ARRAY_SIZE(dd->ipath_pkeys); i++) {
  655. if (!dd->ipath_pkeys[i] &&
  656. atomic_inc_return(&dd->ipath_pkeyrefs[i]) == 1) {
  657. u64 pkey;
  658. /* for ipathstats, etc. */
  659. ipath_stats.sps_pkeys[i] = lkey;
  660. pd->port_pkeys[pidx] = dd->ipath_pkeys[i] = key;
  661. pkey =
  662. (u64) dd->ipath_pkeys[0] |
  663. ((u64) dd->ipath_pkeys[1] << 16) |
  664. ((u64) dd->ipath_pkeys[2] << 32) |
  665. ((u64) dd->ipath_pkeys[3] << 48);
  666. ipath_cdbg(PROC, "p%u set key %x in #%d, "
  667. "portidx %d, new pkey reg %llx\n",
  668. pd->port_port, key, i, pidx,
  669. (unsigned long long) pkey);
  670. ipath_write_kreg(
  671. dd, dd->ipath_kregs->kr_partitionkey, pkey);
  672. ret = 0;
  673. goto bail;
  674. }
  675. }
  676. ipath_dbg("port %u, all pkeys already in use 2nd pass, "
  677. "can't set %x\n", pd->port_port, key);
  678. ret = -EBUSY;
  679. bail:
  680. return ret;
  681. }
  682. /**
  683. * ipath_manage_rcvq - manage a port's receive queue
  684. * @pd: the port
  685. * @subport: the subport
  686. * @start_stop: action to carry out
  687. *
  688. * start_stop == 0 disables receive on the port, for use in queue
  689. * overflow conditions. start_stop==1 re-enables, to be used to
  690. * re-init the software copy of the head register
  691. */
  692. static int ipath_manage_rcvq(struct ipath_portdata *pd, unsigned subport,
  693. int start_stop)
  694. {
  695. struct ipath_devdata *dd = pd->port_dd;
  696. ipath_cdbg(PROC, "%sabling rcv for unit %u port %u:%u\n",
  697. start_stop ? "en" : "dis", dd->ipath_unit,
  698. pd->port_port, subport);
  699. if (subport)
  700. goto bail;
  701. /* atomically clear receive enable port. */
  702. if (start_stop) {
  703. /*
  704. * On enable, force in-memory copy of the tail register to
  705. * 0, so that protocol code doesn't have to worry about
  706. * whether or not the chip has yet updated the in-memory
  707. * copy or not on return from the system call. The chip
  708. * always resets it's tail register back to 0 on a
  709. * transition from disabled to enabled. This could cause a
  710. * problem if software was broken, and did the enable w/o
  711. * the disable, but eventually the in-memory copy will be
  712. * updated and correct itself, even in the face of software
  713. * bugs.
  714. */
  715. if (pd->port_rcvhdrtail_kvaddr)
  716. ipath_clear_rcvhdrtail(pd);
  717. set_bit(dd->ipath_r_portenable_shift + pd->port_port,
  718. &dd->ipath_rcvctrl);
  719. } else
  720. clear_bit(dd->ipath_r_portenable_shift + pd->port_port,
  721. &dd->ipath_rcvctrl);
  722. ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvctrl,
  723. dd->ipath_rcvctrl);
  724. /* now be sure chip saw it before we return */
  725. ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
  726. if (start_stop) {
  727. /*
  728. * And try to be sure that tail reg update has happened too.
  729. * This should in theory interlock with the RXE changes to
  730. * the tail register. Don't assign it to the tail register
  731. * in memory copy, since we could overwrite an update by the
  732. * chip if we did.
  733. */
  734. ipath_read_ureg32(dd, ur_rcvhdrtail, pd->port_port);
  735. }
  736. /* always; new head should be equal to new tail; see above */
  737. bail:
  738. return 0;
  739. }
  740. static void ipath_clean_part_key(struct ipath_portdata *pd,
  741. struct ipath_devdata *dd)
  742. {
  743. int i, j, pchanged = 0;
  744. u64 oldpkey;
  745. /* for debugging only */
  746. oldpkey = (u64) dd->ipath_pkeys[0] |
  747. ((u64) dd->ipath_pkeys[1] << 16) |
  748. ((u64) dd->ipath_pkeys[2] << 32) |
  749. ((u64) dd->ipath_pkeys[3] << 48);
  750. for (i = 0; i < ARRAY_SIZE(pd->port_pkeys); i++) {
  751. if (!pd->port_pkeys[i])
  752. continue;
  753. ipath_cdbg(VERBOSE, "look for key[%d] %hx in pkeys\n", i,
  754. pd->port_pkeys[i]);
  755. for (j = 0; j < ARRAY_SIZE(dd->ipath_pkeys); j++) {
  756. /* check for match independent of the global bit */
  757. if ((dd->ipath_pkeys[j] & 0x7fff) !=
  758. (pd->port_pkeys[i] & 0x7fff))
  759. continue;
  760. if (atomic_dec_and_test(&dd->ipath_pkeyrefs[j])) {
  761. ipath_cdbg(VERBOSE, "p%u clear key "
  762. "%x matches #%d\n",
  763. pd->port_port,
  764. pd->port_pkeys[i], j);
  765. ipath_stats.sps_pkeys[j] =
  766. dd->ipath_pkeys[j] = 0;
  767. pchanged++;
  768. }
  769. else ipath_cdbg(
  770. VERBOSE, "p%u key %x matches #%d, "
  771. "but ref still %d\n", pd->port_port,
  772. pd->port_pkeys[i], j,
  773. atomic_read(&dd->ipath_pkeyrefs[j]));
  774. break;
  775. }
  776. pd->port_pkeys[i] = 0;
  777. }
  778. if (pchanged) {
  779. u64 pkey = (u64) dd->ipath_pkeys[0] |
  780. ((u64) dd->ipath_pkeys[1] << 16) |
  781. ((u64) dd->ipath_pkeys[2] << 32) |
  782. ((u64) dd->ipath_pkeys[3] << 48);
  783. ipath_cdbg(VERBOSE, "p%u old pkey reg %llx, "
  784. "new pkey reg %llx\n", pd->port_port,
  785. (unsigned long long) oldpkey,
  786. (unsigned long long) pkey);
  787. ipath_write_kreg(dd, dd->ipath_kregs->kr_partitionkey,
  788. pkey);
  789. }
  790. }
  791. /*
  792. * Initialize the port data with the receive buffer sizes
  793. * so this can be done while the master port is locked.
  794. * Otherwise, there is a race with a slave opening the port
  795. * and seeing these fields uninitialized.
  796. */
  797. static void init_user_egr_sizes(struct ipath_portdata *pd)
  798. {
  799. struct ipath_devdata *dd = pd->port_dd;
  800. unsigned egrperchunk, egrcnt, size;
  801. /*
  802. * to avoid wasting a lot of memory, we allocate 32KB chunks of
  803. * physically contiguous memory, advance through it until used up
  804. * and then allocate more. Of course, we need memory to store those
  805. * extra pointers, now. Started out with 256KB, but under heavy
  806. * memory pressure (creating large files and then copying them over
  807. * NFS while doing lots of MPI jobs), we hit some allocation
  808. * failures, even though we can sleep... (2.6.10) Still get
  809. * failures at 64K. 32K is the lowest we can go without wasting
  810. * additional memory.
  811. */
  812. size = 0x8000;
  813. egrperchunk = size / dd->ipath_rcvegrbufsize;
  814. egrcnt = dd->ipath_rcvegrcnt;
  815. pd->port_rcvegrbuf_chunks = (egrcnt + egrperchunk - 1) / egrperchunk;
  816. pd->port_rcvegrbufs_perchunk = egrperchunk;
  817. pd->port_rcvegrbuf_size = size;
  818. }
  819. /**
  820. * ipath_create_user_egr - allocate eager TID buffers
  821. * @pd: the port to allocate TID buffers for
  822. *
  823. * This routine is now quite different for user and kernel, because
  824. * the kernel uses skb's, for the accelerated network performance
  825. * This is the user port version
  826. *
  827. * Allocate the eager TID buffers and program them into infinipath
  828. * They are no longer completely contiguous, we do multiple allocation
  829. * calls.
  830. */
  831. static int ipath_create_user_egr(struct ipath_portdata *pd)
  832. {
  833. struct ipath_devdata *dd = pd->port_dd;
  834. unsigned e, egrcnt, egrperchunk, chunk, egrsize, egroff;
  835. size_t size;
  836. int ret;
  837. gfp_t gfp_flags;
  838. /*
  839. * GFP_USER, but without GFP_FS, so buffer cache can be
  840. * coalesced (we hope); otherwise, even at order 4,
  841. * heavy filesystem activity makes these fail, and we can
  842. * use compound pages.
  843. */
  844. gfp_flags = __GFP_WAIT | __GFP_IO | __GFP_COMP;
  845. egrcnt = dd->ipath_rcvegrcnt;
  846. /* TID number offset for this port */
  847. egroff = (pd->port_port - 1) * egrcnt + dd->ipath_p0_rcvegrcnt;
  848. egrsize = dd->ipath_rcvegrbufsize;
  849. ipath_cdbg(VERBOSE, "Allocating %d egr buffers, at egrtid "
  850. "offset %x, egrsize %u\n", egrcnt, egroff, egrsize);
  851. chunk = pd->port_rcvegrbuf_chunks;
  852. egrperchunk = pd->port_rcvegrbufs_perchunk;
  853. size = pd->port_rcvegrbuf_size;
  854. pd->port_rcvegrbuf = kmalloc(chunk * sizeof(pd->port_rcvegrbuf[0]),
  855. GFP_KERNEL);
  856. if (!pd->port_rcvegrbuf) {
  857. ret = -ENOMEM;
  858. goto bail;
  859. }
  860. pd->port_rcvegrbuf_phys =
  861. kmalloc(chunk * sizeof(pd->port_rcvegrbuf_phys[0]),
  862. GFP_KERNEL);
  863. if (!pd->port_rcvegrbuf_phys) {
  864. ret = -ENOMEM;
  865. goto bail_rcvegrbuf;
  866. }
  867. for (e = 0; e < pd->port_rcvegrbuf_chunks; e++) {
  868. pd->port_rcvegrbuf[e] = dma_alloc_coherent(
  869. &dd->pcidev->dev, size, &pd->port_rcvegrbuf_phys[e],
  870. gfp_flags);
  871. if (!pd->port_rcvegrbuf[e]) {
  872. ret = -ENOMEM;
  873. goto bail_rcvegrbuf_phys;
  874. }
  875. }
  876. pd->port_rcvegr_phys = pd->port_rcvegrbuf_phys[0];
  877. for (e = chunk = 0; chunk < pd->port_rcvegrbuf_chunks; chunk++) {
  878. dma_addr_t pa = pd->port_rcvegrbuf_phys[chunk];
  879. unsigned i;
  880. for (i = 0; e < egrcnt && i < egrperchunk; e++, i++) {
  881. dd->ipath_f_put_tid(dd, e + egroff +
  882. (u64 __iomem *)
  883. ((char __iomem *)
  884. dd->ipath_kregbase +
  885. dd->ipath_rcvegrbase),
  886. RCVHQ_RCV_TYPE_EAGER, pa);
  887. pa += egrsize;
  888. }
  889. cond_resched(); /* don't hog the cpu */
  890. }
  891. ret = 0;
  892. goto bail;
  893. bail_rcvegrbuf_phys:
  894. for (e = 0; e < pd->port_rcvegrbuf_chunks &&
  895. pd->port_rcvegrbuf[e]; e++) {
  896. dma_free_coherent(&dd->pcidev->dev, size,
  897. pd->port_rcvegrbuf[e],
  898. pd->port_rcvegrbuf_phys[e]);
  899. }
  900. kfree(pd->port_rcvegrbuf_phys);
  901. pd->port_rcvegrbuf_phys = NULL;
  902. bail_rcvegrbuf:
  903. kfree(pd->port_rcvegrbuf);
  904. pd->port_rcvegrbuf = NULL;
  905. bail:
  906. return ret;
  907. }
  908. /* common code for the mappings on dma_alloc_coherent mem */
  909. static int ipath_mmap_mem(struct vm_area_struct *vma,
  910. struct ipath_portdata *pd, unsigned len, int write_ok,
  911. void *kvaddr, char *what)
  912. {
  913. struct ipath_devdata *dd = pd->port_dd;
  914. unsigned long pfn;
  915. int ret;
  916. if ((vma->vm_end - vma->vm_start) > len) {
  917. dev_info(&dd->pcidev->dev,
  918. "FAIL on %s: len %lx > %x\n", what,
  919. vma->vm_end - vma->vm_start, len);
  920. ret = -EFAULT;
  921. goto bail;
  922. }
  923. if (!write_ok) {
  924. if (vma->vm_flags & VM_WRITE) {
  925. dev_info(&dd->pcidev->dev,
  926. "%s must be mapped readonly\n", what);
  927. ret = -EPERM;
  928. goto bail;
  929. }
  930. /* don't allow them to later change with mprotect */
  931. vma->vm_flags &= ~VM_MAYWRITE;
  932. }
  933. pfn = virt_to_phys(kvaddr) >> PAGE_SHIFT;
  934. ret = remap_pfn_range(vma, vma->vm_start, pfn,
  935. len, vma->vm_page_prot);
  936. if (ret)
  937. dev_info(&dd->pcidev->dev, "%s port%u mmap of %lx, %x "
  938. "bytes r%c failed: %d\n", what, pd->port_port,
  939. pfn, len, write_ok?'w':'o', ret);
  940. else
  941. ipath_cdbg(VERBOSE, "%s port%u mmaped %lx, %x bytes "
  942. "r%c\n", what, pd->port_port, pfn, len,
  943. write_ok?'w':'o');
  944. bail:
  945. return ret;
  946. }
  947. static int mmap_ureg(struct vm_area_struct *vma, struct ipath_devdata *dd,
  948. u64 ureg)
  949. {
  950. unsigned long phys;
  951. int ret;
  952. /*
  953. * This is real hardware, so use io_remap. This is the mechanism
  954. * for the user process to update the head registers for their port
  955. * in the chip.
  956. */
  957. if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) {
  958. dev_info(&dd->pcidev->dev, "FAIL mmap userreg: reqlen "
  959. "%lx > PAGE\n", vma->vm_end - vma->vm_start);
  960. ret = -EFAULT;
  961. } else {
  962. phys = dd->ipath_physaddr + ureg;
  963. vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
  964. vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND;
  965. ret = io_remap_pfn_range(vma, vma->vm_start,
  966. phys >> PAGE_SHIFT,
  967. vma->vm_end - vma->vm_start,
  968. vma->vm_page_prot);
  969. }
  970. return ret;
  971. }
  972. static int mmap_piobufs(struct vm_area_struct *vma,
  973. struct ipath_devdata *dd,
  974. struct ipath_portdata *pd,
  975. unsigned piobufs, unsigned piocnt)
  976. {
  977. unsigned long phys;
  978. int ret;
  979. /*
  980. * When we map the PIO buffers in the chip, we want to map them as
  981. * writeonly, no read possible. This prevents access to previous
  982. * process data, and catches users who might try to read the i/o
  983. * space due to a bug.
  984. */
  985. if ((vma->vm_end - vma->vm_start) > (piocnt * dd->ipath_palign)) {
  986. dev_info(&dd->pcidev->dev, "FAIL mmap piobufs: "
  987. "reqlen %lx > PAGE\n",
  988. vma->vm_end - vma->vm_start);
  989. ret = -EINVAL;
  990. goto bail;
  991. }
  992. phys = dd->ipath_physaddr + piobufs;
  993. #if defined(__powerpc__)
  994. /* There isn't a generic way to specify writethrough mappings */
  995. pgprot_val(vma->vm_page_prot) |= _PAGE_NO_CACHE;
  996. pgprot_val(vma->vm_page_prot) |= _PAGE_WRITETHRU;
  997. pgprot_val(vma->vm_page_prot) &= ~_PAGE_GUARDED;
  998. #endif
  999. /*
  1000. * don't allow them to later change to readable with mprotect (for when
  1001. * not initially mapped readable, as is normally the case)
  1002. */
  1003. vma->vm_flags &= ~VM_MAYREAD;
  1004. vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND;
  1005. ret = io_remap_pfn_range(vma, vma->vm_start, phys >> PAGE_SHIFT,
  1006. vma->vm_end - vma->vm_start,
  1007. vma->vm_page_prot);
  1008. bail:
  1009. return ret;
  1010. }
  1011. static int mmap_rcvegrbufs(struct vm_area_struct *vma,
  1012. struct ipath_portdata *pd)
  1013. {
  1014. struct ipath_devdata *dd = pd->port_dd;
  1015. unsigned long start, size;
  1016. size_t total_size, i;
  1017. unsigned long pfn;
  1018. int ret;
  1019. size = pd->port_rcvegrbuf_size;
  1020. total_size = pd->port_rcvegrbuf_chunks * size;
  1021. if ((vma->vm_end - vma->vm_start) > total_size) {
  1022. dev_info(&dd->pcidev->dev, "FAIL on egr bufs: "
  1023. "reqlen %lx > actual %lx\n",
  1024. vma->vm_end - vma->vm_start,
  1025. (unsigned long) total_size);
  1026. ret = -EINVAL;
  1027. goto bail;
  1028. }
  1029. if (vma->vm_flags & VM_WRITE) {
  1030. dev_info(&dd->pcidev->dev, "Can't map eager buffers as "
  1031. "writable (flags=%lx)\n", vma->vm_flags);
  1032. ret = -EPERM;
  1033. goto bail;
  1034. }
  1035. /* don't allow them to later change to writeable with mprotect */
  1036. vma->vm_flags &= ~VM_MAYWRITE;
  1037. start = vma->vm_start;
  1038. for (i = 0; i < pd->port_rcvegrbuf_chunks; i++, start += size) {
  1039. pfn = virt_to_phys(pd->port_rcvegrbuf[i]) >> PAGE_SHIFT;
  1040. ret = remap_pfn_range(vma, start, pfn, size,
  1041. vma->vm_page_prot);
  1042. if (ret < 0)
  1043. goto bail;
  1044. }
  1045. ret = 0;
  1046. bail:
  1047. return ret;
  1048. }
  1049. /*
  1050. * ipath_file_vma_fault - handle a VMA page fault.
  1051. */
  1052. static int ipath_file_vma_fault(struct vm_area_struct *vma,
  1053. struct vm_fault *vmf)
  1054. {
  1055. struct page *page;
  1056. page = vmalloc_to_page((void *)(vmf->pgoff << PAGE_SHIFT));
  1057. if (!page)
  1058. return VM_FAULT_SIGBUS;
  1059. get_page(page);
  1060. vmf->page = page;
  1061. return 0;
  1062. }
  1063. static const struct vm_operations_struct ipath_file_vm_ops = {
  1064. .fault = ipath_file_vma_fault,
  1065. };
  1066. static int mmap_kvaddr(struct vm_area_struct *vma, u64 pgaddr,
  1067. struct ipath_portdata *pd, unsigned subport)
  1068. {
  1069. unsigned long len;
  1070. struct ipath_devdata *dd;
  1071. void *addr;
  1072. size_t size;
  1073. int ret = 0;
  1074. /* If the port is not shared, all addresses should be physical */
  1075. if (!pd->port_subport_cnt)
  1076. goto bail;
  1077. dd = pd->port_dd;
  1078. size = pd->port_rcvegrbuf_chunks * pd->port_rcvegrbuf_size;
  1079. /*
  1080. * Each process has all the subport uregbase, rcvhdrq, and
  1081. * rcvegrbufs mmapped - as an array for all the processes,
  1082. * and also separately for this process.
  1083. */
  1084. if (pgaddr == cvt_kvaddr(pd->subport_uregbase)) {
  1085. addr = pd->subport_uregbase;
  1086. size = PAGE_SIZE * pd->port_subport_cnt;
  1087. } else if (pgaddr == cvt_kvaddr(pd->subport_rcvhdr_base)) {
  1088. addr = pd->subport_rcvhdr_base;
  1089. size = pd->port_rcvhdrq_size * pd->port_subport_cnt;
  1090. } else if (pgaddr == cvt_kvaddr(pd->subport_rcvegrbuf)) {
  1091. addr = pd->subport_rcvegrbuf;
  1092. size *= pd->port_subport_cnt;
  1093. } else if (pgaddr == cvt_kvaddr(pd->subport_uregbase +
  1094. PAGE_SIZE * subport)) {
  1095. addr = pd->subport_uregbase + PAGE_SIZE * subport;
  1096. size = PAGE_SIZE;
  1097. } else if (pgaddr == cvt_kvaddr(pd->subport_rcvhdr_base +
  1098. pd->port_rcvhdrq_size * subport)) {
  1099. addr = pd->subport_rcvhdr_base +
  1100. pd->port_rcvhdrq_size * subport;
  1101. size = pd->port_rcvhdrq_size;
  1102. } else if (pgaddr == cvt_kvaddr(pd->subport_rcvegrbuf +
  1103. size * subport)) {
  1104. addr = pd->subport_rcvegrbuf + size * subport;
  1105. /* rcvegrbufs are read-only on the slave */
  1106. if (vma->vm_flags & VM_WRITE) {
  1107. dev_info(&dd->pcidev->dev,
  1108. "Can't map eager buffers as "
  1109. "writable (flags=%lx)\n", vma->vm_flags);
  1110. ret = -EPERM;
  1111. goto bail;
  1112. }
  1113. /*
  1114. * Don't allow permission to later change to writeable
  1115. * with mprotect.
  1116. */
  1117. vma->vm_flags &= ~VM_MAYWRITE;
  1118. } else {
  1119. goto bail;
  1120. }
  1121. len = vma->vm_end - vma->vm_start;
  1122. if (len > size) {
  1123. ipath_cdbg(MM, "FAIL: reqlen %lx > %zx\n", len, size);
  1124. ret = -EINVAL;
  1125. goto bail;
  1126. }
  1127. vma->vm_pgoff = (unsigned long) addr >> PAGE_SHIFT;
  1128. vma->vm_ops = &ipath_file_vm_ops;
  1129. vma->vm_flags |= VM_RESERVED | VM_DONTEXPAND;
  1130. ret = 1;
  1131. bail:
  1132. return ret;
  1133. }
  1134. /**
  1135. * ipath_mmap - mmap various structures into user space
  1136. * @fp: the file pointer
  1137. * @vma: the VM area
  1138. *
  1139. * We use this to have a shared buffer between the kernel and the user code
  1140. * for the rcvhdr queue, egr buffers, and the per-port user regs and pio
  1141. * buffers in the chip. We have the open and close entries so we can bump
  1142. * the ref count and keep the driver from being unloaded while still mapped.
  1143. */
  1144. static int ipath_mmap(struct file *fp, struct vm_area_struct *vma)
  1145. {
  1146. struct ipath_portdata *pd;
  1147. struct ipath_devdata *dd;
  1148. u64 pgaddr, ureg;
  1149. unsigned piobufs, piocnt;
  1150. int ret;
  1151. pd = port_fp(fp);
  1152. if (!pd) {
  1153. ret = -EINVAL;
  1154. goto bail;
  1155. }
  1156. dd = pd->port_dd;
  1157. /*
  1158. * This is the ipath_do_user_init() code, mapping the shared buffers
  1159. * into the user process. The address referred to by vm_pgoff is the
  1160. * file offset passed via mmap(). For shared ports, this is the
  1161. * kernel vmalloc() address of the pages to share with the master.
  1162. * For non-shared or master ports, this is a physical address.
  1163. * We only do one mmap for each space mapped.
  1164. */
  1165. pgaddr = vma->vm_pgoff << PAGE_SHIFT;
  1166. /*
  1167. * Check for 0 in case one of the allocations failed, but user
  1168. * called mmap anyway.
  1169. */
  1170. if (!pgaddr) {
  1171. ret = -EINVAL;
  1172. goto bail;
  1173. }
  1174. ipath_cdbg(MM, "pgaddr %llx vm_start=%lx len %lx port %u:%u:%u\n",
  1175. (unsigned long long) pgaddr, vma->vm_start,
  1176. vma->vm_end - vma->vm_start, dd->ipath_unit,
  1177. pd->port_port, subport_fp(fp));
  1178. /*
  1179. * Physical addresses must fit in 40 bits for our hardware.
  1180. * Check for kernel virtual addresses first, anything else must
  1181. * match a HW or memory address.
  1182. */
  1183. ret = mmap_kvaddr(vma, pgaddr, pd, subport_fp(fp));
  1184. if (ret) {
  1185. if (ret > 0)
  1186. ret = 0;
  1187. goto bail;
  1188. }
  1189. ureg = dd->ipath_uregbase + dd->ipath_ureg_align * pd->port_port;
  1190. if (!pd->port_subport_cnt) {
  1191. /* port is not shared */
  1192. piocnt = pd->port_piocnt;
  1193. piobufs = pd->port_piobufs;
  1194. } else if (!subport_fp(fp)) {
  1195. /* caller is the master */
  1196. piocnt = (pd->port_piocnt / pd->port_subport_cnt) +
  1197. (pd->port_piocnt % pd->port_subport_cnt);
  1198. piobufs = pd->port_piobufs +
  1199. dd->ipath_palign * (pd->port_piocnt - piocnt);
  1200. } else {
  1201. unsigned slave = subport_fp(fp) - 1;
  1202. /* caller is a slave */
  1203. piocnt = pd->port_piocnt / pd->port_subport_cnt;
  1204. piobufs = pd->port_piobufs + dd->ipath_palign * piocnt * slave;
  1205. }
  1206. if (pgaddr == ureg)
  1207. ret = mmap_ureg(vma, dd, ureg);
  1208. else if (pgaddr == piobufs)
  1209. ret = mmap_piobufs(vma, dd, pd, piobufs, piocnt);
  1210. else if (pgaddr == dd->ipath_pioavailregs_phys)
  1211. /* in-memory copy of pioavail registers */
  1212. ret = ipath_mmap_mem(vma, pd, PAGE_SIZE, 0,
  1213. (void *) dd->ipath_pioavailregs_dma,
  1214. "pioavail registers");
  1215. else if (pgaddr == pd->port_rcvegr_phys)
  1216. ret = mmap_rcvegrbufs(vma, pd);
  1217. else if (pgaddr == (u64) pd->port_rcvhdrq_phys)
  1218. /*
  1219. * The rcvhdrq itself; readonly except on HT (so have
  1220. * to allow writable mapping), multiple pages, contiguous
  1221. * from an i/o perspective.
  1222. */
  1223. ret = ipath_mmap_mem(vma, pd, pd->port_rcvhdrq_size, 1,
  1224. pd->port_rcvhdrq,
  1225. "rcvhdrq");
  1226. else if (pgaddr == (u64) pd->port_rcvhdrqtailaddr_phys)
  1227. /* in-memory copy of rcvhdrq tail register */
  1228. ret = ipath_mmap_mem(vma, pd, PAGE_SIZE, 0,
  1229. pd->port_rcvhdrtail_kvaddr,
  1230. "rcvhdrq tail");
  1231. else
  1232. ret = -EINVAL;
  1233. vma->vm_private_data = NULL;
  1234. if (ret < 0)
  1235. dev_info(&dd->pcidev->dev,
  1236. "Failure %d on off %llx len %lx\n",
  1237. -ret, (unsigned long long)pgaddr,
  1238. vma->vm_end - vma->vm_start);
  1239. bail:
  1240. return ret;
  1241. }
  1242. static unsigned ipath_poll_hdrqfull(struct ipath_portdata *pd)
  1243. {
  1244. unsigned pollflag = 0;
  1245. if ((pd->poll_type & IPATH_POLL_TYPE_OVERFLOW) &&
  1246. pd->port_hdrqfull != pd->port_hdrqfull_poll) {
  1247. pollflag |= POLLIN | POLLRDNORM;
  1248. pd->port_hdrqfull_poll = pd->port_hdrqfull;
  1249. }
  1250. return pollflag;
  1251. }
  1252. static unsigned int ipath_poll_urgent(struct ipath_portdata *pd,
  1253. struct file *fp,
  1254. struct poll_table_struct *pt)
  1255. {
  1256. unsigned pollflag = 0;
  1257. struct ipath_devdata *dd;
  1258. dd = pd->port_dd;
  1259. /* variable access in ipath_poll_hdrqfull() needs this */
  1260. rmb();
  1261. pollflag = ipath_poll_hdrqfull(pd);
  1262. if (pd->port_urgent != pd->port_urgent_poll) {
  1263. pollflag |= POLLIN | POLLRDNORM;
  1264. pd->port_urgent_poll = pd->port_urgent;
  1265. }
  1266. if (!pollflag) {
  1267. /* this saves a spin_lock/unlock in interrupt handler... */
  1268. set_bit(IPATH_PORT_WAITING_URG, &pd->port_flag);
  1269. /* flush waiting flag so don't miss an event... */
  1270. wmb();
  1271. poll_wait(fp, &pd->port_wait, pt);
  1272. }
  1273. return pollflag;
  1274. }
  1275. static unsigned int ipath_poll_next(struct ipath_portdata *pd,
  1276. struct file *fp,
  1277. struct poll_table_struct *pt)
  1278. {
  1279. u32 head;
  1280. u32 tail;
  1281. unsigned pollflag = 0;
  1282. struct ipath_devdata *dd;
  1283. dd = pd->port_dd;
  1284. /* variable access in ipath_poll_hdrqfull() needs this */
  1285. rmb();
  1286. pollflag = ipath_poll_hdrqfull(pd);
  1287. head = ipath_read_ureg32(dd, ur_rcvhdrhead, pd->port_port);
  1288. if (pd->port_rcvhdrtail_kvaddr)
  1289. tail = ipath_get_rcvhdrtail(pd);
  1290. else
  1291. tail = ipath_read_ureg32(dd, ur_rcvhdrtail, pd->port_port);
  1292. if (head != tail)
  1293. pollflag |= POLLIN | POLLRDNORM;
  1294. else {
  1295. /* this saves a spin_lock/unlock in interrupt handler */
  1296. set_bit(IPATH_PORT_WAITING_RCV, &pd->port_flag);
  1297. /* flush waiting flag so we don't miss an event */
  1298. wmb();
  1299. set_bit(pd->port_port + dd->ipath_r_intravail_shift,
  1300. &dd->ipath_rcvctrl);
  1301. ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvctrl,
  1302. dd->ipath_rcvctrl);
  1303. if (dd->ipath_rhdrhead_intr_off) /* arm rcv interrupt */
  1304. ipath_write_ureg(dd, ur_rcvhdrhead,
  1305. dd->ipath_rhdrhead_intr_off | head,
  1306. pd->port_port);
  1307. poll_wait(fp, &pd->port_wait, pt);
  1308. }
  1309. return pollflag;
  1310. }
  1311. static unsigned int ipath_poll(struct file *fp,
  1312. struct poll_table_struct *pt)
  1313. {
  1314. struct ipath_portdata *pd;
  1315. unsigned pollflag;
  1316. pd = port_fp(fp);
  1317. if (!pd)
  1318. pollflag = 0;
  1319. else if (pd->poll_type & IPATH_POLL_TYPE_URGENT)
  1320. pollflag = ipath_poll_urgent(pd, fp, pt);
  1321. else
  1322. pollflag = ipath_poll_next(pd, fp, pt);
  1323. return pollflag;
  1324. }
  1325. static int ipath_supports_subports(int user_swmajor, int user_swminor)
  1326. {
  1327. /* no subport implementation prior to software version 1.3 */
  1328. return (user_swmajor > 1) || (user_swminor >= 3);
  1329. }
  1330. static int ipath_compatible_subports(int user_swmajor, int user_swminor)
  1331. {
  1332. /* this code is written long-hand for clarity */
  1333. if (IPATH_USER_SWMAJOR != user_swmajor) {
  1334. /* no promise of compatibility if major mismatch */
  1335. return 0;
  1336. }
  1337. if (IPATH_USER_SWMAJOR == 1) {
  1338. switch (IPATH_USER_SWMINOR) {
  1339. case 0:
  1340. case 1:
  1341. case 2:
  1342. /* no subport implementation so cannot be compatible */
  1343. return 0;
  1344. case 3:
  1345. /* 3 is only compatible with itself */
  1346. return user_swminor == 3;
  1347. default:
  1348. /* >= 4 are compatible (or are expected to be) */
  1349. return user_swminor >= 4;
  1350. }
  1351. }
  1352. /* make no promises yet for future major versions */
  1353. return 0;
  1354. }
  1355. static int init_subports(struct ipath_devdata *dd,
  1356. struct ipath_portdata *pd,
  1357. const struct ipath_user_info *uinfo)
  1358. {
  1359. int ret = 0;
  1360. unsigned num_subports;
  1361. size_t size;
  1362. /*
  1363. * If the user is requesting zero subports,
  1364. * skip the subport allocation.
  1365. */
  1366. if (uinfo->spu_subport_cnt <= 0)
  1367. goto bail;
  1368. /* Self-consistency check for ipath_compatible_subports() */
  1369. if (ipath_supports_subports(IPATH_USER_SWMAJOR, IPATH_USER_SWMINOR) &&
  1370. !ipath_compatible_subports(IPATH_USER_SWMAJOR,
  1371. IPATH_USER_SWMINOR)) {
  1372. dev_info(&dd->pcidev->dev,
  1373. "Inconsistent ipath_compatible_subports()\n");
  1374. goto bail;
  1375. }
  1376. /* Check for subport compatibility */
  1377. if (!ipath_compatible_subports(uinfo->spu_userversion >> 16,
  1378. uinfo->spu_userversion & 0xffff)) {
  1379. dev_info(&dd->pcidev->dev,
  1380. "Mismatched user version (%d.%d) and driver "
  1381. "version (%d.%d) while port sharing. Ensure "
  1382. "that driver and library are from the same "
  1383. "release.\n",
  1384. (int) (uinfo->spu_userversion >> 16),
  1385. (int) (uinfo->spu_userversion & 0xffff),
  1386. IPATH_USER_SWMAJOR,
  1387. IPATH_USER_SWMINOR);
  1388. goto bail;
  1389. }
  1390. if (uinfo->spu_subport_cnt > INFINIPATH_MAX_SUBPORT) {
  1391. ret = -EINVAL;
  1392. goto bail;
  1393. }
  1394. num_subports = uinfo->spu_subport_cnt;
  1395. pd->subport_uregbase = vmalloc(PAGE_SIZE * num_subports);
  1396. if (!pd->subport_uregbase) {
  1397. ret = -ENOMEM;
  1398. goto bail;
  1399. }
  1400. /* Note: pd->port_rcvhdrq_size isn't initialized yet. */
  1401. size = ALIGN(dd->ipath_rcvhdrcnt * dd->ipath_rcvhdrentsize *
  1402. sizeof(u32), PAGE_SIZE) * num_subports;
  1403. pd->subport_rcvhdr_base = vmalloc(size);
  1404. if (!pd->subport_rcvhdr_base) {
  1405. ret = -ENOMEM;
  1406. goto bail_ureg;
  1407. }
  1408. pd->subport_rcvegrbuf = vmalloc(pd->port_rcvegrbuf_chunks *
  1409. pd->port_rcvegrbuf_size *
  1410. num_subports);
  1411. if (!pd->subport_rcvegrbuf) {
  1412. ret = -ENOMEM;
  1413. goto bail_rhdr;
  1414. }
  1415. pd->port_subport_cnt = uinfo->spu_subport_cnt;
  1416. pd->port_subport_id = uinfo->spu_subport_id;
  1417. pd->active_slaves = 1;
  1418. set_bit(IPATH_PORT_MASTER_UNINIT, &pd->port_flag);
  1419. memset(pd->subport_uregbase, 0, PAGE_SIZE * num_subports);
  1420. memset(pd->subport_rcvhdr_base, 0, size);
  1421. memset(pd->subport_rcvegrbuf, 0, pd->port_rcvegrbuf_chunks *
  1422. pd->port_rcvegrbuf_size *
  1423. num_subports);
  1424. goto bail;
  1425. bail_rhdr:
  1426. vfree(pd->subport_rcvhdr_base);
  1427. bail_ureg:
  1428. vfree(pd->subport_uregbase);
  1429. pd->subport_uregbase = NULL;
  1430. bail:
  1431. return ret;
  1432. }
  1433. static int try_alloc_port(struct ipath_devdata *dd, int port,
  1434. struct file *fp,
  1435. const struct ipath_user_info *uinfo)
  1436. {
  1437. struct ipath_portdata *pd;
  1438. int ret;
  1439. if (!(pd = dd->ipath_pd[port])) {
  1440. void *ptmp;
  1441. pd = kzalloc(sizeof(struct ipath_portdata), GFP_KERNEL);
  1442. /*
  1443. * Allocate memory for use in ipath_tid_update() just once
  1444. * at open, not per call. Reduces cost of expected send
  1445. * setup.
  1446. */
  1447. ptmp = kmalloc(dd->ipath_rcvtidcnt * sizeof(u16) +
  1448. dd->ipath_rcvtidcnt * sizeof(struct page **),
  1449. GFP_KERNEL);
  1450. if (!pd || !ptmp) {
  1451. ipath_dev_err(dd, "Unable to allocate portdata "
  1452. "memory, failing open\n");
  1453. ret = -ENOMEM;
  1454. kfree(pd);
  1455. kfree(ptmp);
  1456. goto bail;
  1457. }
  1458. dd->ipath_pd[port] = pd;
  1459. dd->ipath_pd[port]->port_port = port;
  1460. dd->ipath_pd[port]->port_dd = dd;
  1461. dd->ipath_pd[port]->port_tid_pg_list = ptmp;
  1462. init_waitqueue_head(&dd->ipath_pd[port]->port_wait);
  1463. }
  1464. if (!pd->port_cnt) {
  1465. pd->userversion = uinfo->spu_userversion;
  1466. init_user_egr_sizes(pd);
  1467. if ((ret = init_subports(dd, pd, uinfo)) != 0)
  1468. goto bail;
  1469. ipath_cdbg(PROC, "%s[%u] opened unit:port %u:%u\n",
  1470. current->comm, current->pid, dd->ipath_unit,
  1471. port);
  1472. pd->port_cnt = 1;
  1473. port_fp(fp) = pd;
  1474. pd->port_pid = get_pid(task_pid(current));
  1475. strlcpy(pd->port_comm, current->comm, sizeof(pd->port_comm));
  1476. ipath_stats.sps_ports++;
  1477. ret = 0;
  1478. } else
  1479. ret = -EBUSY;
  1480. bail:
  1481. return ret;
  1482. }
  1483. static inline int usable(struct ipath_devdata *dd)
  1484. {
  1485. return dd &&
  1486. (dd->ipath_flags & IPATH_PRESENT) &&
  1487. dd->ipath_kregbase &&
  1488. dd->ipath_lid &&
  1489. !(dd->ipath_flags & (IPATH_LINKDOWN | IPATH_DISABLED
  1490. | IPATH_LINKUNK));
  1491. }
  1492. static int find_free_port(int unit, struct file *fp,
  1493. const struct ipath_user_info *uinfo)
  1494. {
  1495. struct ipath_devdata *dd = ipath_lookup(unit);
  1496. int ret, i;
  1497. if (!dd) {
  1498. ret = -ENODEV;
  1499. goto bail;
  1500. }
  1501. if (!usable(dd)) {
  1502. ret = -ENETDOWN;
  1503. goto bail;
  1504. }
  1505. for (i = 1; i < dd->ipath_cfgports; i++) {
  1506. ret = try_alloc_port(dd, i, fp, uinfo);
  1507. if (ret != -EBUSY)
  1508. goto bail;
  1509. }
  1510. ret = -EBUSY;
  1511. bail:
  1512. return ret;
  1513. }
  1514. static int find_best_unit(struct file *fp,
  1515. const struct ipath_user_info *uinfo)
  1516. {
  1517. int ret = 0, i, prefunit = -1, devmax;
  1518. int maxofallports, npresent, nup;
  1519. int ndev;
  1520. devmax = ipath_count_units(&npresent, &nup, &maxofallports);
  1521. /*
  1522. * This code is present to allow a knowledgeable person to
  1523. * specify the layout of processes to processors before opening
  1524. * this driver, and then we'll assign the process to the "closest"
  1525. * InfiniPath chip to that processor (we assume reasonable connectivity,
  1526. * for now). This code assumes that if affinity has been set
  1527. * before this point, that at most one cpu is set; for now this
  1528. * is reasonable. I check for both cpumask_empty() and cpumask_full(),
  1529. * in case some kernel variant sets none of the bits when no
  1530. * affinity is set. 2.6.11 and 12 kernels have all present
  1531. * cpus set. Some day we'll have to fix it up further to handle
  1532. * a cpu subset. This algorithm fails for two HT chips connected
  1533. * in tunnel fashion. Eventually this needs real topology
  1534. * information. There may be some issues with dual core numbering
  1535. * as well. This needs more work prior to release.
  1536. */
  1537. if (!cpumask_empty(&current->cpus_allowed) &&
  1538. !cpumask_full(&current->cpus_allowed)) {
  1539. int ncpus = num_online_cpus(), curcpu = -1, nset = 0;
  1540. for (i = 0; i < ncpus; i++)
  1541. if (cpumask_test_cpu(i, &current->cpus_allowed)) {
  1542. ipath_cdbg(PROC, "%s[%u] affinity set for "
  1543. "cpu %d/%d\n", current->comm,
  1544. current->pid, i, ncpus);
  1545. curcpu = i;
  1546. nset++;
  1547. }
  1548. if (curcpu != -1 && nset != ncpus) {
  1549. if (npresent) {
  1550. prefunit = curcpu / (ncpus / npresent);
  1551. ipath_cdbg(PROC,"%s[%u] %d chips, %d cpus, "
  1552. "%d cpus/chip, select unit %d\n",
  1553. current->comm, current->pid,
  1554. npresent, ncpus, ncpus / npresent,
  1555. prefunit);
  1556. }
  1557. }
  1558. }
  1559. /*
  1560. * user ports start at 1, kernel port is 0
  1561. * For now, we do round-robin access across all chips
  1562. */
  1563. if (prefunit != -1)
  1564. devmax = prefunit + 1;
  1565. recheck:
  1566. for (i = 1; i < maxofallports; i++) {
  1567. for (ndev = prefunit != -1 ? prefunit : 0; ndev < devmax;
  1568. ndev++) {
  1569. struct ipath_devdata *dd = ipath_lookup(ndev);
  1570. if (!usable(dd))
  1571. continue; /* can't use this unit */
  1572. if (i >= dd->ipath_cfgports)
  1573. /*
  1574. * Maxed out on users of this unit. Try
  1575. * next.
  1576. */
  1577. continue;
  1578. ret = try_alloc_port(dd, i, fp, uinfo);
  1579. if (!ret)
  1580. goto done;
  1581. }
  1582. }
  1583. if (npresent) {
  1584. if (nup == 0) {
  1585. ret = -ENETDOWN;
  1586. ipath_dbg("No ports available (none initialized "
  1587. "and ready)\n");
  1588. } else {
  1589. if (prefunit > 0) {
  1590. /* if started above 0, retry from 0 */
  1591. ipath_cdbg(PROC,
  1592. "%s[%u] no ports on prefunit "
  1593. "%d, clear and re-check\n",
  1594. current->comm, current->pid,
  1595. prefunit);
  1596. devmax = ipath_count_units(NULL, NULL,
  1597. NULL);
  1598. prefunit = -1;
  1599. goto recheck;
  1600. }
  1601. ret = -EBUSY;
  1602. ipath_dbg("No ports available\n");
  1603. }
  1604. } else {
  1605. ret = -ENXIO;
  1606. ipath_dbg("No boards found\n");
  1607. }
  1608. done:
  1609. return ret;
  1610. }
  1611. static int find_shared_port(struct file *fp,
  1612. const struct ipath_user_info *uinfo)
  1613. {
  1614. int devmax, ndev, i;
  1615. int ret = 0;
  1616. devmax = ipath_count_units(NULL, NULL, NULL);
  1617. for (ndev = 0; ndev < devmax; ndev++) {
  1618. struct ipath_devdata *dd = ipath_lookup(ndev);
  1619. if (!usable(dd))
  1620. continue;
  1621. for (i = 1; i < dd->ipath_cfgports; i++) {
  1622. struct ipath_portdata *pd = dd->ipath_pd[i];
  1623. /* Skip ports which are not yet open */
  1624. if (!pd || !pd->port_cnt)
  1625. continue;
  1626. /* Skip port if it doesn't match the requested one */
  1627. if (pd->port_subport_id != uinfo->spu_subport_id)
  1628. continue;
  1629. /* Verify the sharing process matches the master */
  1630. if (pd->port_subport_cnt != uinfo->spu_subport_cnt ||
  1631. pd->userversion != uinfo->spu_userversion ||
  1632. pd->port_cnt >= pd->port_subport_cnt) {
  1633. ret = -EINVAL;
  1634. goto done;
  1635. }
  1636. port_fp(fp) = pd;
  1637. subport_fp(fp) = pd->port_cnt++;
  1638. pd->port_subpid[subport_fp(fp)] =
  1639. get_pid(task_pid(current));
  1640. tidcursor_fp(fp) = 0;
  1641. pd->active_slaves |= 1 << subport_fp(fp);
  1642. ipath_cdbg(PROC,
  1643. "%s[%u] %u sharing %s[%u] unit:port %u:%u\n",
  1644. current->comm, current->pid,
  1645. subport_fp(fp),
  1646. pd->port_comm, pid_nr(pd->port_pid),
  1647. dd->ipath_unit, pd->port_port);
  1648. ret = 1;
  1649. goto done;
  1650. }
  1651. }
  1652. done:
  1653. return ret;
  1654. }
  1655. static int ipath_open(struct inode *in, struct file *fp)
  1656. {
  1657. /* The real work is performed later in ipath_assign_port() */
  1658. fp->private_data = kzalloc(sizeof(struct ipath_filedata), GFP_KERNEL);
  1659. return fp->private_data ? 0 : -ENOMEM;
  1660. }
  1661. /* Get port early, so can set affinity prior to memory allocation */
  1662. static int ipath_assign_port(struct file *fp,
  1663. const struct ipath_user_info *uinfo)
  1664. {
  1665. int ret;
  1666. int i_minor;
  1667. unsigned swmajor, swminor;
  1668. /* Check to be sure we haven't already initialized this file */
  1669. if (port_fp(fp)) {
  1670. ret = -EINVAL;
  1671. goto done;
  1672. }
  1673. /* for now, if major version is different, bail */
  1674. swmajor = uinfo->spu_userversion >> 16;
  1675. if (swmajor != IPATH_USER_SWMAJOR) {
  1676. ipath_dbg("User major version %d not same as driver "
  1677. "major %d\n", uinfo->spu_userversion >> 16,
  1678. IPATH_USER_SWMAJOR);
  1679. ret = -ENODEV;
  1680. goto done;
  1681. }
  1682. swminor = uinfo->spu_userversion & 0xffff;
  1683. if (swminor != IPATH_USER_SWMINOR)
  1684. ipath_dbg("User minor version %d not same as driver "
  1685. "minor %d\n", swminor, IPATH_USER_SWMINOR);
  1686. mutex_lock(&ipath_mutex);
  1687. if (ipath_compatible_subports(swmajor, swminor) &&
  1688. uinfo->spu_subport_cnt &&
  1689. (ret = find_shared_port(fp, uinfo))) {
  1690. if (ret > 0)
  1691. ret = 0;
  1692. goto done_chk_sdma;
  1693. }
  1694. i_minor = iminor(fp->f_path.dentry->d_inode) - IPATH_USER_MINOR_BASE;
  1695. ipath_cdbg(VERBOSE, "open on dev %lx (minor %d)\n",
  1696. (long)fp->f_path.dentry->d_inode->i_rdev, i_minor);
  1697. if (i_minor)
  1698. ret = find_free_port(i_minor - 1, fp, uinfo);
  1699. else
  1700. ret = find_best_unit(fp, uinfo);
  1701. done_chk_sdma:
  1702. if (!ret) {
  1703. struct ipath_filedata *fd = fp->private_data;
  1704. const struct ipath_portdata *pd = fd->pd;
  1705. const struct ipath_devdata *dd = pd->port_dd;
  1706. fd->pq = ipath_user_sdma_queue_create(&dd->pcidev->dev,
  1707. dd->ipath_unit,
  1708. pd->port_port,
  1709. fd->subport);
  1710. if (!fd->pq)
  1711. ret = -ENOMEM;
  1712. }
  1713. mutex_unlock(&ipath_mutex);
  1714. done:
  1715. return ret;
  1716. }
  1717. static int ipath_do_user_init(struct file *fp,
  1718. const struct ipath_user_info *uinfo)
  1719. {
  1720. int ret;
  1721. struct ipath_portdata *pd = port_fp(fp);
  1722. struct ipath_devdata *dd;
  1723. u32 head32;
  1724. /* Subports don't need to initialize anything since master did it. */
  1725. if (subport_fp(fp)) {
  1726. ret = wait_event_interruptible(pd->port_wait,
  1727. !test_bit(IPATH_PORT_MASTER_UNINIT, &pd->port_flag));
  1728. goto done;
  1729. }
  1730. dd = pd->port_dd;
  1731. if (uinfo->spu_rcvhdrsize) {
  1732. ret = ipath_setrcvhdrsize(dd, uinfo->spu_rcvhdrsize);
  1733. if (ret)
  1734. goto done;
  1735. }
  1736. /* for now we do nothing with rcvhdrcnt: uinfo->spu_rcvhdrcnt */
  1737. /* some ports may get extra buffers, calculate that here */
  1738. if (pd->port_port <= dd->ipath_ports_extrabuf)
  1739. pd->port_piocnt = dd->ipath_pbufsport + 1;
  1740. else
  1741. pd->port_piocnt = dd->ipath_pbufsport;
  1742. /* for right now, kernel piobufs are at end, so port 1 is at 0 */
  1743. if (pd->port_port <= dd->ipath_ports_extrabuf)
  1744. pd->port_pio_base = (dd->ipath_pbufsport + 1)
  1745. * (pd->port_port - 1);
  1746. else
  1747. pd->port_pio_base = dd->ipath_ports_extrabuf +
  1748. dd->ipath_pbufsport * (pd->port_port - 1);
  1749. pd->port_piobufs = dd->ipath_piobufbase +
  1750. pd->port_pio_base * dd->ipath_palign;
  1751. ipath_cdbg(VERBOSE, "piobuf base for port %u is 0x%x, piocnt %u,"
  1752. " first pio %u\n", pd->port_port, pd->port_piobufs,
  1753. pd->port_piocnt, pd->port_pio_base);
  1754. ipath_chg_pioavailkernel(dd, pd->port_pio_base, pd->port_piocnt, 0);
  1755. /*
  1756. * Now allocate the rcvhdr Q and eager TIDs; skip the TID
  1757. * array for time being. If pd->port_port > chip-supported,
  1758. * we need to do extra stuff here to handle by handling overflow
  1759. * through port 0, someday
  1760. */
  1761. ret = ipath_create_rcvhdrq(dd, pd);
  1762. if (!ret)
  1763. ret = ipath_create_user_egr(pd);
  1764. if (ret)
  1765. goto done;
  1766. /*
  1767. * set the eager head register for this port to the current values
  1768. * of the tail pointers, since we don't know if they were
  1769. * updated on last use of the port.
  1770. */
  1771. head32 = ipath_read_ureg32(dd, ur_rcvegrindextail, pd->port_port);
  1772. ipath_write_ureg(dd, ur_rcvegrindexhead, head32, pd->port_port);
  1773. pd->port_lastrcvhdrqtail = -1;
  1774. ipath_cdbg(VERBOSE, "Wrote port%d egrhead %x from tail regs\n",
  1775. pd->port_port, head32);
  1776. pd->port_tidcursor = 0; /* start at beginning after open */
  1777. /* initialize poll variables... */
  1778. pd->port_urgent = 0;
  1779. pd->port_urgent_poll = 0;
  1780. pd->port_hdrqfull_poll = pd->port_hdrqfull;
  1781. /*
  1782. * Now enable the port for receive.
  1783. * For chips that are set to DMA the tail register to memory
  1784. * when they change (and when the update bit transitions from
  1785. * 0 to 1. So for those chips, we turn it off and then back on.
  1786. * This will (very briefly) affect any other open ports, but the
  1787. * duration is very short, and therefore isn't an issue. We
  1788. * explictly set the in-memory tail copy to 0 beforehand, so we
  1789. * don't have to wait to be sure the DMA update has happened
  1790. * (chip resets head/tail to 0 on transition to enable).
  1791. */
  1792. set_bit(dd->ipath_r_portenable_shift + pd->port_port,
  1793. &dd->ipath_rcvctrl);
  1794. if (!(dd->ipath_flags & IPATH_NODMA_RTAIL)) {
  1795. if (pd->port_rcvhdrtail_kvaddr)
  1796. ipath_clear_rcvhdrtail(pd);
  1797. ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvctrl,
  1798. dd->ipath_rcvctrl &
  1799. ~(1ULL << dd->ipath_r_tailupd_shift));
  1800. }
  1801. ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvctrl,
  1802. dd->ipath_rcvctrl);
  1803. /* Notify any waiting slaves */
  1804. if (pd->port_subport_cnt) {
  1805. clear_bit(IPATH_PORT_MASTER_UNINIT, &pd->port_flag);
  1806. wake_up(&pd->port_wait);
  1807. }
  1808. done:
  1809. return ret;
  1810. }
  1811. /**
  1812. * unlock_exptid - unlock any expected TID entries port still had in use
  1813. * @pd: port
  1814. *
  1815. * We don't actually update the chip here, because we do a bulk update
  1816. * below, using ipath_f_clear_tids.
  1817. */
  1818. static void unlock_expected_tids(struct ipath_portdata *pd)
  1819. {
  1820. struct ipath_devdata *dd = pd->port_dd;
  1821. int port_tidbase = pd->port_port * dd->ipath_rcvtidcnt;
  1822. int i, cnt = 0, maxtid = port_tidbase + dd->ipath_rcvtidcnt;
  1823. ipath_cdbg(VERBOSE, "Port %u unlocking any locked expTID pages\n",
  1824. pd->port_port);
  1825. for (i = port_tidbase; i < maxtid; i++) {
  1826. struct page *ps = dd->ipath_pageshadow[i];
  1827. if (!ps)
  1828. continue;
  1829. dd->ipath_pageshadow[i] = NULL;
  1830. pci_unmap_page(dd->pcidev, dd->ipath_physshadow[i],
  1831. PAGE_SIZE, PCI_DMA_FROMDEVICE);
  1832. ipath_release_user_pages_on_close(&ps, 1);
  1833. cnt++;
  1834. ipath_stats.sps_pageunlocks++;
  1835. }
  1836. if (cnt)
  1837. ipath_cdbg(VERBOSE, "Port %u locked %u expTID entries\n",
  1838. pd->port_port, cnt);
  1839. if (ipath_stats.sps_pagelocks || ipath_stats.sps_pageunlocks)
  1840. ipath_cdbg(VERBOSE, "%llu pages locked, %llu unlocked\n",
  1841. (unsigned long long) ipath_stats.sps_pagelocks,
  1842. (unsigned long long)
  1843. ipath_stats.sps_pageunlocks);
  1844. }
  1845. static int ipath_close(struct inode *in, struct file *fp)
  1846. {
  1847. int ret = 0;
  1848. struct ipath_filedata *fd;
  1849. struct ipath_portdata *pd;
  1850. struct ipath_devdata *dd;
  1851. unsigned long flags;
  1852. unsigned port;
  1853. struct pid *pid;
  1854. ipath_cdbg(VERBOSE, "close on dev %lx, private data %p\n",
  1855. (long)in->i_rdev, fp->private_data);
  1856. mutex_lock(&ipath_mutex);
  1857. fd = (struct ipath_filedata *) fp->private_data;
  1858. fp->private_data = NULL;
  1859. pd = fd->pd;
  1860. if (!pd) {
  1861. mutex_unlock(&ipath_mutex);
  1862. goto bail;
  1863. }
  1864. dd = pd->port_dd;
  1865. /* drain user sdma queue */
  1866. ipath_user_sdma_queue_drain(dd, fd->pq);
  1867. ipath_user_sdma_queue_destroy(fd->pq);
  1868. if (--pd->port_cnt) {
  1869. /*
  1870. * XXX If the master closes the port before the slave(s),
  1871. * revoke the mmap for the eager receive queue so
  1872. * the slave(s) don't wait for receive data forever.
  1873. */
  1874. pd->active_slaves &= ~(1 << fd->subport);
  1875. put_pid(pd->port_subpid[fd->subport]);
  1876. pd->port_subpid[fd->subport] = NULL;
  1877. mutex_unlock(&ipath_mutex);
  1878. goto bail;
  1879. }
  1880. /* early; no interrupt users after this */
  1881. spin_lock_irqsave(&dd->ipath_uctxt_lock, flags);
  1882. port = pd->port_port;
  1883. dd->ipath_pd[port] = NULL;
  1884. pid = pd->port_pid;
  1885. pd->port_pid = NULL;
  1886. spin_unlock_irqrestore(&dd->ipath_uctxt_lock, flags);
  1887. if (pd->port_rcvwait_to || pd->port_piowait_to
  1888. || pd->port_rcvnowait || pd->port_pionowait) {
  1889. ipath_cdbg(VERBOSE, "port%u, %u rcv, %u pio wait timeo; "
  1890. "%u rcv %u, pio already\n",
  1891. pd->port_port, pd->port_rcvwait_to,
  1892. pd->port_piowait_to, pd->port_rcvnowait,
  1893. pd->port_pionowait);
  1894. pd->port_rcvwait_to = pd->port_piowait_to =
  1895. pd->port_rcvnowait = pd->port_pionowait = 0;
  1896. }
  1897. if (pd->port_flag) {
  1898. ipath_cdbg(PROC, "port %u port_flag set: 0x%lx\n",
  1899. pd->port_port, pd->port_flag);
  1900. pd->port_flag = 0;
  1901. }
  1902. if (dd->ipath_kregbase) {
  1903. /* atomically clear receive enable port and intr avail. */
  1904. clear_bit(dd->ipath_r_portenable_shift + port,
  1905. &dd->ipath_rcvctrl);
  1906. clear_bit(pd->port_port + dd->ipath_r_intravail_shift,
  1907. &dd->ipath_rcvctrl);
  1908. ipath_write_kreg( dd, dd->ipath_kregs->kr_rcvctrl,
  1909. dd->ipath_rcvctrl);
  1910. /* and read back from chip to be sure that nothing
  1911. * else is in flight when we do the rest */
  1912. (void)ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
  1913. /* clean up the pkeys for this port user */
  1914. ipath_clean_part_key(pd, dd);
  1915. /*
  1916. * be paranoid, and never write 0's to these, just use an
  1917. * unused part of the port 0 tail page. Of course,
  1918. * rcvhdraddr points to a large chunk of memory, so this
  1919. * could still trash things, but at least it won't trash
  1920. * page 0, and by disabling the port, it should stop "soon",
  1921. * even if a packet or two is in already in flight after we
  1922. * disabled the port.
  1923. */
  1924. ipath_write_kreg_port(dd,
  1925. dd->ipath_kregs->kr_rcvhdrtailaddr, port,
  1926. dd->ipath_dummy_hdrq_phys);
  1927. ipath_write_kreg_port(dd, dd->ipath_kregs->kr_rcvhdraddr,
  1928. pd->port_port, dd->ipath_dummy_hdrq_phys);
  1929. ipath_disarm_piobufs(dd, pd->port_pio_base, pd->port_piocnt);
  1930. ipath_chg_pioavailkernel(dd, pd->port_pio_base,
  1931. pd->port_piocnt, 1);
  1932. dd->ipath_f_clear_tids(dd, pd->port_port);
  1933. if (dd->ipath_pageshadow)
  1934. unlock_expected_tids(pd);
  1935. ipath_stats.sps_ports--;
  1936. ipath_cdbg(PROC, "%s[%u] closed port %u:%u\n",
  1937. pd->port_comm, pid_nr(pid),
  1938. dd->ipath_unit, port);
  1939. }
  1940. put_pid(pid);
  1941. mutex_unlock(&ipath_mutex);
  1942. ipath_free_pddata(dd, pd); /* after releasing the mutex */
  1943. bail:
  1944. kfree(fd);
  1945. return ret;
  1946. }
  1947. static int ipath_port_info(struct ipath_portdata *pd, u16 subport,
  1948. struct ipath_port_info __user *uinfo)
  1949. {
  1950. struct ipath_port_info info;
  1951. int nup;
  1952. int ret;
  1953. size_t sz;
  1954. (void) ipath_count_units(NULL, &nup, NULL);
  1955. info.num_active = nup;
  1956. info.unit = pd->port_dd->ipath_unit;
  1957. info.port = pd->port_port;
  1958. info.subport = subport;
  1959. /* Don't return new fields if old library opened the port. */
  1960. if (ipath_supports_subports(pd->userversion >> 16,
  1961. pd->userversion & 0xffff)) {
  1962. /* Number of user ports available for this device. */
  1963. info.num_ports = pd->port_dd->ipath_cfgports - 1;
  1964. info.num_subports = pd->port_subport_cnt;
  1965. sz = sizeof(info);
  1966. } else
  1967. sz = sizeof(info) - 2 * sizeof(u16);
  1968. if (copy_to_user(uinfo, &info, sz)) {
  1969. ret = -EFAULT;
  1970. goto bail;
  1971. }
  1972. ret = 0;
  1973. bail:
  1974. return ret;
  1975. }
  1976. static int ipath_get_slave_info(struct ipath_portdata *pd,
  1977. void __user *slave_mask_addr)
  1978. {
  1979. int ret = 0;
  1980. if (copy_to_user(slave_mask_addr, &pd->active_slaves, sizeof(u32)))
  1981. ret = -EFAULT;
  1982. return ret;
  1983. }
  1984. static int ipath_sdma_get_inflight(struct ipath_user_sdma_queue *pq,
  1985. u32 __user *inflightp)
  1986. {
  1987. const u32 val = ipath_user_sdma_inflight_counter(pq);
  1988. if (put_user(val, inflightp))
  1989. return -EFAULT;
  1990. return 0;
  1991. }
  1992. static int ipath_sdma_get_complete(struct ipath_devdata *dd,
  1993. struct ipath_user_sdma_queue *pq,
  1994. u32 __user *completep)
  1995. {
  1996. u32 val;
  1997. int err;
  1998. err = ipath_user_sdma_make_progress(dd, pq);
  1999. if (err < 0)
  2000. return err;
  2001. val = ipath_user_sdma_complete_counter(pq);
  2002. if (put_user(val, completep))
  2003. return -EFAULT;
  2004. return 0;
  2005. }
  2006. static ssize_t ipath_write(struct file *fp, const char __user *data,
  2007. size_t count, loff_t *off)
  2008. {
  2009. const struct ipath_cmd __user *ucmd;
  2010. struct ipath_portdata *pd;
  2011. const void __user *src;
  2012. size_t consumed, copy;
  2013. struct ipath_cmd cmd;
  2014. ssize_t ret = 0;
  2015. void *dest;
  2016. if (count < sizeof(cmd.type)) {
  2017. ret = -EINVAL;
  2018. goto bail;
  2019. }
  2020. ucmd = (const struct ipath_cmd __user *) data;
  2021. if (copy_from_user(&cmd.type, &ucmd->type, sizeof(cmd.type))) {
  2022. ret = -EFAULT;
  2023. goto bail;
  2024. }
  2025. consumed = sizeof(cmd.type);
  2026. switch (cmd.type) {
  2027. case IPATH_CMD_ASSIGN_PORT:
  2028. case __IPATH_CMD_USER_INIT:
  2029. case IPATH_CMD_USER_INIT:
  2030. copy = sizeof(cmd.cmd.user_info);
  2031. dest = &cmd.cmd.user_info;
  2032. src = &ucmd->cmd.user_info;
  2033. break;
  2034. case IPATH_CMD_RECV_CTRL:
  2035. copy = sizeof(cmd.cmd.recv_ctrl);
  2036. dest = &cmd.cmd.recv_ctrl;
  2037. src = &ucmd->cmd.recv_ctrl;
  2038. break;
  2039. case IPATH_CMD_PORT_INFO:
  2040. copy = sizeof(cmd.cmd.port_info);
  2041. dest = &cmd.cmd.port_info;
  2042. src = &ucmd->cmd.port_info;
  2043. break;
  2044. case IPATH_CMD_TID_UPDATE:
  2045. case IPATH_CMD_TID_FREE:
  2046. copy = sizeof(cmd.cmd.tid_info);
  2047. dest = &cmd.cmd.tid_info;
  2048. src = &ucmd->cmd.tid_info;
  2049. break;
  2050. case IPATH_CMD_SET_PART_KEY:
  2051. copy = sizeof(cmd.cmd.part_key);
  2052. dest = &cmd.cmd.part_key;
  2053. src = &ucmd->cmd.part_key;
  2054. break;
  2055. case __IPATH_CMD_SLAVE_INFO:
  2056. copy = sizeof(cmd.cmd.slave_mask_addr);
  2057. dest = &cmd.cmd.slave_mask_addr;
  2058. src = &ucmd->cmd.slave_mask_addr;
  2059. break;
  2060. case IPATH_CMD_PIOAVAILUPD: // force an update of PIOAvail reg
  2061. copy = 0;
  2062. src = NULL;
  2063. dest = NULL;
  2064. break;
  2065. case IPATH_CMD_POLL_TYPE:
  2066. copy = sizeof(cmd.cmd.poll_type);
  2067. dest = &cmd.cmd.poll_type;
  2068. src = &ucmd->cmd.poll_type;
  2069. break;
  2070. case IPATH_CMD_ARMLAUNCH_CTRL:
  2071. copy = sizeof(cmd.cmd.armlaunch_ctrl);
  2072. dest = &cmd.cmd.armlaunch_ctrl;
  2073. src = &ucmd->cmd.armlaunch_ctrl;
  2074. break;
  2075. case IPATH_CMD_SDMA_INFLIGHT:
  2076. copy = sizeof(cmd.cmd.sdma_inflight);
  2077. dest = &cmd.cmd.sdma_inflight;
  2078. src = &ucmd->cmd.sdma_inflight;
  2079. break;
  2080. case IPATH_CMD_SDMA_COMPLETE:
  2081. copy = sizeof(cmd.cmd.sdma_complete);
  2082. dest = &cmd.cmd.sdma_complete;
  2083. src = &ucmd->cmd.sdma_complete;
  2084. break;
  2085. default:
  2086. ret = -EINVAL;
  2087. goto bail;
  2088. }
  2089. if (copy) {
  2090. if ((count - consumed) < copy) {
  2091. ret = -EINVAL;
  2092. goto bail;
  2093. }
  2094. if (copy_from_user(dest, src, copy)) {
  2095. ret = -EFAULT;
  2096. goto bail;
  2097. }
  2098. consumed += copy;
  2099. }
  2100. pd = port_fp(fp);
  2101. if (!pd && cmd.type != __IPATH_CMD_USER_INIT &&
  2102. cmd.type != IPATH_CMD_ASSIGN_PORT) {
  2103. ret = -EINVAL;
  2104. goto bail;
  2105. }
  2106. switch (cmd.type) {
  2107. case IPATH_CMD_ASSIGN_PORT:
  2108. ret = ipath_assign_port(fp, &cmd.cmd.user_info);
  2109. if (ret)
  2110. goto bail;
  2111. break;
  2112. case __IPATH_CMD_USER_INIT:
  2113. /* backwards compatibility, get port first */
  2114. ret = ipath_assign_port(fp, &cmd.cmd.user_info);
  2115. if (ret)
  2116. goto bail;
  2117. /* and fall through to current version. */
  2118. case IPATH_CMD_USER_INIT:
  2119. ret = ipath_do_user_init(fp, &cmd.cmd.user_info);
  2120. if (ret)
  2121. goto bail;
  2122. ret = ipath_get_base_info(
  2123. fp, (void __user *) (unsigned long)
  2124. cmd.cmd.user_info.spu_base_info,
  2125. cmd.cmd.user_info.spu_base_info_size);
  2126. break;
  2127. case IPATH_CMD_RECV_CTRL:
  2128. ret = ipath_manage_rcvq(pd, subport_fp(fp), cmd.cmd.recv_ctrl);
  2129. break;
  2130. case IPATH_CMD_PORT_INFO:
  2131. ret = ipath_port_info(pd, subport_fp(fp),
  2132. (struct ipath_port_info __user *)
  2133. (unsigned long) cmd.cmd.port_info);
  2134. break;
  2135. case IPATH_CMD_TID_UPDATE:
  2136. ret = ipath_tid_update(pd, fp, &cmd.cmd.tid_info);
  2137. break;
  2138. case IPATH_CMD_TID_FREE:
  2139. ret = ipath_tid_free(pd, subport_fp(fp), &cmd.cmd.tid_info);
  2140. break;
  2141. case IPATH_CMD_SET_PART_KEY:
  2142. ret = ipath_set_part_key(pd, cmd.cmd.part_key);
  2143. break;
  2144. case __IPATH_CMD_SLAVE_INFO:
  2145. ret = ipath_get_slave_info(pd,
  2146. (void __user *) (unsigned long)
  2147. cmd.cmd.slave_mask_addr);
  2148. break;
  2149. case IPATH_CMD_PIOAVAILUPD:
  2150. ipath_force_pio_avail_update(pd->port_dd);
  2151. break;
  2152. case IPATH_CMD_POLL_TYPE:
  2153. pd->poll_type = cmd.cmd.poll_type;
  2154. break;
  2155. case IPATH_CMD_ARMLAUNCH_CTRL:
  2156. if (cmd.cmd.armlaunch_ctrl)
  2157. ipath_enable_armlaunch(pd->port_dd);
  2158. else
  2159. ipath_disable_armlaunch(pd->port_dd);
  2160. break;
  2161. case IPATH_CMD_SDMA_INFLIGHT:
  2162. ret = ipath_sdma_get_inflight(user_sdma_queue_fp(fp),
  2163. (u32 __user *) (unsigned long)
  2164. cmd.cmd.sdma_inflight);
  2165. break;
  2166. case IPATH_CMD_SDMA_COMPLETE:
  2167. ret = ipath_sdma_get_complete(pd->port_dd,
  2168. user_sdma_queue_fp(fp),
  2169. (u32 __user *) (unsigned long)
  2170. cmd.cmd.sdma_complete);
  2171. break;
  2172. }
  2173. if (ret >= 0)
  2174. ret = consumed;
  2175. bail:
  2176. return ret;
  2177. }
  2178. static ssize_t ipath_writev(struct kiocb *iocb, const struct iovec *iov,
  2179. unsigned long dim, loff_t off)
  2180. {
  2181. struct file *filp = iocb->ki_filp;
  2182. struct ipath_filedata *fp = filp->private_data;
  2183. struct ipath_portdata *pd = port_fp(filp);
  2184. struct ipath_user_sdma_queue *pq = fp->pq;
  2185. if (!dim)
  2186. return -EINVAL;
  2187. return ipath_user_sdma_writev(pd->port_dd, pq, iov, dim);
  2188. }
  2189. static struct class *ipath_class;
  2190. static int init_cdev(int minor, char *name, const struct file_operations *fops,
  2191. struct cdev **cdevp, struct device **devp)
  2192. {
  2193. const dev_t dev = MKDEV(IPATH_MAJOR, minor);
  2194. struct cdev *cdev = NULL;
  2195. struct device *device = NULL;
  2196. int ret;
  2197. cdev = cdev_alloc();
  2198. if (!cdev) {
  2199. printk(KERN_ERR IPATH_DRV_NAME
  2200. ": Could not allocate cdev for minor %d, %s\n",
  2201. minor, name);
  2202. ret = -ENOMEM;
  2203. goto done;
  2204. }
  2205. cdev->owner = THIS_MODULE;
  2206. cdev->ops = fops;
  2207. kobject_set_name(&cdev->kobj, name);
  2208. ret = cdev_add(cdev, dev, 1);
  2209. if (ret < 0) {
  2210. printk(KERN_ERR IPATH_DRV_NAME
  2211. ": Could not add cdev for minor %d, %s (err %d)\n",
  2212. minor, name, -ret);
  2213. goto err_cdev;
  2214. }
  2215. device = device_create(ipath_class, NULL, dev, NULL, name);
  2216. if (IS_ERR(device)) {
  2217. ret = PTR_ERR(device);
  2218. printk(KERN_ERR IPATH_DRV_NAME ": Could not create "
  2219. "device for minor %d, %s (err %d)\n",
  2220. minor, name, -ret);
  2221. goto err_cdev;
  2222. }
  2223. goto done;
  2224. err_cdev:
  2225. cdev_del(cdev);
  2226. cdev = NULL;
  2227. done:
  2228. if (ret >= 0) {
  2229. *cdevp = cdev;
  2230. *devp = device;
  2231. } else {
  2232. *cdevp = NULL;
  2233. *devp = NULL;
  2234. }
  2235. return ret;
  2236. }
  2237. int ipath_cdev_init(int minor, char *name, const struct file_operations *fops,
  2238. struct cdev **cdevp, struct device **devp)
  2239. {
  2240. return init_cdev(minor, name, fops, cdevp, devp);
  2241. }
  2242. static void cleanup_cdev(struct cdev **cdevp,
  2243. struct device **devp)
  2244. {
  2245. struct device *dev = *devp;
  2246. if (dev) {
  2247. device_unregister(dev);
  2248. *devp = NULL;
  2249. }
  2250. if (*cdevp) {
  2251. cdev_del(*cdevp);
  2252. *cdevp = NULL;
  2253. }
  2254. }
  2255. void ipath_cdev_cleanup(struct cdev **cdevp,
  2256. struct device **devp)
  2257. {
  2258. cleanup_cdev(cdevp, devp);
  2259. }
  2260. static struct cdev *wildcard_cdev;
  2261. static struct device *wildcard_dev;
  2262. static const dev_t dev = MKDEV(IPATH_MAJOR, 0);
  2263. static int user_init(void)
  2264. {
  2265. int ret;
  2266. ret = register_chrdev_region(dev, IPATH_NMINORS, IPATH_DRV_NAME);
  2267. if (ret < 0) {
  2268. printk(KERN_ERR IPATH_DRV_NAME ": Could not register "
  2269. "chrdev region (err %d)\n", -ret);
  2270. goto done;
  2271. }
  2272. ipath_class = class_create(THIS_MODULE, IPATH_DRV_NAME);
  2273. if (IS_ERR(ipath_class)) {
  2274. ret = PTR_ERR(ipath_class);
  2275. printk(KERN_ERR IPATH_DRV_NAME ": Could not create "
  2276. "device class (err %d)\n", -ret);
  2277. goto bail;
  2278. }
  2279. goto done;
  2280. bail:
  2281. unregister_chrdev_region(dev, IPATH_NMINORS);
  2282. done:
  2283. return ret;
  2284. }
  2285. static void user_cleanup(void)
  2286. {
  2287. if (ipath_class) {
  2288. class_destroy(ipath_class);
  2289. ipath_class = NULL;
  2290. }
  2291. unregister_chrdev_region(dev, IPATH_NMINORS);
  2292. }
  2293. static atomic_t user_count = ATOMIC_INIT(0);
  2294. static atomic_t user_setup = ATOMIC_INIT(0);
  2295. int ipath_user_add(struct ipath_devdata *dd)
  2296. {
  2297. char name[10];
  2298. int ret;
  2299. if (atomic_inc_return(&user_count) == 1) {
  2300. ret = user_init();
  2301. if (ret < 0) {
  2302. ipath_dev_err(dd, "Unable to set up user support: "
  2303. "error %d\n", -ret);
  2304. goto bail;
  2305. }
  2306. ret = init_cdev(0, "ipath", &ipath_file_ops, &wildcard_cdev,
  2307. &wildcard_dev);
  2308. if (ret < 0) {
  2309. ipath_dev_err(dd, "Could not create wildcard "
  2310. "minor: error %d\n", -ret);
  2311. goto bail_user;
  2312. }
  2313. atomic_set(&user_setup, 1);
  2314. }
  2315. snprintf(name, sizeof(name), "ipath%d", dd->ipath_unit);
  2316. ret = init_cdev(dd->ipath_unit + 1, name, &ipath_file_ops,
  2317. &dd->user_cdev, &dd->user_dev);
  2318. if (ret < 0)
  2319. ipath_dev_err(dd, "Could not create user minor %d, %s\n",
  2320. dd->ipath_unit + 1, name);
  2321. goto bail;
  2322. bail_user:
  2323. user_cleanup();
  2324. bail:
  2325. return ret;
  2326. }
  2327. void ipath_user_remove(struct ipath_devdata *dd)
  2328. {
  2329. cleanup_cdev(&dd->user_cdev, &dd->user_dev);
  2330. if (atomic_dec_return(&user_count) == 0) {
  2331. if (atomic_read(&user_setup) == 0)
  2332. goto bail;
  2333. cleanup_cdev(&wildcard_cdev, &wildcard_dev);
  2334. user_cleanup();
  2335. atomic_set(&user_setup, 0);
  2336. }
  2337. bail:
  2338. return;
  2339. }