PageRenderTime 61ms CodeModel.GetById 20ms RepoModel.GetById 0ms app.codeStats 0ms

/drivers/infiniband/hw/ipath/ipath_cq.c

https://bitbucket.org/cresqo/cm7-p500-kernel
C | 478 lines | 286 code | 54 blank | 138 comment | 60 complexity | 13bdfe150939199abe3edc089ac09b06 MD5 | raw file
Possible License(s): LGPL-2.0, AGPL-1.0, GPL-2.0
  1. /*
  2. * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
  3. * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
  4. *
  5. * This software is available to you under a choice of one of two
  6. * licenses. You may choose to be licensed under the terms of the GNU
  7. * General Public License (GPL) Version 2, available from the file
  8. * COPYING in the main directory of this source tree, or the
  9. * OpenIB.org BSD license below:
  10. *
  11. * Redistribution and use in source and binary forms, with or
  12. * without modification, are permitted provided that the following
  13. * conditions are met:
  14. *
  15. * - Redistributions of source code must retain the above
  16. * copyright notice, this list of conditions and the following
  17. * disclaimer.
  18. *
  19. * - Redistributions in binary form must reproduce the above
  20. * copyright notice, this list of conditions and the following
  21. * disclaimer in the documentation and/or other materials
  22. * provided with the distribution.
  23. *
  24. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31. * SOFTWARE.
  32. */
  33. #include <linux/err.h>
  34. #include <linux/slab.h>
  35. #include <linux/vmalloc.h>
  36. #include "ipath_verbs.h"
  37. /**
  38. * ipath_cq_enter - add a new entry to the completion queue
  39. * @cq: completion queue
  40. * @entry: work completion entry to add
  41. * @sig: true if @entry is a solicitated entry
  42. *
  43. * This may be called with qp->s_lock held.
  44. */
  45. void ipath_cq_enter(struct ipath_cq *cq, struct ib_wc *entry, int solicited)
  46. {
  47. struct ipath_cq_wc *wc;
  48. unsigned long flags;
  49. u32 head;
  50. u32 next;
  51. spin_lock_irqsave(&cq->lock, flags);
  52. /*
  53. * Note that the head pointer might be writable by user processes.
  54. * Take care to verify it is a sane value.
  55. */
  56. wc = cq->queue;
  57. head = wc->head;
  58. if (head >= (unsigned) cq->ibcq.cqe) {
  59. head = cq->ibcq.cqe;
  60. next = 0;
  61. } else
  62. next = head + 1;
  63. if (unlikely(next == wc->tail)) {
  64. spin_unlock_irqrestore(&cq->lock, flags);
  65. if (cq->ibcq.event_handler) {
  66. struct ib_event ev;
  67. ev.device = cq->ibcq.device;
  68. ev.element.cq = &cq->ibcq;
  69. ev.event = IB_EVENT_CQ_ERR;
  70. cq->ibcq.event_handler(&ev, cq->ibcq.cq_context);
  71. }
  72. return;
  73. }
  74. if (cq->ip) {
  75. wc->uqueue[head].wr_id = entry->wr_id;
  76. wc->uqueue[head].status = entry->status;
  77. wc->uqueue[head].opcode = entry->opcode;
  78. wc->uqueue[head].vendor_err = entry->vendor_err;
  79. wc->uqueue[head].byte_len = entry->byte_len;
  80. wc->uqueue[head].ex.imm_data = (__u32 __force) entry->ex.imm_data;
  81. wc->uqueue[head].qp_num = entry->qp->qp_num;
  82. wc->uqueue[head].src_qp = entry->src_qp;
  83. wc->uqueue[head].wc_flags = entry->wc_flags;
  84. wc->uqueue[head].pkey_index = entry->pkey_index;
  85. wc->uqueue[head].slid = entry->slid;
  86. wc->uqueue[head].sl = entry->sl;
  87. wc->uqueue[head].dlid_path_bits = entry->dlid_path_bits;
  88. wc->uqueue[head].port_num = entry->port_num;
  89. /* Make sure entry is written before the head index. */
  90. smp_wmb();
  91. } else
  92. wc->kqueue[head] = *entry;
  93. wc->head = next;
  94. if (cq->notify == IB_CQ_NEXT_COMP ||
  95. (cq->notify == IB_CQ_SOLICITED && solicited)) {
  96. cq->notify = IB_CQ_NONE;
  97. cq->triggered++;
  98. /*
  99. * This will cause send_complete() to be called in
  100. * another thread.
  101. */
  102. tasklet_hi_schedule(&cq->comptask);
  103. }
  104. spin_unlock_irqrestore(&cq->lock, flags);
  105. if (entry->status != IB_WC_SUCCESS)
  106. to_idev(cq->ibcq.device)->n_wqe_errs++;
  107. }
  108. /**
  109. * ipath_poll_cq - poll for work completion entries
  110. * @ibcq: the completion queue to poll
  111. * @num_entries: the maximum number of entries to return
  112. * @entry: pointer to array where work completions are placed
  113. *
  114. * Returns the number of completion entries polled.
  115. *
  116. * This may be called from interrupt context. Also called by ib_poll_cq()
  117. * in the generic verbs code.
  118. */
  119. int ipath_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry)
  120. {
  121. struct ipath_cq *cq = to_icq(ibcq);
  122. struct ipath_cq_wc *wc;
  123. unsigned long flags;
  124. int npolled;
  125. u32 tail;
  126. /* The kernel can only poll a kernel completion queue */
  127. if (cq->ip) {
  128. npolled = -EINVAL;
  129. goto bail;
  130. }
  131. spin_lock_irqsave(&cq->lock, flags);
  132. wc = cq->queue;
  133. tail = wc->tail;
  134. if (tail > (u32) cq->ibcq.cqe)
  135. tail = (u32) cq->ibcq.cqe;
  136. for (npolled = 0; npolled < num_entries; ++npolled, ++entry) {
  137. if (tail == wc->head)
  138. break;
  139. /* The kernel doesn't need a RMB since it has the lock. */
  140. *entry = wc->kqueue[tail];
  141. if (tail >= cq->ibcq.cqe)
  142. tail = 0;
  143. else
  144. tail++;
  145. }
  146. wc->tail = tail;
  147. spin_unlock_irqrestore(&cq->lock, flags);
  148. bail:
  149. return npolled;
  150. }
  151. static void send_complete(unsigned long data)
  152. {
  153. struct ipath_cq *cq = (struct ipath_cq *)data;
  154. /*
  155. * The completion handler will most likely rearm the notification
  156. * and poll for all pending entries. If a new completion entry
  157. * is added while we are in this routine, tasklet_hi_schedule()
  158. * won't call us again until we return so we check triggered to
  159. * see if we need to call the handler again.
  160. */
  161. for (;;) {
  162. u8 triggered = cq->triggered;
  163. cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
  164. if (cq->triggered == triggered)
  165. return;
  166. }
  167. }
  168. /**
  169. * ipath_create_cq - create a completion queue
  170. * @ibdev: the device this completion queue is attached to
  171. * @entries: the minimum size of the completion queue
  172. * @context: unused by the InfiniPath driver
  173. * @udata: unused by the InfiniPath driver
  174. *
  175. * Returns a pointer to the completion queue or negative errno values
  176. * for failure.
  177. *
  178. * Called by ib_create_cq() in the generic verbs code.
  179. */
  180. struct ib_cq *ipath_create_cq(struct ib_device *ibdev, int entries, int comp_vector,
  181. struct ib_ucontext *context,
  182. struct ib_udata *udata)
  183. {
  184. struct ipath_ibdev *dev = to_idev(ibdev);
  185. struct ipath_cq *cq;
  186. struct ipath_cq_wc *wc;
  187. struct ib_cq *ret;
  188. u32 sz;
  189. if (entries < 1 || entries > ib_ipath_max_cqes) {
  190. ret = ERR_PTR(-EINVAL);
  191. goto done;
  192. }
  193. /* Allocate the completion queue structure. */
  194. cq = kmalloc(sizeof(*cq), GFP_KERNEL);
  195. if (!cq) {
  196. ret = ERR_PTR(-ENOMEM);
  197. goto done;
  198. }
  199. /*
  200. * Allocate the completion queue entries and head/tail pointers.
  201. * This is allocated separately so that it can be resized and
  202. * also mapped into user space.
  203. * We need to use vmalloc() in order to support mmap and large
  204. * numbers of entries.
  205. */
  206. sz = sizeof(*wc);
  207. if (udata && udata->outlen >= sizeof(__u64))
  208. sz += sizeof(struct ib_uverbs_wc) * (entries + 1);
  209. else
  210. sz += sizeof(struct ib_wc) * (entries + 1);
  211. wc = vmalloc_user(sz);
  212. if (!wc) {
  213. ret = ERR_PTR(-ENOMEM);
  214. goto bail_cq;
  215. }
  216. /*
  217. * Return the address of the WC as the offset to mmap.
  218. * See ipath_mmap() for details.
  219. */
  220. if (udata && udata->outlen >= sizeof(__u64)) {
  221. int err;
  222. cq->ip = ipath_create_mmap_info(dev, sz, context, wc);
  223. if (!cq->ip) {
  224. ret = ERR_PTR(-ENOMEM);
  225. goto bail_wc;
  226. }
  227. err = ib_copy_to_udata(udata, &cq->ip->offset,
  228. sizeof(cq->ip->offset));
  229. if (err) {
  230. ret = ERR_PTR(err);
  231. goto bail_ip;
  232. }
  233. } else
  234. cq->ip = NULL;
  235. spin_lock(&dev->n_cqs_lock);
  236. if (dev->n_cqs_allocated == ib_ipath_max_cqs) {
  237. spin_unlock(&dev->n_cqs_lock);
  238. ret = ERR_PTR(-ENOMEM);
  239. goto bail_ip;
  240. }
  241. dev->n_cqs_allocated++;
  242. spin_unlock(&dev->n_cqs_lock);
  243. if (cq->ip) {
  244. spin_lock_irq(&dev->pending_lock);
  245. list_add(&cq->ip->pending_mmaps, &dev->pending_mmaps);
  246. spin_unlock_irq(&dev->pending_lock);
  247. }
  248. /*
  249. * ib_create_cq() will initialize cq->ibcq except for cq->ibcq.cqe.
  250. * The number of entries should be >= the number requested or return
  251. * an error.
  252. */
  253. cq->ibcq.cqe = entries;
  254. cq->notify = IB_CQ_NONE;
  255. cq->triggered = 0;
  256. spin_lock_init(&cq->lock);
  257. tasklet_init(&cq->comptask, send_complete, (unsigned long)cq);
  258. wc->head = 0;
  259. wc->tail = 0;
  260. cq->queue = wc;
  261. ret = &cq->ibcq;
  262. goto done;
  263. bail_ip:
  264. kfree(cq->ip);
  265. bail_wc:
  266. vfree(wc);
  267. bail_cq:
  268. kfree(cq);
  269. done:
  270. return ret;
  271. }
  272. /**
  273. * ipath_destroy_cq - destroy a completion queue
  274. * @ibcq: the completion queue to destroy.
  275. *
  276. * Returns 0 for success.
  277. *
  278. * Called by ib_destroy_cq() in the generic verbs code.
  279. */
  280. int ipath_destroy_cq(struct ib_cq *ibcq)
  281. {
  282. struct ipath_ibdev *dev = to_idev(ibcq->device);
  283. struct ipath_cq *cq = to_icq(ibcq);
  284. tasklet_kill(&cq->comptask);
  285. spin_lock(&dev->n_cqs_lock);
  286. dev->n_cqs_allocated--;
  287. spin_unlock(&dev->n_cqs_lock);
  288. if (cq->ip)
  289. kref_put(&cq->ip->ref, ipath_release_mmap_info);
  290. else
  291. vfree(cq->queue);
  292. kfree(cq);
  293. return 0;
  294. }
  295. /**
  296. * ipath_req_notify_cq - change the notification type for a completion queue
  297. * @ibcq: the completion queue
  298. * @notify_flags: the type of notification to request
  299. *
  300. * Returns 0 for success.
  301. *
  302. * This may be called from interrupt context. Also called by
  303. * ib_req_notify_cq() in the generic verbs code.
  304. */
  305. int ipath_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags notify_flags)
  306. {
  307. struct ipath_cq *cq = to_icq(ibcq);
  308. unsigned long flags;
  309. int ret = 0;
  310. spin_lock_irqsave(&cq->lock, flags);
  311. /*
  312. * Don't change IB_CQ_NEXT_COMP to IB_CQ_SOLICITED but allow
  313. * any other transitions (see C11-31 and C11-32 in ch. 11.4.2.2).
  314. */
  315. if (cq->notify != IB_CQ_NEXT_COMP)
  316. cq->notify = notify_flags & IB_CQ_SOLICITED_MASK;
  317. if ((notify_flags & IB_CQ_REPORT_MISSED_EVENTS) &&
  318. cq->queue->head != cq->queue->tail)
  319. ret = 1;
  320. spin_unlock_irqrestore(&cq->lock, flags);
  321. return ret;
  322. }
  323. /**
  324. * ipath_resize_cq - change the size of the CQ
  325. * @ibcq: the completion queue
  326. *
  327. * Returns 0 for success.
  328. */
  329. int ipath_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata)
  330. {
  331. struct ipath_cq *cq = to_icq(ibcq);
  332. struct ipath_cq_wc *old_wc;
  333. struct ipath_cq_wc *wc;
  334. u32 head, tail, n;
  335. int ret;
  336. u32 sz;
  337. if (cqe < 1 || cqe > ib_ipath_max_cqes) {
  338. ret = -EINVAL;
  339. goto bail;
  340. }
  341. /*
  342. * Need to use vmalloc() if we want to support large #s of entries.
  343. */
  344. sz = sizeof(*wc);
  345. if (udata && udata->outlen >= sizeof(__u64))
  346. sz += sizeof(struct ib_uverbs_wc) * (cqe + 1);
  347. else
  348. sz += sizeof(struct ib_wc) * (cqe + 1);
  349. wc = vmalloc_user(sz);
  350. if (!wc) {
  351. ret = -ENOMEM;
  352. goto bail;
  353. }
  354. /* Check that we can write the offset to mmap. */
  355. if (udata && udata->outlen >= sizeof(__u64)) {
  356. __u64 offset = 0;
  357. ret = ib_copy_to_udata(udata, &offset, sizeof(offset));
  358. if (ret)
  359. goto bail_free;
  360. }
  361. spin_lock_irq(&cq->lock);
  362. /*
  363. * Make sure head and tail are sane since they
  364. * might be user writable.
  365. */
  366. old_wc = cq->queue;
  367. head = old_wc->head;
  368. if (head > (u32) cq->ibcq.cqe)
  369. head = (u32) cq->ibcq.cqe;
  370. tail = old_wc->tail;
  371. if (tail > (u32) cq->ibcq.cqe)
  372. tail = (u32) cq->ibcq.cqe;
  373. if (head < tail)
  374. n = cq->ibcq.cqe + 1 + head - tail;
  375. else
  376. n = head - tail;
  377. if (unlikely((u32)cqe < n)) {
  378. ret = -EINVAL;
  379. goto bail_unlock;
  380. }
  381. for (n = 0; tail != head; n++) {
  382. if (cq->ip)
  383. wc->uqueue[n] = old_wc->uqueue[tail];
  384. else
  385. wc->kqueue[n] = old_wc->kqueue[tail];
  386. if (tail == (u32) cq->ibcq.cqe)
  387. tail = 0;
  388. else
  389. tail++;
  390. }
  391. cq->ibcq.cqe = cqe;
  392. wc->head = n;
  393. wc->tail = 0;
  394. cq->queue = wc;
  395. spin_unlock_irq(&cq->lock);
  396. vfree(old_wc);
  397. if (cq->ip) {
  398. struct ipath_ibdev *dev = to_idev(ibcq->device);
  399. struct ipath_mmap_info *ip = cq->ip;
  400. ipath_update_mmap_info(dev, ip, sz, wc);
  401. /*
  402. * Return the offset to mmap.
  403. * See ipath_mmap() for details.
  404. */
  405. if (udata && udata->outlen >= sizeof(__u64)) {
  406. ret = ib_copy_to_udata(udata, &ip->offset,
  407. sizeof(ip->offset));
  408. if (ret)
  409. goto bail;
  410. }
  411. spin_lock_irq(&dev->pending_lock);
  412. if (list_empty(&ip->pending_mmaps))
  413. list_add(&ip->pending_mmaps, &dev->pending_mmaps);
  414. spin_unlock_irq(&dev->pending_lock);
  415. }
  416. ret = 0;
  417. goto bail;
  418. bail_unlock:
  419. spin_unlock_irq(&cq->lock);
  420. bail_free:
  421. vfree(wc);
  422. bail:
  423. return ret;
  424. }