/drivers/infiniband/core/fmr_pool.c

https://bitbucket.org/evzijst/gittest · C · 507 lines · 318 code · 87 blank · 102 comment · 37 complexity · 70bffab036c117755d7d27c1b0531d0a MD5 · raw file

  1. /*
  2. * Copyright (c) 2004 Topspin Communications. All rights reserved.
  3. *
  4. * This software is available to you under a choice of one of two
  5. * licenses. You may choose to be licensed under the terms of the GNU
  6. * General Public License (GPL) Version 2, available from the file
  7. * COPYING in the main directory of this source tree, or the
  8. * OpenIB.org BSD license below:
  9. *
  10. * Redistribution and use in source and binary forms, with or
  11. * without modification, are permitted provided that the following
  12. * conditions are met:
  13. *
  14. * - Redistributions of source code must retain the above
  15. * copyright notice, this list of conditions and the following
  16. * disclaimer.
  17. *
  18. * - Redistributions in binary form must reproduce the above
  19. * copyright notice, this list of conditions and the following
  20. * disclaimer in the documentation and/or other materials
  21. * provided with the distribution.
  22. *
  23. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30. * SOFTWARE.
  31. *
  32. * $Id: fmr_pool.c 1349 2004-12-16 21:09:43Z roland $
  33. */
  34. #include <linux/errno.h>
  35. #include <linux/spinlock.h>
  36. #include <linux/slab.h>
  37. #include <linux/jhash.h>
  38. #include <linux/kthread.h>
  39. #include <ib_fmr_pool.h>
  40. #include "core_priv.h"
  41. enum {
  42. IB_FMR_MAX_REMAPS = 32,
  43. IB_FMR_HASH_BITS = 8,
  44. IB_FMR_HASH_SIZE = 1 << IB_FMR_HASH_BITS,
  45. IB_FMR_HASH_MASK = IB_FMR_HASH_SIZE - 1
  46. };
  47. /*
  48. * If an FMR is not in use, then the list member will point to either
  49. * its pool's free_list (if the FMR can be mapped again; that is,
  50. * remap_count < IB_FMR_MAX_REMAPS) or its pool's dirty_list (if the
  51. * FMR needs to be unmapped before being remapped). In either of
  52. * these cases it is a bug if the ref_count is not 0. In other words,
  53. * if ref_count is > 0, then the list member must not be linked into
  54. * either free_list or dirty_list.
  55. *
  56. * The cache_node member is used to link the FMR into a cache bucket
  57. * (if caching is enabled). This is independent of the reference
  58. * count of the FMR. When a valid FMR is released, its ref_count is
  59. * decremented, and if ref_count reaches 0, the FMR is placed in
  60. * either free_list or dirty_list as appropriate. However, it is not
  61. * removed from the cache and may be "revived" if a call to
  62. * ib_fmr_register_physical() occurs before the FMR is remapped. In
  63. * this case we just increment the ref_count and remove the FMR from
  64. * free_list/dirty_list.
  65. *
  66. * Before we remap an FMR from free_list, we remove it from the cache
  67. * (to prevent another user from obtaining a stale FMR). When an FMR
  68. * is released, we add it to the tail of the free list, so that our
  69. * cache eviction policy is "least recently used."
  70. *
  71. * All manipulation of ref_count, list and cache_node is protected by
  72. * pool_lock to maintain consistency.
  73. */
  74. struct ib_fmr_pool {
  75. spinlock_t pool_lock;
  76. int pool_size;
  77. int max_pages;
  78. int dirty_watermark;
  79. int dirty_len;
  80. struct list_head free_list;
  81. struct list_head dirty_list;
  82. struct hlist_head *cache_bucket;
  83. void (*flush_function)(struct ib_fmr_pool *pool,
  84. void * arg);
  85. void *flush_arg;
  86. struct task_struct *thread;
  87. atomic_t req_ser;
  88. atomic_t flush_ser;
  89. wait_queue_head_t force_wait;
  90. };
  91. static inline u32 ib_fmr_hash(u64 first_page)
  92. {
  93. return jhash_2words((u32) first_page,
  94. (u32) (first_page >> 32),
  95. 0);
  96. }
  97. /* Caller must hold pool_lock */
  98. static inline struct ib_pool_fmr *ib_fmr_cache_lookup(struct ib_fmr_pool *pool,
  99. u64 *page_list,
  100. int page_list_len,
  101. u64 io_virtual_address)
  102. {
  103. struct hlist_head *bucket;
  104. struct ib_pool_fmr *fmr;
  105. struct hlist_node *pos;
  106. if (!pool->cache_bucket)
  107. return NULL;
  108. bucket = pool->cache_bucket + ib_fmr_hash(*page_list);
  109. hlist_for_each_entry(fmr, pos, bucket, cache_node)
  110. if (io_virtual_address == fmr->io_virtual_address &&
  111. page_list_len == fmr->page_list_len &&
  112. !memcmp(page_list, fmr->page_list,
  113. page_list_len * sizeof *page_list))
  114. return fmr;
  115. return NULL;
  116. }
  117. static void ib_fmr_batch_release(struct ib_fmr_pool *pool)
  118. {
  119. int ret;
  120. struct ib_pool_fmr *fmr;
  121. LIST_HEAD(unmap_list);
  122. LIST_HEAD(fmr_list);
  123. spin_lock_irq(&pool->pool_lock);
  124. list_for_each_entry(fmr, &pool->dirty_list, list) {
  125. hlist_del_init(&fmr->cache_node);
  126. fmr->remap_count = 0;
  127. list_add_tail(&fmr->fmr->list, &fmr_list);
  128. #ifdef DEBUG
  129. if (fmr->ref_count !=0) {
  130. printk(KERN_WARNING "Unmapping FMR 0x%08x with ref count %d",
  131. fmr, fmr->ref_count);
  132. }
  133. #endif
  134. }
  135. list_splice(&pool->dirty_list, &unmap_list);
  136. INIT_LIST_HEAD(&pool->dirty_list);
  137. pool->dirty_len = 0;
  138. spin_unlock_irq(&pool->pool_lock);
  139. if (list_empty(&unmap_list)) {
  140. return;
  141. }
  142. ret = ib_unmap_fmr(&fmr_list);
  143. if (ret)
  144. printk(KERN_WARNING "ib_unmap_fmr returned %d", ret);
  145. spin_lock_irq(&pool->pool_lock);
  146. list_splice(&unmap_list, &pool->free_list);
  147. spin_unlock_irq(&pool->pool_lock);
  148. }
  149. static int ib_fmr_cleanup_thread(void *pool_ptr)
  150. {
  151. struct ib_fmr_pool *pool = pool_ptr;
  152. do {
  153. if (pool->dirty_len >= pool->dirty_watermark ||
  154. atomic_read(&pool->flush_ser) - atomic_read(&pool->req_ser) < 0) {
  155. ib_fmr_batch_release(pool);
  156. atomic_inc(&pool->flush_ser);
  157. wake_up_interruptible(&pool->force_wait);
  158. if (pool->flush_function)
  159. pool->flush_function(pool, pool->flush_arg);
  160. }
  161. set_current_state(TASK_INTERRUPTIBLE);
  162. if (pool->dirty_len < pool->dirty_watermark &&
  163. atomic_read(&pool->flush_ser) - atomic_read(&pool->req_ser) >= 0 &&
  164. !kthread_should_stop())
  165. schedule();
  166. __set_current_state(TASK_RUNNING);
  167. } while (!kthread_should_stop());
  168. return 0;
  169. }
  170. /**
  171. * ib_create_fmr_pool - Create an FMR pool
  172. * @pd:Protection domain for FMRs
  173. * @params:FMR pool parameters
  174. *
  175. * Create a pool of FMRs. Return value is pointer to new pool or
  176. * error code if creation failed.
  177. */
  178. struct ib_fmr_pool *ib_create_fmr_pool(struct ib_pd *pd,
  179. struct ib_fmr_pool_param *params)
  180. {
  181. struct ib_device *device;
  182. struct ib_fmr_pool *pool;
  183. int i;
  184. int ret;
  185. if (!params)
  186. return ERR_PTR(-EINVAL);
  187. device = pd->device;
  188. if (!device->alloc_fmr || !device->dealloc_fmr ||
  189. !device->map_phys_fmr || !device->unmap_fmr) {
  190. printk(KERN_WARNING "Device %s does not support fast memory regions",
  191. device->name);
  192. return ERR_PTR(-ENOSYS);
  193. }
  194. pool = kmalloc(sizeof *pool, GFP_KERNEL);
  195. if (!pool) {
  196. printk(KERN_WARNING "couldn't allocate pool struct");
  197. return ERR_PTR(-ENOMEM);
  198. }
  199. pool->cache_bucket = NULL;
  200. pool->flush_function = params->flush_function;
  201. pool->flush_arg = params->flush_arg;
  202. INIT_LIST_HEAD(&pool->free_list);
  203. INIT_LIST_HEAD(&pool->dirty_list);
  204. if (params->cache) {
  205. pool->cache_bucket =
  206. kmalloc(IB_FMR_HASH_SIZE * sizeof *pool->cache_bucket,
  207. GFP_KERNEL);
  208. if (!pool->cache_bucket) {
  209. printk(KERN_WARNING "Failed to allocate cache in pool");
  210. ret = -ENOMEM;
  211. goto out_free_pool;
  212. }
  213. for (i = 0; i < IB_FMR_HASH_SIZE; ++i)
  214. INIT_HLIST_HEAD(pool->cache_bucket + i);
  215. }
  216. pool->pool_size = 0;
  217. pool->max_pages = params->max_pages_per_fmr;
  218. pool->dirty_watermark = params->dirty_watermark;
  219. pool->dirty_len = 0;
  220. spin_lock_init(&pool->pool_lock);
  221. atomic_set(&pool->req_ser, 0);
  222. atomic_set(&pool->flush_ser, 0);
  223. init_waitqueue_head(&pool->force_wait);
  224. pool->thread = kthread_create(ib_fmr_cleanup_thread,
  225. pool,
  226. "ib_fmr(%s)",
  227. device->name);
  228. if (IS_ERR(pool->thread)) {
  229. printk(KERN_WARNING "couldn't start cleanup thread");
  230. ret = PTR_ERR(pool->thread);
  231. goto out_free_pool;
  232. }
  233. {
  234. struct ib_pool_fmr *fmr;
  235. struct ib_fmr_attr attr = {
  236. .max_pages = params->max_pages_per_fmr,
  237. .max_maps = IB_FMR_MAX_REMAPS,
  238. .page_size = PAGE_SHIFT
  239. };
  240. for (i = 0; i < params->pool_size; ++i) {
  241. fmr = kmalloc(sizeof *fmr + params->max_pages_per_fmr * sizeof (u64),
  242. GFP_KERNEL);
  243. if (!fmr) {
  244. printk(KERN_WARNING "failed to allocate fmr struct "
  245. "for FMR %d", i);
  246. goto out_fail;
  247. }
  248. fmr->pool = pool;
  249. fmr->remap_count = 0;
  250. fmr->ref_count = 0;
  251. INIT_HLIST_NODE(&fmr->cache_node);
  252. fmr->fmr = ib_alloc_fmr(pd, params->access, &attr);
  253. if (IS_ERR(fmr->fmr)) {
  254. printk(KERN_WARNING "fmr_create failed for FMR %d", i);
  255. kfree(fmr);
  256. goto out_fail;
  257. }
  258. list_add_tail(&fmr->list, &pool->free_list);
  259. ++pool->pool_size;
  260. }
  261. }
  262. return pool;
  263. out_free_pool:
  264. kfree(pool->cache_bucket);
  265. kfree(pool);
  266. return ERR_PTR(ret);
  267. out_fail:
  268. ib_destroy_fmr_pool(pool);
  269. return ERR_PTR(-ENOMEM);
  270. }
  271. EXPORT_SYMBOL(ib_create_fmr_pool);
  272. /**
  273. * ib_destroy_fmr_pool - Free FMR pool
  274. * @pool:FMR pool to free
  275. *
  276. * Destroy an FMR pool and free all associated resources.
  277. */
  278. int ib_destroy_fmr_pool(struct ib_fmr_pool *pool)
  279. {
  280. struct ib_pool_fmr *fmr;
  281. struct ib_pool_fmr *tmp;
  282. int i;
  283. kthread_stop(pool->thread);
  284. ib_fmr_batch_release(pool);
  285. i = 0;
  286. list_for_each_entry_safe(fmr, tmp, &pool->free_list, list) {
  287. ib_dealloc_fmr(fmr->fmr);
  288. list_del(&fmr->list);
  289. kfree(fmr);
  290. ++i;
  291. }
  292. if (i < pool->pool_size)
  293. printk(KERN_WARNING "pool still has %d regions registered",
  294. pool->pool_size - i);
  295. kfree(pool->cache_bucket);
  296. kfree(pool);
  297. return 0;
  298. }
  299. EXPORT_SYMBOL(ib_destroy_fmr_pool);
  300. /**
  301. * ib_flush_fmr_pool - Invalidate all unmapped FMRs
  302. * @pool:FMR pool to flush
  303. *
  304. * Ensure that all unmapped FMRs are fully invalidated.
  305. */
  306. int ib_flush_fmr_pool(struct ib_fmr_pool *pool)
  307. {
  308. int serial;
  309. atomic_inc(&pool->req_ser);
  310. /*
  311. * It's OK if someone else bumps req_ser again here -- we'll
  312. * just wait a little longer.
  313. */
  314. serial = atomic_read(&pool->req_ser);
  315. wake_up_process(pool->thread);
  316. if (wait_event_interruptible(pool->force_wait,
  317. atomic_read(&pool->flush_ser) -
  318. atomic_read(&pool->req_ser) >= 0))
  319. return -EINTR;
  320. return 0;
  321. }
  322. EXPORT_SYMBOL(ib_flush_fmr_pool);
  323. /**
  324. * ib_fmr_pool_map_phys -
  325. * @pool:FMR pool to allocate FMR from
  326. * @page_list:List of pages to map
  327. * @list_len:Number of pages in @page_list
  328. * @io_virtual_address:I/O virtual address for new FMR
  329. *
  330. * Map an FMR from an FMR pool.
  331. */
  332. struct ib_pool_fmr *ib_fmr_pool_map_phys(struct ib_fmr_pool *pool_handle,
  333. u64 *page_list,
  334. int list_len,
  335. u64 *io_virtual_address)
  336. {
  337. struct ib_fmr_pool *pool = pool_handle;
  338. struct ib_pool_fmr *fmr;
  339. unsigned long flags;
  340. int result;
  341. if (list_len < 1 || list_len > pool->max_pages)
  342. return ERR_PTR(-EINVAL);
  343. spin_lock_irqsave(&pool->pool_lock, flags);
  344. fmr = ib_fmr_cache_lookup(pool,
  345. page_list,
  346. list_len,
  347. *io_virtual_address);
  348. if (fmr) {
  349. /* found in cache */
  350. ++fmr->ref_count;
  351. if (fmr->ref_count == 1) {
  352. list_del(&fmr->list);
  353. }
  354. spin_unlock_irqrestore(&pool->pool_lock, flags);
  355. return fmr;
  356. }
  357. if (list_empty(&pool->free_list)) {
  358. spin_unlock_irqrestore(&pool->pool_lock, flags);
  359. return ERR_PTR(-EAGAIN);
  360. }
  361. fmr = list_entry(pool->free_list.next, struct ib_pool_fmr, list);
  362. list_del(&fmr->list);
  363. hlist_del_init(&fmr->cache_node);
  364. spin_unlock_irqrestore(&pool->pool_lock, flags);
  365. result = ib_map_phys_fmr(fmr->fmr, page_list, list_len,
  366. *io_virtual_address);
  367. if (result) {
  368. spin_lock_irqsave(&pool->pool_lock, flags);
  369. list_add(&fmr->list, &pool->free_list);
  370. spin_unlock_irqrestore(&pool->pool_lock, flags);
  371. printk(KERN_WARNING "fmr_map returns %d",
  372. result);
  373. return ERR_PTR(result);
  374. }
  375. ++fmr->remap_count;
  376. fmr->ref_count = 1;
  377. if (pool->cache_bucket) {
  378. fmr->io_virtual_address = *io_virtual_address;
  379. fmr->page_list_len = list_len;
  380. memcpy(fmr->page_list, page_list, list_len * sizeof(*page_list));
  381. spin_lock_irqsave(&pool->pool_lock, flags);
  382. hlist_add_head(&fmr->cache_node,
  383. pool->cache_bucket + ib_fmr_hash(fmr->page_list[0]));
  384. spin_unlock_irqrestore(&pool->pool_lock, flags);
  385. }
  386. return fmr;
  387. }
  388. EXPORT_SYMBOL(ib_fmr_pool_map_phys);
  389. /**
  390. * ib_fmr_pool_unmap - Unmap FMR
  391. * @fmr:FMR to unmap
  392. *
  393. * Unmap an FMR. The FMR mapping may remain valid until the FMR is
  394. * reused (or until ib_flush_fmr_pool() is called).
  395. */
  396. int ib_fmr_pool_unmap(struct ib_pool_fmr *fmr)
  397. {
  398. struct ib_fmr_pool *pool;
  399. unsigned long flags;
  400. pool = fmr->pool;
  401. spin_lock_irqsave(&pool->pool_lock, flags);
  402. --fmr->ref_count;
  403. if (!fmr->ref_count) {
  404. if (fmr->remap_count < IB_FMR_MAX_REMAPS) {
  405. list_add_tail(&fmr->list, &pool->free_list);
  406. } else {
  407. list_add_tail(&fmr->list, &pool->dirty_list);
  408. ++pool->dirty_len;
  409. wake_up_process(pool->thread);
  410. }
  411. }
  412. #ifdef DEBUG
  413. if (fmr->ref_count < 0)
  414. printk(KERN_WARNING "FMR %p has ref count %d < 0",
  415. fmr, fmr->ref_count);
  416. #endif
  417. spin_unlock_irqrestore(&pool->pool_lock, flags);
  418. return 0;
  419. }
  420. EXPORT_SYMBOL(ib_fmr_pool_unmap);