/drivers/gpu/drm/ttm/ttm_bo.c

https://bitbucket.org/wisechild/galaxy-nexus · C · 1891 lines · 1411 code · 300 blank · 180 comment · 293 complexity · 86bbe99c1cc53be018d80499d0ffc8f6 MD5 · raw file

  1. /**************************************************************************
  2. *
  3. * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
  4. * All Rights Reserved.
  5. *
  6. * Permission is hereby granted, free of charge, to any person obtaining a
  7. * copy of this software and associated documentation files (the
  8. * "Software"), to deal in the Software without restriction, including
  9. * without limitation the rights to use, copy, modify, merge, publish,
  10. * distribute, sub license, and/or sell copies of the Software, and to
  11. * permit persons to whom the Software is furnished to do so, subject to
  12. * the following conditions:
  13. *
  14. * The above copyright notice and this permission notice (including the
  15. * next paragraph) shall be included in all copies or substantial portions
  16. * of the Software.
  17. *
  18. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20. * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  21. * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  22. * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  23. * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  24. * USE OR OTHER DEALINGS IN THE SOFTWARE.
  25. *
  26. **************************************************************************/
  27. /*
  28. * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  29. */
  30. #include "ttm/ttm_module.h"
  31. #include "ttm/ttm_bo_driver.h"
  32. #include "ttm/ttm_placement.h"
  33. #include <linux/jiffies.h>
  34. #include <linux/slab.h>
  35. #include <linux/sched.h>
  36. #include <linux/mm.h>
  37. #include <linux/file.h>
  38. #include <linux/module.h>
  39. #include <asm/atomic.h>
  40. #define TTM_ASSERT_LOCKED(param)
  41. #define TTM_DEBUG(fmt, arg...)
  42. #define TTM_BO_HASH_ORDER 13
  43. static int ttm_bo_setup_vm(struct ttm_buffer_object *bo);
  44. static int ttm_bo_swapout(struct ttm_mem_shrink *shrink);
  45. static void ttm_bo_global_kobj_release(struct kobject *kobj);
  46. static struct attribute ttm_bo_count = {
  47. .name = "bo_count",
  48. .mode = S_IRUGO
  49. };
  50. static inline int ttm_mem_type_from_flags(uint32_t flags, uint32_t *mem_type)
  51. {
  52. int i;
  53. for (i = 0; i <= TTM_PL_PRIV5; i++)
  54. if (flags & (1 << i)) {
  55. *mem_type = i;
  56. return 0;
  57. }
  58. return -EINVAL;
  59. }
  60. static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type)
  61. {
  62. struct ttm_mem_type_manager *man = &bdev->man[mem_type];
  63. printk(KERN_ERR TTM_PFX " has_type: %d\n", man->has_type);
  64. printk(KERN_ERR TTM_PFX " use_type: %d\n", man->use_type);
  65. printk(KERN_ERR TTM_PFX " flags: 0x%08X\n", man->flags);
  66. printk(KERN_ERR TTM_PFX " gpu_offset: 0x%08lX\n", man->gpu_offset);
  67. printk(KERN_ERR TTM_PFX " size: %llu\n", man->size);
  68. printk(KERN_ERR TTM_PFX " available_caching: 0x%08X\n",
  69. man->available_caching);
  70. printk(KERN_ERR TTM_PFX " default_caching: 0x%08X\n",
  71. man->default_caching);
  72. if (mem_type != TTM_PL_SYSTEM)
  73. (*man->func->debug)(man, TTM_PFX);
  74. }
  75. static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
  76. struct ttm_placement *placement)
  77. {
  78. int i, ret, mem_type;
  79. printk(KERN_ERR TTM_PFX "No space for %p (%lu pages, %luK, %luM)\n",
  80. bo, bo->mem.num_pages, bo->mem.size >> 10,
  81. bo->mem.size >> 20);
  82. for (i = 0; i < placement->num_placement; i++) {
  83. ret = ttm_mem_type_from_flags(placement->placement[i],
  84. &mem_type);
  85. if (ret)
  86. return;
  87. printk(KERN_ERR TTM_PFX " placement[%d]=0x%08X (%d)\n",
  88. i, placement->placement[i], mem_type);
  89. ttm_mem_type_debug(bo->bdev, mem_type);
  90. }
  91. }
  92. static ssize_t ttm_bo_global_show(struct kobject *kobj,
  93. struct attribute *attr,
  94. char *buffer)
  95. {
  96. struct ttm_bo_global *glob =
  97. container_of(kobj, struct ttm_bo_global, kobj);
  98. return snprintf(buffer, PAGE_SIZE, "%lu\n",
  99. (unsigned long) atomic_read(&glob->bo_count));
  100. }
  101. static struct attribute *ttm_bo_global_attrs[] = {
  102. &ttm_bo_count,
  103. NULL
  104. };
  105. static const struct sysfs_ops ttm_bo_global_ops = {
  106. .show = &ttm_bo_global_show
  107. };
  108. static struct kobj_type ttm_bo_glob_kobj_type = {
  109. .release = &ttm_bo_global_kobj_release,
  110. .sysfs_ops = &ttm_bo_global_ops,
  111. .default_attrs = ttm_bo_global_attrs
  112. };
  113. static inline uint32_t ttm_bo_type_flags(unsigned type)
  114. {
  115. return 1 << (type);
  116. }
  117. static void ttm_bo_release_list(struct kref *list_kref)
  118. {
  119. struct ttm_buffer_object *bo =
  120. container_of(list_kref, struct ttm_buffer_object, list_kref);
  121. struct ttm_bo_device *bdev = bo->bdev;
  122. BUG_ON(atomic_read(&bo->list_kref.refcount));
  123. BUG_ON(atomic_read(&bo->kref.refcount));
  124. BUG_ON(atomic_read(&bo->cpu_writers));
  125. BUG_ON(bo->sync_obj != NULL);
  126. BUG_ON(bo->mem.mm_node != NULL);
  127. BUG_ON(!list_empty(&bo->lru));
  128. BUG_ON(!list_empty(&bo->ddestroy));
  129. if (bo->ttm)
  130. ttm_tt_destroy(bo->ttm);
  131. atomic_dec(&bo->glob->bo_count);
  132. if (bo->destroy)
  133. bo->destroy(bo);
  134. else {
  135. ttm_mem_global_free(bdev->glob->mem_glob, bo->acc_size);
  136. kfree(bo);
  137. }
  138. }
  139. int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo, bool interruptible)
  140. {
  141. if (interruptible) {
  142. return wait_event_interruptible(bo->event_queue,
  143. atomic_read(&bo->reserved) == 0);
  144. } else {
  145. wait_event(bo->event_queue, atomic_read(&bo->reserved) == 0);
  146. return 0;
  147. }
  148. }
  149. EXPORT_SYMBOL(ttm_bo_wait_unreserved);
  150. void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
  151. {
  152. struct ttm_bo_device *bdev = bo->bdev;
  153. struct ttm_mem_type_manager *man;
  154. BUG_ON(!atomic_read(&bo->reserved));
  155. if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
  156. BUG_ON(!list_empty(&bo->lru));
  157. man = &bdev->man[bo->mem.mem_type];
  158. list_add_tail(&bo->lru, &man->lru);
  159. kref_get(&bo->list_kref);
  160. if (bo->ttm != NULL) {
  161. list_add_tail(&bo->swap, &bo->glob->swap_lru);
  162. kref_get(&bo->list_kref);
  163. }
  164. }
  165. }
  166. int ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
  167. {
  168. int put_count = 0;
  169. if (!list_empty(&bo->swap)) {
  170. list_del_init(&bo->swap);
  171. ++put_count;
  172. }
  173. if (!list_empty(&bo->lru)) {
  174. list_del_init(&bo->lru);
  175. ++put_count;
  176. }
  177. /*
  178. * TODO: Add a driver hook to delete from
  179. * driver-specific LRU's here.
  180. */
  181. return put_count;
  182. }
  183. int ttm_bo_reserve_locked(struct ttm_buffer_object *bo,
  184. bool interruptible,
  185. bool no_wait, bool use_sequence, uint32_t sequence)
  186. {
  187. struct ttm_bo_global *glob = bo->glob;
  188. int ret;
  189. while (unlikely(atomic_cmpxchg(&bo->reserved, 0, 1) != 0)) {
  190. /**
  191. * Deadlock avoidance for multi-bo reserving.
  192. */
  193. if (use_sequence && bo->seq_valid) {
  194. /**
  195. * We've already reserved this one.
  196. */
  197. if (unlikely(sequence == bo->val_seq))
  198. return -EDEADLK;
  199. /**
  200. * Already reserved by a thread that will not back
  201. * off for us. We need to back off.
  202. */
  203. if (unlikely(sequence - bo->val_seq < (1 << 31)))
  204. return -EAGAIN;
  205. }
  206. if (no_wait)
  207. return -EBUSY;
  208. spin_unlock(&glob->lru_lock);
  209. ret = ttm_bo_wait_unreserved(bo, interruptible);
  210. spin_lock(&glob->lru_lock);
  211. if (unlikely(ret))
  212. return ret;
  213. }
  214. if (use_sequence) {
  215. /**
  216. * Wake up waiters that may need to recheck for deadlock,
  217. * if we decreased the sequence number.
  218. */
  219. if (unlikely((bo->val_seq - sequence < (1 << 31))
  220. || !bo->seq_valid))
  221. wake_up_all(&bo->event_queue);
  222. bo->val_seq = sequence;
  223. bo->seq_valid = true;
  224. } else {
  225. bo->seq_valid = false;
  226. }
  227. return 0;
  228. }
  229. EXPORT_SYMBOL(ttm_bo_reserve);
  230. static void ttm_bo_ref_bug(struct kref *list_kref)
  231. {
  232. BUG();
  233. }
  234. void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count,
  235. bool never_free)
  236. {
  237. kref_sub(&bo->list_kref, count,
  238. (never_free) ? ttm_bo_ref_bug : ttm_bo_release_list);
  239. }
  240. int ttm_bo_reserve(struct ttm_buffer_object *bo,
  241. bool interruptible,
  242. bool no_wait, bool use_sequence, uint32_t sequence)
  243. {
  244. struct ttm_bo_global *glob = bo->glob;
  245. int put_count = 0;
  246. int ret;
  247. spin_lock(&glob->lru_lock);
  248. ret = ttm_bo_reserve_locked(bo, interruptible, no_wait, use_sequence,
  249. sequence);
  250. if (likely(ret == 0))
  251. put_count = ttm_bo_del_from_lru(bo);
  252. spin_unlock(&glob->lru_lock);
  253. ttm_bo_list_ref_sub(bo, put_count, true);
  254. return ret;
  255. }
  256. void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo)
  257. {
  258. ttm_bo_add_to_lru(bo);
  259. atomic_set(&bo->reserved, 0);
  260. wake_up_all(&bo->event_queue);
  261. }
  262. void ttm_bo_unreserve(struct ttm_buffer_object *bo)
  263. {
  264. struct ttm_bo_global *glob = bo->glob;
  265. spin_lock(&glob->lru_lock);
  266. ttm_bo_unreserve_locked(bo);
  267. spin_unlock(&glob->lru_lock);
  268. }
  269. EXPORT_SYMBOL(ttm_bo_unreserve);
  270. /*
  271. * Call bo->mutex locked.
  272. */
  273. static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc)
  274. {
  275. struct ttm_bo_device *bdev = bo->bdev;
  276. struct ttm_bo_global *glob = bo->glob;
  277. int ret = 0;
  278. uint32_t page_flags = 0;
  279. TTM_ASSERT_LOCKED(&bo->mutex);
  280. bo->ttm = NULL;
  281. if (bdev->need_dma32)
  282. page_flags |= TTM_PAGE_FLAG_DMA32;
  283. switch (bo->type) {
  284. case ttm_bo_type_device:
  285. if (zero_alloc)
  286. page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC;
  287. case ttm_bo_type_kernel:
  288. bo->ttm = ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT,
  289. page_flags, glob->dummy_read_page);
  290. if (unlikely(bo->ttm == NULL))
  291. ret = -ENOMEM;
  292. break;
  293. case ttm_bo_type_user:
  294. bo->ttm = ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT,
  295. page_flags | TTM_PAGE_FLAG_USER,
  296. glob->dummy_read_page);
  297. if (unlikely(bo->ttm == NULL)) {
  298. ret = -ENOMEM;
  299. break;
  300. }
  301. ret = ttm_tt_set_user(bo->ttm, current,
  302. bo->buffer_start, bo->num_pages);
  303. if (unlikely(ret != 0)) {
  304. ttm_tt_destroy(bo->ttm);
  305. bo->ttm = NULL;
  306. }
  307. break;
  308. default:
  309. printk(KERN_ERR TTM_PFX "Illegal buffer object type\n");
  310. ret = -EINVAL;
  311. break;
  312. }
  313. return ret;
  314. }
  315. static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
  316. struct ttm_mem_reg *mem,
  317. bool evict, bool interruptible,
  318. bool no_wait_reserve, bool no_wait_gpu)
  319. {
  320. struct ttm_bo_device *bdev = bo->bdev;
  321. bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem);
  322. bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem);
  323. struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type];
  324. struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type];
  325. int ret = 0;
  326. if (old_is_pci || new_is_pci ||
  327. ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) {
  328. ret = ttm_mem_io_lock(old_man, true);
  329. if (unlikely(ret != 0))
  330. goto out_err;
  331. ttm_bo_unmap_virtual_locked(bo);
  332. ttm_mem_io_unlock(old_man);
  333. }
  334. /*
  335. * Create and bind a ttm if required.
  336. */
  337. if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
  338. if (bo->ttm == NULL) {
  339. bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED);
  340. ret = ttm_bo_add_ttm(bo, zero);
  341. if (ret)
  342. goto out_err;
  343. }
  344. ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement);
  345. if (ret)
  346. goto out_err;
  347. if (mem->mem_type != TTM_PL_SYSTEM) {
  348. ret = ttm_tt_bind(bo->ttm, mem);
  349. if (ret)
  350. goto out_err;
  351. }
  352. if (bo->mem.mem_type == TTM_PL_SYSTEM) {
  353. if (bdev->driver->move_notify)
  354. bdev->driver->move_notify(bo, mem);
  355. bo->mem = *mem;
  356. mem->mm_node = NULL;
  357. goto moved;
  358. }
  359. }
  360. if (bdev->driver->move_notify)
  361. bdev->driver->move_notify(bo, mem);
  362. if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
  363. !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED))
  364. ret = ttm_bo_move_ttm(bo, evict, no_wait_reserve, no_wait_gpu, mem);
  365. else if (bdev->driver->move)
  366. ret = bdev->driver->move(bo, evict, interruptible,
  367. no_wait_reserve, no_wait_gpu, mem);
  368. else
  369. ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, mem);
  370. if (ret)
  371. goto out_err;
  372. moved:
  373. if (bo->evicted) {
  374. ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement);
  375. if (ret)
  376. printk(KERN_ERR TTM_PFX "Can not flush read caches\n");
  377. bo->evicted = false;
  378. }
  379. if (bo->mem.mm_node) {
  380. bo->offset = (bo->mem.start << PAGE_SHIFT) +
  381. bdev->man[bo->mem.mem_type].gpu_offset;
  382. bo->cur_placement = bo->mem.placement;
  383. } else
  384. bo->offset = 0;
  385. return 0;
  386. out_err:
  387. new_man = &bdev->man[bo->mem.mem_type];
  388. if ((new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm) {
  389. ttm_tt_unbind(bo->ttm);
  390. ttm_tt_destroy(bo->ttm);
  391. bo->ttm = NULL;
  392. }
  393. return ret;
  394. }
  395. /**
  396. * Call bo::reserved.
  397. * Will release GPU memory type usage on destruction.
  398. * This is the place to put in driver specific hooks to release
  399. * driver private resources.
  400. * Will release the bo::reserved lock.
  401. */
  402. static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
  403. {
  404. if (bo->ttm) {
  405. ttm_tt_unbind(bo->ttm);
  406. ttm_tt_destroy(bo->ttm);
  407. bo->ttm = NULL;
  408. }
  409. ttm_bo_mem_put(bo, &bo->mem);
  410. atomic_set(&bo->reserved, 0);
  411. /*
  412. * Make processes trying to reserve really pick it up.
  413. */
  414. smp_mb__after_atomic_dec();
  415. wake_up_all(&bo->event_queue);
  416. }
  417. static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
  418. {
  419. struct ttm_bo_device *bdev = bo->bdev;
  420. struct ttm_bo_global *glob = bo->glob;
  421. struct ttm_bo_driver *driver;
  422. void *sync_obj = NULL;
  423. void *sync_obj_arg;
  424. int put_count;
  425. int ret;
  426. spin_lock(&bdev->fence_lock);
  427. (void) ttm_bo_wait(bo, false, false, true);
  428. if (!bo->sync_obj) {
  429. spin_lock(&glob->lru_lock);
  430. /**
  431. * Lock inversion between bo:reserve and bdev::fence_lock here,
  432. * but that's OK, since we're only trylocking.
  433. */
  434. ret = ttm_bo_reserve_locked(bo, false, true, false, 0);
  435. if (unlikely(ret == -EBUSY))
  436. goto queue;
  437. spin_unlock(&bdev->fence_lock);
  438. put_count = ttm_bo_del_from_lru(bo);
  439. spin_unlock(&glob->lru_lock);
  440. ttm_bo_cleanup_memtype_use(bo);
  441. ttm_bo_list_ref_sub(bo, put_count, true);
  442. return;
  443. } else {
  444. spin_lock(&glob->lru_lock);
  445. }
  446. queue:
  447. driver = bdev->driver;
  448. if (bo->sync_obj)
  449. sync_obj = driver->sync_obj_ref(bo->sync_obj);
  450. sync_obj_arg = bo->sync_obj_arg;
  451. kref_get(&bo->list_kref);
  452. list_add_tail(&bo->ddestroy, &bdev->ddestroy);
  453. spin_unlock(&glob->lru_lock);
  454. spin_unlock(&bdev->fence_lock);
  455. if (sync_obj) {
  456. driver->sync_obj_flush(sync_obj, sync_obj_arg);
  457. driver->sync_obj_unref(&sync_obj);
  458. }
  459. schedule_delayed_work(&bdev->wq,
  460. ((HZ / 100) < 1) ? 1 : HZ / 100);
  461. }
  462. /**
  463. * function ttm_bo_cleanup_refs
  464. * If bo idle, remove from delayed- and lru lists, and unref.
  465. * If not idle, do nothing.
  466. *
  467. * @interruptible Any sleeps should occur interruptibly.
  468. * @no_wait_reserve Never wait for reserve. Return -EBUSY instead.
  469. * @no_wait_gpu Never wait for gpu. Return -EBUSY instead.
  470. */
  471. static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
  472. bool interruptible,
  473. bool no_wait_reserve,
  474. bool no_wait_gpu)
  475. {
  476. struct ttm_bo_device *bdev = bo->bdev;
  477. struct ttm_bo_global *glob = bo->glob;
  478. int put_count;
  479. int ret = 0;
  480. retry:
  481. spin_lock(&bdev->fence_lock);
  482. ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
  483. spin_unlock(&bdev->fence_lock);
  484. if (unlikely(ret != 0))
  485. return ret;
  486. spin_lock(&glob->lru_lock);
  487. ret = ttm_bo_reserve_locked(bo, interruptible,
  488. no_wait_reserve, false, 0);
  489. if (unlikely(ret != 0) || list_empty(&bo->ddestroy)) {
  490. spin_unlock(&glob->lru_lock);
  491. return ret;
  492. }
  493. /**
  494. * We can re-check for sync object without taking
  495. * the bo::lock since setting the sync object requires
  496. * also bo::reserved. A busy object at this point may
  497. * be caused by another thread recently starting an accelerated
  498. * eviction.
  499. */
  500. if (unlikely(bo->sync_obj)) {
  501. atomic_set(&bo->reserved, 0);
  502. wake_up_all(&bo->event_queue);
  503. spin_unlock(&glob->lru_lock);
  504. goto retry;
  505. }
  506. put_count = ttm_bo_del_from_lru(bo);
  507. list_del_init(&bo->ddestroy);
  508. ++put_count;
  509. spin_unlock(&glob->lru_lock);
  510. ttm_bo_cleanup_memtype_use(bo);
  511. ttm_bo_list_ref_sub(bo, put_count, true);
  512. return 0;
  513. }
  514. /**
  515. * Traverse the delayed list, and call ttm_bo_cleanup_refs on all
  516. * encountered buffers.
  517. */
  518. static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)
  519. {
  520. struct ttm_bo_global *glob = bdev->glob;
  521. struct ttm_buffer_object *entry = NULL;
  522. int ret = 0;
  523. spin_lock(&glob->lru_lock);
  524. if (list_empty(&bdev->ddestroy))
  525. goto out_unlock;
  526. entry = list_first_entry(&bdev->ddestroy,
  527. struct ttm_buffer_object, ddestroy);
  528. kref_get(&entry->list_kref);
  529. for (;;) {
  530. struct ttm_buffer_object *nentry = NULL;
  531. if (entry->ddestroy.next != &bdev->ddestroy) {
  532. nentry = list_first_entry(&entry->ddestroy,
  533. struct ttm_buffer_object, ddestroy);
  534. kref_get(&nentry->list_kref);
  535. }
  536. spin_unlock(&glob->lru_lock);
  537. ret = ttm_bo_cleanup_refs(entry, false, !remove_all,
  538. !remove_all);
  539. kref_put(&entry->list_kref, ttm_bo_release_list);
  540. entry = nentry;
  541. if (ret || !entry)
  542. goto out;
  543. spin_lock(&glob->lru_lock);
  544. if (list_empty(&entry->ddestroy))
  545. break;
  546. }
  547. out_unlock:
  548. spin_unlock(&glob->lru_lock);
  549. out:
  550. if (entry)
  551. kref_put(&entry->list_kref, ttm_bo_release_list);
  552. return ret;
  553. }
  554. static void ttm_bo_delayed_workqueue(struct work_struct *work)
  555. {
  556. struct ttm_bo_device *bdev =
  557. container_of(work, struct ttm_bo_device, wq.work);
  558. if (ttm_bo_delayed_delete(bdev, false)) {
  559. schedule_delayed_work(&bdev->wq,
  560. ((HZ / 100) < 1) ? 1 : HZ / 100);
  561. }
  562. }
  563. static void ttm_bo_release(struct kref *kref)
  564. {
  565. struct ttm_buffer_object *bo =
  566. container_of(kref, struct ttm_buffer_object, kref);
  567. struct ttm_bo_device *bdev = bo->bdev;
  568. struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
  569. if (likely(bo->vm_node != NULL)) {
  570. rb_erase(&bo->vm_rb, &bdev->addr_space_rb);
  571. drm_mm_put_block(bo->vm_node);
  572. bo->vm_node = NULL;
  573. }
  574. write_unlock(&bdev->vm_lock);
  575. ttm_mem_io_lock(man, false);
  576. ttm_mem_io_free_vm(bo);
  577. ttm_mem_io_unlock(man);
  578. ttm_bo_cleanup_refs_or_queue(bo);
  579. kref_put(&bo->list_kref, ttm_bo_release_list);
  580. write_lock(&bdev->vm_lock);
  581. }
  582. void ttm_bo_unref(struct ttm_buffer_object **p_bo)
  583. {
  584. struct ttm_buffer_object *bo = *p_bo;
  585. struct ttm_bo_device *bdev = bo->bdev;
  586. *p_bo = NULL;
  587. write_lock(&bdev->vm_lock);
  588. kref_put(&bo->kref, ttm_bo_release);
  589. write_unlock(&bdev->vm_lock);
  590. }
  591. EXPORT_SYMBOL(ttm_bo_unref);
  592. int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev)
  593. {
  594. return cancel_delayed_work_sync(&bdev->wq);
  595. }
  596. EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue);
  597. void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched)
  598. {
  599. if (resched)
  600. schedule_delayed_work(&bdev->wq,
  601. ((HZ / 100) < 1) ? 1 : HZ / 100);
  602. }
  603. EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue);
  604. static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible,
  605. bool no_wait_reserve, bool no_wait_gpu)
  606. {
  607. struct ttm_bo_device *bdev = bo->bdev;
  608. struct ttm_mem_reg evict_mem;
  609. struct ttm_placement placement;
  610. int ret = 0;
  611. spin_lock(&bdev->fence_lock);
  612. ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
  613. spin_unlock(&bdev->fence_lock);
  614. if (unlikely(ret != 0)) {
  615. if (ret != -ERESTARTSYS) {
  616. printk(KERN_ERR TTM_PFX
  617. "Failed to expire sync object before "
  618. "buffer eviction.\n");
  619. }
  620. goto out;
  621. }
  622. BUG_ON(!atomic_read(&bo->reserved));
  623. evict_mem = bo->mem;
  624. evict_mem.mm_node = NULL;
  625. evict_mem.bus.io_reserved_vm = false;
  626. evict_mem.bus.io_reserved_count = 0;
  627. placement.fpfn = 0;
  628. placement.lpfn = 0;
  629. placement.num_placement = 0;
  630. placement.num_busy_placement = 0;
  631. bdev->driver->evict_flags(bo, &placement);
  632. ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible,
  633. no_wait_reserve, no_wait_gpu);
  634. if (ret) {
  635. if (ret != -ERESTARTSYS) {
  636. printk(KERN_ERR TTM_PFX
  637. "Failed to find memory space for "
  638. "buffer 0x%p eviction.\n", bo);
  639. ttm_bo_mem_space_debug(bo, &placement);
  640. }
  641. goto out;
  642. }
  643. ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible,
  644. no_wait_reserve, no_wait_gpu);
  645. if (ret) {
  646. if (ret != -ERESTARTSYS)
  647. printk(KERN_ERR TTM_PFX "Buffer eviction failed\n");
  648. ttm_bo_mem_put(bo, &evict_mem);
  649. goto out;
  650. }
  651. bo->evicted = true;
  652. out:
  653. return ret;
  654. }
  655. static int ttm_mem_evict_first(struct ttm_bo_device *bdev,
  656. uint32_t mem_type,
  657. bool interruptible, bool no_wait_reserve,
  658. bool no_wait_gpu)
  659. {
  660. struct ttm_bo_global *glob = bdev->glob;
  661. struct ttm_mem_type_manager *man = &bdev->man[mem_type];
  662. struct ttm_buffer_object *bo;
  663. int ret, put_count = 0;
  664. retry:
  665. spin_lock(&glob->lru_lock);
  666. if (list_empty(&man->lru)) {
  667. spin_unlock(&glob->lru_lock);
  668. return -EBUSY;
  669. }
  670. bo = list_first_entry(&man->lru, struct ttm_buffer_object, lru);
  671. kref_get(&bo->list_kref);
  672. if (!list_empty(&bo->ddestroy)) {
  673. spin_unlock(&glob->lru_lock);
  674. ret = ttm_bo_cleanup_refs(bo, interruptible,
  675. no_wait_reserve, no_wait_gpu);
  676. kref_put(&bo->list_kref, ttm_bo_release_list);
  677. if (likely(ret == 0 || ret == -ERESTARTSYS))
  678. return ret;
  679. goto retry;
  680. }
  681. ret = ttm_bo_reserve_locked(bo, false, no_wait_reserve, false, 0);
  682. if (unlikely(ret == -EBUSY)) {
  683. spin_unlock(&glob->lru_lock);
  684. if (likely(!no_wait_gpu))
  685. ret = ttm_bo_wait_unreserved(bo, interruptible);
  686. kref_put(&bo->list_kref, ttm_bo_release_list);
  687. /**
  688. * We *need* to retry after releasing the lru lock.
  689. */
  690. if (unlikely(ret != 0))
  691. return ret;
  692. goto retry;
  693. }
  694. put_count = ttm_bo_del_from_lru(bo);
  695. spin_unlock(&glob->lru_lock);
  696. BUG_ON(ret != 0);
  697. ttm_bo_list_ref_sub(bo, put_count, true);
  698. ret = ttm_bo_evict(bo, interruptible, no_wait_reserve, no_wait_gpu);
  699. ttm_bo_unreserve(bo);
  700. kref_put(&bo->list_kref, ttm_bo_release_list);
  701. return ret;
  702. }
  703. void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem)
  704. {
  705. struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type];
  706. if (mem->mm_node)
  707. (*man->func->put_node)(man, mem);
  708. }
  709. EXPORT_SYMBOL(ttm_bo_mem_put);
  710. /**
  711. * Repeatedly evict memory from the LRU for @mem_type until we create enough
  712. * space, or we've evicted everything and there isn't enough space.
  713. */
  714. static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
  715. uint32_t mem_type,
  716. struct ttm_placement *placement,
  717. struct ttm_mem_reg *mem,
  718. bool interruptible,
  719. bool no_wait_reserve,
  720. bool no_wait_gpu)
  721. {
  722. struct ttm_bo_device *bdev = bo->bdev;
  723. struct ttm_mem_type_manager *man = &bdev->man[mem_type];
  724. int ret;
  725. do {
  726. ret = (*man->func->get_node)(man, bo, placement, mem);
  727. if (unlikely(ret != 0))
  728. return ret;
  729. if (mem->mm_node)
  730. break;
  731. ret = ttm_mem_evict_first(bdev, mem_type, interruptible,
  732. no_wait_reserve, no_wait_gpu);
  733. if (unlikely(ret != 0))
  734. return ret;
  735. } while (1);
  736. if (mem->mm_node == NULL)
  737. return -ENOMEM;
  738. mem->mem_type = mem_type;
  739. return 0;
  740. }
  741. static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man,
  742. uint32_t cur_placement,
  743. uint32_t proposed_placement)
  744. {
  745. uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING;
  746. uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING;
  747. /**
  748. * Keep current caching if possible.
  749. */
  750. if ((cur_placement & caching) != 0)
  751. result |= (cur_placement & caching);
  752. else if ((man->default_caching & caching) != 0)
  753. result |= man->default_caching;
  754. else if ((TTM_PL_FLAG_CACHED & caching) != 0)
  755. result |= TTM_PL_FLAG_CACHED;
  756. else if ((TTM_PL_FLAG_WC & caching) != 0)
  757. result |= TTM_PL_FLAG_WC;
  758. else if ((TTM_PL_FLAG_UNCACHED & caching) != 0)
  759. result |= TTM_PL_FLAG_UNCACHED;
  760. return result;
  761. }
  762. static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man,
  763. bool disallow_fixed,
  764. uint32_t mem_type,
  765. uint32_t proposed_placement,
  766. uint32_t *masked_placement)
  767. {
  768. uint32_t cur_flags = ttm_bo_type_flags(mem_type);
  769. if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && disallow_fixed)
  770. return false;
  771. if ((cur_flags & proposed_placement & TTM_PL_MASK_MEM) == 0)
  772. return false;
  773. if ((proposed_placement & man->available_caching) == 0)
  774. return false;
  775. cur_flags |= (proposed_placement & man->available_caching);
  776. *masked_placement = cur_flags;
  777. return true;
  778. }
  779. /**
  780. * Creates space for memory region @mem according to its type.
  781. *
  782. * This function first searches for free space in compatible memory types in
  783. * the priority order defined by the driver. If free space isn't found, then
  784. * ttm_bo_mem_force_space is attempted in priority order to evict and find
  785. * space.
  786. */
  787. int ttm_bo_mem_space(struct ttm_buffer_object *bo,
  788. struct ttm_placement *placement,
  789. struct ttm_mem_reg *mem,
  790. bool interruptible, bool no_wait_reserve,
  791. bool no_wait_gpu)
  792. {
  793. struct ttm_bo_device *bdev = bo->bdev;
  794. struct ttm_mem_type_manager *man;
  795. uint32_t mem_type = TTM_PL_SYSTEM;
  796. uint32_t cur_flags = 0;
  797. bool type_found = false;
  798. bool type_ok = false;
  799. bool has_erestartsys = false;
  800. int i, ret;
  801. mem->mm_node = NULL;
  802. for (i = 0; i < placement->num_placement; ++i) {
  803. ret = ttm_mem_type_from_flags(placement->placement[i],
  804. &mem_type);
  805. if (ret)
  806. return ret;
  807. man = &bdev->man[mem_type];
  808. type_ok = ttm_bo_mt_compatible(man,
  809. bo->type == ttm_bo_type_user,
  810. mem_type,
  811. placement->placement[i],
  812. &cur_flags);
  813. if (!type_ok)
  814. continue;
  815. cur_flags = ttm_bo_select_caching(man, bo->mem.placement,
  816. cur_flags);
  817. /*
  818. * Use the access and other non-mapping-related flag bits from
  819. * the memory placement flags to the current flags
  820. */
  821. ttm_flag_masked(&cur_flags, placement->placement[i],
  822. ~TTM_PL_MASK_MEMTYPE);
  823. if (mem_type == TTM_PL_SYSTEM)
  824. break;
  825. if (man->has_type && man->use_type) {
  826. type_found = true;
  827. ret = (*man->func->get_node)(man, bo, placement, mem);
  828. if (unlikely(ret))
  829. return ret;
  830. }
  831. if (mem->mm_node)
  832. break;
  833. }
  834. if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) {
  835. mem->mem_type = mem_type;
  836. mem->placement = cur_flags;
  837. return 0;
  838. }
  839. if (!type_found)
  840. return -EINVAL;
  841. for (i = 0; i < placement->num_busy_placement; ++i) {
  842. ret = ttm_mem_type_from_flags(placement->busy_placement[i],
  843. &mem_type);
  844. if (ret)
  845. return ret;
  846. man = &bdev->man[mem_type];
  847. if (!man->has_type)
  848. continue;
  849. if (!ttm_bo_mt_compatible(man,
  850. bo->type == ttm_bo_type_user,
  851. mem_type,
  852. placement->busy_placement[i],
  853. &cur_flags))
  854. continue;
  855. cur_flags = ttm_bo_select_caching(man, bo->mem.placement,
  856. cur_flags);
  857. /*
  858. * Use the access and other non-mapping-related flag bits from
  859. * the memory placement flags to the current flags
  860. */
  861. ttm_flag_masked(&cur_flags, placement->busy_placement[i],
  862. ~TTM_PL_MASK_MEMTYPE);
  863. if (mem_type == TTM_PL_SYSTEM) {
  864. mem->mem_type = mem_type;
  865. mem->placement = cur_flags;
  866. mem->mm_node = NULL;
  867. return 0;
  868. }
  869. ret = ttm_bo_mem_force_space(bo, mem_type, placement, mem,
  870. interruptible, no_wait_reserve, no_wait_gpu);
  871. if (ret == 0 && mem->mm_node) {
  872. mem->placement = cur_flags;
  873. return 0;
  874. }
  875. if (ret == -ERESTARTSYS)
  876. has_erestartsys = true;
  877. }
  878. ret = (has_erestartsys) ? -ERESTARTSYS : -ENOMEM;
  879. return ret;
  880. }
  881. EXPORT_SYMBOL(ttm_bo_mem_space);
  882. int ttm_bo_wait_cpu(struct ttm_buffer_object *bo, bool no_wait)
  883. {
  884. if ((atomic_read(&bo->cpu_writers) > 0) && no_wait)
  885. return -EBUSY;
  886. return wait_event_interruptible(bo->event_queue,
  887. atomic_read(&bo->cpu_writers) == 0);
  888. }
  889. EXPORT_SYMBOL(ttm_bo_wait_cpu);
  890. int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
  891. struct ttm_placement *placement,
  892. bool interruptible, bool no_wait_reserve,
  893. bool no_wait_gpu)
  894. {
  895. int ret = 0;
  896. struct ttm_mem_reg mem;
  897. struct ttm_bo_device *bdev = bo->bdev;
  898. BUG_ON(!atomic_read(&bo->reserved));
  899. /*
  900. * FIXME: It's possible to pipeline buffer moves.
  901. * Have the driver move function wait for idle when necessary,
  902. * instead of doing it here.
  903. */
  904. spin_lock(&bdev->fence_lock);
  905. ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
  906. spin_unlock(&bdev->fence_lock);
  907. if (ret)
  908. return ret;
  909. mem.num_pages = bo->num_pages;
  910. mem.size = mem.num_pages << PAGE_SHIFT;
  911. mem.page_alignment = bo->mem.page_alignment;
  912. mem.bus.io_reserved_vm = false;
  913. mem.bus.io_reserved_count = 0;
  914. /*
  915. * Determine where to move the buffer.
  916. */
  917. ret = ttm_bo_mem_space(bo, placement, &mem, interruptible, no_wait_reserve, no_wait_gpu);
  918. if (ret)
  919. goto out_unlock;
  920. ret = ttm_bo_handle_move_mem(bo, &mem, false, interruptible, no_wait_reserve, no_wait_gpu);
  921. out_unlock:
  922. if (ret && mem.mm_node)
  923. ttm_bo_mem_put(bo, &mem);
  924. return ret;
  925. }
  926. static int ttm_bo_mem_compat(struct ttm_placement *placement,
  927. struct ttm_mem_reg *mem)
  928. {
  929. int i;
  930. if (mem->mm_node && placement->lpfn != 0 &&
  931. (mem->start < placement->fpfn ||
  932. mem->start + mem->num_pages > placement->lpfn))
  933. return -1;
  934. for (i = 0; i < placement->num_placement; i++) {
  935. if ((placement->placement[i] & mem->placement &
  936. TTM_PL_MASK_CACHING) &&
  937. (placement->placement[i] & mem->placement &
  938. TTM_PL_MASK_MEM))
  939. return i;
  940. }
  941. return -1;
  942. }
  943. int ttm_bo_validate(struct ttm_buffer_object *bo,
  944. struct ttm_placement *placement,
  945. bool interruptible, bool no_wait_reserve,
  946. bool no_wait_gpu)
  947. {
  948. int ret;
  949. BUG_ON(!atomic_read(&bo->reserved));
  950. /* Check that range is valid */
  951. if (placement->lpfn || placement->fpfn)
  952. if (placement->fpfn > placement->lpfn ||
  953. (placement->lpfn - placement->fpfn) < bo->num_pages)
  954. return -EINVAL;
  955. /*
  956. * Check whether we need to move buffer.
  957. */
  958. ret = ttm_bo_mem_compat(placement, &bo->mem);
  959. if (ret < 0) {
  960. ret = ttm_bo_move_buffer(bo, placement, interruptible, no_wait_reserve, no_wait_gpu);
  961. if (ret)
  962. return ret;
  963. } else {
  964. /*
  965. * Use the access and other non-mapping-related flag bits from
  966. * the compatible memory placement flags to the active flags
  967. */
  968. ttm_flag_masked(&bo->mem.placement, placement->placement[ret],
  969. ~TTM_PL_MASK_MEMTYPE);
  970. }
  971. /*
  972. * We might need to add a TTM.
  973. */
  974. if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
  975. ret = ttm_bo_add_ttm(bo, true);
  976. if (ret)
  977. return ret;
  978. }
  979. return 0;
  980. }
  981. EXPORT_SYMBOL(ttm_bo_validate);
  982. int ttm_bo_check_placement(struct ttm_buffer_object *bo,
  983. struct ttm_placement *placement)
  984. {
  985. BUG_ON((placement->fpfn || placement->lpfn) &&
  986. (bo->mem.num_pages > (placement->lpfn - placement->fpfn)));
  987. return 0;
  988. }
  989. int ttm_bo_init(struct ttm_bo_device *bdev,
  990. struct ttm_buffer_object *bo,
  991. unsigned long size,
  992. enum ttm_bo_type type,
  993. struct ttm_placement *placement,
  994. uint32_t page_alignment,
  995. unsigned long buffer_start,
  996. bool interruptible,
  997. struct file *persistent_swap_storage,
  998. size_t acc_size,
  999. void (*destroy) (struct ttm_buffer_object *))
  1000. {
  1001. int ret = 0;
  1002. unsigned long num_pages;
  1003. size += buffer_start & ~PAGE_MASK;
  1004. num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
  1005. if (num_pages == 0) {
  1006. printk(KERN_ERR TTM_PFX "Illegal buffer object size.\n");
  1007. if (destroy)
  1008. (*destroy)(bo);
  1009. else
  1010. kfree(bo);
  1011. return -EINVAL;
  1012. }
  1013. bo->destroy = destroy;
  1014. kref_init(&bo->kref);
  1015. kref_init(&bo->list_kref);
  1016. atomic_set(&bo->cpu_writers, 0);
  1017. atomic_set(&bo->reserved, 1);
  1018. init_waitqueue_head(&bo->event_queue);
  1019. INIT_LIST_HEAD(&bo->lru);
  1020. INIT_LIST_HEAD(&bo->ddestroy);
  1021. INIT_LIST_HEAD(&bo->swap);
  1022. INIT_LIST_HEAD(&bo->io_reserve_lru);
  1023. bo->bdev = bdev;
  1024. bo->glob = bdev->glob;
  1025. bo->type = type;
  1026. bo->num_pages = num_pages;
  1027. bo->mem.size = num_pages << PAGE_SHIFT;
  1028. bo->mem.mem_type = TTM_PL_SYSTEM;
  1029. bo->mem.num_pages = bo->num_pages;
  1030. bo->mem.mm_node = NULL;
  1031. bo->mem.page_alignment = page_alignment;
  1032. bo->mem.bus.io_reserved_vm = false;
  1033. bo->mem.bus.io_reserved_count = 0;
  1034. bo->buffer_start = buffer_start & PAGE_MASK;
  1035. bo->priv_flags = 0;
  1036. bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED);
  1037. bo->seq_valid = false;
  1038. bo->persistent_swap_storage = persistent_swap_storage;
  1039. bo->acc_size = acc_size;
  1040. atomic_inc(&bo->glob->bo_count);
  1041. ret = ttm_bo_check_placement(bo, placement);
  1042. if (unlikely(ret != 0))
  1043. goto out_err;
  1044. /*
  1045. * For ttm_bo_type_device buffers, allocate
  1046. * address space from the device.
  1047. */
  1048. if (bo->type == ttm_bo_type_device) {
  1049. ret = ttm_bo_setup_vm(bo);
  1050. if (ret)
  1051. goto out_err;
  1052. }
  1053. ret = ttm_bo_validate(bo, placement, interruptible, false, false);
  1054. if (ret)
  1055. goto out_err;
  1056. ttm_bo_unreserve(bo);
  1057. return 0;
  1058. out_err:
  1059. ttm_bo_unreserve(bo);
  1060. ttm_bo_unref(&bo);
  1061. return ret;
  1062. }
  1063. EXPORT_SYMBOL(ttm_bo_init);
  1064. static inline size_t ttm_bo_size(struct ttm_bo_global *glob,
  1065. unsigned long num_pages)
  1066. {
  1067. size_t page_array_size = (num_pages * sizeof(void *) + PAGE_SIZE - 1) &
  1068. PAGE_MASK;
  1069. return glob->ttm_bo_size + 2 * page_array_size;
  1070. }
  1071. int ttm_bo_create(struct ttm_bo_device *bdev,
  1072. unsigned long size,
  1073. enum ttm_bo_type type,
  1074. struct ttm_placement *placement,
  1075. uint32_t page_alignment,
  1076. unsigned long buffer_start,
  1077. bool interruptible,
  1078. struct file *persistent_swap_storage,
  1079. struct ttm_buffer_object **p_bo)
  1080. {
  1081. struct ttm_buffer_object *bo;
  1082. struct ttm_mem_global *mem_glob = bdev->glob->mem_glob;
  1083. int ret;
  1084. size_t acc_size =
  1085. ttm_bo_size(bdev->glob, (size + PAGE_SIZE - 1) >> PAGE_SHIFT);
  1086. ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false);
  1087. if (unlikely(ret != 0))
  1088. return ret;
  1089. bo = kzalloc(sizeof(*bo), GFP_KERNEL);
  1090. if (unlikely(bo == NULL)) {
  1091. ttm_mem_global_free(mem_glob, acc_size);
  1092. return -ENOMEM;
  1093. }
  1094. ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment,
  1095. buffer_start, interruptible,
  1096. persistent_swap_storage, acc_size, NULL);
  1097. if (likely(ret == 0))
  1098. *p_bo = bo;
  1099. return ret;
  1100. }
  1101. static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
  1102. unsigned mem_type, bool allow_errors)
  1103. {
  1104. struct ttm_mem_type_manager *man = &bdev->man[mem_type];
  1105. struct ttm_bo_global *glob = bdev->glob;
  1106. int ret;
  1107. /*
  1108. * Can't use standard list traversal since we're unlocking.
  1109. */
  1110. spin_lock(&glob->lru_lock);
  1111. while (!list_empty(&man->lru)) {
  1112. spin_unlock(&glob->lru_lock);
  1113. ret = ttm_mem_evict_first(bdev, mem_type, false, false, false);
  1114. if (ret) {
  1115. if (allow_errors) {
  1116. return ret;
  1117. } else {
  1118. printk(KERN_ERR TTM_PFX
  1119. "Cleanup eviction failed\n");
  1120. }
  1121. }
  1122. spin_lock(&glob->lru_lock);
  1123. }
  1124. spin_unlock(&glob->lru_lock);
  1125. return 0;
  1126. }
  1127. int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type)
  1128. {
  1129. struct ttm_mem_type_manager *man;
  1130. int ret = -EINVAL;
  1131. if (mem_type >= TTM_NUM_MEM_TYPES) {
  1132. printk(KERN_ERR TTM_PFX "Illegal memory type %d\n", mem_type);
  1133. return ret;
  1134. }
  1135. man = &bdev->man[mem_type];
  1136. if (!man->has_type) {
  1137. printk(KERN_ERR TTM_PFX "Trying to take down uninitialized "
  1138. "memory manager type %u\n", mem_type);
  1139. return ret;
  1140. }
  1141. man->use_type = false;
  1142. man->has_type = false;
  1143. ret = 0;
  1144. if (mem_type > 0) {
  1145. ttm_bo_force_list_clean(bdev, mem_type, false);
  1146. ret = (*man->func->takedown)(man);
  1147. }
  1148. return ret;
  1149. }
  1150. EXPORT_SYMBOL(ttm_bo_clean_mm);
  1151. int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type)
  1152. {
  1153. struct ttm_mem_type_manager *man = &bdev->man[mem_type];
  1154. if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) {
  1155. printk(KERN_ERR TTM_PFX
  1156. "Illegal memory manager memory type %u.\n",
  1157. mem_type);
  1158. return -EINVAL;
  1159. }
  1160. if (!man->has_type) {
  1161. printk(KERN_ERR TTM_PFX
  1162. "Memory type %u has not been initialized.\n",
  1163. mem_type);
  1164. return 0;
  1165. }
  1166. return ttm_bo_force_list_clean(bdev, mem_type, true);
  1167. }
  1168. EXPORT_SYMBOL(ttm_bo_evict_mm);
  1169. int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type,
  1170. unsigned long p_size)
  1171. {
  1172. int ret = -EINVAL;
  1173. struct ttm_mem_type_manager *man;
  1174. BUG_ON(type >= TTM_NUM_MEM_TYPES);
  1175. man = &bdev->man[type];
  1176. BUG_ON(man->has_type);
  1177. man->io_reserve_fastpath = true;
  1178. man->use_io_reserve_lru = false;
  1179. mutex_init(&man->io_reserve_mutex);
  1180. INIT_LIST_HEAD(&man->io_reserve_lru);
  1181. ret = bdev->driver->init_mem_type(bdev, type, man);
  1182. if (ret)
  1183. return ret;
  1184. man->bdev = bdev;
  1185. ret = 0;
  1186. if (type != TTM_PL_SYSTEM) {
  1187. ret = (*man->func->init)(man, p_size);
  1188. if (ret)
  1189. return ret;
  1190. }
  1191. man->has_type = true;
  1192. man->use_type = true;
  1193. man->size = p_size;
  1194. INIT_LIST_HEAD(&man->lru);
  1195. return 0;
  1196. }
  1197. EXPORT_SYMBOL(ttm_bo_init_mm);
  1198. static void ttm_bo_global_kobj_release(struct kobject *kobj)
  1199. {
  1200. struct ttm_bo_global *glob =
  1201. container_of(kobj, struct ttm_bo_global, kobj);
  1202. ttm_mem_unregister_shrink(glob->mem_glob, &glob->shrink);
  1203. __free_page(glob->dummy_read_page);
  1204. kfree(glob);
  1205. }
  1206. void ttm_bo_global_release(struct drm_global_reference *ref)
  1207. {
  1208. struct ttm_bo_global *glob = ref->object;
  1209. kobject_del(&glob->kobj);
  1210. kobject_put(&glob->kobj);
  1211. }
  1212. EXPORT_SYMBOL(ttm_bo_global_release);
  1213. int ttm_bo_global_init(struct drm_global_reference *ref)
  1214. {
  1215. struct ttm_bo_global_ref *bo_ref =
  1216. container_of(ref, struct ttm_bo_global_ref, ref);
  1217. struct ttm_bo_global *glob = ref->object;
  1218. int ret;
  1219. mutex_init(&glob->device_list_mutex);
  1220. spin_lock_init(&glob->lru_lock);
  1221. glob->mem_glob = bo_ref->mem_glob;
  1222. glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32);
  1223. if (unlikely(glob->dummy_read_page == NULL)) {
  1224. ret = -ENOMEM;
  1225. goto out_no_drp;
  1226. }
  1227. INIT_LIST_HEAD(&glob->swap_lru);
  1228. INIT_LIST_HEAD(&glob->device_list);
  1229. ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout);
  1230. ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink);
  1231. if (unlikely(ret != 0)) {
  1232. printk(KERN_ERR TTM_PFX
  1233. "Could not register buffer object swapout.\n");
  1234. goto out_no_shrink;
  1235. }
  1236. glob->ttm_bo_extra_size =
  1237. ttm_round_pot(sizeof(struct ttm_tt)) +
  1238. ttm_round_pot(sizeof(struct ttm_backend));
  1239. glob->ttm_bo_size = glob->ttm_bo_extra_size +
  1240. ttm_round_pot(sizeof(struct ttm_buffer_object));
  1241. atomic_set(&glob->bo_count, 0);
  1242. ret = kobject_init_and_add(
  1243. &glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects");
  1244. if (unlikely(ret != 0))
  1245. kobject_put(&glob->kobj);
  1246. return ret;
  1247. out_no_shrink:
  1248. __free_page(glob->dummy_read_page);
  1249. out_no_drp:
  1250. kfree(glob);
  1251. return ret;
  1252. }
  1253. EXPORT_SYMBOL(ttm_bo_global_init);
  1254. int ttm_bo_device_release(struct ttm_bo_device *bdev)
  1255. {
  1256. int ret = 0;
  1257. unsigned i = TTM_NUM_MEM_TYPES;
  1258. struct ttm_mem_type_manager *man;
  1259. struct ttm_bo_global *glob = bdev->glob;
  1260. while (i--) {
  1261. man = &bdev->man[i];
  1262. if (man->has_type) {
  1263. man->use_type = false;
  1264. if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) {
  1265. ret = -EBUSY;
  1266. printk(KERN_ERR TTM_PFX
  1267. "DRM memory manager type %d "
  1268. "is not clean.\n", i);
  1269. }
  1270. man->has_type = false;
  1271. }
  1272. }
  1273. mutex_lock(&glob->device_list_mutex);
  1274. list_del(&bdev->device_list);
  1275. mutex_unlock(&glob->device_list_mutex);
  1276. cancel_delayed_work_sync(&bdev->wq);
  1277. while (ttm_bo_delayed_delete(bdev, true))
  1278. ;
  1279. spin_lock(&glob->lru_lock);
  1280. if (list_empty(&bdev->ddestroy))
  1281. TTM_DEBUG("Delayed destroy list was clean\n");
  1282. if (list_empty(&bdev->man[0].lru))
  1283. TTM_DEBUG("Swap list was clean\n");
  1284. spin_unlock(&glob->lru_lock);
  1285. BUG_ON(!drm_mm_clean(&bdev->addr_space_mm));
  1286. write_lock(&bdev->vm_lock);
  1287. drm_mm_takedown(&bdev->addr_space_mm);
  1288. write_unlock(&bdev->vm_lock);
  1289. return ret;
  1290. }
  1291. EXPORT_SYMBOL(ttm_bo_device_release);
  1292. int ttm_bo_device_init(struct ttm_bo_device *bdev,
  1293. struct ttm_bo_global *glob,
  1294. struct ttm_bo_driver *driver,
  1295. uint64_t file_page_offset,
  1296. bool need_dma32)
  1297. {
  1298. int ret = -EINVAL;
  1299. rwlock_init(&bdev->vm_lock);
  1300. bdev->driver = driver;
  1301. memset(bdev->man, 0, sizeof(bdev->man));
  1302. /*
  1303. * Initialize the system memory buffer type.
  1304. * Other types need to be driver / IOCTL initialized.
  1305. */
  1306. ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0);
  1307. if (unlikely(ret != 0))
  1308. goto out_no_sys;
  1309. bdev->addr_space_rb = RB_ROOT;
  1310. ret = drm_mm_init(&bdev->addr_space_mm, file_page_offset, 0x10000000);
  1311. if (unlikely(ret != 0))
  1312. goto out_no_addr_mm;
  1313. INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue);
  1314. bdev->nice_mode = true;
  1315. INIT_LIST_HEAD(&bdev->ddestroy);
  1316. bdev->dev_mapping = NULL;
  1317. bdev->glob = glob;
  1318. bdev->need_dma32 = need_dma32;
  1319. bdev->val_seq = 0;
  1320. spin_lock_init(&bdev->fence_lock);
  1321. mutex_lock(&glob->device_list_mutex);
  1322. list_add_tail(&bdev->device_list, &glob->device_list);
  1323. mutex_unlock(&glob->device_list_mutex);
  1324. return 0;
  1325. out_no_addr_mm:
  1326. ttm_bo_clean_mm(bdev, 0);
  1327. out_no_sys:
  1328. return ret;
  1329. }
  1330. EXPORT_SYMBOL(ttm_bo_device_init);
  1331. /*
  1332. * buffer object vm functions.
  1333. */
  1334. bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
  1335. {
  1336. struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
  1337. if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
  1338. if (mem->mem_type == TTM_PL_SYSTEM)
  1339. return false;
  1340. if (man->flags & TTM_MEMTYPE_FLAG_CMA)
  1341. return false;
  1342. if (mem->placement & TTM_PL_FLAG_CACHED)
  1343. return false;
  1344. }
  1345. return true;
  1346. }
  1347. void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo)
  1348. {
  1349. struct ttm_bo_device *bdev = bo->bdev;
  1350. loff_t offset = (loff_t) bo->addr_space_offset;
  1351. loff_t holelen = ((loff_t) bo->mem.num_pages) << PAGE_SHIFT;
  1352. if (!bdev->dev_mapping)
  1353. return;
  1354. unmap_mapping_range(bdev->dev_mapping, offset, holelen, 1);
  1355. ttm_mem_io_free_vm(bo);
  1356. }
  1357. void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
  1358. {
  1359. struct ttm_bo_device *bdev = bo->bdev;
  1360. struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
  1361. ttm_mem_io_lock(man, false);
  1362. ttm_bo_unmap_virtual_locked(bo);
  1363. ttm_mem_io_unlock(man);
  1364. }
  1365. EXPORT_SYMBOL(ttm_bo_unmap_virtual);
  1366. static void ttm_bo_vm_insert_rb(struct ttm_buffer_object *bo)
  1367. {
  1368. struct ttm_bo_device *bdev = bo->bdev;
  1369. struct rb_node **cur = &bdev->addr_space_rb.rb_node;
  1370. struct rb_node *parent = NULL;
  1371. struct ttm_buffer_object *cur_bo;
  1372. unsigned long offset = bo->vm_node->start;
  1373. unsigned long cur_offset;
  1374. while (*cur) {
  1375. parent = *cur;
  1376. cur_bo = rb_entry(parent, struct ttm_buffer_object, vm_rb);
  1377. cur_offset = cur_bo->vm_node->start;
  1378. if (offset < cur_offset)
  1379. cur = &parent->rb_left;
  1380. else if (offset > cur_offset)
  1381. cur = &parent->rb_right;
  1382. else
  1383. BUG();
  1384. }
  1385. rb_link_node(&bo->vm_rb, parent, cur);
  1386. rb_insert_color(&bo->vm_rb, &bdev->addr_space_rb);
  1387. }
  1388. /**
  1389. * ttm_bo_setup_vm:
  1390. *
  1391. * @bo: the buffer to allocate address space for
  1392. *
  1393. * Allocate address space in the drm device so that applications
  1394. * can mmap the buffer and access the contents. This only
  1395. * applies to ttm_bo_type_device objects as others are not
  1396. * placed in the drm device address space.
  1397. */
  1398. static int ttm_bo_setup_vm(struct ttm_buffer_object *bo)
  1399. {
  1400. struct ttm_bo_device *bdev = bo->bdev;
  1401. int ret;
  1402. retry_pre_get:
  1403. ret = drm_mm_pre_get(&bdev->addr_space_mm);
  1404. if (unlikely(ret != 0))
  1405. return ret;
  1406. write_lock(&bdev->vm_lock);
  1407. bo->vm_node = drm_mm_search_free(&bdev->addr_space_mm,
  1408. bo->mem.num_pages, 0, 0);
  1409. if (unlikely(bo->vm_node == NULL)) {
  1410. ret = -ENOMEM;
  1411. goto out_unlock;
  1412. }
  1413. bo->vm_node = drm_mm_get_block_atomic(bo->vm_node,
  1414. bo->mem.num_pages, 0);
  1415. if (unlikely(bo->vm_node == NULL)) {
  1416. write_unlock(&bdev->vm_lock);
  1417. goto retry_pre_get;
  1418. }
  1419. ttm_bo_vm_insert_rb(bo);
  1420. write_unlock(&bdev->vm_lock);
  1421. bo->addr_space_offset = ((uint64_t) bo->vm_node->start) << PAGE_SHIFT;
  1422. return 0;
  1423. out_unlock:
  1424. write_unlock(&bdev->vm_lock);
  1425. return ret;
  1426. }
  1427. int ttm_bo_wait(struct ttm_buffer_object *bo,
  1428. bool lazy, bool interruptible, bool no_wait)
  1429. {
  1430. struct ttm_bo_driver *driver = bo->bdev->driver;
  1431. struct ttm_bo_device *bdev = bo->bdev;
  1432. void *sync_obj;
  1433. void *sync_obj_arg;
  1434. int ret = 0;
  1435. if (likely(bo->sync_obj == NULL))
  1436. return 0;
  1437. while (bo->sync_obj) {
  1438. if (driver->sync_obj_signaled(bo->sync_obj, bo->sync_obj_arg)) {
  1439. void *tmp_obj = bo->sync_obj;
  1440. bo->sync_obj = NULL;
  1441. clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags);
  1442. spin_unlock(&bdev->fence_lock);
  1443. driver->sync_obj_unref(&tmp_obj);
  1444. spin_lock(&bdev->fence_lock);
  1445. continue;
  1446. }
  1447. if (no_wait)
  1448. return -EBUSY;
  1449. sync_obj = driver->sync_obj_ref(bo->sync_obj);
  1450. sync_obj_arg = bo->sync_obj_arg;
  1451. spin_unlock(&bdev->fence_lock);
  1452. ret = driver->sync_obj_wait(sync_obj, sync_obj_arg,
  1453. lazy, interruptible);
  1454. if (unlikely(ret != 0)) {
  1455. driver->sync_obj_unref(&sync_obj);
  1456. spin_lock(&bdev->fence_lock);
  1457. return ret;
  1458. }
  1459. spin_lock(&bdev->fence_lock);
  1460. if (likely(bo->sync_obj == sync_obj &&
  1461. bo->sync_obj_arg == sync_obj_arg)) {
  1462. void *tmp_obj = bo->sync_obj;
  1463. bo->sync_obj = NULL;
  1464. clear_bit(TTM_BO_PRIV_FLAG_MOVING,
  1465. &bo->priv_flags);
  1466. spin_unlock(&bdev->fence_lock);
  1467. driver->sync_obj_unref(&sync_obj);
  1468. driver->sync_obj_unref(&tmp_obj);
  1469. spin_lock(&bdev->fence_lock);
  1470. } else {
  1471. spin_unlock(&bdev->fence_lock);
  1472. driver->sync_obj_unref(&sync_obj);
  1473. spin_lock(&bdev->fence_lock);
  1474. }
  1475. }
  1476. return 0;
  1477. }
  1478. EXPORT_SYMBOL(ttm_bo_wait);
  1479. int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait)
  1480. {
  1481. struct ttm_bo_device *bdev = bo->bdev;
  1482. int ret = 0;
  1483. /*
  1484. * Using ttm_bo_reserve makes sure the lru lists are updated.
  1485. */
  1486. ret = ttm_bo_reserve(bo, true, no_wait, false, 0);
  1487. if (unlikely(ret != 0))
  1488. return ret;
  1489. spin_lock(&bdev->fence_lock);
  1490. ret = ttm_bo_wait(bo, false, true, no_wait);
  1491. spin_unlock(&bdev->fence_lock);
  1492. if (likely(ret == 0))
  1493. atomic_inc(&bo->cpu_writers);
  1494. ttm_bo_unreserve(bo);
  1495. return ret;
  1496. }
  1497. EXPORT_SYMBOL(ttm_bo_synccpu_write_grab);
  1498. void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo)
  1499. {
  1500. if (atomic_dec_and_test(&bo->cpu_writers))
  1501. wake_up_all(&bo->event_queue);
  1502. }
  1503. EXPORT_SYMBOL(ttm_bo_synccpu_write_release);
  1504. /**
  1505. * A buffer object shrink method that tries to swap out the first
  1506. * buffer object on the bo_global::swap_lru list.
  1507. */
  1508. static int ttm_bo_swapout(struct ttm_mem_shrink *shrink)
  1509. {
  1510. struct ttm_bo_global *glob =
  1511. container_of(shrink, struct ttm_bo_global, shrink);
  1512. struct ttm_buffer_object *bo;
  1513. int ret = -EBUSY;
  1514. int put_count;
  1515. uint32_t swap_placement = (TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM);
  1516. spin_lock(&glob->lru_lock);
  1517. while (ret == -EBUSY) {
  1518. if (unlikely(list_empty(&glob->swap_lru))) {
  1519. spin_unlock(&glob->lru_lock);
  1520. return -EBUSY;
  1521. }
  1522. bo = list_first_entry(&glob->swap_lru,
  1523. struct ttm_buffer_object, swap);
  1524. kref_get(&bo->list_kref);
  1525. if (!list_empty(&bo->ddestroy)) {
  1526. spin_unlock(&glob->lru_lock);
  1527. (void) ttm_bo_cleanup_refs(bo, false, false, false);
  1528. kref_put(&bo->list_kref, ttm_bo_release_list);
  1529. continue;
  1530. }
  1531. /**
  1532. * Reserve buffer. Since we unlock while sleeping, we need
  1533. * to re-check that nobody removed us from the swap-list while
  1534. * we slept.
  1535. */
  1536. ret = ttm_bo_reserve_locked(bo, false, true, false, 0);
  1537. if (unlikely(ret == -EBUSY)) {
  1538. spin_unlock(&glob->lru_lock);
  1539. ttm_bo_wait_unreserved(bo, false);
  1540. kref_put(&bo->list_kref, ttm_bo_release_list);
  1541. spin_lock(&glob->lru_lock);
  1542. }
  1543. }
  1544. BUG_ON(ret != 0);
  1545. put_count = ttm_bo_del_from_lru(bo);
  1546. spin_unlock(&glob->lru_lock);
  1547. ttm_bo_list_ref_sub(bo, put_count, true);
  1548. /**
  1549. * Wait for GPU, then move to system cached.
  1550. */
  1551. spin_lock(&bo->bdev->fence_lock);
  1552. ret = ttm_bo_wait(bo, false, false, false);
  1553. spin_unlock(&bo->bdev->fence_lock);
  1554. if (unlikely(ret != 0))
  1555. goto out;
  1556. if ((bo->mem.placement & swap_placement) != swap_placement) {
  1557. struct ttm_mem_reg evict_mem;
  1558. evict_mem = bo->mem;
  1559. evict_mem.mm_node = NULL;
  1560. evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED;
  1561. evict_mem.mem_type = TTM_PL_SYSTEM;
  1562. ret = ttm_bo_handle_move_mem(bo, &evict_mem, true,
  1563. false, false, false);
  1564. if (unlikely(ret != 0))
  1565. goto out;
  1566. }
  1567. ttm_bo_unmap_virtual(bo);
  1568. /**
  1569. * Swap out. Buffer will be swapped in again as soon as
  1570. * anyone tries to access a ttm page.
  1571. */
  1572. if (bo->bdev->driver->swap_notify)
  1573. bo->bdev->driver->swap_notify(bo);
  1574. ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage);
  1575. out:
  1576. /**
  1577. *
  1578. * Unreserve without putting on LRU to avoid swapping out an
  1579. * already swapped buffer.
  1580. */
  1581. atomic_set(&bo->reserved, 0);
  1582. wake_up_all(&bo->event_queue);
  1583. kref_put(&bo->list_kref, ttm_bo_release_list);
  1584. return ret;
  1585. }
  1586. void ttm_bo_swapout_all(struct ttm_bo_device *bdev)
  1587. {
  1588. while (ttm_bo_swapout(&bdev->glob->shrink) == 0)
  1589. ;
  1590. }
  1591. EXPORT_SYMBOL(ttm_bo_swapout_all);