/arch/powerpc/platforms/ps3/mm.c

http://github.com/mirrors/linux · C · 1237 lines · 848 code · 203 blank · 186 comment · 69 complexity · 0b956ded064f5ac93e33341fa9b3fa08 MD5 · raw file

  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * PS3 address space management.
  4. *
  5. * Copyright (C) 2006 Sony Computer Entertainment Inc.
  6. * Copyright 2006 Sony Corp.
  7. */
  8. #include <linux/kernel.h>
  9. #include <linux/export.h>
  10. #include <linux/memblock.h>
  11. #include <linux/slab.h>
  12. #include <asm/cell-regs.h>
  13. #include <asm/firmware.h>
  14. #include <asm/prom.h>
  15. #include <asm/udbg.h>
  16. #include <asm/lv1call.h>
  17. #include <asm/setup.h>
  18. #include "platform.h"
  19. #if defined(DEBUG)
  20. #define DBG udbg_printf
  21. #else
  22. #define DBG pr_devel
  23. #endif
  24. enum {
  25. #if defined(CONFIG_PS3_DYNAMIC_DMA)
  26. USE_DYNAMIC_DMA = 1,
  27. #else
  28. USE_DYNAMIC_DMA = 0,
  29. #endif
  30. };
  31. enum {
  32. PAGE_SHIFT_4K = 12U,
  33. PAGE_SHIFT_64K = 16U,
  34. PAGE_SHIFT_16M = 24U,
  35. };
  36. static unsigned long make_page_sizes(unsigned long a, unsigned long b)
  37. {
  38. return (a << 56) | (b << 48);
  39. }
  40. enum {
  41. ALLOCATE_MEMORY_TRY_ALT_UNIT = 0X04,
  42. ALLOCATE_MEMORY_ADDR_ZERO = 0X08,
  43. };
  44. /* valid htab sizes are {18,19,20} = 256K, 512K, 1M */
  45. enum {
  46. HTAB_SIZE_MAX = 20U, /* HV limit of 1MB */
  47. HTAB_SIZE_MIN = 18U, /* CPU limit of 256KB */
  48. };
  49. /*============================================================================*/
  50. /* virtual address space routines */
  51. /*============================================================================*/
  52. /**
  53. * struct mem_region - memory region structure
  54. * @base: base address
  55. * @size: size in bytes
  56. * @offset: difference between base and rm.size
  57. * @destroy: flag if region should be destroyed upon shutdown
  58. */
  59. struct mem_region {
  60. u64 base;
  61. u64 size;
  62. unsigned long offset;
  63. int destroy;
  64. };
  65. /**
  66. * struct map - address space state variables holder
  67. * @total: total memory available as reported by HV
  68. * @vas_id - HV virtual address space id
  69. * @htab_size: htab size in bytes
  70. *
  71. * The HV virtual address space (vas) allows for hotplug memory regions.
  72. * Memory regions can be created and destroyed in the vas at runtime.
  73. * @rm: real mode (bootmem) region
  74. * @r1: highmem region(s)
  75. *
  76. * ps3 addresses
  77. * virt_addr: a cpu 'translated' effective address
  78. * phys_addr: an address in what Linux thinks is the physical address space
  79. * lpar_addr: an address in the HV virtual address space
  80. * bus_addr: an io controller 'translated' address on a device bus
  81. */
  82. struct map {
  83. u64 total;
  84. u64 vas_id;
  85. u64 htab_size;
  86. struct mem_region rm;
  87. struct mem_region r1;
  88. };
  89. #define debug_dump_map(x) _debug_dump_map(x, __func__, __LINE__)
  90. static void __maybe_unused _debug_dump_map(const struct map *m,
  91. const char *func, int line)
  92. {
  93. DBG("%s:%d: map.total = %llxh\n", func, line, m->total);
  94. DBG("%s:%d: map.rm.size = %llxh\n", func, line, m->rm.size);
  95. DBG("%s:%d: map.vas_id = %llu\n", func, line, m->vas_id);
  96. DBG("%s:%d: map.htab_size = %llxh\n", func, line, m->htab_size);
  97. DBG("%s:%d: map.r1.base = %llxh\n", func, line, m->r1.base);
  98. DBG("%s:%d: map.r1.offset = %lxh\n", func, line, m->r1.offset);
  99. DBG("%s:%d: map.r1.size = %llxh\n", func, line, m->r1.size);
  100. }
  101. static struct map map;
  102. /**
  103. * ps3_mm_phys_to_lpar - translate a linux physical address to lpar address
  104. * @phys_addr: linux physical address
  105. */
  106. unsigned long ps3_mm_phys_to_lpar(unsigned long phys_addr)
  107. {
  108. BUG_ON(is_kernel_addr(phys_addr));
  109. return (phys_addr < map.rm.size || phys_addr >= map.total)
  110. ? phys_addr : phys_addr + map.r1.offset;
  111. }
  112. EXPORT_SYMBOL(ps3_mm_phys_to_lpar);
  113. /**
  114. * ps3_mm_vas_create - create the virtual address space
  115. */
  116. void __init ps3_mm_vas_create(unsigned long* htab_size)
  117. {
  118. int result;
  119. u64 start_address;
  120. u64 size;
  121. u64 access_right;
  122. u64 max_page_size;
  123. u64 flags;
  124. result = lv1_query_logical_partition_address_region_info(0,
  125. &start_address, &size, &access_right, &max_page_size,
  126. &flags);
  127. if (result) {
  128. DBG("%s:%d: lv1_query_logical_partition_address_region_info "
  129. "failed: %s\n", __func__, __LINE__,
  130. ps3_result(result));
  131. goto fail;
  132. }
  133. if (max_page_size < PAGE_SHIFT_16M) {
  134. DBG("%s:%d: bad max_page_size %llxh\n", __func__, __LINE__,
  135. max_page_size);
  136. goto fail;
  137. }
  138. BUILD_BUG_ON(CONFIG_PS3_HTAB_SIZE > HTAB_SIZE_MAX);
  139. BUILD_BUG_ON(CONFIG_PS3_HTAB_SIZE < HTAB_SIZE_MIN);
  140. result = lv1_construct_virtual_address_space(CONFIG_PS3_HTAB_SIZE,
  141. 2, make_page_sizes(PAGE_SHIFT_16M, PAGE_SHIFT_64K),
  142. &map.vas_id, &map.htab_size);
  143. if (result) {
  144. DBG("%s:%d: lv1_construct_virtual_address_space failed: %s\n",
  145. __func__, __LINE__, ps3_result(result));
  146. goto fail;
  147. }
  148. result = lv1_select_virtual_address_space(map.vas_id);
  149. if (result) {
  150. DBG("%s:%d: lv1_select_virtual_address_space failed: %s\n",
  151. __func__, __LINE__, ps3_result(result));
  152. goto fail;
  153. }
  154. *htab_size = map.htab_size;
  155. debug_dump_map(&map);
  156. return;
  157. fail:
  158. panic("ps3_mm_vas_create failed");
  159. }
  160. /**
  161. * ps3_mm_vas_destroy -
  162. */
  163. void ps3_mm_vas_destroy(void)
  164. {
  165. int result;
  166. DBG("%s:%d: map.vas_id = %llu\n", __func__, __LINE__, map.vas_id);
  167. if (map.vas_id) {
  168. result = lv1_select_virtual_address_space(0);
  169. BUG_ON(result);
  170. result = lv1_destruct_virtual_address_space(map.vas_id);
  171. BUG_ON(result);
  172. map.vas_id = 0;
  173. }
  174. }
  175. static int ps3_mm_get_repository_highmem(struct mem_region *r)
  176. {
  177. int result;
  178. /* Assume a single highmem region. */
  179. result = ps3_repository_read_highmem_info(0, &r->base, &r->size);
  180. if (result)
  181. goto zero_region;
  182. if (!r->base || !r->size) {
  183. result = -1;
  184. goto zero_region;
  185. }
  186. r->offset = r->base - map.rm.size;
  187. DBG("%s:%d: Found high region in repository: %llxh %llxh\n",
  188. __func__, __LINE__, r->base, r->size);
  189. return 0;
  190. zero_region:
  191. DBG("%s:%d: No high region in repository.\n", __func__, __LINE__);
  192. r->size = r->base = r->offset = 0;
  193. return result;
  194. }
  195. static int ps3_mm_set_repository_highmem(const struct mem_region *r)
  196. {
  197. /* Assume a single highmem region. */
  198. return r ? ps3_repository_write_highmem_info(0, r->base, r->size) :
  199. ps3_repository_write_highmem_info(0, 0, 0);
  200. }
  201. /**
  202. * ps3_mm_region_create - create a memory region in the vas
  203. * @r: pointer to a struct mem_region to accept initialized values
  204. * @size: requested region size
  205. *
  206. * This implementation creates the region with the vas large page size.
  207. * @size is rounded down to a multiple of the vas large page size.
  208. */
  209. static int ps3_mm_region_create(struct mem_region *r, unsigned long size)
  210. {
  211. int result;
  212. u64 muid;
  213. r->size = _ALIGN_DOWN(size, 1 << PAGE_SHIFT_16M);
  214. DBG("%s:%d requested %lxh\n", __func__, __LINE__, size);
  215. DBG("%s:%d actual %llxh\n", __func__, __LINE__, r->size);
  216. DBG("%s:%d difference %llxh (%lluMB)\n", __func__, __LINE__,
  217. size - r->size, (size - r->size) / 1024 / 1024);
  218. if (r->size == 0) {
  219. DBG("%s:%d: size == 0\n", __func__, __LINE__);
  220. result = -1;
  221. goto zero_region;
  222. }
  223. result = lv1_allocate_memory(r->size, PAGE_SHIFT_16M, 0,
  224. ALLOCATE_MEMORY_TRY_ALT_UNIT, &r->base, &muid);
  225. if (result || r->base < map.rm.size) {
  226. DBG("%s:%d: lv1_allocate_memory failed: %s\n",
  227. __func__, __LINE__, ps3_result(result));
  228. goto zero_region;
  229. }
  230. r->destroy = 1;
  231. r->offset = r->base - map.rm.size;
  232. return result;
  233. zero_region:
  234. r->size = r->base = r->offset = 0;
  235. return result;
  236. }
  237. /**
  238. * ps3_mm_region_destroy - destroy a memory region
  239. * @r: pointer to struct mem_region
  240. */
  241. static void ps3_mm_region_destroy(struct mem_region *r)
  242. {
  243. int result;
  244. if (!r->destroy) {
  245. pr_info("%s:%d: Not destroying high region: %llxh %llxh\n",
  246. __func__, __LINE__, r->base, r->size);
  247. return;
  248. }
  249. DBG("%s:%d: r->base = %llxh\n", __func__, __LINE__, r->base);
  250. if (r->base) {
  251. result = lv1_release_memory(r->base);
  252. BUG_ON(result);
  253. r->size = r->base = r->offset = 0;
  254. map.total = map.rm.size;
  255. }
  256. ps3_mm_set_repository_highmem(NULL);
  257. }
  258. /*============================================================================*/
  259. /* dma routines */
  260. /*============================================================================*/
  261. /**
  262. * dma_sb_lpar_to_bus - Translate an lpar address to ioc mapped bus address.
  263. * @r: pointer to dma region structure
  264. * @lpar_addr: HV lpar address
  265. */
  266. static unsigned long dma_sb_lpar_to_bus(struct ps3_dma_region *r,
  267. unsigned long lpar_addr)
  268. {
  269. if (lpar_addr >= map.rm.size)
  270. lpar_addr -= map.r1.offset;
  271. BUG_ON(lpar_addr < r->offset);
  272. BUG_ON(lpar_addr >= r->offset + r->len);
  273. return r->bus_addr + lpar_addr - r->offset;
  274. }
  275. #define dma_dump_region(_a) _dma_dump_region(_a, __func__, __LINE__)
  276. static void __maybe_unused _dma_dump_region(const struct ps3_dma_region *r,
  277. const char *func, int line)
  278. {
  279. DBG("%s:%d: dev %llu:%llu\n", func, line, r->dev->bus_id,
  280. r->dev->dev_id);
  281. DBG("%s:%d: page_size %u\n", func, line, r->page_size);
  282. DBG("%s:%d: bus_addr %lxh\n", func, line, r->bus_addr);
  283. DBG("%s:%d: len %lxh\n", func, line, r->len);
  284. DBG("%s:%d: offset %lxh\n", func, line, r->offset);
  285. }
  286. /**
  287. * dma_chunk - A chunk of dma pages mapped by the io controller.
  288. * @region - The dma region that owns this chunk.
  289. * @lpar_addr: Starting lpar address of the area to map.
  290. * @bus_addr: Starting ioc bus address of the area to map.
  291. * @len: Length in bytes of the area to map.
  292. * @link: A struct list_head used with struct ps3_dma_region.chunk_list, the
  293. * list of all chuncks owned by the region.
  294. *
  295. * This implementation uses a very simple dma page manager
  296. * based on the dma_chunk structure. This scheme assumes
  297. * that all drivers use very well behaved dma ops.
  298. */
  299. struct dma_chunk {
  300. struct ps3_dma_region *region;
  301. unsigned long lpar_addr;
  302. unsigned long bus_addr;
  303. unsigned long len;
  304. struct list_head link;
  305. unsigned int usage_count;
  306. };
  307. #define dma_dump_chunk(_a) _dma_dump_chunk(_a, __func__, __LINE__)
  308. static void _dma_dump_chunk (const struct dma_chunk* c, const char* func,
  309. int line)
  310. {
  311. DBG("%s:%d: r.dev %llu:%llu\n", func, line,
  312. c->region->dev->bus_id, c->region->dev->dev_id);
  313. DBG("%s:%d: r.bus_addr %lxh\n", func, line, c->region->bus_addr);
  314. DBG("%s:%d: r.page_size %u\n", func, line, c->region->page_size);
  315. DBG("%s:%d: r.len %lxh\n", func, line, c->region->len);
  316. DBG("%s:%d: r.offset %lxh\n", func, line, c->region->offset);
  317. DBG("%s:%d: c.lpar_addr %lxh\n", func, line, c->lpar_addr);
  318. DBG("%s:%d: c.bus_addr %lxh\n", func, line, c->bus_addr);
  319. DBG("%s:%d: c.len %lxh\n", func, line, c->len);
  320. }
  321. static struct dma_chunk * dma_find_chunk(struct ps3_dma_region *r,
  322. unsigned long bus_addr, unsigned long len)
  323. {
  324. struct dma_chunk *c;
  325. unsigned long aligned_bus = _ALIGN_DOWN(bus_addr, 1 << r->page_size);
  326. unsigned long aligned_len = _ALIGN_UP(len+bus_addr-aligned_bus,
  327. 1 << r->page_size);
  328. list_for_each_entry(c, &r->chunk_list.head, link) {
  329. /* intersection */
  330. if (aligned_bus >= c->bus_addr &&
  331. aligned_bus + aligned_len <= c->bus_addr + c->len)
  332. return c;
  333. /* below */
  334. if (aligned_bus + aligned_len <= c->bus_addr)
  335. continue;
  336. /* above */
  337. if (aligned_bus >= c->bus_addr + c->len)
  338. continue;
  339. /* we don't handle the multi-chunk case for now */
  340. dma_dump_chunk(c);
  341. BUG();
  342. }
  343. return NULL;
  344. }
  345. static struct dma_chunk *dma_find_chunk_lpar(struct ps3_dma_region *r,
  346. unsigned long lpar_addr, unsigned long len)
  347. {
  348. struct dma_chunk *c;
  349. unsigned long aligned_lpar = _ALIGN_DOWN(lpar_addr, 1 << r->page_size);
  350. unsigned long aligned_len = _ALIGN_UP(len + lpar_addr - aligned_lpar,
  351. 1 << r->page_size);
  352. list_for_each_entry(c, &r->chunk_list.head, link) {
  353. /* intersection */
  354. if (c->lpar_addr <= aligned_lpar &&
  355. aligned_lpar < c->lpar_addr + c->len) {
  356. if (aligned_lpar + aligned_len <= c->lpar_addr + c->len)
  357. return c;
  358. else {
  359. dma_dump_chunk(c);
  360. BUG();
  361. }
  362. }
  363. /* below */
  364. if (aligned_lpar + aligned_len <= c->lpar_addr) {
  365. continue;
  366. }
  367. /* above */
  368. if (c->lpar_addr + c->len <= aligned_lpar) {
  369. continue;
  370. }
  371. }
  372. return NULL;
  373. }
  374. static int dma_sb_free_chunk(struct dma_chunk *c)
  375. {
  376. int result = 0;
  377. if (c->bus_addr) {
  378. result = lv1_unmap_device_dma_region(c->region->dev->bus_id,
  379. c->region->dev->dev_id, c->bus_addr, c->len);
  380. BUG_ON(result);
  381. }
  382. kfree(c);
  383. return result;
  384. }
  385. static int dma_ioc0_free_chunk(struct dma_chunk *c)
  386. {
  387. int result = 0;
  388. int iopage;
  389. unsigned long offset;
  390. struct ps3_dma_region *r = c->region;
  391. DBG("%s:start\n", __func__);
  392. for (iopage = 0; iopage < (c->len >> r->page_size); iopage++) {
  393. offset = (1 << r->page_size) * iopage;
  394. /* put INVALID entry */
  395. result = lv1_put_iopte(0,
  396. c->bus_addr + offset,
  397. c->lpar_addr + offset,
  398. r->ioid,
  399. 0);
  400. DBG("%s: bus=%#lx, lpar=%#lx, ioid=%d\n", __func__,
  401. c->bus_addr + offset,
  402. c->lpar_addr + offset,
  403. r->ioid);
  404. if (result) {
  405. DBG("%s:%d: lv1_put_iopte failed: %s\n", __func__,
  406. __LINE__, ps3_result(result));
  407. }
  408. }
  409. kfree(c);
  410. DBG("%s:end\n", __func__);
  411. return result;
  412. }
  413. /**
  414. * dma_sb_map_pages - Maps dma pages into the io controller bus address space.
  415. * @r: Pointer to a struct ps3_dma_region.
  416. * @phys_addr: Starting physical address of the area to map.
  417. * @len: Length in bytes of the area to map.
  418. * c_out: A pointer to receive an allocated struct dma_chunk for this area.
  419. *
  420. * This is the lowest level dma mapping routine, and is the one that will
  421. * make the HV call to add the pages into the io controller address space.
  422. */
  423. static int dma_sb_map_pages(struct ps3_dma_region *r, unsigned long phys_addr,
  424. unsigned long len, struct dma_chunk **c_out, u64 iopte_flag)
  425. {
  426. int result;
  427. struct dma_chunk *c;
  428. c = kzalloc(sizeof(*c), GFP_ATOMIC);
  429. if (!c) {
  430. result = -ENOMEM;
  431. goto fail_alloc;
  432. }
  433. c->region = r;
  434. c->lpar_addr = ps3_mm_phys_to_lpar(phys_addr);
  435. c->bus_addr = dma_sb_lpar_to_bus(r, c->lpar_addr);
  436. c->len = len;
  437. BUG_ON(iopte_flag != 0xf800000000000000UL);
  438. result = lv1_map_device_dma_region(c->region->dev->bus_id,
  439. c->region->dev->dev_id, c->lpar_addr,
  440. c->bus_addr, c->len, iopte_flag);
  441. if (result) {
  442. DBG("%s:%d: lv1_map_device_dma_region failed: %s\n",
  443. __func__, __LINE__, ps3_result(result));
  444. goto fail_map;
  445. }
  446. list_add(&c->link, &r->chunk_list.head);
  447. *c_out = c;
  448. return 0;
  449. fail_map:
  450. kfree(c);
  451. fail_alloc:
  452. *c_out = NULL;
  453. DBG(" <- %s:%d\n", __func__, __LINE__);
  454. return result;
  455. }
  456. static int dma_ioc0_map_pages(struct ps3_dma_region *r, unsigned long phys_addr,
  457. unsigned long len, struct dma_chunk **c_out,
  458. u64 iopte_flag)
  459. {
  460. int result;
  461. struct dma_chunk *c, *last;
  462. int iopage, pages;
  463. unsigned long offset;
  464. DBG(KERN_ERR "%s: phy=%#lx, lpar%#lx, len=%#lx\n", __func__,
  465. phys_addr, ps3_mm_phys_to_lpar(phys_addr), len);
  466. c = kzalloc(sizeof(*c), GFP_ATOMIC);
  467. if (!c) {
  468. result = -ENOMEM;
  469. goto fail_alloc;
  470. }
  471. c->region = r;
  472. c->len = len;
  473. c->lpar_addr = ps3_mm_phys_to_lpar(phys_addr);
  474. /* allocate IO address */
  475. if (list_empty(&r->chunk_list.head)) {
  476. /* first one */
  477. c->bus_addr = r->bus_addr;
  478. } else {
  479. /* derive from last bus addr*/
  480. last = list_entry(r->chunk_list.head.next,
  481. struct dma_chunk, link);
  482. c->bus_addr = last->bus_addr + last->len;
  483. DBG("%s: last bus=%#lx, len=%#lx\n", __func__,
  484. last->bus_addr, last->len);
  485. }
  486. /* FIXME: check whether length exceeds region size */
  487. /* build ioptes for the area */
  488. pages = len >> r->page_size;
  489. DBG("%s: pgsize=%#x len=%#lx pages=%#x iopteflag=%#llx\n", __func__,
  490. r->page_size, r->len, pages, iopte_flag);
  491. for (iopage = 0; iopage < pages; iopage++) {
  492. offset = (1 << r->page_size) * iopage;
  493. result = lv1_put_iopte(0,
  494. c->bus_addr + offset,
  495. c->lpar_addr + offset,
  496. r->ioid,
  497. iopte_flag);
  498. if (result) {
  499. pr_warn("%s:%d: lv1_put_iopte failed: %s\n",
  500. __func__, __LINE__, ps3_result(result));
  501. goto fail_map;
  502. }
  503. DBG("%s: pg=%d bus=%#lx, lpar=%#lx, ioid=%#x\n", __func__,
  504. iopage, c->bus_addr + offset, c->lpar_addr + offset,
  505. r->ioid);
  506. }
  507. /* be sure that last allocated one is inserted at head */
  508. list_add(&c->link, &r->chunk_list.head);
  509. *c_out = c;
  510. DBG("%s: end\n", __func__);
  511. return 0;
  512. fail_map:
  513. for (iopage--; 0 <= iopage; iopage--) {
  514. lv1_put_iopte(0,
  515. c->bus_addr + offset,
  516. c->lpar_addr + offset,
  517. r->ioid,
  518. 0);
  519. }
  520. kfree(c);
  521. fail_alloc:
  522. *c_out = NULL;
  523. return result;
  524. }
  525. /**
  526. * dma_sb_region_create - Create a device dma region.
  527. * @r: Pointer to a struct ps3_dma_region.
  528. *
  529. * This is the lowest level dma region create routine, and is the one that
  530. * will make the HV call to create the region.
  531. */
  532. static int dma_sb_region_create(struct ps3_dma_region *r)
  533. {
  534. int result;
  535. u64 bus_addr;
  536. DBG(" -> %s:%d:\n", __func__, __LINE__);
  537. BUG_ON(!r);
  538. if (!r->dev->bus_id) {
  539. pr_info("%s:%d: %llu:%llu no dma\n", __func__, __LINE__,
  540. r->dev->bus_id, r->dev->dev_id);
  541. return 0;
  542. }
  543. DBG("%s:%u: len = 0x%lx, page_size = %u, offset = 0x%lx\n", __func__,
  544. __LINE__, r->len, r->page_size, r->offset);
  545. BUG_ON(!r->len);
  546. BUG_ON(!r->page_size);
  547. BUG_ON(!r->region_ops);
  548. INIT_LIST_HEAD(&r->chunk_list.head);
  549. spin_lock_init(&r->chunk_list.lock);
  550. result = lv1_allocate_device_dma_region(r->dev->bus_id, r->dev->dev_id,
  551. roundup_pow_of_two(r->len), r->page_size, r->region_type,
  552. &bus_addr);
  553. r->bus_addr = bus_addr;
  554. if (result) {
  555. DBG("%s:%d: lv1_allocate_device_dma_region failed: %s\n",
  556. __func__, __LINE__, ps3_result(result));
  557. r->len = r->bus_addr = 0;
  558. }
  559. return result;
  560. }
  561. static int dma_ioc0_region_create(struct ps3_dma_region *r)
  562. {
  563. int result;
  564. u64 bus_addr;
  565. INIT_LIST_HEAD(&r->chunk_list.head);
  566. spin_lock_init(&r->chunk_list.lock);
  567. result = lv1_allocate_io_segment(0,
  568. r->len,
  569. r->page_size,
  570. &bus_addr);
  571. r->bus_addr = bus_addr;
  572. if (result) {
  573. DBG("%s:%d: lv1_allocate_io_segment failed: %s\n",
  574. __func__, __LINE__, ps3_result(result));
  575. r->len = r->bus_addr = 0;
  576. }
  577. DBG("%s: len=%#lx, pg=%d, bus=%#lx\n", __func__,
  578. r->len, r->page_size, r->bus_addr);
  579. return result;
  580. }
  581. /**
  582. * dma_region_free - Free a device dma region.
  583. * @r: Pointer to a struct ps3_dma_region.
  584. *
  585. * This is the lowest level dma region free routine, and is the one that
  586. * will make the HV call to free the region.
  587. */
  588. static int dma_sb_region_free(struct ps3_dma_region *r)
  589. {
  590. int result;
  591. struct dma_chunk *c;
  592. struct dma_chunk *tmp;
  593. BUG_ON(!r);
  594. if (!r->dev->bus_id) {
  595. pr_info("%s:%d: %llu:%llu no dma\n", __func__, __LINE__,
  596. r->dev->bus_id, r->dev->dev_id);
  597. return 0;
  598. }
  599. list_for_each_entry_safe(c, tmp, &r->chunk_list.head, link) {
  600. list_del(&c->link);
  601. dma_sb_free_chunk(c);
  602. }
  603. result = lv1_free_device_dma_region(r->dev->bus_id, r->dev->dev_id,
  604. r->bus_addr);
  605. if (result)
  606. DBG("%s:%d: lv1_free_device_dma_region failed: %s\n",
  607. __func__, __LINE__, ps3_result(result));
  608. r->bus_addr = 0;
  609. return result;
  610. }
  611. static int dma_ioc0_region_free(struct ps3_dma_region *r)
  612. {
  613. int result;
  614. struct dma_chunk *c, *n;
  615. DBG("%s: start\n", __func__);
  616. list_for_each_entry_safe(c, n, &r->chunk_list.head, link) {
  617. list_del(&c->link);
  618. dma_ioc0_free_chunk(c);
  619. }
  620. result = lv1_release_io_segment(0, r->bus_addr);
  621. if (result)
  622. DBG("%s:%d: lv1_free_device_dma_region failed: %s\n",
  623. __func__, __LINE__, ps3_result(result));
  624. r->bus_addr = 0;
  625. DBG("%s: end\n", __func__);
  626. return result;
  627. }
  628. /**
  629. * dma_sb_map_area - Map an area of memory into a device dma region.
  630. * @r: Pointer to a struct ps3_dma_region.
  631. * @virt_addr: Starting virtual address of the area to map.
  632. * @len: Length in bytes of the area to map.
  633. * @bus_addr: A pointer to return the starting ioc bus address of the area to
  634. * map.
  635. *
  636. * This is the common dma mapping routine.
  637. */
  638. static int dma_sb_map_area(struct ps3_dma_region *r, unsigned long virt_addr,
  639. unsigned long len, dma_addr_t *bus_addr,
  640. u64 iopte_flag)
  641. {
  642. int result;
  643. unsigned long flags;
  644. struct dma_chunk *c;
  645. unsigned long phys_addr = is_kernel_addr(virt_addr) ? __pa(virt_addr)
  646. : virt_addr;
  647. unsigned long aligned_phys = _ALIGN_DOWN(phys_addr, 1 << r->page_size);
  648. unsigned long aligned_len = _ALIGN_UP(len + phys_addr - aligned_phys,
  649. 1 << r->page_size);
  650. *bus_addr = dma_sb_lpar_to_bus(r, ps3_mm_phys_to_lpar(phys_addr));
  651. if (!USE_DYNAMIC_DMA) {
  652. unsigned long lpar_addr = ps3_mm_phys_to_lpar(phys_addr);
  653. DBG(" -> %s:%d\n", __func__, __LINE__);
  654. DBG("%s:%d virt_addr %lxh\n", __func__, __LINE__,
  655. virt_addr);
  656. DBG("%s:%d phys_addr %lxh\n", __func__, __LINE__,
  657. phys_addr);
  658. DBG("%s:%d lpar_addr %lxh\n", __func__, __LINE__,
  659. lpar_addr);
  660. DBG("%s:%d len %lxh\n", __func__, __LINE__, len);
  661. DBG("%s:%d bus_addr %llxh (%lxh)\n", __func__, __LINE__,
  662. *bus_addr, len);
  663. }
  664. spin_lock_irqsave(&r->chunk_list.lock, flags);
  665. c = dma_find_chunk(r, *bus_addr, len);
  666. if (c) {
  667. DBG("%s:%d: reusing mapped chunk", __func__, __LINE__);
  668. dma_dump_chunk(c);
  669. c->usage_count++;
  670. spin_unlock_irqrestore(&r->chunk_list.lock, flags);
  671. return 0;
  672. }
  673. result = dma_sb_map_pages(r, aligned_phys, aligned_len, &c, iopte_flag);
  674. if (result) {
  675. *bus_addr = 0;
  676. DBG("%s:%d: dma_sb_map_pages failed (%d)\n",
  677. __func__, __LINE__, result);
  678. spin_unlock_irqrestore(&r->chunk_list.lock, flags);
  679. return result;
  680. }
  681. c->usage_count = 1;
  682. spin_unlock_irqrestore(&r->chunk_list.lock, flags);
  683. return result;
  684. }
  685. static int dma_ioc0_map_area(struct ps3_dma_region *r, unsigned long virt_addr,
  686. unsigned long len, dma_addr_t *bus_addr,
  687. u64 iopte_flag)
  688. {
  689. int result;
  690. unsigned long flags;
  691. struct dma_chunk *c;
  692. unsigned long phys_addr = is_kernel_addr(virt_addr) ? __pa(virt_addr)
  693. : virt_addr;
  694. unsigned long aligned_phys = _ALIGN_DOWN(phys_addr, 1 << r->page_size);
  695. unsigned long aligned_len = _ALIGN_UP(len + phys_addr - aligned_phys,
  696. 1 << r->page_size);
  697. DBG(KERN_ERR "%s: vaddr=%#lx, len=%#lx\n", __func__,
  698. virt_addr, len);
  699. DBG(KERN_ERR "%s: ph=%#lx a_ph=%#lx a_l=%#lx\n", __func__,
  700. phys_addr, aligned_phys, aligned_len);
  701. spin_lock_irqsave(&r->chunk_list.lock, flags);
  702. c = dma_find_chunk_lpar(r, ps3_mm_phys_to_lpar(phys_addr), len);
  703. if (c) {
  704. /* FIXME */
  705. BUG();
  706. *bus_addr = c->bus_addr + phys_addr - aligned_phys;
  707. c->usage_count++;
  708. spin_unlock_irqrestore(&r->chunk_list.lock, flags);
  709. return 0;
  710. }
  711. result = dma_ioc0_map_pages(r, aligned_phys, aligned_len, &c,
  712. iopte_flag);
  713. if (result) {
  714. *bus_addr = 0;
  715. DBG("%s:%d: dma_ioc0_map_pages failed (%d)\n",
  716. __func__, __LINE__, result);
  717. spin_unlock_irqrestore(&r->chunk_list.lock, flags);
  718. return result;
  719. }
  720. *bus_addr = c->bus_addr + phys_addr - aligned_phys;
  721. DBG("%s: va=%#lx pa=%#lx a_pa=%#lx bus=%#llx\n", __func__,
  722. virt_addr, phys_addr, aligned_phys, *bus_addr);
  723. c->usage_count = 1;
  724. spin_unlock_irqrestore(&r->chunk_list.lock, flags);
  725. return result;
  726. }
  727. /**
  728. * dma_sb_unmap_area - Unmap an area of memory from a device dma region.
  729. * @r: Pointer to a struct ps3_dma_region.
  730. * @bus_addr: The starting ioc bus address of the area to unmap.
  731. * @len: Length in bytes of the area to unmap.
  732. *
  733. * This is the common dma unmap routine.
  734. */
  735. static int dma_sb_unmap_area(struct ps3_dma_region *r, dma_addr_t bus_addr,
  736. unsigned long len)
  737. {
  738. unsigned long flags;
  739. struct dma_chunk *c;
  740. spin_lock_irqsave(&r->chunk_list.lock, flags);
  741. c = dma_find_chunk(r, bus_addr, len);
  742. if (!c) {
  743. unsigned long aligned_bus = _ALIGN_DOWN(bus_addr,
  744. 1 << r->page_size);
  745. unsigned long aligned_len = _ALIGN_UP(len + bus_addr
  746. - aligned_bus, 1 << r->page_size);
  747. DBG("%s:%d: not found: bus_addr %llxh\n",
  748. __func__, __LINE__, bus_addr);
  749. DBG("%s:%d: not found: len %lxh\n",
  750. __func__, __LINE__, len);
  751. DBG("%s:%d: not found: aligned_bus %lxh\n",
  752. __func__, __LINE__, aligned_bus);
  753. DBG("%s:%d: not found: aligned_len %lxh\n",
  754. __func__, __LINE__, aligned_len);
  755. BUG();
  756. }
  757. c->usage_count--;
  758. if (!c->usage_count) {
  759. list_del(&c->link);
  760. dma_sb_free_chunk(c);
  761. }
  762. spin_unlock_irqrestore(&r->chunk_list.lock, flags);
  763. return 0;
  764. }
  765. static int dma_ioc0_unmap_area(struct ps3_dma_region *r,
  766. dma_addr_t bus_addr, unsigned long len)
  767. {
  768. unsigned long flags;
  769. struct dma_chunk *c;
  770. DBG("%s: start a=%#llx l=%#lx\n", __func__, bus_addr, len);
  771. spin_lock_irqsave(&r->chunk_list.lock, flags);
  772. c = dma_find_chunk(r, bus_addr, len);
  773. if (!c) {
  774. unsigned long aligned_bus = _ALIGN_DOWN(bus_addr,
  775. 1 << r->page_size);
  776. unsigned long aligned_len = _ALIGN_UP(len + bus_addr
  777. - aligned_bus,
  778. 1 << r->page_size);
  779. DBG("%s:%d: not found: bus_addr %llxh\n",
  780. __func__, __LINE__, bus_addr);
  781. DBG("%s:%d: not found: len %lxh\n",
  782. __func__, __LINE__, len);
  783. DBG("%s:%d: not found: aligned_bus %lxh\n",
  784. __func__, __LINE__, aligned_bus);
  785. DBG("%s:%d: not found: aligned_len %lxh\n",
  786. __func__, __LINE__, aligned_len);
  787. BUG();
  788. }
  789. c->usage_count--;
  790. if (!c->usage_count) {
  791. list_del(&c->link);
  792. dma_ioc0_free_chunk(c);
  793. }
  794. spin_unlock_irqrestore(&r->chunk_list.lock, flags);
  795. DBG("%s: end\n", __func__);
  796. return 0;
  797. }
  798. /**
  799. * dma_sb_region_create_linear - Setup a linear dma mapping for a device.
  800. * @r: Pointer to a struct ps3_dma_region.
  801. *
  802. * This routine creates an HV dma region for the device and maps all available
  803. * ram into the io controller bus address space.
  804. */
  805. static int dma_sb_region_create_linear(struct ps3_dma_region *r)
  806. {
  807. int result;
  808. unsigned long virt_addr, len;
  809. dma_addr_t tmp;
  810. if (r->len > 16*1024*1024) { /* FIXME: need proper fix */
  811. /* force 16M dma pages for linear mapping */
  812. if (r->page_size != PS3_DMA_16M) {
  813. pr_info("%s:%d: forcing 16M pages for linear map\n",
  814. __func__, __LINE__);
  815. r->page_size = PS3_DMA_16M;
  816. r->len = _ALIGN_UP(r->len, 1 << r->page_size);
  817. }
  818. }
  819. result = dma_sb_region_create(r);
  820. BUG_ON(result);
  821. if (r->offset < map.rm.size) {
  822. /* Map (part of) 1st RAM chunk */
  823. virt_addr = map.rm.base + r->offset;
  824. len = map.rm.size - r->offset;
  825. if (len > r->len)
  826. len = r->len;
  827. result = dma_sb_map_area(r, virt_addr, len, &tmp,
  828. CBE_IOPTE_PP_W | CBE_IOPTE_PP_R | CBE_IOPTE_SO_RW |
  829. CBE_IOPTE_M);
  830. BUG_ON(result);
  831. }
  832. if (r->offset + r->len > map.rm.size) {
  833. /* Map (part of) 2nd RAM chunk */
  834. virt_addr = map.rm.size;
  835. len = r->len;
  836. if (r->offset >= map.rm.size)
  837. virt_addr += r->offset - map.rm.size;
  838. else
  839. len -= map.rm.size - r->offset;
  840. result = dma_sb_map_area(r, virt_addr, len, &tmp,
  841. CBE_IOPTE_PP_W | CBE_IOPTE_PP_R | CBE_IOPTE_SO_RW |
  842. CBE_IOPTE_M);
  843. BUG_ON(result);
  844. }
  845. return result;
  846. }
  847. /**
  848. * dma_sb_region_free_linear - Free a linear dma mapping for a device.
  849. * @r: Pointer to a struct ps3_dma_region.
  850. *
  851. * This routine will unmap all mapped areas and free the HV dma region.
  852. */
  853. static int dma_sb_region_free_linear(struct ps3_dma_region *r)
  854. {
  855. int result;
  856. dma_addr_t bus_addr;
  857. unsigned long len, lpar_addr;
  858. if (r->offset < map.rm.size) {
  859. /* Unmap (part of) 1st RAM chunk */
  860. lpar_addr = map.rm.base + r->offset;
  861. len = map.rm.size - r->offset;
  862. if (len > r->len)
  863. len = r->len;
  864. bus_addr = dma_sb_lpar_to_bus(r, lpar_addr);
  865. result = dma_sb_unmap_area(r, bus_addr, len);
  866. BUG_ON(result);
  867. }
  868. if (r->offset + r->len > map.rm.size) {
  869. /* Unmap (part of) 2nd RAM chunk */
  870. lpar_addr = map.r1.base;
  871. len = r->len;
  872. if (r->offset >= map.rm.size)
  873. lpar_addr += r->offset - map.rm.size;
  874. else
  875. len -= map.rm.size - r->offset;
  876. bus_addr = dma_sb_lpar_to_bus(r, lpar_addr);
  877. result = dma_sb_unmap_area(r, bus_addr, len);
  878. BUG_ON(result);
  879. }
  880. result = dma_sb_region_free(r);
  881. BUG_ON(result);
  882. return result;
  883. }
  884. /**
  885. * dma_sb_map_area_linear - Map an area of memory into a device dma region.
  886. * @r: Pointer to a struct ps3_dma_region.
  887. * @virt_addr: Starting virtual address of the area to map.
  888. * @len: Length in bytes of the area to map.
  889. * @bus_addr: A pointer to return the starting ioc bus address of the area to
  890. * map.
  891. *
  892. * This routine just returns the corresponding bus address. Actual mapping
  893. * occurs in dma_region_create_linear().
  894. */
  895. static int dma_sb_map_area_linear(struct ps3_dma_region *r,
  896. unsigned long virt_addr, unsigned long len, dma_addr_t *bus_addr,
  897. u64 iopte_flag)
  898. {
  899. unsigned long phys_addr = is_kernel_addr(virt_addr) ? __pa(virt_addr)
  900. : virt_addr;
  901. *bus_addr = dma_sb_lpar_to_bus(r, ps3_mm_phys_to_lpar(phys_addr));
  902. return 0;
  903. }
  904. /**
  905. * dma_unmap_area_linear - Unmap an area of memory from a device dma region.
  906. * @r: Pointer to a struct ps3_dma_region.
  907. * @bus_addr: The starting ioc bus address of the area to unmap.
  908. * @len: Length in bytes of the area to unmap.
  909. *
  910. * This routine does nothing. Unmapping occurs in dma_sb_region_free_linear().
  911. */
  912. static int dma_sb_unmap_area_linear(struct ps3_dma_region *r,
  913. dma_addr_t bus_addr, unsigned long len)
  914. {
  915. return 0;
  916. };
  917. static const struct ps3_dma_region_ops ps3_dma_sb_region_ops = {
  918. .create = dma_sb_region_create,
  919. .free = dma_sb_region_free,
  920. .map = dma_sb_map_area,
  921. .unmap = dma_sb_unmap_area
  922. };
  923. static const struct ps3_dma_region_ops ps3_dma_sb_region_linear_ops = {
  924. .create = dma_sb_region_create_linear,
  925. .free = dma_sb_region_free_linear,
  926. .map = dma_sb_map_area_linear,
  927. .unmap = dma_sb_unmap_area_linear
  928. };
  929. static const struct ps3_dma_region_ops ps3_dma_ioc0_region_ops = {
  930. .create = dma_ioc0_region_create,
  931. .free = dma_ioc0_region_free,
  932. .map = dma_ioc0_map_area,
  933. .unmap = dma_ioc0_unmap_area
  934. };
  935. int ps3_dma_region_init(struct ps3_system_bus_device *dev,
  936. struct ps3_dma_region *r, enum ps3_dma_page_size page_size,
  937. enum ps3_dma_region_type region_type, void *addr, unsigned long len)
  938. {
  939. unsigned long lpar_addr;
  940. lpar_addr = addr ? ps3_mm_phys_to_lpar(__pa(addr)) : 0;
  941. r->dev = dev;
  942. r->page_size = page_size;
  943. r->region_type = region_type;
  944. r->offset = lpar_addr;
  945. if (r->offset >= map.rm.size)
  946. r->offset -= map.r1.offset;
  947. r->len = len ? len : _ALIGN_UP(map.total, 1 << r->page_size);
  948. switch (dev->dev_type) {
  949. case PS3_DEVICE_TYPE_SB:
  950. r->region_ops = (USE_DYNAMIC_DMA)
  951. ? &ps3_dma_sb_region_ops
  952. : &ps3_dma_sb_region_linear_ops;
  953. break;
  954. case PS3_DEVICE_TYPE_IOC0:
  955. r->region_ops = &ps3_dma_ioc0_region_ops;
  956. break;
  957. default:
  958. BUG();
  959. return -EINVAL;
  960. }
  961. return 0;
  962. }
  963. EXPORT_SYMBOL(ps3_dma_region_init);
  964. int ps3_dma_region_create(struct ps3_dma_region *r)
  965. {
  966. BUG_ON(!r);
  967. BUG_ON(!r->region_ops);
  968. BUG_ON(!r->region_ops->create);
  969. return r->region_ops->create(r);
  970. }
  971. EXPORT_SYMBOL(ps3_dma_region_create);
  972. int ps3_dma_region_free(struct ps3_dma_region *r)
  973. {
  974. BUG_ON(!r);
  975. BUG_ON(!r->region_ops);
  976. BUG_ON(!r->region_ops->free);
  977. return r->region_ops->free(r);
  978. }
  979. EXPORT_SYMBOL(ps3_dma_region_free);
  980. int ps3_dma_map(struct ps3_dma_region *r, unsigned long virt_addr,
  981. unsigned long len, dma_addr_t *bus_addr,
  982. u64 iopte_flag)
  983. {
  984. return r->region_ops->map(r, virt_addr, len, bus_addr, iopte_flag);
  985. }
  986. int ps3_dma_unmap(struct ps3_dma_region *r, dma_addr_t bus_addr,
  987. unsigned long len)
  988. {
  989. return r->region_ops->unmap(r, bus_addr, len);
  990. }
  991. /*============================================================================*/
  992. /* system startup routines */
  993. /*============================================================================*/
  994. /**
  995. * ps3_mm_init - initialize the address space state variables
  996. */
  997. void __init ps3_mm_init(void)
  998. {
  999. int result;
  1000. DBG(" -> %s:%d\n", __func__, __LINE__);
  1001. result = ps3_repository_read_mm_info(&map.rm.base, &map.rm.size,
  1002. &map.total);
  1003. if (result)
  1004. panic("ps3_repository_read_mm_info() failed");
  1005. map.rm.offset = map.rm.base;
  1006. map.vas_id = map.htab_size = 0;
  1007. /* this implementation assumes map.rm.base is zero */
  1008. BUG_ON(map.rm.base);
  1009. BUG_ON(!map.rm.size);
  1010. /* Check if we got the highmem region from an earlier boot step */
  1011. if (ps3_mm_get_repository_highmem(&map.r1)) {
  1012. result = ps3_mm_region_create(&map.r1, map.total - map.rm.size);
  1013. if (!result)
  1014. ps3_mm_set_repository_highmem(&map.r1);
  1015. }
  1016. /* correct map.total for the real total amount of memory we use */
  1017. map.total = map.rm.size + map.r1.size;
  1018. if (!map.r1.size) {
  1019. DBG("%s:%d: No highmem region found\n", __func__, __LINE__);
  1020. } else {
  1021. DBG("%s:%d: Adding highmem region: %llxh %llxh\n",
  1022. __func__, __LINE__, map.rm.size,
  1023. map.total - map.rm.size);
  1024. memblock_add(map.rm.size, map.total - map.rm.size);
  1025. }
  1026. DBG(" <- %s:%d\n", __func__, __LINE__);
  1027. }
  1028. /**
  1029. * ps3_mm_shutdown - final cleanup of address space
  1030. */
  1031. void ps3_mm_shutdown(void)
  1032. {
  1033. ps3_mm_region_destroy(&map.r1);
  1034. }