/Ethereal-msm8939-beta9/arch/metag/kernel/dma.c

https://bitbucket.org/MilosStamenkovic95/etherealos · C · 507 lines · 305 code · 76 blank · 126 comment · 37 complexity · a7866b00a40c73ca63e98f74c772521a MD5 · raw file

  1. /*
  2. * Meta version derived from arch/powerpc/lib/dma-noncoherent.c
  3. * Copyright (C) 2008 Imagination Technologies Ltd.
  4. *
  5. * PowerPC version derived from arch/arm/mm/consistent.c
  6. * Copyright (C) 2001 Dan Malek (dmalek@jlc.net)
  7. *
  8. * Copyright (C) 2000 Russell King
  9. *
  10. * Consistent memory allocators. Used for DMA devices that want to
  11. * share uncached memory with the processor core. The function return
  12. * is the virtual address and 'dma_handle' is the physical address.
  13. * Mostly stolen from the ARM port, with some changes for PowerPC.
  14. * -- Dan
  15. *
  16. * Reorganized to get rid of the arch-specific consistent_* functions
  17. * and provide non-coherent implementations for the DMA API. -Matt
  18. *
  19. * Added in_interrupt() safe dma_alloc_coherent()/dma_free_coherent()
  20. * implementation. This is pulled straight from ARM and barely
  21. * modified. -Matt
  22. *
  23. * This program is free software; you can redistribute it and/or modify
  24. * it under the terms of the GNU General Public License version 2 as
  25. * published by the Free Software Foundation.
  26. */
  27. #include <linux/sched.h>
  28. #include <linux/kernel.h>
  29. #include <linux/errno.h>
  30. #include <linux/export.h>
  31. #include <linux/string.h>
  32. #include <linux/types.h>
  33. #include <linux/highmem.h>
  34. #include <linux/dma-mapping.h>
  35. #include <linux/slab.h>
  36. #include <asm/tlbflush.h>
  37. #include <asm/mmu.h>
  38. #define CONSISTENT_OFFSET(x) (((unsigned long)(x) - CONSISTENT_START) \
  39. >> PAGE_SHIFT)
  40. static u64 get_coherent_dma_mask(struct device *dev)
  41. {
  42. u64 mask = ~0ULL;
  43. if (dev) {
  44. mask = dev->coherent_dma_mask;
  45. /*
  46. * Sanity check the DMA mask - it must be non-zero, and
  47. * must be able to be satisfied by a DMA allocation.
  48. */
  49. if (mask == 0) {
  50. dev_warn(dev, "coherent DMA mask is unset\n");
  51. return 0;
  52. }
  53. }
  54. return mask;
  55. }
  56. /*
  57. * This is the page table (2MB) covering uncached, DMA consistent allocations
  58. */
  59. static pte_t *consistent_pte;
  60. static DEFINE_SPINLOCK(consistent_lock);
  61. /*
  62. * VM region handling support.
  63. *
  64. * This should become something generic, handling VM region allocations for
  65. * vmalloc and similar (ioremap, module space, etc).
  66. *
  67. * I envisage vmalloc()'s supporting vm_struct becoming:
  68. *
  69. * struct vm_struct {
  70. * struct metag_vm_region region;
  71. * unsigned long flags;
  72. * struct page **pages;
  73. * unsigned int nr_pages;
  74. * unsigned long phys_addr;
  75. * };
  76. *
  77. * get_vm_area() would then call metag_vm_region_alloc with an appropriate
  78. * struct metag_vm_region head (eg):
  79. *
  80. * struct metag_vm_region vmalloc_head = {
  81. * .vm_list = LIST_HEAD_INIT(vmalloc_head.vm_list),
  82. * .vm_start = VMALLOC_START,
  83. * .vm_end = VMALLOC_END,
  84. * };
  85. *
  86. * However, vmalloc_head.vm_start is variable (typically, it is dependent on
  87. * the amount of RAM found at boot time.) I would imagine that get_vm_area()
  88. * would have to initialise this each time prior to calling
  89. * metag_vm_region_alloc().
  90. */
  91. struct metag_vm_region {
  92. struct list_head vm_list;
  93. unsigned long vm_start;
  94. unsigned long vm_end;
  95. struct page *vm_pages;
  96. int vm_active;
  97. };
  98. static struct metag_vm_region consistent_head = {
  99. .vm_list = LIST_HEAD_INIT(consistent_head.vm_list),
  100. .vm_start = CONSISTENT_START,
  101. .vm_end = CONSISTENT_END,
  102. };
  103. static struct metag_vm_region *metag_vm_region_alloc(struct metag_vm_region
  104. *head, size_t size,
  105. gfp_t gfp)
  106. {
  107. unsigned long addr = head->vm_start, end = head->vm_end - size;
  108. unsigned long flags;
  109. struct metag_vm_region *c, *new;
  110. new = kmalloc(sizeof(struct metag_vm_region), gfp);
  111. if (!new)
  112. goto out;
  113. spin_lock_irqsave(&consistent_lock, flags);
  114. list_for_each_entry(c, &head->vm_list, vm_list) {
  115. if ((addr + size) < addr)
  116. goto nospc;
  117. if ((addr + size) <= c->vm_start)
  118. goto found;
  119. addr = c->vm_end;
  120. if (addr > end)
  121. goto nospc;
  122. }
  123. found:
  124. /*
  125. * Insert this entry _before_ the one we found.
  126. */
  127. list_add_tail(&new->vm_list, &c->vm_list);
  128. new->vm_start = addr;
  129. new->vm_end = addr + size;
  130. new->vm_active = 1;
  131. spin_unlock_irqrestore(&consistent_lock, flags);
  132. return new;
  133. nospc:
  134. spin_unlock_irqrestore(&consistent_lock, flags);
  135. kfree(new);
  136. out:
  137. return NULL;
  138. }
  139. static struct metag_vm_region *metag_vm_region_find(struct metag_vm_region
  140. *head, unsigned long addr)
  141. {
  142. struct metag_vm_region *c;
  143. list_for_each_entry(c, &head->vm_list, vm_list) {
  144. if (c->vm_active && c->vm_start == addr)
  145. goto out;
  146. }
  147. c = NULL;
  148. out:
  149. return c;
  150. }
  151. /*
  152. * Allocate DMA-coherent memory space and return both the kernel remapped
  153. * virtual and bus address for that space.
  154. */
  155. void *dma_alloc_coherent(struct device *dev, size_t size,
  156. dma_addr_t *handle, gfp_t gfp)
  157. {
  158. struct page *page;
  159. struct metag_vm_region *c;
  160. unsigned long order;
  161. u64 mask = get_coherent_dma_mask(dev);
  162. u64 limit;
  163. if (!consistent_pte) {
  164. pr_err("%s: not initialised\n", __func__);
  165. dump_stack();
  166. return NULL;
  167. }
  168. if (!mask)
  169. goto no_page;
  170. size = PAGE_ALIGN(size);
  171. limit = (mask + 1) & ~mask;
  172. if ((limit && size >= limit)
  173. || size >= (CONSISTENT_END - CONSISTENT_START)) {
  174. pr_warn("coherent allocation too big (requested %#x mask %#Lx)\n",
  175. size, mask);
  176. return NULL;
  177. }
  178. order = get_order(size);
  179. if (mask != 0xffffffff)
  180. gfp |= GFP_DMA;
  181. page = alloc_pages(gfp, order);
  182. if (!page)
  183. goto no_page;
  184. /*
  185. * Invalidate any data that might be lurking in the
  186. * kernel direct-mapped region for device DMA.
  187. */
  188. {
  189. void *kaddr = page_address(page);
  190. memset(kaddr, 0, size);
  191. flush_dcache_region(kaddr, size);
  192. }
  193. /*
  194. * Allocate a virtual address in the consistent mapping region.
  195. */
  196. c = metag_vm_region_alloc(&consistent_head, size,
  197. gfp & ~(__GFP_DMA | __GFP_HIGHMEM));
  198. if (c) {
  199. unsigned long vaddr = c->vm_start;
  200. pte_t *pte = consistent_pte + CONSISTENT_OFFSET(vaddr);
  201. struct page *end = page + (1 << order);
  202. c->vm_pages = page;
  203. split_page(page, order);
  204. /*
  205. * Set the "dma handle"
  206. */
  207. *handle = page_to_bus(page);
  208. do {
  209. BUG_ON(!pte_none(*pte));
  210. SetPageReserved(page);
  211. set_pte_at(&init_mm, vaddr,
  212. pte, mk_pte(page,
  213. pgprot_writecombine
  214. (PAGE_KERNEL)));
  215. page++;
  216. pte++;
  217. vaddr += PAGE_SIZE;
  218. } while (size -= PAGE_SIZE);
  219. /*
  220. * Free the otherwise unused pages.
  221. */
  222. while (page < end) {
  223. __free_page(page);
  224. page++;
  225. }
  226. return (void *)c->vm_start;
  227. }
  228. if (page)
  229. __free_pages(page, order);
  230. no_page:
  231. return NULL;
  232. }
  233. EXPORT_SYMBOL(dma_alloc_coherent);
  234. /*
  235. * free a page as defined by the above mapping.
  236. */
  237. void dma_free_coherent(struct device *dev, size_t size,
  238. void *vaddr, dma_addr_t dma_handle)
  239. {
  240. struct metag_vm_region *c;
  241. unsigned long flags, addr;
  242. pte_t *ptep;
  243. size = PAGE_ALIGN(size);
  244. spin_lock_irqsave(&consistent_lock, flags);
  245. c = metag_vm_region_find(&consistent_head, (unsigned long)vaddr);
  246. if (!c)
  247. goto no_area;
  248. c->vm_active = 0;
  249. if ((c->vm_end - c->vm_start) != size) {
  250. pr_err("%s: freeing wrong coherent size (%ld != %d)\n",
  251. __func__, c->vm_end - c->vm_start, size);
  252. dump_stack();
  253. size = c->vm_end - c->vm_start;
  254. }
  255. ptep = consistent_pte + CONSISTENT_OFFSET(c->vm_start);
  256. addr = c->vm_start;
  257. do {
  258. pte_t pte = ptep_get_and_clear(&init_mm, addr, ptep);
  259. unsigned long pfn;
  260. ptep++;
  261. addr += PAGE_SIZE;
  262. if (!pte_none(pte) && pte_present(pte)) {
  263. pfn = pte_pfn(pte);
  264. if (pfn_valid(pfn)) {
  265. struct page *page = pfn_to_page(pfn);
  266. ClearPageReserved(page);
  267. __free_page(page);
  268. continue;
  269. }
  270. }
  271. pr_crit("%s: bad page in kernel page table\n",
  272. __func__);
  273. } while (size -= PAGE_SIZE);
  274. flush_tlb_kernel_range(c->vm_start, c->vm_end);
  275. list_del(&c->vm_list);
  276. spin_unlock_irqrestore(&consistent_lock, flags);
  277. kfree(c);
  278. return;
  279. no_area:
  280. spin_unlock_irqrestore(&consistent_lock, flags);
  281. pr_err("%s: trying to free invalid coherent area: %p\n",
  282. __func__, vaddr);
  283. dump_stack();
  284. }
  285. EXPORT_SYMBOL(dma_free_coherent);
  286. static int dma_mmap(struct device *dev, struct vm_area_struct *vma,
  287. void *cpu_addr, dma_addr_t dma_addr, size_t size)
  288. {
  289. int ret = -ENXIO;
  290. unsigned long flags, user_size, kern_size;
  291. struct metag_vm_region *c;
  292. user_size = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
  293. spin_lock_irqsave(&consistent_lock, flags);
  294. c = metag_vm_region_find(&consistent_head, (unsigned long)cpu_addr);
  295. spin_unlock_irqrestore(&consistent_lock, flags);
  296. if (c) {
  297. unsigned long off = vma->vm_pgoff;
  298. kern_size = (c->vm_end - c->vm_start) >> PAGE_SHIFT;
  299. if (off < kern_size &&
  300. user_size <= (kern_size - off)) {
  301. ret = remap_pfn_range(vma, vma->vm_start,
  302. page_to_pfn(c->vm_pages) + off,
  303. user_size << PAGE_SHIFT,
  304. vma->vm_page_prot);
  305. }
  306. }
  307. return ret;
  308. }
  309. int dma_mmap_coherent(struct device *dev, struct vm_area_struct *vma,
  310. void *cpu_addr, dma_addr_t dma_addr, size_t size)
  311. {
  312. vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
  313. return dma_mmap(dev, vma, cpu_addr, dma_addr, size);
  314. }
  315. EXPORT_SYMBOL(dma_mmap_coherent);
  316. int dma_mmap_writecombine(struct device *dev, struct vm_area_struct *vma,
  317. void *cpu_addr, dma_addr_t dma_addr, size_t size)
  318. {
  319. vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
  320. return dma_mmap(dev, vma, cpu_addr, dma_addr, size);
  321. }
  322. EXPORT_SYMBOL(dma_mmap_writecombine);
  323. /*
  324. * Initialise the consistent memory allocation.
  325. */
  326. static int __init dma_alloc_init(void)
  327. {
  328. pgd_t *pgd, *pgd_k;
  329. pud_t *pud, *pud_k;
  330. pmd_t *pmd, *pmd_k;
  331. pte_t *pte;
  332. int ret = 0;
  333. do {
  334. int offset = pgd_index(CONSISTENT_START);
  335. pgd = pgd_offset(&init_mm, CONSISTENT_START);
  336. pud = pud_alloc(&init_mm, pgd, CONSISTENT_START);
  337. pmd = pmd_alloc(&init_mm, pud, CONSISTENT_START);
  338. if (!pmd) {
  339. pr_err("%s: no pmd tables\n", __func__);
  340. ret = -ENOMEM;
  341. break;
  342. }
  343. WARN_ON(!pmd_none(*pmd));
  344. pte = pte_alloc_kernel(pmd, CONSISTENT_START);
  345. if (!pte) {
  346. pr_err("%s: no pte tables\n", __func__);
  347. ret = -ENOMEM;
  348. break;
  349. }
  350. pgd_k = ((pgd_t *) mmu_get_base()) + offset;
  351. pud_k = pud_offset(pgd_k, CONSISTENT_START);
  352. pmd_k = pmd_offset(pud_k, CONSISTENT_START);
  353. set_pmd(pmd_k, *pmd);
  354. consistent_pte = pte;
  355. } while (0);
  356. return ret;
  357. }
  358. early_initcall(dma_alloc_init);
  359. /*
  360. * make an area consistent to devices.
  361. */
  362. void dma_sync_for_device(void *vaddr, size_t size, int dma_direction)
  363. {
  364. /*
  365. * Ensure any writes get through the write combiner. This is necessary
  366. * even with DMA_FROM_DEVICE, or the write may dirty the cache after
  367. * we've invalidated it and get written back during the DMA.
  368. */
  369. barrier();
  370. switch (dma_direction) {
  371. case DMA_BIDIRECTIONAL:
  372. /*
  373. * Writeback to ensure the device can see our latest changes and
  374. * so that we have no dirty lines, and invalidate the cache
  375. * lines too in preparation for receiving the buffer back
  376. * (dma_sync_for_cpu) later.
  377. */
  378. flush_dcache_region(vaddr, size);
  379. break;
  380. case DMA_TO_DEVICE:
  381. /*
  382. * Writeback to ensure the device can see our latest changes.
  383. * There's no need to invalidate as the device shouldn't write
  384. * to the buffer.
  385. */
  386. writeback_dcache_region(vaddr, size);
  387. break;
  388. case DMA_FROM_DEVICE:
  389. /*
  390. * Invalidate to ensure we have no dirty lines that could get
  391. * written back during the DMA. It's also safe to flush
  392. * (writeback) here if necessary.
  393. */
  394. invalidate_dcache_region(vaddr, size);
  395. break;
  396. case DMA_NONE:
  397. BUG();
  398. }
  399. wmb();
  400. }
  401. EXPORT_SYMBOL(dma_sync_for_device);
  402. /*
  403. * make an area consistent to the core.
  404. */
  405. void dma_sync_for_cpu(void *vaddr, size_t size, int dma_direction)
  406. {
  407. /*
  408. * Hardware L2 cache prefetch doesn't occur across 4K physical
  409. * boundaries, however according to Documentation/DMA-API-HOWTO.txt
  410. * kmalloc'd memory is DMA'able, so accesses in nearby memory could
  411. * trigger a cache fill in the DMA buffer.
  412. *
  413. * This should never cause dirty lines, so a flush or invalidate should
  414. * be safe to allow us to see data from the device.
  415. */
  416. if (_meta_l2c_pf_is_enabled()) {
  417. switch (dma_direction) {
  418. case DMA_BIDIRECTIONAL:
  419. case DMA_FROM_DEVICE:
  420. invalidate_dcache_region(vaddr, size);
  421. break;
  422. case DMA_TO_DEVICE:
  423. /* The device shouldn't have written to the buffer */
  424. break;
  425. case DMA_NONE:
  426. BUG();
  427. }
  428. }
  429. rmb();
  430. }
  431. EXPORT_SYMBOL(dma_sync_for_cpu);