/arch/arm/include/asm/cacheflush.h

https://bitbucket.org/sammyz/iscream_thunderc-2.6.35-rebase · C++ Header · 475 lines · 254 code · 55 blank · 166 comment · 21 complexity · cc84d4d49cb79400c1a6bdd5883416c4 MD5 · raw file

  1. /*
  2. * arch/arm/include/asm/cacheflush.h
  3. *
  4. * Copyright (C) 1999-2002 Russell King
  5. *
  6. * This program is free software; you can redistribute it and/or modify
  7. * it under the terms of the GNU General Public License version 2 as
  8. * published by the Free Software Foundation.
  9. */
  10. #ifndef _ASMARM_CACHEFLUSH_H
  11. #define _ASMARM_CACHEFLUSH_H
  12. #include <linux/mm.h>
  13. #include <asm/glue.h>
  14. #include <asm/shmparam.h>
  15. #include <asm/cachetype.h>
  16. #include <asm/outercache.h>
  17. #define CACHE_COLOUR(vaddr) ((vaddr & (SHMLBA - 1)) >> PAGE_SHIFT)
  18. /*
  19. * Cache Model
  20. * ===========
  21. */
  22. #undef _CACHE
  23. #undef MULTI_CACHE
  24. #if defined(CONFIG_CPU_CACHE_V3)
  25. # ifdef _CACHE
  26. # define MULTI_CACHE 1
  27. # else
  28. # define _CACHE v3
  29. # endif
  30. #endif
  31. #if defined(CONFIG_CPU_CACHE_V4)
  32. # ifdef _CACHE
  33. # define MULTI_CACHE 1
  34. # else
  35. # define _CACHE v4
  36. # endif
  37. #endif
  38. #if defined(CONFIG_CPU_ARM920T) || defined(CONFIG_CPU_ARM922T) || \
  39. defined(CONFIG_CPU_ARM925T) || defined(CONFIG_CPU_ARM1020) || \
  40. defined(CONFIG_CPU_ARM1026)
  41. # define MULTI_CACHE 1
  42. #endif
  43. #if defined(CONFIG_CPU_FA526)
  44. # ifdef _CACHE
  45. # define MULTI_CACHE 1
  46. # else
  47. # define _CACHE fa
  48. # endif
  49. #endif
  50. #if defined(CONFIG_CPU_ARM926T)
  51. # ifdef _CACHE
  52. # define MULTI_CACHE 1
  53. # else
  54. # define _CACHE arm926
  55. # endif
  56. #endif
  57. #if defined(CONFIG_CPU_ARM940T)
  58. # ifdef _CACHE
  59. # define MULTI_CACHE 1
  60. # else
  61. # define _CACHE arm940
  62. # endif
  63. #endif
  64. #if defined(CONFIG_CPU_ARM946E)
  65. # ifdef _CACHE
  66. # define MULTI_CACHE 1
  67. # else
  68. # define _CACHE arm946
  69. # endif
  70. #endif
  71. #if defined(CONFIG_CPU_CACHE_V4WB)
  72. # ifdef _CACHE
  73. # define MULTI_CACHE 1
  74. # else
  75. # define _CACHE v4wb
  76. # endif
  77. #endif
  78. #if defined(CONFIG_CPU_XSCALE)
  79. # ifdef _CACHE
  80. # define MULTI_CACHE 1
  81. # else
  82. # define _CACHE xscale
  83. # endif
  84. #endif
  85. #if defined(CONFIG_CPU_XSC3)
  86. # ifdef _CACHE
  87. # define MULTI_CACHE 1
  88. # else
  89. # define _CACHE xsc3
  90. # endif
  91. #endif
  92. #if defined(CONFIG_CPU_MOHAWK)
  93. # ifdef _CACHE
  94. # define MULTI_CACHE 1
  95. # else
  96. # define _CACHE mohawk
  97. # endif
  98. #endif
  99. #if defined(CONFIG_CPU_FEROCEON)
  100. # define MULTI_CACHE 1
  101. #endif
  102. #if defined(CONFIG_CPU_V6)
  103. //# ifdef _CACHE
  104. # define MULTI_CACHE 1
  105. //# else
  106. //# define _CACHE v6
  107. //# endif
  108. #endif
  109. #if defined(CONFIG_CPU_V7)
  110. //# ifdef _CACHE
  111. # define MULTI_CACHE 1
  112. //# else
  113. //# define _CACHE v7
  114. //# endif
  115. #endif
  116. #if !defined(_CACHE) && !defined(MULTI_CACHE)
  117. #error Unknown cache maintainence model
  118. #endif
  119. /*
  120. * This flag is used to indicate that the page pointed to by a pte
  121. * is dirty and requires cleaning before returning it to the user.
  122. */
  123. #define PG_dcache_dirty PG_arch_1
  124. /*
  125. * MM Cache Management
  126. * ===================
  127. *
  128. * The arch/arm/mm/cache-*.S and arch/arm/mm/proc-*.S files
  129. * implement these methods.
  130. *
  131. * Start addresses are inclusive and end addresses are exclusive;
  132. * start addresses should be rounded down, end addresses up.
  133. *
  134. * See Documentation/cachetlb.txt for more information.
  135. * Please note that the implementation of these, and the required
  136. * effects are cache-type (VIVT/VIPT/PIPT) specific.
  137. *
  138. * flush_kern_all()
  139. *
  140. * Unconditionally clean and invalidate the entire cache.
  141. *
  142. * flush_user_all()
  143. *
  144. * Clean and invalidate all user space cache entries
  145. * before a change of page tables.
  146. *
  147. * flush_user_range(start, end, flags)
  148. *
  149. * Clean and invalidate a range of cache entries in the
  150. * specified address space before a change of page tables.
  151. * - start - user start address (inclusive, page aligned)
  152. * - end - user end address (exclusive, page aligned)
  153. * - flags - vma->vm_flags field
  154. *
  155. * coherent_kern_range(start, end)
  156. *
  157. * Ensure coherency between the Icache and the Dcache in the
  158. * region described by start, end. If you have non-snooping
  159. * Harvard caches, you need to implement this function.
  160. * - start - virtual start address
  161. * - end - virtual end address
  162. *
  163. * coherent_user_range(start, end)
  164. *
  165. * Ensure coherency between the Icache and the Dcache in the
  166. * region described by start, end. If you have non-snooping
  167. * Harvard caches, you need to implement this function.
  168. * - start - virtual start address
  169. * - end - virtual end address
  170. *
  171. * flush_kern_dcache_area(kaddr, size)
  172. *
  173. * Ensure that the data held in page is written back.
  174. * - kaddr - page address
  175. * - size - region size
  176. *
  177. * DMA Cache Coherency
  178. * ===================
  179. *
  180. * dma_inv_range(start, end)
  181. *
  182. * Invalidate (discard) the specified virtual address range.
  183. * May not write back any entries. If 'start' or 'end'
  184. * are not cache line aligned, those lines must be written
  185. * back.
  186. * - start - virtual start address
  187. * - end - virtual end address
  188. *
  189. * dma_clean_range(start, end)
  190. *
  191. * Clean (write back) the specified virtual address range.
  192. * - start - virtual start address
  193. * - end - virtual end address
  194. *
  195. * dma_flush_range(start, end)
  196. *
  197. * Clean and invalidate the specified virtual address range.
  198. * - start - virtual start address
  199. * - end - virtual end address
  200. */
  201. struct cpu_cache_fns {
  202. void (*flush_kern_all)(void);
  203. void (*flush_user_all)(void);
  204. void (*flush_user_range)(unsigned long, unsigned long, unsigned int);
  205. void (*coherent_kern_range)(unsigned long, unsigned long);
  206. void (*coherent_user_range)(unsigned long, unsigned long);
  207. void (*flush_kern_dcache_area)(void *, size_t);
  208. void (*dma_map_area)(const void *, size_t, int);
  209. void (*dma_unmap_area)(const void *, size_t, int);
  210. void (*dma_inv_range)(const void *, const void *);
  211. void (*dma_clean_range)(const void *, const void *);
  212. void (*dma_flush_range)(const void *, const void *);
  213. };
  214. /*
  215. * Select the calling method
  216. */
  217. #ifdef MULTI_CACHE
  218. extern struct cpu_cache_fns cpu_cache;
  219. #define __cpuc_flush_kern_all cpu_cache.flush_kern_all
  220. #define __cpuc_flush_user_all cpu_cache.flush_user_all
  221. #define __cpuc_flush_user_range cpu_cache.flush_user_range
  222. #define __cpuc_coherent_kern_range cpu_cache.coherent_kern_range
  223. #define __cpuc_coherent_user_range cpu_cache.coherent_user_range
  224. #define __cpuc_flush_dcache_area cpu_cache.flush_kern_dcache_area
  225. /*
  226. * These are private to the dma-mapping API. Do not use directly.
  227. * Their sole purpose is to ensure that data held in the cache
  228. * is visible to DMA, or data written by DMA to system memory is
  229. * visible to the CPU.
  230. */
  231. #define dmac_map_area cpu_cache.dma_map_area
  232. #define dmac_unmap_area cpu_cache.dma_unmap_area
  233. #define dmac_inv_range cpu_cache.dma_inv_range
  234. #define dmac_clean_range cpu_cache.dma_clean_range
  235. #define dmac_flush_range cpu_cache.dma_flush_range
  236. #else
  237. #define __cpuc_flush_kern_all __glue(_CACHE,_flush_kern_cache_all)
  238. #define __cpuc_flush_user_all __glue(_CACHE,_flush_user_cache_all)
  239. #define __cpuc_flush_user_range __glue(_CACHE,_flush_user_cache_range)
  240. #define __cpuc_coherent_kern_range __glue(_CACHE,_coherent_kern_range)
  241. #define __cpuc_coherent_user_range __glue(_CACHE,_coherent_user_range)
  242. #define __cpuc_flush_dcache_area __glue(_CACHE,_flush_kern_dcache_area)
  243. extern void __cpuc_flush_kern_all(void);
  244. extern void __cpuc_flush_user_all(void);
  245. extern void __cpuc_flush_user_range(unsigned long, unsigned long, unsigned int);
  246. extern void __cpuc_coherent_kern_range(unsigned long, unsigned long);
  247. extern void __cpuc_coherent_user_range(unsigned long, unsigned long);
  248. extern void __cpuc_flush_dcache_area(void *, size_t);
  249. /*
  250. * These are private to the dma-mapping API. Do not use directly.
  251. * Their sole purpose is to ensure that data held in the cache
  252. * is visible to DMA, or data written by DMA to system memory is
  253. * visible to the CPU.
  254. */
  255. #define dmac_map_area __glue(_CACHE,_dma_map_area)
  256. #define dmac_unmap_area __glue(_CACHE,_dma_unmap_area)
  257. #define dmac_inv_range __glue(_CACHE,_dma_inv_range)
  258. #define dmac_clean_range __glue(_CACHE,_dma_clean_range)
  259. #define dmac_flush_range __glue(_CACHE,_dma_flush_range)
  260. extern void dmac_map_area(const void *, size_t, int);
  261. extern void dmac_unmap_area(const void *, size_t, int);
  262. extern void dmac_inv_range(const void *, const void *);
  263. extern void dmac_clean_range(const void *, const void *);
  264. extern void dmac_flush_range(const void *, const void *);
  265. #endif
  266. /*
  267. * Copy user data from/to a page which is mapped into a different
  268. * processes address space. Really, we want to allow our "user
  269. * space" model to handle this.
  270. */
  271. extern void copy_to_user_page(struct vm_area_struct *, struct page *,
  272. unsigned long, void *, const void *, unsigned long);
  273. #define copy_from_user_page(vma, page, vaddr, dst, src, len) \
  274. do { \
  275. memcpy(dst, src, len); \
  276. } while (0)
  277. /*
  278. * Convert calls to our calling convention.
  279. */
  280. #define flush_cache_all() __cpuc_flush_kern_all()
  281. static inline void vivt_flush_cache_mm(struct mm_struct *mm)
  282. {
  283. if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(mm)))
  284. __cpuc_flush_user_all();
  285. }
  286. static inline void
  287. vivt_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
  288. {
  289. if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm)))
  290. __cpuc_flush_user_range(start & PAGE_MASK, PAGE_ALIGN(end),
  291. vma->vm_flags);
  292. }
  293. static inline void
  294. vivt_flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsigned long pfn)
  295. {
  296. if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm))) {
  297. unsigned long addr = user_addr & PAGE_MASK;
  298. __cpuc_flush_user_range(addr, addr + PAGE_SIZE, vma->vm_flags);
  299. }
  300. }
  301. #ifndef CONFIG_CPU_CACHE_VIPT
  302. #define flush_cache_mm(mm) \
  303. vivt_flush_cache_mm(mm)
  304. #define flush_cache_range(vma,start,end) \
  305. vivt_flush_cache_range(vma,start,end)
  306. #define flush_cache_page(vma,addr,pfn) \
  307. vivt_flush_cache_page(vma,addr,pfn)
  308. #else
  309. extern void flush_cache_mm(struct mm_struct *mm);
  310. extern void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
  311. extern void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsigned long pfn);
  312. #endif
  313. #define flush_cache_dup_mm(mm) flush_cache_mm(mm)
  314. /*
  315. * flush_cache_user_range is used when we want to ensure that the
  316. * Harvard caches are synchronised for the user space address range.
  317. * This is used for the ARM private sys_cacheflush system call.
  318. */
  319. #define flush_cache_user_range(start,end) \
  320. __cpuc_coherent_user_range((start) & PAGE_MASK, PAGE_ALIGN(end))
  321. /*
  322. * Perform necessary cache operations to ensure that data previously
  323. * stored within this range of addresses can be executed by the CPU.
  324. */
  325. #define flush_icache_range(s,e) __cpuc_coherent_kern_range(s,e)
  326. /*
  327. * Perform necessary cache operations to ensure that the TLB will
  328. * see data written in the specified area.
  329. */
  330. #define clean_dcache_area(start,size) cpu_dcache_clean_area(start, size)
  331. /*
  332. * flush_dcache_page is used when the kernel has written to the page
  333. * cache page at virtual address page->virtual.
  334. *
  335. * If this page isn't mapped (ie, page_mapping == NULL), or it might
  336. * have userspace mappings, then we _must_ always clean + invalidate
  337. * the dcache entries associated with the kernel mapping.
  338. *
  339. * Otherwise we can defer the operation, and clean the cache when we are
  340. * about to change to user space. This is the same method as used on SPARC64.
  341. * See update_mmu_cache for the user space part.
  342. */
  343. #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1
  344. extern void flush_dcache_page(struct page *);
  345. static inline void __flush_icache_all(void)
  346. {
  347. #ifdef CONFIG_ARM_ERRATA_411920
  348. extern void v6_icache_inval_all(void);
  349. v6_icache_inval_all();
  350. #elif defined(CONFIG_SMP) && __LINUX_ARM_ARCH__ >= 7
  351. asm("mcr p15, 0, %0, c7, c1, 0 @ invalidate I-cache inner shareable\n"
  352. :
  353. : "r" (0));
  354. #else
  355. asm("mcr p15, 0, %0, c7, c5, 0 @ invalidate I-cache\n"
  356. :
  357. : "r" (0));
  358. #endif
  359. }
  360. static inline void flush_kernel_vmap_range(void *addr, int size)
  361. {
  362. if ((cache_is_vivt() || cache_is_vipt_aliasing()))
  363. __cpuc_flush_dcache_area(addr, (size_t)size);
  364. }
  365. static inline void invalidate_kernel_vmap_range(void *addr, int size)
  366. {
  367. if ((cache_is_vivt() || cache_is_vipt_aliasing()))
  368. __cpuc_flush_dcache_area(addr, (size_t)size);
  369. }
  370. #define ARCH_HAS_FLUSH_ANON_PAGE
  371. static inline void flush_anon_page(struct vm_area_struct *vma,
  372. struct page *page, unsigned long vmaddr)
  373. {
  374. extern void __flush_anon_page(struct vm_area_struct *vma,
  375. struct page *, unsigned long);
  376. if (PageAnon(page))
  377. __flush_anon_page(vma, page, vmaddr);
  378. }
  379. #define ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE
  380. static inline void flush_kernel_dcache_page(struct page *page)
  381. {
  382. /* highmem pages are always flushed upon kunmap already */
  383. if ((cache_is_vivt() || cache_is_vipt_aliasing()) && !PageHighMem(page))
  384. __cpuc_flush_dcache_area(page_address(page), PAGE_SIZE);
  385. }
  386. #define flush_dcache_mmap_lock(mapping) \
  387. spin_lock_irq(&(mapping)->tree_lock)
  388. #define flush_dcache_mmap_unlock(mapping) \
  389. spin_unlock_irq(&(mapping)->tree_lock)
  390. #define flush_icache_user_range(vma,page,addr,len) \
  391. flush_dcache_page(page)
  392. /*
  393. * We don't appear to need to do anything here. In fact, if we did, we'd
  394. * duplicate cache flushing elsewhere performed by flush_dcache_page().
  395. */
  396. #define flush_icache_page(vma,page) do { } while (0)
  397. /*
  398. * flush_cache_vmap() is used when creating mappings (eg, via vmap,
  399. * vmalloc, ioremap etc) in kernel space for pages. On non-VIPT
  400. * caches, since the direct-mappings of these pages may contain cached
  401. * data, we need to do a full cache flush to ensure that writebacks
  402. * don't corrupt data placed into these pages via the new mappings.
  403. */
  404. static inline void flush_cache_vmap(unsigned long start, unsigned long end)
  405. {
  406. if (!cache_is_vipt_nonaliasing())
  407. flush_cache_all();
  408. else
  409. /*
  410. * set_pte_at() called from vmap_pte_range() does not
  411. * have a DSB after cleaning the cache line.
  412. */
  413. dsb();
  414. }
  415. static inline void flush_cache_vunmap(unsigned long start, unsigned long end)
  416. {
  417. if (!cache_is_vipt_nonaliasing())
  418. flush_cache_all();
  419. }
  420. #endif