/arch/powerpc/include/asm/pgalloc-64.h

http://github.com/mirrors/linux · C Header · 243 lines · 175 code · 40 blank · 28 comment · 4 complexity · bfe678bed8f1c30358742db826d5f9af MD5 · raw file

  1. #ifndef _ASM_POWERPC_PGALLOC_64_H
  2. #define _ASM_POWERPC_PGALLOC_64_H
  3. /*
  4. * This program is free software; you can redistribute it and/or
  5. * modify it under the terms of the GNU General Public License
  6. * as published by the Free Software Foundation; either version
  7. * 2 of the License, or (at your option) any later version.
  8. */
  9. #include <linux/slab.h>
  10. #include <linux/cpumask.h>
  11. #include <linux/percpu.h>
  12. struct vmemmap_backing {
  13. struct vmemmap_backing *list;
  14. unsigned long phys;
  15. unsigned long virt_addr;
  16. };
  17. /*
  18. * Functions that deal with pagetables that could be at any level of
  19. * the table need to be passed an "index_size" so they know how to
  20. * handle allocation. For PTE pages (which are linked to a struct
  21. * page for now, and drawn from the main get_free_pages() pool), the
  22. * allocation size will be (2^index_size * sizeof(pointer)) and
  23. * allocations are drawn from the kmem_cache in PGT_CACHE(index_size).
  24. *
  25. * The maximum index size needs to be big enough to allow any
  26. * pagetable sizes we need, but small enough to fit in the low bits of
  27. * any page table pointer. In other words all pagetables, even tiny
  28. * ones, must be aligned to allow at least enough low 0 bits to
  29. * contain this value. This value is also used as a mask, so it must
  30. * be one less than a power of two.
  31. */
  32. #define MAX_PGTABLE_INDEX_SIZE 0xf
  33. extern struct kmem_cache *pgtable_cache[];
  34. #define PGT_CACHE(shift) ({ \
  35. BUG_ON(!(shift)); \
  36. pgtable_cache[(shift) - 1]; \
  37. })
  38. static inline pgd_t *pgd_alloc(struct mm_struct *mm)
  39. {
  40. return kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE), GFP_KERNEL);
  41. }
  42. static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
  43. {
  44. kmem_cache_free(PGT_CACHE(PGD_INDEX_SIZE), pgd);
  45. }
  46. #ifndef CONFIG_PPC_64K_PAGES
  47. #define pgd_populate(MM, PGD, PUD) pgd_set(PGD, PUD)
  48. static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
  49. {
  50. return kmem_cache_alloc(PGT_CACHE(PUD_INDEX_SIZE),
  51. GFP_KERNEL|__GFP_REPEAT);
  52. }
  53. static inline void pud_free(struct mm_struct *mm, pud_t *pud)
  54. {
  55. kmem_cache_free(PGT_CACHE(PUD_INDEX_SIZE), pud);
  56. }
  57. static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
  58. {
  59. pud_set(pud, (unsigned long)pmd);
  60. }
  61. #define pmd_populate(mm, pmd, pte_page) \
  62. pmd_populate_kernel(mm, pmd, page_address(pte_page))
  63. #define pmd_populate_kernel(mm, pmd, pte) pmd_set(pmd, (unsigned long)(pte))
  64. #define pmd_pgtable(pmd) pmd_page(pmd)
  65. static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
  66. unsigned long address)
  67. {
  68. return (pte_t *)__get_free_page(GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO);
  69. }
  70. static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
  71. unsigned long address)
  72. {
  73. struct page *page;
  74. pte_t *pte;
  75. pte = pte_alloc_one_kernel(mm, address);
  76. if (!pte)
  77. return NULL;
  78. page = virt_to_page(pte);
  79. pgtable_page_ctor(page);
  80. return page;
  81. }
  82. static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
  83. {
  84. free_page((unsigned long)pte);
  85. }
  86. static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage)
  87. {
  88. pgtable_page_dtor(ptepage);
  89. __free_page(ptepage);
  90. }
  91. static inline void pgtable_free(void *table, unsigned index_size)
  92. {
  93. if (!index_size)
  94. free_page((unsigned long)table);
  95. else {
  96. BUG_ON(index_size > MAX_PGTABLE_INDEX_SIZE);
  97. kmem_cache_free(PGT_CACHE(index_size), table);
  98. }
  99. }
  100. #ifdef CONFIG_SMP
  101. static inline void pgtable_free_tlb(struct mmu_gather *tlb,
  102. void *table, int shift)
  103. {
  104. unsigned long pgf = (unsigned long)table;
  105. BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
  106. pgf |= shift;
  107. tlb_remove_table(tlb, (void *)pgf);
  108. }
  109. static inline void __tlb_remove_table(void *_table)
  110. {
  111. void *table = (void *)((unsigned long)_table & ~MAX_PGTABLE_INDEX_SIZE);
  112. unsigned shift = (unsigned long)_table & MAX_PGTABLE_INDEX_SIZE;
  113. pgtable_free(table, shift);
  114. }
  115. #else /* !CONFIG_SMP */
  116. static inline void pgtable_free_tlb(struct mmu_gather *tlb,
  117. void *table, int shift)
  118. {
  119. pgtable_free(table, shift);
  120. }
  121. #endif /* CONFIG_SMP */
  122. static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table,
  123. unsigned long address)
  124. {
  125. struct page *page = page_address(table);
  126. tlb_flush_pgtable(tlb, address);
  127. pgtable_page_dtor(page);
  128. pgtable_free_tlb(tlb, page, 0);
  129. }
  130. #else /* if CONFIG_PPC_64K_PAGES */
  131. /*
  132. * we support 16 fragments per PTE page.
  133. */
  134. #define PTE_FRAG_NR 16
  135. /*
  136. * We use a 2K PTE page fragment and another 2K for storing
  137. * real_pte_t hash index
  138. */
  139. #define PTE_FRAG_SIZE_SHIFT 12
  140. #define PTE_FRAG_SIZE (2 * PTRS_PER_PTE * sizeof(pte_t))
  141. extern pte_t *page_table_alloc(struct mm_struct *, unsigned long, int);
  142. extern void page_table_free(struct mm_struct *, unsigned long *, int);
  143. extern void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift);
  144. #ifdef CONFIG_SMP
  145. extern void __tlb_remove_table(void *_table);
  146. #endif
  147. #define pud_populate(mm, pud, pmd) pud_set(pud, (unsigned long)pmd)
  148. static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd,
  149. pte_t *pte)
  150. {
  151. pmd_set(pmd, (unsigned long)pte);
  152. }
  153. static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
  154. pgtable_t pte_page)
  155. {
  156. pmd_set(pmd, (unsigned long)pte_page);
  157. }
  158. static inline pgtable_t pmd_pgtable(pmd_t pmd)
  159. {
  160. return (pgtable_t)(pmd_val(pmd) & ~PMD_MASKED_BITS);
  161. }
  162. static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
  163. unsigned long address)
  164. {
  165. return (pte_t *)page_table_alloc(mm, address, 1);
  166. }
  167. static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
  168. unsigned long address)
  169. {
  170. return (pgtable_t)page_table_alloc(mm, address, 0);
  171. }
  172. static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
  173. {
  174. page_table_free(mm, (unsigned long *)pte, 1);
  175. }
  176. static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage)
  177. {
  178. page_table_free(mm, (unsigned long *)ptepage, 0);
  179. }
  180. static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table,
  181. unsigned long address)
  182. {
  183. tlb_flush_pgtable(tlb, address);
  184. pgtable_free_tlb(tlb, table, 0);
  185. }
  186. #endif /* CONFIG_PPC_64K_PAGES */
  187. static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
  188. {
  189. return kmem_cache_alloc(PGT_CACHE(PMD_CACHE_INDEX),
  190. GFP_KERNEL|__GFP_REPEAT);
  191. }
  192. static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
  193. {
  194. kmem_cache_free(PGT_CACHE(PMD_CACHE_INDEX), pmd);
  195. }
  196. #define __pmd_free_tlb(tlb, pmd, addr) \
  197. pgtable_free_tlb(tlb, pmd, PMD_CACHE_INDEX)
  198. #ifndef CONFIG_PPC_64K_PAGES
  199. #define __pud_free_tlb(tlb, pud, addr) \
  200. pgtable_free_tlb(tlb, pud, PUD_INDEX_SIZE)
  201. #endif /* CONFIG_PPC_64K_PAGES */
  202. #define check_pgt_cache() do { } while (0)
  203. #endif /* _ASM_POWERPC_PGALLOC_64_H */