/arch/powerpc/kernel/vdso.c

http://github.com/mirrors/linux · C · 793 lines · 570 code · 114 blank · 109 comment · 92 complexity · 4721d352cc3a6de9f359e0cb8c68c8ef MD5 · raw file

  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * Copyright (C) 2004 Benjamin Herrenschmidt, IBM Corp.
  4. * <benh@kernel.crashing.org>
  5. */
  6. #include <linux/errno.h>
  7. #include <linux/sched.h>
  8. #include <linux/kernel.h>
  9. #include <linux/mm.h>
  10. #include <linux/smp.h>
  11. #include <linux/stddef.h>
  12. #include <linux/unistd.h>
  13. #include <linux/slab.h>
  14. #include <linux/user.h>
  15. #include <linux/elf.h>
  16. #include <linux/security.h>
  17. #include <linux/memblock.h>
  18. #include <asm/pgtable.h>
  19. #include <asm/processor.h>
  20. #include <asm/mmu.h>
  21. #include <asm/mmu_context.h>
  22. #include <asm/prom.h>
  23. #include <asm/machdep.h>
  24. #include <asm/cputable.h>
  25. #include <asm/sections.h>
  26. #include <asm/firmware.h>
  27. #include <asm/vdso.h>
  28. #include <asm/vdso_datapage.h>
  29. #include <asm/setup.h>
  30. #undef DEBUG
  31. #ifdef DEBUG
  32. #define DBG(fmt...) printk(fmt)
  33. #else
  34. #define DBG(fmt...)
  35. #endif
  36. /* Max supported size for symbol names */
  37. #define MAX_SYMNAME 64
  38. /* The alignment of the vDSO */
  39. #define VDSO_ALIGNMENT (1 << 16)
  40. static unsigned int vdso32_pages;
  41. static void *vdso32_kbase;
  42. static struct page **vdso32_pagelist;
  43. unsigned long vdso32_sigtramp;
  44. unsigned long vdso32_rt_sigtramp;
  45. #ifdef CONFIG_VDSO32
  46. extern char vdso32_start, vdso32_end;
  47. #endif
  48. #ifdef CONFIG_PPC64
  49. extern char vdso64_start, vdso64_end;
  50. static void *vdso64_kbase = &vdso64_start;
  51. static unsigned int vdso64_pages;
  52. static struct page **vdso64_pagelist;
  53. unsigned long vdso64_rt_sigtramp;
  54. #endif /* CONFIG_PPC64 */
  55. static int vdso_ready;
  56. /*
  57. * The vdso data page (aka. systemcfg for old ppc64 fans) is here.
  58. * Once the early boot kernel code no longer needs to muck around
  59. * with it, it will become dynamically allocated
  60. */
  61. static union {
  62. struct vdso_data data;
  63. u8 page[PAGE_SIZE];
  64. } vdso_data_store __page_aligned_data;
  65. struct vdso_data *vdso_data = &vdso_data_store.data;
  66. /* Format of the patch table */
  67. struct vdso_patch_def
  68. {
  69. unsigned long ftr_mask, ftr_value;
  70. const char *gen_name;
  71. const char *fix_name;
  72. };
  73. /* Table of functions to patch based on the CPU type/revision
  74. *
  75. * Currently, we only change sync_dicache to do nothing on processors
  76. * with a coherent icache
  77. */
  78. static struct vdso_patch_def vdso_patches[] = {
  79. {
  80. CPU_FTR_COHERENT_ICACHE, CPU_FTR_COHERENT_ICACHE,
  81. "__kernel_sync_dicache", "__kernel_sync_dicache_p5"
  82. },
  83. };
  84. /*
  85. * Some infos carried around for each of them during parsing at
  86. * boot time.
  87. */
  88. struct lib32_elfinfo
  89. {
  90. Elf32_Ehdr *hdr; /* ptr to ELF */
  91. Elf32_Sym *dynsym; /* ptr to .dynsym section */
  92. unsigned long dynsymsize; /* size of .dynsym section */
  93. char *dynstr; /* ptr to .dynstr section */
  94. unsigned long text; /* offset of .text section in .so */
  95. };
  96. struct lib64_elfinfo
  97. {
  98. Elf64_Ehdr *hdr;
  99. Elf64_Sym *dynsym;
  100. unsigned long dynsymsize;
  101. char *dynstr;
  102. unsigned long text;
  103. };
  104. /*
  105. * This is called from binfmt_elf, we create the special vma for the
  106. * vDSO and insert it into the mm struct tree
  107. */
  108. int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
  109. {
  110. struct mm_struct *mm = current->mm;
  111. struct page **vdso_pagelist;
  112. unsigned long vdso_pages;
  113. unsigned long vdso_base;
  114. int rc;
  115. if (!vdso_ready)
  116. return 0;
  117. #ifdef CONFIG_PPC64
  118. if (is_32bit_task()) {
  119. vdso_pagelist = vdso32_pagelist;
  120. vdso_pages = vdso32_pages;
  121. vdso_base = VDSO32_MBASE;
  122. } else {
  123. vdso_pagelist = vdso64_pagelist;
  124. vdso_pages = vdso64_pages;
  125. /*
  126. * On 64bit we don't have a preferred map address. This
  127. * allows get_unmapped_area to find an area near other mmaps
  128. * and most likely share a SLB entry.
  129. */
  130. vdso_base = 0;
  131. }
  132. #else
  133. vdso_pagelist = vdso32_pagelist;
  134. vdso_pages = vdso32_pages;
  135. vdso_base = VDSO32_MBASE;
  136. #endif
  137. current->mm->context.vdso_base = 0;
  138. /* vDSO has a problem and was disabled, just don't "enable" it for the
  139. * process
  140. */
  141. if (vdso_pages == 0)
  142. return 0;
  143. /* Add a page to the vdso size for the data page */
  144. vdso_pages ++;
  145. /*
  146. * pick a base address for the vDSO in process space. We try to put it
  147. * at vdso_base which is the "natural" base for it, but we might fail
  148. * and end up putting it elsewhere.
  149. * Add enough to the size so that the result can be aligned.
  150. */
  151. if (down_write_killable(&mm->mmap_sem))
  152. return -EINTR;
  153. vdso_base = get_unmapped_area(NULL, vdso_base,
  154. (vdso_pages << PAGE_SHIFT) +
  155. ((VDSO_ALIGNMENT - 1) & PAGE_MASK),
  156. 0, 0);
  157. if (IS_ERR_VALUE(vdso_base)) {
  158. rc = vdso_base;
  159. goto fail_mmapsem;
  160. }
  161. /* Add required alignment. */
  162. vdso_base = ALIGN(vdso_base, VDSO_ALIGNMENT);
  163. /*
  164. * Put vDSO base into mm struct. We need to do this before calling
  165. * install_special_mapping or the perf counter mmap tracking code
  166. * will fail to recognise it as a vDSO (since arch_vma_name fails).
  167. */
  168. current->mm->context.vdso_base = vdso_base;
  169. /*
  170. * our vma flags don't have VM_WRITE so by default, the process isn't
  171. * allowed to write those pages.
  172. * gdb can break that with ptrace interface, and thus trigger COW on
  173. * those pages but it's then your responsibility to never do that on
  174. * the "data" page of the vDSO or you'll stop getting kernel updates
  175. * and your nice userland gettimeofday will be totally dead.
  176. * It's fine to use that for setting breakpoints in the vDSO code
  177. * pages though.
  178. */
  179. rc = install_special_mapping(mm, vdso_base, vdso_pages << PAGE_SHIFT,
  180. VM_READ|VM_EXEC|
  181. VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC,
  182. vdso_pagelist);
  183. if (rc) {
  184. current->mm->context.vdso_base = 0;
  185. goto fail_mmapsem;
  186. }
  187. up_write(&mm->mmap_sem);
  188. return 0;
  189. fail_mmapsem:
  190. up_write(&mm->mmap_sem);
  191. return rc;
  192. }
  193. const char *arch_vma_name(struct vm_area_struct *vma)
  194. {
  195. if (vma->vm_mm && vma->vm_start == vma->vm_mm->context.vdso_base)
  196. return "[vdso]";
  197. return NULL;
  198. }
  199. #ifdef CONFIG_VDSO32
  200. static void * __init find_section32(Elf32_Ehdr *ehdr, const char *secname,
  201. unsigned long *size)
  202. {
  203. Elf32_Shdr *sechdrs;
  204. unsigned int i;
  205. char *secnames;
  206. /* Grab section headers and strings so we can tell who is who */
  207. sechdrs = (void *)ehdr + ehdr->e_shoff;
  208. secnames = (void *)ehdr + sechdrs[ehdr->e_shstrndx].sh_offset;
  209. /* Find the section they want */
  210. for (i = 1; i < ehdr->e_shnum; i++) {
  211. if (strcmp(secnames+sechdrs[i].sh_name, secname) == 0) {
  212. if (size)
  213. *size = sechdrs[i].sh_size;
  214. return (void *)ehdr + sechdrs[i].sh_offset;
  215. }
  216. }
  217. *size = 0;
  218. return NULL;
  219. }
  220. static Elf32_Sym * __init find_symbol32(struct lib32_elfinfo *lib,
  221. const char *symname)
  222. {
  223. unsigned int i;
  224. char name[MAX_SYMNAME], *c;
  225. for (i = 0; i < (lib->dynsymsize / sizeof(Elf32_Sym)); i++) {
  226. if (lib->dynsym[i].st_name == 0)
  227. continue;
  228. strlcpy(name, lib->dynstr + lib->dynsym[i].st_name,
  229. MAX_SYMNAME);
  230. c = strchr(name, '@');
  231. if (c)
  232. *c = 0;
  233. if (strcmp(symname, name) == 0)
  234. return &lib->dynsym[i];
  235. }
  236. return NULL;
  237. }
  238. /* Note that we assume the section is .text and the symbol is relative to
  239. * the library base
  240. */
  241. static unsigned long __init find_function32(struct lib32_elfinfo *lib,
  242. const char *symname)
  243. {
  244. Elf32_Sym *sym = find_symbol32(lib, symname);
  245. if (sym == NULL) {
  246. printk(KERN_WARNING "vDSO32: function %s not found !\n",
  247. symname);
  248. return 0;
  249. }
  250. return sym->st_value - VDSO32_LBASE;
  251. }
  252. static int __init vdso_do_func_patch32(struct lib32_elfinfo *v32,
  253. struct lib64_elfinfo *v64,
  254. const char *orig, const char *fix)
  255. {
  256. Elf32_Sym *sym32_gen, *sym32_fix;
  257. sym32_gen = find_symbol32(v32, orig);
  258. if (sym32_gen == NULL) {
  259. printk(KERN_ERR "vDSO32: Can't find symbol %s !\n", orig);
  260. return -1;
  261. }
  262. if (fix == NULL) {
  263. sym32_gen->st_name = 0;
  264. return 0;
  265. }
  266. sym32_fix = find_symbol32(v32, fix);
  267. if (sym32_fix == NULL) {
  268. printk(KERN_ERR "vDSO32: Can't find symbol %s !\n", fix);
  269. return -1;
  270. }
  271. sym32_gen->st_value = sym32_fix->st_value;
  272. sym32_gen->st_size = sym32_fix->st_size;
  273. sym32_gen->st_info = sym32_fix->st_info;
  274. sym32_gen->st_other = sym32_fix->st_other;
  275. sym32_gen->st_shndx = sym32_fix->st_shndx;
  276. return 0;
  277. }
  278. #else /* !CONFIG_VDSO32 */
  279. static unsigned long __init find_function32(struct lib32_elfinfo *lib,
  280. const char *symname)
  281. {
  282. return 0;
  283. }
  284. static int __init vdso_do_func_patch32(struct lib32_elfinfo *v32,
  285. struct lib64_elfinfo *v64,
  286. const char *orig, const char *fix)
  287. {
  288. return 0;
  289. }
  290. #endif /* CONFIG_VDSO32 */
  291. #ifdef CONFIG_PPC64
  292. static void * __init find_section64(Elf64_Ehdr *ehdr, const char *secname,
  293. unsigned long *size)
  294. {
  295. Elf64_Shdr *sechdrs;
  296. unsigned int i;
  297. char *secnames;
  298. /* Grab section headers and strings so we can tell who is who */
  299. sechdrs = (void *)ehdr + ehdr->e_shoff;
  300. secnames = (void *)ehdr + sechdrs[ehdr->e_shstrndx].sh_offset;
  301. /* Find the section they want */
  302. for (i = 1; i < ehdr->e_shnum; i++) {
  303. if (strcmp(secnames+sechdrs[i].sh_name, secname) == 0) {
  304. if (size)
  305. *size = sechdrs[i].sh_size;
  306. return (void *)ehdr + sechdrs[i].sh_offset;
  307. }
  308. }
  309. if (size)
  310. *size = 0;
  311. return NULL;
  312. }
  313. static Elf64_Sym * __init find_symbol64(struct lib64_elfinfo *lib,
  314. const char *symname)
  315. {
  316. unsigned int i;
  317. char name[MAX_SYMNAME], *c;
  318. for (i = 0; i < (lib->dynsymsize / sizeof(Elf64_Sym)); i++) {
  319. if (lib->dynsym[i].st_name == 0)
  320. continue;
  321. strlcpy(name, lib->dynstr + lib->dynsym[i].st_name,
  322. MAX_SYMNAME);
  323. c = strchr(name, '@');
  324. if (c)
  325. *c = 0;
  326. if (strcmp(symname, name) == 0)
  327. return &lib->dynsym[i];
  328. }
  329. return NULL;
  330. }
  331. /* Note that we assume the section is .text and the symbol is relative to
  332. * the library base
  333. */
  334. static unsigned long __init find_function64(struct lib64_elfinfo *lib,
  335. const char *symname)
  336. {
  337. Elf64_Sym *sym = find_symbol64(lib, symname);
  338. if (sym == NULL) {
  339. printk(KERN_WARNING "vDSO64: function %s not found !\n",
  340. symname);
  341. return 0;
  342. }
  343. return sym->st_value - VDSO64_LBASE;
  344. }
  345. static int __init vdso_do_func_patch64(struct lib32_elfinfo *v32,
  346. struct lib64_elfinfo *v64,
  347. const char *orig, const char *fix)
  348. {
  349. Elf64_Sym *sym64_gen, *sym64_fix;
  350. sym64_gen = find_symbol64(v64, orig);
  351. if (sym64_gen == NULL) {
  352. printk(KERN_ERR "vDSO64: Can't find symbol %s !\n", orig);
  353. return -1;
  354. }
  355. if (fix == NULL) {
  356. sym64_gen->st_name = 0;
  357. return 0;
  358. }
  359. sym64_fix = find_symbol64(v64, fix);
  360. if (sym64_fix == NULL) {
  361. printk(KERN_ERR "vDSO64: Can't find symbol %s !\n", fix);
  362. return -1;
  363. }
  364. sym64_gen->st_value = sym64_fix->st_value;
  365. sym64_gen->st_size = sym64_fix->st_size;
  366. sym64_gen->st_info = sym64_fix->st_info;
  367. sym64_gen->st_other = sym64_fix->st_other;
  368. sym64_gen->st_shndx = sym64_fix->st_shndx;
  369. return 0;
  370. }
  371. #endif /* CONFIG_PPC64 */
  372. static __init int vdso_do_find_sections(struct lib32_elfinfo *v32,
  373. struct lib64_elfinfo *v64)
  374. {
  375. void *sect;
  376. /*
  377. * Locate symbol tables & text section
  378. */
  379. #ifdef CONFIG_VDSO32
  380. v32->dynsym = find_section32(v32->hdr, ".dynsym", &v32->dynsymsize);
  381. v32->dynstr = find_section32(v32->hdr, ".dynstr", NULL);
  382. if (v32->dynsym == NULL || v32->dynstr == NULL) {
  383. printk(KERN_ERR "vDSO32: required symbol section not found\n");
  384. return -1;
  385. }
  386. sect = find_section32(v32->hdr, ".text", NULL);
  387. if (sect == NULL) {
  388. printk(KERN_ERR "vDSO32: the .text section was not found\n");
  389. return -1;
  390. }
  391. v32->text = sect - vdso32_kbase;
  392. #endif
  393. #ifdef CONFIG_PPC64
  394. v64->dynsym = find_section64(v64->hdr, ".dynsym", &v64->dynsymsize);
  395. v64->dynstr = find_section64(v64->hdr, ".dynstr", NULL);
  396. if (v64->dynsym == NULL || v64->dynstr == NULL) {
  397. printk(KERN_ERR "vDSO64: required symbol section not found\n");
  398. return -1;
  399. }
  400. sect = find_section64(v64->hdr, ".text", NULL);
  401. if (sect == NULL) {
  402. printk(KERN_ERR "vDSO64: the .text section was not found\n");
  403. return -1;
  404. }
  405. v64->text = sect - vdso64_kbase;
  406. #endif /* CONFIG_PPC64 */
  407. return 0;
  408. }
  409. static __init void vdso_setup_trampolines(struct lib32_elfinfo *v32,
  410. struct lib64_elfinfo *v64)
  411. {
  412. /*
  413. * Find signal trampolines
  414. */
  415. #ifdef CONFIG_PPC64
  416. vdso64_rt_sigtramp = find_function64(v64, "__kernel_sigtramp_rt64");
  417. #endif
  418. vdso32_sigtramp = find_function32(v32, "__kernel_sigtramp32");
  419. vdso32_rt_sigtramp = find_function32(v32, "__kernel_sigtramp_rt32");
  420. }
  421. static __init int vdso_fixup_datapage(struct lib32_elfinfo *v32,
  422. struct lib64_elfinfo *v64)
  423. {
  424. #ifdef CONFIG_VDSO32
  425. Elf32_Sym *sym32;
  426. #endif
  427. #ifdef CONFIG_PPC64
  428. Elf64_Sym *sym64;
  429. sym64 = find_symbol64(v64, "__kernel_datapage_offset");
  430. if (sym64 == NULL) {
  431. printk(KERN_ERR "vDSO64: Can't find symbol "
  432. "__kernel_datapage_offset !\n");
  433. return -1;
  434. }
  435. *((int *)(vdso64_kbase + sym64->st_value - VDSO64_LBASE)) =
  436. (vdso64_pages << PAGE_SHIFT) -
  437. (sym64->st_value - VDSO64_LBASE);
  438. #endif /* CONFIG_PPC64 */
  439. #ifdef CONFIG_VDSO32
  440. sym32 = find_symbol32(v32, "__kernel_datapage_offset");
  441. if (sym32 == NULL) {
  442. printk(KERN_ERR "vDSO32: Can't find symbol "
  443. "__kernel_datapage_offset !\n");
  444. return -1;
  445. }
  446. *((int *)(vdso32_kbase + (sym32->st_value - VDSO32_LBASE))) =
  447. (vdso32_pages << PAGE_SHIFT) -
  448. (sym32->st_value - VDSO32_LBASE);
  449. #endif
  450. return 0;
  451. }
  452. static __init int vdso_fixup_features(struct lib32_elfinfo *v32,
  453. struct lib64_elfinfo *v64)
  454. {
  455. unsigned long size;
  456. void *start;
  457. #ifdef CONFIG_PPC64
  458. start = find_section64(v64->hdr, "__ftr_fixup", &size);
  459. if (start)
  460. do_feature_fixups(cur_cpu_spec->cpu_features,
  461. start, start + size);
  462. start = find_section64(v64->hdr, "__mmu_ftr_fixup", &size);
  463. if (start)
  464. do_feature_fixups(cur_cpu_spec->mmu_features,
  465. start, start + size);
  466. start = find_section64(v64->hdr, "__fw_ftr_fixup", &size);
  467. if (start)
  468. do_feature_fixups(powerpc_firmware_features,
  469. start, start + size);
  470. start = find_section64(v64->hdr, "__lwsync_fixup", &size);
  471. if (start)
  472. do_lwsync_fixups(cur_cpu_spec->cpu_features,
  473. start, start + size);
  474. #endif /* CONFIG_PPC64 */
  475. #ifdef CONFIG_VDSO32
  476. start = find_section32(v32->hdr, "__ftr_fixup", &size);
  477. if (start)
  478. do_feature_fixups(cur_cpu_spec->cpu_features,
  479. start, start + size);
  480. start = find_section32(v32->hdr, "__mmu_ftr_fixup", &size);
  481. if (start)
  482. do_feature_fixups(cur_cpu_spec->mmu_features,
  483. start, start + size);
  484. #ifdef CONFIG_PPC64
  485. start = find_section32(v32->hdr, "__fw_ftr_fixup", &size);
  486. if (start)
  487. do_feature_fixups(powerpc_firmware_features,
  488. start, start + size);
  489. #endif /* CONFIG_PPC64 */
  490. start = find_section32(v32->hdr, "__lwsync_fixup", &size);
  491. if (start)
  492. do_lwsync_fixups(cur_cpu_spec->cpu_features,
  493. start, start + size);
  494. #endif
  495. return 0;
  496. }
  497. static __init int vdso_fixup_alt_funcs(struct lib32_elfinfo *v32,
  498. struct lib64_elfinfo *v64)
  499. {
  500. int i;
  501. for (i = 0; i < ARRAY_SIZE(vdso_patches); i++) {
  502. struct vdso_patch_def *patch = &vdso_patches[i];
  503. int match = (cur_cpu_spec->cpu_features & patch->ftr_mask)
  504. == patch->ftr_value;
  505. if (!match)
  506. continue;
  507. DBG("replacing %s with %s...\n", patch->gen_name,
  508. patch->fix_name ? "NONE" : patch->fix_name);
  509. /*
  510. * Patch the 32 bits and 64 bits symbols. Note that we do not
  511. * patch the "." symbol on 64 bits.
  512. * It would be easy to do, but doesn't seem to be necessary,
  513. * patching the OPD symbol is enough.
  514. */
  515. vdso_do_func_patch32(v32, v64, patch->gen_name,
  516. patch->fix_name);
  517. #ifdef CONFIG_PPC64
  518. vdso_do_func_patch64(v32, v64, patch->gen_name,
  519. patch->fix_name);
  520. #endif /* CONFIG_PPC64 */
  521. }
  522. return 0;
  523. }
  524. static __init int vdso_setup(void)
  525. {
  526. struct lib32_elfinfo v32;
  527. struct lib64_elfinfo v64;
  528. v32.hdr = vdso32_kbase;
  529. #ifdef CONFIG_PPC64
  530. v64.hdr = vdso64_kbase;
  531. #endif
  532. if (vdso_do_find_sections(&v32, &v64))
  533. return -1;
  534. if (vdso_fixup_datapage(&v32, &v64))
  535. return -1;
  536. if (vdso_fixup_features(&v32, &v64))
  537. return -1;
  538. if (vdso_fixup_alt_funcs(&v32, &v64))
  539. return -1;
  540. vdso_setup_trampolines(&v32, &v64);
  541. return 0;
  542. }
  543. /*
  544. * Called from setup_arch to initialize the bitmap of available
  545. * syscalls in the systemcfg page
  546. */
  547. static void __init vdso_setup_syscall_map(void)
  548. {
  549. unsigned int i;
  550. extern unsigned long *sys_call_table;
  551. #ifdef CONFIG_PPC64
  552. extern unsigned long *compat_sys_call_table;
  553. #endif
  554. extern unsigned long sys_ni_syscall;
  555. for (i = 0; i < NR_syscalls; i++) {
  556. #ifdef CONFIG_PPC64
  557. if (sys_call_table[i] != sys_ni_syscall)
  558. vdso_data->syscall_map_64[i >> 5] |=
  559. 0x80000000UL >> (i & 0x1f);
  560. if (IS_ENABLED(CONFIG_COMPAT) &&
  561. compat_sys_call_table[i] != sys_ni_syscall)
  562. vdso_data->syscall_map_32[i >> 5] |=
  563. 0x80000000UL >> (i & 0x1f);
  564. #else /* CONFIG_PPC64 */
  565. if (sys_call_table[i] != sys_ni_syscall)
  566. vdso_data->syscall_map_32[i >> 5] |=
  567. 0x80000000UL >> (i & 0x1f);
  568. #endif /* CONFIG_PPC64 */
  569. }
  570. }
  571. #ifdef CONFIG_PPC64
  572. int vdso_getcpu_init(void)
  573. {
  574. unsigned long cpu, node, val;
  575. /*
  576. * SPRG_VDSO contains the CPU in the bottom 16 bits and the NUMA node
  577. * in the next 16 bits. The VDSO uses this to implement getcpu().
  578. */
  579. cpu = get_cpu();
  580. WARN_ON_ONCE(cpu > 0xffff);
  581. node = cpu_to_node(cpu);
  582. WARN_ON_ONCE(node > 0xffff);
  583. val = (cpu & 0xfff) | ((node & 0xffff) << 16);
  584. mtspr(SPRN_SPRG_VDSO_WRITE, val);
  585. get_paca()->sprg_vdso = val;
  586. put_cpu();
  587. return 0;
  588. }
  589. /* We need to call this before SMP init */
  590. early_initcall(vdso_getcpu_init);
  591. #endif
  592. static int __init vdso_init(void)
  593. {
  594. int i;
  595. #ifdef CONFIG_PPC64
  596. /*
  597. * Fill up the "systemcfg" stuff for backward compatibility
  598. */
  599. strcpy((char *)vdso_data->eye_catcher, "SYSTEMCFG:PPC64");
  600. vdso_data->version.major = SYSTEMCFG_MAJOR;
  601. vdso_data->version.minor = SYSTEMCFG_MINOR;
  602. vdso_data->processor = mfspr(SPRN_PVR);
  603. /*
  604. * Fake the old platform number for pSeries and add
  605. * in LPAR bit if necessary
  606. */
  607. vdso_data->platform = 0x100;
  608. if (firmware_has_feature(FW_FEATURE_LPAR))
  609. vdso_data->platform |= 1;
  610. vdso_data->physicalMemorySize = memblock_phys_mem_size();
  611. vdso_data->dcache_size = ppc64_caches.l1d.size;
  612. vdso_data->dcache_line_size = ppc64_caches.l1d.line_size;
  613. vdso_data->icache_size = ppc64_caches.l1i.size;
  614. vdso_data->icache_line_size = ppc64_caches.l1i.line_size;
  615. vdso_data->dcache_block_size = ppc64_caches.l1d.block_size;
  616. vdso_data->icache_block_size = ppc64_caches.l1i.block_size;
  617. vdso_data->dcache_log_block_size = ppc64_caches.l1d.log_block_size;
  618. vdso_data->icache_log_block_size = ppc64_caches.l1i.log_block_size;
  619. /*
  620. * Calculate the size of the 64 bits vDSO
  621. */
  622. vdso64_pages = (&vdso64_end - &vdso64_start) >> PAGE_SHIFT;
  623. DBG("vdso64_kbase: %p, 0x%x pages\n", vdso64_kbase, vdso64_pages);
  624. #endif /* CONFIG_PPC64 */
  625. #ifdef CONFIG_VDSO32
  626. vdso32_kbase = &vdso32_start;
  627. /*
  628. * Calculate the size of the 32 bits vDSO
  629. */
  630. vdso32_pages = (&vdso32_end - &vdso32_start) >> PAGE_SHIFT;
  631. DBG("vdso32_kbase: %p, 0x%x pages\n", vdso32_kbase, vdso32_pages);
  632. #endif
  633. /*
  634. * Setup the syscall map in the vDOS
  635. */
  636. vdso_setup_syscall_map();
  637. /*
  638. * Initialize the vDSO images in memory, that is do necessary
  639. * fixups of vDSO symbols, locate trampolines, etc...
  640. */
  641. if (vdso_setup()) {
  642. printk(KERN_ERR "vDSO setup failure, not enabled !\n");
  643. vdso32_pages = 0;
  644. #ifdef CONFIG_PPC64
  645. vdso64_pages = 0;
  646. #endif
  647. return 0;
  648. }
  649. #ifdef CONFIG_VDSO32
  650. /* Make sure pages are in the correct state */
  651. vdso32_pagelist = kcalloc(vdso32_pages + 2, sizeof(struct page *),
  652. GFP_KERNEL);
  653. BUG_ON(vdso32_pagelist == NULL);
  654. for (i = 0; i < vdso32_pages; i++) {
  655. struct page *pg = virt_to_page(vdso32_kbase + i*PAGE_SIZE);
  656. get_page(pg);
  657. vdso32_pagelist[i] = pg;
  658. }
  659. vdso32_pagelist[i++] = virt_to_page(vdso_data);
  660. vdso32_pagelist[i] = NULL;
  661. #endif
  662. #ifdef CONFIG_PPC64
  663. vdso64_pagelist = kcalloc(vdso64_pages + 2, sizeof(struct page *),
  664. GFP_KERNEL);
  665. BUG_ON(vdso64_pagelist == NULL);
  666. for (i = 0; i < vdso64_pages; i++) {
  667. struct page *pg = virt_to_page(vdso64_kbase + i*PAGE_SIZE);
  668. get_page(pg);
  669. vdso64_pagelist[i] = pg;
  670. }
  671. vdso64_pagelist[i++] = virt_to_page(vdso_data);
  672. vdso64_pagelist[i] = NULL;
  673. #endif /* CONFIG_PPC64 */
  674. get_page(virt_to_page(vdso_data));
  675. smp_wmb();
  676. vdso_ready = 1;
  677. return 0;
  678. }
  679. arch_initcall(vdso_init);