PageRenderTime 67ms CodeModel.GetById 26ms RepoModel.GetById 0ms app.codeStats 0ms

/arch/x86/kvm/svm.c

https://bitbucket.org/thekraven/iscream_thunderc-2.6.35
C | 3428 lines | 2605 code | 613 blank | 210 comment | 283 complexity | c032176705d9a327e97ba8363317307d MD5 | raw file
Possible License(s): GPL-2.0, LGPL-2.0, AGPL-1.0
  1. /*
  2. * Kernel-based Virtual Machine driver for Linux
  3. *
  4. * AMD SVM support
  5. *
  6. * Copyright (C) 2006 Qumranet, Inc.
  7. *
  8. * Authors:
  9. * Yaniv Kamay <yaniv@qumranet.com>
  10. * Avi Kivity <avi@qumranet.com>
  11. *
  12. * This work is licensed under the terms of the GNU GPL, version 2. See
  13. * the COPYING file in the top-level directory.
  14. *
  15. */
  16. #include <linux/kvm_host.h>
  17. #include "irq.h"
  18. #include "mmu.h"
  19. #include "kvm_cache_regs.h"
  20. #include "x86.h"
  21. #include <linux/module.h>
  22. #include <linux/kernel.h>
  23. #include <linux/vmalloc.h>
  24. #include <linux/highmem.h>
  25. #include <linux/sched.h>
  26. #include <linux/ftrace_event.h>
  27. #include <linux/slab.h>
  28. #include <asm/tlbflush.h>
  29. #include <asm/desc.h>
  30. #include <asm/virtext.h>
  31. #include "trace.h"
  32. #define __ex(x) __kvm_handle_fault_on_reboot(x)
  33. MODULE_AUTHOR("Qumranet");
  34. MODULE_LICENSE("GPL");
  35. #define IOPM_ALLOC_ORDER 2
  36. #define MSRPM_ALLOC_ORDER 1
  37. #define SEG_TYPE_LDT 2
  38. #define SEG_TYPE_BUSY_TSS16 3
  39. #define SVM_FEATURE_NPT (1 << 0)
  40. #define SVM_FEATURE_LBRV (1 << 1)
  41. #define SVM_FEATURE_SVML (1 << 2)
  42. #define SVM_FEATURE_NRIP (1 << 3)
  43. #define SVM_FEATURE_PAUSE_FILTER (1 << 10)
  44. #define NESTED_EXIT_HOST 0 /* Exit handled on host level */
  45. #define NESTED_EXIT_DONE 1 /* Exit caused nested vmexit */
  46. #define NESTED_EXIT_CONTINUE 2 /* Further checks needed */
  47. #define DEBUGCTL_RESERVED_BITS (~(0x3fULL))
  48. static bool erratum_383_found __read_mostly;
  49. static const u32 host_save_user_msrs[] = {
  50. #ifdef CONFIG_X86_64
  51. MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE,
  52. MSR_FS_BASE,
  53. #endif
  54. MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
  55. };
  56. #define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs)
  57. struct kvm_vcpu;
  58. struct nested_state {
  59. struct vmcb *hsave;
  60. u64 hsave_msr;
  61. u64 vm_cr_msr;
  62. u64 vmcb;
  63. /* These are the merged vectors */
  64. u32 *msrpm;
  65. /* gpa pointers to the real vectors */
  66. u64 vmcb_msrpm;
  67. u64 vmcb_iopm;
  68. /* A VMEXIT is required but not yet emulated */
  69. bool exit_required;
  70. /* cache for intercepts of the guest */
  71. u16 intercept_cr_read;
  72. u16 intercept_cr_write;
  73. u16 intercept_dr_read;
  74. u16 intercept_dr_write;
  75. u32 intercept_exceptions;
  76. u64 intercept;
  77. };
  78. #define MSRPM_OFFSETS 16
  79. static u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
  80. struct vcpu_svm {
  81. struct kvm_vcpu vcpu;
  82. struct vmcb *vmcb;
  83. unsigned long vmcb_pa;
  84. struct svm_cpu_data *svm_data;
  85. uint64_t asid_generation;
  86. uint64_t sysenter_esp;
  87. uint64_t sysenter_eip;
  88. u64 next_rip;
  89. u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS];
  90. u64 host_gs_base;
  91. u32 *msrpm;
  92. struct nested_state nested;
  93. bool nmi_singlestep;
  94. unsigned int3_injected;
  95. unsigned long int3_rip;
  96. };
  97. #define MSR_INVALID 0xffffffffU
  98. static struct svm_direct_access_msrs {
  99. u32 index; /* Index of the MSR */
  100. bool always; /* True if intercept is always on */
  101. } direct_access_msrs[] = {
  102. { .index = MSR_K6_STAR, .always = true },
  103. { .index = MSR_IA32_SYSENTER_CS, .always = true },
  104. #ifdef CONFIG_X86_64
  105. { .index = MSR_GS_BASE, .always = true },
  106. { .index = MSR_FS_BASE, .always = true },
  107. { .index = MSR_KERNEL_GS_BASE, .always = true },
  108. { .index = MSR_LSTAR, .always = true },
  109. { .index = MSR_CSTAR, .always = true },
  110. { .index = MSR_SYSCALL_MASK, .always = true },
  111. #endif
  112. { .index = MSR_IA32_LASTBRANCHFROMIP, .always = false },
  113. { .index = MSR_IA32_LASTBRANCHTOIP, .always = false },
  114. { .index = MSR_IA32_LASTINTFROMIP, .always = false },
  115. { .index = MSR_IA32_LASTINTTOIP, .always = false },
  116. { .index = MSR_INVALID, .always = false },
  117. };
  118. /* enable NPT for AMD64 and X86 with PAE */
  119. #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
  120. static bool npt_enabled = true;
  121. #else
  122. static bool npt_enabled;
  123. #endif
  124. static int npt = 1;
  125. module_param(npt, int, S_IRUGO);
  126. static int nested = 1;
  127. module_param(nested, int, S_IRUGO);
  128. static void svm_flush_tlb(struct kvm_vcpu *vcpu);
  129. static void svm_complete_interrupts(struct vcpu_svm *svm);
  130. static int nested_svm_exit_handled(struct vcpu_svm *svm);
  131. static int nested_svm_intercept(struct vcpu_svm *svm);
  132. static int nested_svm_vmexit(struct vcpu_svm *svm);
  133. static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
  134. bool has_error_code, u32 error_code);
  135. static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
  136. {
  137. return container_of(vcpu, struct vcpu_svm, vcpu);
  138. }
  139. static inline bool is_nested(struct vcpu_svm *svm)
  140. {
  141. return svm->nested.vmcb;
  142. }
  143. static inline void enable_gif(struct vcpu_svm *svm)
  144. {
  145. svm->vcpu.arch.hflags |= HF_GIF_MASK;
  146. }
  147. static inline void disable_gif(struct vcpu_svm *svm)
  148. {
  149. svm->vcpu.arch.hflags &= ~HF_GIF_MASK;
  150. }
  151. static inline bool gif_set(struct vcpu_svm *svm)
  152. {
  153. return !!(svm->vcpu.arch.hflags & HF_GIF_MASK);
  154. }
  155. static unsigned long iopm_base;
  156. struct kvm_ldttss_desc {
  157. u16 limit0;
  158. u16 base0;
  159. unsigned base1:8, type:5, dpl:2, p:1;
  160. unsigned limit1:4, zero0:3, g:1, base2:8;
  161. u32 base3;
  162. u32 zero1;
  163. } __attribute__((packed));
  164. struct svm_cpu_data {
  165. int cpu;
  166. u64 asid_generation;
  167. u32 max_asid;
  168. u32 next_asid;
  169. struct kvm_ldttss_desc *tss_desc;
  170. struct page *save_area;
  171. };
  172. static DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);
  173. static uint32_t svm_features;
  174. struct svm_init_data {
  175. int cpu;
  176. int r;
  177. };
  178. static u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
  179. #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
  180. #define MSRS_RANGE_SIZE 2048
  181. #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
  182. static u32 svm_msrpm_offset(u32 msr)
  183. {
  184. u32 offset;
  185. int i;
  186. for (i = 0; i < NUM_MSR_MAPS; i++) {
  187. if (msr < msrpm_ranges[i] ||
  188. msr >= msrpm_ranges[i] + MSRS_IN_RANGE)
  189. continue;
  190. offset = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */
  191. offset += (i * MSRS_RANGE_SIZE); /* add range offset */
  192. /* Now we have the u8 offset - but need the u32 offset */
  193. return offset / 4;
  194. }
  195. /* MSR not in any range */
  196. return MSR_INVALID;
  197. }
  198. #define MAX_INST_SIZE 15
  199. static inline u32 svm_has(u32 feat)
  200. {
  201. return svm_features & feat;
  202. }
  203. static inline void clgi(void)
  204. {
  205. asm volatile (__ex(SVM_CLGI));
  206. }
  207. static inline void stgi(void)
  208. {
  209. asm volatile (__ex(SVM_STGI));
  210. }
  211. static inline void invlpga(unsigned long addr, u32 asid)
  212. {
  213. asm volatile (__ex(SVM_INVLPGA) : : "a"(addr), "c"(asid));
  214. }
  215. static inline void force_new_asid(struct kvm_vcpu *vcpu)
  216. {
  217. to_svm(vcpu)->asid_generation--;
  218. }
  219. static inline void flush_guest_tlb(struct kvm_vcpu *vcpu)
  220. {
  221. force_new_asid(vcpu);
  222. }
  223. static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
  224. {
  225. if (!npt_enabled && !(efer & EFER_LMA))
  226. efer &= ~EFER_LME;
  227. to_svm(vcpu)->vmcb->save.efer = efer | EFER_SVME;
  228. vcpu->arch.efer = efer;
  229. }
  230. static int is_external_interrupt(u32 info)
  231. {
  232. info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
  233. return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
  234. }
  235. static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
  236. {
  237. struct vcpu_svm *svm = to_svm(vcpu);
  238. u32 ret = 0;
  239. if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
  240. ret |= KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
  241. return ret & mask;
  242. }
  243. static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
  244. {
  245. struct vcpu_svm *svm = to_svm(vcpu);
  246. if (mask == 0)
  247. svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
  248. else
  249. svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;
  250. }
  251. static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
  252. {
  253. struct vcpu_svm *svm = to_svm(vcpu);
  254. if (svm->vmcb->control.next_rip != 0)
  255. svm->next_rip = svm->vmcb->control.next_rip;
  256. if (!svm->next_rip) {
  257. if (emulate_instruction(vcpu, 0, 0, EMULTYPE_SKIP) !=
  258. EMULATE_DONE)
  259. printk(KERN_DEBUG "%s: NOP\n", __func__);
  260. return;
  261. }
  262. if (svm->next_rip - kvm_rip_read(vcpu) > MAX_INST_SIZE)
  263. printk(KERN_ERR "%s: ip 0x%lx next 0x%llx\n",
  264. __func__, kvm_rip_read(vcpu), svm->next_rip);
  265. kvm_rip_write(vcpu, svm->next_rip);
  266. svm_set_interrupt_shadow(vcpu, 0);
  267. }
  268. static void svm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
  269. bool has_error_code, u32 error_code,
  270. bool reinject)
  271. {
  272. struct vcpu_svm *svm = to_svm(vcpu);
  273. /*
  274. * If we are within a nested VM we'd better #VMEXIT and let the guest
  275. * handle the exception
  276. */
  277. if (!reinject &&
  278. nested_svm_check_exception(svm, nr, has_error_code, error_code))
  279. return;
  280. if (nr == BP_VECTOR && !svm_has(SVM_FEATURE_NRIP)) {
  281. unsigned long rip, old_rip = kvm_rip_read(&svm->vcpu);
  282. /*
  283. * For guest debugging where we have to reinject #BP if some
  284. * INT3 is guest-owned:
  285. * Emulate nRIP by moving RIP forward. Will fail if injection
  286. * raises a fault that is not intercepted. Still better than
  287. * failing in all cases.
  288. */
  289. skip_emulated_instruction(&svm->vcpu);
  290. rip = kvm_rip_read(&svm->vcpu);
  291. svm->int3_rip = rip + svm->vmcb->save.cs.base;
  292. svm->int3_injected = rip - old_rip;
  293. }
  294. svm->vmcb->control.event_inj = nr
  295. | SVM_EVTINJ_VALID
  296. | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
  297. | SVM_EVTINJ_TYPE_EXEPT;
  298. svm->vmcb->control.event_inj_err = error_code;
  299. }
  300. static void svm_init_erratum_383(void)
  301. {
  302. u32 low, high;
  303. int err;
  304. u64 val;
  305. /* Only Fam10h is affected */
  306. if (boot_cpu_data.x86 != 0x10)
  307. return;
  308. /* Use _safe variants to not break nested virtualization */
  309. val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err);
  310. if (err)
  311. return;
  312. val |= (1ULL << 47);
  313. low = lower_32_bits(val);
  314. high = upper_32_bits(val);
  315. native_write_msr_safe(MSR_AMD64_DC_CFG, low, high);
  316. erratum_383_found = true;
  317. }
  318. static int has_svm(void)
  319. {
  320. const char *msg;
  321. if (!cpu_has_svm(&msg)) {
  322. printk(KERN_INFO "has_svm: %s\n", msg);
  323. return 0;
  324. }
  325. return 1;
  326. }
  327. static void svm_hardware_disable(void *garbage)
  328. {
  329. cpu_svm_disable();
  330. }
  331. static int svm_hardware_enable(void *garbage)
  332. {
  333. struct svm_cpu_data *sd;
  334. uint64_t efer;
  335. struct desc_ptr gdt_descr;
  336. struct desc_struct *gdt;
  337. int me = raw_smp_processor_id();
  338. rdmsrl(MSR_EFER, efer);
  339. if (efer & EFER_SVME)
  340. return -EBUSY;
  341. if (!has_svm()) {
  342. printk(KERN_ERR "svm_hardware_enable: err EOPNOTSUPP on %d\n",
  343. me);
  344. return -EINVAL;
  345. }
  346. sd = per_cpu(svm_data, me);
  347. if (!sd) {
  348. printk(KERN_ERR "svm_hardware_enable: svm_data is NULL on %d\n",
  349. me);
  350. return -EINVAL;
  351. }
  352. sd->asid_generation = 1;
  353. sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
  354. sd->next_asid = sd->max_asid + 1;
  355. native_store_gdt(&gdt_descr);
  356. gdt = (struct desc_struct *)gdt_descr.address;
  357. sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
  358. wrmsrl(MSR_EFER, efer | EFER_SVME);
  359. wrmsrl(MSR_VM_HSAVE_PA, page_to_pfn(sd->save_area) << PAGE_SHIFT);
  360. svm_init_erratum_383();
  361. return 0;
  362. }
  363. static void svm_cpu_uninit(int cpu)
  364. {
  365. struct svm_cpu_data *sd = per_cpu(svm_data, raw_smp_processor_id());
  366. if (!sd)
  367. return;
  368. per_cpu(svm_data, raw_smp_processor_id()) = NULL;
  369. __free_page(sd->save_area);
  370. kfree(sd);
  371. }
  372. static int svm_cpu_init(int cpu)
  373. {
  374. struct svm_cpu_data *sd;
  375. int r;
  376. sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
  377. if (!sd)
  378. return -ENOMEM;
  379. sd->cpu = cpu;
  380. sd->save_area = alloc_page(GFP_KERNEL);
  381. r = -ENOMEM;
  382. if (!sd->save_area)
  383. goto err_1;
  384. per_cpu(svm_data, cpu) = sd;
  385. return 0;
  386. err_1:
  387. kfree(sd);
  388. return r;
  389. }
  390. static bool valid_msr_intercept(u32 index)
  391. {
  392. int i;
  393. for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++)
  394. if (direct_access_msrs[i].index == index)
  395. return true;
  396. return false;
  397. }
  398. static void set_msr_interception(u32 *msrpm, unsigned msr,
  399. int read, int write)
  400. {
  401. u8 bit_read, bit_write;
  402. unsigned long tmp;
  403. u32 offset;
  404. /*
  405. * If this warning triggers extend the direct_access_msrs list at the
  406. * beginning of the file
  407. */
  408. WARN_ON(!valid_msr_intercept(msr));
  409. offset = svm_msrpm_offset(msr);
  410. bit_read = 2 * (msr & 0x0f);
  411. bit_write = 2 * (msr & 0x0f) + 1;
  412. tmp = msrpm[offset];
  413. BUG_ON(offset == MSR_INVALID);
  414. read ? clear_bit(bit_read, &tmp) : set_bit(bit_read, &tmp);
  415. write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp);
  416. msrpm[offset] = tmp;
  417. }
  418. static void svm_vcpu_init_msrpm(u32 *msrpm)
  419. {
  420. int i;
  421. memset(msrpm, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER));
  422. for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
  423. if (!direct_access_msrs[i].always)
  424. continue;
  425. set_msr_interception(msrpm, direct_access_msrs[i].index, 1, 1);
  426. }
  427. }
  428. static void add_msr_offset(u32 offset)
  429. {
  430. int i;
  431. for (i = 0; i < MSRPM_OFFSETS; ++i) {
  432. /* Offset already in list? */
  433. if (msrpm_offsets[i] == offset)
  434. return;
  435. /* Slot used by another offset? */
  436. if (msrpm_offsets[i] != MSR_INVALID)
  437. continue;
  438. /* Add offset to list */
  439. msrpm_offsets[i] = offset;
  440. return;
  441. }
  442. /*
  443. * If this BUG triggers the msrpm_offsets table has an overflow. Just
  444. * increase MSRPM_OFFSETS in this case.
  445. */
  446. BUG();
  447. }
  448. static void init_msrpm_offsets(void)
  449. {
  450. int i;
  451. memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets));
  452. for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
  453. u32 offset;
  454. offset = svm_msrpm_offset(direct_access_msrs[i].index);
  455. BUG_ON(offset == MSR_INVALID);
  456. add_msr_offset(offset);
  457. }
  458. }
  459. static void svm_enable_lbrv(struct vcpu_svm *svm)
  460. {
  461. u32 *msrpm = svm->msrpm;
  462. svm->vmcb->control.lbr_ctl = 1;
  463. set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
  464. set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
  465. set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
  466. set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
  467. }
  468. static void svm_disable_lbrv(struct vcpu_svm *svm)
  469. {
  470. u32 *msrpm = svm->msrpm;
  471. svm->vmcb->control.lbr_ctl = 0;
  472. set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
  473. set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
  474. set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
  475. set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
  476. }
  477. static __init int svm_hardware_setup(void)
  478. {
  479. int cpu;
  480. struct page *iopm_pages;
  481. void *iopm_va;
  482. int r;
  483. iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER);
  484. if (!iopm_pages)
  485. return -ENOMEM;
  486. iopm_va = page_address(iopm_pages);
  487. memset(iopm_va, 0xff, PAGE_SIZE * (1 << IOPM_ALLOC_ORDER));
  488. iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;
  489. init_msrpm_offsets();
  490. if (boot_cpu_has(X86_FEATURE_NX))
  491. kvm_enable_efer_bits(EFER_NX);
  492. if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
  493. kvm_enable_efer_bits(EFER_FFXSR);
  494. if (nested) {
  495. printk(KERN_INFO "kvm: Nested Virtualization enabled\n");
  496. kvm_enable_efer_bits(EFER_SVME);
  497. }
  498. for_each_possible_cpu(cpu) {
  499. r = svm_cpu_init(cpu);
  500. if (r)
  501. goto err;
  502. }
  503. svm_features = cpuid_edx(SVM_CPUID_FUNC);
  504. if (!svm_has(SVM_FEATURE_NPT))
  505. npt_enabled = false;
  506. if (npt_enabled && !npt) {
  507. printk(KERN_INFO "kvm: Nested Paging disabled\n");
  508. npt_enabled = false;
  509. }
  510. if (npt_enabled) {
  511. printk(KERN_INFO "kvm: Nested Paging enabled\n");
  512. kvm_enable_tdp();
  513. } else
  514. kvm_disable_tdp();
  515. return 0;
  516. err:
  517. __free_pages(iopm_pages, IOPM_ALLOC_ORDER);
  518. iopm_base = 0;
  519. return r;
  520. }
  521. static __exit void svm_hardware_unsetup(void)
  522. {
  523. int cpu;
  524. for_each_possible_cpu(cpu)
  525. svm_cpu_uninit(cpu);
  526. __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER);
  527. iopm_base = 0;
  528. }
  529. static void init_seg(struct vmcb_seg *seg)
  530. {
  531. seg->selector = 0;
  532. seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
  533. SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
  534. seg->limit = 0xffff;
  535. seg->base = 0;
  536. }
  537. static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
  538. {
  539. seg->selector = 0;
  540. seg->attrib = SVM_SELECTOR_P_MASK | type;
  541. seg->limit = 0xffff;
  542. seg->base = 0;
  543. }
  544. static void init_vmcb(struct vcpu_svm *svm)
  545. {
  546. struct vmcb_control_area *control = &svm->vmcb->control;
  547. struct vmcb_save_area *save = &svm->vmcb->save;
  548. svm->vcpu.fpu_active = 1;
  549. control->intercept_cr_read = INTERCEPT_CR0_MASK |
  550. INTERCEPT_CR3_MASK |
  551. INTERCEPT_CR4_MASK;
  552. control->intercept_cr_write = INTERCEPT_CR0_MASK |
  553. INTERCEPT_CR3_MASK |
  554. INTERCEPT_CR4_MASK |
  555. INTERCEPT_CR8_MASK;
  556. control->intercept_dr_read = INTERCEPT_DR0_MASK |
  557. INTERCEPT_DR1_MASK |
  558. INTERCEPT_DR2_MASK |
  559. INTERCEPT_DR3_MASK |
  560. INTERCEPT_DR4_MASK |
  561. INTERCEPT_DR5_MASK |
  562. INTERCEPT_DR6_MASK |
  563. INTERCEPT_DR7_MASK;
  564. control->intercept_dr_write = INTERCEPT_DR0_MASK |
  565. INTERCEPT_DR1_MASK |
  566. INTERCEPT_DR2_MASK |
  567. INTERCEPT_DR3_MASK |
  568. INTERCEPT_DR4_MASK |
  569. INTERCEPT_DR5_MASK |
  570. INTERCEPT_DR6_MASK |
  571. INTERCEPT_DR7_MASK;
  572. control->intercept_exceptions = (1 << PF_VECTOR) |
  573. (1 << UD_VECTOR) |
  574. (1 << MC_VECTOR);
  575. control->intercept = (1ULL << INTERCEPT_INTR) |
  576. (1ULL << INTERCEPT_NMI) |
  577. (1ULL << INTERCEPT_SMI) |
  578. (1ULL << INTERCEPT_SELECTIVE_CR0) |
  579. (1ULL << INTERCEPT_CPUID) |
  580. (1ULL << INTERCEPT_INVD) |
  581. (1ULL << INTERCEPT_HLT) |
  582. (1ULL << INTERCEPT_INVLPG) |
  583. (1ULL << INTERCEPT_INVLPGA) |
  584. (1ULL << INTERCEPT_IOIO_PROT) |
  585. (1ULL << INTERCEPT_MSR_PROT) |
  586. (1ULL << INTERCEPT_TASK_SWITCH) |
  587. (1ULL << INTERCEPT_SHUTDOWN) |
  588. (1ULL << INTERCEPT_VMRUN) |
  589. (1ULL << INTERCEPT_VMMCALL) |
  590. (1ULL << INTERCEPT_VMLOAD) |
  591. (1ULL << INTERCEPT_VMSAVE) |
  592. (1ULL << INTERCEPT_STGI) |
  593. (1ULL << INTERCEPT_CLGI) |
  594. (1ULL << INTERCEPT_SKINIT) |
  595. (1ULL << INTERCEPT_WBINVD) |
  596. (1ULL << INTERCEPT_MONITOR) |
  597. (1ULL << INTERCEPT_MWAIT);
  598. control->iopm_base_pa = iopm_base;
  599. control->msrpm_base_pa = __pa(svm->msrpm);
  600. control->tsc_offset = 0;
  601. control->int_ctl = V_INTR_MASKING_MASK;
  602. init_seg(&save->es);
  603. init_seg(&save->ss);
  604. init_seg(&save->ds);
  605. init_seg(&save->fs);
  606. init_seg(&save->gs);
  607. save->cs.selector = 0xf000;
  608. /* Executable/Readable Code Segment */
  609. save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
  610. SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
  611. save->cs.limit = 0xffff;
  612. /*
  613. * cs.base should really be 0xffff0000, but vmx can't handle that, so
  614. * be consistent with it.
  615. *
  616. * Replace when we have real mode working for vmx.
  617. */
  618. save->cs.base = 0xf0000;
  619. save->gdtr.limit = 0xffff;
  620. save->idtr.limit = 0xffff;
  621. init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
  622. init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
  623. save->efer = EFER_SVME;
  624. save->dr6 = 0xffff0ff0;
  625. save->dr7 = 0x400;
  626. save->rflags = 2;
  627. save->rip = 0x0000fff0;
  628. svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip;
  629. /*
  630. * This is the guest-visible cr0 value.
  631. * svm_set_cr0() sets PG and WP and clears NW and CD on save->cr0.
  632. */
  633. svm->vcpu.arch.cr0 = X86_CR0_NW | X86_CR0_CD | X86_CR0_ET;
  634. kvm_set_cr0(&svm->vcpu, svm->vcpu.arch.cr0);
  635. save->cr4 = X86_CR4_PAE;
  636. /* rdx = ?? */
  637. if (npt_enabled) {
  638. /* Setup VMCB for Nested Paging */
  639. control->nested_ctl = 1;
  640. control->intercept &= ~((1ULL << INTERCEPT_TASK_SWITCH) |
  641. (1ULL << INTERCEPT_INVLPG));
  642. control->intercept_exceptions &= ~(1 << PF_VECTOR);
  643. control->intercept_cr_read &= ~INTERCEPT_CR3_MASK;
  644. control->intercept_cr_write &= ~INTERCEPT_CR3_MASK;
  645. save->g_pat = 0x0007040600070406ULL;
  646. save->cr3 = 0;
  647. save->cr4 = 0;
  648. }
  649. force_new_asid(&svm->vcpu);
  650. svm->nested.vmcb = 0;
  651. svm->vcpu.arch.hflags = 0;
  652. if (svm_has(SVM_FEATURE_PAUSE_FILTER)) {
  653. control->pause_filter_count = 3000;
  654. control->intercept |= (1ULL << INTERCEPT_PAUSE);
  655. }
  656. enable_gif(svm);
  657. }
  658. static int svm_vcpu_reset(struct kvm_vcpu *vcpu)
  659. {
  660. struct vcpu_svm *svm = to_svm(vcpu);
  661. init_vmcb(svm);
  662. if (!kvm_vcpu_is_bsp(vcpu)) {
  663. kvm_rip_write(vcpu, 0);
  664. svm->vmcb->save.cs.base = svm->vcpu.arch.sipi_vector << 12;
  665. svm->vmcb->save.cs.selector = svm->vcpu.arch.sipi_vector << 8;
  666. }
  667. vcpu->arch.regs_avail = ~0;
  668. vcpu->arch.regs_dirty = ~0;
  669. return 0;
  670. }
  671. static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
  672. {
  673. struct vcpu_svm *svm;
  674. struct page *page;
  675. struct page *msrpm_pages;
  676. struct page *hsave_page;
  677. struct page *nested_msrpm_pages;
  678. int err;
  679. svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
  680. if (!svm) {
  681. err = -ENOMEM;
  682. goto out;
  683. }
  684. err = kvm_vcpu_init(&svm->vcpu, kvm, id);
  685. if (err)
  686. goto free_svm;
  687. err = -ENOMEM;
  688. page = alloc_page(GFP_KERNEL);
  689. if (!page)
  690. goto uninit;
  691. msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
  692. if (!msrpm_pages)
  693. goto free_page1;
  694. nested_msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
  695. if (!nested_msrpm_pages)
  696. goto free_page2;
  697. hsave_page = alloc_page(GFP_KERNEL);
  698. if (!hsave_page)
  699. goto free_page3;
  700. svm->nested.hsave = page_address(hsave_page);
  701. svm->msrpm = page_address(msrpm_pages);
  702. svm_vcpu_init_msrpm(svm->msrpm);
  703. svm->nested.msrpm = page_address(nested_msrpm_pages);
  704. svm_vcpu_init_msrpm(svm->nested.msrpm);
  705. svm->vmcb = page_address(page);
  706. clear_page(svm->vmcb);
  707. svm->vmcb_pa = page_to_pfn(page) << PAGE_SHIFT;
  708. svm->asid_generation = 0;
  709. init_vmcb(svm);
  710. fx_init(&svm->vcpu);
  711. svm->vcpu.arch.apic_base = 0xfee00000 | MSR_IA32_APICBASE_ENABLE;
  712. if (kvm_vcpu_is_bsp(&svm->vcpu))
  713. svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP;
  714. return &svm->vcpu;
  715. free_page3:
  716. __free_pages(nested_msrpm_pages, MSRPM_ALLOC_ORDER);
  717. free_page2:
  718. __free_pages(msrpm_pages, MSRPM_ALLOC_ORDER);
  719. free_page1:
  720. __free_page(page);
  721. uninit:
  722. kvm_vcpu_uninit(&svm->vcpu);
  723. free_svm:
  724. kmem_cache_free(kvm_vcpu_cache, svm);
  725. out:
  726. return ERR_PTR(err);
  727. }
  728. static void svm_free_vcpu(struct kvm_vcpu *vcpu)
  729. {
  730. struct vcpu_svm *svm = to_svm(vcpu);
  731. __free_page(pfn_to_page(svm->vmcb_pa >> PAGE_SHIFT));
  732. __free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER);
  733. __free_page(virt_to_page(svm->nested.hsave));
  734. __free_pages(virt_to_page(svm->nested.msrpm), MSRPM_ALLOC_ORDER);
  735. kvm_vcpu_uninit(vcpu);
  736. kmem_cache_free(kvm_vcpu_cache, svm);
  737. }
  738. static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
  739. {
  740. struct vcpu_svm *svm = to_svm(vcpu);
  741. int i;
  742. if (unlikely(cpu != vcpu->cpu)) {
  743. u64 delta;
  744. if (check_tsc_unstable()) {
  745. /*
  746. * Make sure that the guest sees a monotonically
  747. * increasing TSC.
  748. */
  749. delta = vcpu->arch.host_tsc - native_read_tsc();
  750. svm->vmcb->control.tsc_offset += delta;
  751. if (is_nested(svm))
  752. svm->nested.hsave->control.tsc_offset += delta;
  753. }
  754. vcpu->cpu = cpu;
  755. kvm_migrate_timers(vcpu);
  756. svm->asid_generation = 0;
  757. }
  758. for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
  759. rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
  760. }
  761. static void svm_vcpu_put(struct kvm_vcpu *vcpu)
  762. {
  763. struct vcpu_svm *svm = to_svm(vcpu);
  764. int i;
  765. ++vcpu->stat.host_state_reload;
  766. for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
  767. wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
  768. vcpu->arch.host_tsc = native_read_tsc();
  769. }
  770. static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
  771. {
  772. return to_svm(vcpu)->vmcb->save.rflags;
  773. }
  774. static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
  775. {
  776. to_svm(vcpu)->vmcb->save.rflags = rflags;
  777. }
  778. static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
  779. {
  780. switch (reg) {
  781. case VCPU_EXREG_PDPTR:
  782. BUG_ON(!npt_enabled);
  783. load_pdptrs(vcpu, vcpu->arch.cr3);
  784. break;
  785. default:
  786. BUG();
  787. }
  788. }
  789. static void svm_set_vintr(struct vcpu_svm *svm)
  790. {
  791. svm->vmcb->control.intercept |= 1ULL << INTERCEPT_VINTR;
  792. }
  793. static void svm_clear_vintr(struct vcpu_svm *svm)
  794. {
  795. svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_VINTR);
  796. }
  797. static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
  798. {
  799. struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
  800. switch (seg) {
  801. case VCPU_SREG_CS: return &save->cs;
  802. case VCPU_SREG_DS: return &save->ds;
  803. case VCPU_SREG_ES: return &save->es;
  804. case VCPU_SREG_FS: return &save->fs;
  805. case VCPU_SREG_GS: return &save->gs;
  806. case VCPU_SREG_SS: return &save->ss;
  807. case VCPU_SREG_TR: return &save->tr;
  808. case VCPU_SREG_LDTR: return &save->ldtr;
  809. }
  810. BUG();
  811. return NULL;
  812. }
  813. static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
  814. {
  815. struct vmcb_seg *s = svm_seg(vcpu, seg);
  816. return s->base;
  817. }
  818. static void svm_get_segment(struct kvm_vcpu *vcpu,
  819. struct kvm_segment *var, int seg)
  820. {
  821. struct vmcb_seg *s = svm_seg(vcpu, seg);
  822. var->base = s->base;
  823. var->limit = s->limit;
  824. var->selector = s->selector;
  825. var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
  826. var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
  827. var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
  828. var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
  829. var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
  830. var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
  831. var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
  832. var->g = (s->attrib >> SVM_SELECTOR_G_SHIFT) & 1;
  833. /*
  834. * AMD's VMCB does not have an explicit unusable field, so emulate it
  835. * for cross vendor migration purposes by "not present"
  836. */
  837. var->unusable = !var->present || (var->type == 0);
  838. switch (seg) {
  839. case VCPU_SREG_CS:
  840. /*
  841. * SVM always stores 0 for the 'G' bit in the CS selector in
  842. * the VMCB on a VMEXIT. This hurts cross-vendor migration:
  843. * Intel's VMENTRY has a check on the 'G' bit.
  844. */
  845. var->g = s->limit > 0xfffff;
  846. break;
  847. case VCPU_SREG_TR:
  848. /*
  849. * Work around a bug where the busy flag in the tr selector
  850. * isn't exposed
  851. */
  852. var->type |= 0x2;
  853. break;
  854. case VCPU_SREG_DS:
  855. case VCPU_SREG_ES:
  856. case VCPU_SREG_FS:
  857. case VCPU_SREG_GS:
  858. /*
  859. * The accessed bit must always be set in the segment
  860. * descriptor cache, although it can be cleared in the
  861. * descriptor, the cached bit always remains at 1. Since
  862. * Intel has a check on this, set it here to support
  863. * cross-vendor migration.
  864. */
  865. if (!var->unusable)
  866. var->type |= 0x1;
  867. break;
  868. case VCPU_SREG_SS:
  869. /*
  870. * On AMD CPUs sometimes the DB bit in the segment
  871. * descriptor is left as 1, although the whole segment has
  872. * been made unusable. Clear it here to pass an Intel VMX
  873. * entry check when cross vendor migrating.
  874. */
  875. if (var->unusable)
  876. var->db = 0;
  877. break;
  878. }
  879. }
  880. static int svm_get_cpl(struct kvm_vcpu *vcpu)
  881. {
  882. struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
  883. return save->cpl;
  884. }
  885. static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
  886. {
  887. struct vcpu_svm *svm = to_svm(vcpu);
  888. dt->size = svm->vmcb->save.idtr.limit;
  889. dt->address = svm->vmcb->save.idtr.base;
  890. }
  891. static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
  892. {
  893. struct vcpu_svm *svm = to_svm(vcpu);
  894. svm->vmcb->save.idtr.limit = dt->size;
  895. svm->vmcb->save.idtr.base = dt->address ;
  896. }
  897. static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
  898. {
  899. struct vcpu_svm *svm = to_svm(vcpu);
  900. dt->size = svm->vmcb->save.gdtr.limit;
  901. dt->address = svm->vmcb->save.gdtr.base;
  902. }
  903. static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
  904. {
  905. struct vcpu_svm *svm = to_svm(vcpu);
  906. svm->vmcb->save.gdtr.limit = dt->size;
  907. svm->vmcb->save.gdtr.base = dt->address ;
  908. }
  909. static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
  910. {
  911. }
  912. static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
  913. {
  914. }
  915. static void update_cr0_intercept(struct vcpu_svm *svm)
  916. {
  917. struct vmcb *vmcb = svm->vmcb;
  918. ulong gcr0 = svm->vcpu.arch.cr0;
  919. u64 *hcr0 = &svm->vmcb->save.cr0;
  920. if (!svm->vcpu.fpu_active)
  921. *hcr0 |= SVM_CR0_SELECTIVE_MASK;
  922. else
  923. *hcr0 = (*hcr0 & ~SVM_CR0_SELECTIVE_MASK)
  924. | (gcr0 & SVM_CR0_SELECTIVE_MASK);
  925. if (gcr0 == *hcr0 && svm->vcpu.fpu_active) {
  926. vmcb->control.intercept_cr_read &= ~INTERCEPT_CR0_MASK;
  927. vmcb->control.intercept_cr_write &= ~INTERCEPT_CR0_MASK;
  928. if (is_nested(svm)) {
  929. struct vmcb *hsave = svm->nested.hsave;
  930. hsave->control.intercept_cr_read &= ~INTERCEPT_CR0_MASK;
  931. hsave->control.intercept_cr_write &= ~INTERCEPT_CR0_MASK;
  932. vmcb->control.intercept_cr_read |= svm->nested.intercept_cr_read;
  933. vmcb->control.intercept_cr_write |= svm->nested.intercept_cr_write;
  934. }
  935. } else {
  936. svm->vmcb->control.intercept_cr_read |= INTERCEPT_CR0_MASK;
  937. svm->vmcb->control.intercept_cr_write |= INTERCEPT_CR0_MASK;
  938. if (is_nested(svm)) {
  939. struct vmcb *hsave = svm->nested.hsave;
  940. hsave->control.intercept_cr_read |= INTERCEPT_CR0_MASK;
  941. hsave->control.intercept_cr_write |= INTERCEPT_CR0_MASK;
  942. }
  943. }
  944. }
  945. static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
  946. {
  947. struct vcpu_svm *svm = to_svm(vcpu);
  948. if (is_nested(svm)) {
  949. /*
  950. * We are here because we run in nested mode, the host kvm
  951. * intercepts cr0 writes but the l1 hypervisor does not.
  952. * But the L1 hypervisor may intercept selective cr0 writes.
  953. * This needs to be checked here.
  954. */
  955. unsigned long old, new;
  956. /* Remove bits that would trigger a real cr0 write intercept */
  957. old = vcpu->arch.cr0 & SVM_CR0_SELECTIVE_MASK;
  958. new = cr0 & SVM_CR0_SELECTIVE_MASK;
  959. if (old == new) {
  960. /* cr0 write with ts and mp unchanged */
  961. svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE;
  962. if (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE)
  963. return;
  964. }
  965. }
  966. #ifdef CONFIG_X86_64
  967. if (vcpu->arch.efer & EFER_LME) {
  968. if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
  969. vcpu->arch.efer |= EFER_LMA;
  970. svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
  971. }
  972. if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
  973. vcpu->arch.efer &= ~EFER_LMA;
  974. svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
  975. }
  976. }
  977. #endif
  978. vcpu->arch.cr0 = cr0;
  979. if (!npt_enabled)
  980. cr0 |= X86_CR0_PG | X86_CR0_WP;
  981. if (!vcpu->fpu_active)
  982. cr0 |= X86_CR0_TS;
  983. /*
  984. * re-enable caching here because the QEMU bios
  985. * does not do it - this results in some delay at
  986. * reboot
  987. */
  988. cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
  989. svm->vmcb->save.cr0 = cr0;
  990. update_cr0_intercept(svm);
  991. }
  992. static void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
  993. {
  994. unsigned long host_cr4_mce = read_cr4() & X86_CR4_MCE;
  995. unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4;
  996. if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
  997. force_new_asid(vcpu);
  998. vcpu->arch.cr4 = cr4;
  999. if (!npt_enabled)
  1000. cr4 |= X86_CR4_PAE;
  1001. cr4 |= host_cr4_mce;
  1002. to_svm(vcpu)->vmcb->save.cr4 = cr4;
  1003. }
  1004. static void svm_set_segment(struct kvm_vcpu *vcpu,
  1005. struct kvm_segment *var, int seg)
  1006. {
  1007. struct vcpu_svm *svm = to_svm(vcpu);
  1008. struct vmcb_seg *s = svm_seg(vcpu, seg);
  1009. s->base = var->base;
  1010. s->limit = var->limit;
  1011. s->selector = var->selector;
  1012. if (var->unusable)
  1013. s->attrib = 0;
  1014. else {
  1015. s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
  1016. s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
  1017. s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
  1018. s->attrib |= (var->present & 1) << SVM_SELECTOR_P_SHIFT;
  1019. s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
  1020. s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
  1021. s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
  1022. s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
  1023. }
  1024. if (seg == VCPU_SREG_CS)
  1025. svm->vmcb->save.cpl
  1026. = (svm->vmcb->save.cs.attrib
  1027. >> SVM_SELECTOR_DPL_SHIFT) & 3;
  1028. }
  1029. static void update_db_intercept(struct kvm_vcpu *vcpu)
  1030. {
  1031. struct vcpu_svm *svm = to_svm(vcpu);
  1032. svm->vmcb->control.intercept_exceptions &=
  1033. ~((1 << DB_VECTOR) | (1 << BP_VECTOR));
  1034. if (svm->nmi_singlestep)
  1035. svm->vmcb->control.intercept_exceptions |= (1 << DB_VECTOR);
  1036. if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
  1037. if (vcpu->guest_debug &
  1038. (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
  1039. svm->vmcb->control.intercept_exceptions |=
  1040. 1 << DB_VECTOR;
  1041. if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
  1042. svm->vmcb->control.intercept_exceptions |=
  1043. 1 << BP_VECTOR;
  1044. } else
  1045. vcpu->guest_debug = 0;
  1046. }
  1047. static void svm_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg)
  1048. {
  1049. struct vcpu_svm *svm = to_svm(vcpu);
  1050. if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
  1051. svm->vmcb->save.dr7 = dbg->arch.debugreg[7];
  1052. else
  1053. svm->vmcb->save.dr7 = vcpu->arch.dr7;
  1054. update_db_intercept(vcpu);
  1055. }
  1056. static void load_host_msrs(struct kvm_vcpu *vcpu)
  1057. {
  1058. #ifdef CONFIG_X86_64
  1059. wrmsrl(MSR_GS_BASE, to_svm(vcpu)->host_gs_base);
  1060. #endif
  1061. }
  1062. static void save_host_msrs(struct kvm_vcpu *vcpu)
  1063. {
  1064. #ifdef CONFIG_X86_64
  1065. rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host_gs_base);
  1066. #endif
  1067. }
  1068. static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
  1069. {
  1070. if (sd->next_asid > sd->max_asid) {
  1071. ++sd->asid_generation;
  1072. sd->next_asid = 1;
  1073. svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
  1074. }
  1075. svm->asid_generation = sd->asid_generation;
  1076. svm->vmcb->control.asid = sd->next_asid++;
  1077. }
  1078. static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
  1079. {
  1080. struct vcpu_svm *svm = to_svm(vcpu);
  1081. svm->vmcb->save.dr7 = value;
  1082. }
  1083. static int pf_interception(struct vcpu_svm *svm)
  1084. {
  1085. u64 fault_address;
  1086. u32 error_code;
  1087. fault_address = svm->vmcb->control.exit_info_2;
  1088. error_code = svm->vmcb->control.exit_info_1;
  1089. trace_kvm_page_fault(fault_address, error_code);
  1090. if (!npt_enabled && kvm_event_needs_reinjection(&svm->vcpu))
  1091. kvm_mmu_unprotect_page_virt(&svm->vcpu, fault_address);
  1092. return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code);
  1093. }
  1094. static int db_interception(struct vcpu_svm *svm)
  1095. {
  1096. struct kvm_run *kvm_run = svm->vcpu.run;
  1097. if (!(svm->vcpu.guest_debug &
  1098. (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
  1099. !svm->nmi_singlestep) {
  1100. kvm_queue_exception(&svm->vcpu, DB_VECTOR);
  1101. return 1;
  1102. }
  1103. if (svm->nmi_singlestep) {
  1104. svm->nmi_singlestep = false;
  1105. if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP))
  1106. svm->vmcb->save.rflags &=
  1107. ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
  1108. update_db_intercept(&svm->vcpu);
  1109. }
  1110. if (svm->vcpu.guest_debug &
  1111. (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
  1112. kvm_run->exit_reason = KVM_EXIT_DEBUG;
  1113. kvm_run->debug.arch.pc =
  1114. svm->vmcb->save.cs.base + svm->vmcb->save.rip;
  1115. kvm_run->debug.arch.exception = DB_VECTOR;
  1116. return 0;
  1117. }
  1118. return 1;
  1119. }
  1120. static int bp_interception(struct vcpu_svm *svm)
  1121. {
  1122. struct kvm_run *kvm_run = svm->vcpu.run;
  1123. kvm_run->exit_reason = KVM_EXIT_DEBUG;
  1124. kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
  1125. kvm_run->debug.arch.exception = BP_VECTOR;
  1126. return 0;
  1127. }
  1128. static int ud_interception(struct vcpu_svm *svm)
  1129. {
  1130. int er;
  1131. er = emulate_instruction(&svm->vcpu, 0, 0, EMULTYPE_TRAP_UD);
  1132. if (er != EMULATE_DONE)
  1133. kvm_queue_exception(&svm->vcpu, UD_VECTOR);
  1134. return 1;
  1135. }
  1136. static void svm_fpu_activate(struct kvm_vcpu *vcpu)
  1137. {
  1138. struct vcpu_svm *svm = to_svm(vcpu);
  1139. u32 excp;
  1140. if (is_nested(svm)) {
  1141. u32 h_excp, n_excp;
  1142. h_excp = svm->nested.hsave->control.intercept_exceptions;
  1143. n_excp = svm->nested.intercept_exceptions;
  1144. h_excp &= ~(1 << NM_VECTOR);
  1145. excp = h_excp | n_excp;
  1146. } else {
  1147. excp = svm->vmcb->control.intercept_exceptions;
  1148. excp &= ~(1 << NM_VECTOR);
  1149. }
  1150. svm->vmcb->control.intercept_exceptions = excp;
  1151. svm->vcpu.fpu_active = 1;
  1152. update_cr0_intercept(svm);
  1153. }
  1154. static int nm_interception(struct vcpu_svm *svm)
  1155. {
  1156. svm_fpu_activate(&svm->vcpu);
  1157. return 1;
  1158. }
  1159. static bool is_erratum_383(void)
  1160. {
  1161. int err, i;
  1162. u64 value;
  1163. if (!erratum_383_found)
  1164. return false;
  1165. value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err);
  1166. if (err)
  1167. return false;
  1168. /* Bit 62 may or may not be set for this mce */
  1169. value &= ~(1ULL << 62);
  1170. if (value != 0xb600000000010015ULL)
  1171. return false;
  1172. /* Clear MCi_STATUS registers */
  1173. for (i = 0; i < 6; ++i)
  1174. native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0);
  1175. value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err);
  1176. if (!err) {
  1177. u32 low, high;
  1178. value &= ~(1ULL << 2);
  1179. low = lower_32_bits(value);
  1180. high = upper_32_bits(value);
  1181. native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high);
  1182. }
  1183. /* Flush tlb to evict multi-match entries */
  1184. __flush_tlb_all();
  1185. return true;
  1186. }
  1187. static void svm_handle_mce(struct vcpu_svm *svm)
  1188. {
  1189. if (is_erratum_383()) {
  1190. /*
  1191. * Erratum 383 triggered. Guest state is corrupt so kill the
  1192. * guest.
  1193. */
  1194. pr_err("KVM: Guest triggered AMD Erratum 383\n");
  1195. set_bit(KVM_REQ_TRIPLE_FAULT, &svm->vcpu.requests);
  1196. return;
  1197. }
  1198. /*
  1199. * On an #MC intercept the MCE handler is not called automatically in
  1200. * the host. So do it by hand here.
  1201. */
  1202. asm volatile (
  1203. "int $0x12\n");
  1204. /* not sure if we ever come back to this point */
  1205. return;
  1206. }
  1207. static int mc_interception(struct vcpu_svm *svm)
  1208. {
  1209. return 1;
  1210. }
  1211. static int shutdown_interception(struct vcpu_svm *svm)
  1212. {
  1213. struct kvm_run *kvm_run = svm->vcpu.run;
  1214. /*
  1215. * VMCB is undefined after a SHUTDOWN intercept
  1216. * so reinitialize it.
  1217. */
  1218. clear_page(svm->vmcb);
  1219. init_vmcb(svm);
  1220. kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
  1221. return 0;
  1222. }
  1223. static int io_interception(struct vcpu_svm *svm)
  1224. {
  1225. struct kvm_vcpu *vcpu = &svm->vcpu;
  1226. u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
  1227. int size, in, string;
  1228. unsigned port;
  1229. ++svm->vcpu.stat.io_exits;
  1230. string = (io_info & SVM_IOIO_STR_MASK) != 0;
  1231. in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
  1232. if (string || in)
  1233. return !(emulate_instruction(vcpu, 0, 0, 0) == EMULATE_DO_MMIO);
  1234. port = io_info >> 16;
  1235. size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
  1236. svm->next_rip = svm->vmcb->control.exit_info_2;
  1237. skip_emulated_instruction(&svm->vcpu);
  1238. return kvm_fast_pio_out(vcpu, size, port);
  1239. }
  1240. static int nmi_interception(struct vcpu_svm *svm)
  1241. {
  1242. return 1;
  1243. }
  1244. static int intr_interception(struct vcpu_svm *svm)
  1245. {
  1246. ++svm->vcpu.stat.irq_exits;
  1247. return 1;
  1248. }
  1249. static int nop_on_interception(struct vcpu_svm *svm)
  1250. {
  1251. return 1;
  1252. }
  1253. static int halt_interception(struct vcpu_svm *svm)
  1254. {
  1255. svm->next_rip = kvm_rip_read(&svm->vcpu) + 1;
  1256. skip_emulated_instruction(&svm->vcpu);
  1257. return kvm_emulate_halt(&svm->vcpu);
  1258. }
  1259. static int vmmcall_interception(struct vcpu_svm *svm)
  1260. {
  1261. svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
  1262. skip_emulated_instruction(&svm->vcpu);
  1263. kvm_emulate_hypercall(&svm->vcpu);
  1264. return 1;
  1265. }
  1266. static int nested_svm_check_permissions(struct vcpu_svm *svm)
  1267. {
  1268. if (!(svm->vcpu.arch.efer & EFER_SVME)
  1269. || !is_paging(&svm->vcpu)) {
  1270. kvm_queue_exception(&svm->vcpu, UD_VECTOR);
  1271. return 1;
  1272. }
  1273. if (svm->vmcb->save.cpl) {
  1274. kvm_inject_gp(&svm->vcpu, 0);
  1275. return 1;
  1276. }
  1277. return 0;
  1278. }
  1279. static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
  1280. bool has_error_code, u32 error_code)
  1281. {
  1282. int vmexit;
  1283. if (!is_nested(svm))
  1284. return 0;
  1285. svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr;
  1286. svm->vmcb->control.exit_code_hi = 0;
  1287. svm->vmcb->control.exit_info_1 = error_code;
  1288. svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2;
  1289. vmexit = nested_svm_intercept(svm);
  1290. if (vmexit == NESTED_EXIT_DONE)
  1291. svm->nested.exit_required = true;
  1292. return vmexit;
  1293. }
  1294. /* This function returns true if it is save to enable the irq window */
  1295. static inline bool nested_svm_intr(struct vcpu_svm *svm)
  1296. {
  1297. if (!is_nested(svm))
  1298. return true;
  1299. if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
  1300. return true;
  1301. if (!(svm->vcpu.arch.hflags & HF_HIF_MASK))
  1302. return false;
  1303. svm->vmcb->control.exit_code = SVM_EXIT_INTR;
  1304. svm->vmcb->control.exit_info_1 = 0;
  1305. svm->vmcb->control.exit_info_2 = 0;
  1306. if (svm->nested.intercept & 1ULL) {
  1307. /*
  1308. * The #vmexit can't be emulated here directly because this
  1309. * code path runs with irqs and preemtion disabled. A
  1310. * #vmexit emulation might sleep. Only signal request for
  1311. * the #vmexit here.
  1312. */
  1313. svm->nested.exit_required = true;
  1314. trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip);
  1315. return false;
  1316. }
  1317. return true;
  1318. }
  1319. /* This function returns true if it is save to enable the nmi window */
  1320. static inline bool nested_svm_nmi(struct vcpu_svm *svm)
  1321. {
  1322. if (!is_nested(svm))
  1323. return true;
  1324. if (!(svm->nested.intercept & (1ULL << INTERCEPT_NMI)))
  1325. return true;
  1326. svm->vmcb->control.exit_code = SVM_EXIT_NMI;
  1327. svm->nested.exit_required = true;
  1328. return false;
  1329. }
  1330. static void *nested_svm_map(struct vcpu_svm *svm, u64 gpa, struct page **_page)
  1331. {
  1332. struct page *page;
  1333. might_sleep();
  1334. page = gfn_to_page(svm->vcpu.kvm, gpa >> PAGE_SHIFT);
  1335. if (is_error_page(page))
  1336. goto error;
  1337. *_page = page;
  1338. return kmap(page);
  1339. error:
  1340. kvm_release_page_clean(page);
  1341. kvm_inject_gp(&svm->vcpu, 0);
  1342. return NULL;
  1343. }
  1344. static void nested_svm_unmap(struct page *page)
  1345. {
  1346. kunmap(page);
  1347. kvm_release_page_dirty(page);
  1348. }
  1349. static int nested_svm_intercept_ioio(struct vcpu_svm *svm)
  1350. {
  1351. unsigned port;
  1352. u8 val, bit;
  1353. u64 gpa;
  1354. if (!(svm->nested.intercept & (1ULL << INTERCEPT_IOIO_PROT)))
  1355. return NESTED_EXIT_HOST;
  1356. port = svm->vmcb->control.exit_info_1 >> 16;
  1357. gpa = svm->nested.vmcb_iopm + (port / 8);
  1358. bit = port % 8;
  1359. val = 0;
  1360. if (kvm_read_guest(svm->vcpu.kvm, gpa, &val, 1))
  1361. val &= (1 << bit);
  1362. return val ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
  1363. }
  1364. static int nested_svm_exit_handled_msr(struct vcpu_svm *svm)
  1365. {
  1366. u32 offset, msr, value;
  1367. int write, mask;
  1368. if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
  1369. return NESTED_EXIT_HOST;
  1370. msr = svm->vcpu.arch.regs[VCPU_REGS_RCX];
  1371. offset = svm_msrpm_offset(msr);
  1372. write = svm->vmcb->control.exit_info_1 & 1;
  1373. mask = 1 << ((2 * (msr & 0xf)) + write);
  1374. if (offset == MSR_INVALID)
  1375. return NESTED_EXIT_DONE;
  1376. /* Offset is in 32 bit units but need in 8 bit units */
  1377. offset *= 4;
  1378. if (kvm_read_guest(svm->vcpu.kvm, svm->nested.vmcb_msrpm + offset, &value, 4))
  1379. return NESTED_EXIT_DONE;
  1380. return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
  1381. }
  1382. static int nested_svm_exit_special(struct vcpu_svm *svm)
  1383. {
  1384. u32 exit_code = svm->vmcb->control.exit_code;
  1385. switch (exit_code) {
  1386. case SVM_EXIT_INTR:
  1387. case SVM_EXIT_NMI:
  1388. case SVM_EXIT_EXCP_BASE + MC_VECTOR:
  1389. return NESTED_EXIT_HOST;
  1390. case SVM_EXIT_NPF:
  1391. /* For now we are always handling NPFs when using them */
  1392. if (npt_enabled)
  1393. return NESTED_EXIT_HOST;
  1394. break;
  1395. case SVM_EXIT_EXCP_BASE + PF_VECTOR:
  1396. /* When we're shadowing, trap PFs */
  1397. if (!npt_enabled)
  1398. return NESTED_EXIT_HOST;
  1399. break;
  1400. case SVM_EXIT_EXCP_BASE + NM_VECTOR:
  1401. nm_interception(svm);
  1402. break;
  1403. default:
  1404. break;
  1405. }
  1406. return NESTED_EXIT_CONTINUE;
  1407. }
  1408. /*
  1409. * If this function returns true, this #vmexit was already handled
  1410. */
  1411. static int nested_svm_intercept(struct vcpu_svm *svm)
  1412. {
  1413. u32 exit_code = svm->vmcb->control.exit_code;
  1414. int vmexit = NESTED_EXIT_HOST;
  1415. switch (exit_code) {
  1416. case SVM_EXIT_MSR:
  1417. vmexit = nested_svm_exit_handled_msr(svm);
  1418. break;
  1419. case SVM_EXIT_IOIO:
  1420. vmexit = nested_svm_intercept_ioio(svm);
  1421. break;
  1422. case SVM_EXIT_READ_CR0 ... SVM_EXIT_READ_CR8: {
  1423. u32 cr_bits = 1 << (exit_code - SVM_EXIT_READ_CR0);
  1424. if (svm->nested.intercept_cr_read & cr_bits)
  1425. vmexit = NESTED_EXIT_DONE;
  1426. break;
  1427. }
  1428. case SVM_EXIT_WRITE_CR0 ... SVM_EXIT_WRITE_CR8: {
  1429. u32 cr_bits = 1 << (exit_code - SVM_EXIT_WRITE_CR0);
  1430. if (svm->nested.intercept_cr_write & cr_bits)
  1431. vmexit = NESTED_EXIT_DONE;
  1432. break;
  1433. }
  1434. case SVM_EXIT_READ_DR0 ... SVM_EXIT_READ_DR7: {
  1435. u32 dr_bits = 1 << (exit_code - SVM_EXIT_READ_DR0);
  1436. if (svm->nested.intercept_dr_read & dr_bits)
  1437. vmexit = NESTED_EXIT_DONE;
  1438. break;
  1439. }
  1440. case SVM_EXIT_WRITE_DR0 ... SVM_EXIT_WRITE_DR7: {
  1441. u32 dr_bits = 1 << (exit_code - SVM_EXIT_WRITE_DR0);
  1442. if (svm->nested.intercept_dr_write & dr_bits)
  1443. vmexit = NESTED_EXIT_DONE;
  1444. break;
  1445. }
  1446. case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
  1447. u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE);
  1448. if (svm->nested.intercept_exceptions & excp_bits)
  1449. vmexit = NESTED_EXIT_DONE;
  1450. break;
  1451. }
  1452. case SVM_EXIT_ERR: {
  1453. vmexit = NESTED_EXIT_DONE;
  1454. break;
  1455. }
  1456. default: {
  1457. u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR);
  1458. if (svm->nested.intercept & exit_bits)
  1459. vmexit = NESTED_EXIT_DONE;
  1460. }
  1461. }
  1462. return vmexit;
  1463. }
  1464. static int nested_svm_exit_handled(struct vcpu_svm *svm)
  1465. {
  1466. int vmexit;
  1467. vmexit = nested_svm_intercept(svm);
  1468. if (vmexit == NESTED_EXIT_DONE)
  1469. nested_svm_vmexit(svm);
  1470. return vmexit;
  1471. }
  1472. static inline void copy_vmcb_control_area(struct vmcb *dst_vmcb, struct vmcb *from_vmcb)
  1473. {
  1474. struct vmcb_control_area *dst = &dst_vmcb->control;
  1475. struct vmcb_control_area *from = &from_vmcb->control;
  1476. dst->intercept_cr_read = from->intercept_cr_read;
  1477. dst->intercept_cr_write = from->intercept_cr_write;
  1478. dst->intercept_dr_read = from->intercept_dr_read;
  1479. dst->intercept_dr_write = from->intercept_dr_write;
  1480. dst->intercept_exceptions = from->intercept_exceptions;
  1481. dst->intercept = from->intercept;
  1482. dst->iopm_base_pa = from->iopm_base_pa;
  1483. dst->msrpm_base_pa = from->msrpm_base_pa;
  1484. dst->tsc_offset = from->tsc_offset;
  1485. dst->asid = from->asid;
  1486. dst->tlb_ctl = from->tlb_ctl;
  1487. dst->int_ctl = from->int_ctl;
  1488. dst->int_vector = from->int_vector;
  1489. dst->int_state = from->int_state;
  1490. dst->exit_code = from->exit_code;
  1491. dst->exit_code_hi = from->exit_code_hi;
  1492. dst->exit_info_1 = from->exit_info_1;
  1493. dst->exit_info_2 = from->exit_info_2;
  1494. dst->exit_int_info = from->exit_int_info;
  1495. dst->exit_int_info_err = from->exit_int_info_err;
  1496. dst->nested_ctl = from->nested_ctl;
  1497. dst->event_inj = from->event_inj;
  1498. dst->event_inj_err = from->event_inj_err;
  1499. dst->nested_cr3 = from->nested_cr3;
  1500. dst->lbr_ctl = from->lbr_ctl;
  1501. }
  1502. static int nested_svm_vmexit(struct vcpu_svm *svm)
  1503. {
  1504. struct vmcb *nested_vmcb;
  1505. struct vmcb *hsave = svm->nested.hsave;
  1506. struct vmcb *vmcb = svm->vmcb;
  1507. struct page *page;
  1508. trace_kvm_nested_vmexit_inject(vmcb->control.exit_code,
  1509. vmcb->control.exit_info_1,
  1510. vmcb->control.exit_info_2,
  1511. vmcb->control.exit_int_info,
  1512. vmcb->control.exit_int_info_err);
  1513. nested_vmcb = nested_svm_map(svm, svm->nested.vmcb, &page);
  1514. if (!nested_vmcb)
  1515. return 1;
  1516. /* Exit nested SVM mode */
  1517. svm->nested.vmcb = 0;
  1518. /* Give the current vmcb to the guest */
  1519. disable_gif(svm);
  1520. nested_vmcb->save.es = vmcb->save.es;
  1521. nested_vmcb->save.cs = vmcb->save.cs;
  1522. nested_vmcb->save.ss = vmcb->save.ss;
  1523. nested_vmcb->save.ds = vmcb->save.ds;
  1524. nested_vmcb->save.gdtr = vmcb->save.gdtr;
  1525. nested_vmcb->save.idtr = vmcb->save.idtr;
  1526. nested_vmcb->save.cr0 = kvm_read_cr0(&svm->vcpu);
  1527. nested_vmcb->save.cr3 = svm->vcpu.arch.cr3;
  1528. nested_vmcb->save.cr2 = vmcb->save.cr2;
  1529. nested_vmcb->save.cr4 = svm->vcpu.arch.cr4;
  1530. nested_vmcb->save.rflags = vmcb->save.rflags;
  1531. nested_vmcb->save.rip = vmcb->save.rip;
  1532. nested_vmcb->save.rsp = vmcb->save.rsp;
  1533. nested_vmcb->save.rax = vmcb->save.rax;
  1534. nested_vmcb->save.dr7 = vmcb->save.dr7;
  1535. nested_vmcb->save.dr6 = vmcb->save.dr6;
  1536. nested_vmcb->save.cpl = vmcb->save.cpl;
  1537. nested_vmcb->control.int_ctl = vmcb->control.int_ctl;
  1538. nested_vmcb->control.int_vector = vmcb->control.int_vector;
  1539. nested_vmcb->control.int_state = vmcb->control.int_state;
  1540. nested_vmcb->control.exit_code = vmcb->control.exit_code;
  1541. nested_vmcb->control.exit_code_hi = vmcb->control.exit_code_hi;
  1542. nested_vmcb->control.exit_info_1 = vmcb->control.exit_info_1;
  1543. nested_vmcb->control.exit_info_2 = vmcb->control.exit_info_2;
  1544. nested_vmcb->control.exit_int_info = vmcb->control.exit_int_info;
  1545. nested_vmcb->control.exit_int_info_err = vmcb->control.exit_int_info_err;
  1546. /*
  1547. * If we emulate a VMRUN/#VMEXIT in the same host #vmexit cycle we have
  1548. * to make sure that we do not lose injected events. So check event_inj
  1549. * here and copy it to exit_int_info if it is valid.
  1550. * Exit_int_info and event_inj can't be both valid because the case
  1551. * below only happens on a VMRUN instruction intercept which has
  1552. * no valid exit_int_info set.
  1553. */
  1554. if (vmcb->control.event_inj & SVM_EVTINJ_VALID) {
  1555. struct vmcb_control_area *nc = &nested_vmcb->control;
  1556. nc->exit_int_info = vmcb->control.event_inj;
  1557. nc->exit_int_info_err = vmcb->control.event_inj_err;
  1558. }
  1559. nested_vmcb->control.tlb_ctl = 0;
  1560. nested_vmcb->control.event_inj = 0;
  1561. nested_vmcb->control.event_inj_err = 0;
  1562. /* We always set V_INTR_MASKING and remember the old value in hflags */
  1563. if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
  1564. nested_vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;
  1565. /* Restore the original control entries */
  1566. copy_vmcb_control_area(vmcb, hsave);
  1567. kvm_clear_exception_queue(&svm->vcpu);
  1568. kvm_clear_interrupt_queue(&svm->vcpu);
  1569. /* Restore selected save entries */
  1570. svm->vmcb->save.es = hsave->save.es;
  1571. svm->vmcb->save.cs = hsave->save.cs;
  1572. svm->vmcb->save.ss = hsave->save.ss;
  1573. svm->vmcb->save.ds = hsave->save.ds;
  1574. svm->vmcb->save.gdtr = hsave->save.gdtr;
  1575. svm->vmcb->save.idtr = hsave->save.idtr;
  1576. svm->vmcb->save.rflags = hsave->save.rflags;
  1577. svm_set_efer(&svm->vcpu, hsave->save.efer);
  1578. svm_set_cr0(&svm->vcpu, hsave->save.cr0 | X86_CR0_PE);
  1579. svm_set_cr4(&svm->vcpu, hsave->save.cr4);
  1580. if (npt_enabled) {
  1581. svm->vmcb->save.cr3 = hsave->save.cr3;
  1582. svm->vcpu.arch.cr3 = hsave->save.cr3;
  1583. } else {
  1584. kvm_set_cr3(&svm->vcpu, hsave->save.cr3);
  1585. }
  1586. kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, hsave->save.rax);
  1587. kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, hsave->save.rsp);
  1588. kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, hsave->save.rip);
  1589. svm->vmcb->save.dr7 = 0;
  1590. svm->vmcb->save.cpl = 0;
  1591. svm->vmcb->control.exit_int_info = 0;
  1592. nested_svm_unmap(page);
  1593. kvm_mmu_reset_context(&svm->vcpu);
  1594. kvm_mmu_load(&svm->vcpu);
  1595. return 0;
  1596. }
  1597. static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm)
  1598. {
  1599. /*
  1600. * This function merges the msr permission bitmaps of kvm and the
  1601. * nested vmcb. It is omptimized in that it only merges the parts where
  1602. * the kvm msr permission bitmap may contain zero bits
  1603. */
  1604. int i;
  1605. if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
  1606. return true;
  1607. for (i = 0; i < MSRPM_OFFSETS; i++) {
  1608. u32 value, p;
  1609. u64 offset;
  1610. if (msrpm_offsets[i] == 0xffffffff)
  1611. break;
  1612. p = msrpm_offsets[i];
  1613. offset = svm->nested.vmcb_msrpm + (p * 4);
  1614. if (kvm_read_guest(svm->vcpu.kvm, offset, &value, 4))
  1615. return false;
  1616. svm->nested.msrpm[p] = svm->msrpm[p] | value;
  1617. }
  1618. svm->vmcb->control.msrpm_base_pa = __pa(svm->nested.msrpm);
  1619. return true;
  1620. }
  1621. static bool nested_svm_vmrun(struct vcpu_svm *svm)
  1622. {
  1623. struct vmcb *nested_vmcb;
  1624. struct vmcb *hsave = svm->nested.hsave;
  1625. struct vmcb *vmcb = svm->vmcb;
  1626. struct page *page;
  1627. u64 vmcb_gpa;
  1628. vmcb_gpa = svm->vmcb->save.rax;
  1629. nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
  1630. if (!nested_vmcb)
  1631. return false;
  1632. trace_kvm_nested_vmrun(svm->vmcb->save.rip - 3, vmcb_gpa,
  1633. nested_vmcb->save.rip,
  1634. nested_vmcb->control.int_ctl,
  1635. nested_vmcb->control.event_inj,
  1636. nested_vmcb->control.nested_ctl);
  1637. trace_kvm_nested_intercepts(nested_vmcb->control.intercept_cr_read,
  1638. nested_vmcb->control.intercept_cr_write,
  1639. nested_vmcb->control.intercept_exceptions,
  1640. nested_vmcb->control.intercept);
  1641. /* Clear internal status */
  1642. kvm_clear_exception_queue(&svm->vcpu);
  1643. kvm_clear_interrupt_queue(&svm->vcpu);
  1644. /*
  1645. * Save the old vmcb, so we don't need to pick what we save, but can
  1646. * restore everything when a VMEXIT occurs
  1647. */
  1648. hsave->save.es = vmcb->save.es;
  1649. hsave->save.cs = vmcb->save.cs;
  1650. hsave->save.ss = vmcb->save.ss;
  1651. hsave->save.ds = vmcb->save.ds;
  1652. hsave->save.gdtr = vmcb->save.gdtr;
  1653. hsave->save.idtr = vmcb->save.idtr;
  1654. hsave->save.efer = svm->vcpu.arch.efer;
  1655. hsave->save.cr0 = kvm_read_cr0(&svm->vcpu);
  1656. hsave->save.cr4 = svm->vcpu.arch.cr4;
  1657. hsave->save.rflags = vmcb->save.rflags;
  1658. hsave->save.rip = svm->next_rip;
  1659. hsave->save.rsp = vmcb->save.rsp;
  1660. hsave->save.rax = vmcb->save.rax;
  1661. if (npt_enabled)
  1662. hsave->save.cr3 = vmcb->save.cr3;
  1663. else
  1664. hsave->save.cr3 = svm->vcpu.arch.cr3;
  1665. copy_vmcb_control_area(hsave, vmcb);
  1666. if (svm->vmcb->save.rflags & X86_EFLAGS_IF)
  1667. svm->vcpu.arch.hflags |= HF_HIF_MASK;
  1668. else
  1669. svm->vcpu.arch.hflags &= ~HF_HIF_MASK;
  1670. /* Load the nested guest state */
  1671. svm->vmcb->save.es = nested_vmcb->save.es;
  1672. svm->vmcb->save.cs = nested_vmcb->save.cs;
  1673. svm->vmcb->save.ss = nested_vmcb->save.ss;
  1674. svm->vmcb->save.ds = nested_vmcb->save.ds;
  1675. svm->vmcb->save.gdtr = nested_vmcb->save.gdtr;
  1676. svm->vmcb->save.idtr = nested_vmcb->save.idtr;
  1677. svm->vmcb->save.rflags = nested_vmcb->save.rflags;
  1678. svm_set_efer(&svm->vcpu, nested_vmcb->save.efer);
  1679. svm_set_cr0(&svm->vcpu, nested_vmcb->save.cr0);
  1680. svm_set_cr4(&svm->vcpu, nested_vmcb->save.cr4);
  1681. if (npt_enabled) {
  1682. svm->vmcb->save.cr3 = nested_vmcb->save.cr3;
  1683. svm->vcpu.arch.cr3 = nested_vmcb->save.cr3;
  1684. } else
  1685. kvm_set_cr3(&svm->vcpu, nested_vmcb->save.cr3);
  1686. /* Guest paging mode is active - reset mmu */
  1687. kvm_mmu_reset_context(&svm->vcpu);
  1688. svm->vmcb->save.cr2 = svm->vcpu.arch.cr2 = nested_vmcb->save.cr2;
  1689. kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, nested_vmcb->save.rax);
  1690. kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, nested_vmcb->save.rsp);
  1691. kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, nested_vmcb->save.rip);
  1692. /* In case we don't even reach vcpu_run, the fields are not updated */
  1693. svm->vmcb->save.rax = nested_vmcb->save.rax;
  1694. svm->vmcb->save.rsp = nested_vmcb->save.rsp;
  1695. svm->vmcb->save.rip = nested_vmcb->save.rip;
  1696. svm->vmcb->save.dr7 = nested_vmcb->save.dr7;
  1697. svm->vmcb->save.dr6 = nested_vmcb->save.dr6;
  1698. svm->vmcb->save.cpl = nested_vmcb->save.cpl;
  1699. svm->nested.vmcb_msrpm = nested_vmcb->control.msrpm_base_pa & ~0x0fffULL;
  1700. svm->nested.vmcb_iopm = nested_vmcb->control.iopm_base_pa & ~0x0fffULL;
  1701. /* cache intercepts */
  1702. svm->nested.intercept_cr_read = nested_vmcb->control.intercept_cr_read;
  1703. svm->nested.intercept_cr_write = nested_vmcb->control.intercept_cr_write;
  1704. svm->nested.intercept_dr_read = nested_vmcb->control.intercept_dr_read;
  1705. svm->nested.intercept_dr_write = nested_vmcb->control.intercept_dr_write;
  1706. svm->nested.intercept_exceptions = nested_vmcb->control.intercept_exceptions;
  1707. svm->nested.intercept = nested_vmcb->control.intercept;
  1708. force_new_asid(&svm->vcpu);
  1709. svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK;
  1710. if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK)
  1711. svm->vcpu.arch.hflags |= HF_VINTR_MASK;
  1712. else
  1713. svm->vcpu.arch.hflags &= ~HF_VINTR_MASK;
  1714. if (svm->vcpu.arch.hflags & HF_VINTR_MASK) {
  1715. /* We only want the cr8 intercept bits of the guest */
  1716. svm->vmcb->control.intercept_cr_read &= ~INTERCEPT_CR8_MASK;
  1717. svm->vmcb->control.intercept_cr_write &= ~INTERCEPT_CR8_MASK;
  1718. }
  1719. /* We don't want to see VMMCALLs from a nested guest */
  1720. svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_VMMCALL);
  1721. /*
  1722. * We don't want a nested guest to be more powerful than the guest, so
  1723. * all intercepts are ORed
  1724. */
  1725. svm->vmcb->control.intercept_cr_read |=
  1726. nested_vmcb->control.intercept_cr_read;
  1727. svm->vmcb->control.intercept_cr_write |=
  1728. nested_vmcb->control.intercept_cr_write;
  1729. svm->vmcb->control.intercept_dr_read |=
  1730. nested_vmcb->control.intercept_dr_read;
  1731. svm->vmcb->control.intercept_dr_write |=
  1732. nested_vmcb->control.intercept_dr_write;
  1733. svm->vmcb->control.intercept_exceptions |=
  1734. nested_vmcb->control.intercept_exceptions;
  1735. svm->vmcb->control.intercept |= nested_vmcb->control.intercept;
  1736. svm->vmcb->control.lbr_ctl = nested_vmcb->control.lbr_ctl;
  1737. svm->vmcb->control.int_vector = nested_vmcb->control.int_vector;
  1738. svm->vmcb->control.int_state = nested_vmcb->control.int_state;
  1739. svm->vmcb->control.tsc_offset += nested_vmcb->control.tsc_offset;
  1740. svm->vmcb->control.event_inj = nested_vmcb->control.event_inj;
  1741. svm->vmcb->control.event_inj_err = nested_vmcb->control.event_inj_err;
  1742. nested_svm_unmap(page);
  1743. /* nested_vmcb is our indicator if nested SVM is activated */
  1744. svm->nested.vmcb = vmcb_gpa;
  1745. enable_gif(svm);
  1746. return true;
  1747. }
  1748. static void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
  1749. {
  1750. to_vmcb->save.fs = from_vmcb->save.fs;
  1751. to_vmcb->save.gs = from_vmcb->save.gs;
  1752. to_vmcb->save.tr = from_vmcb->save.tr;
  1753. to_vmcb->save.ldtr = from_vmcb->save.ldtr;
  1754. to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base;
  1755. to_vmcb->save.star = from_vmcb->save.star;
  1756. to_vmcb->save.lstar = from_vmcb->save.lstar;
  1757. to_vmcb->save.cstar = from_vmcb->save.cstar;
  1758. to_vmcb->save.sfmask = from_vmcb->save.sfmask;
  1759. to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs;
  1760. to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp;
  1761. to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip;
  1762. }
  1763. static int vmload_interception(struct vcpu_svm *svm)
  1764. {
  1765. struct vmcb *nested_vmcb;
  1766. struct page *page;
  1767. if (nested_svm_check_permissions(svm))
  1768. return 1;
  1769. svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
  1770. skip_emulated_instruction(&svm->vcpu);
  1771. nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
  1772. if (!nested_vmcb)
  1773. return 1;
  1774. nested_svm_vmloadsave(nested_vmcb, svm->vmcb);
  1775. nested_svm_unmap(page);
  1776. return 1;
  1777. }
  1778. static int vmsave_interception(struct vcpu_svm *svm)
  1779. {
  1780. struct vmcb *nested_vmcb;
  1781. struct page *page;
  1782. if (nested_svm_check_permissions(svm))
  1783. return 1;
  1784. svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
  1785. skip_emulated_instruction(&svm->vcpu);
  1786. nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
  1787. if (!nested_vmcb)
  1788. return 1;
  1789. nested_svm_vmloadsave(svm->vmcb, nested_vmcb);
  1790. nested_svm_unmap(page);
  1791. return 1;
  1792. }
  1793. static int vmrun_interception(struct vcpu_svm *svm)
  1794. {
  1795. if (nested_svm_check_permissions(svm))
  1796. return 1;
  1797. svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
  1798. skip_emulated_instruction(&svm->vcpu);
  1799. if (!nested_svm_vmrun(svm))
  1800. return 1;
  1801. if (!nested_svm_vmrun_msrpm(svm))
  1802. goto failed;
  1803. return 1;
  1804. failed:
  1805. svm->vmcb->control.exit_code = SVM_EXIT_ERR;
  1806. svm->vmcb->control.exit_code_hi = 0;
  1807. svm->vmcb->control.exit_info_1 = 0;
  1808. svm->vmcb->control.exit_info_2 = 0;
  1809. nested_svm_vmexit(svm);
  1810. return 1;
  1811. }
  1812. static int stgi_interception(struct vcpu_svm *svm)
  1813. {
  1814. if (nested_svm_check_permissions(svm))
  1815. return 1;
  1816. svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
  1817. skip_emulated_instruction(&svm->vcpu);
  1818. enable_gif(svm);
  1819. return 1;
  1820. }
  1821. static int clgi_interception(struct vcpu_svm *svm)
  1822. {
  1823. if (nested_svm_check_permissions(svm))
  1824. return 1;
  1825. svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
  1826. skip_emulated_instruction(&svm->vcpu);
  1827. disable_gif(svm);
  1828. /* After a CLGI no interrupts should come */
  1829. svm_clear_vintr(svm);
  1830. svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
  1831. return 1;
  1832. }
  1833. static int invlpga_interception(struct vcpu_svm *svm)
  1834. {
  1835. struct kvm_vcpu *vcpu = &svm->vcpu;
  1836. trace_kvm_invlpga(svm->vmcb->save.rip, vcpu->arch.regs[VCPU_REGS_RCX],
  1837. vcpu->arch.regs[VCPU_REGS_RAX]);
  1838. /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
  1839. kvm_mmu_invlpg(vcpu, vcpu->arch.regs[VCPU_REGS_RAX]);
  1840. svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
  1841. skip_emulated_instruction(&svm->vcpu);
  1842. return 1;
  1843. }
  1844. static int skinit_interception(struct vcpu_svm *svm)
  1845. {
  1846. trace_kvm_skinit(svm->vmcb->save.rip, svm->vcpu.arch.regs[VCPU_REGS_RAX]);
  1847. kvm_queue_exception(&svm->vcpu, UD_VECTOR);
  1848. return 1;
  1849. }
  1850. static int invalid_op_interception(struct vcpu_svm *svm)
  1851. {
  1852. kvm_queue_exception(&svm->vcpu, UD_VECTOR);
  1853. return 1;
  1854. }
  1855. static int task_switch_interception(struct vcpu_svm *svm)
  1856. {
  1857. u16 tss_selector;
  1858. int reason;
  1859. int int_type = svm->vmcb->control.exit_int_info &
  1860. SVM_EXITINTINFO_TYPE_MASK;
  1861. int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
  1862. uint32_t type =
  1863. svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
  1864. uint32_t idt_v =
  1865. svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
  1866. bool has_error_code = false;
  1867. u32 error_code = 0;
  1868. tss_selector = (u16)svm->vmcb->control.exit_info_1;
  1869. if (svm->vmcb->control.exit_info_2 &
  1870. (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
  1871. reason = TASK_SWITCH_IRET;
  1872. else if (svm->vmcb->control.exit_info_2 &
  1873. (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
  1874. reason = TASK_SWITCH_JMP;
  1875. else if (idt_v)
  1876. reason = TASK_SWITCH_GATE;
  1877. else
  1878. reason = TASK_SWITCH_CALL;
  1879. if (reason == TASK_SWITCH_GATE) {
  1880. switch (type) {
  1881. case SVM_EXITINTINFO_TYPE_NMI:
  1882. svm->vcpu.arch.nmi_injected = false;
  1883. break;
  1884. case SVM_EXITINTINFO_TYPE_EXEPT:
  1885. if (svm->vmcb->control.exit_info_2 &
  1886. (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) {
  1887. has_error_code = true;
  1888. error_code =
  1889. (u32)svm->vmcb->control.exit_info_2;
  1890. }
  1891. kvm_clear_exception_queue(&svm->vcpu);
  1892. break;
  1893. case SVM_EXITINTINFO_TYPE_INTR:
  1894. kvm_clear_interrupt_queue(&svm->vcpu);
  1895. break;
  1896. default:
  1897. break;
  1898. }
  1899. }
  1900. if (reason != TASK_SWITCH_GATE ||
  1901. int_type == SVM_EXITINTINFO_TYPE_SOFT ||
  1902. (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
  1903. (int_vec == OF_VECTOR || int_vec == BP_VECTOR)))
  1904. skip_emulated_instruction(&svm->vcpu);
  1905. if (kvm_task_switch(&svm->vcpu, tss_selector, reason,
  1906. has_error_code, error_code) == EMULATE_FAIL) {
  1907. svm->vcpu.run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
  1908. svm->vcpu.run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
  1909. svm->vcpu.run->internal.ndata = 0;
  1910. return 0;
  1911. }
  1912. return 1;
  1913. }
  1914. static int cpuid_interception(struct vcpu_svm *svm)
  1915. {
  1916. svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
  1917. kvm_emulate_cpuid(&svm->vcpu);
  1918. return 1;
  1919. }
  1920. static int iret_interception(struct vcpu_svm *svm)
  1921. {
  1922. ++svm->vcpu.stat.nmi_window_exits;
  1923. svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_IRET);
  1924. svm->vcpu.arch.hflags |= HF_IRET_MASK;
  1925. return 1;
  1926. }
  1927. static int invlpg_interception(struct vcpu_svm *svm)
  1928. {
  1929. if (emulate_instruction(&svm->vcpu, 0, 0, 0) != EMULATE_DONE)
  1930. pr_unimpl(&svm->vcpu, "%s: failed\n", __func__);
  1931. return 1;
  1932. }
  1933. static int emulate_on_interception(struct vcpu_svm *svm)
  1934. {
  1935. if (emulate_instruction(&svm->vcpu, 0, 0, 0) != EMULATE_DONE)
  1936. pr_unimpl(&svm->vcpu, "%s: failed\n", __func__);
  1937. return 1;
  1938. }
  1939. static int cr8_write_interception(struct vcpu_svm *svm)
  1940. {
  1941. struct kvm_run *kvm_run = svm->vcpu.run;
  1942. u8 cr8_prev = kvm_get_cr8(&svm->vcpu);
  1943. /* instruction emulation calls kvm_set_cr8() */
  1944. emulate_instruction(&svm->vcpu, 0, 0, 0);
  1945. if (irqchip_in_kernel(svm->vcpu.kvm)) {
  1946. svm->vmcb->control.intercept_cr_write &= ~INTERCEPT_CR8_MASK;
  1947. return 1;
  1948. }
  1949. if (cr8_prev <= kvm_get_cr8(&svm->vcpu))
  1950. return 1;
  1951. kvm_run->exit_reason = KVM_EXIT_SET_TPR;
  1952. return 0;
  1953. }
  1954. static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data)
  1955. {
  1956. struct vcpu_svm *svm = to_svm(vcpu);
  1957. switch (ecx) {
  1958. case MSR_IA32_TSC: {
  1959. u64 tsc_offset;
  1960. if (is_nested(svm))
  1961. tsc_offset = svm->nested.hsave->control.tsc_offset;
  1962. else
  1963. tsc_offset = svm->vmcb->control.tsc_offset;
  1964. *data = tsc_offset + native_read_tsc();
  1965. break;
  1966. }
  1967. case MSR_K6_STAR:
  1968. *data = svm->vmcb->save.star;
  1969. break;
  1970. #ifdef CONFIG_X86_64
  1971. case MSR_LSTAR:
  1972. *data = svm->vmcb->save.lstar;
  1973. break;
  1974. case MSR_CSTAR:
  1975. *data = svm->vmcb->save.cstar;
  1976. break;
  1977. case MSR_KERNEL_GS_BASE:
  1978. *data = svm->vmcb->save.kernel_gs_base;
  1979. break;
  1980. case MSR_SYSCALL_MASK:
  1981. *data = svm->vmcb->save.sfmask;
  1982. break;
  1983. #endif
  1984. case MSR_IA32_SYSENTER_CS:
  1985. *data = svm->vmcb->save.sysenter_cs;
  1986. break;
  1987. case MSR_IA32_SYSENTER_EIP:
  1988. *data = svm->sysenter_eip;
  1989. break;
  1990. case MSR_IA32_SYSENTER_ESP:
  1991. *data = svm->sysenter_esp;
  1992. break;
  1993. /*
  1994. * Nobody will change the following 5 values in the VMCB so we can
  1995. * safely return them on rdmsr. They will always be 0 until LBRV is
  1996. * implemented.
  1997. */
  1998. case MSR_IA32_DEBUGCTLMSR:
  1999. *data = svm->vmcb->save.dbgctl;
  2000. break;
  2001. case MSR_IA32_LASTBRANCHFROMIP:
  2002. *data = svm->vmcb->save.br_from;
  2003. break;
  2004. case MSR_IA32_LASTBRANCHTOIP:
  2005. *data = svm->vmcb->save.br_to;
  2006. break;
  2007. case MSR_IA32_LASTINTFROMIP:
  2008. *data = svm->vmcb->save.last_excp_from;
  2009. break;
  2010. case MSR_IA32_LASTINTTOIP:
  2011. *data = svm->vmcb->save.last_excp_to;
  2012. break;
  2013. case MSR_VM_HSAVE_PA:
  2014. *data = svm->nested.hsave_msr;
  2015. break;
  2016. case MSR_VM_CR:
  2017. *data = svm->nested.vm_cr_msr;
  2018. break;
  2019. case MSR_IA32_UCODE_REV:
  2020. *data = 0x01000065;
  2021. break;
  2022. default:
  2023. return kvm_get_msr_common(vcpu, ecx, data);
  2024. }
  2025. return 0;
  2026. }
  2027. static int rdmsr_interception(struct vcpu_svm *svm)
  2028. {
  2029. u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX];
  2030. u64 data;
  2031. if (svm_get_msr(&svm->vcpu, ecx, &data)) {
  2032. trace_kvm_msr_read_ex(ecx);
  2033. kvm_inject_gp(&svm->vcpu, 0);
  2034. } else {
  2035. trace_kvm_msr_read(ecx, data);
  2036. svm->vcpu.arch.regs[VCPU_REGS_RAX] = data & 0xffffffff;
  2037. svm->vcpu.arch.regs[VCPU_REGS_RDX] = data >> 32;
  2038. svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
  2039. skip_emulated_instruction(&svm->vcpu);
  2040. }
  2041. return 1;
  2042. }
  2043. static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)
  2044. {
  2045. struct vcpu_svm *svm = to_svm(vcpu);
  2046. int svm_dis, chg_mask;
  2047. if (data & ~SVM_VM_CR_VALID_MASK)
  2048. return 1;
  2049. chg_mask = SVM_VM_CR_VALID_MASK;
  2050. if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK)
  2051. chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK);
  2052. svm->nested.vm_cr_msr &= ~chg_mask;
  2053. svm->nested.vm_cr_msr |= (data & chg_mask);
  2054. svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK;
  2055. /* check for svm_disable while efer.svme is set */
  2056. if (svm_dis && (vcpu->arch.efer & EFER_SVME))
  2057. return 1;
  2058. return 0;
  2059. }
  2060. static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data)
  2061. {
  2062. struct vcpu_svm *svm = to_svm(vcpu);
  2063. switch (ecx) {
  2064. case MSR_IA32_TSC: {
  2065. u64 tsc_offset = data - native_read_tsc();
  2066. u64 g_tsc_offset = 0;
  2067. if (is_nested(svm)) {
  2068. g_tsc_offset = svm->vmcb->control.tsc_offset -
  2069. svm->nested.hsave->control.tsc_offset;
  2070. svm->nested.hsave->control.tsc_offset = tsc_offset;
  2071. }
  2072. svm->vmcb->control.tsc_offset = tsc_offset + g_tsc_offset;
  2073. break;
  2074. }
  2075. case MSR_K6_STAR:
  2076. svm->vmcb->save.star = data;
  2077. break;
  2078. #ifdef CONFIG_X86_64
  2079. case MSR_LSTAR:
  2080. svm->vmcb->save.lstar = data;
  2081. break;
  2082. case MSR_CSTAR:
  2083. svm->vmcb->save.cstar = data;
  2084. break;
  2085. case MSR_KERNEL_GS_BASE:
  2086. svm->vmcb->save.kernel_gs_base = data;
  2087. break;
  2088. case MSR_SYSCALL_MASK:
  2089. svm->vmcb->save.sfmask = data;
  2090. break;
  2091. #endif
  2092. case MSR_IA32_SYSENTER_CS:
  2093. svm->vmcb->save.sysenter_cs = data;
  2094. break;
  2095. case MSR_IA32_SYSENTER_EIP:
  2096. svm->sysenter_eip = data;
  2097. svm->vmcb->save.sysenter_eip = data;
  2098. break;
  2099. case MSR_IA32_SYSENTER_ESP:
  2100. svm->sysenter_esp = data;
  2101. svm->vmcb->save.sysenter_esp = data;
  2102. break;
  2103. case MSR_IA32_DEBUGCTLMSR:
  2104. if (!svm_has(SVM_FEATURE_LBRV)) {
  2105. pr_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
  2106. __func__, data);
  2107. break;
  2108. }
  2109. if (data & DEBUGCTL_RESERVED_BITS)
  2110. return 1;
  2111. svm->vmcb->save.dbgctl = data;
  2112. if (data & (1ULL<<0))
  2113. svm_enable_lbrv(svm);
  2114. else
  2115. svm_disable_lbrv(svm);
  2116. break;
  2117. case MSR_VM_HSAVE_PA:
  2118. svm->nested.hsave_msr = data;
  2119. break;
  2120. case MSR_VM_CR:
  2121. return svm_set_vm_cr(vcpu, data);
  2122. case MSR_VM_IGNNE:
  2123. pr_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data);
  2124. break;
  2125. default:
  2126. return kvm_set_msr_common(vcpu, ecx, data);
  2127. }
  2128. return 0;
  2129. }
  2130. static int wrmsr_interception(struct vcpu_svm *svm)
  2131. {
  2132. u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX];
  2133. u64 data = (svm->vcpu.arch.regs[VCPU_REGS_RAX] & -1u)
  2134. | ((u64)(svm->vcpu.arch.regs[VCPU_REGS_RDX] & -1u) << 32);
  2135. svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
  2136. if (svm_set_msr(&svm->vcpu, ecx, data)) {
  2137. trace_kvm_msr_write_ex(ecx, data);
  2138. kvm_inject_gp(&svm->vcpu, 0);
  2139. } else {
  2140. trace_kvm_msr_write(ecx, data);
  2141. skip_emulated_instruction(&svm->vcpu);
  2142. }
  2143. return 1;
  2144. }
  2145. static int msr_interception(struct vcpu_svm *svm)
  2146. {
  2147. if (svm->vmcb->control.exit_info_1)
  2148. return wrmsr_interception(svm);
  2149. else
  2150. return rdmsr_interception(svm);
  2151. }
  2152. static int interrupt_window_interception(struct vcpu_svm *svm)
  2153. {
  2154. struct kvm_run *kvm_run = svm->vcpu.run;
  2155. svm_clear_vintr(svm);
  2156. svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
  2157. /*
  2158. * If the user space waits to inject interrupts, exit as soon as
  2159. * possible
  2160. */
  2161. if (!irqchip_in_kernel(svm->vcpu.kvm) &&
  2162. kvm_run->request_interrupt_window &&
  2163. !kvm_cpu_has_interrupt(&svm->vcpu)) {
  2164. ++svm->vcpu.stat.irq_window_exits;
  2165. kvm_run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
  2166. return 0;
  2167. }
  2168. return 1;
  2169. }
  2170. static int pause_interception(struct vcpu_svm *svm)
  2171. {
  2172. kvm_vcpu_on_spin(&(svm->vcpu));
  2173. return 1;
  2174. }
  2175. static int (*svm_exit_handlers[])(struct vcpu_svm *svm) = {
  2176. [SVM_EXIT_READ_CR0] = emulate_on_interception,
  2177. [SVM_EXIT_READ_CR3] = emulate_on_interception,
  2178. [SVM_EXIT_READ_CR4] = emulate_on_interception,
  2179. [SVM_EXIT_READ_CR8] = emulate_on_interception,
  2180. [SVM_EXIT_CR0_SEL_WRITE] = emulate_on_interception,
  2181. [SVM_EXIT_WRITE_CR0] = emulate_on_interception,
  2182. [SVM_EXIT_WRITE_CR3] = emulate_on_interception,
  2183. [SVM_EXIT_WRITE_CR4] = emulate_on_interception,
  2184. [SVM_EXIT_WRITE_CR8] = cr8_write_interception,
  2185. [SVM_EXIT_READ_DR0] = emulate_on_interception,
  2186. [SVM_EXIT_READ_DR1] = emulate_on_interception,
  2187. [SVM_EXIT_READ_DR2] = emulate_on_interception,
  2188. [SVM_EXIT_READ_DR3] = emulate_on_interception,
  2189. [SVM_EXIT_READ_DR4] = emulate_on_interception,
  2190. [SVM_EXIT_READ_DR5] = emulate_on_interception,
  2191. [SVM_EXIT_READ_DR6] = emulate_on_interception,
  2192. [SVM_EXIT_READ_DR7] = emulate_on_interception,
  2193. [SVM_EXIT_WRITE_DR0] = emulate_on_interception,
  2194. [SVM_EXIT_WRITE_DR1] = emulate_on_interception,
  2195. [SVM_EXIT_WRITE_DR2] = emulate_on_interception,
  2196. [SVM_EXIT_WRITE_DR3] = emulate_on_interception,
  2197. [SVM_EXIT_WRITE_DR4] = emulate_on_interception,
  2198. [SVM_EXIT_WRITE_DR5] = emulate_on_interception,
  2199. [SVM_EXIT_WRITE_DR6] = emulate_on_interception,
  2200. [SVM_EXIT_WRITE_DR7] = emulate_on_interception,
  2201. [SVM_EXIT_EXCP_BASE + DB_VECTOR] = db_interception,
  2202. [SVM_EXIT_EXCP_BASE + BP_VECTOR] = bp_interception,
  2203. [SVM_EXIT_EXCP_BASE + UD_VECTOR] = ud_interception,
  2204. [SVM_EXIT_EXCP_BASE + PF_VECTOR] = pf_interception,
  2205. [SVM_EXIT_EXCP_BASE + NM_VECTOR] = nm_interception,
  2206. [SVM_EXIT_EXCP_BASE + MC_VECTOR] = mc_interception,
  2207. [SVM_EXIT_INTR] = intr_interception,
  2208. [SVM_EXIT_NMI] = nmi_interception,
  2209. [SVM_EXIT_SMI] = nop_on_interception,
  2210. [SVM_EXIT_INIT] = nop_on_interception,
  2211. [SVM_EXIT_VINTR] = interrupt_window_interception,
  2212. [SVM_EXIT_CPUID] = cpuid_interception,
  2213. [SVM_EXIT_IRET] = iret_interception,
  2214. [SVM_EXIT_INVD] = emulate_on_interception,
  2215. [SVM_EXIT_PAUSE] = pause_interception,
  2216. [SVM_EXIT_HLT] = halt_interception,
  2217. [SVM_EXIT_INVLPG] = invlpg_interception,
  2218. [SVM_EXIT_INVLPGA] = invlpga_interception,
  2219. [SVM_EXIT_IOIO] = io_interception,
  2220. [SVM_EXIT_MSR] = msr_interception,
  2221. [SVM_EXIT_TASK_SWITCH] = task_switch_interception,
  2222. [SVM_EXIT_SHUTDOWN] = shutdown_interception,
  2223. [SVM_EXIT_VMRUN] = vmrun_interception,
  2224. [SVM_EXIT_VMMCALL] = vmmcall_interception,
  2225. [SVM_EXIT_VMLOAD] = vmload_interception,
  2226. [SVM_EXIT_VMSAVE] = vmsave_interception,
  2227. [SVM_EXIT_STGI] = stgi_interception,
  2228. [SVM_EXIT_CLGI] = clgi_interception,
  2229. [SVM_EXIT_SKINIT] = skinit_interception,
  2230. [SVM_EXIT_WBINVD] = emulate_on_interception,
  2231. [SVM_EXIT_MONITOR] = invalid_op_interception,
  2232. [SVM_EXIT_MWAIT] = invalid_op_interception,
  2233. [SVM_EXIT_NPF] = pf_interception,
  2234. };
  2235. static int handle_exit(struct kvm_vcpu *vcpu)
  2236. {
  2237. struct vcpu_svm *svm = to_svm(vcpu);
  2238. struct kvm_run *kvm_run = vcpu->run;
  2239. u32 exit_code = svm->vmcb->control.exit_code;
  2240. trace_kvm_exit(exit_code, vcpu);
  2241. if (!(svm->vmcb->control.intercept_cr_write & INTERCEPT_CR0_MASK))
  2242. vcpu->arch.cr0 = svm->vmcb->save.cr0;
  2243. if (npt_enabled)
  2244. vcpu->arch.cr3 = svm->vmcb->save.cr3;
  2245. if (unlikely(svm->nested.exit_required)) {
  2246. nested_svm_vmexit(svm);
  2247. svm->nested.exit_required = false;
  2248. return 1;
  2249. }
  2250. if (is_nested(svm)) {
  2251. int vmexit;
  2252. trace_kvm_nested_vmexit(svm->vmcb->save.rip, exit_code,
  2253. svm->vmcb->control.exit_info_1,
  2254. svm->vmcb->control.exit_info_2,
  2255. svm->vmcb->control.exit_int_info,
  2256. svm->vmcb->control.exit_int_info_err);
  2257. vmexit = nested_svm_exit_special(svm);
  2258. if (vmexit == NESTED_EXIT_CONTINUE)
  2259. vmexit = nested_svm_exit_handled(svm);
  2260. if (vmexit == NESTED_EXIT_DONE)
  2261. return 1;
  2262. }
  2263. svm_complete_interrupts(svm);
  2264. if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
  2265. kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
  2266. kvm_run->fail_entry.hardware_entry_failure_reason
  2267. = svm->vmcb->control.exit_code;
  2268. return 0;
  2269. }
  2270. if (is_external_interrupt(svm->vmcb->control.exit_int_info) &&
  2271. exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR &&
  2272. exit_code != SVM_EXIT_NPF && exit_code != SVM_EXIT_TASK_SWITCH)
  2273. printk(KERN_ERR "%s: unexpected exit_ini_info 0x%x "
  2274. "exit_code 0x%x\n",
  2275. __func__, svm->vmcb->control.exit_int_info,
  2276. exit_code);
  2277. if (exit_code >= ARRAY_SIZE(svm_exit_handlers)
  2278. || !svm_exit_handlers[exit_code]) {
  2279. kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
  2280. kvm_run->hw.hardware_exit_reason = exit_code;
  2281. return 0;
  2282. }
  2283. return svm_exit_handlers[exit_code](svm);
  2284. }
  2285. static void reload_tss(struct kvm_vcpu *vcpu)
  2286. {
  2287. int cpu = raw_smp_processor_id();
  2288. struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
  2289. sd->tss_desc->type = 9; /* available 32/64-bit TSS */
  2290. load_TR_desc();
  2291. }
  2292. static void pre_svm_run(struct vcpu_svm *svm)
  2293. {
  2294. int cpu = raw_smp_processor_id();
  2295. struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
  2296. svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
  2297. /* FIXME: handle wraparound of asid_generation */
  2298. if (svm->asid_generation != sd->asid_generation)
  2299. new_asid(svm, sd);
  2300. }
  2301. static void svm_inject_nmi(struct kvm_vcpu *vcpu)
  2302. {
  2303. struct vcpu_svm *svm = to_svm(vcpu);
  2304. svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
  2305. vcpu->arch.hflags |= HF_NMI_MASK;
  2306. svm->vmcb->control.intercept |= (1ULL << INTERCEPT_IRET);
  2307. ++vcpu->stat.nmi_injections;
  2308. }
  2309. static inline void svm_inject_irq(struct vcpu_svm *svm, int irq)
  2310. {
  2311. struct vmcb_control_area *control;
  2312. trace_kvm_inj_virq(irq);
  2313. ++svm->vcpu.stat.irq_injections;
  2314. control = &svm->vmcb->control;
  2315. control->int_vector = irq;
  2316. control->int_ctl &= ~V_INTR_PRIO_MASK;
  2317. control->int_ctl |= V_IRQ_MASK |
  2318. ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
  2319. }
  2320. static void svm_set_irq(struct kvm_vcpu *vcpu)
  2321. {
  2322. struct vcpu_svm *svm = to_svm(vcpu);
  2323. BUG_ON(!(gif_set(svm)));
  2324. svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
  2325. SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR;
  2326. }
  2327. static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
  2328. {
  2329. struct vcpu_svm *svm = to_svm(vcpu);
  2330. if (is_nested(svm) && (vcpu->arch.hflags & HF_VINTR_MASK))
  2331. return;
  2332. if (irr == -1)
  2333. return;
  2334. if (tpr >= irr)
  2335. svm->vmcb->control.intercept_cr_write |= INTERCEPT_CR8_MASK;
  2336. }
  2337. static int svm_nmi_allowed(struct kvm_vcpu *vcpu)
  2338. {
  2339. struct vcpu_svm *svm = to_svm(vcpu);
  2340. struct vmcb *vmcb = svm->vmcb;
  2341. int ret;
  2342. ret = !(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) &&
  2343. !(svm->vcpu.arch.hflags & HF_NMI_MASK);
  2344. ret = ret && gif_set(svm) && nested_svm_nmi(svm);
  2345. return ret;
  2346. }
  2347. static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu)
  2348. {
  2349. struct vcpu_svm *svm = to_svm(vcpu);
  2350. return !!(svm->vcpu.arch.hflags & HF_NMI_MASK);
  2351. }
  2352. static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
  2353. {
  2354. struct vcpu_svm *svm = to_svm(vcpu);
  2355. if (masked) {
  2356. svm->vcpu.arch.hflags |= HF_NMI_MASK;
  2357. svm->vmcb->control.intercept |= (1ULL << INTERCEPT_IRET);
  2358. } else {
  2359. svm->vcpu.arch.hflags &= ~HF_NMI_MASK;
  2360. svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_IRET);
  2361. }
  2362. }
  2363. static int svm_interrupt_allowed(struct kvm_vcpu *vcpu)
  2364. {
  2365. struct vcpu_svm *svm = to_svm(vcpu);
  2366. struct vmcb *vmcb = svm->vmcb;
  2367. int ret;
  2368. if (!gif_set(svm) ||
  2369. (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK))
  2370. return 0;
  2371. ret = !!(vmcb->save.rflags & X86_EFLAGS_IF);
  2372. if (is_nested(svm))
  2373. return ret && !(svm->vcpu.arch.hflags & HF_VINTR_MASK);
  2374. return ret;
  2375. }
  2376. static void enable_irq_window(struct kvm_vcpu *vcpu)
  2377. {
  2378. struct vcpu_svm *svm = to_svm(vcpu);
  2379. /*
  2380. * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
  2381. * 1, because that's a separate STGI/VMRUN intercept. The next time we
  2382. * get that intercept, this function will be called again though and
  2383. * we'll get the vintr intercept.
  2384. */
  2385. if (gif_set(svm) && nested_svm_intr(svm)) {
  2386. svm_set_vintr(svm);
  2387. svm_inject_irq(svm, 0x0);
  2388. }
  2389. }
  2390. static void enable_nmi_window(struct kvm_vcpu *vcpu)
  2391. {
  2392. struct vcpu_svm *svm = to_svm(vcpu);
  2393. if ((svm->vcpu.arch.hflags & (HF_NMI_MASK | HF_IRET_MASK))
  2394. == HF_NMI_MASK)
  2395. return; /* IRET will cause a vm exit */
  2396. /*
  2397. * Something prevents NMI from been injected. Single step over possible
  2398. * problem (IRET or exception injection or interrupt shadow)
  2399. */
  2400. svm->nmi_singlestep = true;
  2401. svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
  2402. update_db_intercept(vcpu);
  2403. }
  2404. static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
  2405. {
  2406. return 0;
  2407. }
  2408. static void svm_flush_tlb(struct kvm_vcpu *vcpu)
  2409. {
  2410. force_new_asid(vcpu);
  2411. }
  2412. static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
  2413. {
  2414. }
  2415. static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
  2416. {
  2417. struct vcpu_svm *svm = to_svm(vcpu);
  2418. if (is_nested(svm) && (vcpu->arch.hflags & HF_VINTR_MASK))
  2419. return;
  2420. if (!(svm->vmcb->control.intercept_cr_write & INTERCEPT_CR8_MASK)) {
  2421. int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
  2422. kvm_set_cr8(vcpu, cr8);
  2423. }
  2424. }
  2425. static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
  2426. {
  2427. struct vcpu_svm *svm = to_svm(vcpu);
  2428. u64 cr8;
  2429. if (is_nested(svm) && (vcpu->arch.hflags & HF_VINTR_MASK))
  2430. return;
  2431. cr8 = kvm_get_cr8(vcpu);
  2432. svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
  2433. svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
  2434. }
  2435. static void svm_complete_interrupts(struct vcpu_svm *svm)
  2436. {
  2437. u8 vector;
  2438. int type;
  2439. u32 exitintinfo = svm->vmcb->control.exit_int_info;
  2440. unsigned int3_injected = svm->int3_injected;
  2441. svm->int3_injected = 0;
  2442. if (svm->vcpu.arch.hflags & HF_IRET_MASK)
  2443. svm->vcpu.arch.hflags &= ~(HF_NMI_MASK | HF_IRET_MASK);
  2444. svm->vcpu.arch.nmi_injected = false;
  2445. kvm_clear_exception_queue(&svm->vcpu);
  2446. kvm_clear_interrupt_queue(&svm->vcpu);
  2447. if (!(exitintinfo & SVM_EXITINTINFO_VALID))
  2448. return;
  2449. vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
  2450. type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;
  2451. switch (type) {
  2452. case SVM_EXITINTINFO_TYPE_NMI:
  2453. svm->vcpu.arch.nmi_injected = true;
  2454. break;
  2455. case SVM_EXITINTINFO_TYPE_EXEPT:
  2456. /*
  2457. * In case of software exceptions, do not reinject the vector,
  2458. * but re-execute the instruction instead. Rewind RIP first
  2459. * if we emulated INT3 before.
  2460. */
  2461. if (kvm_exception_is_soft(vector)) {
  2462. if (vector == BP_VECTOR && int3_injected &&
  2463. kvm_is_linear_rip(&svm->vcpu, svm->int3_rip))
  2464. kvm_rip_write(&svm->vcpu,
  2465. kvm_rip_read(&svm->vcpu) -
  2466. int3_injected);
  2467. break;
  2468. }
  2469. if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
  2470. u32 err = svm->vmcb->control.exit_int_info_err;
  2471. kvm_requeue_exception_e(&svm->vcpu, vector, err);
  2472. } else
  2473. kvm_requeue_exception(&svm->vcpu, vector);
  2474. break;
  2475. case SVM_EXITINTINFO_TYPE_INTR:
  2476. kvm_queue_interrupt(&svm->vcpu, vector, false);
  2477. break;
  2478. default:
  2479. break;
  2480. }
  2481. }
  2482. #ifdef CONFIG_X86_64
  2483. #define R "r"
  2484. #else
  2485. #define R "e"
  2486. #endif
  2487. static void svm_vcpu_run(struct kvm_vcpu *vcpu)
  2488. {
  2489. struct vcpu_svm *svm = to_svm(vcpu);
  2490. u16 fs_selector;
  2491. u16 gs_selector;
  2492. u16 ldt_selector;
  2493. svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
  2494. svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
  2495. svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
  2496. /*
  2497. * A vmexit emulation is required before the vcpu can be executed
  2498. * again.
  2499. */
  2500. if (unlikely(svm->nested.exit_required))
  2501. return;
  2502. pre_svm_run(svm);
  2503. sync_lapic_to_cr8(vcpu);
  2504. save_host_msrs(vcpu);
  2505. fs_selector = kvm_read_fs();
  2506. gs_selector = kvm_read_gs();
  2507. ldt_selector = kvm_read_ldt();
  2508. svm->vmcb->save.cr2 = vcpu->arch.cr2;
  2509. /* required for live migration with NPT */
  2510. if (npt_enabled)
  2511. svm->vmcb->save.cr3 = vcpu->arch.cr3;
  2512. clgi();
  2513. local_irq_enable();
  2514. asm volatile (
  2515. "push %%"R"bp; \n\t"
  2516. "mov %c[rbx](%[svm]), %%"R"bx \n\t"
  2517. "mov %c[rcx](%[svm]), %%"R"cx \n\t"
  2518. "mov %c[rdx](%[svm]), %%"R"dx \n\t"
  2519. "mov %c[rsi](%[svm]), %%"R"si \n\t"
  2520. "mov %c[rdi](%[svm]), %%"R"di \n\t"
  2521. "mov %c[rbp](%[svm]), %%"R"bp \n\t"
  2522. #ifdef CONFIG_X86_64
  2523. "mov %c[r8](%[svm]), %%r8 \n\t"
  2524. "mov %c[r9](%[svm]), %%r9 \n\t"
  2525. "mov %c[r10](%[svm]), %%r10 \n\t"
  2526. "mov %c[r11](%[svm]), %%r11 \n\t"
  2527. "mov %c[r12](%[svm]), %%r12 \n\t"
  2528. "mov %c[r13](%[svm]), %%r13 \n\t"
  2529. "mov %c[r14](%[svm]), %%r14 \n\t"
  2530. "mov %c[r15](%[svm]), %%r15 \n\t"
  2531. #endif
  2532. /* Enter guest mode */
  2533. "push %%"R"ax \n\t"
  2534. "mov %c[vmcb](%[svm]), %%"R"ax \n\t"
  2535. __ex(SVM_VMLOAD) "\n\t"
  2536. __ex(SVM_VMRUN) "\n\t"
  2537. __ex(SVM_VMSAVE) "\n\t"
  2538. "pop %%"R"ax \n\t"
  2539. /* Save guest registers, load host registers */
  2540. "mov %%"R"bx, %c[rbx](%[svm]) \n\t"
  2541. "mov %%"R"cx, %c[rcx](%[svm]) \n\t"
  2542. "mov %%"R"dx, %c[rdx](%[svm]) \n\t"
  2543. "mov %%"R"si, %c[rsi](%[svm]) \n\t"
  2544. "mov %%"R"di, %c[rdi](%[svm]) \n\t"
  2545. "mov %%"R"bp, %c[rbp](%[svm]) \n\t"
  2546. #ifdef CONFIG_X86_64
  2547. "mov %%r8, %c[r8](%[svm]) \n\t"
  2548. "mov %%r9, %c[r9](%[svm]) \n\t"
  2549. "mov %%r10, %c[r10](%[svm]) \n\t"
  2550. "mov %%r11, %c[r11](%[svm]) \n\t"
  2551. "mov %%r12, %c[r12](%[svm]) \n\t"
  2552. "mov %%r13, %c[r13](%[svm]) \n\t"
  2553. "mov %%r14, %c[r14](%[svm]) \n\t"
  2554. "mov %%r15, %c[r15](%[svm]) \n\t"
  2555. #endif
  2556. "pop %%"R"bp"
  2557. :
  2558. : [svm]"a"(svm),
  2559. [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)),
  2560. [rbx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBX])),
  2561. [rcx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RCX])),
  2562. [rdx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDX])),
  2563. [rsi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RSI])),
  2564. [rdi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDI])),
  2565. [rbp]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBP]))
  2566. #ifdef CONFIG_X86_64
  2567. , [r8]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R8])),
  2568. [r9]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R9])),
  2569. [r10]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R10])),
  2570. [r11]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R11])),
  2571. [r12]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R12])),
  2572. [r13]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R13])),
  2573. [r14]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R14])),
  2574. [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15]))
  2575. #endif
  2576. : "cc", "memory"
  2577. , R"bx", R"cx", R"dx", R"si", R"di"
  2578. #ifdef CONFIG_X86_64
  2579. , "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15"
  2580. #endif
  2581. );
  2582. vcpu->arch.cr2 = svm->vmcb->save.cr2;
  2583. vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
  2584. vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
  2585. vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
  2586. kvm_load_fs(fs_selector);
  2587. kvm_load_gs(gs_selector);
  2588. kvm_load_ldt(ldt_selector);
  2589. load_host_msrs(vcpu);
  2590. reload_tss(vcpu);
  2591. local_irq_disable();
  2592. stgi();
  2593. sync_cr8_to_lapic(vcpu);
  2594. svm->next_rip = 0;
  2595. if (npt_enabled) {
  2596. vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR);
  2597. vcpu->arch.regs_dirty &= ~(1 << VCPU_EXREG_PDPTR);
  2598. }
  2599. /*
  2600. * We need to handle MC intercepts here before the vcpu has a chance to
  2601. * change the physical cpu
  2602. */
  2603. if (unlikely(svm->vmcb->control.exit_code ==
  2604. SVM_EXIT_EXCP_BASE + MC_VECTOR))
  2605. svm_handle_mce(svm);
  2606. }
  2607. #undef R
  2608. static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
  2609. {
  2610. struct vcpu_svm *svm = to_svm(vcpu);
  2611. if (npt_enabled) {
  2612. svm->vmcb->control.nested_cr3 = root;
  2613. force_new_asid(vcpu);
  2614. return;
  2615. }
  2616. svm->vmcb->save.cr3 = root;
  2617. force_new_asid(vcpu);
  2618. }
  2619. static int is_disabled(void)
  2620. {
  2621. u64 vm_cr;
  2622. rdmsrl(MSR_VM_CR, vm_cr);
  2623. if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE))
  2624. return 1;
  2625. return 0;
  2626. }
  2627. static void
  2628. svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
  2629. {
  2630. /*
  2631. * Patch in the VMMCALL instruction:
  2632. */
  2633. hypercall[0] = 0x0f;
  2634. hypercall[1] = 0x01;
  2635. hypercall[2] = 0xd9;
  2636. }
  2637. static void svm_check_processor_compat(void *rtn)
  2638. {
  2639. *(int *)rtn = 0;
  2640. }
  2641. static bool svm_cpu_has_accelerated_tpr(void)
  2642. {
  2643. return false;
  2644. }
  2645. static int get_npt_level(void)
  2646. {
  2647. #ifdef CONFIG_X86_64
  2648. return PT64_ROOT_LEVEL;
  2649. #else
  2650. return PT32E_ROOT_LEVEL;
  2651. #endif
  2652. }
  2653. static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
  2654. {
  2655. return 0;
  2656. }
  2657. static void svm_cpuid_update(struct kvm_vcpu *vcpu)
  2658. {
  2659. }
  2660. static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
  2661. {
  2662. switch (func) {
  2663. case 0x8000000A:
  2664. entry->eax = 1; /* SVM revision 1 */
  2665. entry->ebx = 8; /* Lets support 8 ASIDs in case we add proper
  2666. ASID emulation to nested SVM */
  2667. entry->ecx = 0; /* Reserved */
  2668. entry->edx = 0; /* Do not support any additional features */
  2669. break;
  2670. }
  2671. }
  2672. static const struct trace_print_flags svm_exit_reasons_str[] = {
  2673. { SVM_EXIT_READ_CR0, "read_cr0" },
  2674. { SVM_EXIT_READ_CR3, "read_cr3" },
  2675. { SVM_EXIT_READ_CR4, "read_cr4" },
  2676. { SVM_EXIT_READ_CR8, "read_cr8" },
  2677. { SVM_EXIT_WRITE_CR0, "write_cr0" },
  2678. { SVM_EXIT_WRITE_CR3, "write_cr3" },
  2679. { SVM_EXIT_WRITE_CR4, "write_cr4" },
  2680. { SVM_EXIT_WRITE_CR8, "write_cr8" },
  2681. { SVM_EXIT_READ_DR0, "read_dr0" },
  2682. { SVM_EXIT_READ_DR1, "read_dr1" },
  2683. { SVM_EXIT_READ_DR2, "read_dr2" },
  2684. { SVM_EXIT_READ_DR3, "read_dr3" },
  2685. { SVM_EXIT_WRITE_DR0, "write_dr0" },
  2686. { SVM_EXIT_WRITE_DR1, "write_dr1" },
  2687. { SVM_EXIT_WRITE_DR2, "write_dr2" },
  2688. { SVM_EXIT_WRITE_DR3, "write_dr3" },
  2689. { SVM_EXIT_WRITE_DR5, "write_dr5" },
  2690. { SVM_EXIT_WRITE_DR7, "write_dr7" },
  2691. { SVM_EXIT_EXCP_BASE + DB_VECTOR, "DB excp" },
  2692. { SVM_EXIT_EXCP_BASE + BP_VECTOR, "BP excp" },
  2693. { SVM_EXIT_EXCP_BASE + UD_VECTOR, "UD excp" },
  2694. { SVM_EXIT_EXCP_BASE + PF_VECTOR, "PF excp" },
  2695. { SVM_EXIT_EXCP_BASE + NM_VECTOR, "NM excp" },
  2696. { SVM_EXIT_EXCP_BASE + MC_VECTOR, "MC excp" },
  2697. { SVM_EXIT_INTR, "interrupt" },
  2698. { SVM_EXIT_NMI, "nmi" },
  2699. { SVM_EXIT_SMI, "smi" },
  2700. { SVM_EXIT_INIT, "init" },
  2701. { SVM_EXIT_VINTR, "vintr" },
  2702. { SVM_EXIT_CPUID, "cpuid" },
  2703. { SVM_EXIT_INVD, "invd" },
  2704. { SVM_EXIT_HLT, "hlt" },
  2705. { SVM_EXIT_INVLPG, "invlpg" },
  2706. { SVM_EXIT_INVLPGA, "invlpga" },
  2707. { SVM_EXIT_IOIO, "io" },
  2708. { SVM_EXIT_MSR, "msr" },
  2709. { SVM_EXIT_TASK_SWITCH, "task_switch" },
  2710. { SVM_EXIT_SHUTDOWN, "shutdown" },
  2711. { SVM_EXIT_VMRUN, "vmrun" },
  2712. { SVM_EXIT_VMMCALL, "hypercall" },
  2713. { SVM_EXIT_VMLOAD, "vmload" },
  2714. { SVM_EXIT_VMSAVE, "vmsave" },
  2715. { SVM_EXIT_STGI, "stgi" },
  2716. { SVM_EXIT_CLGI, "clgi" },
  2717. { SVM_EXIT_SKINIT, "skinit" },
  2718. { SVM_EXIT_WBINVD, "wbinvd" },
  2719. { SVM_EXIT_MONITOR, "monitor" },
  2720. { SVM_EXIT_MWAIT, "mwait" },
  2721. { SVM_EXIT_NPF, "npf" },
  2722. { -1, NULL }
  2723. };
  2724. static int svm_get_lpage_level(void)
  2725. {
  2726. return PT_PDPE_LEVEL;
  2727. }
  2728. static bool svm_rdtscp_supported(void)
  2729. {
  2730. return false;
  2731. }
  2732. static void svm_fpu_deactivate(struct kvm_vcpu *vcpu)
  2733. {
  2734. struct vcpu_svm *svm = to_svm(vcpu);
  2735. svm->vmcb->control.intercept_exceptions |= 1 << NM_VECTOR;
  2736. if (is_nested(svm))
  2737. svm->nested.hsave->control.intercept_exceptions |= 1 << NM_VECTOR;
  2738. update_cr0_intercept(svm);
  2739. }
  2740. static struct kvm_x86_ops svm_x86_ops = {
  2741. .cpu_has_kvm_support = has_svm,
  2742. .disabled_by_bios = is_disabled,
  2743. .hardware_setup = svm_hardware_setup,
  2744. .hardware_unsetup = svm_hardware_unsetup,
  2745. .check_processor_compatibility = svm_check_processor_compat,
  2746. .hardware_enable = svm_hardware_enable,
  2747. .hardware_disable = svm_hardware_disable,
  2748. .cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr,
  2749. .vcpu_create = svm_create_vcpu,
  2750. .vcpu_free = svm_free_vcpu,
  2751. .vcpu_reset = svm_vcpu_reset,
  2752. .prepare_guest_switch = svm_prepare_guest_switch,
  2753. .vcpu_load = svm_vcpu_load,
  2754. .vcpu_put = svm_vcpu_put,
  2755. .set_guest_debug = svm_guest_debug,
  2756. .get_msr = svm_get_msr,
  2757. .set_msr = svm_set_msr,
  2758. .get_segment_base = svm_get_segment_base,
  2759. .get_segment = svm_get_segment,
  2760. .set_segment = svm_set_segment,
  2761. .get_cpl = svm_get_cpl,
  2762. .get_cs_db_l_bits = kvm_get_cs_db_l_bits,
  2763. .decache_cr0_guest_bits = svm_decache_cr0_guest_bits,
  2764. .decache_cr4_guest_bits = svm_decache_cr4_guest_bits,
  2765. .set_cr0 = svm_set_cr0,
  2766. .set_cr3 = svm_set_cr3,
  2767. .set_cr4 = svm_set_cr4,
  2768. .set_efer = svm_set_efer,
  2769. .get_idt = svm_get_idt,
  2770. .set_idt = svm_set_idt,
  2771. .get_gdt = svm_get_gdt,
  2772. .set_gdt = svm_set_gdt,
  2773. .set_dr7 = svm_set_dr7,
  2774. .cache_reg = svm_cache_reg,
  2775. .get_rflags = svm_get_rflags,
  2776. .set_rflags = svm_set_rflags,
  2777. .fpu_activate = svm_fpu_activate,
  2778. .fpu_deactivate = svm_fpu_deactivate,
  2779. .tlb_flush = svm_flush_tlb,
  2780. .run = svm_vcpu_run,
  2781. .handle_exit = handle_exit,
  2782. .skip_emulated_instruction = skip_emulated_instruction,
  2783. .set_interrupt_shadow = svm_set_interrupt_shadow,
  2784. .get_interrupt_shadow = svm_get_interrupt_shadow,
  2785. .patch_hypercall = svm_patch_hypercall,
  2786. .set_irq = svm_set_irq,
  2787. .set_nmi = svm_inject_nmi,
  2788. .queue_exception = svm_queue_exception,
  2789. .interrupt_allowed = svm_interrupt_allowed,
  2790. .nmi_allowed = svm_nmi_allowed,
  2791. .get_nmi_mask = svm_get_nmi_mask,
  2792. .set_nmi_mask = svm_set_nmi_mask,
  2793. .enable_nmi_window = enable_nmi_window,
  2794. .enable_irq_window = enable_irq_window,
  2795. .update_cr8_intercept = update_cr8_intercept,
  2796. .set_tss_addr = svm_set_tss_addr,
  2797. .get_tdp_level = get_npt_level,
  2798. .get_mt_mask = svm_get_mt_mask,
  2799. .exit_reasons_str = svm_exit_reasons_str,
  2800. .get_lpage_level = svm_get_lpage_level,
  2801. .cpuid_update = svm_cpuid_update,
  2802. .rdtscp_supported = svm_rdtscp_supported,
  2803. .set_supported_cpuid = svm_set_supported_cpuid,
  2804. };
  2805. static int __init svm_init(void)
  2806. {
  2807. return kvm_init(&svm_x86_ops, sizeof(struct vcpu_svm),
  2808. __alignof__(struct vcpu_svm), THIS_MODULE);
  2809. }
  2810. static void __exit svm_exit(void)
  2811. {
  2812. kvm_exit();
  2813. }
  2814. module_init(svm_init)
  2815. module_exit(svm_exit)