diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index f148cb8c9a6e0..09e3f0574b92d 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -2138,6 +2138,38 @@ bool kvm_cpuid(struct kvm_vcpu *vcpu, u32 *eax, u32 *ebx, *eax = vcpu->arch.hw_tsc_khz; } #endif + } else if (pkvm_is_protected_vcpu(vcpu) && function == 0x15) { + /* + * Since protected VMs cannot use kvmclock to find out the + * TSC frequency, expose the native TSC frequency directly + * via cpuid. For now this is not for security (pKVM does not + * provide secure TSC yet) but just to let the guest know the + * TSC frequency so that it doesn't need to calibrate the TSC + * against host-emulated PIT/HPET timers which are inherently + * inaccurate as they are sensitive to the system load in the + * host. + */ + if (tsc_khz) { + /* + * TSC crystal clock must match the LAPIC clock. + * See native_calibrate_tsc(). + */ + *ecx = 1000000000 / vcpu->kvm->arch.apic_bus_cycle_ns; + /* + * Try to minimize calculation error: use 38.4 MHz + * which is the typical TSC crystal clock frequency + * on today's Intel's desktop CPUs. + * + * TODO: we could avoid any calculation error, by + * exposing tsc_khz via some PV (akin to kvmclock) + * instead of cpuid. + */ + *eax = *ecx / 38400000; /* denominator */ + *ebx = tsc_khz / 38400; /* numerator */ + *edx = 0; + } else { + *eax = *ebx = *ecx = *edx = 0; + } } } else { *eax = *ebx = *ecx = *edx = 0; diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 3d341944b5f6d..f3a74971600c8 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -1645,7 +1645,7 @@ static bool __kvm_rmap_zap_gfn_range(struct kvm *kvm, start, end - 1, can_yield, true, flush); } #ifdef CONFIG_PKVM_X86 -static bool pkvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) +static bool __pkvm_unmap_gfn_range(struct kvm *kvm, gfn_t start, gfn_t end) { struct pkvm_mapping *m; @@ -1654,7 +1654,7 @@ static bool pkvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) if (pkvm_is_protected_vm(kvm)) return false; - for_each_pkvm_mapping(kvm, range->start, range->end, m) { + for_each_pkvm_mapping(kvm, start, end, m) { int err = pkvm_hypercall(vm_mmu_unmap, kvm->arch.pkvm.handle, m->gfn << PAGE_SHIFT, m->nr_pages << PAGE_SHIFT); @@ -1669,6 +1669,11 @@ static bool pkvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) return false; } +static bool pkvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) +{ + return __pkvm_unmap_gfn_range(kvm, range->start, range->end); +} + static bool pkvm_age_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range, bool mkold) { @@ -7488,6 +7493,15 @@ static void kvm_mmu_zap_all(struct kvm *kvm) LIST_HEAD(invalid_list); int ign; +#ifdef CONFIG_PKVM_X86 + if (enable_pkvm) { + write_lock(&kvm->mmu_lock); + __pkvm_unmap_gfn_range(kvm, 0, U64_MAX); + write_unlock(&kvm->mmu_lock); + return; + } +#endif + write_lock(&kvm->mmu_lock); restart: list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link) { @@ -7509,10 +7523,6 @@ static void kvm_mmu_zap_all(struct kvm *kvm) void kvm_arch_flush_shadow_all(struct kvm *kvm) { - /* pKVM hypervisor takes care of MMU teardown when destroying VM. */ - if (enable_pkvm) - return; - kvm_mmu_zap_all(kvm); } diff --git a/arch/x86/kvm/pkvm/pkvm.c b/arch/x86/kvm/pkvm/pkvm.c index 953322537c98a..b0e5cb9bce2d8 100644 --- a/arch/x86/kvm/pkvm/pkvm.c +++ b/arch/x86/kvm/pkvm/pkvm.c @@ -163,6 +163,7 @@ static int pkvm_vm_init(phys_addr_t host_kvm_pa, phys_addr_t pkvm_vm_pa, kvm->arch.disabled_quirks = (kvm_caps.inapplicable_quirks | pkvm_vm->shared_kvm->arch.disabled_quirks) & kvm_caps.supported_quirks; + kvm->arch.apic_bus_cycle_ns = APIC_BUS_CYCLE_NS_DEFAULT; kvm->arch.pkvm.pvmfw_load_addr = INVALID_GPA; pkvm_spin_lock_init(&pkvm_vm->lock); @@ -462,6 +463,8 @@ static int __vcpu_create(struct kvm *kvm, struct kvm_vcpu *vcpu, struct fpstate kvm->arch.notify_window = pkvm_vm->shared_kvm->arch.notify_window; kvm->arch.notify_vmexit_flags = pkvm_vm->shared_kvm->arch.notify_vmexit_flags; } + if (pkvm_vm->shared_kvm->arch.apic_bus_cycle_ns) + kvm->arch.apic_bus_cycle_ns = pkvm_vm->shared_kvm->arch.apic_bus_cycle_ns; if (!pkvm_is_protected_vm(kvm)) kvm->arch.disabled_exits = pkvm_vm->shared_kvm->arch.disabled_exits; @@ -1800,7 +1803,8 @@ static int pkvm_vcpu_handle_host_hypercall(struct kvm_vcpu *hvcpu, enum pkvm_hc void pkvm_handle_host_hypercall(struct kvm_vcpu *vcpu) { enum pkvm_hc hc = pkvm_hc(vcpu); - union pkvm_hc_data in, out; + /* Zero 'out' to prevent leaking stack data on error */ + union pkvm_hc_data in, out = {0}; int ret = 0; pkvm_hc_get_input(vcpu, hc, &in); @@ -1930,19 +1934,6 @@ int pkvm_x86_vendor_init(struct kvm_x86_init_ops *ops) memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops)); - if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES)) - kvm_caps.supported_xss = 0; - - if (!kvm_cpu_cap_has(X86_FEATURE_SHSTK) && - !kvm_cpu_cap_has(X86_FEATURE_IBT)) - kvm_caps.supported_xss &= ~XFEATURE_MASK_CET_ALL; - - if ((kvm_caps.supported_xss & XFEATURE_MASK_CET_ALL) != XFEATURE_MASK_CET_ALL) { - kvm_cpu_cap_clear(X86_FEATURE_SHSTK); - kvm_cpu_cap_clear(X86_FEATURE_IBT); - kvm_caps.supported_xss &= ~XFEATURE_MASK_CET_ALL; - } - return 0; } diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 3b215c5b5b01d..d758bff6e068b 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -5285,6 +5285,8 @@ static __init void svm_set_cpu_caps(void) */ kvm_cpu_cap_clear(X86_FEATURE_BUS_LOCK_DETECT); kvm_cpu_cap_clear(X86_FEATURE_MSR_IMM); + + kvm_setup_xss_caps(); } static __init int svm_hardware_setup(void) diff --git a/arch/x86/kvm/vmx/pkvm_host.c b/arch/x86/kvm/vmx/pkvm_host.c index ab301ff460b16..10f81b10a3c71 100644 --- a/arch/x86/kvm/vmx/pkvm_host.c +++ b/arch/x86/kvm/vmx/pkvm_host.c @@ -1629,7 +1629,7 @@ static int pkvm_vcpu_realloc_fpstate(struct kvm_vcpu *vcpu) void *fps; int ret; - fpsize = PAGE_ALIGN(vcpu->arch.guest_fpu.fpstate->size + + fpsize = PAGE_ALIGN(fpu_user_cfg.max_size + ALIGN(offsetof(struct fpstate, regs), 64)); fps = alloc_pages_exact(fpsize, GFP_KERNEL_ACCOUNT); if (!fps) @@ -1663,7 +1663,7 @@ static void pkvm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) * fpstate is only necessary for the pVM, and should be done before * adding the new cpuid entries to the pKVM hypervisor. */ - if ((vcpu->arch.guest_fpu.xfeatures & XFEATURE_MASK_USER_DYNAMIC) && + if ((fpu_user_cfg.max_features & XFEATURE_MASK_USER_DYNAMIC) && pkvm_is_protected_vcpu(vcpu) && pkvm_vcpu_realloc_fpstate(vcpu)) return; diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 554c67a3498c4..5348c94273576 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -8856,6 +8856,8 @@ static __init void vmx_set_cpu_caps(void) kvm_cpu_cap_clear(X86_FEATURE_SHSTK); kvm_cpu_cap_clear(X86_FEATURE_IBT); } + + kvm_setup_xss_caps(); } #ifndef __PKVM_HYP__ diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 015da5732f216..b1170c0aa2c67 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -10234,7 +10234,26 @@ static struct notifier_block pvclock_gtod_notifier = { .notifier_call = pvclock_gtod_notify, }; #endif +#endif /* !__PKVM_HYP__ */ + +void kvm_setup_xss_caps(void) +{ + if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES)) + kvm_caps.supported_xss = 0; + + if (!kvm_cpu_cap_has(X86_FEATURE_SHSTK) && + !kvm_cpu_cap_has(X86_FEATURE_IBT)) + kvm_caps.supported_xss &= ~XFEATURE_MASK_CET_ALL; + + if ((kvm_caps.supported_xss & XFEATURE_MASK_CET_ALL) != XFEATURE_MASK_CET_ALL) { + kvm_cpu_cap_clear(X86_FEATURE_SHSTK); + kvm_cpu_cap_clear(X86_FEATURE_IBT); + kvm_caps.supported_xss &= ~XFEATURE_MASK_CET_ALL; + } +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_setup_xss_caps); +#ifndef __PKVM_HYP__ static inline void kvm_ops_update(struct kvm_x86_init_ops *ops) { memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops)); @@ -10416,19 +10435,6 @@ int kvm_x86_vendor_init(struct kvm_x86_init_ops *ops) if (!tdp_enabled) kvm_caps.supported_quirks &= ~KVM_X86_QUIRK_IGNORE_GUEST_PAT; - if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES)) - kvm_caps.supported_xss = 0; - - if (!kvm_cpu_cap_has(X86_FEATURE_SHSTK) && - !kvm_cpu_cap_has(X86_FEATURE_IBT)) - kvm_caps.supported_xss &= ~XFEATURE_MASK_CET_ALL; - - if ((kvm_caps.supported_xss & XFEATURE_MASK_CET_ALL) != XFEATURE_MASK_CET_ALL) { - kvm_cpu_cap_clear(X86_FEATURE_SHSTK); - kvm_cpu_cap_clear(X86_FEATURE_IBT); - kvm_caps.supported_xss &= ~XFEATURE_MASK_CET_ALL; - } - if (kvm_caps.has_tsc_control) { /* * Make sure the user can only configure tsc_khz values that diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index 8a6dad689b03b..663fed049123a 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -475,6 +475,8 @@ extern struct kvm_host_values kvm_host; extern bool enable_pmu; +void kvm_setup_xss_caps(void); + /* * Get a filtered version of KVM's supported XCR0 that strips out dynamic * features for which the current process doesn't (yet) have permission to use.