Skip to content

KVM: x86: pKVM: align AMX CPUID with host-provided model #88

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
i-yyi wants to merge 1 commit into
base: pkvm-v6.18
Choose a base branch
from
Open

KVM: x86: pKVM: align AMX CPUID with host-provided model #88

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
137 changes: 134 additions & 3 deletions arch/x86/kvm/cpuid.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2303,8 +2303,130 @@ static void pkvm_fixup_cpuid_entry(struct kvm_cpuid_entry2 *entry)
#define CPUID_1_ECX_TSC_DLTIMER (1 << 24)
#define CPUID_1_ECX_HYP (1 << 31)
#define CPUID_1_EDX_HTT (1 << 28)
static bool cpuid_entry_is_empty(struct kvm_cpuid_entry2 *e2);

static bool pkvm_cpuid_entry_has_amx(struct kvm_cpuid_entry2 *entry)
{
u64 xfeatures;

switch (entry->function) {
case 7:
if (!entry->index)
return cpuid_entry_has(entry, X86_FEATURE_AMX_TILE) ||
cpuid_entry_has(entry, X86_FEATURE_AMX_INT8) ||
cpuid_entry_has(entry, X86_FEATURE_AMX_BF16);
if (entry->index == 1)
return cpuid_entry_has(entry, X86_FEATURE_AMX_FP16) ||
cpuid_entry_has(entry, X86_FEATURE_AMX_COMPLEX);
return false;
case 0xd:
if (entry->index == 0) {
xfeatures = entry->eax | ((u64)entry->edx << 32);
return xfeatures & XFEATURE_MASK_XTILE;
}

return entry->index == XFEATURE_XTILE_CFG ||
entry->index == XFEATURE_XTILE_DATA;
case 0x1d:
case 0x1e:
return entry->function || entry->eax || entry->ebx ||
entry->ecx || entry->edx;
default:
return false;
}
}

static bool pkvm_cpuid_has_amx(struct kvm_cpuid_entry2 *e2, int nent)
{
int i;

for (i = 0; i < nent; i++) {
if (cpuid_entry_is_empty(&e2[i]))
continue;
if (pkvm_cpuid_entry_has_amx(&e2[i]))
return true;
}

return false;
}

static void pkvm_cpuid_align_amx(struct kvm_cpuid_entry2 *entry,
struct kvm_cpuid_entry2 *host,
bool has_amx)
{
u64 xfeatures;

switch (entry->function) {
case 7:
if (!entry->index) {
bool keep_tile = host && cpuid_entry_has(host, X86_FEATURE_AMX_TILE);
bool keep_int8 = host && cpuid_entry_has(host, X86_FEATURE_AMX_INT8);
bool keep_bf16 = host && cpuid_entry_has(host, X86_FEATURE_AMX_BF16);

cpuid_entry_change(entry, X86_FEATURE_AMX_TILE, keep_tile);
cpuid_entry_change(entry, X86_FEATURE_AMX_INT8, keep_int8);
cpuid_entry_change(entry, X86_FEATURE_AMX_BF16, keep_bf16);
} else if (entry->index == 1) {
bool keep_fp16 = host && cpuid_entry_has(host, X86_FEATURE_AMX_FP16);
bool keep_complex = host && cpuid_entry_has(host, X86_FEATURE_AMX_COMPLEX);

cpuid_entry_change(entry, X86_FEATURE_AMX_FP16, keep_fp16);
cpuid_entry_change(entry, X86_FEATURE_AMX_COMPLEX, keep_complex);
}
break;
case 0xd:
if (!has_amx) {
if (entry->index == 0) {
xfeatures = entry->eax | ((u64)entry->edx << 32);
xfeatures &= ~XFEATURE_MASK_XTILE;
entry->eax = xfeatures;
entry->edx = xfeatures >> 32;
entry->ebx = xstate_required_size(xfeatures, false);
entry->ecx = entry->ebx;
} else if (entry->index == XFEATURE_XTILE_CFG ||
entry->index == XFEATURE_XTILE_DATA) {
memset(entry, 0, sizeof(*entry));
}
}
break;
case 0x1d:
case 0x1e:
if (!has_amx)
memset(entry, 0, sizeof(*entry));
break;
default:
break;
}
}

static void pkvm_refresh_xstate_cpuid(struct kvm_cpuid_entry2 *e2, int nent)
{
struct kvm_cpuid_entry2 *d0, *d1;
u64 xcr0, xss;

d0 = kvm_find_cpuid_entry2(e2, nent, 0xd, 0);
if (!d0)
return;

xcr0 = d0->eax | ((u64)d0->edx << 32);
d0->ebx = xstate_required_size(xcr0, false);
d0->ecx = d0->ebx;

d1 = kvm_find_cpuid_entry2(e2, nent, 0xd, 1);
if (!d1)
return;

xss = d1->ecx | ((u64)d1->edx << 32);
if (cpuid_entry_has(d1, X86_FEATURE_XSAVES) ||
cpuid_entry_has(d1, X86_FEATURE_XSAVEC))
d1->ebx = xstate_required_size(xcr0 | xss, true);
else
d1->ebx = 0;
}

static void pkvm_enforce_cpuid_entry(struct kvm_cpuid_entry2 *entry,
struct kvm_cpuid_entry2 *def)
struct kvm_cpuid_entry2 *def,
bool has_amx)
{
struct kvm_cpuid_entry2 tmp = *def;

Expand All @@ -2323,6 +2445,7 @@ static void pkvm_enforce_cpuid_entry(struct kvm_cpuid_entry2 *entry,
break;
}

pkvm_cpuid_align_amx(&tmp, entry, has_amx);
*entry = tmp;
}

Expand Down Expand Up @@ -2385,6 +2508,7 @@ int pkvm_enforce_cpuid(struct kvm_cpuid_entry2 *e2, int *nent, int max_nent)
int def_nent, r, i, n;
int orig_nent = *nent;
bool has_func4 = false;
bool has_amx = pkvm_cpuid_has_amx(e2, orig_nent);

memset(de2, 0, KVM_MAX_CPUID_ENTRIES * sizeof(struct kvm_cpuid_entry2));
def_nent = KVM_MAX_CPUID_ENTRIES;
Expand Down Expand Up @@ -2416,7 +2540,7 @@ int pkvm_enforce_cpuid(struct kvm_cpuid_entry2 *e2, int *nent, int max_nent)

tmp = find_cpuid_entry(de2, def_nent, &e2[i]);
if (tmp)
pkvm_enforce_cpuid_entry(&e2[i], tmp);
pkvm_enforce_cpuid_entry(&e2[i], tmp, has_amx);
else
memset(&e2[i], 0, sizeof(struct kvm_cpuid_entry2));
}
Expand Down Expand Up @@ -2448,12 +2572,19 @@ int pkvm_enforce_cpuid(struct kvm_cpuid_entry2 *e2, int *nent, int max_nent)
if (n == max_nent)
return -ENOSPC;

e2[n++] = de2[i];
e2[n] = de2[i];
pkvm_cpuid_align_amx(&e2[n], NULL, has_amx);
if (cpuid_entry_is_empty(&e2[n]))
continue;
n++;
}

if (n > orig_nent)
*nent = n;

if (!has_amx)
pkvm_refresh_xstate_cpuid(e2, *nent);

/* Apply fixed values to the final set of entries */
for (i = 0; i < *nent; i++) {
if (cpuid_entry_is_empty(&e2[i]) ||
Expand Down
73 changes: 63 additions & 10 deletions arch/x86/kvm/vmx/pkvm_host.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1622,15 +1622,71 @@ static void pkvm_get_entry_info(struct kvm_vcpu *vcpu, u32 *intr_info, u32 *erro
*error_code = 0;
}

static bool pkvm_cpuid_entry_has_amx(struct kvm_cpuid_entry2 *entry)
{
u64 xfeatures;

switch (entry->function) {
case 7:
if (!entry->index)
return cpuid_entry_has(entry, X86_FEATURE_AMX_TILE) ||
cpuid_entry_has(entry, X86_FEATURE_AMX_INT8) ||
cpuid_entry_has(entry, X86_FEATURE_AMX_BF16);
if (entry->index == 1)
return cpuid_entry_has(entry, X86_FEATURE_AMX_FP16) ||
cpuid_entry_has(entry, X86_FEATURE_AMX_COMPLEX);
return false;
case 0xd:
if (entry->index == 0) {
xfeatures = entry->eax | ((u64)entry->edx << 32);
return xfeatures & XFEATURE_MASK_XTILE;
}

return entry->index == XFEATURE_XTILE_CFG ||
entry->index == XFEATURE_XTILE_DATA;
case 0x1d:
case 0x1e:
return entry->function || entry->eax || entry->ebx ||
entry->ecx || entry->edx;
default:
return false;
}
}

static bool pkvm_cpuid_has_amx(struct kvm_cpuid_entry2 *e2, int nent)
{
int i;

for (i = 0; i < nent; i++) {
if (!e2[i].function && !e2[i].eax)
continue;
if (pkvm_cpuid_entry_has_amx(&e2[i]))
return true;
}

return false;
}

static size_t pkvm_vcpu_fpstate_size(struct kvm_vcpu *vcpu)
{
size_t size = vcpu->arch.guest_fpu.fpstate->size;

if (pkvm_is_protected_vcpu(vcpu) &&
(kvm_caps.supported_xcr0 & XFEATURE_MASK_USER_DYNAMIC))
size = max_t(size_t, size,
xstate_required_size(kvm_caps.supported_xcr0, false));

return PAGE_ALIGN(size + ALIGN(offsetof(struct fpstate, regs), 64));
}

static int pkvm_vcpu_realloc_fpstate(struct kvm_vcpu *vcpu)
{
union pkvm_hc_data out;
size_t fpsize;
void *fps;
int ret;

fpsize = PAGE_ALIGN(vcpu->arch.guest_fpu.fpstate->size +
ALIGN(offsetof(struct fpstate, regs), 64));
fpsize = pkvm_vcpu_fpstate_size(vcpu);
fps = alloc_pages_exact(fpsize, GFP_KERNEL_ACCOUNT);
if (!fps)
return -ENOMEM;
Expand All @@ -1656,15 +1712,12 @@ static void pkvm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
return;

/*
* With exposing the FPU dynamic feature via the cpuid, the fpstate
* allocated when creating the vcpu may not be sufficient for the
* guest. As the pVM's FPU state is managed by the pKVM hypervisor
* while the npVM's FPU state is managed by the host, re-allocating the
* fpstate is only necessary for the pVM, and should be done before
* adding the new cpuid entries to the pKVM hypervisor.
* pKVM may append AMX-related CPUID state when the host/VMM chooses to
* expose AMX to the pVM. Size the donated fpstate buffer up front for
* that case before synchronizing CPUID into the hypervisor.
*/
if ((vcpu->arch.guest_fpu.xfeatures & XFEATURE_MASK_USER_DYNAMIC) &&
pkvm_is_protected_vcpu(vcpu) &&
if (pkvm_is_protected_vcpu(vcpu) &&
pkvm_cpuid_has_amx(e2, nent) &&
pkvm_vcpu_realloc_fpstate(vcpu))
return;

Expand Down