In this patch seried, modified kvm selftests code to enable guest code to run in vEL2(As guest Hypervisor). Also added test cases to test guest code booting in vEL2 and register access of VNCR mapped registers.
This patchset is created as per discussions over ml[1]. Posting RFC patch for the early feedback and to further explore requirements and test cases.
Ganapatrao Kulkarni (2): KVM: arm64: nv: selftests: Add guest hypervisor test KVM: arm64: nv: selftests: Access VNCR mapped registers
tools/testing/selftests/kvm/Makefile.kvm | 2 + .../selftests/kvm/arm64/nv_guest_hypervisor.c | 83 ++++++ .../selftests/kvm/arm64/nv_vncr_regs_test.c | 255 ++++++++++++++++++ .../kvm/include/arm64/kvm_util_arch.h | 3 + .../selftests/kvm/include/arm64/nv_util.h | 28 ++ .../testing/selftests/kvm/include/kvm_util.h | 1 + .../selftests/kvm/lib/arm64/processor.c | 59 +++- 7 files changed, 417 insertions(+), 14 deletions(-) create mode 100644 tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c create mode 100644 tools/testing/selftests/kvm/arm64/nv_vncr_regs_test.c create mode 100644 tools/testing/selftests/kvm/include/arm64/nv_util.h
This patch adds the required changes to init vcpu in vEL2 context. Also adds a KVM selftest to execute guest code as a guest hypervisor(L1).
Signed-off-by: Ganapatrao Kulkarni gankulkarni@os.amperecomputing.com --- tools/testing/selftests/kvm/Makefile.kvm | 1 + .../selftests/kvm/arm64/nv_guest_hypervisor.c | 83 +++++++++++++++++++ .../kvm/include/arm64/kvm_util_arch.h | 3 + .../selftests/kvm/include/arm64/nv_util.h | 28 +++++++ .../testing/selftests/kvm/include/kvm_util.h | 1 + .../selftests/kvm/lib/arm64/processor.c | 59 +++++++++---- 6 files changed, 161 insertions(+), 14 deletions(-) create mode 100644 tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c create mode 100644 tools/testing/selftests/kvm/include/arm64/nv_util.h
diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index 4277b983cace..a85d3bec9fb1 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -154,6 +154,7 @@ TEST_GEN_PROGS_arm64 += arm64/vgic_irq TEST_GEN_PROGS_arm64 += arm64/vgic_lpi_stress TEST_GEN_PROGS_arm64 += arm64/vpmu_counter_access TEST_GEN_PROGS_arm64 += arm64/no-vgic-v3 +TEST_GEN_PROGS_arm64 += arm64/nv_guest_hypervisor TEST_GEN_PROGS_arm64 += access_tracking_perf_test TEST_GEN_PROGS_arm64 += arch_timer TEST_GEN_PROGS_arm64 += coalesced_io_test diff --git a/tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c b/tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c new file mode 100644 index 000000000000..5aeefe43aff7 --- /dev/null +++ b/tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2025 Ampere Computing LLC + */ +#include <kvm_util.h> +#include <nv_util.h> +#include <processor.h> +#include <vgic.h> + +static void guest_code(void) +{ + if (read_sysreg(CurrentEL) == CurrentEL_EL2) + GUEST_PRINTF("Executing guest code in vEL2\n"); + else + GUEST_FAIL("Fail to run in vEL2\n"); + + GUEST_DONE(); +} + +static void guest_undef_handler(struct ex_regs *regs) +{ + GUEST_FAIL("Unexpected exception far_el1 = 0x%lx", read_sysreg(far_el1)); +} + +static void test_run_vcpu(struct kvm_vcpu *vcpu) +{ + struct ucall uc; + + do { + vcpu_run(vcpu); + + switch (get_ucall(vcpu, &uc)) { + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + break; + case UCALL_PRINTF: + printf("%s", uc.buffer); + break; + case UCALL_DONE: + printf("Test PASS\n"); + break; + default: + TEST_FAIL("Unknown ucall %lu", uc.cmd); + } + } while (uc.cmd != UCALL_DONE); +} + +static void test_nv_guest_hypervisor(void) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + struct kvm_vcpu_init init; + int gic_fd; + + vm = vm_create(1); + vm_ioctl(vm, KVM_ARM_PREFERRED_TARGET, &init); + + init.features[0] = 0; + init_vcpu_nested(&init); + vcpu = aarch64_vcpu_add(vm, 0, &init, guest_code); + + __TEST_REQUIRE(is_vcpu_nested(vcpu), "Failed to Enable NV"); + + vm_init_descriptor_tables(vm); + vcpu_init_descriptor_tables(vcpu); + gic_fd = vgic_v3_setup(vm, 1, 64); + __TEST_REQUIRE(gic_fd >= 0, "Failed to create vgic-v3"); + + vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT, + ESR_ELx_EC_UNKNOWN, guest_undef_handler); + + test_run_vcpu(vcpu); + kvm_vm_free(vm); +} + +int main(int argc, char *argv[]) +{ + TEST_REQUIRE(kvm_has_cap(KVM_CAP_ARM_EL2)); + + test_nv_guest_hypervisor(); + + return 0; +} diff --git a/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h b/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h index e43a57d99b56..ab5279c24413 100644 --- a/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h +++ b/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h @@ -2,6 +2,9 @@ #ifndef SELFTEST_KVM_UTIL_ARCH_H #define SELFTEST_KVM_UTIL_ARCH_H
+#define CurrentEL_EL1 (1 << 2) +#define CurrentEL_EL2 (2 << 2) + struct kvm_vm_arch {};
#endif // SELFTEST_KVM_UTIL_ARCH_H diff --git a/tools/testing/selftests/kvm/include/arm64/nv_util.h b/tools/testing/selftests/kvm/include/arm64/nv_util.h new file mode 100644 index 000000000000..4fecf1f18554 --- /dev/null +++ b/tools/testing/selftests/kvm/include/arm64/nv_util.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2025 Ampere Computing + */ +#ifndef SELFTEST_NV_UTIL_H +#define SELFTEST_NV_UTIL_H + +#include <linux/bitmap.h> + +/* NV helpers */ +static inline void init_vcpu_nested(struct kvm_vcpu_init *init) +{ + init->features[0] |= (1 << KVM_ARM_VCPU_HAS_EL2); +} + +static inline bool kvm_arm_vcpu_has_el2(struct kvm_vcpu_init *init) +{ + unsigned long features = init->features[0]; + + return test_bit(KVM_ARM_VCPU_HAS_EL2, &features); +} + +static inline bool is_vcpu_nested(struct kvm_vcpu *vcpu) +{ + return vcpu->nested; +} + +#endif /* SELFTEST_NV_UTIL_H */ diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 4c4e5a847f67..8c53dbc17f8f 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -58,6 +58,7 @@ struct kvm_vcpu { struct kvm_dirty_gfn *dirty_gfns; uint32_t fetch_index; uint32_t dirty_gfns_count; + bool nested; };
struct userspace_mem_regions { diff --git a/tools/testing/selftests/kvm/lib/arm64/processor.c b/tools/testing/selftests/kvm/lib/arm64/processor.c index 7ba3aa3755f3..35ba2ace61a2 100644 --- a/tools/testing/selftests/kvm/lib/arm64/processor.c +++ b/tools/testing/selftests/kvm/lib/arm64/processor.c @@ -10,6 +10,7 @@
#include "guest_modes.h" #include "kvm_util.h" +#include "nv_util.h" #include "processor.h" #include "ucall_common.h"
@@ -258,14 +259,47 @@ void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent) } }
+static void aarch64_vcpu_set_reg(struct kvm_vcpu *vcpu, uint64_t sctlr_el1, + uint64_t tcr_el1, uint64_t ttbr0_el1) +{ + uint64_t fpen; + + /* + * Enable FP/ASIMD to avoid trapping when accessing Q0-Q15 + * registers, which the variable argument list macros do. + */ + fpen = 3 << 20; + + if (is_vcpu_nested(vcpu)) { + vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_CPTR_EL2), fpen); + vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL2), sctlr_el1); + vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL2), tcr_el1); + vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_MAIR_EL2), DEFAULT_MAIR_EL1); + vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TTBR0_EL2), ttbr0_el1); + vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TPIDR_EL2), vcpu->id); + } else { + vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_CPACR_EL1), fpen); + vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL1), sctlr_el1); + vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL1), tcr_el1); + vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_MAIR_EL1), DEFAULT_MAIR_EL1); + vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TTBR0_EL1), ttbr0_el1); + vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TPIDR_EL1), vcpu->id); +} +} + void aarch64_vcpu_setup(struct kvm_vcpu *vcpu, struct kvm_vcpu_init *init) { struct kvm_vcpu_init default_init = { .target = -1, }; struct kvm_vm *vm = vcpu->vm; uint64_t sctlr_el1, tcr_el1, ttbr0_el1;
- if (!init) + if (!init) { init = &default_init; + } else { + /* Is this vcpu a Guest-Hypersior */ + if (kvm_arm_vcpu_has_el2(init)) + vcpu->nested = true; + }
if (init->target == -1) { struct kvm_vcpu_init preferred; @@ -275,12 +309,6 @@ void aarch64_vcpu_setup(struct kvm_vcpu *vcpu, struct kvm_vcpu_init *init)
vcpu_ioctl(vcpu, KVM_ARM_VCPU_INIT, init);
- /* - * Enable FP/ASIMD to avoid trapping when accessing Q0-Q15 - * registers, which the variable argument list macros do. - */ - vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_CPACR_EL1), 3 << 20); - sctlr_el1 = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL1)); tcr_el1 = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL1));
@@ -349,11 +377,7 @@ void aarch64_vcpu_setup(struct kvm_vcpu *vcpu, struct kvm_vcpu_init *init) if (use_lpa2_pte_format(vm)) tcr_el1 |= (1ul << 59) /* DS */;
- vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL1), sctlr_el1); - vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL1), tcr_el1); - vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_MAIR_EL1), DEFAULT_MAIR_EL1); - vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TTBR0_EL1), ttbr0_el1); - vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TPIDR_EL1), vcpu->id); + aarch64_vcpu_set_reg(vcpu, sctlr_el1, tcr_el1, ttbr0_el1); }
void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, uint8_t indent) @@ -387,7 +411,11 @@ static struct kvm_vcpu *__aarch64_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id,
aarch64_vcpu_setup(vcpu, init);
- vcpu_set_reg(vcpu, ARM64_CORE_REG(sp_el1), stack_vaddr + stack_size); + if (is_vcpu_nested(vcpu)) + vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SP_EL2), stack_vaddr + stack_size); + else + vcpu_set_reg(vcpu, ARM64_CORE_REG(sp_el1), stack_vaddr + stack_size); + return vcpu; }
@@ -457,7 +485,10 @@ void vcpu_init_descriptor_tables(struct kvm_vcpu *vcpu) { extern char vectors;
- vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_VBAR_EL1), (uint64_t)&vectors); + if (is_vcpu_nested(vcpu)) + vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_VBAR_EL2), (uint64_t)&vectors); + else + vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_VBAR_EL1), (uint64_t)&vectors); }
void route_exception(struct ex_regs *regs, int vector)
On Thu, 06 Feb 2025 16:41:19 +0000, Ganapatrao Kulkarni gankulkarni@os.amperecomputing.com wrote:
This patch adds the required changes to init vcpu in vEL2 context. Also adds a KVM selftest to execute guest code as a guest hypervisor(L1).
Signed-off-by: Ganapatrao Kulkarni gankulkarni@os.amperecomputing.com
tools/testing/selftests/kvm/Makefile.kvm | 1 + .../selftests/kvm/arm64/nv_guest_hypervisor.c | 83 +++++++++++++++++++ .../kvm/include/arm64/kvm_util_arch.h | 3 + .../selftests/kvm/include/arm64/nv_util.h | 28 +++++++ .../testing/selftests/kvm/include/kvm_util.h | 1 + .../selftests/kvm/lib/arm64/processor.c | 59 +++++++++---- 6 files changed, 161 insertions(+), 14 deletions(-) create mode 100644 tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c create mode 100644 tools/testing/selftests/kvm/include/arm64/nv_util.h
diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index 4277b983cace..a85d3bec9fb1 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -154,6 +154,7 @@ TEST_GEN_PROGS_arm64 += arm64/vgic_irq TEST_GEN_PROGS_arm64 += arm64/vgic_lpi_stress TEST_GEN_PROGS_arm64 += arm64/vpmu_counter_access TEST_GEN_PROGS_arm64 += arm64/no-vgic-v3 +TEST_GEN_PROGS_arm64 += arm64/nv_guest_hypervisor TEST_GEN_PROGS_arm64 += access_tracking_perf_test TEST_GEN_PROGS_arm64 += arch_timer TEST_GEN_PROGS_arm64 += coalesced_io_test diff --git a/tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c b/tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c new file mode 100644 index 000000000000..5aeefe43aff7 --- /dev/null +++ b/tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0-only +/*
- Copyright (c) 2025 Ampere Computing LLC
- */
+#include <kvm_util.h> +#include <nv_util.h> +#include <processor.h> +#include <vgic.h>
+static void guest_code(void) +{
- if (read_sysreg(CurrentEL) == CurrentEL_EL2)
GUEST_PRINTF("Executing guest code in vEL2\n");- else
GUEST_FAIL("Fail to run in vEL2\n");- GUEST_DONE();
+}
+static void guest_undef_handler(struct ex_regs *regs) +{
- GUEST_FAIL("Unexpected exception far_el1 = 0x%lx", read_sysreg(far_el1));
+}
+static void test_run_vcpu(struct kvm_vcpu *vcpu) +{
- struct ucall uc;
- do {
vcpu_run(vcpu);switch (get_ucall(vcpu, &uc)) {case UCALL_ABORT:REPORT_GUEST_ASSERT(uc);break;case UCALL_PRINTF:printf("%s", uc.buffer);break;case UCALL_DONE:printf("Test PASS\n");break;default:TEST_FAIL("Unknown ucall %lu", uc.cmd);}- } while (uc.cmd != UCALL_DONE);
+}
+static void test_nv_guest_hypervisor(void) +{
- struct kvm_vcpu *vcpu;
- struct kvm_vm *vm;
- struct kvm_vcpu_init init;
- int gic_fd;
- vm = vm_create(1);
- vm_ioctl(vm, KVM_ARM_PREFERRED_TARGET, &init);
- init.features[0] = 0;
- init_vcpu_nested(&init);
- vcpu = aarch64_vcpu_add(vm, 0, &init, guest_code);
- __TEST_REQUIRE(is_vcpu_nested(vcpu), "Failed to Enable NV");
- vm_init_descriptor_tables(vm);
- vcpu_init_descriptor_tables(vcpu);
- gic_fd = vgic_v3_setup(vm, 1, 64);
- __TEST_REQUIRE(gic_fd >= 0, "Failed to create vgic-v3");
- vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT,
ESR_ELx_EC_UNKNOWN, guest_undef_handler);- test_run_vcpu(vcpu);
- kvm_vm_free(vm);
+}
+int main(int argc, char *argv[]) +{
- TEST_REQUIRE(kvm_has_cap(KVM_CAP_ARM_EL2));
- test_nv_guest_hypervisor();
- return 0;
+} diff --git a/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h b/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h index e43a57d99b56..ab5279c24413 100644 --- a/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h +++ b/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h @@ -2,6 +2,9 @@ #ifndef SELFTEST_KVM_UTIL_ARCH_H #define SELFTEST_KVM_UTIL_ARCH_H +#define CurrentEL_EL1 (1 << 2) +#define CurrentEL_EL2 (2 << 2)
struct kvm_vm_arch {}; #endif // SELFTEST_KVM_UTIL_ARCH_H diff --git a/tools/testing/selftests/kvm/include/arm64/nv_util.h b/tools/testing/selftests/kvm/include/arm64/nv_util.h new file mode 100644 index 000000000000..4fecf1f18554 --- /dev/null +++ b/tools/testing/selftests/kvm/include/arm64/nv_util.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/*
- Copyright (c) 2025 Ampere Computing
- */
+#ifndef SELFTEST_NV_UTIL_H +#define SELFTEST_NV_UTIL_H
+#include <linux/bitmap.h>
+/* NV helpers */ +static inline void init_vcpu_nested(struct kvm_vcpu_init *init) +{
- init->features[0] |= (1 << KVM_ARM_VCPU_HAS_EL2);
+}
+static inline bool kvm_arm_vcpu_has_el2(struct kvm_vcpu_init *init) +{
- unsigned long features = init->features[0];
- return test_bit(KVM_ARM_VCPU_HAS_EL2, &features);
+}
+static inline bool is_vcpu_nested(struct kvm_vcpu *vcpu) +{
- return vcpu->nested;
+}
+#endif /* SELFTEST_NV_UTIL_H */ diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 4c4e5a847f67..8c53dbc17f8f 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -58,6 +58,7 @@ struct kvm_vcpu { struct kvm_dirty_gfn *dirty_gfns; uint32_t fetch_index; uint32_t dirty_gfns_count;
- bool nested;
}; struct userspace_mem_regions { diff --git a/tools/testing/selftests/kvm/lib/arm64/processor.c b/tools/testing/selftests/kvm/lib/arm64/processor.c index 7ba3aa3755f3..35ba2ace61a2 100644 --- a/tools/testing/selftests/kvm/lib/arm64/processor.c +++ b/tools/testing/selftests/kvm/lib/arm64/processor.c @@ -10,6 +10,7 @@ #include "guest_modes.h" #include "kvm_util.h" +#include "nv_util.h" #include "processor.h" #include "ucall_common.h" @@ -258,14 +259,47 @@ void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent) } } +static void aarch64_vcpu_set_reg(struct kvm_vcpu *vcpu, uint64_t sctlr_el1,
uint64_t tcr_el1, uint64_t ttbr0_el1)+{
- uint64_t fpen;
- /*
* Enable FP/ASIMD to avoid trapping when accessing Q0-Q15* registers, which the variable argument list macros do.*/- fpen = 3 << 20;
- if (is_vcpu_nested(vcpu)) {
vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_CPTR_EL2), fpen);vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL2), sctlr_el1);vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL2), tcr_el1);vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_MAIR_EL2), DEFAULT_MAIR_EL1);vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TTBR0_EL2), ttbr0_el1);vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TPIDR_EL2), vcpu->id);
How about some of the basics such as HCR_EL2, MDCR_EL2? A bunch of things there do have an impact on how the guest behaves, and relying on defaults feels like a bad idea.
This also assumes VHE, without trying to enforce it.
Finally, how to you plan to make all the existing tests run as EL2 guests if TPIDR_EL1 isn't populated with the expected value? Surely you need to change the read side...
M.
Hi Marc,
On 07-02-2025 02:44 am, Marc Zyngier wrote:
On Thu, 06 Feb 2025 16:41:19 +0000, Ganapatrao Kulkarni gankulkarni@os.amperecomputing.com wrote:
This patch adds the required changes to init vcpu in vEL2 context. Also adds a KVM selftest to execute guest code as a guest hypervisor(L1).
Signed-off-by: Ganapatrao Kulkarni gankulkarni@os.amperecomputing.com
tools/testing/selftests/kvm/Makefile.kvm | 1 + .../selftests/kvm/arm64/nv_guest_hypervisor.c | 83 +++++++++++++++++++ .../kvm/include/arm64/kvm_util_arch.h | 3 + .../selftests/kvm/include/arm64/nv_util.h | 28 +++++++ .../testing/selftests/kvm/include/kvm_util.h | 1 + .../selftests/kvm/lib/arm64/processor.c | 59 +++++++++---- 6 files changed, 161 insertions(+), 14 deletions(-) create mode 100644 tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c create mode 100644 tools/testing/selftests/kvm/include/arm64/nv_util.h
diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index 4277b983cace..a85d3bec9fb1 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -154,6 +154,7 @@ TEST_GEN_PROGS_arm64 += arm64/vgic_irq TEST_GEN_PROGS_arm64 += arm64/vgic_lpi_stress TEST_GEN_PROGS_arm64 += arm64/vpmu_counter_access TEST_GEN_PROGS_arm64 += arm64/no-vgic-v3 +TEST_GEN_PROGS_arm64 += arm64/nv_guest_hypervisor TEST_GEN_PROGS_arm64 += access_tracking_perf_test TEST_GEN_PROGS_arm64 += arch_timer TEST_GEN_PROGS_arm64 += coalesced_io_test diff --git a/tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c b/tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c new file mode 100644 index 000000000000..5aeefe43aff7 --- /dev/null +++ b/tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0-only +/*
- Copyright (c) 2025 Ampere Computing LLC
- */
+#include <kvm_util.h> +#include <nv_util.h> +#include <processor.h> +#include <vgic.h>
+static void guest_code(void) +{
- if (read_sysreg(CurrentEL) == CurrentEL_EL2)
GUEST_PRINTF("Executing guest code in vEL2\n");- else
GUEST_FAIL("Fail to run in vEL2\n");- GUEST_DONE();
+}
+static void guest_undef_handler(struct ex_regs *regs) +{
- GUEST_FAIL("Unexpected exception far_el1 = 0x%lx", read_sysreg(far_el1));
+}
+static void test_run_vcpu(struct kvm_vcpu *vcpu) +{
- struct ucall uc;
- do {
vcpu_run(vcpu);switch (get_ucall(vcpu, &uc)) {case UCALL_ABORT:REPORT_GUEST_ASSERT(uc);break;case UCALL_PRINTF:printf("%s", uc.buffer);break;case UCALL_DONE:printf("Test PASS\n");break;default:TEST_FAIL("Unknown ucall %lu", uc.cmd);}- } while (uc.cmd != UCALL_DONE);
+}
+static void test_nv_guest_hypervisor(void) +{
- struct kvm_vcpu *vcpu;
- struct kvm_vm *vm;
- struct kvm_vcpu_init init;
- int gic_fd;
- vm = vm_create(1);
- vm_ioctl(vm, KVM_ARM_PREFERRED_TARGET, &init);
- init.features[0] = 0;
- init_vcpu_nested(&init);
- vcpu = aarch64_vcpu_add(vm, 0, &init, guest_code);
- __TEST_REQUIRE(is_vcpu_nested(vcpu), "Failed to Enable NV");
- vm_init_descriptor_tables(vm);
- vcpu_init_descriptor_tables(vcpu);
- gic_fd = vgic_v3_setup(vm, 1, 64);
- __TEST_REQUIRE(gic_fd >= 0, "Failed to create vgic-v3");
- vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT,
ESR_ELx_EC_UNKNOWN, guest_undef_handler);- test_run_vcpu(vcpu);
- kvm_vm_free(vm);
+}
+int main(int argc, char *argv[]) +{
- TEST_REQUIRE(kvm_has_cap(KVM_CAP_ARM_EL2));
- test_nv_guest_hypervisor();
- return 0;
+} diff --git a/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h b/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h index e43a57d99b56..ab5279c24413 100644 --- a/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h +++ b/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h @@ -2,6 +2,9 @@ #ifndef SELFTEST_KVM_UTIL_ARCH_H #define SELFTEST_KVM_UTIL_ARCH_H +#define CurrentEL_EL1 (1 << 2) +#define CurrentEL_EL2 (2 << 2)
- struct kvm_vm_arch {};
#endif // SELFTEST_KVM_UTIL_ARCH_H diff --git a/tools/testing/selftests/kvm/include/arm64/nv_util.h b/tools/testing/selftests/kvm/include/arm64/nv_util.h new file mode 100644 index 000000000000..4fecf1f18554 --- /dev/null +++ b/tools/testing/selftests/kvm/include/arm64/nv_util.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/*
- Copyright (c) 2025 Ampere Computing
- */
+#ifndef SELFTEST_NV_UTIL_H +#define SELFTEST_NV_UTIL_H
+#include <linux/bitmap.h>
+/* NV helpers */ +static inline void init_vcpu_nested(struct kvm_vcpu_init *init) +{
- init->features[0] |= (1 << KVM_ARM_VCPU_HAS_EL2);
+}
+static inline bool kvm_arm_vcpu_has_el2(struct kvm_vcpu_init *init) +{
- unsigned long features = init->features[0];
- return test_bit(KVM_ARM_VCPU_HAS_EL2, &features);
+}
+static inline bool is_vcpu_nested(struct kvm_vcpu *vcpu) +{
- return vcpu->nested;
+}
+#endif /* SELFTEST_NV_UTIL_H */ diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 4c4e5a847f67..8c53dbc17f8f 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -58,6 +58,7 @@ struct kvm_vcpu { struct kvm_dirty_gfn *dirty_gfns; uint32_t fetch_index; uint32_t dirty_gfns_count;
- bool nested; };
struct userspace_mem_regions { diff --git a/tools/testing/selftests/kvm/lib/arm64/processor.c b/tools/testing/selftests/kvm/lib/arm64/processor.c index 7ba3aa3755f3..35ba2ace61a2 100644 --- a/tools/testing/selftests/kvm/lib/arm64/processor.c +++ b/tools/testing/selftests/kvm/lib/arm64/processor.c @@ -10,6 +10,7 @@ #include "guest_modes.h" #include "kvm_util.h" +#include "nv_util.h" #include "processor.h" #include "ucall_common.h" @@ -258,14 +259,47 @@ void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent) } } +static void aarch64_vcpu_set_reg(struct kvm_vcpu *vcpu, uint64_t sctlr_el1,
uint64_t tcr_el1, uint64_t ttbr0_el1)+{
- uint64_t fpen;
- /*
* Enable FP/ASIMD to avoid trapping when accessing Q0-Q15* registers, which the variable argument list macros do.*/- fpen = 3 << 20;
- if (is_vcpu_nested(vcpu)) {
vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_CPTR_EL2), fpen);vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL2), sctlr_el1);vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL2), tcr_el1);vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_MAIR_EL2), DEFAULT_MAIR_EL1);vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TTBR0_EL2), ttbr0_el1);vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TPIDR_EL2), vcpu->id);How about some of the basics such as HCR_EL2, MDCR_EL2? A bunch of things there do have an impact on how the guest behaves, and relying on defaults feels like a bad idea.
Sure, I will try to have these registers also set to required value explicitly.
This also assumes VHE, without trying to enforce it.
Ok, I will try to set specific bits of HCR_EL2 to force it run in VHE.
Finally, how to you plan to make all the existing tests run as EL2 guests if TPIDR_EL1 isn't populated with the expected value? Surely you need to change the read side...
OK, I have not yet tried running existing tests modifying to run as EL2 guests. I will try to run them modifying to run in vEL2.
Should we modify them to run as EL2 guests by default, if the host supports/detected NV? or command line argument based run? either in El1(default) or in EL2?
BTW, I have also ran all existing tests on L1, most of the tests are passing(atleast I did not see any failure prints).
arm64/debug-exceptions is failing on L1 and needs to be debugged/fixed/skipped. arm64/arch_timer_edge_cases fails on both L0 and L1.
On Fri, 07 Feb 2025 13:26:41 +0000, Ganapatrao Kulkarni gankulkarni@os.amperecomputing.com wrote:
- if (is_vcpu_nested(vcpu)) {
vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_CPTR_EL2), fpen);vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL2), sctlr_el1);vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL2), tcr_el1);vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_MAIR_EL2), DEFAULT_MAIR_EL1);vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TTBR0_EL2), ttbr0_el1);vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TPIDR_EL2), vcpu->id);How about some of the basics such as HCR_EL2, MDCR_EL2? A bunch of things there do have an impact on how the guest behaves, and relying on defaults feels like a bad idea.
Sure, I will try to have these registers also set to required value explicitly.
This also assumes VHE, without trying to enforce it.
Ok, I will try to set specific bits of HCR_EL2 to force it run in VHE.
Finally, how to you plan to make all the existing tests run as EL2 guests if TPIDR_EL1 isn't populated with the expected value? Surely you need to change the read side...
OK, I have not yet tried running existing tests modifying to run as EL2 guests. I will try to run them modifying to run in vEL2.
You shouldn't try to modify them. Just make them take a parameter so that the initialisation is done by configuring everything at EL2.
Should we modify them to run as EL2 guests by default, if the host supports/detected NV? or command line argument based run? either in El1(default) or in EL2?
EL1 by default.
BTW, I have also ran all existing tests on L1, most of the tests are passing(atleast I did not see any failure prints).
arm64/debug-exceptions is failing on L1 and needs to be debugged/fixed/skipped. arm64/arch_timer_edge_cases fails on both L0 and L1.
Then I guess you have some work to do to debug these problems, and it once more means that NV is not ready for merging.
M.
On 07-02-2025 07:29 pm, Marc Zyngier wrote:
On Fri, 07 Feb 2025 13:26:41 +0000, Ganapatrao Kulkarni gankulkarni@os.amperecomputing.com wrote:
- if (is_vcpu_nested(vcpu)) {
vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_CPTR_EL2), fpen);vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL2), sctlr_el1);vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL2), tcr_el1);vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_MAIR_EL2), DEFAULT_MAIR_EL1);vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TTBR0_EL2), ttbr0_el1);vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TPIDR_EL2), vcpu->id);How about some of the basics such as HCR_EL2, MDCR_EL2? A bunch of things there do have an impact on how the guest behaves, and relying on defaults feels like a bad idea.
Sure, I will try to have these registers also set to required value explicitly.
This also assumes VHE, without trying to enforce it.
Ok, I will try to set specific bits of HCR_EL2 to force it run in VHE.
Finally, how to you plan to make all the existing tests run as EL2 guests if TPIDR_EL1 isn't populated with the expected value? Surely you need to change the read side...
OK, I have not yet tried running existing tests modifying to run as EL2 guests. I will try to run them modifying to run in vEL2.
You shouldn't try to modify them. Just make them take a parameter so that the initialisation is done by configuring everything at EL2.
OK, make sense, thanks. I will work on one of the test case and post it in the v2.
Should we modify them to run as EL2 guests by default, if the host supports/detected NV? or command line argument based run? either in El1(default) or in EL2?
EL1 by default.
OK.
BTW, I have also ran all existing tests on L1, most of the tests are passing(atleast I did not see any failure prints).
arm64/debug-exceptions is failing on L1 and needs to be debugged/fixed/skipped. arm64/arch_timer_edge_cases fails on both L0 and L1.
Then I guess you have some work to do to debug these problems, and it once more means that NV is not ready for merging.
I will debug and share the findings/fix at the earliest.
On 07-02-2025 10:16 pm, Ganapatrao Kulkarni wrote:
On 07-02-2025 07:29 pm, Marc Zyngier wrote:
On Fri, 07 Feb 2025 13:26:41 +0000, Ganapatrao Kulkarni gankulkarni@os.amperecomputing.com wrote:
+ if (is_vcpu_nested(vcpu)) { + vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_CPTR_EL2), fpen); + vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL2), sctlr_el1); + vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL2), tcr_el1); + vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_MAIR_EL2), DEFAULT_MAIR_EL1); + vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TTBR0_EL2), ttbr0_el1); + vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TPIDR_EL2), vcpu-
id);
How about some of the basics such as HCR_EL2, MDCR_EL2? A bunch of things there do have an impact on how the guest behaves, and relying on defaults feels like a bad idea.
Sure, I will try to have these registers also set to required value explicitly.
This also assumes VHE, without trying to enforce it.
Ok, I will try to set specific bits of HCR_EL2 to force it run in VHE.
Finally, how to you plan to make all the existing tests run as EL2 guests if TPIDR_EL1 isn't populated with the expected value? Surely you need to change the read side...
IIUC, we need not write to TPIDR_EL2, instead write always to TPIDR_EL1 and in guest code(like function cpu_copy_el2regs in Linux kernel) copy TPIDR_EL1 value to TPIDR_EL2.
OR, Write to both TPIDR_EL1 and TPIDR_EL2 in the test code itself during vcpu setup.
With NV2 enabled, some of the EL1/EL2/EL12 register accesses are transformed to memory accesses. This test code accesses all those registers in guest code to validate that they are not trapped to L0.
Signed-off-by: Ganapatrao Kulkarni gankulkarni@os.amperecomputing.com --- tools/testing/selftests/kvm/Makefile.kvm | 1 + .../selftests/kvm/arm64/nv_vncr_regs_test.c | 255 ++++++++++++++++++ 2 files changed, 256 insertions(+) create mode 100644 tools/testing/selftests/kvm/arm64/nv_vncr_regs_test.c
diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index a85d3bec9fb1..7790e4021013 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -155,6 +155,7 @@ TEST_GEN_PROGS_arm64 += arm64/vgic_lpi_stress TEST_GEN_PROGS_arm64 += arm64/vpmu_counter_access TEST_GEN_PROGS_arm64 += arm64/no-vgic-v3 TEST_GEN_PROGS_arm64 += arm64/nv_guest_hypervisor +TEST_GEN_PROGS_arm64 += arm64/nv_vncr_regs_test TEST_GEN_PROGS_arm64 += access_tracking_perf_test TEST_GEN_PROGS_arm64 += arch_timer TEST_GEN_PROGS_arm64 += coalesced_io_test diff --git a/tools/testing/selftests/kvm/arm64/nv_vncr_regs_test.c b/tools/testing/selftests/kvm/arm64/nv_vncr_regs_test.c new file mode 100644 index 000000000000..d05b20b828ff --- /dev/null +++ b/tools/testing/selftests/kvm/arm64/nv_vncr_regs_test.c @@ -0,0 +1,255 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2025 Ampere Computing LLC + * + * This is a test to validate Nested Virtualization. + */ +#include <kvm_util.h> +#include <nv_util.h> +#include <processor.h> +#include <vgic.h> + +#define __check_sr_read(r) \ + ({ \ + uint64_t val; \ + \ + handled = false; \ + dsb(sy); \ + val = read_sysreg_s(SYS_ ## r); \ + val; \ + }) + +#define __check_sr_write(r) \ + do { \ + handled = false; \ + dsb(sy); \ + write_sysreg_s(0, SYS_ ## r); \ + isb(); \ + } while (0) + + +#define check_sr_read(r) \ + do { \ + __check_sr_read(r); \ + __GUEST_ASSERT(!handled, #r "Read Test Failed"); \ + } while (0) + +#define check_sr_write(r) \ + do { \ + __check_sr_write(r); \ + __GUEST_ASSERT(!handled, #r "Write Test Failed"); \ + } while (0) + +#define check_sr_rw(r) \ + do { \ + GUEST_PRINTF("%s\n", #r); \ + check_sr_write(r); \ + check_sr_read(r); \ + } while (0) + +static void test_vncr_mapped_regs(void); +static void regs_test_ich_lr(void); + +static volatile bool handled; + +static void regs_test_ich_lr(void) +{ + int nr_lr, lr; + + nr_lr = (read_sysreg_s(SYS_ICH_VTR_EL2) & 0xf); + + for (lr = 0; lr <= nr_lr; lr++) { + switch (lr) { + case 0: + check_sr_rw(ICH_LR0_EL2); + break; + case 1: + check_sr_rw(ICH_LR1_EL2); + break; + case 2: + check_sr_rw(ICH_LR2_EL2); + break; + case 3: + check_sr_rw(ICH_LR3_EL2); + break; + case 4: + check_sr_rw(ICH_LR4_EL2); + break; + case 5: + check_sr_rw(ICH_LR5_EL2); + break; + case 6: + check_sr_rw(ICH_LR6_EL2); + break; + case 7: + check_sr_rw(ICH_LR7_EL2); + break; + case 8: + check_sr_rw(ICH_LR8_EL2); + break; + case 9: + check_sr_rw(ICH_LR9_EL2); + break; + case 10: + check_sr_rw(ICH_LR10_EL2); + break; + case 11: + check_sr_rw(ICH_LR11_EL2); + break; + case 12: + check_sr_rw(ICH_LR12_EL2); + break; + case 13: + check_sr_rw(ICH_LR13_EL2); + break; + case 14: + check_sr_rw(ICH_LR14_EL2); + break; + case 15: + check_sr_rw(ICH_LR15_EL2); + break; + default: + break; + } + } +} + +/* + * Validate READ/WRITE to VNCR Mapped registers for NV1=0 + */ + +static void test_vncr_mapped_regs(void) +{ + /* + * Access all VNCR Mapped registers, and fail if we get an UNDEF. + */ + + GUEST_PRINTF("VNCR Mapped registers access test:\n"); + check_sr_rw(VTTBR_EL2); + check_sr_rw(VTCR_EL2); + check_sr_rw(VMPIDR_EL2); + check_sr_rw(CNTVOFF_EL2); + check_sr_rw(HCR_EL2); + check_sr_rw(HSTR_EL2); + check_sr_rw(VPIDR_EL2); + check_sr_rw(TPIDR_EL2); + check_sr_rw(VNCR_EL2); + check_sr_rw(CPACR_EL12); + check_sr_rw(CONTEXTIDR_EL12); + check_sr_rw(SCTLR_EL12); + check_sr_rw(ACTLR_EL1); + check_sr_rw(TCR_EL12); + check_sr_rw(AFSR0_EL12); + check_sr_rw(AFSR1_EL12); + check_sr_rw(ESR_EL12); + check_sr_rw(MAIR_EL12); + check_sr_rw(AMAIR_EL12); + check_sr_rw(MDSCR_EL1); + check_sr_rw(SPSR_EL12); + check_sr_rw(CNTV_CVAL_EL02); + check_sr_rw(CNTV_CTL_EL02); + check_sr_rw(CNTP_CVAL_EL02); + check_sr_rw(CNTP_CTL_EL02); + check_sr_rw(HAFGRTR_EL2); + check_sr_rw(TTBR0_EL12); + check_sr_rw(TTBR1_EL12); + check_sr_rw(FAR_EL12); + check_sr_rw(ELR_EL12); + check_sr_rw(SP_EL1); + check_sr_rw(VBAR_EL12); + + regs_test_ich_lr(); + + check_sr_rw(ICH_AP0R0_EL2); + check_sr_rw(ICH_AP1R0_EL2); + check_sr_rw(ICH_HCR_EL2); + check_sr_rw(ICH_VMCR_EL2); + check_sr_rw(VDISR_EL2); + check_sr_rw(MPAM1_EL12); + check_sr_rw(MPAMHCR_EL2); + check_sr_rw(MPAMVPMV_EL2); + check_sr_rw(MPAMVPM0_EL2); + check_sr_rw(MPAMVPM1_EL2); + check_sr_rw(MPAMVPM2_EL2); + check_sr_rw(MPAMVPM3_EL2); + check_sr_rw(MPAMVPM4_EL2); + check_sr_rw(MPAMVPM5_EL2); + check_sr_rw(MPAMVPM6_EL2); + check_sr_rw(MPAMVPM7_EL2); +} + +static void guest_code(void) +{ + if (read_sysreg(CurrentEL) != CurrentEL_EL2) + GUEST_FAIL("Fail to run in vEL2\n"); + + test_vncr_mapped_regs(); + GUEST_DONE(); +} + +static void guest_undef_handler(struct ex_regs *regs) +{ + handled = true; + regs->pc += 4; + GUEST_FAIL("TEST FAIL: register access trap to EL2"); +} + +static void test_run_vcpu(struct kvm_vcpu *vcpu) +{ + struct ucall uc; + + do { + vcpu_run(vcpu); + + switch (get_ucall(vcpu, &uc)) { + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + break; + case UCALL_PRINTF: + printf("%s", uc.buffer); + break; + case UCALL_DONE: + printf("TEST PASS\n"); + break; + default: + TEST_FAIL("Unknown ucall %lu", uc.cmd); + } + } while (uc.cmd != UCALL_DONE); +} + +static void test_nv_vncr(void) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + struct kvm_vcpu_init init; + int gic_fd; + + vm = vm_create(1); + vm_ioctl(vm, KVM_ARM_PREFERRED_TARGET, &init); + + init.features[0] = 0; + init_vcpu_nested(&init); + vcpu = aarch64_vcpu_add(vm, 0, &init, guest_code); + + __TEST_REQUIRE(is_vcpu_nested(vcpu), "Failed to Enable NV"); + + vm_init_descriptor_tables(vm); + vcpu_init_descriptor_tables(vcpu); + gic_fd = vgic_v3_setup(vm, 1, 64); + __TEST_REQUIRE(gic_fd >= 0, "Failed to create vgic-v3"); + + vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT, + ESR_ELx_EC_UNKNOWN, guest_undef_handler); + + test_run_vcpu(vcpu); + kvm_vm_free(vm); +} + +int main(int argc, char *argv[]) +{ + TEST_REQUIRE(kvm_has_cap(KVM_CAP_ARM_EL2)); + + test_nv_vncr(); + + return 0; +}
On Thu, 06 Feb 2025 16:41:20 +0000, Ganapatrao Kulkarni gankulkarni@os.amperecomputing.com wrote:
With NV2 enabled, some of the EL1/EL2/EL12 register accesses are transformed to memory accesses. This test code accesses all those registers in guest code to validate that they are not trapped to L0.
Traps to L0 are invisible to the guest -- by definition. What the guest can observe is an exception, be it injected by L0 or directly delivered by the HW.
But the *absence* of an exception doesn't mean things are fine. Quite the opposite.
Signed-off-by: Ganapatrao Kulkarni gankulkarni@os.amperecomputing.com
tools/testing/selftests/kvm/Makefile.kvm | 1 + .../selftests/kvm/arm64/nv_vncr_regs_test.c | 255 ++++++++++++++++++ 2 files changed, 256 insertions(+) create mode 100644 tools/testing/selftests/kvm/arm64/nv_vncr_regs_test.c
diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index a85d3bec9fb1..7790e4021013 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -155,6 +155,7 @@ TEST_GEN_PROGS_arm64 += arm64/vgic_lpi_stress TEST_GEN_PROGS_arm64 += arm64/vpmu_counter_access TEST_GEN_PROGS_arm64 += arm64/no-vgic-v3 TEST_GEN_PROGS_arm64 += arm64/nv_guest_hypervisor +TEST_GEN_PROGS_arm64 += arm64/nv_vncr_regs_test TEST_GEN_PROGS_arm64 += access_tracking_perf_test TEST_GEN_PROGS_arm64 += arch_timer TEST_GEN_PROGS_arm64 += coalesced_io_test diff --git a/tools/testing/selftests/kvm/arm64/nv_vncr_regs_test.c b/tools/testing/selftests/kvm/arm64/nv_vncr_regs_test.c new file mode 100644 index 000000000000..d05b20b828ff --- /dev/null +++ b/tools/testing/selftests/kvm/arm64/nv_vncr_regs_test.c @@ -0,0 +1,255 @@ +// SPDX-License-Identifier: GPL-2.0-only +/*
- Copyright (c) 2025 Ampere Computing LLC
- This is a test to validate Nested Virtualization.
- */
+#include <kvm_util.h> +#include <nv_util.h> +#include <processor.h> +#include <vgic.h>
+#define __check_sr_read(r) \
- ({ \
uint64_t val; \\handled = false; \dsb(sy); \val = read_sysreg_s(SYS_ ## r); \val; \- })
+#define __check_sr_write(r) \
- do { \
handled = false; \dsb(sy); \write_sysreg_s(0, SYS_ ## r); \isb(); \- } while (0)
+#define check_sr_read(r) \
- do { \
__check_sr_read(r); \__GUEST_ASSERT(!handled, #r "Read Test Failed"); \- } while (0)
+#define check_sr_write(r) \
- do { \
__check_sr_write(r); \__GUEST_ASSERT(!handled, #r "Write Test Failed"); \- } while (0)
+#define check_sr_rw(r) \
- do { \
GUEST_PRINTF("%s\n", #r); \check_sr_write(r); \check_sr_read(r); \- } while (0)
Instead of lifting things from existing tests, you could move these things to an include file for everybody's benefit.
+static void test_vncr_mapped_regs(void); +static void regs_test_ich_lr(void);
+static volatile bool handled;
+static void regs_test_ich_lr(void) +{
- int nr_lr, lr;
- nr_lr = (read_sysreg_s(SYS_ICH_VTR_EL2) & 0xf);
- for (lr = 0; lr <= nr_lr; lr++) {
switch (lr) {case 0:check_sr_rw(ICH_LR0_EL2);break;case 1:check_sr_rw(ICH_LR1_EL2);break;case 2:check_sr_rw(ICH_LR2_EL2);break;case 3:check_sr_rw(ICH_LR3_EL2);break;case 4:check_sr_rw(ICH_LR4_EL2);break;case 5:check_sr_rw(ICH_LR5_EL2);break;case 6:check_sr_rw(ICH_LR6_EL2);break;case 7:check_sr_rw(ICH_LR7_EL2);break;case 8:check_sr_rw(ICH_LR8_EL2);break;case 9:check_sr_rw(ICH_LR9_EL2);break;case 10:check_sr_rw(ICH_LR10_EL2);break;case 11:check_sr_rw(ICH_LR11_EL2);break;case 12:check_sr_rw(ICH_LR12_EL2);break;case 13:check_sr_rw(ICH_LR13_EL2);break;case 14:check_sr_rw(ICH_LR14_EL2);break;case 15:check_sr_rw(ICH_LR15_EL2);break;default:break;}- }
+}
+/*
- Validate READ/WRITE to VNCR Mapped registers for NV1=0
- */
+static void test_vncr_mapped_regs(void) +{
- /*
* Access all VNCR Mapped registers, and fail if we get an UNDEF.*/
No. You are accessing a lot of random registers, irrespective of the configuration exposed to the guest. Being able to access it is not an indication of correctness. Seeing it UNDEF is not an indication of a bug.
Also, this is supposed to be an EL2 test that doesn't deal with NV *itself*. There is no VNCR page in this test. The fact that the host uses VNCR as a way to virtualise EL2 has nothing to do with the test at all.
- GUEST_PRINTF("VNCR Mapped registers access test:\n");
- check_sr_rw(VTTBR_EL2);
- check_sr_rw(VTCR_EL2);
- check_sr_rw(VMPIDR_EL2);
- check_sr_rw(CNTVOFF_EL2);
- check_sr_rw(HCR_EL2);
- check_sr_rw(HSTR_EL2);
- check_sr_rw(VPIDR_EL2);
- check_sr_rw(TPIDR_EL2);
- check_sr_rw(VNCR_EL2);
- check_sr_rw(CPACR_EL12);
- check_sr_rw(CONTEXTIDR_EL12);
- check_sr_rw(SCTLR_EL12);
- check_sr_rw(ACTLR_EL1);
- check_sr_rw(TCR_EL12);
- check_sr_rw(AFSR0_EL12);
- check_sr_rw(AFSR1_EL12);
- check_sr_rw(ESR_EL12);
- check_sr_rw(MAIR_EL12);
- check_sr_rw(AMAIR_EL12);
- check_sr_rw(MDSCR_EL1);
- check_sr_rw(SPSR_EL12);
- check_sr_rw(CNTV_CVAL_EL02);
- check_sr_rw(CNTV_CTL_EL02);
- check_sr_rw(CNTP_CVAL_EL02);
- check_sr_rw(CNTP_CTL_EL02);
- check_sr_rw(HAFGRTR_EL2);
- check_sr_rw(TTBR0_EL12);
- check_sr_rw(TTBR1_EL12);
- check_sr_rw(FAR_EL12);
- check_sr_rw(ELR_EL12);
- check_sr_rw(SP_EL1);
- check_sr_rw(VBAR_EL12);
- regs_test_ich_lr();
- check_sr_rw(ICH_AP0R0_EL2);
- check_sr_rw(ICH_AP1R0_EL2);
- check_sr_rw(ICH_HCR_EL2);
- check_sr_rw(ICH_VMCR_EL2);
- check_sr_rw(VDISR_EL2);
This should absolutely UNDEF in the absence of FEAT_RAS exposed to the guest. And yet it won't. Why? That's because the architecture doesn't allow this to be trapped. Does it mean being able to access it is right? Absolutely not. This is an *architecture* bug.
- check_sr_rw(MPAM1_EL12);
This should UNDEF in the test if the configuration doesn't expose MPAM to the guest. And I really hope it explodes or this is a glaring bug, because MPAM should never be exposed to a guest.
The conclusion is that blindly testing that you can R/W registers buys us nothing if you are not checking the validity of the access against the architecture rules.
Any test should take as input:
- the configuration of the guest
- the expected outcome of the access for that particular configuration and trap configuration, as described in the ARM ARM pseudocode (or, even better, in the BSD-licensed JSON file that has all that information)
The result of the access must then be matched against these inputs and any discrepancy reported, either as fatal (because the outcome of the access wasn't the expected one) or as an expected, non-fatal failure (because we know that the architecture is not self-consistent).
Yes, this looks a lot like a full CPU validation suite. What a surprise.
M.
On 06-02-2025 10:11 pm, Ganapatrao Kulkarni wrote:
In this patch seried, modified kvm selftests code to enable guest code to run in vEL2(As guest Hypervisor). Also added test cases to test guest code booting in vEL2 and register access of VNCR mapped registers.
This patchset is created as per discussions over ml[1]. Posting RFC patch for the early feedback and to further explore requirements and test cases.
[1] https://lore.kernel.org/linux-arm-kernel/871pz2th4b.wl-maz@kernel.org/
Ganapatrao Kulkarni (2): KVM: arm64: nv: selftests: Add guest hypervisor test KVM: arm64: nv: selftests: Access VNCR mapped registers
tools/testing/selftests/kvm/Makefile.kvm | 2 + .../selftests/kvm/arm64/nv_guest_hypervisor.c | 83 ++++++ .../selftests/kvm/arm64/nv_vncr_regs_test.c | 255 ++++++++++++++++++ .../kvm/include/arm64/kvm_util_arch.h | 3 + .../selftests/kvm/include/arm64/nv_util.h | 28 ++ .../testing/selftests/kvm/include/kvm_util.h | 1 + .../selftests/kvm/lib/arm64/processor.c | 59 +++- 7 files changed, 417 insertions(+), 14 deletions(-) create mode 100644 tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c create mode 100644 tools/testing/selftests/kvm/arm64/nv_vncr_regs_test.c create mode 100644 tools/testing/selftests/kvm/include/arm64/nv_util.h
linux-kselftest-mirror@lists.linaro.org
-
Ganapatrao Kulkarni -
Marc Zyngier