Port UPM tests from rfc v2 series posted here: https://lore.kernel.org/lkml/20220511000811.384766-1-vannapurve@google.com/T... to execute with SEV/SEV-ES VMs.
Major changes from original series: 1) SEV/SEV-ES VM creation logic is hooked into setup_and_execute_test. 2) Shared ucall is used for communication between vmm and the guest. 3) C bit toggle logic is added at places where memory conversion is needed. 4) memory size is passed via vcpu registers instead of MMIO.
Example invocation on AMD SEV/SEV-ES capable machine: ./sev_priv_memfd_test -> Runs UPM selftests with SEV VMs ./sev_priv_memfd_test -e -> Runs UPM selftests with SEV-ES VMs
Signed-off-by: Vishal Annapurve vannapurve@google.com --- tools/testing/selftests/kvm/.gitignore | 1 + tools/testing/selftests/kvm/Makefile | 1 + .../kvm/x86_64/sev_priv_memfd_test.c | 1511 +++++++++++++++++ 3 files changed, 1513 insertions(+) create mode 100644 tools/testing/selftests/kvm/x86_64/sev_priv_memfd_test.c
diff --git a/tools/testing/selftests/kvm/.gitignore b/tools/testing/selftests/kvm/.gitignore index 8d4fda1ace8f..3cd4a6678663 100644 --- a/tools/testing/selftests/kvm/.gitignore +++ b/tools/testing/selftests/kvm/.gitignore @@ -56,6 +56,7 @@ /x86_64/xss_msr_test /x86_64/vmx_pmu_msrs_test /x86_64/sev_all_boot_test +/x86_64/sev_priv_memfd_test /access_tracking_perf_test /demand_paging_test /dirty_log_test diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index 6d269c3159bf..1840f6a4c6f5 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -94,6 +94,7 @@ TEST_GEN_PROGS_x86_64 += x86_64/xen_shinfo_test TEST_GEN_PROGS_x86_64 += x86_64/xen_vmcall_test TEST_GEN_PROGS_x86_64 += x86_64/sev_migrate_tests TEST_GEN_PROGS_x86_64 += x86_64/sev_all_boot_test +TEST_GEN_PROGS_x86_64 += x86_64/sev_priv_memfd_test TEST_GEN_PROGS_x86_64 += demand_paging_test TEST_GEN_PROGS_x86_64 += dirty_log_test TEST_GEN_PROGS_x86_64 += dirty_log_perf_test diff --git a/tools/testing/selftests/kvm/x86_64/sev_priv_memfd_test.c b/tools/testing/selftests/kvm/x86_64/sev_priv_memfd_test.c new file mode 100644 index 000000000000..9255a6a3ce41 --- /dev/null +++ b/tools/testing/selftests/kvm/x86_64/sev_priv_memfd_test.c @@ -0,0 +1,1511 @@ +// SPDX-License-Identifier: GPL-2.0 +#define _GNU_SOURCE /* for program_invocation_short_name */ +#include <fcntl.h> +#include <limits.h> +#include <sched.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/ioctl.h> + +#include <linux/compiler.h> +#include <linux/kernel.h> +#include <linux/kvm_para.h> +#include <linux/memfd.h> + +#include <test_util.h> +#include <kvm_util.h> +#include <processor.h> + +#include <linux/psp-sev.h> +#include <sev.h> +#include <sev_exitlib.h> + +#define BYTE_MASK 0xFF + +// flags for mmap +#define MAP_HUGE_2MB (21 << MAP_HUGE_SHIFT) +#define MAP_HUGE_1GB (30 << MAP_HUGE_SHIFT) + +// page sizes +#define PAGE_SIZE_4KB ((size_t)0x1000) +#define PAGE_SIZE_2MB (PAGE_SIZE_4KB * (size_t)512) +#define PAGE_SIZE_1GB ((PAGE_SIZE_4KB * 256) * 1024) + +#define TEST_MEM_GPA 0xb0000000 +#define TEST_MEM_DATA_PAT1 0x6666666666666666 +#define TEST_MEM_DATA_PAT2 0x9999999999999999 +#define TEST_MEM_DATA_PAT3 0x3333333333333333 +#define TEST_MEM_DATA_PAT4 0xaaaaaaaaaaaaaaaa + +#define TOTAL_PAGES (1024) +#define GUEST_PGT_MIN_VADDR 0x10000 + +enum mem_op { + SET_PAT, + VERIFY_PAT +}; + +#define TEST_MEM_SLOT 10 + +#define VCPU_ID 0 + +#define VM_STAGE_PROCESSED(x) pr_info("Processed stage %s\n", #x) + +// global used for storing the current mem allocation size +// for the running test +static size_t test_mem_size; + +typedef bool (*vm_stage_handler_fn)(struct kvm_vm *, struct ucall *, void *, + uint64_t); + +static uint64_t g_ghcb_gpa; +static void *g_ghcb_gva; + +/* guest code function will accept 4 arguments: + * - pointer to shared ucall struct + * - encryption bit shift + * - pointer to guest_pgt_info struct + * - size of the test memory buffer + * - guest physical address of ghcb page + * - guest virtual address of ghcb page + */ +typedef void (*guest_code_fn)(struct ucall *, uint8_t, + struct guest_pgt_info *, uint64_t, uint64_t, void *); + +struct test_run_helper { + char *test_desc; + vm_stage_handler_fn vmst_handler; + guest_code_fn guest_fn; + void *shared_mem; + int priv_memfd; + bool disallow_boot_shared_access; + bool toggle_shared_mem_state; +}; + +enum page_size { + PAGE_4KB, + PAGE_2MB, + PAGE_1GB +}; + +struct page_combo { + enum page_size shared; + enum page_size private; +}; + +static char *page_size_to_str(enum page_size x) +{ + switch (x) { + case PAGE_4KB: + return "PAGE_4KB"; + case PAGE_2MB: + return "PAGE_2MB"; + case PAGE_1GB: + return "PAGE_1GB"; + default: + return "UNKNOWN"; + } +} + +static uint64_t test_mem_end(const uint64_t start, const uint64_t size) +{ + return start + size; +} + +/* Guest code in selftests is loaded to guest memory using kvm_vm_elf_load + * which doesn't handle global offset table updates. Calling standard libc + * functions would normally result in referring to the global offset table. + * Adding O1 here seems to prohibit compiler from replacing the memory + * operations with standard libc functions such as memset. + */ +static bool __attribute__((optimize("O1"))) do_mem_op(enum mem_op op, void *mem, + uint64_t pat, uint32_t size) +{ + uint64_t *buf = (uint64_t *)mem; + uint32_t chunk_size = sizeof(pat); + uint64_t mem_addr = (uint64_t)mem; + + if (((mem_addr % chunk_size) != 0) || ((size % chunk_size) != 0)) + return false; + + for (uint32_t i = 0; i < (size / chunk_size); i++) { + if (op == SET_PAT) + buf[i] = pat; + if (op == VERIFY_PAT) { + if (buf[i] != pat) + return false; + } + } + + return true; +} + +static inline uint64_t guest_hypercall(uint64_t nr, uint64_t a0, uint64_t a1, + uint64_t a2, uint64_t a3) +{ + uint64_t r; + + asm volatile("vmmcall" + : "=a"(r) + : "a"(nr), "b"(a0), "c"(a1), "d"(a2), "S"(a3)); + return r; +} + +static void guest_set_clr_pte_bit(struct ucall *uc, + struct guest_pgt_info *gpgt_info, uint64_t vaddr_start, + uint64_t mem_size, bool set, uint32_t bit) +{ + uint64_t vaddr = vaddr_start; + uint32_t guest_page_size = gpgt_info->page_size; + uint32_t num_pages; + + GUEST_SHARED_ASSERT(uc, !(mem_size % guest_page_size)); + num_pages = mem_size / guest_page_size; + for (uint32_t i = 0; i < num_pages; i++) { + uint64_t *pte = guest_code_get_pte(gpgt_info, vaddr); + + GUEST_SHARED_ASSERT(uc, pte); + if (set) + *pte |= (1ULL << bit); + else + *pte &= ~(1ULL << bit); + asm volatile("invlpg (%0)" :: "r"(vaddr) : "memory"); + vaddr += guest_page_size; + } +} + +static void guest_verify_sev_vm_boot(struct ucall *uc, bool sev_es) +{ + uint32_t eax, ebx, ecx, edx; + uint64_t sev_status; + + /* Check CPUID values via GHCB MSR protocol. */ + eax = 0x8000001f; + ecx = 0; + cpuid(&eax, &ebx, &ecx, &edx); + + /* Check SEV bit. */ + GUEST_SHARED_ASSERT(uc, eax & (1 << 1)); + + /* Check SEV-ES bit. */ + if (sev_es) + GUEST_SHARED_ASSERT(uc, eax & (1 << 3)); + + /* Check SEV and SEV-ES enabled bits (bits 0 and 1, respectively). */ + sev_status = rdmsr(MSR_AMD64_SEV); + GUEST_SHARED_ASSERT(uc, (sev_status & 0x1) == 1); + + if (sev_es) + GUEST_SHARED_ASSERT(uc, (sev_status & 0x2) == 2); +} + +/* Test to verify guest private accesses on private memory with following steps: + * 1) Upon entry, guest signals VMM that it has started. + * 2) VMM populates the shared memory with known pattern and continues guest + * execution. + * 3) Guest writes a different pattern on the private memory and signals VMM + * that it has updated private memory. + * 4) VMM verifies its shared memory contents to be same as the data populated + * in step 2 and continues guest execution. + * 5) Guest verifies its private memory contents to be same as the data + * populated in step 3 and marks the end of the guest execution. + */ +#define PMPAT_ID 0 +#define PMPAT_DESC "PrivateMemoryPrivateAccessTest" + +/* Guest code execution stages for private mem access test */ +#define PMPAT_GUEST_STARTED 0ULL +#define PMPAT_GUEST_PRIV_MEM_UPDATED 1ULL + +static bool pmpat_handle_vm_stage(struct kvm_vm *vm, struct ucall *uc, + void *test_info, uint64_t stage) +{ + void *shared_mem = ((struct test_run_helper *)test_info)->shared_mem; + + switch (stage) { + case PMPAT_GUEST_STARTED: { + /* Initialize the contents of shared memory */ + TEST_ASSERT(do_mem_op(SET_PAT, shared_mem, + TEST_MEM_DATA_PAT1, test_mem_size), + "Shared memory update failure"); + VM_STAGE_PROCESSED(PMPAT_GUEST_STARTED); + break; + } + case PMPAT_GUEST_PRIV_MEM_UPDATED: { + /* verify host updated data is still intact */ + TEST_ASSERT(do_mem_op(VERIFY_PAT, shared_mem, + TEST_MEM_DATA_PAT1, test_mem_size), + "Shared memory view mismatch"); + VM_STAGE_PROCESSED(PMPAT_GUEST_PRIV_MEM_UPDATED); + break; + } + default: + TEST_FAIL("Unhandled VM stage %ld\n", stage); + return false; + } + + return true; +} + +static void pmpat_guest_code(struct ucall *uc, uint8_t enc_bit_shift, + struct guest_pgt_info *gpgt_info, uint64_t mem_size, uint64_t ghcb_gpa, + void *ghcb_gva) +{ + void *priv_mem = (void *)TEST_MEM_GPA; + int ret; + + g_ghcb_gva = ghcb_gva; + g_ghcb_gpa = ghcb_gpa; + GUEST_SHARED_SYNC(uc, PMPAT_GUEST_STARTED); + guest_verify_sev_vm_boot(uc, g_ghcb_gva != NULL); + + guest_set_clr_pte_bit(uc, gpgt_info, TEST_MEM_GPA, mem_size, true, + enc_bit_shift); + /* Mark the GPA range to be treated as always accessed privately */ + ret = guest_hypercall(KVM_HC_MAP_GPA_RANGE, TEST_MEM_GPA, + mem_size >> MIN_PAGE_SHIFT, KVM_MARK_GPA_RANGE_ENC_ACCESS, 0); + GUEST_SHARED_ASSERT_1(uc, ret == 0, ret); + + GUEST_SHARED_ASSERT(uc, do_mem_op(SET_PAT, priv_mem, TEST_MEM_DATA_PAT2, + mem_size)); + GUEST_SHARED_SYNC(uc, PMPAT_GUEST_PRIV_MEM_UPDATED); + + GUEST_SHARED_ASSERT(uc, do_mem_op(VERIFY_PAT, priv_mem, + TEST_MEM_DATA_PAT2, mem_size)); + + GUEST_SHARED_DONE(uc); +} + +/* Test to verify guest shared accesses on private memory with following steps: + * 1) Upon entry, guest signals VMM that it has started. + * 2) VMM populates the shared memory with known pattern and continues guest + * execution. + * 3) Guest reads private gpa range in a shared fashion and verifies that it + * reads what VMM has written in step2. + * 3) Guest writes a different pattern on the shared memory and signals VMM + * that it has updated the shared memory. + * 4) VMM verifies shared memory contents to be same as the data populated + * in step 3 and continues guest execution. + */ +#define PMSAT_ID 1 +#define PMSAT_DESC "PrivateMemorySharedAccessTest" + +/* Guest code execution stages for private mem access test */ +#define PMSAT_GUEST_STARTED 0ULL +#define PMSAT_GUEST_TEST_MEM_UPDATED 1ULL + +static bool pmsat_handle_vm_stage(struct kvm_vm *vm, struct ucall *uc, + void *test_info, uint64_t stage) +{ + void *shared_mem = ((struct test_run_helper *)test_info)->shared_mem; + + switch (stage) { + case PMSAT_GUEST_STARTED: { + /* Initialize the contents of shared memory */ + TEST_ASSERT(do_mem_op(SET_PAT, shared_mem, + TEST_MEM_DATA_PAT1, test_mem_size), + "Shared memory update failed"); + VM_STAGE_PROCESSED(PMSAT_GUEST_STARTED); + break; + } + case PMSAT_GUEST_TEST_MEM_UPDATED: { + /* verify data to be same as what guest wrote */ + TEST_ASSERT(do_mem_op(VERIFY_PAT, shared_mem, + TEST_MEM_DATA_PAT2, test_mem_size), + "Shared memory view mismatch"); + VM_STAGE_PROCESSED(PMSAT_GUEST_TEST_MEM_UPDATED); + break; + } + default: + TEST_FAIL("Unhandled VM stage %ld\n", stage); + return false; + } + + return true; +} + +static void pmsat_guest_code(struct ucall *uc, uint8_t enc_bit_shift, + struct guest_pgt_info *gpgt_info, uint64_t mem_size, uint64_t ghcb_gpa, + void *ghcb_gva) +{ + void *shared_mem = (void *)TEST_MEM_GPA; + + g_ghcb_gva = ghcb_gva; + g_ghcb_gpa = ghcb_gpa; + GUEST_SHARED_SYNC(uc, PMSAT_GUEST_STARTED); + guest_verify_sev_vm_boot(uc, g_ghcb_gva != NULL); + GUEST_SHARED_ASSERT(uc, do_mem_op(VERIFY_PAT, shared_mem, + TEST_MEM_DATA_PAT1, mem_size)); + + GUEST_SHARED_ASSERT(uc, do_mem_op(SET_PAT, shared_mem, + TEST_MEM_DATA_PAT2, mem_size)); + GUEST_SHARED_SYNC(uc, PMSAT_GUEST_TEST_MEM_UPDATED); + + GUEST_SHARED_DONE(uc); +} + +/* Test to verify guest shared accesses on shared memory with following steps: + * 1) Upon entry, guest signals VMM that it has started. + * 2) VMM deallocates the backing private memory and populates the shared memory + * with known pattern and continues guest execution. + * 3) Guest reads shared gpa range in a shared fashion and verifies that it + * reads what VMM has written in step2. + * 3) Guest writes a different pattern on the shared memory and signals VMM + * that it has updated the shared memory. + * 4) VMM verifies shared memory contents to be same as the data populated + * in step 3 and continues guest execution. + */ +#define SMSAT_ID 2 +#define SMSAT_DESC "SharedMemorySharedAccessTest" + +#define SMSAT_GUEST_STARTED 0ULL +#define SMSAT_GUEST_TEST_MEM_UPDATED 1ULL + +static bool smsat_handle_vm_stage(struct kvm_vm *vm, struct ucall *uc, + void *test_info, uint64_t stage) +{ + void *shared_mem = ((struct test_run_helper *)test_info)->shared_mem; + int priv_memfd = ((struct test_run_helper *)test_info)->priv_memfd; + + switch (stage) { + case SMSAT_GUEST_STARTED: { + /* Remove the backing private memory storage */ + int ret = fallocate(priv_memfd, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + 0, test_mem_size); + TEST_ASSERT(ret != -1, "fallocate failed in smsat handling"); + /* Initialize the contents of shared memory */ + TEST_ASSERT(do_mem_op(SET_PAT, shared_mem, + TEST_MEM_DATA_PAT1, test_mem_size), + "Shared memory updated failed"); + VM_STAGE_PROCESSED(SMSAT_GUEST_STARTED); + break; + } + case SMSAT_GUEST_TEST_MEM_UPDATED: { + /* verify data to be same as what guest wrote */ + TEST_ASSERT(do_mem_op(VERIFY_PAT, shared_mem, + TEST_MEM_DATA_PAT2, test_mem_size), + "Shared memory view mismatch"); + VM_STAGE_PROCESSED(SMSAT_GUEST_TEST_MEM_UPDATED); + break; + } + default: + TEST_FAIL("Unhandled VM stage %ld\n", stage); + return false; + } + + return true; +} + +static void smsat_guest_code(struct ucall *uc, uint8_t enc_bit_shift, + struct guest_pgt_info *gpgt_info, uint64_t mem_size, uint64_t ghcb_gpa, + void *ghcb_gva) +{ + void *shared_mem = (void *)TEST_MEM_GPA; + + g_ghcb_gva = ghcb_gva; + g_ghcb_gpa = ghcb_gpa; + GUEST_SHARED_SYNC(uc, SMSAT_GUEST_STARTED); + guest_verify_sev_vm_boot(uc, g_ghcb_gva != NULL); + GUEST_SHARED_ASSERT(uc, do_mem_op(VERIFY_PAT, shared_mem, + TEST_MEM_DATA_PAT1, mem_size)); + + GUEST_SHARED_ASSERT(uc, do_mem_op(SET_PAT, shared_mem, + TEST_MEM_DATA_PAT2, mem_size)); + GUEST_SHARED_SYNC(uc, SMSAT_GUEST_TEST_MEM_UPDATED); + + GUEST_SHARED_DONE(uc); +} + +/* Test to verify guest private accesses on shared memory with following steps: + * 1) Upon entry, guest signals VMM that it has started. + * 2) VMM deallocates the backing private memory and populates the shared memory + * with known pattern and continues guest execution. + * 3) Guest writes gpa range via private access and signals VMM. + * 4) VMM verifies shared memory contents to be same as the data populated + * in step 2 and continues guest execution. + * 5) Guest reads gpa range via private access and verifies that the contents + * are same as written in step 3. + */ +#define SMPAT_ID 3 +#define SMPAT_DESC "SharedMemoryPrivateAccessTest" + +#define SMPAT_GUEST_STARTED 0ULL +#define SMPAT_GUEST_TEST_MEM_UPDATED 1ULL + +static bool smpat_handle_vm_stage(struct kvm_vm *vm, struct ucall *uc, + void *test_info, uint64_t stage) +{ + void *shared_mem = ((struct test_run_helper *)test_info)->shared_mem; + int priv_memfd = ((struct test_run_helper *)test_info)->priv_memfd; + + switch (stage) { + case SMPAT_GUEST_STARTED: { + /* Remove the backing private memory storage */ + int ret = fallocate(priv_memfd, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + 0, test_mem_size); + TEST_ASSERT(ret != -1, "fallocate failed in smpat handling"); + /* Initialize the contents of shared memory */ + TEST_ASSERT(do_mem_op(SET_PAT, shared_mem, TEST_MEM_DATA_PAT1, + test_mem_size), "Shared memory updated failed"); + VM_STAGE_PROCESSED(SMPAT_GUEST_STARTED); + break; + } + case SMPAT_GUEST_TEST_MEM_UPDATED: { + /* verify data to be same as what vmm wrote earlier */ + TEST_ASSERT(do_mem_op(VERIFY_PAT, shared_mem, + TEST_MEM_DATA_PAT1, test_mem_size), + "Shared memory view mismatch"); + VM_STAGE_PROCESSED(SMPAT_GUEST_TEST_MEM_UPDATED); + break; + } + default: + TEST_FAIL("Unhandled VM stage %ld\n", stage); + return false; + } + + return true; +} + +static void smpat_guest_code(struct ucall *uc, uint8_t enc_bit_shift, + struct guest_pgt_info *gpgt_info, uint64_t mem_size, uint64_t ghcb_gpa, + void *ghcb_gva) +{ + void *shared_mem = (void *)TEST_MEM_GPA; + int ret; + + g_ghcb_gva = ghcb_gva; + g_ghcb_gpa = ghcb_gpa; + GUEST_SHARED_SYNC(uc, SMPAT_GUEST_STARTED); + guest_verify_sev_vm_boot(uc, g_ghcb_gva != NULL); + + guest_set_clr_pte_bit(uc, gpgt_info, TEST_MEM_GPA, + mem_size, true, enc_bit_shift); + /* Mark the GPA range to be treated as always accessed privately */ + ret = guest_hypercall(KVM_HC_MAP_GPA_RANGE, TEST_MEM_GPA, + mem_size >> MIN_PAGE_SHIFT, KVM_MARK_GPA_RANGE_ENC_ACCESS, 0); + GUEST_SHARED_ASSERT_1(uc, ret == 0, ret); + + GUEST_SHARED_ASSERT(uc, do_mem_op(SET_PAT, shared_mem, + TEST_MEM_DATA_PAT2, mem_size)); + GUEST_SHARED_SYNC(uc, SMPAT_GUEST_TEST_MEM_UPDATED); + GUEST_SHARED_ASSERT(uc, do_mem_op(VERIFY_PAT, shared_mem, + TEST_MEM_DATA_PAT2, mem_size)); + + GUEST_SHARED_DONE(uc); +} + +/* Test to verify guest shared and private accesses on memory with following + * steps: + * 1) Upon entry, guest signals VMM that it has started. + * 2) VMM populates the shared memory with known pattern and continues guest + * execution. + * 3) Guest writes shared gpa range in a private fashion and signals VMM + * 4) VMM verifies that shared memory still contains the pattern written in + * step 2 and continues guest execution. + * 5) Guest verifies private memory contents to be same as the data populated + * in step 3 and signals VMM. + * 6) VMM removes the private memory backing which should also clear out the + * second stage mappings for the VM + * 6) Guest does shared write access on shared memory and signals vmm + * 7) VMM reads the shared memory and verifies that the data is same as what + * guest wrote in step 6 and continues guest execution. + * 8) Guest reads the private memory and verifies that the data is same as + * written in step 6. + */ +#define PSAT_ID 4 +#define PSAT_DESC "PrivateSharedAccessTest" + +#define PSAT_GUEST_STARTED 0ULL +#define PSAT_GUEST_PRIVATE_MEM_UPDATED 1ULL +#define PSAT_GUEST_PRIVATE_MEM_VERIFIED 2ULL +#define PSAT_GUEST_SHARED_MEM_UPDATED 3ULL + +static bool psat_handle_vm_stage(struct kvm_vm *vm, struct ucall *uc, + void *test_info, uint64_t stage) +{ + void *shared_mem = ((struct test_run_helper *)test_info)->shared_mem; + int priv_memfd = ((struct test_run_helper *)test_info)->priv_memfd; + + switch (stage) { + case PSAT_GUEST_STARTED: { + /* Initialize the contents of shared memory */ + TEST_ASSERT(do_mem_op(SET_PAT, shared_mem, + TEST_MEM_DATA_PAT1, test_mem_size), + "Shared memory update failed"); + VM_STAGE_PROCESSED(PSAT_GUEST_STARTED); + break; + } + case PSAT_GUEST_PRIVATE_MEM_UPDATED: { + /* verify data to be same as what vmm wrote earlier */ + TEST_ASSERT(do_mem_op(VERIFY_PAT, shared_mem, + TEST_MEM_DATA_PAT1, test_mem_size), + "Shared memory view mismatch"); + VM_STAGE_PROCESSED(PSAT_GUEST_PRIVATE_MEM_UPDATED); + break; + } + case PSAT_GUEST_PRIVATE_MEM_VERIFIED: { + /* Remove the backing private memory storage so that + * subsequent accesses from guest cause a second stage + * page fault + */ + int ret = fallocate(priv_memfd, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + 0, test_mem_size); + TEST_ASSERT(ret != -1, + "fallocate failed in smpat handling"); + VM_STAGE_PROCESSED(PSAT_GUEST_PRIVATE_MEM_VERIFIED); + break; + } + case PSAT_GUEST_SHARED_MEM_UPDATED: { + /* verify data to be same as what guest wrote */ + TEST_ASSERT(do_mem_op(VERIFY_PAT, shared_mem, + TEST_MEM_DATA_PAT2, test_mem_size), + "Shared memory view mismatch"); + VM_STAGE_PROCESSED(PSAT_GUEST_SHARED_MEM_UPDATED); + break; + } + default: + TEST_FAIL("Unhandled VM stage %ld\n", stage); + return false; + } + + return true; +} + +static void psat_guest_code(struct ucall *uc, uint8_t enc_bit_shift, + struct guest_pgt_info *gpgt_info, uint64_t mem_size, uint64_t ghcb_gpa, + void *ghcb_gva) +{ + void *shared_mem = (void *)TEST_MEM_GPA; + int ret; + + g_ghcb_gva = ghcb_gva; + g_ghcb_gpa = ghcb_gpa; + GUEST_SHARED_SYNC(uc, PSAT_GUEST_STARTED); + guest_verify_sev_vm_boot(uc, g_ghcb_gva != NULL); + + guest_set_clr_pte_bit(uc, gpgt_info, TEST_MEM_GPA, mem_size, true, + enc_bit_shift); + /* Mark the GPA range to be treated as always accessed privately */ + ret = guest_hypercall(KVM_HC_MAP_GPA_RANGE, TEST_MEM_GPA, + mem_size >> MIN_PAGE_SHIFT, KVM_MARK_GPA_RANGE_ENC_ACCESS, 0); + GUEST_SHARED_ASSERT_1(uc, ret == 0, ret); + + GUEST_SHARED_ASSERT(uc, do_mem_op(SET_PAT, shared_mem, + TEST_MEM_DATA_PAT2, mem_size)); + GUEST_SHARED_SYNC(uc, PSAT_GUEST_PRIVATE_MEM_UPDATED); + GUEST_SHARED_ASSERT(uc, do_mem_op(VERIFY_PAT, shared_mem, + TEST_MEM_DATA_PAT2, mem_size)); + + GUEST_SHARED_SYNC(uc, PSAT_GUEST_PRIVATE_MEM_VERIFIED); + + guest_set_clr_pte_bit(uc, gpgt_info, TEST_MEM_GPA, mem_size, false, + enc_bit_shift); + /* Mark no GPA range to be treated as accessed privately */ + ret = guest_hypercall(KVM_HC_MAP_GPA_RANGE, 0, 0, + KVM_MARK_GPA_RANGE_ENC_ACCESS, 0); + GUEST_SHARED_ASSERT_1(uc, ret == 0, ret); + GUEST_SHARED_ASSERT(uc, do_mem_op(SET_PAT, shared_mem, + TEST_MEM_DATA_PAT2, mem_size)); + GUEST_SHARED_SYNC(uc, PSAT_GUEST_SHARED_MEM_UPDATED); + GUEST_SHARED_ASSERT(uc, do_mem_op(VERIFY_PAT, shared_mem, + TEST_MEM_DATA_PAT2, mem_size)); + + GUEST_SHARED_DONE(uc); +} + +/* Test to verify guest shared and private accesses on memory with following + * steps: + * 1) Upon entry, guest signals VMM that it has started. + * 2) VMM removes the private memory backing and populates the shared memory + * with known pattern and continues guest execution. + * 3) Guest reads shared gpa range in a shared fashion and verifies that it + * reads what VMM has written in step2. + * 4) Guest writes a different pattern on the shared memory and signals VMM + * that it has updated the shared memory. + * 5) VMM verifies shared memory contents to be same as the data populated + * in step 4 and installs private memory backing again to allow guest + * to do private access and invalidate second stage mappings. + * 6) Guest does private write access on shared memory and signals vmm + * 7) VMM reads the shared memory and verified that the data is still same + * as in step 4 and continues guest execution. + * 8) Guest reads the private memory and verifies that the data is same as + * written in step 6. + */ +#define SPAT_ID 5 +#define SPAT_DESC "SharedPrivateAccessTest" + +#define SPAT_GUEST_STARTED 0ULL +#define SPAT_GUEST_SHARED_MEM_UPDATED 1ULL +#define SPAT_GUEST_PRIVATE_MEM_UPDATED 2ULL + +static bool spat_handle_vm_stage(struct kvm_vm *vm, struct ucall *uc, + void *test_info, uint64_t stage) +{ + void *shared_mem = ((struct test_run_helper *)test_info)->shared_mem; + int priv_memfd = ((struct test_run_helper *)test_info)->priv_memfd; + int ret; + + switch (stage) { + case SPAT_GUEST_STARTED: { + /* Remove the backing private memory storage so that + * subsequent accesses from guest cause a second stage + * page fault + */ + ret = fallocate(priv_memfd, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + 0, test_mem_size); + TEST_ASSERT(ret != -1, + "fallocate failed in spat handling"); + + /* Initialize the contents of shared memory */ + TEST_ASSERT(do_mem_op(SET_PAT, shared_mem, + TEST_MEM_DATA_PAT1, test_mem_size), + "Shared memory updated failed"); + VM_STAGE_PROCESSED(SPAT_GUEST_STARTED); + break; + } + case SPAT_GUEST_SHARED_MEM_UPDATED: { + /* verify data to be same as what guest wrote earlier */ + TEST_ASSERT(do_mem_op(VERIFY_PAT, shared_mem, + TEST_MEM_DATA_PAT2, test_mem_size), + "Shared memory view mismatch"); + /* Allocate memory for private backing store */ + ret = fallocate(priv_memfd, 0, 0, test_mem_size); + TEST_ASSERT(ret != -1, "fallocate failed in spat handling"); + VM_STAGE_PROCESSED(SPAT_GUEST_SHARED_MEM_UPDATED); + break; + } + case SPAT_GUEST_PRIVATE_MEM_UPDATED: { + /* verify data to be same as what guest wrote earlier */ + TEST_ASSERT(do_mem_op(VERIFY_PAT, shared_mem, + TEST_MEM_DATA_PAT2, test_mem_size), + "Shared memory view mismatch"); + VM_STAGE_PROCESSED(SPAT_GUEST_PRIVATE_MEM_UPDATED); + break; + } + default: + TEST_FAIL("Unhandled VM stage %ld\n", stage); + return false; + } + + return true; +} + +static void spat_guest_code(struct ucall *uc, uint8_t enc_bit_shift, + struct guest_pgt_info *gpgt_info, uint64_t mem_size, uint64_t ghcb_gpa, + void *ghcb_gva) +{ + void *shared_mem = (void *)TEST_MEM_GPA; + int ret; + + g_ghcb_gva = ghcb_gva; + g_ghcb_gpa = ghcb_gpa; + GUEST_SHARED_SYNC(uc, SPAT_GUEST_STARTED); + guest_verify_sev_vm_boot(uc, g_ghcb_gva != NULL); + GUEST_SHARED_ASSERT(uc, do_mem_op(VERIFY_PAT, shared_mem, + TEST_MEM_DATA_PAT1, mem_size)); + GUEST_SHARED_ASSERT(uc, do_mem_op(SET_PAT, shared_mem, + TEST_MEM_DATA_PAT2, mem_size)); + + GUEST_SHARED_SYNC(uc, SPAT_GUEST_SHARED_MEM_UPDATED); + + guest_set_clr_pte_bit(uc, gpgt_info, TEST_MEM_GPA, mem_size, true, + enc_bit_shift); + /* Mark the GPA range to be treated as always accessed privately */ + ret = guest_hypercall(KVM_HC_MAP_GPA_RANGE, TEST_MEM_GPA, + mem_size >> MIN_PAGE_SHIFT, KVM_MARK_GPA_RANGE_ENC_ACCESS, 0); + GUEST_SHARED_ASSERT_1(uc, ret == 0, ret); + + GUEST_SHARED_ASSERT(uc, do_mem_op(SET_PAT, shared_mem, + TEST_MEM_DATA_PAT1, mem_size)); + GUEST_SHARED_SYNC(uc, PSAT_GUEST_PRIVATE_MEM_UPDATED); + GUEST_SHARED_ASSERT(uc, do_mem_op(VERIFY_PAT, shared_mem, + TEST_MEM_DATA_PAT1, mem_size)); + GUEST_SHARED_DONE(uc); +} + +/* Test to verify guest private, shared, private accesses on memory with + * following steps: + * 1) Upon entry, guest signals VMM that it has started. + * 2) VMM initializes the shared memory with known pattern and continues guest + * execution + * 3) Guest writes the private memory privately via a known pattern and + * signals VMM + * 4) VMM reads the shared memory and verifies that it's same as whats written + * in step 2 and continues guest execution + * 5) Guest reads the private memory privately and verifies that the contents + * are same as written in step 3. + * 6) Guest invokes KVM_HC_MAP_GPA_RANGE to map the hpa range as shared + * and marks the range to be accessed via shared access. + * 7) Guest does a shared access to shared memory and verifies that the + * contents are same as written in step 2. + * 8) Guest writes known pattern to test memory and signals VMM. + * 9) VMM verifies the memory contents to be same as written by guest in step + * 8 + * 10) Guest invokes KVM_HC_MAP_GPA_RANGE to map the hpa range as private + * and marks the range to be accessed via private access. + * 11) Guest writes a known pattern to the test memory and signals VMM. + * 12) VMM verifies the memory contents to be same as written by guest in step + * 8 and continues guest execution. + * 13) Guest verififes the memory pattern to be same as written in step 11. + */ +#define PSPAHCT_ID 6 +#define PSPAHCT_DESC "PrivateSharedPrivateAccessHyperCallTest" + +#define PSPAHCT_GUEST_STARTED 0ULL +#define PSPAHCT_GUEST_PRIVATE_MEM_UPDATED 1ULL +#define PSPAHCT_GUEST_SHARED_MEM_UPDATED 2ULL +#define PSPAHCT_GUEST_PRIVATE_MEM_UPDATED2 3ULL + +static bool pspahct_handle_vm_stage(struct kvm_vm *vm, struct ucall *uc, + void *test_info, uint64_t stage) +{ + void *shared_mem = ((struct test_run_helper *)test_info)->shared_mem; + + switch (stage) { + case PSPAHCT_GUEST_STARTED: { + /* Initialize the contents of shared memory */ + TEST_ASSERT(do_mem_op(SET_PAT, shared_mem, + TEST_MEM_DATA_PAT1, test_mem_size), + "Shared memory update failed"); + VM_STAGE_PROCESSED(PSPAHCT_GUEST_STARTED); + break; + } + case PSPAHCT_GUEST_PRIVATE_MEM_UPDATED: { + /* verify data to be same as what guest wrote earlier */ + TEST_ASSERT(do_mem_op(VERIFY_PAT, shared_mem, + TEST_MEM_DATA_PAT1, test_mem_size), + "Shared memory view mismatch"); + VM_STAGE_PROCESSED(PSPAHCT_GUEST_PRIVATE_MEM_UPDATED); + break; + } + case PSPAHCT_GUEST_SHARED_MEM_UPDATED: { + /* verify data to be same as what guest wrote earlier */ + TEST_ASSERT(do_mem_op(VERIFY_PAT, shared_mem, + TEST_MEM_DATA_PAT2, test_mem_size), + "Shared memory view mismatch"); + VM_STAGE_PROCESSED(PSPAHCT_GUEST_SHARED_MEM_UPDATED); + break; + } + case PSPAHCT_GUEST_PRIVATE_MEM_UPDATED2: { + /* verify data to be same as what guest wrote earlier */ + TEST_ASSERT(do_mem_op(VERIFY_PAT, shared_mem, + TEST_MEM_DATA_PAT2, test_mem_size), + "Shared memory view mismatch"); + VM_STAGE_PROCESSED(PSPAHCT_GUEST_PRIVATE_MEM_UPDATED2); + break; + } + default: + TEST_FAIL("Unhandled VM stage %ld\n", stage); + return false; + } + + return true; +} + +static void pspahct_guest_code(struct ucall *uc, uint8_t enc_bit_shift, + struct guest_pgt_info *gpgt_info, uint64_t mem_size, uint64_t ghcb_gpa, + void *ghcb_gva) +{ + void *test_mem = (void *)TEST_MEM_GPA; + int ret; + + g_ghcb_gva = ghcb_gva; + g_ghcb_gpa = ghcb_gpa; + GUEST_SHARED_SYNC(uc, PSPAHCT_GUEST_STARTED); + guest_verify_sev_vm_boot(uc, g_ghcb_gva != NULL); + + guest_set_clr_pte_bit(uc, gpgt_info, TEST_MEM_GPA, mem_size, true, + enc_bit_shift); + /* Mark the GPA range to be treated as always accessed privately */ + ret = guest_hypercall(KVM_HC_MAP_GPA_RANGE, TEST_MEM_GPA, + mem_size >> MIN_PAGE_SHIFT, KVM_MARK_GPA_RANGE_ENC_ACCESS, 0); + GUEST_SHARED_ASSERT_1(uc, ret == 0, ret); + GUEST_SHARED_ASSERT(uc, do_mem_op(SET_PAT, test_mem, TEST_MEM_DATA_PAT2, + mem_size)); + + GUEST_SHARED_SYNC(uc, PSPAHCT_GUEST_PRIVATE_MEM_UPDATED); + GUEST_SHARED_ASSERT(uc, do_mem_op(VERIFY_PAT, test_mem, + TEST_MEM_DATA_PAT2, mem_size)); + + /* Map the GPA range to be treated as shared */ + ret = guest_hypercall(KVM_HC_MAP_GPA_RANGE, TEST_MEM_GPA, + mem_size >> MIN_PAGE_SHIFT, + KVM_MAP_GPA_RANGE_DECRYPTED | KVM_MAP_GPA_RANGE_PAGE_SZ_4K, 0); + GUEST_SHARED_ASSERT_1(uc, ret == 0, ret); + + guest_set_clr_pte_bit(uc, gpgt_info, TEST_MEM_GPA, mem_size, false, + enc_bit_shift); + /* Mark the GPA range to be treated as always accessed via shared + * access + */ + ret = guest_hypercall(KVM_HC_MAP_GPA_RANGE, 0, 0, + KVM_MARK_GPA_RANGE_ENC_ACCESS, 0); + GUEST_SHARED_ASSERT_1(uc, ret == 0, ret); + + GUEST_SHARED_ASSERT(uc, do_mem_op(VERIFY_PAT, test_mem, + TEST_MEM_DATA_PAT1, mem_size)); + GUEST_SHARED_ASSERT(uc, do_mem_op(SET_PAT, test_mem, + TEST_MEM_DATA_PAT2, mem_size)); + GUEST_SHARED_SYNC(uc, PSPAHCT_GUEST_SHARED_MEM_UPDATED); + + GUEST_SHARED_ASSERT(uc, do_mem_op(VERIFY_PAT, test_mem, + TEST_MEM_DATA_PAT2, mem_size)); + + guest_set_clr_pte_bit(uc, gpgt_info, TEST_MEM_GPA, mem_size, true, + enc_bit_shift); + /* Map the GPA range to be treated as private */ + ret = guest_hypercall(KVM_HC_MAP_GPA_RANGE, TEST_MEM_GPA, + mem_size >> MIN_PAGE_SHIFT, + KVM_MAP_GPA_RANGE_ENCRYPTED | KVM_MAP_GPA_RANGE_PAGE_SZ_4K, 0); + GUEST_SHARED_ASSERT_1(uc, ret == 0, ret); + + /* Mark the GPA range to be treated as always accessed via private + * access + */ + ret = guest_hypercall(KVM_HC_MAP_GPA_RANGE, TEST_MEM_GPA, + mem_size >> MIN_PAGE_SHIFT, KVM_MARK_GPA_RANGE_ENC_ACCESS, 0); + GUEST_SHARED_ASSERT_1(uc, ret == 0, ret); + + GUEST_SHARED_ASSERT(uc, do_mem_op(SET_PAT, test_mem, + TEST_MEM_DATA_PAT1, mem_size)); + GUEST_SHARED_SYNC(uc, PSPAHCT_GUEST_PRIVATE_MEM_UPDATED2); + GUEST_SHARED_ASSERT(uc, do_mem_op(VERIFY_PAT, test_mem, + TEST_MEM_DATA_PAT1, mem_size)); + GUEST_SHARED_DONE(uc); +} + +/* Test to verify guest accesses without double allocation: + * Guest starts with shared memory access disallowed by default. + * 1) Guest writes the private memory privately via a known pattern + * 3) Guest reads the private memory privately and verifies that the contents + * are same as written. + * 4) Guest invokes KVM_HC_MAP_GPA_RANGE to map the hpa range as shared + * and marks the range to be accessed via shared access. + * 5) Guest writes shared memory with another pattern and signals VMM + * 6) VMM verifies the memory contents to be same as written by guest in step + * 5 and updates the memory with a different pattern + * 7) Guest verifies the memory contents to be same as written in step 6. + * 8) Guest invokes KVM_HC_MAP_GPA_RANGE to map the hpa range as private + * and marks the range to be accessed via private access. + * 9) Guest writes a known pattern to the test memory and verifies the contents + * to be same as written. + * 10) Guest invokes KVM_HC_MAP_GPA_RANGE to map the hpa range as shared + * and marks the range to be accessed via shared access. + * 11) Guest writes shared memory with another pattern and signals VMM + * 12) VMM verifies the memory contents to be same as written by guest in step + * 5 and updates the memory with a different pattern + * 13) Guest verifies the memory contents to be same as written in step 6. + */ +#define PSAWDAT_ID 7 +#define PSAWDAT_DESC "PrivateSharedAccessWithoutDoubleAllocationTest" + +#define PSAWDAT_GUEST_SHARED_MEM_UPDATED1 1ULL +#define PSAWDAT_GUEST_SHARED_MEM_UPDATED2 2ULL + +static bool psawdat_handle_vm_stage(struct kvm_vm *vm, struct ucall *uc, + void *test_info, uint64_t stage) +{ + void *shared_mem = ((struct test_run_helper *)test_info)->shared_mem; + + switch (stage) { + case PSAWDAT_GUEST_SHARED_MEM_UPDATED1: { + /* verify data to be same as what guest wrote earlier */ + TEST_ASSERT(do_mem_op(VERIFY_PAT, shared_mem, + TEST_MEM_DATA_PAT2, test_mem_size), + "Shared memory view mismatch"); + TEST_ASSERT(do_mem_op(SET_PAT, shared_mem, + TEST_MEM_DATA_PAT1, test_mem_size), + "Shared mem update Failure"); + VM_STAGE_PROCESSED(PSAWDAT_GUEST_SHARED_MEM_UPDATED); + break; + } + case PSAWDAT_GUEST_SHARED_MEM_UPDATED2: { + /* verify data to be same as what guest wrote earlier */ + TEST_ASSERT(do_mem_op(VERIFY_PAT, shared_mem, + TEST_MEM_DATA_PAT3, test_mem_size), + "Shared memory view mismatch"); + TEST_ASSERT(do_mem_op(SET_PAT, shared_mem, + TEST_MEM_DATA_PAT4, test_mem_size), + "Shared mem update Failure"); + VM_STAGE_PROCESSED(PSAWDAT_GUEST_SHARED_MEM_UPDATED2); + break; + } + default: + TEST_FAIL("Unhandled VM stage %ld\n", stage); + return false; + } + + return true; +} + +static void psawdat_guest_code(struct ucall *uc, uint8_t enc_bit_shift, + struct guest_pgt_info *gpgt_info, uint64_t mem_size, uint64_t ghcb_gpa, + void *ghcb_gva) +{ + void *test_mem = (void *)TEST_MEM_GPA; + int ret; + + g_ghcb_gva = ghcb_gva; + g_ghcb_gpa = ghcb_gpa; + guest_verify_sev_vm_boot(uc, g_ghcb_gva != NULL); + guest_set_clr_pte_bit(uc, gpgt_info, TEST_MEM_GPA, mem_size, true, + enc_bit_shift); + /* Mark the GPA range to be treated as always accessed privately */ + ret = guest_hypercall(KVM_HC_MAP_GPA_RANGE, TEST_MEM_GPA, + mem_size >> MIN_PAGE_SHIFT, KVM_MARK_GPA_RANGE_ENC_ACCESS, 0); + GUEST_SHARED_ASSERT_1(uc, ret == 0, ret); + GUEST_SHARED_ASSERT(uc, do_mem_op(SET_PAT, test_mem, TEST_MEM_DATA_PAT1, + mem_size)); + + GUEST_SHARED_ASSERT(uc, do_mem_op(VERIFY_PAT, test_mem, + TEST_MEM_DATA_PAT1, mem_size)); + + /* Map the GPA range to be treated as shared */ + ret = guest_hypercall(KVM_HC_MAP_GPA_RANGE, TEST_MEM_GPA, + mem_size >> MIN_PAGE_SHIFT, + KVM_MAP_GPA_RANGE_DECRYPTED | KVM_MAP_GPA_RANGE_PAGE_SZ_4K, 0); + GUEST_SHARED_ASSERT_1(uc, ret == 0, ret); + + guest_set_clr_pte_bit(uc, gpgt_info, TEST_MEM_GPA, mem_size, false, + enc_bit_shift); + /* Mark the GPA range to be treated as always accessed via shared + * access + */ + ret = guest_hypercall(KVM_HC_MAP_GPA_RANGE, 0, 0, + KVM_MARK_GPA_RANGE_ENC_ACCESS, 0); + GUEST_SHARED_ASSERT_1(uc, ret == 0, ret); + + GUEST_SHARED_ASSERT(uc, do_mem_op(SET_PAT, test_mem, + TEST_MEM_DATA_PAT2, mem_size)); + GUEST_SHARED_SYNC(uc, PSAWDAT_GUEST_SHARED_MEM_UPDATED1); + + GUEST_SHARED_ASSERT(uc, do_mem_op(VERIFY_PAT, test_mem, + TEST_MEM_DATA_PAT1, mem_size)); + + /* Map the GPA range to be treated as private */ + ret = guest_hypercall(KVM_HC_MAP_GPA_RANGE, TEST_MEM_GPA, + mem_size >> MIN_PAGE_SHIFT, + KVM_MAP_GPA_RANGE_ENCRYPTED | KVM_MAP_GPA_RANGE_PAGE_SZ_4K, 0); + GUEST_SHARED_ASSERT_1(uc, ret == 0, ret); + + guest_set_clr_pte_bit(uc, gpgt_info, TEST_MEM_GPA, mem_size, true, + enc_bit_shift); + /* Mark the GPA range to be treated as always accessed via private + * access + */ + ret = guest_hypercall(KVM_HC_MAP_GPA_RANGE, TEST_MEM_GPA, + mem_size >> MIN_PAGE_SHIFT, KVM_MARK_GPA_RANGE_ENC_ACCESS, 0); + GUEST_SHARED_ASSERT_1(uc, ret == 0, ret); + + GUEST_SHARED_ASSERT(uc, do_mem_op(SET_PAT, test_mem, + TEST_MEM_DATA_PAT2, mem_size)); + GUEST_SHARED_ASSERT(uc, do_mem_op(VERIFY_PAT, test_mem, + TEST_MEM_DATA_PAT2, mem_size)); + + /* Map the GPA range to be treated as shared */ + ret = guest_hypercall(KVM_HC_MAP_GPA_RANGE, TEST_MEM_GPA, + mem_size >> MIN_PAGE_SHIFT, + KVM_MAP_GPA_RANGE_DECRYPTED | KVM_MAP_GPA_RANGE_PAGE_SZ_4K, 0); + GUEST_SHARED_ASSERT_1(uc, ret == 0, ret); + + guest_set_clr_pte_bit(uc, gpgt_info, TEST_MEM_GPA, mem_size, false, + enc_bit_shift); + /* Mark the GPA range to be treated as always accessed via shared + * access + */ + ret = guest_hypercall(KVM_HC_MAP_GPA_RANGE, 0, 0, + KVM_MARK_GPA_RANGE_ENC_ACCESS, 0); + GUEST_SHARED_ASSERT_1(uc, ret == 0, ret); + + GUEST_SHARED_ASSERT(uc, do_mem_op(SET_PAT, test_mem, + TEST_MEM_DATA_PAT3, mem_size)); + GUEST_SHARED_SYNC(uc, PSAWDAT_GUEST_SHARED_MEM_UPDATED2); + + GUEST_SHARED_ASSERT(uc, do_mem_op(VERIFY_PAT, test_mem, + TEST_MEM_DATA_PAT4, mem_size)); + + GUEST_SHARED_DONE(uc); +} + +static struct test_run_helper priv_memfd_testsuite[] = { + [PMPAT_ID] = { + .test_desc = PMPAT_DESC, + .vmst_handler = pmpat_handle_vm_stage, + .guest_fn = pmpat_guest_code, + }, + [PMSAT_ID] = { + .test_desc = PMSAT_DESC, + .vmst_handler = pmsat_handle_vm_stage, + .guest_fn = pmsat_guest_code, + }, + [SMSAT_ID] = { + .test_desc = SMSAT_DESC, + .vmst_handler = smsat_handle_vm_stage, + .guest_fn = smsat_guest_code, + }, + [SMPAT_ID] = { + .test_desc = SMPAT_DESC, + .vmst_handler = smpat_handle_vm_stage, + .guest_fn = smpat_guest_code, + }, + [PSAT_ID] = { + .test_desc = PSAT_DESC, + .vmst_handler = psat_handle_vm_stage, + .guest_fn = psat_guest_code, + }, + [SPAT_ID] = { + .test_desc = SPAT_DESC, + .vmst_handler = spat_handle_vm_stage, + .guest_fn = spat_guest_code, + }, + [PSPAHCT_ID] = { + .test_desc = PSPAHCT_DESC, + .vmst_handler = pspahct_handle_vm_stage, + .guest_fn = pspahct_guest_code, + }, + [PSAWDAT_ID] = { + .test_desc = PSAWDAT_DESC, + .vmst_handler = psawdat_handle_vm_stage, + .guest_fn = psawdat_guest_code, + .toggle_shared_mem_state = true, + .disallow_boot_shared_access = true, + }, +}; + +static void handle_vm_exit_hypercall(struct kvm_run *run, + uint32_t test_id) +{ + uint64_t gpa, npages, attrs, mem_end; + int priv_memfd = priv_memfd_testsuite[test_id].priv_memfd; + int ret; + int fallocate_mode; + void *shared_mem = priv_memfd_testsuite[test_id].shared_mem; + bool toggle_shared_mem_state = + priv_memfd_testsuite[test_id].toggle_shared_mem_state; + int mprotect_mode; + + if (run->hypercall.nr != KVM_HC_MAP_GPA_RANGE) + TEST_FAIL("Unhandled Hypercall %lld\n", run->hypercall.nr); + + gpa = run->hypercall.args[0]; + npages = run->hypercall.args[1]; + attrs = run->hypercall.args[2]; + mem_end = test_mem_end(gpa, test_mem_size); + + if ((gpa < TEST_MEM_GPA) || + ((gpa + (npages << MIN_PAGE_SHIFT)) > mem_end)) + TEST_FAIL("Unhandled gpa 0x%lx npages %ld\n", gpa, npages); + + if (attrs & KVM_MAP_GPA_RANGE_ENCRYPTED) { + fallocate_mode = 0; + mprotect_mode = PROT_NONE; + } else { + fallocate_mode = (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE); + mprotect_mode = PROT_READ | PROT_WRITE; + } + pr_info("Converting off 0x%lx pages 0x%lx to %s\n", + (gpa - TEST_MEM_GPA), npages, fallocate_mode ? + "shared" : "private"); + ret = fallocate(priv_memfd, fallocate_mode, (gpa - TEST_MEM_GPA), + npages << MIN_PAGE_SHIFT); + TEST_ASSERT(ret != -1, "fallocate failed in hc handling"); + if (toggle_shared_mem_state) { + if (fallocate_mode) { + ret = madvise(shared_mem, test_mem_size, MADV_DONTNEED); + TEST_ASSERT(ret != -1, "madvise failed in hc handling"); + } + ret = mprotect(shared_mem, test_mem_size, mprotect_mode); + TEST_ASSERT(ret != -1, "mprotect failed in hc handling"); + } + run->hypercall.ret = 0; +} + +static void handle_vm_exit_memory_error(struct kvm_run *run, + uint32_t test_id) +{ + uint64_t gpa, size, flags, mem_end; + int ret; + int priv_memfd = + priv_memfd_testsuite[test_id].priv_memfd; + void *shared_mem = priv_memfd_testsuite[test_id].shared_mem; + bool toggle_shared_mem_state = + priv_memfd_testsuite[test_id].toggle_shared_mem_state; + int fallocate_mode; + int mprotect_mode; + + gpa = run->memory.gpa; + size = run->memory.size; + flags = run->memory.flags; + mem_end = test_mem_end(gpa, test_mem_size); + + if ((gpa < TEST_MEM_GPA) || ((gpa + size) > mem_end)) + TEST_FAIL("Unhandled gpa 0x%lx size 0x%lx\n", gpa, size); + + if (flags & KVM_MEMORY_EXIT_FLAG_PRIVATE) { + fallocate_mode = 0; + mprotect_mode = PROT_NONE; + } else { + fallocate_mode = (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE); + mprotect_mode = PROT_READ | PROT_WRITE; + } + pr_info("Converting off 0x%lx size 0x%lx to %s\n", (gpa - TEST_MEM_GPA), + size, fallocate_mode ? "shared" : "private"); + ret = fallocate(priv_memfd, fallocate_mode, (gpa - TEST_MEM_GPA), size); + TEST_ASSERT(ret != -1, "fallocate failed in memory error handling"); + + if (toggle_shared_mem_state) { + if (fallocate_mode) { + ret = madvise(shared_mem, test_mem_size, MADV_DONTNEED); + TEST_ASSERT(ret != -1, + "madvise failed in memory error handling"); + } + ret = mprotect(shared_mem, test_mem_size, mprotect_mode); + TEST_ASSERT(ret != -1, + "mprotect failed in memory error handling"); + } +} + +static void vcpu_work(struct kvm_vm *vm, struct ucall *uc, uint32_t test_id) +{ + struct kvm_run *run; + uint64_t cmd; + + /* + * Loop until the guest is done. + */ + run = vcpu_state(vm, VCPU_ID); + + while (true) { + vcpu_run(vm, VCPU_ID); + + if (run->exit_reason == KVM_EXIT_HLT) { + cmd = get_ucall_shared(vm, VCPU_ID, uc); + if (cmd != UCALL_SYNC) + break; + + if (!priv_memfd_testsuite[test_id].vmst_handler( + vm, uc, &priv_memfd_testsuite[test_id], + uc->args[1])) + break; + + continue; + } + + if (run->exit_reason == KVM_EXIT_HYPERCALL) { + handle_vm_exit_hypercall(run, test_id); + continue; + } + + if (run->exit_reason == KVM_EXIT_MEMORY_FAULT) { + handle_vm_exit_memory_error(run, test_id); + continue; + } + + TEST_FAIL("Unhandled VCPU exit reason %d\n", run->exit_reason); + break; + } + + if (run->exit_reason == KVM_EXIT_HLT && cmd == UCALL_ABORT) + TEST_FAIL("%s at %s:%ld, val = %lu", (const char *)uc->args[0], + __FILE__, uc->args[1], uc->args[2]); +} + +static void priv_memory_region_add(struct kvm_vm *vm, void *mem, uint32_t slot, + uint32_t size, uint64_t guest_addr, + uint32_t priv_fd, uint64_t priv_offset) +{ + struct kvm_userspace_memory_region_ext region_ext; + int ret; + + region_ext.region.slot = slot; + region_ext.region.flags = KVM_MEM_PRIVATE; + region_ext.region.guest_phys_addr = guest_addr; + region_ext.region.memory_size = size; + region_ext.region.userspace_addr = (uintptr_t) mem; + region_ext.private_fd = priv_fd; + region_ext.private_offset = priv_offset; + ret = ioctl(vm_get_fd(vm), KVM_SET_USER_MEMORY_REGION, ®ion_ext); + TEST_ASSERT(ret == 0, "Failed to register user region for gpa 0x%lx\n", + guest_addr); +} + +static void vc_handler(struct ex_regs *regs) +{ + sev_es_handle_vc(g_ghcb_gva, g_ghcb_gpa, regs); +} + +static void setup_and_execute_test(uint32_t test_id, + const enum page_size shared, + const enum page_size private, + uint32_t policy) +{ + struct kvm_vm *vm; + vm_vaddr_t uc_vaddr; + struct sev_vm *sev; + int priv_memfd; + int ret; + void *shared_mem; + struct kvm_enable_cap cap; + bool disallow_boot_shared_access = + priv_memfd_testsuite[test_id].disallow_boot_shared_access; + int prot_flags = PROT_READ | PROT_WRITE; + uint8_t measurement[512]; + uint32_t vm_page_size, num_test_pages; + vm_vaddr_t ghcb_vaddr = 0; + uint8_t enc_bit; + + sev = sev_vm_create(policy, TOTAL_PAGES); + TEST_ASSERT(sev, "Failed to create SEV VM"); + vm = sev_get_vm(sev); + + vm_set_pgt_alloc_tracking(vm); + + /* Set up VCPU and initial guest kernel. */ + vm_vcpu_add_default(vm, VCPU_ID, + priv_memfd_testsuite[test_id].guest_fn); + kvm_vm_elf_load(vm, program_invocation_name); + + // use 2 pages by default + size_t mem_size = PAGE_SIZE_4KB * 2; + bool using_hugepages = false; + + int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE; + + switch (shared) { + case PAGE_4KB: + // no additional flags are needed + break; + case PAGE_2MB: + mmap_flags |= MAP_HUGETLB | MAP_HUGE_2MB | MAP_POPULATE; + mem_size = max(mem_size, PAGE_SIZE_2MB); + using_hugepages = true; + break; + case PAGE_1GB: + mmap_flags |= MAP_HUGETLB | MAP_HUGE_1GB | MAP_POPULATE; + mem_size = max(mem_size, PAGE_SIZE_1GB); + using_hugepages = true; + break; + default: + TEST_FAIL("unknown page size for shared memory\n"); + } + + unsigned int memfd_flags = MFD_INACCESSIBLE; + + switch (private) { + case PAGE_4KB: + // no additional flags are needed + break; + case PAGE_2MB: + memfd_flags |= MFD_HUGETLB | MFD_HUGE_2MB; + mem_size = PAGE_SIZE_2MB; + using_hugepages = true; + break; + case PAGE_1GB: + memfd_flags |= MFD_HUGETLB | MFD_HUGE_1GB; + mem_size = PAGE_SIZE_1GB; + using_hugepages = true; + break; + default: + TEST_FAIL("unknown page size for private memory\n"); + } + + // set global for mem size to use later + test_mem_size = mem_size; + + if (disallow_boot_shared_access) + prot_flags = PROT_NONE; + + /* Allocate shared memory */ + shared_mem = mmap(NULL, mem_size, prot_flags, mmap_flags, -1, 0); + TEST_ASSERT(shared_mem != MAP_FAILED, "Failed to mmap() host"); + + if (using_hugepages) { + ret = madvise(shared_mem, mem_size, MADV_WILLNEED); + TEST_ASSERT(ret == 0, "madvise failed"); + } + + /* Allocate private memory */ + priv_memfd = memfd_create("vm_private_mem", memfd_flags); + TEST_ASSERT(priv_memfd != -1, "Failed to create priv_memfd"); + ret = fallocate(priv_memfd, 0, 0, mem_size); + TEST_ASSERT(ret != -1, "fallocate failed"); + + priv_memory_region_add(vm, shared_mem, TEST_MEM_SLOT, mem_size, + TEST_MEM_GPA, priv_memfd, 0); + + vm_page_size = vm_get_page_size(vm); + num_test_pages = mem_size / vm_page_size; + TEST_ASSERT(!(mem_size % vm_page_size), + "mem_size unaligned with vm page size"); + pr_info("Mapping test memory pages 0x%x page_size 0x%x\n", + num_test_pages, vm_page_size); + virt_map(vm, TEST_MEM_GPA, TEST_MEM_GPA, num_test_pages); + + /* Set up shared ucall buffer. */ + uc_vaddr = ucall_shared_alloc(vm, 1); + + /* Enable exit on KVM_HC_MAP_GPA_RANGE */ + pr_info("Enabling exit on map_gpa_range hypercall\n"); + ret = ioctl(vm_get_fd(vm), KVM_CHECK_EXTENSION, KVM_CAP_EXIT_HYPERCALL); + TEST_ASSERT(ret & (1 << KVM_HC_MAP_GPA_RANGE), + "VM exit on MAP_GPA_RANGE HC not supported"); + cap.cap = KVM_CAP_EXIT_HYPERCALL; + cap.flags = 0; + cap.args[0] = (1 << KVM_HC_MAP_GPA_RANGE); + ret = ioctl(vm_get_fd(vm), KVM_ENABLE_CAP, &cap); + TEST_ASSERT(ret == 0, + "Failed to enable exit on MAP_GPA_RANGE hypercall\n"); + + vm_map_page_table(vm, GUEST_PGT_MIN_VADDR); + vm_vaddr_t pgt_info_vaddr = vm_setup_pgt_info_buf(vm, + GUEST_PGT_MIN_VADDR); + + if (policy & SEV_POLICY_ES) { + ghcb_vaddr = vm_vaddr_alloc_shared(vm, vm_page_size, + vm_page_size); + /* Set up VC handler. */ + vm_init_descriptor_tables(vm); + vm_install_exception_handler(vm, 29, vc_handler); + vcpu_init_descriptor_tables(vm, VCPU_ID); + } + + /* Set up guest params. */ + enc_bit = sev_get_enc_bit(sev); + vcpu_args_set(vm, VCPU_ID, 6, uc_vaddr, enc_bit, pgt_info_vaddr, + test_mem_size, ghcb_vaddr ? addr_gva2gpa(vm, ghcb_vaddr) : 0, + ghcb_vaddr); + struct ucall *uc = (struct ucall *)addr_gva2hva(vm, uc_vaddr); + + priv_memfd_testsuite[test_id].shared_mem = shared_mem; + priv_memfd_testsuite[test_id].priv_memfd = priv_memfd; + + /* Allocations/setup done. Encrypt initial guest payload. */ + sev_vm_launch(sev); + sev_vm_launch_measure(sev, measurement); + pr_info("guest measurement: "); + for (uint32_t i = 0; i < 32; ++i) + pr_info("%02x", measurement[i]); + pr_info("\n"); + + sev_vm_launch_finish(sev); + + vcpu_work(vm, uc, test_id); + + munmap(shared_mem, mem_size); + priv_memfd_testsuite[test_id].shared_mem = NULL; + close(priv_memfd); + priv_memfd_testsuite[test_id].priv_memfd = -1; + sev_vm_free(sev); +} + +static void hugepage_requirements_text(const struct page_combo matrix) +{ + int pages_needed_2mb = 0; + int pages_needed_1gb = 0; + enum page_size sizes[] = { matrix.shared, matrix.private }; + + for (int i = 0; i < ARRAY_SIZE(sizes); ++i) { + if (sizes[i] == PAGE_2MB) + ++pages_needed_2mb; + if (sizes[i] == PAGE_1GB) + ++pages_needed_1gb; + } + if (pages_needed_2mb != 0 && pages_needed_1gb != 0) { + pr_info("This test requires %d 2MB page(s) and %d 1GB page(s)\n", + pages_needed_2mb, pages_needed_1gb); + } else if (pages_needed_2mb != 0) { + pr_info("This test requires %d 2MB page(s)\n", pages_needed_2mb); + } else if (pages_needed_1gb != 0) { + pr_info("This test requires %d 1GB page(s)\n", pages_needed_1gb); + } +} + +static bool should_skip_test(const struct page_combo matrix, + const bool use_2mb_pages, + const bool use_1gb_pages) +{ + if ((matrix.shared == PAGE_2MB || matrix.private == PAGE_2MB) + && !use_2mb_pages) + return true; + if ((matrix.shared == PAGE_1GB || matrix.private == PAGE_1GB) + && !use_1gb_pages) + return true; + return false; +} + +static void print_help(const char *const name) +{ + puts(""); + printf("usage %s [-h] [-m] [-g]\n", name); + puts(""); + printf(" -h: Display this help message\n"); + printf(" -m: include test runs using 2MB page permutations\n"); + printf(" -g: include test runs using 1GB page permutations\n"); + printf(" -e: Run test with SEV-ES VM\n"); + exit(0); +} + +int main(int argc, char *argv[]) +{ + /* Tell stdout not to buffer its content */ + setbuf(stdout, NULL); + + // arg parsing + int opt; + bool use_2mb_pages = false; + bool use_1gb_pages = false; + uint32_t policy = 0; + + while ((opt = getopt(argc, argv, "emgh")) != -1) { + switch (opt) { + case 'm': + use_2mb_pages = true; + break; + case 'g': + use_1gb_pages = true; + break; + case 'e': + policy |= SEV_POLICY_ES; + break; + case 'h': + default: + print_help(argv[0]); + } + } + + struct page_combo page_size_matrix[] = { + { .shared = PAGE_4KB, .private = PAGE_4KB }, + { .shared = PAGE_2MB, .private = PAGE_4KB }, + }; + + for (uint32_t i = 0; i < ARRAY_SIZE(priv_memfd_testsuite); i++) { + for (uint32_t j = 0; j < ARRAY_SIZE(page_size_matrix); j++) { + const struct page_combo current_page_matrix = + page_size_matrix[j]; + + if (should_skip_test(current_page_matrix, + use_2mb_pages, use_1gb_pages)) + break; + pr_info("=== Starting test %s... ===\n", + priv_memfd_testsuite[i].test_desc); + pr_info("using page sizes shared: %s private: %s\n", + page_size_to_str( + current_page_matrix.shared), + page_size_to_str( + current_page_matrix.private)); + hugepage_requirements_text(current_page_matrix); + setup_and_execute_test(i, current_page_matrix.shared, + current_page_matrix.private, policy); + pr_info("--- completed test %s ---\n\n", + priv_memfd_testsuite[i].test_desc); + } + } + + return 0; +}