From: David Hildenbrand david@redhat.com

alloc_anon_secure_inode() returns an inode after running checks in security_inode_init_security_anon().

Also refactor secretmem's file creation process to use the new function.

Signed-off-by: David Hildenbrand david@redhat.com Signed-off-by: Ackerley Tng ackerleytng@google.com Signed-off-by: Ryan Afranji afranji@google.com --- fs/anon_inodes.c | 23 ++++++++++++++++------- include/linux/fs.h | 13 +++++++------ mm/secretmem.c | 9 +-------- 3 files changed, 24 insertions(+), 21 deletions(-)

diff --git a/fs/anon_inodes.c b/fs/anon_inodes.c index 583ac81669c2..0ce28959c43a 100644 --- a/fs/anon_inodes.c +++ b/fs/anon_inodes.c @@ -55,17 +55,20 @@ static struct file_system_type anon_inode_fs_type = { .kill_sb = kill_anon_super, };

-static struct inode *anon_inode_make_secure_inode( - const char *name, - const struct inode *context_inode) +static struct inode *anon_inode_make_secure_inode(struct super_block *s, + const char *name, const struct inode *context_inode, + bool fs_internal) { struct inode *inode; int error;

- inode = alloc_anon_inode(anon_inode_mnt->mnt_sb); + inode = alloc_anon_inode(s); if (IS_ERR(inode)) return inode; - inode->i_flags &= ~S_PRIVATE; + + if (!fs_internal) + inode->i_flags &= ~S_PRIVATE; + error = security_inode_init_security_anon(inode, &QSTR(name), context_inode); if (error) { @@ -75,6 +78,12 @@ static struct inode *anon_inode_make_secure_inode( return inode; }

+struct inode *alloc_anon_secure_inode(struct super_block *s, const char *name) +{ + return anon_inode_make_secure_inode(s, name, NULL, true); +} +EXPORT_SYMBOL_GPL(alloc_anon_secure_inode); + static struct file *__anon_inode_getfile(const char *name, const struct file_operations *fops, void *priv, int flags, @@ -88,7 +97,8 @@ static struct file *__anon_inode_getfile(const char *name, return ERR_PTR(-ENOENT);

if (make_inode) { - inode = anon_inode_make_secure_inode(name, context_inode); + inode = anon_inode_make_secure_inode(anon_inode_mnt->mnt_sb, + name, context_inode, false); if (IS_ERR(inode)) { file = ERR_CAST(inode); goto err; @@ -318,4 +328,3 @@ static int __init anon_inode_init(void) }

fs_initcall(anon_inode_init); - diff --git a/include/linux/fs.h b/include/linux/fs.h index 016b0fe1536e..8eeef9a7fe07 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -309,7 +309,7 @@ struct iattr { */ #define FILESYSTEM_MAX_STACK_DEPTH 2

-/** +/** * enum positive_aop_returns - aop return codes with specific semantics * * @AOP_WRITEPAGE_ACTIVATE: Informs the caller that page writeback has @@ -319,7 +319,7 @@ struct iattr { * be a candidate for writeback again in the near * future. Other callers must be careful to unlock * the page if they get this return. Returned by - * writepage(); + * writepage(); * * @AOP_TRUNCATED_PAGE: The AOP method that was handed a locked page has * unlocked it and the page might have been truncated. @@ -1141,8 +1141,8 @@ struct file *get_file_active(struct file **f);

#define MAX_NON_LFS ((1UL<<31) - 1)

-/* Page cache limit. The filesystems should put that into their s_maxbytes - limits, otherwise bad things can happen in VM. */ +/* Page cache limit. The filesystems should put that into their s_maxbytes + limits, otherwise bad things can happen in VM. */ #if BITS_PER_LONG==32 #define MAX_LFS_FILESIZE ((loff_t)ULONG_MAX << PAGE_SHIFT) #elif BITS_PER_LONG==64 @@ -2607,7 +2607,7 @@ int sync_inode_metadata(struct inode *inode, int wait); struct file_system_type { const char *name; int fs_flags; -#define FS_REQUIRES_DEV 1 +#define FS_REQUIRES_DEV 1 #define FS_BINARY_MOUNTDATA 2 #define FS_HAS_SUBTYPE 4 #define FS_USERNS_MOUNT 8 /* Can be mounted by userns root */ @@ -3195,7 +3195,7 @@ ssize_t __kernel_read(struct file *file, void *buf, size_t count, loff_t *pos); extern ssize_t kernel_write(struct file *, const void *, size_t, loff_t *); extern ssize_t __kernel_write(struct file *, const void *, size_t, loff_t *); extern struct file * open_exec(const char *); - + /* fs/dcache.c -- generic fs support functions */ extern bool is_subdir(struct dentry *, struct dentry *); extern bool path_is_under(const struct path *, const struct path *); @@ -3550,6 +3550,7 @@ extern int simple_write_begin(struct file *file, struct address_space *mapping, extern const struct address_space_operations ram_aops; extern int always_delete_dentry(const struct dentry *); extern struct inode *alloc_anon_inode(struct super_block *); +extern struct inode *alloc_anon_secure_inode(struct super_block *, const char *); extern int simple_nosetlease(struct file *, int, struct file_lease **, void **); extern const struct dentry_operations simple_dentry_operations;

diff --git a/mm/secretmem.c b/mm/secretmem.c index 1b0a214ee558..c0e459e58cb6 100644 --- a/mm/secretmem.c +++ b/mm/secretmem.c @@ -195,18 +195,11 @@ static struct file *secretmem_file_create(unsigned long flags) struct file *file; struct inode *inode; const char *anon_name = "[secretmem]"; - int err;

- inode = alloc_anon_inode(secretmem_mnt->mnt_sb); + inode = alloc_anon_secure_inode(secretmem_mnt->mnt_sb, anon_name); if (IS_ERR(inode)) return ERR_CAST(inode);

- err = security_inode_init_security_anon(inode, &QSTR(anon_name), NULL); - if (err) { - file = ERR_PTR(err); - goto err_free_inode; - } - file = alloc_file_pseudo(inode, secretmem_mnt, "secretmem", O_RDWR, &secretmem_fops); if (IS_ERR(file))

From: Ackerley Tng ackerleytng@google.com

kvm_gmem_alloc_view() will allocate and build a file out of an inode.

Will be reused later by kvm_gmem_link()

Signed-off-by: Ackerley Tng ackerleytng@google.com Co-developed-by: Ryan Afranji afranji@google.com Signed-off-by: Ryan Afranji afranji@google.com --- virt/kvm/guest_memfd.c | 61 +++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 34 deletions(-)

diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 2ee26695dc31..a3918d1695b9 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -470,49 +470,47 @@ static struct inode *kvm_gmem_inode_make_secure_inode(const char *name, return inode; }

-static struct file *kvm_gmem_inode_create_getfile(void *priv, loff_t size, - u64 flags) +static struct file *kvm_gmem_alloc_view(struct kvm *kvm, struct inode *inode, + const char *name) { - static const char *name = "[kvm-gmem]"; - struct inode *inode; + struct kvm_gmem *gmem; struct file *file; - int err;

- err = -ENOENT; if (!try_module_get(kvm_gmem_fops.owner)) - goto err; + return ERR_PTR(-ENOENT);

- inode = kvm_gmem_inode_make_secure_inode(name, size, flags); - if (IS_ERR(inode)) { - err = PTR_ERR(inode); + gmem = kzalloc(sizeof(*gmem), GFP_KERNEL); + if (!gmem) { + file = ERR_PTR(-ENOMEM); goto err_put_module; }

file = alloc_file_pseudo(inode, kvm_gmem_mnt, name, O_RDWR, &kvm_gmem_fops); - if (IS_ERR(file)) { - err = PTR_ERR(file); - goto err_put_inode; - } + if (IS_ERR(file)) + goto err_gmem;

file->f_flags |= O_LARGEFILE; - file->private_data = priv; + file->private_data = gmem; + + kvm_get_kvm(kvm); + gmem->kvm = kvm; + xa_init(&gmem->bindings); + list_add(&gmem->entry, &file_inode(file)->i_mapping->i_private_list);

-out: return file;

-err_put_inode: - iput(inode); +err_gmem: + kfree(gmem); err_put_module: module_put(kvm_gmem_fops.owner); -err: - file = ERR_PTR(err); - goto out; + return file; }

static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags) { - struct kvm_gmem *gmem; + static const char *name = "[kvm-gmem]"; + struct inode *inode; struct file *file; int fd, err;

@@ -520,28 +518,23 @@ static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags) if (fd < 0) return fd;

- gmem = kzalloc(sizeof(*gmem), GFP_KERNEL); - if (!gmem) { - err = -ENOMEM; + inode = kvm_gmem_inode_make_secure_inode(name, size, flags); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); goto err_fd; }

- file = kvm_gmem_inode_create_getfile(gmem, size, flags); + file = kvm_gmem_alloc_view(kvm, inode, name); if (IS_ERR(file)) { err = PTR_ERR(file); - goto err_gmem; + goto err_put_inode; }

- kvm_get_kvm(kvm); - gmem->kvm = kvm; - xa_init(&gmem->bindings); - list_add(&gmem->entry, &file_inode(file)->i_mapping->i_private_list); - fd_install(fd, file); return fd;

-err_gmem: - kfree(gmem); +err_put_inode: + iput(inode); err_fd: put_unused_fd(fd); return err;

From: Ackerley Tng ackerleytng@google.com

Test that

+ Invalid inputs should be rejected with EINVAL + Successful inputs return a new (destination) fd + Destination and source fds have the same inode number + No crash on program exit

Signed-off-by: Ackerley Tng ackerleytng@google.com Signed-off-by: Ryan Afranji afranji@google.com --- .../testing/selftests/kvm/guest_memfd_test.c | 43 +++++++++++++++++++ .../testing/selftests/kvm/include/kvm_util.h | 18 ++++++++ 2 files changed, 61 insertions(+)

diff --git a/tools/testing/selftests/kvm/guest_memfd_test.c b/tools/testing/selftests/kvm/guest_memfd_test.c index ce687f8d248f..9b2a58cd9b64 100644 --- a/tools/testing/selftests/kvm/guest_memfd_test.c +++ b/tools/testing/selftests/kvm/guest_memfd_test.c @@ -170,6 +170,48 @@ static void test_create_guest_memfd_multiple(struct kvm_vm *vm) close(fd1); }

+static void test_link(struct kvm_vm *src_vm, int src_fd, size_t total_size) +{ + int ret; + int dst_fd; + struct kvm_vm *dst_vm; + struct stat src_stat; + struct stat dst_stat; + + dst_vm = vm_create_barebones(); + + /* Linking with a nonexistent fd */ + dst_fd = __vm_link_guest_memfd(dst_vm, 99, 0); + TEST_ASSERT_EQ(dst_fd, -1); + TEST_ASSERT_EQ(errno, EINVAL); + + /* Linking with a non-gmem fd */ + dst_fd = __vm_link_guest_memfd(dst_vm, 0, 1); + TEST_ASSERT_EQ(dst_fd, -1); + TEST_ASSERT_EQ(errno, EINVAL); + + /* Linking with invalid flags */ + dst_fd = __vm_link_guest_memfd(dst_vm, src_fd, 1); + TEST_ASSERT_EQ(dst_fd, -1); + TEST_ASSERT_EQ(errno, EINVAL); + + /* Linking with an already-associated vm */ + dst_fd = __vm_link_guest_memfd(src_vm, src_fd, 1); + TEST_ASSERT_EQ(dst_fd, -1); + TEST_ASSERT_EQ(errno, EINVAL); + + dst_fd = __vm_link_guest_memfd(dst_vm, src_fd, 0); + TEST_ASSERT(dst_vm > 0, "linking should succeed with valid inputs"); + TEST_ASSERT(src_fd != dst_fd, "linking should return a different fd"); + + ret = fstat(src_fd, &src_stat); + TEST_ASSERT_EQ(ret, 0); + ret = fstat(dst_fd, &dst_stat); + TEST_ASSERT_EQ(ret, 0); + TEST_ASSERT(src_stat.st_ino == dst_stat.st_ino, + "src and dst files should have the same inode number"); +} + int main(int argc, char *argv[]) { size_t page_size; @@ -194,6 +236,7 @@ int main(int argc, char *argv[]) test_file_size(fd, page_size, total_size); test_fallocate(fd, page_size, total_size); test_invalid_punch_hole(fd, page_size, total_size); + test_link(vm, fd, total_size);

close(fd); } diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 373912464fb4..68faa658b69e 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -571,6 +571,24 @@ static inline int vm_create_guest_memfd(struct kvm_vm *vm, uint64_t size, return fd; }

+static inline int __vm_link_guest_memfd(struct kvm_vm *vm, int fd, uint64_t flags) +{ + struct kvm_link_guest_memfd params = { + .fd = fd, + .flags = flags, + }; + + return __vm_ioctl(vm, KVM_LINK_GUEST_MEMFD, &params); +} + +static inline int vm_link_guest_memfd(struct kvm_vm *vm, int fd, uint64_t flags) +{ + int new_fd = __vm_link_guest_memfd(vm, fd, flags); + + TEST_ASSERT(new_fd >= 0, KVM_IOCTL_ERROR(KVM_LINK_GUEST_MEMFD, new_fd)); + return new_fd; +} + void vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags, uint64_t gpa, uint64_t size, void *hva); int __vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags,

From: Ackerley Tng ackerleytng@google.com

The migration_in_progress flag will also be needed for migration of non-sev VMs.

Co-developed-by: Sagi Shahar sagis@google.com Signed-off-by: Sagi Shahar sagis@google.com Co-developed-by: Vishal Annapurve vannapurve@google.com Signed-off-by: Vishal Annapurve vannapurve@google.com Signed-off-by: Ackerley Tng ackerleytng@google.com Signed-off-by: Ryan Afranji afranji@google.com --- arch/x86/kvm/svm/sev.c | 17 ++++++----------- arch/x86/kvm/svm/svm.h | 1 - include/linux/kvm_host.h | 1 + 3 files changed, 7 insertions(+), 12 deletions(-)

diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 0bc708ee2788..89c06cfcc200 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -1838,8 +1838,6 @@ static bool is_cmd_allowed_from_mirror(u32 cmd_id)

static int sev_lock_two_vms(struct kvm *dst_kvm, struct kvm *src_kvm) { - struct kvm_sev_info *dst_sev = to_kvm_sev_info(dst_kvm); - struct kvm_sev_info *src_sev = to_kvm_sev_info(src_kvm); int r = -EBUSY;

if (dst_kvm == src_kvm) @@ -1849,10 +1847,10 @@ static int sev_lock_two_vms(struct kvm *dst_kvm, struct kvm *src_kvm) * Bail if these VMs are already involved in a migration to avoid * deadlock between two VMs trying to migrate to/from each other. */ - if (atomic_cmpxchg_acquire(&dst_sev->migration_in_progress, 0, 1)) + if (atomic_cmpxchg_acquire(&dst_kvm->migration_in_progress, 0, 1)) return -EBUSY;

- if (atomic_cmpxchg_acquire(&src_sev->migration_in_progress, 0, 1)) + if (atomic_cmpxchg_acquire(&src_kvm->migration_in_progress, 0, 1)) goto release_dst;

r = -EINTR; @@ -1865,21 +1863,18 @@ static int sev_lock_two_vms(struct kvm *dst_kvm, struct kvm *src_kvm) unlock_dst: mutex_unlock(&dst_kvm->lock); release_src: - atomic_set_release(&src_sev->migration_in_progress, 0); + atomic_set_release(&src_kvm->migration_in_progress, 0); release_dst: - atomic_set_release(&dst_sev->migration_in_progress, 0); + atomic_set_release(&dst_kvm->migration_in_progress, 0); return r; }

static void sev_unlock_two_vms(struct kvm *dst_kvm, struct kvm *src_kvm) { - struct kvm_sev_info *dst_sev = to_kvm_sev_info(dst_kvm); - struct kvm_sev_info *src_sev = to_kvm_sev_info(src_kvm); - mutex_unlock(&dst_kvm->lock); mutex_unlock(&src_kvm->lock); - atomic_set_release(&dst_sev->migration_in_progress, 0); - atomic_set_release(&src_sev->migration_in_progress, 0); + atomic_set_release(&dst_kvm->migration_in_progress, 0); + atomic_set_release(&src_kvm->migration_in_progress, 0); }

/* vCPU mutex subclasses. */ diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index d4490eaed55d..35df8be621c5 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -107,7 +107,6 @@ struct kvm_sev_info { struct list_head mirror_vms; /* List of VMs mirroring */ struct list_head mirror_entry; /* Use as a list entry of mirrors */ struct misc_cg *misc_cg; /* For misc cgroup accounting */ - atomic_t migration_in_progress; void *snp_context; /* SNP guest context page */ void *guest_req_buf; /* Bounce buffer for SNP Guest Request input */ void *guest_resp_buf; /* Bounce buffer for SNP Guest Request output */ diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 1dedc421b3e3..0c1d637a6e7d 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -862,6 +862,7 @@ struct kvm { /* Protected by slots_locks (for writes) and RCU (for reads) */ struct xarray mem_attr_array; #endif + atomic_t migration_in_progress; char stats_id[KVM_STATS_NAME_SIZE]; };

From: Ackerley Tng ackerleytng@google.com

Co-developed-by: Sagi Shahar sagis@google.com Signed-off-by: Sagi Shahar sagis@google.com Co-developed-by: Vishal Annapurve vannapurve@google.com Signed-off-by: Vishal Annapurve vannapurve@google.com Signed-off-by: Ackerley Tng ackerleytng@google.com Signed-off-by: Ryan Afranji afranji@google.com --- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/svm/sev.c | 29 +++--------------------- arch/x86/kvm/svm/svm.h | 2 +- arch/x86/kvm/x86.c | 39 ++++++++++++++++++++++++++++++++- 4 files changed, 43 insertions(+), 29 deletions(-)

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 6c06f3d6e081..179618300270 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1871,7 +1871,7 @@ struct kvm_x86_ops { int (*mem_enc_register_region)(struct kvm *kvm, struct kvm_enc_region *argp); int (*mem_enc_unregister_region)(struct kvm *kvm, struct kvm_enc_region *argp); int (*vm_copy_enc_context_from)(struct kvm *kvm, unsigned int source_fd); - int (*vm_move_enc_context_from)(struct kvm *kvm, unsigned int source_fd); + int (*vm_move_enc_context_from)(struct kvm *kvm, struct kvm *source_kvm); void (*guest_memory_reclaimed)(struct kvm *kvm);

int (*get_feature_msr)(u32 msr, u64 *data); diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index b3048ec411e2..689521d9e26f 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -2000,34 +2000,15 @@ static int sev_check_source_vcpus(struct kvm *dst, struct kvm *src) return 0; }

-int sev_vm_move_enc_context_from(struct kvm *kvm, unsigned int source_fd) +int sev_vm_move_enc_context_from(struct kvm *kvm, struct kvm *source_kvm) { struct kvm_sev_info *dst_sev = to_kvm_sev_info(kvm); struct kvm_sev_info *src_sev, *cg_cleanup_sev; - CLASS(fd, f)(source_fd); - struct kvm *source_kvm; bool charged = false; int ret;

- if (fd_empty(f)) - return -EBADF; - - if (!file_is_kvm(fd_file(f))) - return -EBADF; - - source_kvm = fd_file(f)->private_data; - ret = kvm_mark_migration_in_progress(kvm, source_kvm); - if (ret) - return ret; - ret = kvm_lock_two_vms(kvm, source_kvm); - if (ret) - goto out_mark_migration_done; - - if (kvm->arch.vm_type != source_kvm->arch.vm_type || - sev_guest(kvm) || !sev_guest(source_kvm)) { - ret = -EINVAL; - goto out_unlock; - } + if (sev_guest(kvm) || !sev_guest(source_kvm)) + return -EINVAL;

src_sev = to_kvm_sev_info(source_kvm);

@@ -2066,10 +2047,6 @@ int sev_vm_move_enc_context_from(struct kvm *kvm, unsigned int source_fd) sev_misc_cg_uncharge(cg_cleanup_sev); put_misc_cg(cg_cleanup_sev->misc_cg); cg_cleanup_sev->misc_cg = NULL; -out_unlock: - kvm_unlock_two_vms(kvm, source_kvm); -out_mark_migration_done: - kvm_mark_migration_done(kvm, source_kvm); return ret; }

diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 35df8be621c5..7bd31c0b135a 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -757,7 +757,7 @@ int sev_mem_enc_register_region(struct kvm *kvm, int sev_mem_enc_unregister_region(struct kvm *kvm, struct kvm_enc_region *range); int sev_vm_copy_enc_context_from(struct kvm *kvm, unsigned int source_fd); -int sev_vm_move_enc_context_from(struct kvm *kvm, unsigned int source_fd); +int sev_vm_move_enc_context_from(struct kvm *kvm, struct kvm *source_kvm); void sev_guest_memory_reclaimed(struct kvm *kvm); int sev_handle_vmgexit(struct kvm_vcpu *vcpu);

diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 422c66a033d2..637540309456 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -6597,6 +6597,43 @@ int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event, return 0; }

+static int kvm_vm_move_enc_context_from(struct kvm *kvm, unsigned int source_fd) +{ + int r; + struct kvm *source_kvm; + struct fd f = fdget(source_fd); + struct file *file = fd_file(f); + + r = -EBADF; + if (!file) + return r; + + if (!file_is_kvm(file)) + goto out_fdput; + + r = -EINVAL; + source_kvm = file->private_data; + if (kvm->arch.vm_type != source_kvm->arch.vm_type) + goto out_fdput; + + r = kvm_mark_migration_in_progress(kvm, source_kvm); + if (r) + goto out_fdput; + + r = kvm_lock_two_vms(kvm, source_kvm); + if (r) + goto out_mark_migration_done; + + r = kvm_x86_call(vm_move_enc_context_from)(kvm, source_kvm); + + kvm_unlock_two_vms(kvm, source_kvm); +out_mark_migration_done: + kvm_mark_migration_done(kvm, source_kvm); +out_fdput: + fdput(f); + return r; +} + int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap) { @@ -6738,7 +6775,7 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, if (!kvm_x86_ops.vm_move_enc_context_from) break;

- r = kvm_x86_call(vm_move_enc_context_from)(kvm, cap->args[0]); + r = kvm_vm_move_enc_context_from(kvm, cap->args[0]); break; case KVM_CAP_EXIT_HYPERCALL: if (cap->args[0] & ~KVM_EXIT_HYPERCALL_VALID_MASK) {

From: Ackerley Tng ackerleytng@google.com

Migration of memory context involves moving lpage_info and mem_attr_array from source to destination VM.

Co-developed-by: Sagi Shahar sagis@google.com Signed-off-by: Sagi Shahar sagis@google.com Co-developed-by: Vishal Annapurve vannapurve@google.com Signed-off-by: Vishal Annapurve vannapurve@google.com Signed-off-by: Ackerley Tng ackerleytng@google.com Signed-off-by: Ryan Afranji afranji@google.com --- arch/x86/kvm/x86.c | 110 +++++++++++++++++++++++++++++++++++++++ include/linux/kvm_host.h | 17 ++++++ virt/kvm/guest_memfd.c | 25 +++++++++ 3 files changed, 152 insertions(+)

diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 3a7e05c47aa8..887702781465 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -4564,6 +4564,33 @@ void kvm_unlock_two_vms(struct kvm *dst_kvm, struct kvm *src_kvm) } EXPORT_SYMBOL_GPL(kvm_unlock_two_vms);

+static int kvm_lock_vm_memslots(struct kvm *dst_kvm, struct kvm *src_kvm) +{ + int r = -EINVAL; + + if (dst_kvm == src_kvm) + return r; + + r = -EINTR; + if (mutex_lock_killable(&dst_kvm->slots_lock)) + return r; + + if (mutex_lock_killable_nested(&src_kvm->slots_lock, SINGLE_DEPTH_NESTING)) + goto unlock_dst; + + return 0; + +unlock_dst: + mutex_unlock(&dst_kvm->slots_lock); + return r; +} + +static void kvm_unlock_vm_memslots(struct kvm *dst_kvm, struct kvm *src_kvm) +{ + mutex_unlock(&src_kvm->slots_lock); + mutex_unlock(&dst_kvm->slots_lock); +} + /* * Read or write a bunch of msrs. All parameters are kernel addresses. * @@ -6597,6 +6624,78 @@ int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event, return 0; }

+static bool memslot_configurations_match(struct kvm_memslots *src_slots, + struct kvm_memslots *dst_slots) +{ + struct kvm_memslot_iter src_iter; + struct kvm_memslot_iter dst_iter; + + kvm_for_each_memslot_pair(&src_iter, src_slots, &dst_iter, dst_slots) { + if (src_iter.slot->base_gfn != dst_iter.slot->base_gfn || + src_iter.slot->npages != dst_iter.slot->npages || + src_iter.slot->flags != dst_iter.slot->flags) + return false; + + if (kvm_slot_can_be_private(dst_iter.slot) && + !kvm_gmem_params_match(src_iter.slot, dst_iter.slot)) + return false; + } + + /* There should be no more nodes to iterate if configurations match */ + return !src_iter.node && !dst_iter.node; +} + +static int kvm_move_memory_ctxt_from(struct kvm *dst, struct kvm *src) +{ + struct kvm_memslot_iter src_iter; + struct kvm_memslot_iter dst_iter; + struct kvm_memslots *src_slots, *dst_slots; + int i; + + /* TODO: Do we also need to check consistency for as_id == SMM? */ + src_slots = __kvm_memslots(src, 0); + dst_slots = __kvm_memslots(dst, 0); + + if (!memslot_configurations_match(src_slots, dst_slots)) + return -EINVAL; + + /* + * Transferring lpage_info is an optimization, lpage_info can be rebuilt + * by the destination VM. + */ + kvm_for_each_memslot_pair(&src_iter, src_slots, &dst_iter, dst_slots) { + for (i = 1; i < KVM_NR_PAGE_SIZES; ++i) { + unsigned long ugfn = dst_iter.slot->userspace_addr >> PAGE_SHIFT; + int level = i + 1; + + /* + * If the gfn and userspace address are not aligned wrt each + * other, skip migrating lpage_info. + */ + if ((dst_iter.slot->base_gfn ^ ugfn) & + (KVM_PAGES_PER_HPAGE(level) - 1)) + continue; + + kvfree(dst_iter.slot->arch.lpage_info[i - 1]); + dst_iter.slot->arch.lpage_info[i - 1] = + src_iter.slot->arch.lpage_info[i - 1]; + src_iter.slot->arch.lpage_info[i - 1] = NULL; + } + } + +#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES + /* + * For VMs that don't use private memory, this will just be moving an + * empty xarray pointer. + */ + dst->mem_attr_array.xa_head = src->mem_attr_array.xa_head; + src->mem_attr_array.xa_head = NULL; +#endif + + kvm_vm_dead(src); + return 0; +} + static int kvm_vm_move_enc_context_from(struct kvm *kvm, unsigned int source_fd) { int r; @@ -6624,6 +6723,14 @@ static int kvm_vm_move_enc_context_from(struct kvm *kvm, unsigned int source_fd) if (r) goto out_mark_migration_done;

+ r = kvm_lock_vm_memslots(kvm, source_kvm); + if (r) + goto out_unlock; + + r = kvm_move_memory_ctxt_from(kvm, source_kvm); + if (r) + goto out_unlock_memslots; + /* * Different types of VMs will allow userspace to define if moving * encryption context should be required. @@ -6633,6 +6740,9 @@ static int kvm_vm_move_enc_context_from(struct kvm *kvm, unsigned int source_fd) r = kvm_x86_call(vm_move_enc_context_from)(kvm, source_kvm); }

+out_unlock_memslots: + kvm_unlock_vm_memslots(kvm, source_kvm); +out_unlock: kvm_unlock_two_vms(kvm, source_kvm); out_mark_migration_done: kvm_mark_migration_done(kvm, source_kvm); diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 0c1d637a6e7d..99abe9879856 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -1197,6 +1197,16 @@ struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn); struct kvm_memslots *kvm_vcpu_memslots(struct kvm_vcpu *vcpu); struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn);

+ +/* Iterate over a pair of memslots in gfn order until one of the trees end */ +#define kvm_for_each_memslot_pair(iter1, slots1, iter2, slots2) \ + for (kvm_memslot_iter_start(iter1, slots1, 0), \ + kvm_memslot_iter_start(iter2, slots2, 0); \ + kvm_memslot_iter_is_valid(iter1, U64_MAX) && \ + kvm_memslot_iter_is_valid(iter2, U64_MAX); \ + kvm_memslot_iter_next(iter1), \ + kvm_memslot_iter_next(iter2)) + /* * KVM_SET_USER_MEMORY_REGION ioctl allows the following operations: * - create a new memory slot @@ -2521,6 +2531,8 @@ static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn) int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn, kvm_pfn_t *pfn, struct page **page, int *max_order); +bool kvm_gmem_params_match(struct kvm_memory_slot *slot1, + struct kvm_memory_slot *slot2); #else static inline int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn, @@ -2530,6 +2542,11 @@ static inline int kvm_gmem_get_pfn(struct kvm *kvm, KVM_BUG_ON(1, kvm); return -EIO; } +static inline bool kvm_gmem_params_match(struct kvm_memory_slot *slot1, + struct kvm_memory_slot *slot2) +{ + return false; +} #endif /* CONFIG_KVM_PRIVATE_MEM */

#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index d76bd1119198..1a4198c4a4dd 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -778,6 +778,31 @@ int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, } EXPORT_SYMBOL_GPL(kvm_gmem_get_pfn);

+bool kvm_gmem_params_match(struct kvm_memory_slot *slot1, + struct kvm_memory_slot *slot2) +{ + bool ret; + struct file *file1; + struct file *file2; + + if (slot1->gmem.pgoff != slot2->gmem.pgoff) + return false; + + file1 = kvm_gmem_get_file(slot1); + file2 = kvm_gmem_get_file(slot2); + + ret = (file1 && file2 && + file_inode(file1) == file_inode(file2)); + + if (file1) + fput(file1); + if (file2) + fput(file2); + + return ret; +} +EXPORT_SYMBOL_GPL(kvm_gmem_params_match); + #ifdef CONFIG_KVM_GENERIC_PRIVATE_MEM long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, long npages, kvm_gmem_populate_cb post_populate, void *opaque)

From: Ackerley Tng ackerleytng@google.com

Tests that private mem (in guest_mem files) can be migrated. Also demonstrates the migration flow.

Signed-off-by: Ackerley Tng ackerleytng@google.com Signed-off-by: Ryan Afranji afranji@google.com --- tools/testing/selftests/kvm/Makefile.kvm | 1 + .../kvm/x86/private_mem_migrate_tests.c | 56 ++++++++++--------- 2 files changed, 32 insertions(+), 25 deletions(-)

diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index f62b0a5aba35..e9d53ea6c6c8 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -85,6 +85,7 @@ TEST_GEN_PROGS_x86 += x86/platform_info_test TEST_GEN_PROGS_x86 += x86/pmu_counters_test TEST_GEN_PROGS_x86 += x86/pmu_event_filter_test TEST_GEN_PROGS_x86 += x86/private_mem_conversions_test +TEST_GEN_PROGS_x86 += x86/private_mem_migrate_tests TEST_GEN_PROGS_x86 += x86/private_mem_kvm_exits_test TEST_GEN_PROGS_x86 += x86/set_boot_cpu_id TEST_GEN_PROGS_x86 += x86/set_sregs_test diff --git a/tools/testing/selftests/kvm/x86/private_mem_migrate_tests.c b/tools/testing/selftests/kvm/x86/private_mem_migrate_tests.c index 4226de3ebd41..4ad94ea04b66 100644 --- a/tools/testing/selftests/kvm/x86/private_mem_migrate_tests.c +++ b/tools/testing/selftests/kvm/x86/private_mem_migrate_tests.c @@ -1,32 +1,32 @@ // SPDX-License-Identifier: GPL-2.0 -#include "kvm_util_base.h" +#include "kvm_util.h" #include "test_util.h" #include "ucall_common.h" #include <linux/kvm.h> #include <linux/sizes.h>

-#define TRANSFER_PRIVATE_MEM_TEST_SLOT 10 -#define TRANSFER_PRIVATE_MEM_GPA ((uint64_t)(1ull << 32)) -#define TRANSFER_PRIVATE_MEM_GVA TRANSFER_PRIVATE_MEM_GPA -#define TRANSFER_PRIVATE_MEM_VALUE 0xdeadbeef +#define MIGRATE_PRIVATE_MEM_TEST_SLOT 10 +#define MIGRATE_PRIVATE_MEM_GPA ((uint64_t)(1ull << 32)) +#define MIGRATE_PRIVATE_MEM_GVA MIGRATE_PRIVATE_MEM_GPA +#define MIGRATE_PRIVATE_MEM_VALUE 0xdeadbeef

-static void transfer_private_mem_guest_code_src(void) +static void migrate_private_mem_data_guest_code_src(void) { - uint64_t volatile *const ptr = (uint64_t *)TRANSFER_PRIVATE_MEM_GVA; + uint64_t volatile *const ptr = (uint64_t *)MIGRATE_PRIVATE_MEM_GVA;

- *ptr = TRANSFER_PRIVATE_MEM_VALUE; + *ptr = MIGRATE_PRIVATE_MEM_VALUE;

GUEST_SYNC1(*ptr); }

-static void transfer_private_mem_guest_code_dst(void) +static void migrate_private_mem_guest_code_dst(void) { - uint64_t volatile *const ptr = (uint64_t *)TRANSFER_PRIVATE_MEM_GVA; + uint64_t volatile *const ptr = (uint64_t *)MIGRATE_PRIVATE_MEM_GVA;

GUEST_SYNC1(*ptr); }

-static void test_transfer_private_mem(void) +static void test_migrate_private_mem_data(bool migrate) { struct kvm_vm *src_vm, *dst_vm; struct kvm_vcpu *src_vcpu, *dst_vcpu; @@ -40,40 +40,43 @@ static void test_transfer_private_mem(void)

/* Build the source VM, use it to write to private memory */ src_vm = __vm_create_shape_with_one_vcpu( - shape, &src_vcpu, 0, transfer_private_mem_guest_code_src); + shape, &src_vcpu, 0, migrate_private_mem_data_guest_code_src); src_memfd = vm_create_guest_memfd(src_vm, SZ_4K, 0);

- vm_mem_add(src_vm, DEFAULT_VM_MEM_SRC, TRANSFER_PRIVATE_MEM_GPA, - TRANSFER_PRIVATE_MEM_TEST_SLOT, 1, KVM_MEM_PRIVATE, + vm_mem_add(src_vm, DEFAULT_VM_MEM_SRC, MIGRATE_PRIVATE_MEM_GPA, + MIGRATE_PRIVATE_MEM_TEST_SLOT, 1, KVM_MEM_GUEST_MEMFD, src_memfd, 0);

- virt_map(src_vm, TRANSFER_PRIVATE_MEM_GVA, TRANSFER_PRIVATE_MEM_GPA, 1); - vm_set_memory_attributes(src_vm, TRANSFER_PRIVATE_MEM_GPA, SZ_4K, + virt_map(src_vm, MIGRATE_PRIVATE_MEM_GVA, MIGRATE_PRIVATE_MEM_GPA, 1); + vm_set_memory_attributes(src_vm, MIGRATE_PRIVATE_MEM_GPA, SZ_4K, KVM_MEMORY_ATTRIBUTE_PRIVATE);

vcpu_run(src_vcpu); TEST_ASSERT_KVM_EXIT_REASON(src_vcpu, KVM_EXIT_IO); get_ucall(src_vcpu, &uc); - TEST_ASSERT(uc.args[0] == TRANSFER_PRIVATE_MEM_VALUE, + TEST_ASSERT(uc.args[0] == MIGRATE_PRIVATE_MEM_VALUE, "Source VM should be able to write to private memory");

/* Build the destination VM with linked fd */ dst_vm = __vm_create_shape_with_one_vcpu( - shape, &dst_vcpu, 0, transfer_private_mem_guest_code_dst); + shape, &dst_vcpu, 0, migrate_private_mem_guest_code_dst); dst_memfd = vm_link_guest_memfd(dst_vm, src_memfd, 0);

- vm_mem_add(dst_vm, DEFAULT_VM_MEM_SRC, TRANSFER_PRIVATE_MEM_GPA, - TRANSFER_PRIVATE_MEM_TEST_SLOT, 1, KVM_MEM_PRIVATE, + vm_mem_add(dst_vm, DEFAULT_VM_MEM_SRC, MIGRATE_PRIVATE_MEM_GPA, + MIGRATE_PRIVATE_MEM_TEST_SLOT, 1, KVM_MEM_GUEST_MEMFD, dst_memfd, 0);

- virt_map(dst_vm, TRANSFER_PRIVATE_MEM_GVA, TRANSFER_PRIVATE_MEM_GPA, 1); - vm_set_memory_attributes(dst_vm, TRANSFER_PRIVATE_MEM_GPA, SZ_4K, - KVM_MEMORY_ATTRIBUTE_PRIVATE); + virt_map(dst_vm, MIGRATE_PRIVATE_MEM_GVA, MIGRATE_PRIVATE_MEM_GPA, 1); + if (migrate) + vm_migrate_from(dst_vm, src_vm); + else + vm_set_memory_attributes(dst_vm, MIGRATE_PRIVATE_MEM_GPA, SZ_4K, + KVM_MEMORY_ATTRIBUTE_PRIVATE);

vcpu_run(dst_vcpu); TEST_ASSERT_KVM_EXIT_REASON(dst_vcpu, KVM_EXIT_IO); get_ucall(dst_vcpu, &uc); - TEST_ASSERT(uc.args[0] == TRANSFER_PRIVATE_MEM_VALUE, + TEST_ASSERT(uc.args[0] == MIGRATE_PRIVATE_MEM_VALUE, "Destination VM should be able to read value transferred"); }

@@ -81,7 +84,10 @@ int main(int argc, char *argv[]) { TEST_REQUIRE(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM));

- test_transfer_private_mem(); + test_migrate_private_mem_data(false); + + if (kvm_check_cap(KVM_CAP_VM_MOVE_ENC_CONTEXT_FROM)) + test_migrate_private_mem_data(true);

return 0; }