From: Jiri Pirko jiri@nvidia.com

dma_addr is unitialized in dma_direct_map_phys() when swiotlb is forced and DMA_ATTR_MMIO is set which leads to random value print out in warning. Fix that by just returning DMA_MAPPING_ERROR.

Fixes: e53d29f957b3 ("dma-mapping: convert dma_direct_*map_page to be phys_addr_t based") Signed-off-by: Jiri Pirko jiri@nvidia.com --- kernel/dma/direct.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kernel/dma/direct.h b/kernel/dma/direct.h index da2fadf45bcd..62f0d9d0ba02 100644 --- a/kernel/dma/direct.h +++ b/kernel/dma/direct.h @@ -88,7 +88,7 @@ static inline dma_addr_t dma_direct_map_phys(struct device *dev,

if (is_swiotlb_force_bounce(dev)) { if (attrs & DMA_ATTR_MMIO) - goto err_overflow; + return DMA_MAPPING_ERROR;

return swiotlb_map(dev, phys, size, dir, attrs); }

From: Jiri Pirko jiri@nvidia.com

This is only relevant inside confidential computing (CoCo) virtual machines, not on the hypervisor side.

Current CoCo designs don't place a vIOMMU in front of untrusted devices. Instead, the DMA API forces all untrusted device DMA through swiotlb bounce buffers (is_swiotlb_force_bounce()) which copies data into decrypted memory on behalf of the device.

When a caller has already arranged for the memory to be decrypted via set_memory_decrypted(), the DMA API needs to know so it can map directly using the unencrypted physical address rather than bounce buffering. Following the pattern of DMA_ATTR_MMIO, add DMA_ATTR_CC_DECRYPTED for this purpose. Like the MMIO case, only the caller knows what kind of memory it has and must inform the DMA API for it to work correctly.

Signed-off-by: Jiri Pirko jiri@nvidia.com --- include/linux/dma-mapping.h | 7 +++++++ include/trace/events/dma.h | 3 ++- kernel/dma/direct.h | 14 +++++++++++--- 3 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index aa36a0d1d9df..052235feb853 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -78,6 +78,13 @@ */ #define DMA_ATTR_MMIO (1UL << 10)

+/* + * DMA_ATTR_CC_DECRYPTED: Indicates memory that has been explicitly decrypted + * (shared) for confidential computing guests. The caller must have + * called set_memory_decrypted(). A struct page is required. + */ +#define DMA_ATTR_CC_DECRYPTED (1UL << 11) + /* * A dma_addr_t can hold any valid DMA or bus address for the platform. It can * be given to a device to use as a DMA source or target. It is specific to a diff --git a/include/trace/events/dma.h b/include/trace/events/dma.h index b3fef140ae15..b3c2cee8841a 100644 --- a/include/trace/events/dma.h +++ b/include/trace/events/dma.h @@ -32,7 +32,8 @@ TRACE_DEFINE_ENUM(DMA_NONE); { DMA_ATTR_ALLOC_SINGLE_PAGES, "ALLOC_SINGLE_PAGES" }, \ { DMA_ATTR_NO_WARN, "NO_WARN" }, \ { DMA_ATTR_PRIVILEGED, "PRIVILEGED" }, \ - { DMA_ATTR_MMIO, "MMIO" }) + { DMA_ATTR_MMIO, "MMIO" }, \ + { DMA_ATTR_CC_DECRYPTED, "CC_DECRYPTED" })

DECLARE_EVENT_CLASS(dma_map, TP_PROTO(struct device *dev, phys_addr_t phys_addr, dma_addr_t dma_addr, diff --git a/kernel/dma/direct.h b/kernel/dma/direct.h index 62f0d9d0ba02..ae5bc1919e1c 100644 --- a/kernel/dma/direct.h +++ b/kernel/dma/direct.h @@ -87,16 +87,24 @@ static inline dma_addr_t dma_direct_map_phys(struct device *dev, dma_addr_t dma_addr;

if (is_swiotlb_force_bounce(dev)) { - if (attrs & DMA_ATTR_MMIO) - return DMA_MAPPING_ERROR; + if (!(attrs & DMA_ATTR_CC_DECRYPTED)) { + if (attrs & DMA_ATTR_MMIO) + return DMA_MAPPING_ERROR;

- return swiotlb_map(dev, phys, size, dir, attrs); + return swiotlb_map(dev, phys, size, dir, attrs); + } + } else if (attrs & DMA_ATTR_CC_DECRYPTED) { + return DMA_MAPPING_ERROR; }

if (attrs & DMA_ATTR_MMIO) { dma_addr = phys; if (unlikely(!dma_capable(dev, dma_addr, size, false))) goto err_overflow; + } else if (attrs & DMA_ATTR_CC_DECRYPTED) { + dma_addr = phys_to_dma_unencrypted(dev, phys); + if (unlikely(!dma_capable(dev, dma_addr, size, false))) + goto err_overflow; } else { dma_addr = phys_to_dma(dev, phys); if (unlikely(!dma_capable(dev, dma_addr, size, true)) ||

From: Jiri Pirko jiri@nvidia.com

Currently the flags, which are unused, are validated for all heaps. Since the follow-up patch introduces a flag valid for only one of the heaps, allow to specify the valid flags per-heap.

Signed-off-by: Jiri Pirko jiri@nvidia.com --- drivers/dma-buf/dma-heap.c | 5 ++++- include/linux/dma-heap.h | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/drivers/dma-buf/dma-heap.c b/drivers/dma-buf/dma-heap.c index 8ab49924f8b7..4751bcef4b19 100644 --- a/drivers/dma-buf/dma-heap.c +++ b/drivers/dma-buf/dma-heap.c @@ -28,6 +28,7 @@ * @name: used for debugging/device-node name * @ops: ops struct for this heap * @priv: private data for this heap + * @valid_heap_flags: valid heap flags for this heap * @heap_devt: heap device node * @list: list head connecting to list of heaps * @heap_cdev: heap char device @@ -38,6 +39,7 @@ struct dma_heap { const char *name; const struct dma_heap_ops *ops; void *priv; + u64 valid_heap_flags; dev_t heap_devt; struct list_head list; struct cdev heap_cdev; @@ -105,7 +107,7 @@ static long dma_heap_ioctl_allocate(struct file *file, void *data) if (heap_allocation->fd_flags & ~DMA_HEAP_VALID_FD_FLAGS) return -EINVAL;

- if (heap_allocation->heap_flags & ~DMA_HEAP_VALID_HEAP_FLAGS) + if (heap_allocation->heap_flags & ~heap->valid_heap_flags) return -EINVAL;

fd = dma_heap_buffer_alloc(heap, heap_allocation->len, @@ -246,6 +248,7 @@ struct dma_heap *dma_heap_add(const struct dma_heap_export_info *exp_info) heap->name = exp_info->name; heap->ops = exp_info->ops; heap->priv = exp_info->priv; + heap->valid_heap_flags = exp_info->valid_heap_flags;

/* Find unused minor number */ ret = xa_alloc(&dma_heap_minors, &minor, heap, diff --git a/include/linux/dma-heap.h b/include/linux/dma-heap.h index 27d15f60950a..7cfb531a9281 100644 --- a/include/linux/dma-heap.h +++ b/include/linux/dma-heap.h @@ -31,6 +31,7 @@ struct dma_heap_ops { * @name: used for debugging/device-node name * @ops: ops struct for this heap * @priv: heap exporter private data + * @valid_heap_flags: valid heap flags for this heap * * Information needed to export a new dmabuf heap. */ @@ -38,6 +39,7 @@ struct dma_heap_export_info { const char *name; const struct dma_heap_ops *ops; void *priv; + u64 valid_heap_flags; };

void *dma_heap_get_drvdata(struct dma_heap *heap);

From: Jiri Pirko jiri@nvidia.com

Add a new DMA_HEAP_FLAG_DECRYPTED heap flag to allow userspace to allocate decrypted (shared) memory from the dma-buf system heap for confidential computing (CoCo) VMs.

On CoCo VMs, guest memory is encrypted by default. The hardware uses an encryption bit in page table entries (C-bit on AMD SEV, "shared" bit on Intel TDX) to control whether a given memory access is encrypted or decrypted. The kernel's direct map is set up with encryption enabled, so pages returned by alloc_pages() are encrypted in the direct map by default. To make this memory usable for devices that do not support DMA to encrypted memory (no TDISP support), it has to be explicitly decrypted. A couple of things are needed to properly handle decrypted memory for the dma-buf use case:

- set_memory_decrypted() on the direct map after allocation: Besides clearing the encryption bit in the direct map PTEs, this also notifies the hypervisor about the page state change. On free, the inverse set_memory_encrypted() must be called before returning pages to the allocator. If re-encryption fails, pages are intentionally leaked to prevent decrypted memory from being reused as private.

- pgprot_decrypted() for userspace and kernel virtual mappings: Any new mapping of the decrypted pages, be it to userspace via mmap or to kernel vmalloc space via vmap, creates PTEs independent of the direct map. These must also have the encryption bit cleared, otherwise accesses through them would see encrypted (garbage) data.

- DMA_ATTR_CC_DECRYPTED for DMA mapping: Since the pages are already decrypted, the DMA API needs to be informed via DMA_ATTR_CC_DECRYPTED so it can map them correctly as unencrypted for device access.

On non-CoCo VMs the flag is rejected with -EOPNOTSUPP to prevent misuse by userspace that does not understand the security implications of explicitly decrypted memory.

Signed-off-by: Jiri Pirko jiri@nvidia.com --- drivers/dma-buf/heaps/system_heap.c | 87 +++++++++++++++++++++++++++-- include/linux/dma-heap.h | 1 + include/uapi/linux/dma-heap.h | 12 +++- 3 files changed, 94 insertions(+), 6 deletions(-)

diff --git a/drivers/dma-buf/heaps/system_heap.c b/drivers/dma-buf/heaps/system_heap.c index 124dca56e4d8..0f80ecb660ec 100644 --- a/drivers/dma-buf/heaps/system_heap.c +++ b/drivers/dma-buf/heaps/system_heap.c @@ -10,6 +10,7 @@ * Andrew F. Davis afd@ti.com */

+#include <linux/cc_platform.h> #include <linux/dma-buf.h> #include <linux/dma-mapping.h> #include <linux/dma-heap.h> @@ -17,7 +18,9 @@ #include <linux/highmem.h> #include <linux/mm.h> #include <linux/module.h> +#include <linux/pgtable.h> #include <linux/scatterlist.h> +#include <linux/set_memory.h> #include <linux/slab.h> #include <linux/vmalloc.h>

@@ -29,6 +32,7 @@ struct system_heap_buffer { struct sg_table sg_table; int vmap_cnt; void *vaddr; + bool decrypted; };

struct dma_heap_attachment { @@ -36,6 +40,7 @@ struct dma_heap_attachment { struct sg_table table; struct list_head list; bool mapped; + bool decrypted; };

#define LOW_ORDER_GFP (GFP_HIGHUSER | __GFP_ZERO) @@ -52,6 +57,34 @@ static gfp_t order_flags[] = {HIGH_ORDER_GFP, HIGH_ORDER_GFP, LOW_ORDER_GFP}; static const unsigned int orders[] = {8, 4, 0}; #define NUM_ORDERS ARRAY_SIZE(orders)

+static int system_heap_set_page_decrypted(struct page *page) +{ + unsigned long addr = (unsigned long)page_address(page); + unsigned int nr_pages = 1 << compound_order(page); + int ret; + + ret = set_memory_decrypted(addr, nr_pages); + if (ret) + pr_warn_ratelimited("dma-buf system heap: failed to decrypt page at %p\n", + page_address(page)); + + return ret; +} + +static int system_heap_set_page_encrypted(struct page *page) +{ + unsigned long addr = (unsigned long)page_address(page); + unsigned int nr_pages = 1 << compound_order(page); + int ret; + + ret = set_memory_encrypted(addr, nr_pages); + if (ret) + pr_warn_ratelimited("dma-buf system heap: failed to re-encrypt page at %p, leaking memory\n", + page_address(page)); + + return ret; +} + static int dup_sg_table(struct sg_table *from, struct sg_table *to) { struct scatterlist *sg, *new_sg; @@ -90,6 +123,7 @@ static int system_heap_attach(struct dma_buf *dmabuf, a->dev = attachment->dev; INIT_LIST_HEAD(&a->list); a->mapped = false; + a->decrypted = buffer->decrypted;

attachment->priv = a;

@@ -119,9 +153,11 @@ static struct sg_table *system_heap_map_dma_buf(struct dma_buf_attachment *attac { struct dma_heap_attachment *a = attachment->priv; struct sg_table *table = &a->table; + unsigned long attrs; int ret;

- ret = dma_map_sgtable(attachment->dev, table, direction, 0); + attrs = a->decrypted ? DMA_ATTR_CC_DECRYPTED : 0; + ret = dma_map_sgtable(attachment->dev, table, direction, attrs); if (ret) return ERR_PTR(ret);

@@ -188,8 +224,13 @@ static int system_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) unsigned long addr = vma->vm_start; unsigned long pgoff = vma->vm_pgoff; struct scatterlist *sg; + pgprot_t prot; int i, ret;

+ prot = vma->vm_page_prot; + if (buffer->decrypted) + prot = pgprot_decrypted(prot); + for_each_sgtable_sg(table, sg, i) { unsigned long n = sg->length >> PAGE_SHIFT;

@@ -206,8 +247,7 @@ static int system_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) if (addr + size > vma->vm_end) size = vma->vm_end - addr;

- ret = remap_pfn_range(vma, addr, page_to_pfn(page), - size, vma->vm_page_prot); + ret = remap_pfn_range(vma, addr, page_to_pfn(page), size, prot); if (ret) return ret;

@@ -225,6 +265,7 @@ static void *system_heap_do_vmap(struct system_heap_buffer *buffer) struct page **pages = vmalloc(sizeof(struct page *) * npages); struct page **tmp = pages; struct sg_page_iter piter; + pgprot_t prot; void *vaddr;

if (!pages) @@ -235,7 +276,10 @@ static void *system_heap_do_vmap(struct system_heap_buffer *buffer) *tmp++ = sg_page_iter_page(&piter); }

- vaddr = vmap(pages, npages, VM_MAP, PAGE_KERNEL); + prot = PAGE_KERNEL; + if (buffer->decrypted) + prot = pgprot_decrypted(prot); + vaddr = vmap(pages, npages, VM_MAP, prot); vfree(pages);

if (!vaddr) @@ -296,6 +340,14 @@ static void system_heap_dma_buf_release(struct dma_buf *dmabuf) for_each_sgtable_sg(table, sg, i) { struct page *page = sg_page(sg);

+ /* + * Intentionally leak pages that cannot be re-encrypted + * to prevent decrypted memory from being reused. + */ + if (buffer->decrypted && + system_heap_set_page_encrypted(page)) + continue; + __free_pages(page, compound_order(page)); } sg_free_table(table); @@ -344,6 +396,7 @@ static struct dma_buf *system_heap_allocate(struct dma_heap *heap, DEFINE_DMA_BUF_EXPORT_INFO(exp_info); unsigned long size_remaining = len; unsigned int max_order = orders[0]; + bool decrypted = heap_flags & DMA_HEAP_FLAG_DECRYPTED; struct dma_buf *dmabuf; struct sg_table *table; struct scatterlist *sg; @@ -351,6 +404,15 @@ static struct dma_buf *system_heap_allocate(struct dma_heap *heap, struct page *page, *tmp_page; int i, ret = -ENOMEM;

+ if (decrypted) { + if (!cc_platform_has(CC_ATTR_MEM_ENCRYPT)) + return ERR_PTR(-EOPNOTSUPP); +#ifdef CONFIG_HIGHMEM + /* Sanity check, should not happen. */ + return ERR_PTR(-EINVAL); +#endif + } + buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); if (!buffer) return ERR_PTR(-ENOMEM); @@ -359,6 +421,7 @@ static struct dma_buf *system_heap_allocate(struct dma_heap *heap, mutex_init(&buffer->lock); buffer->heap = heap; buffer->len = len; + buffer->decrypted = decrypted;

INIT_LIST_HEAD(&pages); i = 0; @@ -393,6 +456,14 @@ static struct dma_buf *system_heap_allocate(struct dma_heap *heap, list_del(&page->lru); }

+ if (decrypted) { + for_each_sgtable_sg(table, sg, i) { + ret = system_heap_set_page_decrypted(sg_page(sg)); + if (ret) + goto free_pages; + } + } + /* create the dmabuf */ exp_info.exp_name = dma_heap_get_name(heap); exp_info.ops = &system_heap_buf_ops; @@ -410,6 +481,13 @@ static struct dma_buf *system_heap_allocate(struct dma_heap *heap, for_each_sgtable_sg(table, sg, i) { struct page *p = sg_page(sg);

+ /* + * Intentionally leak pages that cannot be re-encrypted + * to prevent decrypted memory from being reused. + */ + if (buffer->decrypted && + system_heap_set_page_encrypted(p)) + continue; __free_pages(p, compound_order(p)); } sg_free_table(table); @@ -430,6 +508,7 @@ static int __init system_heap_create(void) struct dma_heap_export_info exp_info = { .name = "system", .ops = &system_heap_ops, + .valid_heap_flags = DMA_HEAP_FLAG_DECRYPTED, }; struct dma_heap *sys_heap;

diff --git a/include/linux/dma-heap.h b/include/linux/dma-heap.h index 7cfb531a9281..295a7eaa19ca 100644 --- a/include/linux/dma-heap.h +++ b/include/linux/dma-heap.h @@ -10,6 +10,7 @@ #define _DMA_HEAPS_H

#include <linux/types.h> +#include <uapi/linux/dma-heap.h>

struct dma_heap;

diff --git a/include/uapi/linux/dma-heap.h b/include/uapi/linux/dma-heap.h index a4cf716a49fa..6552c88e52f6 100644 --- a/include/uapi/linux/dma-heap.h +++ b/include/uapi/linux/dma-heap.h @@ -18,8 +18,16 @@ /* Valid FD_FLAGS are O_CLOEXEC, O_RDONLY, O_WRONLY, O_RDWR */ #define DMA_HEAP_VALID_FD_FLAGS (O_CLOEXEC | O_ACCMODE)

-/* Currently no heap flags */ -#define DMA_HEAP_VALID_HEAP_FLAGS (0ULL) +/** + * DMA_HEAP_FLAG_DECRYPTED - Allocate decrypted (shared) memory + * + * For confidential computing guests (AMD SEV, Intel TDX), this flag + * requests that the allocated memory be marked as decrypted (shared + * with the host). + */ +#define DMA_HEAP_FLAG_DECRYPTED (1ULL << 0) + +#define DMA_HEAP_VALID_HEAP_FLAGS (DMA_HEAP_FLAG_DECRYPTED)

/** * struct dma_heap_allocation_data - metadata passed from userspace for