On Sun, Nov 02, 2025 at 10:00:48AM +0200, Leon Romanovsky wrote:
Changelog: v6:
- Fixed wrong error check from pcim_p2pdma_init().
- Documented pcim_p2pdma_provider() function.
- Improved commit messages.
- Added VFIO DMA-BUF selftest.
- Added __counted_by(nr_ranges) annotation to struct vfio_device_feature_dma_buf.
- Fixed error unwind when dma_buf_fd() fails.
- Document latest changes to p2pmem.
- Removed EXPORT_SYMBOL_GPL from pci_p2pdma_map_type.
- Moved DMA mapping logic to DMA-BUF.
- Removed types patch to avoid dependencies between subsystems.
- Moved vfio_pci_dma_buf_move() in err_undo block.
- Added nvgrace patch.
Thanks Leon. Attaching a toy program which sanity tests the dma-buf export UAPI by feeding the allocated dma-buf into an dma-buf importer (libibverbs + CX-7).
Tested-by: Alex Mastro amastro@fb.com
$ cc -Og -Wall -Wextra $(pkg-config --cflags --libs libibverbs) test_dmabuf.c -o test_dmabuf $ ./test_dmabuf 0000:05:00.0 3 4 0 0x1000 opening 0000:05:00.0 via /dev/vfio/56 allocating dma_buf bar_idx=4, bar_offset=0x0, size=0x1000 allocated dma_buf fd=6 discovered 4 ibv devices: mlx5_0 mlx5_1 mlx5_2 mlx5_3 opened ibv device 3: mlx5_3 registered dma_buf unregistered dma_buf closed dma_buf fd
--- #include <fcntl.h> #include <infiniband/verbs.h> #include <libgen.h> #include <linux/limits.h> #include <linux/types.h> #include <linux/vfio.h> #include <stdbool.h> #include <stdint.h> #include <stdio.h> #include <stdlib.h> #include <sys/ioctl.h> #include <unistd.h>
#define ensure(cond) \ do { \ if (!(cond)) { \ fprintf(stderr, \ "%s:%d Condition failed: '%s' (errno=%d: %s)\n", \ __FILE__, __LINE__, #cond, errno, \ strerror(errno)); \ exit(EXIT_FAILURE); \ } \ } while (0)
#ifndef VFIO_DEVICE_FEATURE_DMA_BUF #define VFIO_DEVICE_FEATURE_DMA_BUF 11
struct vfio_region_dma_range { __u64 offset; __u64 length; };
struct vfio_device_feature_dma_buf { __u32 region_index; __u32 open_flags; __u32 flags; __u32 nr_ranges; struct vfio_region_dma_range dma_ranges[]; }; #endif
static uint32_t group_for_bdf(const char *bdf) { char path[PATH_MAX]; char link[PATH_MAX]; int ret;
snprintf(path, sizeof(path), "/sys/bus/pci/devices/%s/iommu_group", bdf); ret = readlink(path, link, sizeof(link)); ensure(ret > 0);
const char *filename = basename(link); ensure(filename);
return strtoul(filename, NULL, 0); }
int main(int argc, char **argv) { int ret;
if (argc != 6) { printf("usage: %s <vfio_bdf> <ibv_device_idx> <bar_idx> <bar_offset> <size>\n", argv[0]); printf("example: %s 0000:05:00.0 3 2 0x20000 0x1000\n", argv[0]); return 1; }
const char *bdf = argv[1]; uint32_t ibv_idx = strtoul(argv[2], NULL, 0); uint32_t bar_idx = strtoul(argv[3], NULL, 0); uint64_t bar_offs = strtoull(argv[4], NULL, 0); uint64_t dmabuf_len = strtoull(argv[5], NULL, 0);
uint32_t group_num = group_for_bdf(bdf); char group_path[PATH_MAX]; snprintf(group_path, sizeof(group_path), "/dev/vfio/%u", group_num);
int container_fd = open("/dev/vfio/vfio", O_RDWR); ensure(container_fd >= 0);
printf("opening %s via %s\n", bdf, group_path); int group_fd = open(group_path, O_RDWR); ensure(group_fd >= 0);
ret = ioctl(group_fd, VFIO_GROUP_SET_CONTAINER, &container_fd); ensure(!ret);
ret = ioctl(container_fd, VFIO_SET_IOMMU, VFIO_TYPE1v2_IOMMU); ensure(!ret);
int device_fd = ioctl(group_fd, VFIO_GROUP_GET_DEVICE_FD, bdf); ensure(device_fd >= 0);
uint8_t buf[sizeof(struct vfio_device_feature) + sizeof(struct vfio_device_feature_dma_buf) + sizeof(struct vfio_region_dma_range)] __attribute__((aligned(32)));
struct vfio_device_feature *ft = (struct vfio_device_feature *)buf; *ft = (struct vfio_device_feature){ .argsz = sizeof(buf), .flags = VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_DMA_BUF, };
struct vfio_device_feature_dma_buf *ft_dma_buf = (struct vfio_device_feature_dma_buf *)ft->data; *ft_dma_buf = (struct vfio_device_feature_dma_buf){ .region_index = bar_idx, .open_flags = O_RDWR, .nr_ranges = 1, };
ft_dma_buf->dma_ranges[0] = (struct vfio_region_dma_range){ .length = dmabuf_len, .offset = bar_offs, };
printf("allocating dma_buf bar_idx=%u, bar_offset=0x%lx, size=0x%lx\n", bar_idx, bar_offs, dmabuf_len); int dmabuf_fd = ioctl(device_fd, VFIO_DEVICE_FEATURE, buf); ensure(dmabuf_fd >= 0); printf("allocated dma_buf fd=%d\n", dmabuf_fd);
int num; struct ibv_device **devs = ibv_get_device_list(&num); ensure(devs && num > 0); printf("discovered %d ibv devices:", num); for (int i = 0; i < num; i++) { printf(" %s", ibv_get_device_name(devs[i])); } printf("\n"); ensure(ibv_idx < (uint32_t)num);
struct ibv_context *ctx = ibv_open_device(devs[ibv_idx]); ensure(ctx); printf("opened ibv device %d: %s\n", ibv_idx, ibv_get_device_name(devs[ibv_idx]));
struct ibv_pd *pd = ibv_alloc_pd(ctx); ensure(pd);
uint64_t offset = 0; uint64_t iova = 0; int access = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE;
struct ibv_mr *mr = ibv_reg_dmabuf_mr(pd, offset, dmabuf_len, iova, dmabuf_fd, access); ensure(mr); printf("registered dma_buf\n");
ret = ibv_dereg_mr(mr); ensure(!ret); printf("unregistered dma_buf\n");
ret = close(dmabuf_fd); ensure(!ret); printf("closed dma_buf fd\n");
return 0; } ---