Add support for IOV interconnect by provding ops for map/unmap and match interconnect. Note that the xarray is populated with entries of type struct range. The range type contains the start and end addresses of the memory region.
Cc: Jason Gunthorpe jgg@nvidia.com Cc: Christian Koenig christian.koenig@amd.com Cc: Sumit Semwal sumit.semwal@linaro.org Cc: Thomas Hellström thomas.hellstrom@linux.intel.com Cc: Simona Vetter simona.vetter@ffwll.ch Signed-off-by: Vivek Kasireddy vivek.kasireddy@intel.com --- drivers/vfio/pci/vfio_pci_dmabuf.c | 135 ++++++++++++++++++++++++++++- 1 file changed, 134 insertions(+), 1 deletion(-)
diff --git a/drivers/vfio/pci/vfio_pci_dmabuf.c b/drivers/vfio/pci/vfio_pci_dmabuf.c index eaba010777f3..d2b7b5410e5a 100644 --- a/drivers/vfio/pci/vfio_pci_dmabuf.c +++ b/drivers/vfio/pci/vfio_pci_dmabuf.c @@ -4,6 +4,7 @@ #include <linux/dma-buf.h> #include <linux/pci-p2pdma.h> #include <linux/dma-resv.h> +#include <linux/range.h>
#include "vfio_pci_priv.h"
@@ -16,15 +17,132 @@ struct vfio_pci_dma_buf { size_t size; struct phys_vec *phys_vec; struct p2pdma_provider *provider; + struct dma_buf_interconnect_match *ic_match; u32 nr_ranges; u8 revoked : 1; };
+static int +vfio_pci_create_match(struct vfio_pci_dma_buf *priv, + struct vfio_device_feature_dma_buf *dma_buf) +{ + struct dma_buf_interconnect_match *ic_match; + + ic_match = kzalloc(sizeof(*ic_match), GFP_KERNEL); + if (!ic_match) + return -ENOMEM; + + ic_match->dev = &priv->vdev->pdev->dev; + ic_match->bar = dma_buf->region_index; + + priv->ic_match = ic_match; + return 0; +} + +static int vfio_pci_map_iov_interconnect(struct vfio_pci_dma_buf *priv, + struct xarray *ranges) +{ + struct phys_vec *phys_vec = priv->phys_vec; + struct range *range; + unsigned long i; + void *entry; + int ret; + + range = kmalloc_array(priv->nr_ranges, sizeof(*range), GFP_KERNEL); + if (!range) + return -ENOMEM; + + for (i = 0; i < priv->nr_ranges; i++) { + entry = &range[i]; + range[i].start = phys_vec[i].paddr; + range[i].end = phys_vec[i].paddr + phys_vec[i].len - 1; + + entry = xa_store(ranges, i, entry, GFP_KERNEL); + if (xa_is_err(entry)) { + ret = xa_err(entry); + goto err_free_range; + } + } + return 0; + +err_free_range: + kfree(range); + return ret; +} + +static int vfio_pci_map_interconnect(struct dma_buf_attachment *attachment, + struct dma_buf_ranges *ranges) +{ + const struct dma_buf_interconnect *ic = attachment->ic_match->type; + struct vfio_pci_dma_buf *priv = attachment->dmabuf->priv; + int ret = -EINVAL; + + ranges->nranges = priv->nr_ranges; + + if (ic == iov_interconnect) + ret = vfio_pci_map_iov_interconnect(priv, &ranges->ranges); + + return ret; +} + +static void vfio_pci_unmap_interconnect(struct dma_buf_attachment *attachment, + struct dma_buf_ranges *ranges) +{ + void *entry; + + entry = xa_load(&ranges->ranges, 0); + kfree(entry); +} + +static bool +vfio_pci_match_iov_interconnect(const struct dma_buf_interconnect_match *exp, + const struct dma_buf_interconnect_match *imp) +{ + struct pci_dev *exp_pdev = to_pci_dev(exp->dev); + struct pci_dev *imp_pdev = to_pci_dev(imp->dev); + + return imp_pdev == pci_physfn(exp_pdev) && imp->bar == exp->bar; +} + +static bool +vfio_pci_match_interconnect(const struct dma_buf_interconnect_match *exp, + const struct dma_buf_interconnect_match *imp) +{ + const struct dma_buf_interconnect *ic = exp->type; + + if (ic == iov_interconnect) + return vfio_pci_match_iov_interconnect(exp, imp); + + return false; +} + +static bool +vfio_pci_match_interconnects(struct vfio_pci_dma_buf *priv, + struct dma_buf_attachment *attachment) +{ + const struct dma_buf_attach_ops *aops = attachment->importer_ops; + const struct dma_buf_interconnect_match supports_ics[] = { + MATCH_INTERCONNECT(iov_interconnect, + priv->ic_match->dev, priv->ic_match->bar), + }; + + if (attachment->allow_ic) { + if (aops->supports_interconnects(attachment, supports_ics, + ARRAY_SIZE(supports_ics))) + return true; + } + return false; +} + static int vfio_pci_dma_buf_attach(struct dma_buf *dmabuf, struct dma_buf_attachment *attachment) { struct vfio_pci_dma_buf *priv = dmabuf->priv;
+ if (vfio_pci_match_interconnects(priv, attachment)) { + return 0; + } + if (!attachment->peer2peer) return -EOPNOTSUPP;
@@ -189,6 +307,7 @@ vfio_pci_dma_buf_map(struct dma_buf_attachment *attachment, return ERR_PTR(ret); }
+ static void vfio_pci_dma_buf_unmap(struct dma_buf_attachment *attachment, struct sg_table *sgt, enum dma_data_direction dir) @@ -228,15 +347,23 @@ static void vfio_pci_dma_buf_release(struct dma_buf *dmabuf) vfio_device_put_registration(&priv->vdev->vdev); } kfree(priv->phys_vec); + kfree(priv->ic_match); kfree(priv); }
+static const struct dma_buf_interconnect_ops vfio_pci_interconnect_ops = { + .match_interconnect = vfio_pci_match_interconnect, + .map_interconnect = vfio_pci_map_interconnect, + .unmap_interconnect = vfio_pci_unmap_interconnect, +}; + static const struct dma_buf_ops vfio_pci_dmabuf_ops = { .attach = vfio_pci_dma_buf_attach, .detach = vfio_pci_dma_buf_detach, .map_dma_buf = vfio_pci_dma_buf_map, .release = vfio_pci_dma_buf_release, .unmap_dma_buf = vfio_pci_dma_buf_unmap, + .interconnect_ops = &vfio_pci_interconnect_ops, };
static void dma_ranges_to_p2p_phys(struct vfio_pci_dma_buf *priv, @@ -365,6 +492,10 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags, goto err_free_phys; }
+ ret = vfio_pci_create_match(priv, &get_dma_buf); + if (ret) + goto err_dev_put; + exp_info.ops = &vfio_pci_dmabuf_ops; exp_info.size = priv->size; exp_info.flags = get_dma_buf.open_flags; @@ -373,7 +504,7 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags, priv->dmabuf = dma_buf_export(&exp_info); if (IS_ERR(priv->dmabuf)) { ret = PTR_ERR(priv->dmabuf); - goto err_dev_put; + goto err_free_iov; }
/* dma_buf_put() now frees priv */ @@ -391,6 +522,8 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags, */ return dma_buf_fd(priv->dmabuf, get_dma_buf.open_flags);
+err_free_iov: + kfree(priv->ic_match); err_dev_put: vfio_device_put_registration(&vdev->vdev); err_free_phys: