The fault object was designed exclusively for hwpt's IO page faults (PRI). But its queue implementation can be reused for other purposes too, such as hardware IRQ and event injections to user space.
Meanwhile, a fault object holds a list of faults. So it's more accurate to call it a "fault queue". Combining the reusing idea above, abstract a new iommufd_eventq as a common structure embedded into struct iommufd_fault, similar to hwpt_paging holding a common hwpt.
Add a common iommufd_eventq_ops and iommufd_eventq_init to prepare for an IOMMUFD_OBJ_VEVENTQ (vIOMMU Event Queue).
Also, add missing xa_destroy and mutex_destroy in iommufd_fault_destroy().
Signed-off-by: Nicolin Chen nicolinc@nvidia.com --- drivers/iommu/iommufd/iommufd_private.h | 52 ++++++--- drivers/iommu/iommufd/fault.c | 142 +++++++++++++++--------- drivers/iommu/iommufd/hw_pagetable.c | 6 +- 3 files changed, 130 insertions(+), 70 deletions(-)
diff --git a/drivers/iommu/iommufd/iommufd_private.h b/drivers/iommu/iommufd/iommufd_private.h index 8b378705ee71..dfbc5cfbd164 100644 --- a/drivers/iommu/iommufd/iommufd_private.h +++ b/drivers/iommu/iommufd/iommufd_private.h @@ -18,6 +18,7 @@ struct iommu_domain; struct iommu_group; struct iommu_option; struct iommufd_device; +struct iommufd_eventq;
struct iommufd_ctx { struct file *file; @@ -433,32 +434,35 @@ void iopt_remove_access(struct io_pagetable *iopt, u32 iopt_access_list_id); void iommufd_access_destroy_object(struct iommufd_object *obj);
-/* - * An iommufd_fault object represents an interface to deliver I/O page faults - * to the user space. These objects are created/destroyed by the user space and - * associated with hardware page table objects during page-table allocation. - */ -struct iommufd_fault { +struct iommufd_eventq_ops { + ssize_t (*read)(struct iommufd_eventq *eventq, char __user *buf, + size_t count, loff_t *ppos); /* Mandatory op */ + ssize_t (*write)(struct iommufd_eventq *eventq, const char __user *buf, + size_t count, loff_t *ppos); /* Optional op */ +}; + +struct iommufd_eventq { struct iommufd_object obj; struct iommufd_ctx *ictx; struct file *filep;
- /* The lists of outstanding faults protected by below mutex. */ + const struct iommufd_eventq_ops *ops; + + /* The lists of outstanding events protected by below mutex. */ struct mutex mutex; struct list_head deliver; - struct xarray response;
struct wait_queue_head wait_queue; };
-static inline int iommufd_fault_notify(struct iommufd_fault *fault, - struct list_head *new_fault) +static inline int iommufd_eventq_notify(struct iommufd_eventq *eventq, + struct list_head *new_event) { - mutex_lock(&fault->mutex); - list_add_tail(new_fault, &fault->deliver); - mutex_unlock(&fault->mutex); + mutex_lock(&eventq->mutex); + list_add_tail(new_event, &eventq->deliver); + mutex_unlock(&eventq->mutex);
- wake_up_interruptible(&fault->wait_queue); + wake_up_interruptible(&eventq->wait_queue); return 0; }
@@ -470,12 +474,28 @@ struct iommufd_attach_handle { /* Convert an iommu attach handle to iommufd handle. */ #define to_iommufd_handle(hdl) container_of(hdl, struct iommufd_attach_handle, handle)
+/* + * An iommufd_fault object represents an interface to deliver I/O page faults + * to the user space. These objects are created/destroyed by the user space and + * associated with hardware page table objects during page-table allocation. + */ +struct iommufd_fault { + struct iommufd_eventq common; + struct xarray response; +}; + +static inline struct iommufd_fault * +eventq_to_fault(struct iommufd_eventq *eventq) +{ + return container_of(eventq, struct iommufd_fault, common); +} + static inline struct iommufd_fault * iommufd_get_fault(struct iommufd_ucmd *ucmd, u32 id) { return container_of(iommufd_get_object(ucmd->ictx, id, IOMMUFD_OBJ_FAULT), - struct iommufd_fault, obj); + struct iommufd_fault, common.obj); }
int iommufd_fault_alloc(struct iommufd_ucmd *ucmd); @@ -486,7 +506,7 @@ static inline int iommufd_fault_iopf_handler(struct iopf_group *group) struct iommufd_hw_pagetable *hwpt = group->attach_handle->domain->fault_data;
- return iommufd_fault_notify(hwpt->fault, &group->node); + return iommufd_eventq_notify(&hwpt->fault->common, &group->node); }
int iommufd_fault_domain_attach_dev(struct iommufd_hw_pagetable *hwpt, diff --git a/drivers/iommu/iommufd/fault.c b/drivers/iommu/iommufd/fault.c index d188994e4e84..e386b6c3e6ab 100644 --- a/drivers/iommu/iommufd/fault.c +++ b/drivers/iommu/iommufd/fault.c @@ -17,6 +17,8 @@ #include "../iommu-priv.h" #include "iommufd_private.h"
+/* IOMMUFD_OBJ_FAULT Functions */ + static int iommufd_fault_iopf_enable(struct iommufd_device *idev) { struct device *dev = idev->dev; @@ -108,8 +110,8 @@ static void iommufd_auto_response_faults(struct iommufd_hw_pagetable *hwpt, if (!fault) return;
- mutex_lock(&fault->mutex); - list_for_each_entry_safe(group, next, &fault->deliver, node) { + mutex_lock(&fault->common.mutex); + list_for_each_entry_safe(group, next, &fault->common.deliver, node) { if (group->attach_handle != &handle->handle) continue; list_del(&group->node); @@ -124,7 +126,7 @@ static void iommufd_auto_response_faults(struct iommufd_hw_pagetable *hwpt, iopf_group_response(group, IOMMU_PAGE_RESP_INVALID); iopf_free_group(group); } - mutex_unlock(&fault->mutex); + mutex_unlock(&fault->common.mutex); }
static struct iommufd_attach_handle * @@ -211,7 +213,8 @@ int iommufd_fault_domain_replace_dev(struct iommufd_device *idev,
void iommufd_fault_destroy(struct iommufd_object *obj) { - struct iommufd_fault *fault = container_of(obj, struct iommufd_fault, obj); + struct iommufd_eventq *eventq = + container_of(obj, struct iommufd_eventq, obj); struct iopf_group *group, *next;
/* @@ -220,11 +223,13 @@ void iommufd_fault_destroy(struct iommufd_object *obj) * accessing this pointer. Therefore, acquiring the mutex here * is unnecessary. */ - list_for_each_entry_safe(group, next, &fault->deliver, node) { + list_for_each_entry_safe(group, next, &eventq->deliver, node) { list_del(&group->node); iopf_group_response(group, IOMMU_PAGE_RESP_INVALID); iopf_free_group(group); } + xa_destroy(&eventq_to_fault(eventq)->response); + mutex_destroy(&eventq->mutex); }
static void iommufd_compose_fault_message(struct iommu_fault *fault, @@ -242,11 +247,12 @@ static void iommufd_compose_fault_message(struct iommu_fault *fault, hwpt_fault->cookie = cookie; }
-static ssize_t iommufd_fault_fops_read(struct file *filep, char __user *buf, - size_t count, loff_t *ppos) +static ssize_t iommufd_fault_fops_read(struct iommufd_eventq *eventq, + char __user *buf, size_t count, + loff_t *ppos) { size_t fault_size = sizeof(struct iommu_hwpt_pgfault); - struct iommufd_fault *fault = filep->private_data; + struct iommufd_fault *fault = eventq_to_fault(eventq); struct iommu_hwpt_pgfault data; struct iommufd_device *idev; struct iopf_group *group; @@ -257,10 +263,10 @@ static ssize_t iommufd_fault_fops_read(struct file *filep, char __user *buf, if (*ppos || count % fault_size) return -ESPIPE;
- mutex_lock(&fault->mutex); - while (!list_empty(&fault->deliver) && count > done) { - group = list_first_entry(&fault->deliver, - struct iopf_group, node); + mutex_lock(&eventq->mutex); + while (!list_empty(&eventq->deliver) && count > done) { + group = list_first_entry(&eventq->deliver, struct iopf_group, + node);
if (group->fault_count * fault_size > count - done) break; @@ -285,16 +291,17 @@ static ssize_t iommufd_fault_fops_read(struct file *filep, char __user *buf,
list_del(&group->node); } - mutex_unlock(&fault->mutex); + mutex_unlock(&eventq->mutex);
return done == 0 ? rc : done; }
-static ssize_t iommufd_fault_fops_write(struct file *filep, const char __user *buf, - size_t count, loff_t *ppos) +static ssize_t iommufd_fault_fops_write(struct iommufd_eventq *eventq, + const char __user *buf, size_t count, + loff_t *ppos) { size_t response_size = sizeof(struct iommu_hwpt_page_response); - struct iommufd_fault *fault = filep->private_data; + struct iommufd_fault *fault = eventq_to_fault(eventq); struct iommu_hwpt_page_response response; struct iopf_group *group; size_t done = 0; @@ -303,7 +310,7 @@ static ssize_t iommufd_fault_fops_write(struct file *filep, const char __user *b if (*ppos || count % response_size) return -ESPIPE;
- mutex_lock(&fault->mutex); + mutex_lock(&eventq->mutex); while (count > done) { rc = copy_from_user(&response, buf + done, response_size); if (rc) @@ -329,62 +336,93 @@ static ssize_t iommufd_fault_fops_write(struct file *filep, const char __user *b iopf_free_group(group); done += response_size; } - mutex_unlock(&fault->mutex); + mutex_unlock(&eventq->mutex);
return done == 0 ? rc : done; }
-static __poll_t iommufd_fault_fops_poll(struct file *filep, - struct poll_table_struct *wait) +static const struct iommufd_eventq_ops iommufd_fault_ops = { + .read = &iommufd_fault_fops_read, + .write = &iommufd_fault_fops_write, +}; + +/* Common Event Queue Functions */ + +static ssize_t iommufd_eventq_fops_read(struct file *filep, char __user *buf, + size_t count, loff_t *ppos) { - struct iommufd_fault *fault = filep->private_data; + struct iommufd_eventq *eventq = filep->private_data; + + return eventq->ops->read(eventq, buf, count, ppos); +} + +static ssize_t iommufd_eventq_fops_write(struct file *filep, + const char __user *buf, size_t count, + loff_t *ppos) +{ + struct iommufd_eventq *eventq = filep->private_data; + + if (!eventq->ops->write) + return -EOPNOTSUPP; + return eventq->ops->write(eventq, buf, count, ppos); +} + +static __poll_t iommufd_eventq_fops_poll(struct file *filep, + struct poll_table_struct *wait) +{ + struct iommufd_eventq *eventq = filep->private_data; __poll_t pollflags = EPOLLOUT;
- poll_wait(filep, &fault->wait_queue, wait); - mutex_lock(&fault->mutex); - if (!list_empty(&fault->deliver)) + poll_wait(filep, &eventq->wait_queue, wait); + mutex_lock(&eventq->mutex); + if (!list_empty(&eventq->deliver)) pollflags |= EPOLLIN | EPOLLRDNORM; - mutex_unlock(&fault->mutex); + mutex_unlock(&eventq->mutex);
return pollflags; }
-static int iommufd_fault_fops_release(struct inode *inode, struct file *filep) +static int iommufd_eventq_fops_release(struct inode *inode, struct file *filep) { - struct iommufd_fault *fault = filep->private_data; + struct iommufd_eventq *eventq = filep->private_data;
- refcount_dec(&fault->obj.users); - iommufd_ctx_put(fault->ictx); + refcount_dec(&eventq->obj.users); + iommufd_ctx_put(eventq->ictx); return 0; }
-static const struct file_operations iommufd_fault_fops = { +static const struct file_operations iommufd_eventq_fops = { .owner = THIS_MODULE, .open = nonseekable_open, - .read = iommufd_fault_fops_read, - .write = iommufd_fault_fops_write, - .poll = iommufd_fault_fops_poll, - .release = iommufd_fault_fops_release, + .read = iommufd_eventq_fops_read, + .write = iommufd_eventq_fops_write, + .poll = iommufd_eventq_fops_poll, + .release = iommufd_eventq_fops_release, };
-static int iommufd_fault_init(struct iommufd_fault *fault, char *name, - struct iommufd_ctx *ictx) +static int iommufd_eventq_init(struct iommufd_eventq *eventq, char *name, + struct iommufd_ctx *ictx, + const struct iommufd_eventq_ops *ops) { struct file *filep; int fdno;
- mutex_init(&fault->mutex); - INIT_LIST_HEAD(&fault->deliver); - init_waitqueue_head(&fault->wait_queue); + if (WARN_ON_ONCE(!ops || !ops->read)) + return -EINVAL; + + mutex_init(&eventq->mutex); + INIT_LIST_HEAD(&eventq->deliver); + init_waitqueue_head(&eventq->wait_queue);
- filep = anon_inode_getfile(name, &iommufd_fault_fops, fault, O_RDWR); + filep = anon_inode_getfile(name, &iommufd_eventq_fops, eventq, O_RDWR); if (IS_ERR(filep)) return PTR_ERR(filep);
- fault->ictx = ictx; - iommufd_ctx_get(fault->ictx); - fault->filep = filep; - refcount_inc(&fault->obj.users); + eventq->ops = ops; + eventq->ictx = ictx; + iommufd_ctx_get(eventq->ictx); + refcount_inc(&eventq->obj.users); + eventq->filep = filep;
fdno = get_unused_fd_flags(O_CLOEXEC); if (fdno < 0) @@ -402,34 +440,36 @@ int iommufd_fault_alloc(struct iommufd_ucmd *ucmd) if (cmd->flags) return -EOPNOTSUPP;
- fault = iommufd_object_alloc(ucmd->ictx, fault, IOMMUFD_OBJ_FAULT); + fault = __iommufd_object_alloc(ucmd->ictx, fault, IOMMUFD_OBJ_FAULT, + common.obj); if (IS_ERR(fault)) return PTR_ERR(fault);
xa_init_flags(&fault->response, XA_FLAGS_ALLOC1);
- fdno = iommufd_fault_init(fault, "[iommufd-pgfault]", ucmd->ictx); + fdno = iommufd_eventq_init(&fault->common, "[iommufd-pgfault]", + ucmd->ictx, &iommufd_fault_ops); if (fdno < 0) { rc = fdno; goto out_abort; }
- cmd->out_fault_id = fault->obj.id; + cmd->out_fault_id = fault->common.obj.id; cmd->out_fault_fd = fdno;
rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); if (rc) goto out_put_fdno; - iommufd_object_finalize(ucmd->ictx, &fault->obj); + iommufd_object_finalize(ucmd->ictx, &fault->common.obj);
- fd_install(fdno, fault->filep); + fd_install(fdno, fault->common.filep);
return 0; out_put_fdno: put_unused_fd(fdno); - fput(fault->filep); + fput(fault->common.filep); out_abort: - iommufd_object_abort_and_destroy(ucmd->ictx, &fault->obj); + iommufd_object_abort_and_destroy(ucmd->ictx, &fault->common.obj);
return rc; } diff --git a/drivers/iommu/iommufd/hw_pagetable.c b/drivers/iommu/iommufd/hw_pagetable.c index ce03c3804651..12a576f1f13d 100644 --- a/drivers/iommu/iommufd/hw_pagetable.c +++ b/drivers/iommu/iommufd/hw_pagetable.c @@ -14,7 +14,7 @@ static void __iommufd_hwpt_destroy(struct iommufd_hw_pagetable *hwpt) iommu_domain_free(hwpt->domain);
if (hwpt->fault) - refcount_dec(&hwpt->fault->obj.users); + refcount_dec(&hwpt->fault->common.obj.users); }
void iommufd_hwpt_paging_destroy(struct iommufd_object *obj) @@ -403,8 +403,8 @@ int iommufd_hwpt_alloc(struct iommufd_ucmd *ucmd) hwpt->fault = fault; hwpt->domain->iopf_handler = iommufd_fault_iopf_handler; hwpt->domain->fault_data = hwpt; - refcount_inc(&fault->obj.users); - iommufd_put_object(ucmd->ictx, &fault->obj); + refcount_inc(&fault->common.obj.users); + iommufd_put_object(ucmd->ictx, &fault->common.obj); }
cmd->out_hwpt_id = hwpt->obj.id;