There are two main objects. struct io_dmabuf_token and struct io_dmabuf_map. The token is used during initial registration and serves as an interface between the upper layer user like io_uring and to the importer subsystem / driver. io_dmabuf_map represens the actual dma map established for the target device[s] with dma_buf_map_attachment() and stored in a device specific format.
The separation into two different objects exists to support map invalidation (see dma_buf_invalidate_mappings()). A token can create multiple maps during its lifetime, but there can only be one (active) map attached to it. It's aslo possible to not have an active map. Invalidation drops the active map if present, and the next map will only be attempted to be created once there is a new request that wants to use the token.
The primary task of the io_dmabuf_map object is to count all requests currently using it, which is done with percpu refcounts. When a map is invalidated, we remove it from the token, so there can be no new requests, then it adds a fence to the dmabuf reservation object. Once all the requests complete, we signal the fence and unmap it.
[un]mapping and any work with dma addresses is delegated to the importer driver via an ops table stored in the token, see struct io_dmabuf_token_dev_ops. That's required because the generic layer doesn't have knowledge about the device it's going to be use with, and there will be more complex use cases with multiple devices.
Signed-off-by: Pavel Begunkov asml.silence@gmail.com --- include/linux/io_dmabuf_token.h | 92 +++++++++++ lib/Kconfig | 4 + lib/Makefile | 2 + lib/io_dmabuf_token.c | 272 ++++++++++++++++++++++++++++++++ 4 files changed, 370 insertions(+) create mode 100644 include/linux/io_dmabuf_token.h create mode 100644 lib/io_dmabuf_token.c
diff --git a/include/linux/io_dmabuf_token.h b/include/linux/io_dmabuf_token.h new file mode 100644 index 000000000000..b94bda684812 --- /dev/null +++ b/include/linux/io_dmabuf_token.h @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_DMA_TOKEN_H +#define _LINUX_DMA_TOKEN_H + +#include <linux/dma-buf.h> + +struct io_dmabuf_fence; +struct io_dmabuf_token; +struct io_dmabuf_map; + +struct io_dmabuf_token_dev_ops { + /* + * Create a new map for the given token. It should be initialised + * with io_dmabuf_init_map(). The callback is executed with the + * reservation lock held. + */ + struct io_dmabuf_map *(*map)(struct io_dmabuf_token *); + + /* + * Clean up device specific parts of the map. The callback is + * executed with the reservation lock held. + */ + void (*unmap)(struct io_dmabuf_token *, struct io_dmabuf_map *); + + /* + * The user tries to destroy the token. Release all device specific + * parts of the token. + */ + void (*release)(struct io_dmabuf_token *); +}; + +struct io_dmabuf_map { + /* + * Counts attached requests and other users. Device specific unmapping + * is deferred until all refs are dropped. + */ + struct percpu_ref refs; + + struct work_struct release_work; + struct io_dmabuf_fence *fence; + struct io_dmabuf_token *token; +}; + +struct io_dmabuf_token { + struct io_dmabuf_map __rcu *map; + struct dma_buf *dmabuf; + enum dma_data_direction dir; + + atomic_t fence_seq; + u64 fence_ctx; + struct work_struct release_work; + refcount_t refs; + + void *dev_priv; + const struct io_dmabuf_token_dev_ops *dev_ops; +}; + +int io_dmabuf_token_create(struct file *file, + struct io_dmabuf_token *token, + struct dma_buf *dmabuf, + enum dma_data_direction dir); +void io_dmabuf_token_release(struct io_dmabuf_token *token); + +struct io_dmabuf_map *io_dmabuf_create_map(struct io_dmabuf_token *token); + +static inline struct io_dmabuf_map *io_dmabuf_get_map(struct io_dmabuf_token *token) +{ + struct io_dmabuf_map *map; + + guard(rcu)(); + + map = rcu_dereference(token->map); + if (unlikely(!map || !percpu_ref_tryget_live_rcu(&map->refs))) + return NULL; + + return map; +} + +static inline void io_dmabuf_map_drop(struct io_dmabuf_map *map) +{ + percpu_ref_put(&map->refs); +} + +/* + * Device API + */ + +void io_dmabuf_token_invalidate_mappings(struct io_dmabuf_token *token); +int io_dmabuf_init_map(struct io_dmabuf_token *token, struct io_dmabuf_map *map); + + +#endif diff --git a/lib/Kconfig b/lib/Kconfig index 0f2fb9610647..853f10bf8e1a 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -636,3 +636,7 @@ config UNION_FIND
config MIN_HEAP bool + +config DMABUF_TOKEN + def_bool y + depends on DMA_SHARED_BUFFER diff --git a/lib/Makefile b/lib/Makefile index ea660cca04f4..4a42cfcaa80c 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -246,6 +246,8 @@ obj-$(CONFIG_IRQ_POLL) += irq_poll.o
obj-$(CONFIG_POLYNOMIAL) += polynomial.o
+obj-$(CONFIG_DMABUF_TOKEN) += io_dmabuf_token.o + # stackdepot.c should not be instrumented or call instrumented functions. # Prevent the compiler from calling builtins like memcmp() or bcmp() from this # file. diff --git a/lib/io_dmabuf_token.c b/lib/io_dmabuf_token.c new file mode 100644 index 000000000000..808b5ad33dbc --- /dev/null +++ b/lib/io_dmabuf_token.c @@ -0,0 +1,272 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Common infrastructure for supporing dma-buf in the I/O path. + * + * Copyright (C) 2026 Pavel Begunkov asml.silence@gmail.com + */ +#include <linux/io_dmabuf_token.h> +#include <linux/dma-resv.h> + +struct io_dmabuf_fence { + struct dma_fence base; + spinlock_t lock; +}; + +static const char *io_dmabuf_fence_drv_name(struct dma_fence *fence) +{ + /* default fence release kfree's the base pointer */ + BUILD_BUG_ON(offsetof(struct io_dmabuf_fence, base)); + + return "DMABUF token"; +} + +static const char *io_dmabuf_fence_timeline_name(struct dma_fence *fence) +{ + return "DMABUF token"; +} + +const struct dma_fence_ops io_dmabuf_fence_ops = { + .get_driver_name = io_dmabuf_fence_drv_name, + .get_timeline_name = io_dmabuf_fence_timeline_name, +}; + +static void io_dmabuf_token_destroy_work(struct work_struct *work) +{ + struct io_dmabuf_token *token = container_of(work, struct io_dmabuf_token, + release_work); + + if (WARN_ON_ONCE(refcount_read(&token->refs))) + return; + + token->dev_ops->release(token); + dma_buf_put(token->dmabuf); + kfree(token); +} + +static void io_dmabuf_map_release_work(struct work_struct *work) +{ + struct io_dmabuf_map *map = container_of(work, struct io_dmabuf_map, + release_work); + struct io_dmabuf_fence *fence = map->fence; + struct io_dmabuf_token *token = map->token; + struct dma_buf *dmabuf = token->dmabuf; + + /* the release path must wait for fences */ + if (WARN_ON_ONCE(refcount_read(&token->refs) == 0)) + return; + + /* Prevent from destoying the token while unmapping */ + refcount_inc(&token->refs); + + /* + * There are no more requests using the map, we can signal the fence. + * It should be done before taking the resv lock as someone could be + * waiting for the fence while holding the lock. + */ + dma_fence_signal(&fence->base); + + dma_resv_lock(dmabuf->resv, NULL); + token->dev_ops->unmap(token, map); + dma_resv_unlock(dmabuf->resv); + + dma_fence_put(&fence->base); + percpu_ref_exit(&map->refs); + kfree(map); + + if (refcount_dec_and_test(&token->refs)) { + /* + * Destruction needs to wait for I/O and dma fences. Defer it to + * simplify locking. + */ + INIT_WORK(&token->release_work, io_dmabuf_token_destroy_work); + queue_work(system_wq, &token->release_work); + } +} + +static void io_dmabuf_map_refs_release(struct percpu_ref *ref) +{ + struct io_dmabuf_map *map = container_of(ref, struct io_dmabuf_map, refs); + + /* might sleep, use a worker */ + INIT_WORK(&map->release_work, io_dmabuf_map_release_work); + queue_work(system_wq, &map->release_work); +} + +int io_dmabuf_init_map(struct io_dmabuf_token *token, struct io_dmabuf_map *map) +{ + struct io_dmabuf_fence *fence = NULL; + int ret; + + fence = kzalloc(sizeof(*fence), GFP_KERNEL); + if (!fence) + return -ENOMEM; + + ret = percpu_ref_init(&map->refs, io_dmabuf_map_refs_release, 0, GFP_KERNEL); + if (ret) { + kfree(fence); + return ret; + } + + spin_lock_init(&fence->lock); + dma_fence_init(&fence->base, &io_dmabuf_fence_ops, &fence->lock, + token->fence_ctx, atomic_inc_return(&token->fence_seq)); + map->fence = fence; + map->token = token; + return 0; +} +EXPORT_SYMBOL_NS_GPL(io_dmabuf_init_map, "DMA_BUF"); + +struct io_dmabuf_map *io_dmabuf_create_map(struct io_dmabuf_token *token) +{ + struct dma_buf *dmabuf = token->dmabuf; + struct io_dmabuf_map *map; + long ret; + +retry: + /* + * ->dmabuf_map() will be calling dma_buf_map_attachment(), for which + * we'll need to wait for fences. Do a bit nicer and try to wait + * without the resv lock first. + */ + ret = dma_resv_wait_timeout(dmabuf->resv, DMA_RESV_USAGE_KERNEL, + true, MAX_SCHEDULE_TIMEOUT); + if (!ret) + ret = -EAGAIN; + if (ret < 0) + return ERR_PTR(ret); + + dma_resv_lock(dmabuf->resv, NULL); + map = io_dmabuf_get_map(token); + if (map) { + ret = 0; + goto out; + } + + if (dma_resv_wait_timeout(dmabuf->resv, DMA_RESV_USAGE_KERNEL, + true, 0) < 0) { + dma_resv_unlock(dmabuf->resv); + goto retry; + } + + map = token->dev_ops->map(token); + if (IS_ERR(map)) { + ret = PTR_ERR(map); + goto out; + } + + percpu_ref_get(&map->refs); + rcu_assign_pointer(token->map, map); +out: + dma_resv_unlock(dmabuf->resv); + if (ret < 0) + return ERR_PTR(ret); + return map; +} + +static void io_dmabuf_drop_map(struct io_dmabuf_token *token) +{ + struct dma_buf *dmabuf = token->dmabuf; + struct io_dmabuf_map *map; + int ret; + + dma_resv_assert_held(dmabuf->resv); + + map = rcu_dereference_protected(token->map, + dma_resv_held(dmabuf->resv)); + if (!map) + return; + rcu_assign_pointer(token->map, NULL); + + ret = dma_resv_reserve_fences(dmabuf->resv, 1); + if (WARN_ON_ONCE(ret)) { + struct dma_fence *fence = &map->fence->base; + + dma_fence_get(fence); + percpu_ref_kill(&map->refs); + dma_fence_wait(fence, false); + dma_fence_put(fence); + return; + } + + dma_resv_add_fence(dmabuf->resv, &map->fence->base, + DMA_RESV_USAGE_KERNEL); + /* + * Delay destruction until all inflight requests using the map are + * gone. It'll also signal the fence then. + */ + percpu_ref_kill(&map->refs); +} + +void io_dmabuf_token_invalidate_mappings(struct io_dmabuf_token *token) +{ + io_dmabuf_drop_map(token); +} +EXPORT_SYMBOL_NS_GPL(io_dmabuf_token_invalidate_mappings, "DMA_BUF"); + +static void io_dmabuf_token_release_work(struct work_struct *work) +{ + struct io_dmabuf_token *token = container_of(work, struct io_dmabuf_token, + release_work); + struct dma_buf *dmabuf = token->dmabuf; + long ret; + + dma_resv_lock(dmabuf->resv, NULL); + /* Remove the last map, there should be no new ones going forward. */ + io_dmabuf_drop_map(token); + dma_resv_unlock(dmabuf->resv); + + /* Wait until all maps are destroyed. */ + ret = dma_resv_wait_timeout(dmabuf->resv, DMA_RESV_USAGE_KERNEL, + false, MAX_SCHEDULE_TIMEOUT); + + if (WARN_ON_ONCE(ret <= 0)) + return; + if (WARN_ON_ONCE(rcu_dereference_protected(token->map, true))) + return; + + if (refcount_dec_and_test(&token->refs)) + io_dmabuf_token_destroy_work(&token->release_work); +} + +void io_dmabuf_token_release(struct io_dmabuf_token *token) +{ + /* + * Destruction needs to wait for I/O and dma fences. Defer it to + * simplify locking. + */ + INIT_WORK(&token->release_work, io_dmabuf_token_release_work); + queue_work(system_wq, &token->release_work); +} + +int io_dmabuf_token_create(struct file *file, + struct io_dmabuf_token *token, + struct dma_buf *dmabuf, + enum dma_data_direction dir) +{ + int ret; + + if (!file->f_op->create_dmabuf_token) + return -EOPNOTSUPP; + + memset(token, 0, sizeof(*token)); + token->fence_ctx = dma_fence_context_alloc(1); + token->dir = dir; + token->dmabuf = dmabuf; + refcount_set(&token->refs, 1); + get_dma_buf(dmabuf); + + ret = file->f_op->create_dmabuf_token(file, token); + if (ret) { + memset(token, 0, sizeof(*token)); + dma_buf_put(dmabuf); + return ret; + } + + if (WARN_ON_ONCE(!token->dev_ops || + !token->dev_ops->map || + !token->dev_ops->unmap || + !token->dev_ops->release)) + return -EINVAL; + + return ret; +}