From: Bobby Eshleman bobbyeshleman@meta.com

Every devmem dmabuf binding today hands the page_pool PAGE_SIZE niovs. This caps a single RX descriptor at PAGE_SIZE, burning CPU on buffer churn for large flows.

Add a bind-time netlink attribute, NETDEV_A_DMABUF_RX_BUF_SIZE, that lets userspace request a larger niov size. The value must be a power of two >= PAGE_SIZE.

Measurements: Setup: kperf in devmem RX/TX cuda mode, 4 flows, 64 MB messages, 60s, dctcp, num-rx-queues=4, dmabuf-rx/tx-size-mb=2048, 10 runs per niov size, mlx5.

CPU Util:

niov net sirq % net idle % app sys % app idle % ----- ---------------- ---------------- ---------------- ---------------- 4K 62.38 +/- 8.27 33.40 +/- 7.51 54.15 +/- 10.23 43.67 +/- 10.53 16K 58.91 +/- 5.35 35.23 +/- 5.88 41.05 +/- 8.87 56.42 +/- 9.24 32K 64.12 +/- 0.68 31.09 +/- 1.48 44.54 +/- 3.51 52.63 +/- 3.65 64K 54.69 +/- 5.54 39.67 +/- 5.81 35.47 +/- 3.11 61.97 +/- 3.27

RX app sys % drops ~19% from 4K to 64K.

Throughput:

niov RX dev Gbps RX flow avg Gbps ----- ---------------- ----------------- 4K 300.63 +/- 53.21 75.16 +/- 13.30 16K 321.35 +/- 28.20 80.34 +/- 7.05 32K 347.63 +/- 2.20 86.91 +/- 0.55 64K 332.11 +/- 14.26 83.03 +/- 3.56

Throughput seems to increase, but the stdev is pretty wide so could just be noise.

kperf support (not yet merged): https://github.com/facebookexperimental/kperf/commit/8837577f920876bce6986ec...

Signed-off-by: Bobby Eshleman bobbyeshleman@meta.com Acked-by: Stanislav Fomichev sdf@fomichev.me Reviewed-by: Mina Almasry almasrymina@google.com --- Documentation/netlink/specs/netdev.yaml | 8 ++++++ include/uapi/linux/netdev.h | 1 + net/core/devmem.c | 51 +++++++++++++++++++-------------- net/core/devmem.h | 13 ++++++--- net/core/netdev-genl-gen.c | 5 ++-- net/core/netdev-genl.c | 19 ++++++++++-- tools/include/uapi/linux/netdev.h | 1 + 7 files changed, 69 insertions(+), 29 deletions(-)

diff --git a/Documentation/netlink/specs/netdev.yaml b/Documentation/netlink/specs/netdev.yaml index 5f143da7458c..70b902008bd3 100644 --- a/Documentation/netlink/specs/netdev.yaml +++ b/Documentation/netlink/specs/netdev.yaml @@ -598,6 +598,13 @@ attribute-sets: type: u32 checks: min: 1 + - + name: rx-buf-size + doc: | + Size in bytes of each RX buffer the NIC writes into from the bound + dmabuf. Must be a power of two and >= PAGE_SIZE; defaults to + PAGE_SIZE. + type: u32

operations: list: @@ -812,6 +819,7 @@ operations: - ifindex - fd - queues + - rx-buf-size reply: attributes: - id diff --git a/include/uapi/linux/netdev.h b/include/uapi/linux/netdev.h index 2f3ab75e8cc0..85e1d20c6268 100644 --- a/include/uapi/linux/netdev.h +++ b/include/uapi/linux/netdev.h @@ -219,6 +219,7 @@ enum { NETDEV_A_DMABUF_QUEUES, NETDEV_A_DMABUF_FD, NETDEV_A_DMABUF_ID, + NETDEV_A_DMABUF_RX_BUF_SIZE,

__NETDEV_A_DMABUF_MAX, NETDEV_A_DMABUF_MAX = (__NETDEV_A_DMABUF_MAX - 1) diff --git a/net/core/devmem.c b/net/core/devmem.c index 957d6b96216b..3ce3cc14bec0 100644 --- a/net/core/devmem.c +++ b/net/core/devmem.c @@ -46,7 +46,7 @@ static dma_addr_t net_devmem_get_dma_addr(const struct net_iov *niov)

owner = net_devmem_iov_to_chunk_owner(niov); return owner->base_dma_addr + - ((dma_addr_t)net_iov_idx(niov) << PAGE_SHIFT); + ((dma_addr_t)net_iov_idx(niov) << owner->binding->niov_shift); }

static void net_devmem_dmabuf_binding_release(struct percpu_ref *ref) @@ -93,13 +93,14 @@ net_devmem_alloc_dmabuf(struct net_devmem_dmabuf_binding *binding) ssize_t offset; ssize_t index;

- dma_addr = gen_pool_alloc_owner(binding->chunk_pool, PAGE_SIZE, + dma_addr = gen_pool_alloc_owner(binding->chunk_pool, + 1UL << binding->niov_shift, (void **)&owner); if (!dma_addr) return NULL;

offset = dma_addr - owner->base_dma_addr; - index = offset / PAGE_SIZE; + index = offset >> binding->niov_shift; niov = &owner->area.niovs[index];

niov->desc.pp_magic = 0; @@ -113,12 +114,13 @@ void net_devmem_free_dmabuf(struct net_iov *niov) { struct net_devmem_dmabuf_binding *binding = net_devmem_iov_binding(niov); unsigned long dma_addr = net_devmem_get_dma_addr(niov); + size_t niov_size = 1UL << binding->niov_shift;

if (WARN_ON(!gen_pool_has_addr(binding->chunk_pool, dma_addr, - PAGE_SIZE))) + niov_size))) return;

- gen_pool_free(binding->chunk_pool, dma_addr, PAGE_SIZE); + gen_pool_free(binding->chunk_pool, dma_addr, niov_size); }

void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding) @@ -163,6 +165,9 @@ int net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx, u32 xa_idx; int err;

+ if (binding->niov_shift != PAGE_SHIFT) + mp_params.rx_page_size = 1U << binding->niov_shift; + err = netif_mp_open_rxq(dev, rxq_idx, &mp_params, extack); if (err) return err; @@ -184,14 +189,16 @@ struct net_devmem_dmabuf_binding * net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, struct device *dma_dev, enum dma_data_direction direction, - unsigned int dmabuf_fd, struct netdev_nl_sock *priv, + unsigned int dmabuf_fd, unsigned int niov_shift, + struct netdev_nl_sock *priv, struct netlink_ext_ack *extack) { struct net_devmem_dmabuf_binding *binding; + size_t niov_size = 1UL << niov_shift; static u32 id_alloc_next; + unsigned int sg_idx, i; struct scatterlist *sg; struct dma_buf *dmabuf; - unsigned int sg_idx, i; unsigned long virtual; int err;

@@ -213,6 +220,7 @@ net_devmem_bind_dmabuf(struct net_device *dev, void *vdev,

binding->dev = dev; binding->vdev = vdev; + binding->niov_shift = niov_shift; xa_init_flags(&binding->bound_rxqs, XA_FLAGS_ALLOC);

err = percpu_ref_init(&binding->ref, @@ -248,18 +256,14 @@ net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, goto err_unmap; } binding->tx_vec = kvmalloc_objs(struct net_iov *, - dmabuf->size / PAGE_SIZE); + dmabuf->size >> niov_shift); if (!binding->tx_vec) { err = -ENOMEM; goto err_unmap; } }

- /* For simplicity we expect to make PAGE_SIZE allocations, but the - * binding can be much more flexible than that. We may be able to - * allocate MTU sized chunks here. Leave that for future work... - */ - binding->chunk_pool = gen_pool_create(PAGE_SHIFT, + binding->chunk_pool = gen_pool_create(niov_shift, dev_to_node(&dev->dev)); if (!binding->chunk_pool) { err = -ENOMEM; @@ -273,9 +277,12 @@ net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, size_t len = sg_dma_len(sg); struct net_iov *niov;

- if (!IS_ALIGNED(len, PAGE_SIZE)) { + if (!IS_ALIGNED(dma_addr, niov_size) || + !IS_ALIGNED(len, niov_size)) { err = -EINVAL; - NL_SET_ERR_MSG(extack, "dma-buf SG length must be PAGE_SIZE aligned"); + NL_SET_ERR_MSG_FMT(extack, + "dmabuf sg entry (addr=%pad, len=%zu) not aligned to niov size %zu", + &dma_addr, len, niov_size); goto err_free_chunks; }

@@ -288,7 +295,7 @@ net_devmem_bind_dmabuf(struct net_device *dev, void *vdev,

owner->area.base_virtual = virtual; owner->base_dma_addr = dma_addr; - owner->area.num_niovs = len / PAGE_SIZE; + owner->area.num_niovs = len >> niov_shift; owner->binding = binding;

err = gen_pool_add_owner(binding->chunk_pool, dma_addr, @@ -313,7 +320,7 @@ net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, page_pool_set_dma_addr_netmem(net_iov_to_netmem(niov), net_devmem_get_dma_addr(niov)); if (direction == DMA_TO_DEVICE) - binding->tx_vec[owner->area.base_virtual / PAGE_SIZE + i] = niov; + binding->tx_vec[(owner->area.base_virtual >> niov_shift) + i] = niov; }

virtual += len; @@ -430,13 +437,15 @@ struct net_iov * net_devmem_get_niov_at(struct net_devmem_dmabuf_binding *binding, size_t virt_addr, size_t *off, size_t *size) { + size_t niov_size = 1UL << binding->niov_shift; + if (virt_addr >= binding->dmabuf->size) return NULL;

- *off = virt_addr % PAGE_SIZE; - *size = PAGE_SIZE - *off; + *off = virt_addr & (niov_size - 1); + *size = niov_size - *off;

- return binding->tx_vec[virt_addr / PAGE_SIZE]; + return binding->tx_vec[virt_addr >> binding->niov_shift]; }

/*** "Dmabuf devmem memory provider" ***/ @@ -454,7 +463,7 @@ int mp_dmabuf_devmem_init(struct page_pool *pool) pool->dma_sync = false; pool->dma_sync_for_cpu = false;

- if (pool->p.order != 0) + if (pool->p.order != binding->niov_shift - PAGE_SHIFT) return -E2BIG;

net_devmem_dmabuf_binding_get(binding); diff --git a/net/core/devmem.h b/net/core/devmem.h index 3852a56036cb..4a293a7d1149 100644 --- a/net/core/devmem.h +++ b/net/core/devmem.h @@ -71,6 +71,8 @@ struct net_devmem_dmabuf_binding { */ struct net_iov **tx_vec;

+ unsigned int niov_shift; + struct work_struct unbind_w; };

@@ -93,7 +95,8 @@ struct net_devmem_dmabuf_binding * net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, struct device *dma_dev, enum dma_data_direction direction, - unsigned int dmabuf_fd, struct netdev_nl_sock *priv, + unsigned int dmabuf_fd, unsigned int niov_shift, + struct netdev_nl_sock *priv, struct netlink_ext_ack *extack); struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id); void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding); @@ -122,10 +125,11 @@ static inline u32 net_devmem_iov_binding_id(const struct net_iov *niov)

static inline unsigned long net_iov_virtual_addr(const struct net_iov *niov) { - struct net_iov_area *owner = net_iov_owner(niov); + struct dmabuf_genpool_chunk_owner *co = + net_devmem_iov_to_chunk_owner(niov);

- return owner->base_virtual + - ((unsigned long)net_iov_idx(niov) << PAGE_SHIFT); + return net_iov_owner(niov)->base_virtual + + ((unsigned long)net_iov_idx(niov) << co->binding->niov_shift); }

static inline bool @@ -175,6 +179,7 @@ net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, struct device *dma_dev, enum dma_data_direction direction, unsigned int dmabuf_fd, + unsigned int niov_shift, struct netdev_nl_sock *priv, struct netlink_ext_ack *extack) { diff --git a/net/core/netdev-genl-gen.c b/net/core/netdev-genl-gen.c index d18c89b5a6c7..447ed06d8c74 100644 --- a/net/core/netdev-genl-gen.c +++ b/net/core/netdev-genl-gen.c @@ -106,10 +106,11 @@ static const struct nla_policy netdev_qstats_get_nl_policy[NETDEV_A_QSTATS_SCOPE };

/* NETDEV_CMD_BIND_RX - do */ -static const struct nla_policy netdev_bind_rx_nl_policy[NETDEV_A_DMABUF_FD + 1] = { +static const struct nla_policy netdev_bind_rx_nl_policy[NETDEV_A_DMABUF_RX_BUF_SIZE + 1] = { [NETDEV_A_DMABUF_IFINDEX] = NLA_POLICY_MIN(NLA_U32, 1), [NETDEV_A_DMABUF_FD] = { .type = NLA_U32, }, [NETDEV_A_DMABUF_QUEUES] = NLA_POLICY_NESTED(netdev_queue_id_nl_policy), + [NETDEV_A_DMABUF_RX_BUF_SIZE] = { .type = NLA_U32, }, };

/* NETDEV_CMD_NAPI_SET - do */ @@ -219,7 +220,7 @@ static const struct genl_split_ops netdev_nl_ops[] = { .cmd = NETDEV_CMD_BIND_RX, .doit = netdev_nl_bind_rx_doit, .policy = netdev_bind_rx_nl_policy, - .maxattr = NETDEV_A_DMABUF_FD, + .maxattr = NETDEV_A_DMABUF_RX_BUF_SIZE, .flags = GENL_UNS_ADMIN_PERM | GENL_CMD_CAP_DO, }, { diff --git a/net/core/netdev-genl.c b/net/core/netdev-genl.c index c15d8d4ca1f8..82089dac000f 100644 --- a/net/core/netdev-genl.c +++ b/net/core/netdev-genl.c @@ -1013,6 +1013,7 @@ netdev_nl_get_dma_dev(struct net_device *netdev, unsigned long *rxq_bitmap, int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info) { struct net_devmem_dmabuf_binding *binding; + unsigned int niov_shift = PAGE_SHIFT; u32 ifindex, dmabuf_fd, rxq_idx; struct netdev_nl_sock *priv; struct net_device *netdev; @@ -1030,6 +1031,19 @@ int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info) ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]); dmabuf_fd = nla_get_u32(info->attrs[NETDEV_A_DMABUF_FD]);

+ if (info->attrs[NETDEV_A_DMABUF_RX_BUF_SIZE]) { + u32 rx_buf_size = nla_get_u32(info->attrs[NETDEV_A_DMABUF_RX_BUF_SIZE]); + + if (!rx_buf_size || !is_power_of_2(rx_buf_size) || + rx_buf_size < PAGE_SIZE) { + NL_SET_ERR_MSG_FMT(info->extack, + "rx_buf_size %u must be a power of 2 >= page size (%lu)", + rx_buf_size, PAGE_SIZE); + return -EINVAL; + } + niov_shift = ilog2(rx_buf_size); + } + priv = genl_sk_priv_get(&netdev_nl_family, NETLINK_CB(skb).sk); if (IS_ERR(priv)) return PTR_ERR(priv); @@ -1080,7 +1094,8 @@ int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info) }

binding = net_devmem_bind_dmabuf(netdev, NULL, dma_dev, DMA_FROM_DEVICE, - dmabuf_fd, priv, info->extack); + dmabuf_fd, niov_shift, priv, + info->extack); if (IS_ERR(binding)) { err = PTR_ERR(binding); goto err_rxq_bitmap; @@ -1221,7 +1236,7 @@ int netdev_nl_bind_tx_doit(struct sk_buff *skb, struct genl_info *info) binding = net_devmem_bind_dmabuf(bind_dev, bind_dev != netdev ? netdev : NULL, dma_dev, DMA_TO_DEVICE, dmabuf_fd, - priv, info->extack); + PAGE_SHIFT, priv, info->extack); if (IS_ERR(binding)) { err = PTR_ERR(binding); goto err_unlock_bind_dev; diff --git a/tools/include/uapi/linux/netdev.h b/tools/include/uapi/linux/netdev.h index 2f3ab75e8cc0..85e1d20c6268 100644 --- a/tools/include/uapi/linux/netdev.h +++ b/tools/include/uapi/linux/netdev.h @@ -219,6 +219,7 @@ enum { NETDEV_A_DMABUF_QUEUES, NETDEV_A_DMABUF_FD, NETDEV_A_DMABUF_ID, + NETDEV_A_DMABUF_RX_BUF_SIZE,

__NETDEV_A_DMABUF_MAX, NETDEV_A_DMABUF_MAX = (__NETDEV_A_DMABUF_MAX - 1)

From: Bobby Eshleman bobbyeshleman@meta.com

Add -b <bytes> to request a non-default niov size via NETDEV_A_DMABUF_RX_BUF_SIZE. When the value exceeds PAGE_SIZE, udmabuf_alloc() switches to an MFD_HUGETLB-backed memfd so each 2 MB hugepage produces one naturally-aligned sg entry.

Signed-off-by: Bobby Eshleman bobbyeshleman@meta.com Acked-by: Stanislav Fomichev sdf@fomichev.me --- tools/testing/selftests/drivers/net/hw/ncdevmem.c | 36 +++++++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-)

diff --git a/tools/testing/selftests/drivers/net/hw/ncdevmem.c b/tools/testing/selftests/drivers/net/hw/ncdevmem.c index d96e8a3b5a65..a16e55af51ee 100644 --- a/tools/testing/selftests/drivers/net/hw/ncdevmem.c +++ b/tools/testing/selftests/drivers/net/hw/ncdevmem.c @@ -40,6 +40,7 @@

#include <linux/uio.h> #include <stdarg.h> +#include <stdint.h> #include <stdio.h> #include <stdlib.h> #include <unistd.h> @@ -61,6 +62,7 @@ #include <sys/time.h>

#include <linux/memfd.h> +#include <sys/param.h> #include <linux/dma-buf.h> #include <linux/errqueue.h> #include <linux/udmabuf.h> @@ -79,6 +81,7 @@ #define PAGE_SHIFT 12 #define TEST_PREFIX "ncdevmem" #define NUM_PAGES 16000 +#define MB(x) ((x) << 20)

#ifndef MSG_SOCK_DEVMEM #define MSG_SOCK_DEVMEM 0x2000000 @@ -100,6 +103,7 @@ static unsigned int dmabuf_id; static uint32_t tx_dmabuf_id; static int waittime_ms = 500; static bool fail_on_linear; +static uint32_t rx_buf_size;

/* System state loaded by current_config_load() */ #define MAX_FLOWS 8 @@ -142,6 +146,7 @@ static struct memory_buffer *udmabuf_alloc(size_t size) { struct udmabuf_create create; struct memory_buffer *ctx; + unsigned int memfd_flags; int ret;

ctx = malloc(sizeof(*ctx)); @@ -156,9 +161,14 @@ static struct memory_buffer *udmabuf_alloc(size_t size) goto err_free_ctx; }

- ctx->memfd = memfd_create("udmabuf-test", MFD_ALLOW_SEALING); + memfd_flags = MFD_ALLOW_SEALING; + if (rx_buf_size > getpagesize()) + memfd_flags |= MFD_HUGETLB | MFD_HUGE_2MB; + + ctx->memfd = memfd_create("udmabuf-test", memfd_flags); if (ctx->memfd < 0) { - pr_err("[skip,no-memfd]"); + pr_err("[skip,no-memfd%s]", + (memfd_flags & MFD_HUGETLB) ? " (need hugepages)" : ""); goto err_close_dev; }

@@ -168,6 +178,11 @@ static struct memory_buffer *udmabuf_alloc(size_t size) goto err_close_memfd; }

+ if (memfd_flags & MFD_HUGETLB) { + size = roundup(size, MB(2)); + ctx->size = size; + } + ret = ftruncate(ctx->memfd, size); if (ret == -1) { pr_err("[FAIL,memfd-truncate]"); @@ -699,6 +714,8 @@ static int bind_rx_queue(unsigned int ifindex, unsigned int dmabuf_fd, netdev_bind_rx_req_set_ifindex(req, ifindex); netdev_bind_rx_req_set_fd(req, dmabuf_fd); __netdev_bind_rx_req_set_queues(req, queues, n_queue_index); + if (rx_buf_size) + netdev_bind_rx_req_set_rx_buf_size(req, rx_buf_size);

rsp = netdev_bind_rx(*ys, req); if (!rsp) { @@ -1411,7 +1428,7 @@ int main(int argc, char *argv[]) int is_server = 0, opt; int ret, err = 1;

- while ((opt = getopt(argc, argv, "Lls:c:p:v:q:t:f:z:n")) != -1) { + while ((opt = getopt(argc, argv, "Lls:c:p:v:q:t:f:z:nb:")) != -1) { switch (opt) { case 'L': fail_on_linear = true; @@ -1446,6 +1463,19 @@ int main(int argc, char *argv[]) case 'n': skip_config = 1; break; + case 'b': { + unsigned long val; + + errno = 0; + val = strtoul(optarg, NULL, 0); + if ((val == ULONG_MAX && errno == ERANGE) || + val > UINT32_MAX) { + pr_err("invalid rx_buf_size: %s", optarg); + return 1; + } + rx_buf_size = val; + break; + } case '?': fprintf(stderr, "unknown option: %c\n", optopt); break;