IB/core: Introduce FRMR pools (ce5df0b8) · Commits · git / linux-net

drivers/infiniband/core/Makefile

+1 −1

Original line number	Diff line number	Diff line
		@@ -12,7 +12,7 @@ ib_core-y := packer.o ud_header.o verbs.o cq.o rw.o sysfs.o \
		roce_gid_mgmt.o mr_pool.o addr.o sa_query.o \
		multicast.o mad.o smi.o agent.o mad_rmpp.o \
		nldev.o restrack.o counters.o ib_core_uverbs.o \
		trace.o lag.o iter.o
		trace.o lag.o iter.o frmr_pools.o

		ib_core-$(CONFIG_SECURITY_INFINIBAND) += security.o
		ib_core-$(CONFIG_CGROUP_RDMA) += cgroup.o

drivers/infiniband/core/frmr_pools.c

0 → 100644

+319 −0

Original line number	Diff line number	Diff line
		// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
		/*
		* Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
		*/

		#include <linux/slab.h>
		#include <linux/rbtree.h>
		#include <linux/sort.h>
		#include <linux/spinlock.h>
		#include <rdma/ib_verbs.h>

		#include "frmr_pools.h"

		static int push_handle_to_queue_locked(struct frmr_queue *queue, u32 handle)
		{
		u32 tmp = queue->ci % NUM_HANDLES_PER_PAGE;
		struct frmr_handles_page *page;

		if (queue->ci >= queue->num_pages * NUM_HANDLES_PER_PAGE) {
		page = kzalloc_obj(*page, GFP_ATOMIC);
		if (!page)
		return -ENOMEM;
		queue->num_pages++;
		list_add_tail(&page->list, &queue->pages_list);
		} else {
		page = list_last_entry(&queue->pages_list,
		struct frmr_handles_page, list);
		}

		page->handles[tmp] = handle;
		queue->ci++;
		return 0;
		}

		static u32 pop_handle_from_queue_locked(struct frmr_queue *queue)
		{
		u32 tmp = (queue->ci - 1) % NUM_HANDLES_PER_PAGE;
		struct frmr_handles_page *page;
		u32 handle;

		page = list_last_entry(&queue->pages_list, struct frmr_handles_page,
		list);
		handle = page->handles[tmp];
		queue->ci--;

		if (!tmp) {
		list_del(&page->list);
		queue->num_pages--;
		kfree(page);
		}

		return handle;
		}

		static bool pop_frmr_handles_page(struct ib_frmr_pool *pool,
		struct frmr_queue *queue,
		struct frmr_handles_page *page, u32 count)
		{
		spin_lock(&pool->lock);
		if (list_empty(&queue->pages_list)) {
		spin_unlock(&pool->lock);
		return false;
		}

		*page = list_first_entry(&queue->pages_list, struct frmr_handles_page,
		list);
		list_del(&(*page)->list);
		queue->num_pages--;

		/* If this is the last page, count may be less than
		* NUM_HANDLES_PER_PAGE.
		*/
		if (queue->ci >= NUM_HANDLES_PER_PAGE)
		*count = NUM_HANDLES_PER_PAGE;
		else
		*count = queue->ci;

		queue->ci -= *count;
		spin_unlock(&pool->lock);
		return true;
		}

		static void destroy_frmr_pool(struct ib_device *device,
		struct ib_frmr_pool *pool)
		{
		struct ib_frmr_pools *pools = device->frmr_pools;
		struct frmr_handles_page *page;
		u32 count;

		while (pop_frmr_handles_page(pool, &pool->queue, &page, &count)) {
		pools->pool_ops->destroy_frmrs(device, page->handles, count);
		kfree(page);
		}

		kfree(pool);
		}

		/*
		* Initialize the FRMR pools for a device.
		*
		* @device: The device to initialize the FRMR pools for.
		* @pool_ops: The pool operations to use.
		*
		* Returns 0 on success, negative error code on failure.
		*/
		int ib_frmr_pools_init(struct ib_device *device,
		const struct ib_frmr_pool_ops *pool_ops)
		{
		struct ib_frmr_pools *pools;

		pools = kzalloc_obj(*pools);
		if (!pools)
		return -ENOMEM;

		pools->rb_root = RB_ROOT;
		rwlock_init(&pools->rb_lock);
		pools->pool_ops = pool_ops;

		device->frmr_pools = pools;
		return 0;
		}
		EXPORT_SYMBOL(ib_frmr_pools_init);

		/*
		* Clean up the FRMR pools for a device.
		*
		* @device: The device to clean up the FRMR pools for.
		*
		* Call cleanup only after all FRMR handles have been pushed back to the pool
		* and no other FRMR operations are allowed to run in parallel.
		* Ensuring this allows us to save synchronization overhead in pop and push
		* operations.
		*/
		void ib_frmr_pools_cleanup(struct ib_device *device)
		{
		struct ib_frmr_pools *pools = device->frmr_pools;
		struct ib_frmr_pool pool, next;

		if (!pools)
		return;

		rbtree_postorder_for_each_entry_safe(pool, next, &pools->rb_root, node)
		destroy_frmr_pool(device, pool);

		kfree(pools);
		device->frmr_pools = NULL;
		}
		EXPORT_SYMBOL(ib_frmr_pools_cleanup);

		static inline int compare_keys(struct ib_frmr_key *key1,
		struct ib_frmr_key *key2)
		{
		int res;

		res = cmp_int(key1->ats, key2->ats);
		if (res)
		return res;

		res = cmp_int(key1->access_flags, key2->access_flags);
		if (res)
		return res;

		res = cmp_int(key1->vendor_key, key2->vendor_key);
		if (res)
		return res;

		res = cmp_int(key1->kernel_vendor_key, key2->kernel_vendor_key);
		if (res)
		return res;

		/*
		* allow using handles that support more DMA blocks, up to twice the
		* requested number
		*/
		res = cmp_int(key1->num_dma_blocks, key2->num_dma_blocks);
		if (res > 0) {
		if (key1->num_dma_blocks - key2->num_dma_blocks <
		key2->num_dma_blocks)
		return 0;
		}

		return res;
		}

		static int frmr_pool_cmp_find(const void key, const struct rb_node node)
		{
		struct ib_frmr_pool *pool = rb_entry(node, struct ib_frmr_pool, node);

		return compare_keys(&pool->key, (struct ib_frmr_key *)key);
		}

		static int frmr_pool_cmp_add(struct rb_node new, const struct rb_node node)
		{
		struct ib_frmr_pool *new_pool =
		rb_entry(new, struct ib_frmr_pool, node);
		struct ib_frmr_pool *pool = rb_entry(node, struct ib_frmr_pool, node);

		return compare_keys(&pool->key, &new_pool->key);
		}

		static struct ib_frmr_pool ib_frmr_pool_find(struct ib_frmr_pools pools,
		struct ib_frmr_key *key)
		{
		struct ib_frmr_pool *pool;
		struct rb_node *node;

		/* find operation is done under read lock for performance reasons.
		* The case of threads failing to find the same pool and creating it
		* is handled by the create_frmr_pool function.
		*/
		read_lock(&pools->rb_lock);
		node = rb_find(key, &pools->rb_root, frmr_pool_cmp_find);
		pool = rb_entry_safe(node, struct ib_frmr_pool, node);
		read_unlock(&pools->rb_lock);

		return pool;
		}

		static struct ib_frmr_pool create_frmr_pool(struct ib_device device,
		struct ib_frmr_key *key)
		{
		struct ib_frmr_pools *pools = device->frmr_pools;
		struct ib_frmr_pool *pool;
		struct rb_node *existing;

		pool = kzalloc_obj(*pool);
		if (!pool)
		return ERR_PTR(-ENOMEM);

		memcpy(&pool->key, key, sizeof(*key));
		INIT_LIST_HEAD(&pool->queue.pages_list);
		spin_lock_init(&pool->lock);

		write_lock(&pools->rb_lock);
		existing = rb_find_add(&pool->node, &pools->rb_root, frmr_pool_cmp_add);
		write_unlock(&pools->rb_lock);

		/* If a different thread has already created the pool, return it.
		* The insert operation is done under the write lock so we are sure
		* that the pool is not inserted twice.
		*/
		if (existing) {
		kfree(pool);
		return rb_entry(existing, struct ib_frmr_pool, node);
		}

		return pool;
		}

		static int get_frmr_from_pool(struct ib_device *device,
		struct ib_frmr_pool pool, struct ib_mr mr)
		{
		struct ib_frmr_pools *pools = device->frmr_pools;
		u32 handle;
		int err;

		spin_lock(&pool->lock);
		if (pool->queue.ci == 0) {
		spin_unlock(&pool->lock);
		err = pools->pool_ops->create_frmrs(device, &pool->key, &handle,
		1);
		if (err)
		return err;
		} else {
		handle = pop_handle_from_queue_locked(&pool->queue);
		spin_unlock(&pool->lock);
		}

		mr->frmr.pool = pool;
		mr->frmr.handle = handle;

		return 0;
		}

		/*
		* Pop an FRMR handle from the pool.
		*
		* @device: The device to pop the FRMR handle from.
		* @mr: The MR to pop the FRMR handle from.
		*
		* Returns 0 on success, negative error code on failure.
		*/
		int ib_frmr_pool_pop(struct ib_device device, struct ib_mr mr)
		{
		struct ib_frmr_pools *pools = device->frmr_pools;
		struct ib_frmr_pool *pool;

		WARN_ON_ONCE(!device->frmr_pools);
		pool = ib_frmr_pool_find(pools, &mr->frmr.key);
		if (!pool) {
		pool = create_frmr_pool(device, &mr->frmr.key);
		if (IS_ERR(pool))
		return PTR_ERR(pool);
		}

		return get_frmr_from_pool(device, pool, mr);
		}
		EXPORT_SYMBOL(ib_frmr_pool_pop);

		/*
		* Push an FRMR handle back to the pool.
		*
		* @device: The device to push the FRMR handle to.
		* @mr: The MR containing the FRMR handle to push back to the pool.
		*
		* Returns 0 on success, negative error code on failure.
		*/
		int ib_frmr_pool_push(struct ib_device device, struct ib_mr mr)
		{
		struct ib_frmr_pool *pool = mr->frmr.pool;
		int ret;

		spin_lock(&pool->lock);
		ret = push_handle_to_queue_locked(&pool->queue, mr->frmr.handle);
		spin_unlock(&pool->lock);

		return ret;
		}
		EXPORT_SYMBOL(ib_frmr_pool_push);

drivers/infiniband/core/frmr_pools.h

0 → 100644

+48 −0

Original line number	Diff line number	Diff line
		/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
		*
		* Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
		*/

		#ifndef RDMA_CORE_FRMR_POOLS_H
		#define RDMA_CORE_FRMR_POOLS_H

		#include <rdma/frmr_pools.h>
		#include <linux/rbtree_types.h>
		#include <linux/spinlock_types.h>
		#include <linux/types.h>
		#include <asm/page.h>

		#define NUM_HANDLES_PER_PAGE \
		((PAGE_SIZE - sizeof(struct list_head)) / sizeof(u32))

		struct frmr_handles_page {
		struct list_head list;
		u32 handles[NUM_HANDLES_PER_PAGE];
		};

		/* FRMR queue holds a list of frmr_handles_page.
		* num_pages: number of pages in the queue.
		* ci: current index in the handles array across all pages.
		*/
		struct frmr_queue {
		struct list_head pages_list;
		u32 num_pages;
		unsigned long ci;
		};

		struct ib_frmr_pool {
		struct rb_node node;
		struct ib_frmr_key key; /* Pool key */

		/* Protect access to the queue */
		spinlock_t lock;
		struct frmr_queue queue;
		};

		struct ib_frmr_pools {
		struct rb_root rb_root;
		rwlock_t rb_lock;
		const struct ib_frmr_pool_ops *pool_ops;
		};

		#endif /* RDMA_CORE_FRMR_POOLS_H */

include/rdma/frmr_pools.h

0 → 100644

+37 −0

Original line number	Diff line number	Diff line
		/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
		*
		* Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
		*/

		#ifndef FRMR_POOLS_H
		#define FRMR_POOLS_H

		#include <linux/types.h>
		#include <asm/page.h>

		struct ib_device;
		struct ib_mr;

		struct ib_frmr_key {
		u64 vendor_key;
		/* A pool with non-zero kernel_vendor_key is a kernel-only pool. */
		u64 kernel_vendor_key;
		size_t num_dma_blocks;
		int access_flags;
		u8 ats:1;
		};

		struct ib_frmr_pool_ops {
		int (create_frmrs)(struct ib_device device, struct ib_frmr_key *key,
		u32 *handles, u32 count);
		void (destroy_frmrs)(struct ib_device device, u32 *handles,
		u32 count);
		};

		int ib_frmr_pools_init(struct ib_device *device,
		const struct ib_frmr_pool_ops *pool_ops);
		void ib_frmr_pools_cleanup(struct ib_device *device);
		int ib_frmr_pool_pop(struct ib_device device, struct ib_mr mr);
		int ib_frmr_pool_push(struct ib_device device, struct ib_mr mr);

		#endif /* FRMR_POOLS_H */

include/rdma/ib_verbs.h

+8 −0

Original line number	Diff line number	Diff line
		@@ -44,6 +44,7 @@
		#include <uapi/rdma/rdma_user_ioctl.h>
		#include <uapi/rdma/ib_user_ioctl_verbs.h>
		#include <linux/pci-tph.h>
		#include <rdma/frmr_pools.h>
		#include <linux/dma-buf.h>

		#define IB_FW_VERSION_NAME_MAX ETHTOOL_FWVERS_LEN
		@@ -1905,6 +1906,11 @@ struct ib_mr {
		struct ib_dm *dm;
		struct ib_sig_attrs sig_attrs; / only for IB_MR_TYPE_INTEGRITY MRs */
		struct ib_dmah *dmah;
		struct {
		struct ib_frmr_pool *pool;
		struct ib_frmr_key key;
		u32 handle;
		} frmr;
		/*
		* Implementation details of the RDMA core, don't use in drivers:
		*/
		@@ -2907,6 +2913,8 @@ struct ib_device {
		struct list_head subdev_list;

		enum rdma_nl_name_assign_type name_assign_type;

		struct ib_frmr_pools *frmr_pools;
		};

		static inline void rdma_zalloc_obj(struct ib_device dev, size_t size,