Commit 2307157c authored by Michael Margolin's avatar Michael Margolin Committed by Leon Romanovsky
Browse files

RDMA/efa: Add EFA query MR support 



Add EFA driver uapi definitions and register a new query MR method that
currently returns the physical interconnects the device is using to
reach the MR. Update admin definitions and efa-abi accordingly.

Reviewed-by: default avatarAnas Mousa <anasmous@amazon.com>
Reviewed-by: default avatarFiras Jahjah <firasj@amazon.com>
Signed-off-by: default avatarMichael Margolin <mrgolin@amazon.com>
Link: https://lore.kernel.org/r/20240104095155.10676-1-mrgolin@amazon.com


Signed-off-by: default avatarLeon Romanovsky <leon@kernel.org>
parent 2f188828

drivers/infiniband/hw/efa/efa.h

+11 −1
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */

/*

 * Copyright 2018-2021 Amazon.com, Inc. or its affiliates. All rights reserved.

 * Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved.

 */



#ifndef _EFA_H_
@@ -80,9 +80,19 @@ struct efa_pd { 
	u16 pdn;

};



struct efa_mr_interconnect_info {

	u16 recv_ic_id;

	u16 rdma_read_ic_id;

	u16 rdma_recv_ic_id;

	u8 recv_ic_id_valid : 1;

	u8 rdma_read_ic_id_valid : 1;

	u8 rdma_recv_ic_id_valid : 1;

};



struct efa_mr {

	struct ib_mr ibmr;

	struct ib_umem *umem;

	struct efa_mr_interconnect_info ic_info;

};



struct efa_cq {

drivers/infiniband/hw/efa/efa_admin_cmds_defs.h

+32 −1
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */

/*

 * Copyright 2018-2023 Amazon.com, Inc. or its affiliates. All rights reserved.

 * Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved.

 */



#ifndef _EFA_ADMIN_CMDS_H_
@@ -415,6 +415,32 @@ struct efa_admin_reg_mr_resp { 
	 * memory region

	 */

	u32 r_key;



	/*

	 * Mask indicating which fields have valid values

	 * 0 : recv_ic_id

	 * 1 : rdma_read_ic_id

	 * 2 : rdma_recv_ic_id

	 */

	u8 validity;



	/*

	 * Physical interconnect used by the device to reach the MR for receive

	 * operation

	 */

	u8 recv_ic_id;



	/*

	 * Physical interconnect used by the device to reach the MR for RDMA

	 * read operation

	 */

	u8 rdma_read_ic_id;



	/*

	 * Physical interconnect used by the device to reach the MR for RDMA

	 * write receive

	 */

	u8 rdma_recv_ic_id;

};



struct efa_admin_dereg_mr_cmd {
@@ -999,6 +1025,11 @@ struct efa_admin_host_info { 
#define EFA_ADMIN_REG_MR_CMD_REMOTE_WRITE_ENABLE_MASK       BIT(1)

#define EFA_ADMIN_REG_MR_CMD_REMOTE_READ_ENABLE_MASK        BIT(2)



/* reg_mr_resp */

#define EFA_ADMIN_REG_MR_RESP_RECV_IC_ID_MASK               BIT(0)

#define EFA_ADMIN_REG_MR_RESP_RDMA_READ_IC_ID_MASK          BIT(1)

#define EFA_ADMIN_REG_MR_RESP_RDMA_RECV_IC_ID_MASK          BIT(2)



/* create_cq_cmd */

#define EFA_ADMIN_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_MASK BIT(5)

#define EFA_ADMIN_CREATE_CQ_CMD_VIRT_MASK                   BIT(6)

drivers/infiniband/hw/efa/efa_com_cmd.c

+10 −1
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause

/*

 * Copyright 2018-2023 Amazon.com, Inc. or its affiliates. All rights reserved.

 * Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved.

 */



#include "efa_com.h"
@@ -270,6 +270,15 @@ int efa_com_register_mr(struct efa_com_dev *edev, 


	result->l_key = cmd_completion.l_key;

	result->r_key = cmd_completion.r_key;

	result->ic_info.recv_ic_id = cmd_completion.recv_ic_id;

	result->ic_info.rdma_read_ic_id = cmd_completion.rdma_read_ic_id;

	result->ic_info.rdma_recv_ic_id = cmd_completion.rdma_recv_ic_id;

	result->ic_info.recv_ic_id_valid = EFA_GET(&cmd_completion.validity,

						   EFA_ADMIN_REG_MR_RESP_RECV_IC_ID);

	result->ic_info.rdma_read_ic_id_valid = EFA_GET(&cmd_completion.validity,

							EFA_ADMIN_REG_MR_RESP_RDMA_READ_IC_ID);

	result->ic_info.rdma_recv_ic_id_valid = EFA_GET(&cmd_completion.validity,

							EFA_ADMIN_REG_MR_RESP_RDMA_RECV_IC_ID);



	return 0;

}

drivers/infiniband/hw/efa/efa_com_cmd.h

+11 −1
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */

/*

 * Copyright 2018-2023 Amazon.com, Inc. or its affiliates. All rights reserved.

 * Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved.

 */



#ifndef _EFA_COM_CMD_H_
@@ -199,6 +199,15 @@ struct efa_com_reg_mr_params { 
	u8 indirect;

};



struct efa_com_mr_interconnect_info {

	u16 recv_ic_id;

	u16 rdma_read_ic_id;

	u16 rdma_recv_ic_id;

	u8 recv_ic_id_valid : 1;

	u8 rdma_read_ic_id_valid : 1;

	u8 rdma_recv_ic_id_valid : 1;

};



struct efa_com_reg_mr_result {

	/*

	 * To be used in conjunction with local buffers references in SQ and
@@ -210,6 +219,7 @@ struct efa_com_reg_mr_result { 
	 * accessed memory region

	 */

	u32 r_key;

	struct efa_com_mr_interconnect_info ic_info;

};



struct efa_com_dereg_mr_params {

drivers/infiniband/hw/efa/efa_main.c

+6 −1
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause

/*

 * Copyright 2018-2022 Amazon.com, Inc. or its affiliates. All rights reserved.

 * Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved.

 */



#include <linux/module.h>
@@ -9,6 +9,7 @@ 
#include <linux/version.h>



#include <rdma/ib_user_verbs.h>

#include <rdma/uverbs_ioctl.h>



#include "efa.h"
@@ -36,6 +37,8 @@ MODULE_DEVICE_TABLE(pci, efa_pci_tbl); 
	(BIT(EFA_ADMIN_FATAL_ERROR) | BIT(EFA_ADMIN_WARNING) | \

	 BIT(EFA_ADMIN_NOTIFICATION) | BIT(EFA_ADMIN_KEEP_ALIVE))



extern const struct uapi_definition efa_uapi_defs[];



/* This handler will called for unknown event group or unimplemented handlers */

static void unimplemented_aenq_handler(void *data,

				       struct efa_admin_aenq_entry *aenq_e)
@@ -432,6 +435,8 @@ static int efa_ib_device_add(struct efa_dev *dev) 


	ib_set_device_ops(&dev->ibdev, &efa_dev_ops);



	dev->ibdev.driver_def = efa_uapi_defs;



	err = ib_register_device(&dev->ibdev, "efa_%d", &pdev->dev);

	if (err)

		goto err_destroy_eqs;