Commit a24a29c8 authored by Faisal Latif's avatar Faisal Latif Committed by Leon Romanovsky
Browse files

RDMA/irdma: Add Atomic Operations support 



Extend irdma to support atomic operations, namely Compare and Swap and
Fetch and Add, for GEN3 devices.

Signed-off-by: default avatarFaisal Latif <faisal.latif@intel.com>
Signed-off-by: default avatarTatyana Nikolova <tatyana.e.nikolova@intel.com>
Link: https://patch.msgid.link/20250827152545.2056-15-tatyana.e.nikolova@intel.com


Tested-by: default avatarJacob Moroni <jmoroni@google.com>
Signed-off-by: default avatarLeon Romanovsky <leon@kernel.org>
parent eb31dfc2

drivers/infiniband/hw/irdma/ctrl.c

+11 −0
Original line number Diff line number Diff line
@@ -1110,6 +1110,8 @@ static void irdma_sc_qp_setctx_roce_gen_3(struct irdma_sc_qp *qp, 
		      FIELD_PREP(IRDMAQPC_UDPRIVCQENABLE,

				 roce_info->udprivcq_en) |

		      FIELD_PREP(IRDMAQPC_PRIVEN, roce_info->priv_mode_en) |

		      FIELD_PREP(IRDMAQPC_REMOTE_ATOMIC_EN,

				 info->remote_atomics_en) |

		      FIELD_PREP(IRDMAQPC_TIMELYENABLE, roce_info->timely_en));

	set_64bit_val(qp_ctx, 168,

		      FIELD_PREP(IRDMAQPC_QPCOMPCTX, info->qp_compl_ctx));
@@ -1490,6 +1492,8 @@ static int irdma_sc_alloc_stag(struct irdma_sc_dev *dev, 
	      FIELD_PREP(IRDMA_CQPSQ_STAG_REMACCENABLED, info->remote_access) |

	      FIELD_PREP(IRDMA_CQPSQ_STAG_USEHMCFNIDX, info->use_hmc_fcn_index) |

	      FIELD_PREP(IRDMA_CQPSQ_STAG_USEPFRID, info->use_pf_rid) |

	      FIELD_PREP(IRDMA_CQPSQ_STAG_REMOTE_ATOMIC_EN,

			 info->remote_atomics_en) |

	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);

	dma_wmb(); /* make sure WQE is written before valid bit is set */
@@ -1582,6 +1586,8 @@ static int irdma_sc_mr_reg_non_shared(struct irdma_sc_dev *dev, 
	      FIELD_PREP(IRDMA_CQPSQ_STAG_VABASEDTO, addr_type) |

	      FIELD_PREP(IRDMA_CQPSQ_STAG_USEHMCFNIDX, info->use_hmc_fcn_index) |

	      FIELD_PREP(IRDMA_CQPSQ_STAG_USEPFRID, info->use_pf_rid) |

	      FIELD_PREP(IRDMA_CQPSQ_STAG_REMOTE_ATOMIC_EN,

			 info->remote_atomics_en) |

	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);

	dma_wmb(); /* make sure WQE is written before valid bit is set */
@@ -1740,6 +1746,7 @@ int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp, 
	      FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) |

	      FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |

	      FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |

	      FIELD_PREP(IRDMAQPSQ_REMOTE_ATOMICS_EN, info->remote_atomics_en) |

	      FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);

	dma_wmb(); /* make sure WQE is written before valid bit is set */
@@ -5542,6 +5549,10 @@ int irdma_get_rdma_features(struct irdma_sc_dev *dev) 
		}

		dev->feature_info[feat_type] = temp;

	}



	if (dev->feature_info[IRDMA_FTN_FLAGS] & IRDMA_ATOMICS_ALLOWED_BIT)

		dev->hw_attrs.uk_attrs.feature_flags |= IRDMA_FEATURE_ATOMIC_OPS;



exit:

	dma_free_coherent(dev->hw->device, feat_buf.size, feat_buf.va,

			  feat_buf.pa);

drivers/infiniband/hw/irdma/defs.h

+9 −1
Original line number Diff line number Diff line
@@ -189,6 +189,8 @@ enum irdma_protocol_used { 
#define IRDMAQP_OP_RDMA_READ_LOC_INV		0x0b

#define IRDMAQP_OP_NOP				0x0c

#define IRDMAQP_OP_RDMA_WRITE_SOL		0x0d

#define IRDMAQP_OP_ATOMIC_FETCH_ADD		0x0f

#define IRDMAQP_OP_ATOMIC_COMPARE_SWAP_ADD	0x11

#define IRDMAQP_OP_GEN_RTS_AE			0x30



enum irdma_cqp_op_type {
@@ -694,7 +696,8 @@ enum irdma_cqp_op_type { 
#define IRDMA_CQPSQ_STAG_USEPFRID BIT_ULL(61)



#define IRDMA_CQPSQ_STAG_PBA IRDMA_CQPHC_QPCTX

#define IRDMA_CQPSQ_STAG_HMCFNIDX GENMASK_ULL(5, 0)

#define IRDMA_CQPSQ_STAG_HMCFNIDX GENMASK_ULL(15, 0)

#define IRDMA_CQPSQ_STAG_REMOTE_ATOMIC_EN BIT_ULL(61)



#define IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX GENMASK_ULL(27, 0)

#define IRDMA_CQPSQ_QUERYSTAG_IDX IRDMA_CQPSQ_STAG_IDX
@@ -981,6 +984,9 @@ enum irdma_cqp_op_type { 


#define IRDMAQPSQ_REMTO IRDMA_CQPHC_QPCTX



#define IRDMAQPSQ_STAG GENMASK_ULL(31, 0)

#define IRDMAQPSQ_REMOTE_STAG GENMASK_ULL(31, 0)



#define IRDMAQPSQ_STAGRIGHTS GENMASK_ULL(52, 48)

#define IRDMAQPSQ_VABASEDTO BIT_ULL(53)

#define IRDMAQPSQ_MEMWINDOWTYPE BIT_ULL(54)
@@ -991,6 +997,8 @@ enum irdma_cqp_op_type { 


#define IRDMAQPSQ_BASEVA_TO_FBO IRDMA_CQPHC_QPCTX



#define IRDMAQPSQ_REMOTE_ATOMICS_EN BIT_ULL(55)



#define IRDMAQPSQ_LOCSTAG GENMASK_ULL(31, 0)



#define IRDMAQPSQ_STAGKEY GENMASK_ULL(7, 0)

drivers/infiniband/hw/irdma/ig3rdma_hw.c

+0 −3
Original line number Diff line number Diff line
@@ -120,9 +120,6 @@ void ig3rdma_init_hw(struct irdma_sc_dev *dev) 
	dev->hw_attrs.first_hw_vf_fpm_id = 0;

	dev->hw_attrs.max_hw_vf_fpm_id = IG3_MAX_APFS + IG3_MAX_AVFS;

	dev->hw_attrs.uk_attrs.feature_flags |= IRDMA_FEATURE_64_BYTE_CQE;

	if (dev->feature_info[IRDMA_FTN_FLAGS] & IRDMA_ATOMICS_ALLOWED_BIT)

		dev->hw_attrs.uk_attrs.feature_flags |=

			IRDMA_FEATURE_ATOMIC_OPS;

	dev->hw_attrs.uk_attrs.feature_flags |= IRDMA_FEATURE_CQE_TIMESTAMPING;



	dev->hw_attrs.uk_attrs.feature_flags |= IRDMA_FEATURE_SRQ;

drivers/infiniband/hw/irdma/type.h

+4 −0
Original line number Diff line number Diff line
@@ -1087,6 +1087,7 @@ struct irdma_qp_host_ctx_info { 
	u32 srq_id;

	u32 rem_endpoint_idx;

	u16 stats_idx;

	bool remote_atomics_en:1;

	bool srq_valid:1;

	bool tcp_info_valid:1;

	bool iwarp_info_valid:1;
@@ -1127,6 +1128,7 @@ struct irdma_allocate_stag_info { 
	bool use_hmc_fcn_index:1;

	bool use_pf_rid:1;

	bool all_memory:1;

	bool remote_atomics_en:1;

	u16 hmc_fcn_index;

};
@@ -1155,6 +1157,7 @@ struct irdma_reg_ns_stag_info { 
	u8 hmc_fcn_index;

	bool use_pf_rid:1;

	bool all_memory:1;

	bool remote_atomics_en:1;

};



struct irdma_fast_reg_stag_info {
@@ -1178,6 +1181,7 @@ struct irdma_fast_reg_stag_info { 
	u8 hmc_fcn_index;

	bool use_pf_rid:1;

	bool defer_flag:1;

	bool remote_atomics_en:1;

};



struct irdma_dealloc_stag_info {

drivers/infiniband/hw/irdma/uk.c

+102 −0
Original line number Diff line number Diff line
@@ -337,6 +337,108 @@ int irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, 
	return 0;

}



/**

 * irdma_uk_atomic_fetch_add - atomic fetch and add operation

 * @qp: hw qp ptr

 * @info: post sq information

 * @post_sq: flag to post sq

 */

int irdma_uk_atomic_fetch_add(struct irdma_qp_uk *qp,

			      struct irdma_post_sq_info *info, bool post_sq)

{

	struct irdma_atomic_fetch_add *op_info;

	u32 total_size = 0;

	u16 quanta = 2;

	u32 wqe_idx;

	__le64 *wqe;

	u64 hdr;



	op_info = &info->op.atomic_fetch_add;

	wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,

					 info);

	if (!wqe)

		return -ENOMEM;



	set_64bit_val(wqe, 0, op_info->tagged_offset);

	set_64bit_val(wqe, 8,

		      FIELD_PREP(IRDMAQPSQ_STAG, op_info->stag));

	set_64bit_val(wqe, 16, op_info->remote_tagged_offset);



	hdr = FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, 1) |

	      FIELD_PREP(IRDMAQPSQ_REMOTE_STAG, op_info->remote_stag) |

	      FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_ATOMIC_FETCH_ADD) |

	      FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) |

	      FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |

	      FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |

	      FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);



	set_64bit_val(wqe, 32, op_info->fetch_add_data_bytes);

	set_64bit_val(wqe, 40, 0);

	set_64bit_val(wqe, 48, 0);

	set_64bit_val(wqe, 56,

		      FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity));



	dma_wmb(); /* make sure WQE is populated before valid bit is set */



	set_64bit_val(wqe, 24, hdr);



	if (post_sq)

		irdma_uk_qp_post_wr(qp);



	return 0;

}



/**

 * irdma_uk_atomic_compare_swap - atomic compare and swap operation

 * @qp: hw qp ptr

 * @info: post sq information

 * @post_sq: flag to post sq

 */

int irdma_uk_atomic_compare_swap(struct irdma_qp_uk *qp,

				 struct irdma_post_sq_info *info, bool post_sq)

{

	struct irdma_atomic_compare_swap *op_info;

	u32 total_size = 0;

	u16 quanta = 2;

	u32 wqe_idx;

	__le64 *wqe;

	u64 hdr;



	op_info = &info->op.atomic_compare_swap;

	wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,

					 info);

	if (!wqe)

		return -ENOMEM;



	set_64bit_val(wqe, 0, op_info->tagged_offset);

	set_64bit_val(wqe, 8,

		      FIELD_PREP(IRDMAQPSQ_STAG, op_info->stag));

	set_64bit_val(wqe, 16, op_info->remote_tagged_offset);



	hdr = FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, 1) |

	      FIELD_PREP(IRDMAQPSQ_REMOTE_STAG, op_info->remote_stag) |

	      FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_ATOMIC_COMPARE_SWAP_ADD) |

	      FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) |

	      FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |

	      FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |

	      FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);



	set_64bit_val(wqe, 32, op_info->swap_data_bytes);

	set_64bit_val(wqe, 40, op_info->compare_data_bytes);

	set_64bit_val(wqe, 48, 0);

	set_64bit_val(wqe, 56,

		      FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity));



	dma_wmb(); /* make sure WQE is populated before valid bit is set */



	set_64bit_val(wqe, 24, hdr);



	if (post_sq)

		irdma_uk_qp_post_wr(qp);



	return 0;

}



/**

 * irdma_uk_srq_post_receive - post a receive wqe to a shared rq

 * @srq: shared rq ptr