Commit 92508e7f authored by Srujana Challa's avatar Srujana Challa Committed by Herbert Xu
Browse files

crypto: octeontx2 - add SGv2 support for CN10KB or CN10KA B0 



Scatter Gather input format for CPT has changed on CN10KB/CN10KA B0 HW
to make it compatible with NIX Scatter Gather format to support SG mode
for inline IPsec. This patch modifies the code to make the driver works
for the same. This patch also enables CPT firmware load for these chips.

Signed-off-by: default avatarSrujana Challa <schalla@marvell.com>
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent 711b2e2d

drivers/crypto/marvell/octeontx2/cn10k_cpt.c

+12 −4
Original line number Diff line number Diff line
@@ -14,12 +14,14 @@ static struct cpt_hw_ops otx2_hw_ops = { 
	.send_cmd = otx2_cpt_send_cmd,

	.cpt_get_compcode = otx2_cpt_get_compcode,

	.cpt_get_uc_compcode = otx2_cpt_get_uc_compcode,

	.cpt_sg_info_create = otx2_sg_info_create,

};



static struct cpt_hw_ops cn10k_hw_ops = {

	.send_cmd = cn10k_cpt_send_cmd,

	.cpt_get_compcode = cn10k_cpt_get_compcode,

	.cpt_get_uc_compcode = cn10k_cpt_get_uc_compcode,

	.cpt_sg_info_create = otx2_sg_info_create,

};



static void cn10k_cpt_send_cmd(union otx2_cpt_inst_s *cptinst, u32 insts_num,
@@ -78,12 +80,9 @@ int cn10k_cptvf_lmtst_init(struct otx2_cptvf_dev *cptvf) 
	struct pci_dev *pdev = cptvf->pdev;

	resource_size_t offset, size;



	if (!test_bit(CN10K_LMTST, &cptvf->cap_flag)) {

		cptvf->lfs.ops = &otx2_hw_ops;

	if (!test_bit(CN10K_LMTST, &cptvf->cap_flag))

		return 0;

	}



	cptvf->lfs.ops = &cn10k_hw_ops;

	offset = pci_resource_start(pdev, PCI_MBOX_BAR_NUM);

	size = pci_resource_len(pdev, PCI_MBOX_BAR_NUM);

	/* Map VF LMILINE region */
@@ -96,3 +95,12 @@ int cn10k_cptvf_lmtst_init(struct otx2_cptvf_dev *cptvf) 
	return 0;

}

EXPORT_SYMBOL_NS_GPL(cn10k_cptvf_lmtst_init, CRYPTO_DEV_OCTEONTX2_CPT);



void cptvf_hw_ops_get(struct otx2_cptvf_dev *cptvf)

{

	if (test_bit(CN10K_LMTST, &cptvf->cap_flag))

		cptvf->lfs.ops = &cn10k_hw_ops;

	else

		cptvf->lfs.ops = &otx2_hw_ops;

}

EXPORT_SYMBOL_NS_GPL(cptvf_hw_ops_get, CRYPTO_DEV_OCTEONTX2_CPT);

drivers/crypto/marvell/octeontx2/cn10k_cpt.h

+1 −0
Original line number Diff line number Diff line
@@ -30,5 +30,6 @@ static inline u8 otx2_cpt_get_uc_compcode(union otx2_cpt_res_s *result) 


int cn10k_cptpf_lmtst_init(struct otx2_cptpf_dev *cptpf);

int cn10k_cptvf_lmtst_init(struct otx2_cptvf_dev *cptvf);

void cptvf_hw_ops_get(struct otx2_cptvf_dev *cptvf);



#endif /* __CN10K_CPTLF_H */

drivers/crypto/marvell/octeontx2/otx2_cpt_common.h

+33 −1
Original line number Diff line number Diff line
@@ -102,7 +102,10 @@ union otx2_cpt_eng_caps { 
		u64 kasumi:1;

		u64 des:1;

		u64 crc:1;

		u64 reserved_14_63:50;

		u64 mmul:1;

		u64 reserved_15_33:19;

		u64 pdcp_chain:1;

		u64 reserved_35_63:29;

	};

};
@@ -145,6 +148,35 @@ static inline bool is_dev_otx2(struct pci_dev *pdev) 
	return false;

}



static inline bool is_dev_cn10ka(struct pci_dev *pdev)

{

	return pdev->subsystem_device == CPT_PCI_SUBSYS_DEVID_CN10K_A;

}



static inline bool is_dev_cn10ka_ax(struct pci_dev *pdev)

{

	if (pdev->subsystem_device == CPT_PCI_SUBSYS_DEVID_CN10K_A &&

	    ((pdev->revision & 0xFF) == 4 || (pdev->revision & 0xFF) == 0x50 ||

	     (pdev->revision & 0xff) == 0x51))

		return true;



	return false;

}



static inline bool is_dev_cn10kb(struct pci_dev *pdev)

{

	return pdev->subsystem_device == CPT_PCI_SUBSYS_DEVID_CN10K_B;

}



static inline bool is_dev_cn10ka_b0(struct pci_dev *pdev)

{

	if (pdev->subsystem_device == CPT_PCI_SUBSYS_DEVID_CN10K_A &&

	    (pdev->revision & 0xFF) == 0x54)

		return true;



	return false;

}



static inline void otx2_cpt_set_hw_caps(struct pci_dev *pdev,

					unsigned long *cap_flag)

{

drivers/crypto/marvell/octeontx2/otx2_cpt_hw_types.h

+3 −0
Original line number Diff line number Diff line
@@ -13,6 +13,9 @@ 
#define CN10K_CPT_PCI_PF_DEVICE_ID 0xA0F2

#define CN10K_CPT_PCI_VF_DEVICE_ID 0xA0F3



#define CPT_PCI_SUBSYS_DEVID_CN10K_A 0xB900

#define CPT_PCI_SUBSYS_DEVID_CN10K_B 0xBD00



/* Mailbox interrupts offset */

#define OTX2_CPT_PF_MBOX_INT	6

#define OTX2_CPT_PF_INT_VEC_E_MBOXX(x, a) ((x) + (a))

drivers/crypto/marvell/octeontx2/otx2_cpt_reqmgr.h

+296 −0
Original line number Diff line number Diff line
@@ -27,6 +27,13 @@ 


#define OTX2_CPT_MAX_REQ_SIZE 65535



#define SG_COMPS_MAX    4

#define SGV2_COMPS_MAX  3



#define SG_COMP_3    3

#define SG_COMP_2    2

#define SG_COMP_1    1



union otx2_cpt_opcode {

	u16 flags;

	struct {
@@ -143,6 +150,8 @@ struct otx2_cpt_inst_info { 
	unsigned long time_in;

	u32 dlen;

	u32 dma_len;

	u64 gthr_sz;

	u64 sctr_sz;

	u8 extra_time;

};
@@ -157,6 +166,16 @@ struct otx2_cpt_sglist_component { 
	__be64 ptr3;

};



struct cn10kb_cpt_sglist_component {

	u16 len0;

	u16 len1;

	u16 len2;

	u16 valid_segs;

	u64 ptr0;

	u64 ptr1;

	u64 ptr2;

};



static inline void otx2_cpt_info_destroy(struct pci_dev *pdev,

					 struct otx2_cpt_inst_info *info)

{
@@ -188,6 +207,283 @@ static inline void otx2_cpt_info_destroy(struct pci_dev *pdev, 
	kfree(info);

}



static inline int setup_sgio_components(struct pci_dev *pdev,

					struct otx2_cpt_buf_ptr *list,

					int buf_count, u8 *buffer)

{

	struct otx2_cpt_sglist_component *sg_ptr;

	int components;

	int i, j;



	if (unlikely(!list)) {

		dev_err(&pdev->dev, "Input list pointer is NULL\n");

		return -EINVAL;

	}



	for (i = 0; i < buf_count; i++) {

		if (unlikely(!list[i].vptr))

			continue;

		list[i].dma_addr = dma_map_single(&pdev->dev, list[i].vptr,

						  list[i].size,

						  DMA_BIDIRECTIONAL);

		if (unlikely(dma_mapping_error(&pdev->dev, list[i].dma_addr))) {

			dev_err(&pdev->dev, "Dma mapping failed\n");

			goto sg_cleanup;

		}

	}

	components = buf_count / SG_COMPS_MAX;

	sg_ptr = (struct otx2_cpt_sglist_component *)buffer;

	for (i = 0; i < components; i++) {

		sg_ptr->len0 = cpu_to_be16(list[i * SG_COMPS_MAX + 0].size);

		sg_ptr->len1 = cpu_to_be16(list[i * SG_COMPS_MAX + 1].size);

		sg_ptr->len2 = cpu_to_be16(list[i * SG_COMPS_MAX + 2].size);

		sg_ptr->len3 = cpu_to_be16(list[i * SG_COMPS_MAX + 3].size);

		sg_ptr->ptr0 = cpu_to_be64(list[i * SG_COMPS_MAX + 0].dma_addr);

		sg_ptr->ptr1 = cpu_to_be64(list[i * SG_COMPS_MAX + 1].dma_addr);

		sg_ptr->ptr2 = cpu_to_be64(list[i * SG_COMPS_MAX + 2].dma_addr);

		sg_ptr->ptr3 = cpu_to_be64(list[i * SG_COMPS_MAX + 3].dma_addr);

		sg_ptr++;

	}

	components = buf_count % SG_COMPS_MAX;



	switch (components) {

	case SG_COMP_3:

		sg_ptr->len2 = cpu_to_be16(list[i * SG_COMPS_MAX + 2].size);

		sg_ptr->ptr2 = cpu_to_be64(list[i * SG_COMPS_MAX + 2].dma_addr);

		fallthrough;

	case SG_COMP_2:

		sg_ptr->len1 = cpu_to_be16(list[i * SG_COMPS_MAX + 1].size);

		sg_ptr->ptr1 = cpu_to_be64(list[i * SG_COMPS_MAX + 1].dma_addr);

		fallthrough;

	case SG_COMP_1:

		sg_ptr->len0 = cpu_to_be16(list[i * SG_COMPS_MAX + 0].size);

		sg_ptr->ptr0 = cpu_to_be64(list[i * SG_COMPS_MAX + 0].dma_addr);

		break;

	default:

		break;

	}

	return 0;



sg_cleanup:

	for (j = 0; j < i; j++) {

		if (list[j].dma_addr) {

			dma_unmap_single(&pdev->dev, list[j].dma_addr,

					 list[j].size, DMA_BIDIRECTIONAL);

		}



		list[j].dma_addr = 0;

	}

	return -EIO;

}



static inline int sgv2io_components_setup(struct pci_dev *pdev,

					  struct otx2_cpt_buf_ptr *list,

					  int buf_count, u8 *buffer)

{

	struct cn10kb_cpt_sglist_component *sg_ptr;

	int components;

	int i, j;



	if (unlikely(!list)) {

		dev_err(&pdev->dev, "Input list pointer is NULL\n");

		return -EFAULT;

	}



	for (i = 0; i < buf_count; i++) {

		if (unlikely(!list[i].vptr))

			continue;

		list[i].dma_addr = dma_map_single(&pdev->dev, list[i].vptr,

						  list[i].size,

						  DMA_BIDIRECTIONAL);

		if (unlikely(dma_mapping_error(&pdev->dev, list[i].dma_addr))) {

			dev_err(&pdev->dev, "Dma mapping failed\n");

			goto sg_cleanup;

		}

	}

	components = buf_count / SGV2_COMPS_MAX;

	sg_ptr = (struct cn10kb_cpt_sglist_component *)buffer;

	for (i = 0; i < components; i++) {

		sg_ptr->len0 = list[i * SGV2_COMPS_MAX + 0].size;

		sg_ptr->len1 = list[i * SGV2_COMPS_MAX + 1].size;

		sg_ptr->len2 = list[i * SGV2_COMPS_MAX + 2].size;

		sg_ptr->ptr0 = list[i * SGV2_COMPS_MAX + 0].dma_addr;

		sg_ptr->ptr1 = list[i * SGV2_COMPS_MAX + 1].dma_addr;

		sg_ptr->ptr2 = list[i * SGV2_COMPS_MAX + 2].dma_addr;

		sg_ptr->valid_segs = SGV2_COMPS_MAX;

		sg_ptr++;

	}

	components = buf_count % SGV2_COMPS_MAX;



	sg_ptr->valid_segs = components;

	switch (components) {

	case SG_COMP_2:

		sg_ptr->len1 = list[i * SGV2_COMPS_MAX + 1].size;

		sg_ptr->ptr1 = list[i * SGV2_COMPS_MAX + 1].dma_addr;

		fallthrough;

	case SG_COMP_1:

		sg_ptr->len0 = list[i * SGV2_COMPS_MAX + 0].size;

		sg_ptr->ptr0 = list[i * SGV2_COMPS_MAX + 0].dma_addr;

		break;

	default:

		break;

	}

	return 0;



sg_cleanup:

	for (j = 0; j < i; j++) {

		if (list[j].dma_addr) {

			dma_unmap_single(&pdev->dev, list[j].dma_addr,

					 list[j].size, DMA_BIDIRECTIONAL);

		}



		list[j].dma_addr = 0;

	}

	return -EIO;

}



static inline struct otx2_cpt_inst_info *

cn10k_sgv2_info_create(struct pci_dev *pdev, struct otx2_cpt_req_info *req,

		       gfp_t gfp)

{

	u32 dlen = 0, g_len, sg_len, info_len;

	int align = OTX2_CPT_DMA_MINALIGN;

	struct otx2_cpt_inst_info *info;

	u16 g_sz_bytes, s_sz_bytes;

	u32 total_mem_len;

	int i;



	g_sz_bytes = ((req->in_cnt + 2) / 3) *

		      sizeof(struct cn10kb_cpt_sglist_component);

	s_sz_bytes = ((req->out_cnt + 2) / 3) *

		      sizeof(struct cn10kb_cpt_sglist_component);



	g_len = ALIGN(g_sz_bytes, align);

	sg_len = ALIGN(g_len + s_sz_bytes, align);

	info_len = ALIGN(sizeof(*info), align);

	total_mem_len = sg_len + info_len + sizeof(union otx2_cpt_res_s);



	info = kzalloc(total_mem_len, gfp);

	if (unlikely(!info))

		return NULL;



	for (i = 0; i < req->in_cnt; i++)

		dlen += req->in[i].size;



	info->dlen = dlen;

	info->in_buffer = (u8 *)info + info_len;

	info->gthr_sz = req->in_cnt;

	info->sctr_sz = req->out_cnt;



	/* Setup gather (input) components */

	if (sgv2io_components_setup(pdev, req->in, req->in_cnt,

				    info->in_buffer)) {

		dev_err(&pdev->dev, "Failed to setup gather list\n");

		goto destroy_info;

	}



	if (sgv2io_components_setup(pdev, req->out, req->out_cnt,

				    &info->in_buffer[g_len])) {

		dev_err(&pdev->dev, "Failed to setup scatter list\n");

		goto destroy_info;

	}



	info->dma_len = total_mem_len - info_len;

	info->dptr_baddr = dma_map_single(&pdev->dev, info->in_buffer,

					  info->dma_len, DMA_BIDIRECTIONAL);

	if (unlikely(dma_mapping_error(&pdev->dev, info->dptr_baddr))) {

		dev_err(&pdev->dev, "DMA Mapping failed for cpt req\n");

		goto destroy_info;

	}

	info->rptr_baddr = info->dptr_baddr + g_len;

	/*

	 * Get buffer for union otx2_cpt_res_s response

	 * structure and its physical address

	 */

	info->completion_addr = info->in_buffer + sg_len;

	info->comp_baddr = info->dptr_baddr + sg_len;



	return info;



destroy_info:

	otx2_cpt_info_destroy(pdev, info);

	return NULL;

}



/* SG list header size in bytes */

#define SG_LIST_HDR_SIZE	8

static inline struct otx2_cpt_inst_info *

otx2_sg_info_create(struct pci_dev *pdev, struct otx2_cpt_req_info *req,

		    gfp_t gfp)

{

	int align = OTX2_CPT_DMA_MINALIGN;

	struct otx2_cpt_inst_info *info;

	u32 dlen, align_dlen, info_len;

	u16 g_sz_bytes, s_sz_bytes;

	u32 total_mem_len;



	if (unlikely(req->in_cnt > OTX2_CPT_MAX_SG_IN_CNT ||

		     req->out_cnt > OTX2_CPT_MAX_SG_OUT_CNT)) {

		dev_err(&pdev->dev, "Error too many sg components\n");

		return NULL;

	}



	g_sz_bytes = ((req->in_cnt + 3) / 4) *

		      sizeof(struct otx2_cpt_sglist_component);

	s_sz_bytes = ((req->out_cnt + 3) / 4) *

		      sizeof(struct otx2_cpt_sglist_component);



	dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE;

	align_dlen = ALIGN(dlen, align);

	info_len = ALIGN(sizeof(*info), align);

	total_mem_len = align_dlen + info_len + sizeof(union otx2_cpt_res_s);



	info = kzalloc(total_mem_len, gfp);

	if (unlikely(!info))

		return NULL;



	info->dlen = dlen;

	info->in_buffer = (u8 *)info + info_len;



	((u16 *)info->in_buffer)[0] = req->out_cnt;

	((u16 *)info->in_buffer)[1] = req->in_cnt;

	((u16 *)info->in_buffer)[2] = 0;

	((u16 *)info->in_buffer)[3] = 0;

	cpu_to_be64s((u64 *)info->in_buffer);



	/* Setup gather (input) components */

	if (setup_sgio_components(pdev, req->in, req->in_cnt,

				  &info->in_buffer[8])) {

		dev_err(&pdev->dev, "Failed to setup gather list\n");

		goto destroy_info;

	}



	if (setup_sgio_components(pdev, req->out, req->out_cnt,

				  &info->in_buffer[8 + g_sz_bytes])) {

		dev_err(&pdev->dev, "Failed to setup scatter list\n");

		goto destroy_info;

	}



	info->dma_len = total_mem_len - info_len;

	info->dptr_baddr = dma_map_single(&pdev->dev, info->in_buffer,

					  info->dma_len, DMA_BIDIRECTIONAL);

	if (unlikely(dma_mapping_error(&pdev->dev, info->dptr_baddr))) {

		dev_err(&pdev->dev, "DMA Mapping failed for cpt req\n");

		goto destroy_info;

	}

	/*

	 * Get buffer for union otx2_cpt_res_s response

	 * structure and its physical address

	 */

	info->completion_addr = info->in_buffer + align_dlen;

	info->comp_baddr = info->dptr_baddr + align_dlen;



	return info;



destroy_info:

	otx2_cpt_info_destroy(pdev, info);

	return NULL;

}



struct otx2_cptlf_wqe;

int otx2_cpt_do_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req,

			int cpu_num);