Commit 179a8b51 authored by David S. Miller's avatar David S. Miller
Browse files

Merge tag 'mlx5-fixes-2023-12-04' of git://git.kernel.org/pub/scm/linux/kernel/git/saeed/linux 



Saeed Mahameed says:

====================
mlx5 fixes 2023-12-04

This series provides bug fixes to mlx5 driver.

V1->V2:
  - Drop commit #9 ("net/mlx5e: Forbid devlink reload if IPSec rules are
    offloaded"), we are working on a better fix

Please pull and let me know if there is any problem.
====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 5e3f5b81 ca4ef28d

drivers/net/ethernet/mellanox/mlx5/core/en.h

+1 −0
Original line number Diff line number Diff line
@@ -826,6 +826,7 @@ enum { 
	MLX5E_STATE_DESTROYING,

	MLX5E_STATE_XDP_TX_ENABLED,

	MLX5E_STATE_XDP_ACTIVE,

	MLX5E_STATE_CHANNELS_ACTIVE,

};



struct mlx5e_modify_sq_param {

drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_act.c

+6 −0
Original line number Diff line number Diff line
@@ -83,6 +83,9 @@ mlx5e_tc_post_act_offload(struct mlx5e_post_act *post_act, 
	struct mlx5_flow_spec *spec;

	int err;



	if (IS_ERR(post_act))

		return PTR_ERR(post_act);



	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);

	if (!spec)

		return -ENOMEM;
@@ -111,6 +114,9 @@ mlx5e_tc_post_act_add(struct mlx5e_post_act *post_act, struct mlx5_flow_attr *po 
	struct mlx5e_post_act_handle *handle;

	int err;



	if (IS_ERR(post_act))

		return ERR_CAST(post_act);



	handle = kzalloc(sizeof(*handle), GFP_KERNEL);

	if (!handle)

		return ERR_PTR(-ENOMEM);

drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c

+40 −16
Original line number Diff line number Diff line
@@ -121,7 +121,14 @@ static bool mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry) 
	if (x->xso.type == XFRM_DEV_OFFLOAD_CRYPTO)

		esn_msb = xfrm_replay_seqhi(x, htonl(seq_bottom));



	if (sa_entry->esn_state.esn_msb)

		sa_entry->esn_state.esn = esn;

	else

		/* According to RFC4303, section "3.3.3. Sequence Number Generation",

		 * the first packet sent using a given SA will contain a sequence

		 * number of 1.

		 */

		sa_entry->esn_state.esn = max_t(u32, esn, 1);

	sa_entry->esn_state.esn_msb = esn_msb;



	if (unlikely(overlap && seq_bottom < MLX5E_IPSEC_ESN_SCOPE_MID)) {
@@ -335,6 +342,27 @@ void mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, 
		attrs->replay_esn.esn = sa_entry->esn_state.esn;

		attrs->replay_esn.esn_msb = sa_entry->esn_state.esn_msb;

		attrs->replay_esn.overlap = sa_entry->esn_state.overlap;

		switch (x->replay_esn->replay_window) {

		case 32:

			attrs->replay_esn.replay_window =

				MLX5_IPSEC_ASO_REPLAY_WIN_32BIT;

			break;

		case 64:

			attrs->replay_esn.replay_window =

				MLX5_IPSEC_ASO_REPLAY_WIN_64BIT;

			break;

		case 128:

			attrs->replay_esn.replay_window =

				MLX5_IPSEC_ASO_REPLAY_WIN_128BIT;

			break;

		case 256:

			attrs->replay_esn.replay_window =

				MLX5_IPSEC_ASO_REPLAY_WIN_256BIT;

			break;

		default:

			WARN_ON(true);

			return;

		}

	}



	attrs->dir = x->xso.dir;
@@ -907,9 +935,11 @@ void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv) 
		return;



	mlx5e_accel_ipsec_fs_cleanup(ipsec);

	if (mlx5_ipsec_device_caps(priv->mdev) & MLX5_IPSEC_CAP_TUNNEL)

	if (ipsec->netevent_nb.notifier_call) {

		unregister_netevent_notifier(&ipsec->netevent_nb);

	if (mlx5_ipsec_device_caps(priv->mdev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD)

		ipsec->netevent_nb.notifier_call = NULL;

	}

	if (ipsec->aso)

		mlx5e_ipsec_aso_cleanup(ipsec);

	destroy_workqueue(ipsec->wq);

	kfree(ipsec);
@@ -1018,6 +1048,12 @@ static int mlx5e_xfrm_validate_policy(struct mlx5_core_dev *mdev, 
		}

	}



	if (x->xdo.type == XFRM_DEV_OFFLOAD_PACKET &&

	    !(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD)) {

		NL_SET_ERR_MSG_MOD(extack, "Packet offload is not supported");

		return -EINVAL;

	}



	return 0;

}
@@ -1113,14 +1149,6 @@ static const struct xfrmdev_ops mlx5e_ipsec_xfrmdev_ops = { 
	.xdo_dev_state_free	= mlx5e_xfrm_free_state,

	.xdo_dev_offload_ok	= mlx5e_ipsec_offload_ok,

	.xdo_dev_state_advance_esn = mlx5e_xfrm_advance_esn_state,

};



static const struct xfrmdev_ops mlx5e_ipsec_packet_xfrmdev_ops = {

	.xdo_dev_state_add	= mlx5e_xfrm_add_state,

	.xdo_dev_state_delete	= mlx5e_xfrm_del_state,

	.xdo_dev_state_free	= mlx5e_xfrm_free_state,

	.xdo_dev_offload_ok	= mlx5e_ipsec_offload_ok,

	.xdo_dev_state_advance_esn = mlx5e_xfrm_advance_esn_state,



	.xdo_dev_state_update_curlft = mlx5e_xfrm_update_curlft,

	.xdo_dev_policy_add = mlx5e_xfrm_add_policy,
@@ -1138,11 +1166,7 @@ void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv) 


	mlx5_core_info(mdev, "mlx5e: IPSec ESP acceleration enabled\n");



	if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD)

		netdev->xfrmdev_ops = &mlx5e_ipsec_packet_xfrmdev_ops;

	else

	netdev->xfrmdev_ops = &mlx5e_ipsec_xfrmdev_ops;



	netdev->features |= NETIF_F_HW_ESP;

	netdev->hw_enc_features |= NETIF_F_HW_ESP;

drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h

+9 −13
Original line number Diff line number Diff line
@@ -189,11 +189,19 @@ struct mlx5e_ipsec_ft { 
	u32 refcnt;

};



struct mlx5e_ipsec_drop {

	struct mlx5_flow_handle *rule;

	struct mlx5_fc *fc;

};



struct mlx5e_ipsec_rule {

	struct mlx5_flow_handle *rule;

	struct mlx5_modify_hdr *modify_hdr;

	struct mlx5_pkt_reformat *pkt_reformat;

	struct mlx5_fc *fc;

	struct mlx5e_ipsec_drop replay;

	struct mlx5e_ipsec_drop auth;

	struct mlx5e_ipsec_drop trailer;

};



struct mlx5e_ipsec_miss {
@@ -201,19 +209,6 @@ struct mlx5e_ipsec_miss { 
	struct mlx5_flow_handle *rule;

};



struct mlx5e_ipsec_rx {

	struct mlx5e_ipsec_ft ft;

	struct mlx5e_ipsec_miss pol;

	struct mlx5e_ipsec_miss sa;

	struct mlx5e_ipsec_rule status;

	struct mlx5e_ipsec_miss status_drop;

	struct mlx5_fc *status_drop_cnt;

	struct mlx5e_ipsec_fc *fc;

	struct mlx5_fs_chains *chains;

	u8 allow_tunnel_mode : 1;

	struct xarray ipsec_obj_id_map;

};



struct mlx5e_ipsec_tx_create_attr {

	int prio;

	int pol_level;
@@ -248,6 +243,7 @@ struct mlx5e_ipsec { 
	struct mlx5_ipsec_fs *roce;

	u8 is_uplink_rep: 1;

	struct mlx5e_ipsec_mpv_work mpv_work;

	struct xarray ipsec_obj_id_map;

};



struct mlx5e_ipsec_esn_state {

drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c

+378 −63
Original line number Diff line number Diff line
@@ -32,6 +32,22 @@ struct mlx5e_ipsec_tx { 
	u8 allow_tunnel_mode : 1;

};



struct mlx5e_ipsec_status_checks {

	struct mlx5_flow_group *drop_all_group;

	struct mlx5e_ipsec_drop all;

};



struct mlx5e_ipsec_rx {

	struct mlx5e_ipsec_ft ft;

	struct mlx5e_ipsec_miss pol;

	struct mlx5e_ipsec_miss sa;

	struct mlx5e_ipsec_rule status;

	struct mlx5e_ipsec_status_checks status_drops;

	struct mlx5e_ipsec_fc *fc;

	struct mlx5_fs_chains *chains;

	u8 allow_tunnel_mode : 1;

};



/* IPsec RX flow steering */

static enum mlx5_traffic_types family2tt(u32 family)

{
@@ -128,14 +144,37 @@ static struct mlx5_flow_table *ipsec_ft_create(struct mlx5_flow_namespace *ns, 
	return mlx5_create_auto_grouped_flow_table(ns, &ft_attr);

}



static int ipsec_status_rule(struct mlx5_core_dev *mdev,

			     struct mlx5e_ipsec_rx *rx,

			     struct mlx5_flow_destination *dest)

static void ipsec_rx_status_drop_destroy(struct mlx5e_ipsec *ipsec,

					 struct mlx5e_ipsec_rx *rx)

{

	u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};

	mlx5_del_flow_rules(rx->status_drops.all.rule);

	mlx5_fc_destroy(ipsec->mdev, rx->status_drops.all.fc);

	mlx5_destroy_flow_group(rx->status_drops.drop_all_group);

}



static void ipsec_rx_status_pass_destroy(struct mlx5e_ipsec *ipsec,

					 struct mlx5e_ipsec_rx *rx)

{

	mlx5_del_flow_rules(rx->status.rule);



	if (rx != ipsec->rx_esw)

		return;



#ifdef CONFIG_MLX5_ESWITCH

	mlx5_chains_put_table(esw_chains(ipsec->mdev->priv.eswitch), 0, 1, 0);

#endif

}



static int rx_add_rule_drop_auth_trailer(struct mlx5e_ipsec_sa_entry *sa_entry,

					 struct mlx5e_ipsec_rx *rx)

{

	struct mlx5e_ipsec *ipsec = sa_entry->ipsec;

	struct mlx5_flow_table *ft = rx->ft.status;

	struct mlx5_core_dev *mdev = ipsec->mdev;

	struct mlx5_flow_destination dest = {};

	struct mlx5_flow_act flow_act = {};

	struct mlx5_modify_hdr *modify_hdr;

	struct mlx5_flow_handle *fte;

	struct mlx5_flow_handle *rule;

	struct mlx5_fc *flow_counter;

	struct mlx5_flow_spec *spec;

	int err;
@@ -143,48 +182,273 @@ static int ipsec_status_rule(struct mlx5_core_dev *mdev, 
	if (!spec)

		return -ENOMEM;



	/* Action to copy 7 bit ipsec_syndrome to regB[24:30] */

	MLX5_SET(copy_action_in, action, action_type, MLX5_ACTION_TYPE_COPY);

	MLX5_SET(copy_action_in, action, src_field, MLX5_ACTION_IN_FIELD_IPSEC_SYNDROME);

	MLX5_SET(copy_action_in, action, src_offset, 0);

	MLX5_SET(copy_action_in, action, length, 7);

	MLX5_SET(copy_action_in, action, dst_field, MLX5_ACTION_IN_FIELD_METADATA_REG_B);

	MLX5_SET(copy_action_in, action, dst_offset, 24);

	flow_counter = mlx5_fc_create(mdev, true);

	if (IS_ERR(flow_counter)) {

		err = PTR_ERR(flow_counter);

		mlx5_core_err(mdev,

			      "Failed to add ipsec rx status drop rule counter, err=%d\n", err);

		goto err_cnt;

	}

	sa_entry->ipsec_rule.auth.fc = flow_counter;



	modify_hdr = mlx5_modify_header_alloc(mdev, MLX5_FLOW_NAMESPACE_KERNEL,

					      1, action);

	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP | MLX5_FLOW_CONTEXT_ACTION_COUNT;

	flow_act.flags = FLOW_ACT_NO_APPEND;

	dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;

	dest.counter_id = mlx5_fc_id(flow_counter);

	if (rx == ipsec->rx_esw)

		spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK;



	if (IS_ERR(modify_hdr)) {

		err = PTR_ERR(modify_hdr);

	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters_2.ipsec_syndrome);

	MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.ipsec_syndrome, 1);

	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters_2.metadata_reg_c_2);

	MLX5_SET(fte_match_param, spec->match_value,

		 misc_parameters_2.metadata_reg_c_2,

		 sa_entry->ipsec_obj_id | BIT(31));

	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;

	rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);

	if (IS_ERR(rule)) {

		err = PTR_ERR(rule);

		mlx5_core_err(mdev,

			      "fail to alloc ipsec copy modify_header_id err=%d\n", err);

		goto out_spec;

			      "Failed to add ipsec rx status drop rule, err=%d\n", err);

		goto err_rule;

	}

	sa_entry->ipsec_rule.auth.rule = rule;



	/* create fte */

	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_MOD_HDR |

			  MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |

	flow_counter = mlx5_fc_create(mdev, true);

	if (IS_ERR(flow_counter)) {

		err = PTR_ERR(flow_counter);

		mlx5_core_err(mdev,

			      "Failed to add ipsec rx status drop rule counter, err=%d\n", err);

		goto err_cnt_2;

	}

	sa_entry->ipsec_rule.trailer.fc = flow_counter;



	dest.counter_id = mlx5_fc_id(flow_counter);

	MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.ipsec_syndrome, 2);

	rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);

	if (IS_ERR(rule)) {

		err = PTR_ERR(rule);

		mlx5_core_err(mdev,

			      "Failed to add ipsec rx status drop rule, err=%d\n", err);

		goto err_rule_2;

	}

	sa_entry->ipsec_rule.trailer.rule = rule;



	kvfree(spec);

	return 0;



err_rule_2:

	mlx5_fc_destroy(mdev, sa_entry->ipsec_rule.trailer.fc);

err_cnt_2:

	mlx5_del_flow_rules(sa_entry->ipsec_rule.auth.rule);

err_rule:

	mlx5_fc_destroy(mdev, sa_entry->ipsec_rule.auth.fc);

err_cnt:

	kvfree(spec);

	return err;

}



static int rx_add_rule_drop_replay(struct mlx5e_ipsec_sa_entry *sa_entry, struct mlx5e_ipsec_rx *rx)

{

	struct mlx5e_ipsec *ipsec = sa_entry->ipsec;

	struct mlx5_flow_table *ft = rx->ft.status;

	struct mlx5_core_dev *mdev = ipsec->mdev;

	struct mlx5_flow_destination dest = {};

	struct mlx5_flow_act flow_act = {};

	struct mlx5_flow_handle *rule;

	struct mlx5_fc *flow_counter;

	struct mlx5_flow_spec *spec;

	int err;



	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);

	if (!spec)

		return -ENOMEM;



	flow_counter = mlx5_fc_create(mdev, true);

	if (IS_ERR(flow_counter)) {

		err = PTR_ERR(flow_counter);

		mlx5_core_err(mdev,

			      "Failed to add ipsec rx status drop rule counter, err=%d\n", err);

		goto err_cnt;

	}



	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP | MLX5_FLOW_CONTEXT_ACTION_COUNT;

	flow_act.flags = FLOW_ACT_NO_APPEND;

	dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;

	dest.counter_id = mlx5_fc_id(flow_counter);

	if (rx == ipsec->rx_esw)

		spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK;



	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters_2.metadata_reg_c_4);

	MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_4, 1);

	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters_2.metadata_reg_c_2);

	MLX5_SET(fte_match_param, spec->match_value,  misc_parameters_2.metadata_reg_c_2,

		 sa_entry->ipsec_obj_id | BIT(31));

	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;

	rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);

	if (IS_ERR(rule)) {

		err = PTR_ERR(rule);

		mlx5_core_err(mdev,

			      "Failed to add ipsec rx status drop rule, err=%d\n", err);

		goto err_rule;

	}



	sa_entry->ipsec_rule.replay.rule = rule;

	sa_entry->ipsec_rule.replay.fc = flow_counter;



	kvfree(spec);

	return 0;



err_rule:

	mlx5_fc_destroy(mdev, flow_counter);

err_cnt:

	kvfree(spec);

	return err;

}



static int ipsec_rx_status_drop_all_create(struct mlx5e_ipsec *ipsec,

					   struct mlx5e_ipsec_rx *rx)

{

	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);

	struct mlx5_flow_table *ft = rx->ft.status;

	struct mlx5_core_dev *mdev = ipsec->mdev;

	struct mlx5_flow_destination dest = {};

	struct mlx5_flow_act flow_act = {};

	struct mlx5_flow_handle *rule;

	struct mlx5_fc *flow_counter;

	struct mlx5_flow_spec *spec;

	struct mlx5_flow_group *g;

	u32 *flow_group_in;

	int err = 0;



	flow_group_in = kvzalloc(inlen, GFP_KERNEL);

	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);

	if (!flow_group_in || !spec) {

		err = -ENOMEM;

		goto err_out;

	}



	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ft->max_fte - 1);

	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ft->max_fte - 1);

	g = mlx5_create_flow_group(ft, flow_group_in);

	if (IS_ERR(g)) {

		err = PTR_ERR(g);

		mlx5_core_err(mdev,

			      "Failed to add ipsec rx status drop flow group, err=%d\n", err);

		goto err_out;

	}



	flow_counter = mlx5_fc_create(mdev, false);

	if (IS_ERR(flow_counter)) {

		err = PTR_ERR(flow_counter);

		mlx5_core_err(mdev,

			      "Failed to add ipsec rx status drop rule counter, err=%d\n", err);

		goto err_cnt;

	}



	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP | MLX5_FLOW_CONTEXT_ACTION_COUNT;

	dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;

	dest.counter_id = mlx5_fc_id(flow_counter);

	if (rx == ipsec->rx_esw)

		spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK;

	rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);

	if (IS_ERR(rule)) {

		err = PTR_ERR(rule);

		mlx5_core_err(mdev,

			      "Failed to add ipsec rx status drop rule, err=%d\n", err);

		goto err_rule;

	}



	rx->status_drops.drop_all_group = g;

	rx->status_drops.all.rule = rule;

	rx->status_drops.all.fc = flow_counter;



	kvfree(flow_group_in);

	kvfree(spec);

	return 0;



err_rule:

	mlx5_fc_destroy(mdev, flow_counter);

err_cnt:

	mlx5_destroy_flow_group(g);

err_out:

	kvfree(flow_group_in);

	kvfree(spec);

	return err;

}



static int ipsec_rx_status_pass_create(struct mlx5e_ipsec *ipsec,

				       struct mlx5e_ipsec_rx *rx,

				       struct mlx5_flow_destination *dest)

{

	struct mlx5_flow_act flow_act = {};

	struct mlx5_flow_handle *rule;

	struct mlx5_flow_spec *spec;

	int err;



	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);

	if (!spec)

		return -ENOMEM;



	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,

			 misc_parameters_2.ipsec_syndrome);

	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,

			 misc_parameters_2.metadata_reg_c_4);

	MLX5_SET(fte_match_param, spec->match_value,

		 misc_parameters_2.ipsec_syndrome, 0);

	MLX5_SET(fte_match_param, spec->match_value,

		 misc_parameters_2.metadata_reg_c_4, 0);

	if (rx == ipsec->rx_esw)

		spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK;

	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;

	flow_act.flags = FLOW_ACT_NO_APPEND;

	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |

			  MLX5_FLOW_CONTEXT_ACTION_COUNT;

	flow_act.modify_hdr = modify_hdr;

	fte = mlx5_add_flow_rules(rx->ft.status, spec, &flow_act, dest, 2);

	if (IS_ERR(fte)) {

		err = PTR_ERR(fte);

		mlx5_core_err(mdev, "fail to add ipsec rx err copy rule err=%d\n", err);

		goto out;

	rule = mlx5_add_flow_rules(rx->ft.status, spec, &flow_act, dest, 2);

	if (IS_ERR(rule)) {

		err = PTR_ERR(rule);

		mlx5_core_warn(ipsec->mdev,

			       "Failed to add ipsec rx status pass rule, err=%d\n", err);

		goto err_rule;

	}



	rx->status.rule = rule;

	kvfree(spec);

	rx->status.rule = fte;

	rx->status.modify_hdr = modify_hdr;

	return 0;



out:

	mlx5_modify_header_dealloc(mdev, modify_hdr);

out_spec:

err_rule:

	kvfree(spec);

	return err;

}



static void mlx5_ipsec_rx_status_destroy(struct mlx5e_ipsec *ipsec,

					 struct mlx5e_ipsec_rx *rx)

{

	ipsec_rx_status_pass_destroy(ipsec, rx);

	ipsec_rx_status_drop_destroy(ipsec, rx);

}



static int mlx5_ipsec_rx_status_create(struct mlx5e_ipsec *ipsec,

				       struct mlx5e_ipsec_rx *rx,

				       struct mlx5_flow_destination *dest)

{

	int err;



	err = ipsec_rx_status_drop_all_create(ipsec, rx);

	if (err)

		return err;



	err = ipsec_rx_status_pass_create(ipsec, rx, dest);

	if (err)

		goto err_pass_create;



	return 0;



err_pass_create:

	ipsec_rx_status_drop_destroy(ipsec, rx);

	return err;

}



static int ipsec_miss_create(struct mlx5_core_dev *mdev,

			     struct mlx5_flow_table *ft,

			     struct mlx5e_ipsec_miss *miss,
@@ -333,12 +597,7 @@ static void rx_destroy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, 
	mlx5_destroy_flow_table(rx->ft.sa);

	if (rx->allow_tunnel_mode)

		mlx5_eswitch_unblock_encap(mdev);

	if (rx == ipsec->rx_esw) {

		mlx5_esw_ipsec_rx_status_destroy(ipsec, rx);

	} else {

		mlx5_del_flow_rules(rx->status.rule);

		mlx5_modify_header_dealloc(mdev, rx->status.modify_hdr);

	}

	mlx5_ipsec_rx_status_destroy(ipsec, rx);

	mlx5_destroy_flow_table(rx->ft.status);



	mlx5_ipsec_fs_roce_rx_destroy(ipsec->roce, family, mdev);
@@ -419,7 +678,7 @@ static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, 
	if (err)

		return err;



	ft = ipsec_ft_create(attr.ns, attr.status_level, attr.prio, 1, 0);

	ft = ipsec_ft_create(attr.ns, attr.status_level, attr.prio, 3, 0);

	if (IS_ERR(ft)) {

		err = PTR_ERR(ft);

		goto err_fs_ft_status;
@@ -428,10 +687,7 @@ static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, 


	dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;

	dest[1].counter_id = mlx5_fc_id(rx->fc->cnt);

	if (rx == ipsec->rx_esw)

		err = mlx5_esw_ipsec_rx_status_create(ipsec, rx, dest);

	else

		err = ipsec_status_rule(mdev, rx, dest);

	err = mlx5_ipsec_rx_status_create(ipsec, rx, dest);

	if (err)

		goto err_add;
@@ -956,13 +1212,22 @@ static void setup_fte_esp(struct mlx5_flow_spec *spec) 
	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_ESP);

}



static void setup_fte_spi(struct mlx5_flow_spec *spec, u32 spi)

static void setup_fte_spi(struct mlx5_flow_spec *spec, u32 spi, bool encap)

{

	/* SPI number */

	spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;



	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters.outer_esp_spi);

	MLX5_SET(fte_match_param, spec->match_value, misc_parameters.outer_esp_spi, spi);

	if (encap) {

		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,

				 misc_parameters.inner_esp_spi);

		MLX5_SET(fte_match_param, spec->match_value,

			 misc_parameters.inner_esp_spi, spi);

	} else {

		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,

				 misc_parameters.outer_esp_spi);

		MLX5_SET(fte_match_param, spec->match_value,

			 misc_parameters.outer_esp_spi, spi);

	}

}



static void setup_fte_no_frags(struct mlx5_flow_spec *spec)
@@ -1052,29 +1317,48 @@ static int setup_modify_header(struct mlx5e_ipsec *ipsec, int type, u32 val, u8 
			       struct mlx5_flow_act *flow_act)

{

	enum mlx5_flow_namespace_type ns_type = ipsec_fs_get_ns(ipsec, type, dir);

	u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};

	u8 action[3][MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};

	struct mlx5_core_dev *mdev = ipsec->mdev;

	struct mlx5_modify_hdr *modify_hdr;

	u8 num_of_actions = 1;



	MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET);

	MLX5_SET(set_action_in, action[0], action_type, MLX5_ACTION_TYPE_SET);

	switch (dir) {

	case XFRM_DEV_OFFLOAD_IN:

		MLX5_SET(set_action_in, action, field,

		MLX5_SET(set_action_in, action[0], field,

			 MLX5_ACTION_IN_FIELD_METADATA_REG_B);



		num_of_actions++;

		MLX5_SET(set_action_in, action[1], action_type, MLX5_ACTION_TYPE_SET);

		MLX5_SET(set_action_in, action[1], field, MLX5_ACTION_IN_FIELD_METADATA_REG_C_2);

		MLX5_SET(set_action_in, action[1], data, val);

		MLX5_SET(set_action_in, action[1], offset, 0);

		MLX5_SET(set_action_in, action[1], length, 32);



		if (type == XFRM_DEV_OFFLOAD_CRYPTO) {

			num_of_actions++;

			MLX5_SET(set_action_in, action[2], action_type,

				 MLX5_ACTION_TYPE_SET);

			MLX5_SET(set_action_in, action[2], field,

				 MLX5_ACTION_IN_FIELD_METADATA_REG_C_4);

			MLX5_SET(set_action_in, action[2], data, 0);

			MLX5_SET(set_action_in, action[2], offset, 0);

			MLX5_SET(set_action_in, action[2], length, 32);

		}

		break;

	case XFRM_DEV_OFFLOAD_OUT:

		MLX5_SET(set_action_in, action, field,

		MLX5_SET(set_action_in, action[0], field,

			 MLX5_ACTION_IN_FIELD_METADATA_REG_C_4);

		break;

	default:

		return -EINVAL;

	}



	MLX5_SET(set_action_in, action, data, val);

	MLX5_SET(set_action_in, action, offset, 0);

	MLX5_SET(set_action_in, action, length, 32);

	MLX5_SET(set_action_in, action[0], data, val);

	MLX5_SET(set_action_in, action[0], offset, 0);

	MLX5_SET(set_action_in, action[0], length, 32);



	modify_hdr = mlx5_modify_header_alloc(mdev, ns_type, 1, action);

	modify_hdr = mlx5_modify_header_alloc(mdev, ns_type, num_of_actions, action);

	if (IS_ERR(modify_hdr)) {

		mlx5_core_err(mdev, "Failed to allocate modify_header %ld\n",

			      PTR_ERR(modify_hdr));
@@ -1321,7 +1605,8 @@ static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) 
	else

		setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6);



	setup_fte_spi(spec, attrs->spi);

	setup_fte_spi(spec, attrs->spi, attrs->encap);

	if (!attrs->encap)

		setup_fte_esp(spec);

	setup_fte_no_frags(spec);

	setup_fte_upper_proto_match(spec, &attrs->upspec);
@@ -1372,6 +1657,15 @@ static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) 
		mlx5_core_err(mdev, "fail to add RX ipsec rule err=%d\n", err);

		goto err_add_flow;

	}

	if (attrs->type == XFRM_DEV_OFFLOAD_PACKET)

		err = rx_add_rule_drop_replay(sa_entry, rx);

	if (err)

		goto err_add_replay;



	err = rx_add_rule_drop_auth_trailer(sa_entry, rx);

	if (err)

		goto err_drop_reason;



	kvfree(spec);



	sa_entry->ipsec_rule.rule = rule;
@@ -1380,6 +1674,13 @@ static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) 
	sa_entry->ipsec_rule.pkt_reformat = flow_act.pkt_reformat;

	return 0;



err_drop_reason:

	if (sa_entry->ipsec_rule.replay.rule) {

		mlx5_del_flow_rules(sa_entry->ipsec_rule.replay.rule);

		mlx5_fc_destroy(mdev, sa_entry->ipsec_rule.replay.fc);

	}

err_add_replay:

	mlx5_del_flow_rules(rule);

err_add_flow:

	mlx5_fc_destroy(mdev, counter);

err_add_cnt:
@@ -1428,7 +1729,7 @@ static int tx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) 


	switch (attrs->type) {

	case XFRM_DEV_OFFLOAD_CRYPTO:

		setup_fte_spi(spec, attrs->spi);

		setup_fte_spi(spec, attrs->spi, false);

		setup_fte_esp(spec);

		setup_fte_reg_a(spec);

		break;
@@ -1809,8 +2110,11 @@ static int mlx5e_ipsec_block_tc_offload(struct mlx5_core_dev *mdev) 
	struct mlx5_eswitch *esw = mdev->priv.eswitch;

	int err = 0;



	if (esw)

		down_write(&esw->mode_lock);

	if (esw) {

		err = mlx5_esw_lock(esw);

		if (err)

			return err;

	}



	if (mdev->num_block_ipsec) {

		err = -EBUSY;
@@ -1821,7 +2125,7 @@ static int mlx5e_ipsec_block_tc_offload(struct mlx5_core_dev *mdev) 


unlock:

	if (esw)

		up_write(&esw->mode_lock);

		mlx5_esw_unlock(esw);



	return err;

}
@@ -1887,6 +2191,17 @@ void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_ipsec_sa_entry *sa_entry) 


	if (ipsec_rule->modify_hdr)

		mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr);



	mlx5_del_flow_rules(ipsec_rule->trailer.rule);

	mlx5_fc_destroy(mdev, ipsec_rule->trailer.fc);



	mlx5_del_flow_rules(ipsec_rule->auth.rule);

	mlx5_fc_destroy(mdev, ipsec_rule->auth.fc);



	if (ipsec_rule->replay.rule) {

		mlx5_del_flow_rules(ipsec_rule->replay.rule);

		mlx5_fc_destroy(mdev, ipsec_rule->replay.fc);

	}

	mlx5_esw_ipsec_rx_id_mapping_remove(sa_entry);

	rx_ft_put(sa_entry->ipsec, sa_entry->attrs.family, sa_entry->attrs.type);

}
@@ -1957,7 +2272,7 @@ void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec) 
	kfree(ipsec->rx_ipv6);



	if (ipsec->is_uplink_rep) {

		xa_destroy(&ipsec->rx_esw->ipsec_obj_id_map);

		xa_destroy(&ipsec->ipsec_obj_id_map);



		mutex_destroy(&ipsec->tx_esw->ft.mutex);

		WARN_ON(ipsec->tx_esw->ft.refcnt);
@@ -2020,7 +2335,7 @@ int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec, 
		mutex_init(&ipsec->tx_esw->ft.mutex);

		mutex_init(&ipsec->rx_esw->ft.mutex);

		ipsec->tx_esw->ns = ns_esw;

		xa_init_flags(&ipsec->rx_esw->ipsec_obj_id_map, XA_FLAGS_ALLOC1);

		xa_init_flags(&ipsec->ipsec_obj_id_map, XA_FLAGS_ALLOC1);

	} else if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_ROCE) {

		ipsec->roce = mlx5_ipsec_fs_roce_init(mdev, devcom);

	} else {