Commit 906154ca authored by Vlad Dogaru's avatar Vlad Dogaru Committed by Jakub Kicinski
Browse files

net/mlx5: HWS, Generalize complex matchers 



The existing solution of complex matchers splits the match parameters
across two, and exactly two, matchers. For some rather extreme cases
(e.g. IPv6-in-IPv6 tunnels), even two matchers are not enough.

Generalize complex matchers to up to 4 submatchers, and allow easy
extension to more if needed. This resulted in rewriting a large part
of the high-level complex matchers logic, but the original concepts
were rock solid and still hold.

Key characteristics of the new implementation:

* Rework complex matchers to include multiple submatchers. All
  submatchers but the first are isolated, in keeping with the existing
  paradigm of handing off to specialized matchers that are not otherwise
  reachable by regular rules.

* Similarly, rework complex rules to allow splitting them into more than
  two simple rules. Rules continue to be refcounted to allow for
  multiple complex rules matching on identical parts of the match
  params.

* Rely on the match tag, as opposed to the entire match_param, to hash
  subrules. This results in lower memory usage.

* Prefer to split the original user-supplied match parameters rather
  than the internal field descriptors. This avoids the awkward
  transition back and forth between the two formats.

* Allow splitting multi-dword fields across matchers. The only
  restrictions that the new implementation impose are: a) any fragment
  of an IP address must be accompanied by a match on the IP version; and
  b) a single lower dword of an IPv6 address cannot be present in a
  submatcher as it would be interpreted as an IPv4 address.

* Employ a greedy algorithm to split the match params, as opposed to
  complete search. The results are not optimal, but the algorithm is now
  linear compared to exponential. Consequently, we see complex matcher
  creation time drops two orders of magnitude in our tests.

Signed-off-by: default avatarVlad Dogaru <vdogaru@nvidia.com>
Signed-off-by: default avatarYevgeny Kliteynik <kliteyn@nvidia.com>
Reviewed-by: default avatarMark Bloch <mbloch@nvidia.com>
Signed-off-by: default avatarTariq Toukan <tariqt@nvidia.com>
Link: https://patch.msgid.link/1759094723-843774-2-git-send-email-tariqt@nvidia.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent fd8c8216

drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.c

+19 −18
Original line number Diff line number Diff line
@@ -51,9 +51,6 @@ static void hws_bwc_matcher_init_attr(struct mlx5hws_bwc_matcher *bwc_matcher, 
				      u8 size_log_rx, u8 size_log_tx,

				      struct mlx5hws_matcher_attr *attr)

{

	struct mlx5hws_bwc_matcher *first_matcher =

		bwc_matcher->complex_first_bwc_matcher;



	memset(attr, 0, sizeof(*attr));



	attr->priority = priority;
@@ -66,9 +63,6 @@ static void hws_bwc_matcher_init_attr(struct mlx5hws_bwc_matcher *bwc_matcher, 
	attr->size[MLX5HWS_MATCHER_SIZE_TYPE_TX].rule.num_log = size_log_tx;

	attr->resizable = true;

	attr->max_num_of_at_attach = MLX5HWS_BWC_MATCHER_ATTACH_AT_NUM;



	attr->isolated_matcher_end_ft_id =

		first_matcher ? first_matcher->matcher->end_ft_id : 0;

}



static int
@@ -171,10 +165,16 @@ hws_bwc_matcher_move_all_simple(struct mlx5hws_bwc_matcher *bwc_matcher) 


static int hws_bwc_matcher_move_all(struct mlx5hws_bwc_matcher *bwc_matcher)

{

	if (!bwc_matcher->complex)

	switch (bwc_matcher->matcher_type) {

	case MLX5HWS_BWC_MATCHER_SIMPLE:

		return hws_bwc_matcher_move_all_simple(bwc_matcher);



	return mlx5hws_bwc_matcher_move_all_complex(bwc_matcher);

	case MLX5HWS_BWC_MATCHER_COMPLEX_FIRST:

		return mlx5hws_bwc_matcher_complex_move_first(bwc_matcher);

	case MLX5HWS_BWC_MATCHER_COMPLEX_SUBMATCHER:

		return mlx5hws_bwc_matcher_complex_move(bwc_matcher);

	default:

		return -EINVAL;

	}

}



static int hws_bwc_matcher_move(struct mlx5hws_bwc_matcher *bwc_matcher)
@@ -249,6 +249,7 @@ int mlx5hws_bwc_matcher_create_simple(struct mlx5hws_bwc_matcher *bwc_matcher, 
				  bwc_matcher->tx_size.size_log,

				  &attr);



	bwc_matcher->matcher_type = MLX5HWS_BWC_MATCHER_SIMPLE;

	bwc_matcher->priority = priority;



	bwc_matcher->size_of_at_array = MLX5HWS_BWC_MATCHER_ATTACH_AT_NUM;
@@ -393,7 +394,7 @@ int mlx5hws_bwc_matcher_destroy(struct mlx5hws_bwc_matcher *bwc_matcher) 
			    "BWC matcher destroy: matcher still has %u RX and %u TX rules\n",

			    rx_rules, tx_rules);



	if (bwc_matcher->complex)

	if (bwc_matcher->matcher_type == MLX5HWS_BWC_MATCHER_COMPLEX_FIRST)

		mlx5hws_bwc_matcher_destroy_complex(bwc_matcher);

	else

		mlx5hws_bwc_matcher_destroy_simple(bwc_matcher);
@@ -651,7 +652,8 @@ int mlx5hws_bwc_rule_destroy_simple(struct mlx5hws_bwc_rule *bwc_rule) 


int mlx5hws_bwc_rule_destroy(struct mlx5hws_bwc_rule *bwc_rule)

{

	bool is_complex = !!bwc_rule->bwc_matcher->complex;

	bool is_complex = bwc_rule->bwc_matcher->matcher_type ==

			  MLX5HWS_BWC_MATCHER_COMPLEX_FIRST;

	int ret = 0;



	if (is_complex)
@@ -1147,7 +1149,7 @@ mlx5hws_bwc_rule_create(struct mlx5hws_bwc_matcher *bwc_matcher, 


	bwc_queue_idx = hws_bwc_gen_queue_idx(ctx);



	if (bwc_matcher->complex)

	if (bwc_matcher->matcher_type == MLX5HWS_BWC_MATCHER_COMPLEX_FIRST)

		ret = mlx5hws_bwc_rule_create_complex(bwc_rule,

						      params,

						      flow_source,
@@ -1216,10 +1218,9 @@ int mlx5hws_bwc_rule_action_update(struct mlx5hws_bwc_rule *bwc_rule, 
		return -EINVAL;

	}



	/* For complex rule, the update should happen on the second matcher */

	if (bwc_rule->isolated_bwc_rule)

		return hws_bwc_rule_action_update(bwc_rule->isolated_bwc_rule,

						  rule_actions);

	else

	/* For complex rules, the update should happen on the last subrule. */

	while (bwc_rule->next_subrule)

		bwc_rule = bwc_rule->next_subrule;



	return hws_bwc_rule_action_update(bwc_rule, rule_actions);

}

drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.h

+18 −3
Original line number Diff line number Diff line
@@ -18,6 +18,21 @@ 


#define MLX5HWS_BWC_POLLING_TIMEOUT 60



enum mlx5hws_bwc_matcher_type {

	/* Standalone bwc matcher. */

	MLX5HWS_BWC_MATCHER_SIMPLE,

	/* The first matcher of a complex matcher. When rules are inserted into

	 * a matcher of this type, they are split into subrules and inserted

	 * into their corresponding submatchers.

	 */

	MLX5HWS_BWC_MATCHER_COMPLEX_FIRST,

	/* A submatcher that is part of a complex matcher. For most purposes

	 * these are treated as simple matchers, except when it comes to moving

	 * rules during resize.

	 */

	MLX5HWS_BWC_MATCHER_COMPLEX_SUBMATCHER,

};



struct mlx5hws_bwc_matcher_complex_data;



struct mlx5hws_bwc_matcher_size {
@@ -31,9 +46,9 @@ struct mlx5hws_bwc_matcher { 
	struct mlx5hws_match_template *mt;

	struct mlx5hws_action_template **at;

	struct mlx5hws_bwc_matcher_complex_data *complex;

	struct mlx5hws_bwc_matcher *complex_first_bwc_matcher;

	u8 num_of_at;

	u8 size_of_at_array;

	enum mlx5hws_bwc_matcher_type matcher_type;

	u32 priority;

	struct mlx5hws_bwc_matcher_size rx_size;

	struct mlx5hws_bwc_matcher_size tx_size;
@@ -43,8 +58,8 @@ struct mlx5hws_bwc_matcher { 
struct mlx5hws_bwc_rule {

	struct mlx5hws_bwc_matcher *bwc_matcher;

	struct mlx5hws_rule *rule;

	struct mlx5hws_bwc_rule *isolated_bwc_rule;

	struct mlx5hws_bwc_complex_rule_hash_node *complex_hash_node;

	struct mlx5hws_bwc_rule *next_subrule;

	struct mlx5hws_bwc_complex_subrule_data *subrule_data;

	u32 flow_source;

	u16 bwc_queue_idx;

	bool skip_rx;

drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc_complex.c

+739 −1082

File changed.

Preview size limit exceeded, changes collapsed.

drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc_complex.h

+49 −11
Original line number Diff line number Diff line
@@ -4,25 +4,60 @@ 
#ifndef HWS_BWC_COMPLEX_H_

#define HWS_BWC_COMPLEX_H_



struct mlx5hws_bwc_complex_rule_hash_node {

	u32 match_buf[MLX5_ST_SZ_DW_MATCH_PARAM];

	u32 tag;

#define MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS 4



/* A matcher can't contain two rules with the same match tag, but it is possible

 * that two different complex rules' subrules have the same match tag. In that

 * case, those subrules correspond to a single rule, and we need to refcount.

 */

struct mlx5hws_bwc_complex_subrule_data {

	struct mlx5hws_rule_match_tag match_tag;

	refcount_t refcount;

	bool rtc_valid;

	/* The chain_id is what glues individual subrules into larger complex

	 * rules. It is the value that this subrule writes to register C6, and

	 * that the next subrule matches against.

	 */

	u32 chain_id;

	u32 rtc_0;

	u32 rtc_1;

	/* During rehash we iterate through all the subrules to move them. But

	 * two or more subrules can share the same physical rule in the

	 * submatcher, so we use `was_moved` to keep track if a given rule was

	 * already moved.

	 */

	bool was_moved;

	struct rhash_head hash_node;

};



struct mlx5hws_bwc_complex_submatcher {

	/* Isolated table that the matcher lives in. Not set for the first

	 * matcher, which lives in the original table.

	 */

	struct mlx5hws_table *tbl;

	/* Match a rule with this action to go to `tbl`. This is set in all

	 * submatchers but the first.

	 */

	struct mlx5hws_action *action_tbl;

	/* This submatcher's simple matcher. The first submatcher points to the

	 * outer (complex) matcher.

	 */

	struct mlx5hws_bwc_matcher *bwc_matcher;

	struct rhashtable rules_hash;

	struct ida chain_ida;

	struct mutex hash_lock; /* Protect the hash and ida. */

};



struct mlx5hws_bwc_matcher_complex_data {

	struct mlx5hws_table *isolated_tbl;

	struct mlx5hws_bwc_matcher *isolated_bwc_matcher;

	struct mlx5hws_bwc_complex_submatcher

		submatchers[MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS];

	int num_submatchers;

	/* Actions used by all but the last submatcher to point to the next

	 * submatcher in the chain. The last submatcher uses the action template

	 * from the complex matcher, to perform the actions that the user

	 * originally requested.

	 */

	struct mlx5hws_action *action_metadata;

	struct mlx5hws_action *action_go_to_tbl;

	struct mlx5hws_action *action_last;

	struct rhashtable refcount_hash;

	struct mutex hash_lock; /* Protect the refcount rhashtable */

	struct ida metadata_ida;

};



bool mlx5hws_bwc_match_params_is_complex(struct mlx5hws_context *ctx,
@@ -37,7 +72,10 @@ int mlx5hws_bwc_matcher_create_complex(struct mlx5hws_bwc_matcher *bwc_matcher, 


void mlx5hws_bwc_matcher_destroy_complex(struct mlx5hws_bwc_matcher *bwc_matcher);



int mlx5hws_bwc_matcher_move_all_complex(struct mlx5hws_bwc_matcher *bwc_matcher);

int mlx5hws_bwc_matcher_complex_move(struct mlx5hws_bwc_matcher *bwc_matcher);



int

mlx5hws_bwc_matcher_complex_move_first(struct mlx5hws_bwc_matcher *bwc_matcher);



int mlx5hws_bwc_rule_create_complex(struct mlx5hws_bwc_rule *bwc_rule,

				    struct mlx5hws_match_parameters *params,

drivers/net/ethernet/mellanox/mlx5/core/steering/hws/definer.c

+9 −78
Original line number Diff line number Diff line
@@ -1831,80 +1831,6 @@ hws_definer_conv_match_params_to_hl(struct mlx5hws_context *ctx, 
	return ret;

}



struct mlx5hws_definer_fc *

mlx5hws_definer_conv_match_params_to_compressed_fc(struct mlx5hws_context *ctx,

						   u8 match_criteria_enable,

						   u32 *match_param,

						   int *fc_sz)

{

	struct mlx5hws_definer_fc *compressed_fc = NULL;

	struct mlx5hws_definer_conv_data cd = {0};

	struct mlx5hws_definer_fc *fc;

	int ret;



	fc = hws_definer_alloc_fc(ctx, MLX5HWS_DEFINER_FNAME_MAX);

	if (!fc)

		return NULL;



	cd.fc = fc;

	cd.ctx = ctx;



	if (match_criteria_enable & MLX5HWS_DEFINER_MATCH_CRITERIA_OUTER) {

		ret = hws_definer_conv_outer(&cd, match_param);

		if (ret)

			goto err_free_fc;

	}



	if (match_criteria_enable & MLX5HWS_DEFINER_MATCH_CRITERIA_INNER) {

		ret = hws_definer_conv_inner(&cd, match_param);

		if (ret)

			goto err_free_fc;

	}



	if (match_criteria_enable & MLX5HWS_DEFINER_MATCH_CRITERIA_MISC) {

		ret = hws_definer_conv_misc(&cd, match_param);

		if (ret)

			goto err_free_fc;

	}



	if (match_criteria_enable & MLX5HWS_DEFINER_MATCH_CRITERIA_MISC2) {

		ret = hws_definer_conv_misc2(&cd, match_param);

		if (ret)

			goto err_free_fc;

	}



	if (match_criteria_enable & MLX5HWS_DEFINER_MATCH_CRITERIA_MISC3) {

		ret = hws_definer_conv_misc3(&cd, match_param);

		if (ret)

			goto err_free_fc;

	}



	if (match_criteria_enable & MLX5HWS_DEFINER_MATCH_CRITERIA_MISC4) {

		ret = hws_definer_conv_misc4(&cd, match_param);

		if (ret)

			goto err_free_fc;

	}



	if (match_criteria_enable & MLX5HWS_DEFINER_MATCH_CRITERIA_MISC5) {

		ret = hws_definer_conv_misc5(&cd, match_param);

		if (ret)

			goto err_free_fc;

	}



	/* Allocate fc array on mt */

	compressed_fc = hws_definer_alloc_compressed_fc(fc);

	if (!compressed_fc) {

		mlx5hws_err(ctx,

			    "Convert to compressed fc: failed to set field copy to match template\n");

		goto err_free_fc;

	}

	*fc_sz = hws_definer_get_fc_size(fc);



err_free_fc:

	kfree(fc);

	return compressed_fc;

}



static int

hws_definer_find_byte_in_tag(struct mlx5hws_definer *definer,

			     u32 hl_byte_off,
@@ -2067,7 +1993,7 @@ hws_definer_copy_sel_ctrl(struct mlx5hws_definer_sel_ctrl *ctrl, 
static int

hws_definer_find_best_match_fit(struct mlx5hws_context *ctx,

				struct mlx5hws_definer *definer,

				u8 *hl)

				u8 *hl, bool allow_jumbo)

{

	struct mlx5hws_definer_sel_ctrl ctrl = {0};

	bool found;
@@ -2084,6 +2010,9 @@ hws_definer_find_best_match_fit(struct mlx5hws_context *ctx, 
		return 0;

	}



	if (!allow_jumbo)

		return -E2BIG;



	/* Try to create a full/limited jumbo definer */

	ctrl.allowed_full_dw = ctx->caps->full_dw_jumbo_support ? DW_SELECTORS :

								  DW_SELECTORS_MATCH;
@@ -2160,7 +2089,8 @@ int mlx5hws_definer_compare(struct mlx5hws_definer *definer_a, 
int

mlx5hws_definer_calc_layout(struct mlx5hws_context *ctx,

			    struct mlx5hws_match_template *mt,

			    struct mlx5hws_definer *match_definer)

			    struct mlx5hws_definer *match_definer,

			    bool allow_jumbo)

{

	u8 *match_hl;

	int ret;
@@ -2182,7 +2112,8 @@ mlx5hws_definer_calc_layout(struct mlx5hws_context *ctx, 
	}



	/* Find the match definer layout for header layout match union */

	ret = hws_definer_find_best_match_fit(ctx, match_definer, match_hl);

	ret = hws_definer_find_best_match_fit(ctx, match_definer, match_hl,

					      allow_jumbo);

	if (ret) {

		if (ret == -E2BIG)

			mlx5hws_dbg(ctx,
@@ -2370,7 +2301,7 @@ int mlx5hws_definer_mt_init(struct mlx5hws_context *ctx, 
	struct mlx5hws_definer match_layout = {0};

	int ret;



	ret = mlx5hws_definer_calc_layout(ctx, mt, &match_layout);

	ret = mlx5hws_definer_calc_layout(ctx, mt, &match_layout, true);

	if (ret) {

		mlx5hws_err(ctx, "Failed to calculate matcher definer layout\n");

		return ret;