Commit ddb23e34 authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'sfc-conntrack-offloads' 



Edward Cree says:

====================
sfc: conntrack offload for tunnels

This series adds support for offloading TC flower rules which require
both connection tracking and tunnel decapsulation.  Depending on the
match keys required, the left-hand-side rule may go in either the
Outer Rule table or the Action Rule table.
====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 2a92fccd e4470561

drivers/net/ethernet/sfc/mae.c

+57 −2
Original line number Diff line number Diff line
@@ -1705,8 +1705,10 @@ static int efx_mae_insert_lhs_outer_rule(struct efx_nic *efx, 


	/* action */

	act = &rule->lhs_act;

	MCDI_SET_DWORD(inbuf, MAE_OUTER_RULE_INSERT_IN_ENCAP_TYPE,

		       MAE_MCDI_ENCAP_TYPE_NONE);

	rc = efx_mae_encap_type_to_mae_type(act->tun_type);

	if (rc < 0)

		return rc;

	MCDI_SET_DWORD(inbuf, MAE_OUTER_RULE_INSERT_IN_ENCAP_TYPE, rc);

	/* We always inhibit CT lookup on TCP_INTERESTING_FLAGS, since the

	 * SW path needs to process the packet to update the conntrack tables

	 * on connection establishment (SYN) or termination (FIN, RST).
@@ -1734,9 +1736,60 @@ static int efx_mae_insert_lhs_outer_rule(struct efx_nic *efx, 
	return 0;

}



static int efx_mae_populate_match_criteria(MCDI_DECLARE_STRUCT_PTR(match_crit),

					   const struct efx_tc_match *match);



static int efx_mae_insert_lhs_action_rule(struct efx_nic *efx,

					  struct efx_tc_lhs_rule *rule,

					  u32 prio)

{

	MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_ACTION_RULE_INSERT_IN_LEN(MAE_FIELD_MASK_VALUE_PAIRS_V2_LEN));

	MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_ACTION_RULE_INSERT_OUT_LEN);

	struct efx_tc_lhs_action *act = &rule->lhs_act;

	MCDI_DECLARE_STRUCT_PTR(match_crit);

	MCDI_DECLARE_STRUCT_PTR(response);

	size_t outlen;

	int rc;



	match_crit = _MCDI_DWORD(inbuf, MAE_ACTION_RULE_INSERT_IN_MATCH_CRITERIA);

	response = _MCDI_DWORD(inbuf, MAE_ACTION_RULE_INSERT_IN_RESPONSE);

	MCDI_STRUCT_SET_DWORD(response, MAE_ACTION_RULE_RESPONSE_ASL_ID,

			      MC_CMD_MAE_ACTION_SET_LIST_ALLOC_OUT_ACTION_SET_LIST_ID_NULL);

	MCDI_STRUCT_SET_DWORD(response, MAE_ACTION_RULE_RESPONSE_AS_ID,

			      MC_CMD_MAE_ACTION_SET_ALLOC_OUT_ACTION_SET_ID_NULL);

	EFX_POPULATE_DWORD_5(*_MCDI_STRUCT_DWORD(response, MAE_ACTION_RULE_RESPONSE_LOOKUP_CONTROL),

			     MAE_ACTION_RULE_RESPONSE_DO_CT, !!act->zone,

			     MAE_ACTION_RULE_RESPONSE_DO_RECIRC,

			     act->rid && !act->zone,

			     MAE_ACTION_RULE_RESPONSE_CT_VNI_MODE,

			     MAE_CT_VNI_MODE_ZERO,

			     MAE_ACTION_RULE_RESPONSE_RECIRC_ID,

			     act->rid ? act->rid->fw_id : 0,

			     MAE_ACTION_RULE_RESPONSE_CT_DOMAIN,

			     act->zone ? act->zone->zone : 0);

	MCDI_STRUCT_SET_DWORD(response, MAE_ACTION_RULE_RESPONSE_COUNTER_ID,

			      act->count ? act->count->cnt->fw_id :

			      MC_CMD_MAE_COUNTER_ALLOC_OUT_COUNTER_ID_NULL);

	MCDI_SET_DWORD(inbuf, MAE_ACTION_RULE_INSERT_IN_PRIO, prio);

	rc = efx_mae_populate_match_criteria(match_crit, &rule->match);

	if (rc)

		return rc;



	rc = efx_mcdi_rpc(efx, MC_CMD_MAE_ACTION_RULE_INSERT, inbuf, sizeof(inbuf),

			  outbuf, sizeof(outbuf), &outlen);

	if (rc)

		return rc;

	if (outlen < sizeof(outbuf))

		return -EIO;

	rule->fw_id = MCDI_DWORD(outbuf, MAE_ACTION_RULE_INSERT_OUT_AR_ID);

	return 0;

}



int efx_mae_insert_lhs_rule(struct efx_nic *efx, struct efx_tc_lhs_rule *rule,

			    u32 prio)

{

	if (rule->is_ar)

		return efx_mae_insert_lhs_action_rule(efx, rule, prio);

	return efx_mae_insert_lhs_outer_rule(efx, rule, prio);

}
@@ -1770,6 +1823,8 @@ static int efx_mae_remove_lhs_outer_rule(struct efx_nic *efx, 


int efx_mae_remove_lhs_rule(struct efx_nic *efx, struct efx_tc_lhs_rule *rule)

{

	if (rule->is_ar)

		return efx_mae_delete_rule(efx, rule->fw_id);

	return efx_mae_remove_lhs_outer_rule(efx, rule);

}

drivers/net/ethernet/sfc/tc.c

+325 −4
Original line number Diff line number Diff line
@@ -642,6 +642,15 @@ static int efx_tc_flower_record_encap_match(struct efx_nic *efx, 
				return -EEXIST;

			}

			break;

		case EFX_TC_EM_PSEUDO_OR:

			/* old EM corresponds to an OR that has to be unique

			 * (it must not overlap with any other OR, whether

			 * direct-EM or pseudo).

			 */

			NL_SET_ERR_MSG_FMT_MOD(extack,

					       "%s encap match conflicts with existing pseudo(OR) entry",

					       em_type ? "Pseudo" : "Direct");

			return -EEXIST;

		default: /* Unrecognised pseudo-type.  Just say no */

			NL_SET_ERR_MSG_FMT_MOD(extack,

					       "%s encap match conflicts with existing pseudo(%d) entry",
@@ -872,6 +881,93 @@ static bool efx_tc_rule_is_lhs_rule(struct flow_rule *fr, 
	return false;

}



/* A foreign LHS rule has matches on enc_ keys at the TC layer (including an

 * implied match on enc_ip_proto UDP).  Translate these into non-enc_ keys,

 * so that we can use the same MAE machinery as local LHS rules (and so that

 * the lhs_rules entries have uniform semantics).  It may seem odd to do it

 * this way round, given that the corresponding fields in the MAE MCDIs are

 * all ENC_, but (a) we don't have enc_L2 or enc_ip_proto in struct

 * efx_tc_match_fields and (b) semantically an LHS rule doesn't have inner

 * fields so it's just matching on *the* header rather than the outer header.

 * Make sure that the non-enc_ keys were not already being matched on, as that

 * would imply a rule that needed a triple lookup.  (Hardware can do that,

 * with OR-AR-CT-AR, but it halves packet rate so we avoid it where possible;

 * see efx_tc_flower_flhs_needs_ar().)

 */

static int efx_tc_flower_translate_flhs_match(struct efx_tc_match *match)

{

	int rc = 0;



#define COPY_MASK_AND_VALUE(_key, _ekey)	({	\

	if (match->mask._key) {				\

		rc = -EOPNOTSUPP;			\

	} else {					\

		match->mask._key = match->mask._ekey;	\

		match->mask._ekey = 0;			\

		match->value._key = match->value._ekey;	\

		match->value._ekey = 0;			\

	}						\

	rc;						\

})

#define COPY_FROM_ENC(_key)	COPY_MASK_AND_VALUE(_key, enc_##_key)

	if (match->mask.ip_proto)

		return -EOPNOTSUPP;

	match->mask.ip_proto = ~0;

	match->value.ip_proto = IPPROTO_UDP;

	if (COPY_FROM_ENC(src_ip) || COPY_FROM_ENC(dst_ip))

		return rc;

#ifdef CONFIG_IPV6

	if (!ipv6_addr_any(&match->mask.src_ip6))

		return -EOPNOTSUPP;

	match->mask.src_ip6 = match->mask.enc_src_ip6;

	memset(&match->mask.enc_src_ip6, 0, sizeof(struct in6_addr));

	if (!ipv6_addr_any(&match->mask.dst_ip6))

		return -EOPNOTSUPP;

	match->mask.dst_ip6 = match->mask.enc_dst_ip6;

	memset(&match->mask.enc_dst_ip6, 0, sizeof(struct in6_addr));

#endif

	if (COPY_FROM_ENC(ip_tos) || COPY_FROM_ENC(ip_ttl))

		return rc;

	/* should really copy enc_ip_frag but we don't have that in

	 * parse_match yet

	 */

	if (COPY_MASK_AND_VALUE(l4_sport, enc_sport) ||

	    COPY_MASK_AND_VALUE(l4_dport, enc_dport))

		return rc;

	return 0;

#undef COPY_FROM_ENC

#undef COPY_MASK_AND_VALUE

}



/* If a foreign LHS rule wants to match on keys that are only available after

 * encap header identification and parsing, then it can't be done in the Outer

 * Rule lookup, because that lookup determines the encap type used to parse

 * beyond the outer headers.  Thus, such rules must use the OR-AR-CT-AR lookup

 * sequence, with an EM (struct efx_tc_encap_match) in the OR step.

 * Return true iff the passed match requires this.

 */

static bool efx_tc_flower_flhs_needs_ar(struct efx_tc_match *match)

{

	/* matches on inner-header keys can't be done in OR */

	return match->mask.eth_proto ||

	       match->mask.vlan_tci[0] || match->mask.vlan_tci[1] ||

	       match->mask.vlan_proto[0] || match->mask.vlan_proto[1] ||

	       memchr_inv(match->mask.eth_saddr, 0, ETH_ALEN) ||

	       memchr_inv(match->mask.eth_daddr, 0, ETH_ALEN) ||

	       match->mask.ip_proto ||

	       match->mask.ip_tos || match->mask.ip_ttl ||

	       match->mask.src_ip || match->mask.dst_ip ||

#ifdef CONFIG_IPV6

	       !ipv6_addr_any(&match->mask.src_ip6) ||

	       !ipv6_addr_any(&match->mask.dst_ip6) ||

#endif

	       match->mask.ip_frag || match->mask.ip_firstfrag ||

	       match->mask.l4_sport || match->mask.l4_dport ||

	       match->mask.tcp_flags ||

	/* nor can VNI */

	       match->mask.enc_keyid;

}



static int efx_tc_flower_handle_lhs_actions(struct efx_nic *efx,

					    struct flow_cls_offload *tc,

					    struct flow_rule *fr,
@@ -882,9 +978,12 @@ static int efx_tc_flower_handle_lhs_actions(struct efx_nic *efx, 
	struct netlink_ext_ack *extack = tc->common.extack;

	struct efx_tc_lhs_action *act = &rule->lhs_act;

	const struct flow_action_entry *fa;

	enum efx_tc_counter_type ctype;

	bool pipe = true;

	int i;



	ctype = rule->is_ar ? EFX_TC_COUNTER_TYPE_AR : EFX_TC_COUNTER_TYPE_OR;



	flow_action_for_each(i, fa, &fr->action) {

		struct efx_tc_ct_zone *ct_zone;

		struct efx_tc_recirc_id *rid;
@@ -917,7 +1016,7 @@ static int efx_tc_flower_handle_lhs_actions(struct efx_nic *efx, 
					return -EOPNOTSUPP;

				}

				cnt = efx_tc_flower_get_counter_index(efx, tc->cookie,

								      EFX_TC_COUNTER_TYPE_OR);

								      ctype);

				if (IS_ERR(cnt)) {

					NL_SET_ERR_MSG_MOD(extack, "Failed to obtain a counter");

					return PTR_ERR(cnt);
@@ -1354,6 +1453,222 @@ static int efx_tc_incomplete_mangle(struct efx_tc_mangler_state *mung, 
	return 0;

}



static int efx_tc_flower_replace_foreign_lhs_ar(struct efx_nic *efx,

						struct flow_cls_offload *tc,

						struct flow_rule *fr,

						struct efx_tc_match *match,

						struct net_device *net_dev)

{

	struct netlink_ext_ack *extack = tc->common.extack;

	struct efx_tc_lhs_rule *rule, *old;

	enum efx_encap_type type;

	int rc;



	type = efx_tc_indr_netdev_type(net_dev);

	if (type == EFX_ENCAP_TYPE_NONE) {

		NL_SET_ERR_MSG_MOD(extack, "Egress encap match on unsupported tunnel device");

		return -EOPNOTSUPP;

	}



	rc = efx_mae_check_encap_type_supported(efx, type);

	if (rc) {

		NL_SET_ERR_MSG_FMT_MOD(extack,

				       "Firmware reports no support for %s encap match",

				       efx_tc_encap_type_name(type));

		return rc;

	}

	/* This is an Action Rule, so it needs a separate Encap Match in the

	 * Outer Rule table.  Insert that now.

	 */

	rc = efx_tc_flower_record_encap_match(efx, match, type,

					      EFX_TC_EM_DIRECT, 0, 0, extack);

	if (rc)

		return rc;



	match->mask.recirc_id = 0xff;

	if (match->mask.ct_state_trk && match->value.ct_state_trk) {

		NL_SET_ERR_MSG_MOD(extack, "LHS rule can never match +trk");

		rc = -EOPNOTSUPP;

		goto release_encap_match;

	}

	/* LHS rules are always -trk, so we don't need to match on that */

	match->mask.ct_state_trk = 0;

	match->value.ct_state_trk = 0;

	/* We must inhibit match on TCP SYN/FIN/RST, so that SW can see

	 * the packet and update the conntrack table.

	 * Outer Rules will do that with CT_TCP_FLAGS_INHIBIT, but Action

	 * Rules don't have that; instead they support matching on

	 * TCP_SYN_FIN_RST (aka TCP_INTERESTING_FLAGS), so use that.

	 * This is only strictly needed if there will be a DO_CT action,

	 * which we don't know yet, but typically there will be and it's

	 * simpler not to bother checking here.

	 */

	match->mask.tcp_syn_fin_rst = true;



	rc = efx_mae_match_check_caps(efx, &match->mask, extack);

	if (rc)

		goto release_encap_match;



	rule = kzalloc(sizeof(*rule), GFP_USER);

	if (!rule) {

		rc = -ENOMEM;

		goto release_encap_match;

	}

	rule->cookie = tc->cookie;

	rule->is_ar = true;

	old = rhashtable_lookup_get_insert_fast(&efx->tc->lhs_rule_ht,

						&rule->linkage,

						efx_tc_lhs_rule_ht_params);

	if (old) {

		netif_dbg(efx, drv, efx->net_dev,

			  "Already offloaded rule (cookie %lx)\n", tc->cookie);

		rc = -EEXIST;

		NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded");

		goto release;

	}



	/* Parse actions */

	rc = efx_tc_flower_handle_lhs_actions(efx, tc, fr, net_dev, rule);

	if (rc)

		goto release;



	rule->match = *match;

	rule->lhs_act.tun_type = type;



	rc = efx_mae_insert_lhs_rule(efx, rule, EFX_TC_PRIO_TC);

	if (rc) {

		NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");

		goto release;

	}

	netif_dbg(efx, drv, efx->net_dev,

		  "Successfully parsed lhs rule (cookie %lx)\n",

		  tc->cookie);

	return 0;



release:

	efx_tc_flower_release_lhs_actions(efx, &rule->lhs_act);

	if (!old)

		rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &rule->linkage,

				       efx_tc_lhs_rule_ht_params);

	kfree(rule);

release_encap_match:

	if (match->encap)

		efx_tc_flower_release_encap_match(efx, match->encap);

	return rc;

}



static int efx_tc_flower_replace_foreign_lhs(struct efx_nic *efx,

					     struct flow_cls_offload *tc,

					     struct flow_rule *fr,

					     struct efx_tc_match *match,

					     struct net_device *net_dev)

{

	struct netlink_ext_ack *extack = tc->common.extack;

	struct efx_tc_lhs_rule *rule, *old;

	enum efx_encap_type type;

	int rc;



	if (tc->common.chain_index) {

		NL_SET_ERR_MSG_MOD(extack, "LHS rule only allowed in chain 0");

		return -EOPNOTSUPP;

	}



	if (!efx_tc_match_is_encap(&match->mask)) {

		/* This is not a tunnel decap rule, ignore it */

		netif_dbg(efx, drv, efx->net_dev, "Ignoring foreign LHS filter without encap match\n");

		return -EOPNOTSUPP;

	}



	if (efx_tc_flower_flhs_needs_ar(match))

		return efx_tc_flower_replace_foreign_lhs_ar(efx, tc, fr, match,

							    net_dev);



	type = efx_tc_indr_netdev_type(net_dev);

	if (type == EFX_ENCAP_TYPE_NONE) {

		NL_SET_ERR_MSG_MOD(extack, "Egress encap match on unsupported tunnel device\n");

		return -EOPNOTSUPP;

	}



	rc = efx_mae_check_encap_type_supported(efx, type);

	if (rc) {

		NL_SET_ERR_MSG_FMT_MOD(extack,

				       "Firmware reports no support for %s encap match",

				       efx_tc_encap_type_name(type));

		return rc;

	}

	/* Reserve the outer tuple with a pseudo Encap Match */

	rc = efx_tc_flower_record_encap_match(efx, match, type,

					      EFX_TC_EM_PSEUDO_OR, 0, 0,

					      extack);

	if (rc)

		return rc;



	if (match->mask.ct_state_trk && match->value.ct_state_trk) {

		NL_SET_ERR_MSG_MOD(extack, "LHS rule can never match +trk");

		rc = -EOPNOTSUPP;

		goto release_encap_match;

	}

	/* LHS rules are always -trk, so we don't need to match on that */

	match->mask.ct_state_trk = 0;

	match->value.ct_state_trk = 0;



	rc = efx_tc_flower_translate_flhs_match(match);

	if (rc) {

		NL_SET_ERR_MSG_MOD(extack, "LHS rule cannot match on inner fields");

		goto release_encap_match;

	}



	rc = efx_mae_match_check_caps_lhs(efx, &match->mask, extack);

	if (rc)

		goto release_encap_match;



	rule = kzalloc(sizeof(*rule), GFP_USER);

	if (!rule) {

		rc = -ENOMEM;

		goto release_encap_match;

	}

	rule->cookie = tc->cookie;

	old = rhashtable_lookup_get_insert_fast(&efx->tc->lhs_rule_ht,

						&rule->linkage,

						efx_tc_lhs_rule_ht_params);

	if (old) {

		netif_dbg(efx, drv, efx->net_dev,

			  "Already offloaded rule (cookie %lx)\n", tc->cookie);

		rc = -EEXIST;

		NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded");

		goto release;

	}



	/* Parse actions */

	rc = efx_tc_flower_handle_lhs_actions(efx, tc, fr, net_dev, rule);

	if (rc)

		goto release;



	rule->match = *match;

	rule->lhs_act.tun_type = type;



	rc = efx_mae_insert_lhs_rule(efx, rule, EFX_TC_PRIO_TC);

	if (rc) {

		NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");

		goto release;

	}

	netif_dbg(efx, drv, efx->net_dev,

		  "Successfully parsed lhs rule (cookie %lx)\n",

		  tc->cookie);

	return 0;



release:

	efx_tc_flower_release_lhs_actions(efx, &rule->lhs_act);

	if (!old)

		rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &rule->linkage,

				       efx_tc_lhs_rule_ht_params);

	kfree(rule);

release_encap_match:

	if (match->encap)

		efx_tc_flower_release_encap_match(efx, match->encap);

	return rc;

}



static int efx_tc_flower_replace_foreign(struct efx_nic *efx,

					 struct net_device *net_dev,

					 struct flow_cls_offload *tc)
@@ -1371,7 +1686,7 @@ static int efx_tc_flower_replace_foreign(struct efx_nic *efx, 


	/* Parse match */

	memset(&match, 0, sizeof(match));

	rc = efx_tc_flower_parse_match(efx, fr, &match, NULL);

	rc = efx_tc_flower_parse_match(efx, fr, &match, extack);

	if (rc)

		return rc;

	/* The rule as given to us doesn't specify a source netdevice.
@@ -1387,6 +1702,10 @@ static int efx_tc_flower_replace_foreign(struct efx_nic *efx, 
	match.value.ingress_port = rc;

	match.mask.ingress_port = ~0;



	if (efx_tc_rule_is_lhs_rule(fr, &match))

		return efx_tc_flower_replace_foreign_lhs(efx, tc, fr, &match,

							 net_dev);



	if (tc->common.chain_index) {

		struct efx_tc_recirc_id *rid;
@@ -1416,6 +1735,7 @@ static int efx_tc_flower_replace_foreign(struct efx_nic *efx, 
	if (match.mask.ct_state_est && !match.value.ct_state_est) {

		if (match.value.tcp_syn_fin_rst) {

			/* Can't offload this combination */

			NL_SET_ERR_MSG_MOD(extack, "TCP flags and -est conflict for offload");

			rc = -EOPNOTSUPP;

			goto release;

		}
@@ -1442,7 +1762,7 @@ static int efx_tc_flower_replace_foreign(struct efx_nic *efx, 
		goto release;

	}



	rc = efx_mae_match_check_caps(efx, &match.mask, NULL);

	rc = efx_mae_match_check_caps(efx, &match.mask, extack);

	if (rc)

		goto release;
@@ -1470,7 +1790,7 @@ static int efx_tc_flower_replace_foreign(struct efx_nic *efx, 
						      extack);

		if (rc)

			goto release;

	} else {

	} else if (!tc->common.chain_index) {

		/* This is not a tunnel decap rule, ignore it */

		netif_dbg(efx, drv, efx->net_dev,

			  "Ignoring foreign filter without encap match\n");
@@ -1530,6 +1850,7 @@ static int efx_tc_flower_replace_foreign(struct efx_nic *efx, 
					goto release;

				}

				if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_COUNT)) {

					NL_SET_ERR_MSG_MOD(extack, "Count action violates action order (can't happen)");

					rc = -EOPNOTSUPP;

					goto release;

				}

drivers/net/ethernet/sfc/tc.h

+6 −0
Original line number Diff line number Diff line
@@ -140,10 +140,14 @@ static inline bool efx_tc_match_is_encap(const struct efx_tc_match_fields *mask) 
 *	The pseudo encap match may be referenced again by an encap match

 *	with different values for these fields, but all masks must match the

 *	first (stored in our child_* fields).

 * @EFX_TC_EM_PSEUDO_OR: registered by an fLHS rule that fits in the OR

 *	table.  The &struct efx_tc_lhs_rule already holds the HW OR entry.

 *	Only one reference to this encap match may exist.

 */

enum efx_tc_em_pseudo_type {

	EFX_TC_EM_DIRECT,

	EFX_TC_EM_PSEUDO_MASK,

	EFX_TC_EM_PSEUDO_OR,

};



struct efx_tc_encap_match {
@@ -183,6 +187,7 @@ struct efx_tc_action_set_list { 
};



struct efx_tc_lhs_action {

	enum efx_encap_type tun_type;

	struct efx_tc_recirc_id *rid;

	struct efx_tc_ct_zone *zone;

	struct efx_tc_counter_index *count;
@@ -203,6 +208,7 @@ struct efx_tc_lhs_rule { 
	struct efx_tc_lhs_action lhs_act;

	struct rhash_head linkage;

	u32 fw_id;

	bool is_ar; /* Action Rule (for OR-AR-CT-AR sequence) */

};



enum efx_tc_rule_prios {