Commit b81e0f2b authored by Lorenzo Bianconi's avatar Lorenzo Bianconi Committed by Jakub Kicinski
Browse files

net: airoha: Add FLOW_CLS_STATS callback support 



Introduce per-flow stats accounting to the flowtable hw offload in
the airoha_eth driver. Flow stats are split in the PPE and NPU modules:
- PPE: accounts for high 32bit of per-flow stats
- NPU: accounts for low 32bit of per-flow stats

FLOW_CLS_STATS can be enabled or disabled at compile time.

Signed-off-by: default avatarLorenzo Bianconi <lorenzo@kernel.org>
Reviewed-by: default avatarSimon Horman <horms@kernel.org>
Link: https://patch.msgid.link/20250516-airoha-en7581-flowstats-v2-2-06d5fbf28984@kernel.org


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent c5291874

drivers/net/ethernet/airoha/Kconfig

+7 −0
Original line number Diff line number Diff line
@@ -24,4 +24,11 @@ config NET_AIROHA 
	  This driver supports the gigabit ethernet MACs in the

	  Airoha SoC family.



config NET_AIROHA_FLOW_STATS

	default y

	bool "Airoha flow stats"

	depends on NET_AIROHA && NET_AIROHA_NPU

	help

	  Enable Aiorha flowtable statistic counters.



endif #NET_VENDOR_AIROHA

drivers/net/ethernet/airoha/airoha_eth.h

+33 −0
Original line number Diff line number Diff line
@@ -50,6 +50,14 @@ 
#define PPE_NUM				2

#define PPE1_SRAM_NUM_ENTRIES		(8 * 1024)

#define PPE_SRAM_NUM_ENTRIES		(2 * PPE1_SRAM_NUM_ENTRIES)

#ifdef CONFIG_NET_AIROHA_FLOW_STATS

#define PPE1_STATS_NUM_ENTRIES		(4 * 1024)

#else

#define PPE1_STATS_NUM_ENTRIES		0

#endif /* CONFIG_NET_AIROHA_FLOW_STATS */

#define PPE_STATS_NUM_ENTRIES		(2 * PPE1_STATS_NUM_ENTRIES)

#define PPE1_SRAM_NUM_DATA_ENTRIES	(PPE1_SRAM_NUM_ENTRIES - PPE1_STATS_NUM_ENTRIES)

#define PPE_SRAM_NUM_DATA_ENTRIES	(2 * PPE1_SRAM_NUM_DATA_ENTRIES)

#define PPE_DRAM_NUM_ENTRIES		(16 * 1024)

#define PPE_NUM_ENTRIES			(PPE_SRAM_NUM_ENTRIES + PPE_DRAM_NUM_ENTRIES)

#define PPE_HASH_MASK			(PPE_NUM_ENTRIES - 1)
@@ -261,6 +269,8 @@ struct airoha_foe_mac_info { 


	u16 pppoe_id;

	u16 src_mac_lo;



	u32 meter;

};



#define AIROHA_FOE_IB1_UNBIND_PREBIND		BIT(24)
@@ -296,6 +306,11 @@ struct airoha_foe_mac_info { 
#define AIROHA_FOE_TUNNEL			BIT(6)

#define AIROHA_FOE_TUNNEL_ID			GENMASK(5, 0)



#define AIROHA_FOE_TUNNEL_MTU			GENMASK(31, 16)

#define AIROHA_FOE_ACNT_GRP3			GENMASK(15, 9)

#define AIROHA_FOE_METER_GRP3			GENMASK(8, 5)

#define AIROHA_FOE_METER_GRP2			GENMASK(4, 0)



struct airoha_foe_bridge {

	u32 dest_mac_hi;
@@ -379,6 +394,8 @@ struct airoha_foe_ipv6 { 
	u32 ib2;



	struct airoha_foe_mac_info_common l2;



	u32 meter;

};



struct airoha_foe_entry {
@@ -397,6 +414,16 @@ struct airoha_foe_entry { 
	};

};



struct airoha_foe_stats {

	u32 bytes;

	u32 packets;

};



struct airoha_foe_stats64 {

	u64 bytes;

	u64 packets;

};



struct airoha_flow_data {

	struct ethhdr eth;
@@ -447,6 +474,7 @@ struct airoha_flow_table_entry { 
	struct hlist_node l2_subflow_node; /* PPE L2 subflow entry */

	u32 hash;



	struct airoha_foe_stats64 stats;

	enum airoha_flow_entry_type type;



	struct rhash_head node;
@@ -523,6 +551,9 @@ struct airoha_ppe { 
	struct hlist_head *foe_flow;

	u16 foe_check_time[PPE_NUM_ENTRIES];



	struct airoha_foe_stats *foe_stats;

	dma_addr_t foe_stats_dma;



	struct dentry *debugfs_dir;

};
@@ -582,6 +613,8 @@ int airoha_ppe_init(struct airoha_eth *eth); 
void airoha_ppe_deinit(struct airoha_eth *eth);

struct airoha_foe_entry *airoha_ppe_foe_get_entry(struct airoha_ppe *ppe,

						  u32 hash);

void airoha_ppe_foe_entry_get_stats(struct airoha_ppe *ppe, u32 hash,

				    struct airoha_foe_stats64 *stats);



#ifdef CONFIG_DEBUG_FS

int airoha_ppe_debugfs_init(struct airoha_ppe *ppe);

drivers/net/ethernet/airoha/airoha_npu.c

+51 −1
Original line number Diff line number Diff line
@@ -12,6 +12,7 @@ 
#include <linux/of_reserved_mem.h>

#include <linux/regmap.h>



#include "airoha_eth.h"

#include "airoha_npu.h"



#define NPU_EN7581_FIRMWARE_DATA		"airoha/en7581_npu_data.bin"
@@ -72,6 +73,7 @@ enum { 
	PPE_FUNC_SET_WAIT_HWNAT_INIT,

	PPE_FUNC_SET_WAIT_HWNAT_DEINIT,

	PPE_FUNC_SET_WAIT_API,

	PPE_FUNC_SET_WAIT_FLOW_STATS_SETUP,

};



enum {
@@ -115,6 +117,10 @@ struct ppe_mbox_data { 
			u32 size;

			u32 data;

		} set_info;

		struct {

			u32 npu_stats_addr;

			u32 foe_stats_addr;

		} stats_info;

	};

};
@@ -351,7 +357,40 @@ static int airoha_npu_foe_commit_entry(struct airoha_npu *npu, 
	return err;

}



struct airoha_npu *airoha_npu_get(struct device *dev)

static int airoha_npu_stats_setup(struct airoha_npu *npu,

				  dma_addr_t foe_stats_addr)

{

	int err, size = PPE_STATS_NUM_ENTRIES * sizeof(*npu->stats);

	struct ppe_mbox_data *ppe_data;



	if (!size) /* flow stats are disabled */

		return 0;



	ppe_data = kzalloc(sizeof(*ppe_data), GFP_ATOMIC);

	if (!ppe_data)

		return -ENOMEM;



	ppe_data->func_type = NPU_OP_SET;

	ppe_data->func_id = PPE_FUNC_SET_WAIT_FLOW_STATS_SETUP;

	ppe_data->stats_info.foe_stats_addr = foe_stats_addr;



	err = airoha_npu_send_msg(npu, NPU_FUNC_PPE, ppe_data,

				  sizeof(*ppe_data));

	if (err)

		goto out;



	npu->stats = devm_ioremap(npu->dev,

				  ppe_data->stats_info.npu_stats_addr,

				  size);

	if (!npu->stats)

		err = -ENOMEM;

out:

	kfree(ppe_data);



	return err;

}



struct airoha_npu *airoha_npu_get(struct device *dev, dma_addr_t *stats_addr)

{

	struct platform_device *pdev;

	struct device_node *np;
@@ -389,6 +428,17 @@ struct airoha_npu *airoha_npu_get(struct device *dev) 
		goto error_module_put;

	}



	if (stats_addr) {

		int err;



		err = airoha_npu_stats_setup(npu, *stats_addr);

		if (err) {

			dev_err(dev, "failed to allocate npu stats buffer\n");

			npu = ERR_PTR(err);

			goto error_module_put;

		}

	}



	return npu;



error_module_put:

drivers/net/ethernet/airoha/airoha_npu.h

+3 −1
Original line number Diff line number Diff line
@@ -17,6 +17,8 @@ struct airoha_npu { 
		struct work_struct wdt_work;

	} cores[NPU_NUM_CORES];



	struct airoha_foe_stats __iomem *stats;



	struct {

		int (*ppe_init)(struct airoha_npu *npu);

		int (*ppe_deinit)(struct airoha_npu *npu);
@@ -30,5 +32,5 @@ struct airoha_npu { 
	} ops;

};



struct airoha_npu *airoha_npu_get(struct device *dev);

struct airoha_npu *airoha_npu_get(struct device *dev, dma_addr_t *stats_addr);

void airoha_npu_put(struct airoha_npu *npu);

drivers/net/ethernet/airoha/airoha_ppe.c

+253 −16
Original line number Diff line number Diff line
@@ -102,7 +102,7 @@ static void airoha_ppe_hw_init(struct airoha_ppe *ppe) 


	if (airoha_ppe2_is_enabled(eth)) {

		sram_num_entries =

			PPE_RAM_NUM_ENTRIES_SHIFT(PPE1_SRAM_NUM_ENTRIES);

			PPE_RAM_NUM_ENTRIES_SHIFT(PPE1_SRAM_NUM_DATA_ENTRIES);

		airoha_fe_rmw(eth, REG_PPE_TB_CFG(0),

			      PPE_SRAM_TB_NUM_ENTRY_MASK |

			      PPE_DRAM_TB_NUM_ENTRY_MASK,
@@ -119,7 +119,7 @@ static void airoha_ppe_hw_init(struct airoha_ppe *ppe) 
					 dram_num_entries));

	} else {

		sram_num_entries =

			PPE_RAM_NUM_ENTRIES_SHIFT(PPE_SRAM_NUM_ENTRIES);

			PPE_RAM_NUM_ENTRIES_SHIFT(PPE_SRAM_NUM_DATA_ENTRIES);

		airoha_fe_rmw(eth, REG_PPE_TB_CFG(0),

			      PPE_SRAM_TB_NUM_ENTRY_MASK |

			      PPE_DRAM_TB_NUM_ENTRY_MASK,
@@ -417,6 +417,77 @@ static u32 airoha_ppe_foe_get_entry_hash(struct airoha_foe_entry *hwe) 
	return hash;

}



static u32 airoha_ppe_foe_get_flow_stats_index(struct airoha_ppe *ppe, u32 hash)

{

	if (!airoha_ppe2_is_enabled(ppe->eth))

		return hash;



	return hash >= PPE_STATS_NUM_ENTRIES ? hash - PPE1_STATS_NUM_ENTRIES

					     : hash;

}



static void airoha_ppe_foe_flow_stat_entry_reset(struct airoha_ppe *ppe,

						 struct airoha_npu *npu,

						 int index)

{

	memset_io(&npu->stats[index], 0, sizeof(*npu->stats));

	memset(&ppe->foe_stats[index], 0, sizeof(*ppe->foe_stats));

}



static void airoha_ppe_foe_flow_stats_reset(struct airoha_ppe *ppe,

					    struct airoha_npu *npu)

{

	int i;



	for (i = 0; i < PPE_STATS_NUM_ENTRIES; i++)

		airoha_ppe_foe_flow_stat_entry_reset(ppe, npu, i);

}



static void airoha_ppe_foe_flow_stats_update(struct airoha_ppe *ppe,

					     struct airoha_npu *npu,

					     struct airoha_foe_entry *hwe,

					     u32 hash)

{

	int type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, hwe->ib1);

	u32 index, pse_port, val, *data, *ib2, *meter;

	u8 nbq;



	index = airoha_ppe_foe_get_flow_stats_index(ppe, hash);

	if (index >= PPE_STATS_NUM_ENTRIES)

		return;



	if (type == PPE_PKT_TYPE_BRIDGE) {

		data = &hwe->bridge.data;

		ib2 = &hwe->bridge.ib2;

		meter = &hwe->bridge.l2.meter;

	} else if (type >= PPE_PKT_TYPE_IPV6_ROUTE_3T) {

		data = &hwe->ipv6.data;

		ib2 = &hwe->ipv6.ib2;

		meter = &hwe->ipv6.meter;

	} else {

		data = &hwe->ipv4.data;

		ib2 = &hwe->ipv4.ib2;

		meter = &hwe->ipv4.l2.meter;

	}



	airoha_ppe_foe_flow_stat_entry_reset(ppe, npu, index);



	val = FIELD_GET(AIROHA_FOE_CHANNEL | AIROHA_FOE_QID, *data);

	*data = (*data & ~AIROHA_FOE_ACTDP) |

		FIELD_PREP(AIROHA_FOE_ACTDP, val);



	val = *ib2 & (AIROHA_FOE_IB2_NBQ | AIROHA_FOE_IB2_PSE_PORT |

		      AIROHA_FOE_IB2_PSE_QOS | AIROHA_FOE_IB2_FAST_PATH);

	*meter |= FIELD_PREP(AIROHA_FOE_TUNNEL_MTU, val);



	pse_port = FIELD_GET(AIROHA_FOE_IB2_PSE_PORT, *ib2);

	nbq = pse_port == 1 ? 6 : 5;

	*ib2 &= ~(AIROHA_FOE_IB2_NBQ | AIROHA_FOE_IB2_PSE_PORT |

		  AIROHA_FOE_IB2_PSE_QOS);

	*ib2 |= FIELD_PREP(AIROHA_FOE_IB2_PSE_PORT, 6) |

		FIELD_PREP(AIROHA_FOE_IB2_NBQ, nbq);

}



struct airoha_foe_entry *airoha_ppe_foe_get_entry(struct airoha_ppe *ppe,

						  u32 hash)

{
@@ -470,6 +541,8 @@ static int airoha_ppe_foe_commit_entry(struct airoha_ppe *ppe, 
	struct airoha_foe_entry *hwe = ppe->foe + hash * sizeof(*hwe);

	u32 ts = airoha_ppe_get_timestamp(ppe);

	struct airoha_eth *eth = ppe->eth;

	struct airoha_npu *npu;

	int err = 0;



	memcpy(&hwe->d, &e->d, sizeof(*hwe) - sizeof(hwe->ib1));

	wmb();
@@ -478,27 +551,30 @@ static int airoha_ppe_foe_commit_entry(struct airoha_ppe *ppe, 
	e->ib1 |= FIELD_PREP(AIROHA_FOE_IB1_BIND_TIMESTAMP, ts);

	hwe->ib1 = e->ib1;



	rcu_read_lock();



	npu = rcu_dereference(eth->npu);

	if (!npu) {

		err = -ENODEV;

		goto unlock;

	}



	airoha_ppe_foe_flow_stats_update(ppe, npu, hwe, hash);



	if (hash < PPE_SRAM_NUM_ENTRIES) {

		dma_addr_t addr = ppe->foe_dma + hash * sizeof(*hwe);

		bool ppe2 = airoha_ppe2_is_enabled(eth) &&

			    hash >= PPE1_SRAM_NUM_ENTRIES;

		struct airoha_npu *npu;

		int err = -ENODEV;



		rcu_read_lock();

		npu = rcu_dereference(eth->npu);

		if (npu)

			err = npu->ops.ppe_foe_commit_entry(npu, addr,

							    sizeof(*hwe), hash,

							    ppe2);

		err = npu->ops.ppe_foe_commit_entry(npu, addr, sizeof(*hwe),

						    hash, ppe2);

	}

unlock:

	rcu_read_unlock();



	return err;

}



	return 0;

}



static void airoha_ppe_foe_remove_flow(struct airoha_ppe *ppe,

				       struct airoha_flow_table_entry *e)

{
@@ -582,6 +658,7 @@ airoha_ppe_foe_commit_subflow_entry(struct airoha_ppe *ppe, 
		l2->common.etype = ETH_P_IPV6;



	hwe.bridge.ib2 = e->data.bridge.ib2;

	hwe.bridge.data = e->data.bridge.data;

	airoha_ppe_foe_commit_entry(ppe, &hwe, hash);



	return 0;
@@ -681,6 +758,98 @@ static int airoha_ppe_foe_flow_commit_entry(struct airoha_ppe *ppe, 
	return 0;

}



static int airoha_ppe_get_entry_idle_time(struct airoha_ppe *ppe, u32 ib1)

{

	u32 state = FIELD_GET(AIROHA_FOE_IB1_BIND_STATE, ib1);

	u32 ts, ts_mask, now = airoha_ppe_get_timestamp(ppe);

	int idle;



	if (state == AIROHA_FOE_STATE_BIND) {

		ts = FIELD_GET(AIROHA_FOE_IB1_BIND_TIMESTAMP, ib1);

		ts_mask = AIROHA_FOE_IB1_BIND_TIMESTAMP;

	} else {

		ts = FIELD_GET(AIROHA_FOE_IB1_UNBIND_TIMESTAMP, ib1);

		now = FIELD_GET(AIROHA_FOE_IB1_UNBIND_TIMESTAMP, now);

		ts_mask = AIROHA_FOE_IB1_UNBIND_TIMESTAMP;

	}

	idle = now - ts;



	return idle < 0 ? idle + ts_mask + 1 : idle;

}



static void

airoha_ppe_foe_flow_l2_entry_update(struct airoha_ppe *ppe,

				    struct airoha_flow_table_entry *e)

{

	int min_idle = airoha_ppe_get_entry_idle_time(ppe, e->data.ib1);

	struct airoha_flow_table_entry *iter;

	struct hlist_node *n;



	lockdep_assert_held(&ppe_lock);



	hlist_for_each_entry_safe(iter, n, &e->l2_flows, l2_subflow_node) {

		struct airoha_foe_entry *hwe;

		u32 ib1, state;

		int idle;



		hwe = airoha_ppe_foe_get_entry(ppe, iter->hash);

		ib1 = READ_ONCE(hwe->ib1);



		state = FIELD_GET(AIROHA_FOE_IB1_BIND_STATE, ib1);

		if (state != AIROHA_FOE_STATE_BIND) {

			iter->hash = 0xffff;

			airoha_ppe_foe_remove_flow(ppe, iter);

			continue;

		}



		idle = airoha_ppe_get_entry_idle_time(ppe, ib1);

		if (idle >= min_idle)

			continue;



		min_idle = idle;

		e->data.ib1 &= ~AIROHA_FOE_IB1_BIND_TIMESTAMP;

		e->data.ib1 |= ib1 & AIROHA_FOE_IB1_BIND_TIMESTAMP;

	}

}



static void airoha_ppe_foe_flow_entry_update(struct airoha_ppe *ppe,

					     struct airoha_flow_table_entry *e)

{

	struct airoha_foe_entry *hwe_p, hwe = {};



	spin_lock_bh(&ppe_lock);



	if (e->type == FLOW_TYPE_L2) {

		airoha_ppe_foe_flow_l2_entry_update(ppe, e);

		goto unlock;

	}



	if (e->hash == 0xffff)

		goto unlock;



	hwe_p = airoha_ppe_foe_get_entry(ppe, e->hash);

	if (!hwe_p)

		goto unlock;



	memcpy(&hwe, hwe_p, sizeof(*hwe_p));

	if (!airoha_ppe_foe_compare_entry(e, &hwe)) {

		e->hash = 0xffff;

		goto unlock;

	}



	e->data.ib1 = hwe.ib1;

unlock:

	spin_unlock_bh(&ppe_lock);

}



static int airoha_ppe_entry_idle_time(struct airoha_ppe *ppe,

				      struct airoha_flow_table_entry *e)

{

	airoha_ppe_foe_flow_entry_update(ppe, e);



	return airoha_ppe_get_entry_idle_time(ppe, e->data.ib1);

}



static int airoha_ppe_flow_offload_replace(struct airoha_gdm_port *port,

					   struct flow_cls_offload *f)

{
@@ -896,6 +1065,60 @@ static int airoha_ppe_flow_offload_destroy(struct airoha_gdm_port *port, 
	return 0;

}



void airoha_ppe_foe_entry_get_stats(struct airoha_ppe *ppe, u32 hash,

				    struct airoha_foe_stats64 *stats)

{

	u32 index = airoha_ppe_foe_get_flow_stats_index(ppe, hash);

	struct airoha_eth *eth = ppe->eth;

	struct airoha_npu *npu;



	if (index >= PPE_STATS_NUM_ENTRIES)

		return;



	rcu_read_lock();



	npu = rcu_dereference(eth->npu);

	if (npu) {

		u64 packets = ppe->foe_stats[index].packets;

		u64 bytes = ppe->foe_stats[index].bytes;

		struct airoha_foe_stats npu_stats;



		memcpy_fromio(&npu_stats, &npu->stats[index],

			      sizeof(*npu->stats));

		stats->packets = packets << 32 | npu_stats.packets;

		stats->bytes = bytes << 32 | npu_stats.bytes;

	}



	rcu_read_unlock();

}



static int airoha_ppe_flow_offload_stats(struct airoha_gdm_port *port,

					 struct flow_cls_offload *f)

{

	struct airoha_eth *eth = port->qdma->eth;

	struct airoha_flow_table_entry *e;

	u32 idle;



	e = rhashtable_lookup(&eth->flow_table, &f->cookie,

			      airoha_flow_table_params);

	if (!e)

		return -ENOENT;



	idle = airoha_ppe_entry_idle_time(eth->ppe, e);

	f->stats.lastused = jiffies - idle * HZ;



	if (e->hash != 0xffff) {

		struct airoha_foe_stats64 stats = {};



		airoha_ppe_foe_entry_get_stats(eth->ppe, e->hash, &stats);

		f->stats.pkts += (stats.packets - e->stats.packets);

		f->stats.bytes += (stats.bytes - e->stats.bytes);

		e->stats = stats;

	}



	return 0;

}



static int airoha_ppe_flow_offload_cmd(struct airoha_gdm_port *port,

				       struct flow_cls_offload *f)

{
@@ -904,6 +1127,8 @@ static int airoha_ppe_flow_offload_cmd(struct airoha_gdm_port *port, 
		return airoha_ppe_flow_offload_replace(port, f);

	case FLOW_CLS_DESTROY:

		return airoha_ppe_flow_offload_destroy(port, f);

	case FLOW_CLS_STATS:

		return airoha_ppe_flow_offload_stats(port, f);

	default:

		break;

	}
@@ -929,11 +1154,12 @@ static int airoha_ppe_flush_sram_entries(struct airoha_ppe *ppe, 


static struct airoha_npu *airoha_ppe_npu_get(struct airoha_eth *eth)

{

	struct airoha_npu *npu = airoha_npu_get(eth->dev);

	struct airoha_npu *npu = airoha_npu_get(eth->dev,

						&eth->ppe->foe_stats_dma);



	if (IS_ERR(npu)) {

		request_module("airoha-npu");

		npu = airoha_npu_get(eth->dev);

		npu = airoha_npu_get(eth->dev, &eth->ppe->foe_stats_dma);

	}



	return npu;
@@ -956,6 +1182,8 @@ static int airoha_ppe_offload_setup(struct airoha_eth *eth) 
	if (err)

		goto error_npu_put;



	airoha_ppe_foe_flow_stats_reset(eth->ppe, npu);



	rcu_assign_pointer(eth->npu, npu);

	synchronize_rcu();
@@ -1027,6 +1255,15 @@ int airoha_ppe_init(struct airoha_eth *eth) 
	if (!ppe->foe_flow)

		return -ENOMEM;



	foe_size = PPE_STATS_NUM_ENTRIES * sizeof(*ppe->foe_stats);

	if (foe_size) {

		ppe->foe_stats = dmam_alloc_coherent(eth->dev, foe_size,

						     &ppe->foe_stats_dma,

						     GFP_KERNEL);

		if (!ppe->foe_stats)

			return -ENOMEM;

	}



	err = rhashtable_init(&eth->flow_table, &airoha_flow_table_params);

	if (err)

		return err;