Commit 9963117a authored by Alexander Duyck's avatar Alexander Duyck Committed by Paolo Abeni
Browse files

fbnic: Add logic to track PMD state via MAC/PCS signals 



One complication with the design of our part is that the PMD doesn't
provide a direct signal to the host. Instead we have visibility to signals
that the PCS provides to the MAC that allow us to check the link state
through that.

We will need to account for several things in the PMD and firmware when
managing the link. Specifically when the link first starts to come up the
PMD will cause the link to flap. This is due to the firmware starting a
training cycle when the link is first detected. This will cause link
flapping if we were to immediately report link up when the PCS first
detects it.

To address that we are adding a pmd_state variable that is meant to be a
countdown of sorts indicating the state of the PMD. If the link is down or
has been reconfigured the PMD will start out in the initialize state. By
default the link is assumed to be in the SEND_DATA state if it is available
on initial link inspection. If link is detected while in the initialize
state the PMD state will switch to training, and if after 4 seconds the
link is still stable we will transition to link_ready, and finally the
send_data state.  With this we can avoid link flapping when a cable is
first connected to the NIC.

One side effect of this is that we need to pull the link state away from
the PCS. For now we use a union of the PCS link state register value and
the pmd_state. The plan is to add a PMD register to report the pmd_state
to the phylink interface. With that we can then look at switching over to
the use of the XPCS driver for fbnic instead of having an internal one.

Signed-off-by: default avatarAlexander Duyck <alexanderduyck@fb.com>
Link: https://patch.msgid.link/176374323107.959489.14951134213387615059.stgit@ahduyck-xeon-server.home.arpa


Signed-off-by: default avatarPaolo Abeni <pabeni@redhat.com>
parent f18dd1b1

drivers/net/ethernet/meta/fbnic/fbnic.h

+4 −0
Original line number Diff line number Diff line
@@ -83,6 +83,10 @@ struct fbnic_dev { 
	/* Last @time_high refresh time in jiffies (to catch stalls) */

	unsigned long last_read;



	/* PMD specific data */

	unsigned long end_of_pmd_training;

	u8 pmd_state;



	/* Local copy of hardware statistics */

	struct fbnic_hw_stats hw_stats;

drivers/net/ethernet/meta/fbnic/fbnic_csr.h

+2 −0
Original line number Diff line number Diff line
@@ -787,6 +787,8 @@ enum { 


/* MAC PCS registers */

#define FBNIC_CSR_START_PCS		0x10000 /* CSR section delimiter */

#define FBNIC_PCS_PAGE(n)	(0x10000 + 0x400 * (n))	/* 0x40000 + 1024*n */

#define FBNIC_PCS(reg, n)	((reg) + FBNIC_PCS_PAGE(n))

#define FBNIC_CSR_END_PCS		0x10668 /* CSR section delimiter */



#define FBNIC_CSR_START_RSFEC		0x10800 /* CSR section delimiter */

drivers/net/ethernet/meta/fbnic/fbnic_irq.c

+3 −1
Original line number Diff line number Diff line
@@ -131,6 +131,8 @@ static irqreturn_t fbnic_mac_msix_intr(int __always_unused irq, void *data) 


	fbn = netdev_priv(fbd->netdev);



	/* Record link down events */

	if (!fbd->mac->get_link(fbd, fbn->aui, fbn->fec))

		phylink_pcs_change(&fbn->phylink_pcs, false);



	return IRQ_HANDLED;

drivers/net/ethernet/meta/fbnic/fbnic_mac.c

+51 −18
Original line number Diff line number Diff line
@@ -466,9 +466,8 @@ static u32 __fbnic_mac_cmd_config_asic(struct fbnic_dev *fbd, 
	return command_config;

}



static bool fbnic_mac_get_pcs_link_status(struct fbnic_dev *fbd)

static bool fbnic_mac_get_link_status(struct fbnic_dev *fbd, u8 aui, u8 fec)

{

	struct fbnic_net *fbn = netdev_priv(fbd->netdev);

	u32 pcs_status, lane_mask = ~0;



	pcs_status = rd32(fbd, FBNIC_SIG_PCS_OUT0);
@@ -476,7 +475,7 @@ static bool fbnic_mac_get_pcs_link_status(struct fbnic_dev *fbd) 
		return false;



	/* Define the expected lane mask for the status bits we need to check */

	switch (fbn->aui) {

	switch (aui) {

	case FBNIC_AUI_100GAUI2:

		lane_mask = 0xf;

		break;
@@ -484,7 +483,7 @@ static bool fbnic_mac_get_pcs_link_status(struct fbnic_dev *fbd) 
		lane_mask = 3;

		break;

	case FBNIC_AUI_LAUI2:

		switch (fbn->fec) {

		switch (fec) {

		case FBNIC_FEC_OFF:

			lane_mask = 0x63;

			break;
@@ -502,7 +501,7 @@ static bool fbnic_mac_get_pcs_link_status(struct fbnic_dev *fbd) 
	}



	/* Use an XOR to remove the bits we expect to see set */

	switch (fbn->fec) {

	switch (fec) {

	case FBNIC_FEC_OFF:

		lane_mask ^= FIELD_GET(FBNIC_SIG_PCS_OUT0_BLOCK_LOCK,

				       pcs_status);
@@ -521,7 +520,46 @@ static bool fbnic_mac_get_pcs_link_status(struct fbnic_dev *fbd) 
	return !lane_mask;

}



static bool fbnic_mac_get_link(struct fbnic_dev *fbd)

static bool fbnic_pmd_update_state(struct fbnic_dev *fbd, bool signal_detect)

{

	/* Delay link up for 4 seconds to allow for link training.

	 * The state transitions for this are as follows:

	 *

	 * All states have the following two transitions in common:

	 *	Loss of signal -> FBNIC_PMD_INITIALIZE

	 *		The condition handled below (!signal)

	 *	Reconfiguration -> FBNIC_PMD_INITIALIZE

	 *		Occurs when mac_prepare starts a PHY reconfig

	 * FBNIC_PMD_TRAINING:

	 *	signal still detected && 4s have passed -> Report link up

	 *	When link is brought up in link_up -> FBNIC_PMD_SEND_DATA

	 * FBNIC_PMD_INITIALIZE:

	 *	signal detected -> FBNIC_PMD_TRAINING

	 */

	if (!signal_detect) {

		fbd->pmd_state = FBNIC_PMD_INITIALIZE;

		return false;

	}



	switch (fbd->pmd_state) {

	case FBNIC_PMD_TRAINING:

		return time_before(fbd->end_of_pmd_training, jiffies);

	case FBNIC_PMD_LINK_READY:

	case FBNIC_PMD_SEND_DATA:

		return true;

	}



	fbd->end_of_pmd_training = jiffies + 4 * HZ;



	/* Ensure end_of_training is visible before the state change */

	smp_wmb();



	fbd->pmd_state = FBNIC_PMD_TRAINING;



	return false;

}



static bool fbnic_mac_get_link(struct fbnic_dev *fbd, u8 aui, u8 fec)

{

	bool link;
@@ -538,7 +576,8 @@ static bool fbnic_mac_get_link(struct fbnic_dev *fbd) 
	wr32(fbd, FBNIC_SIG_PCS_INTR_STS,

	     FBNIC_SIG_PCS_INTR_LINK_DOWN | FBNIC_SIG_PCS_INTR_LINK_UP);



	link = fbnic_mac_get_pcs_link_status(fbd);

	link = fbnic_mac_get_link_status(fbd, aui, fec);

	link = fbnic_pmd_update_state(fbd, link);



	/* Enable interrupt to only capture changes in link state */

	wr32(fbd, FBNIC_SIG_PCS_INTR_MASK,
@@ -586,20 +625,15 @@ void fbnic_mac_get_fw_settings(struct fbnic_dev *fbd, u8 *aui, u8 *fec) 
	}

}



static int fbnic_pcs_enable_asic(struct fbnic_dev *fbd)

static void fbnic_mac_prepare(struct fbnic_dev *fbd, u8 aui, u8 fec)

{

	/* Mask and clear the PCS interrupt, will be enabled by link handler */

	wr32(fbd, FBNIC_SIG_PCS_INTR_MASK, ~0);

	wr32(fbd, FBNIC_SIG_PCS_INTR_STS, ~0);



	return 0;

}



static void fbnic_pcs_disable_asic(struct fbnic_dev *fbd)

{

	/* Mask and clear the PCS interrupt */

	wr32(fbd, FBNIC_SIG_PCS_INTR_MASK, ~0);

	wr32(fbd, FBNIC_SIG_PCS_INTR_STS, ~0);

	/* If we don't have link tear it all down and start over */

	if (!fbnic_mac_get_link_status(fbd, aui, fec))

		fbd->pmd_state = FBNIC_PMD_INITIALIZE;

}



static void fbnic_mac_link_down_asic(struct fbnic_dev *fbd)
@@ -867,10 +901,9 @@ static int fbnic_mac_get_sensor_asic(struct fbnic_dev *fbd, int id, 


static const struct fbnic_mac fbnic_mac_asic = {

	.init_regs = fbnic_mac_init_regs,

	.pcs_enable = fbnic_pcs_enable_asic,

	.pcs_disable = fbnic_pcs_disable_asic,

	.get_link = fbnic_mac_get_link,

	.get_link_event = fbnic_mac_get_link_event,

	.prepare = fbnic_mac_prepare,

	.get_fec_stats = fbnic_mac_get_fec_stats,

	.get_pcs_stats = fbnic_mac_get_pcs_stats,

	.get_eth_mac_stats = fbnic_mac_get_eth_mac_stats,

drivers/net/ethernet/meta/fbnic/fbnic_mac.h

+27 −9
Original line number Diff line number Diff line
@@ -10,6 +10,24 @@ struct fbnic_dev; 


#define FBNIC_MAX_JUMBO_FRAME_SIZE	9742



/* States loosely based on section 136.8.11.7.5 of IEEE 802.3-2022 Ethernet

 * Standard.  These are needed to track the state of the PHY as it has a delay

 * of several seconds from the time link comes up until it has completed

 * training that we need to wait to report the link.

 *

 * Currently we treat training as a single block as this is managed by the

 * firmware.

 *

 * We have FBNIC_PMD_SEND_DATA set to 0 as the expected default at driver load

 * and we initialize the structure containing it to zero at allocation.

 */

enum {

	FBNIC_PMD_SEND_DATA	= 0x0,

	FBNIC_PMD_INITIALIZE	= 0x1,

	FBNIC_PMD_TRAINING	= 0x2,

	FBNIC_PMD_LINK_READY	= 0x3,

};



enum {

	FBNIC_LINK_EVENT_NONE	= 0,

	FBNIC_LINK_EVENT_UP	= 1,
@@ -55,15 +73,15 @@ enum fbnic_sensor_id { 
 * void (*init_regs)(struct fbnic_dev *fbd);

 *	Initialize MAC registers to enable Tx/Rx paths and FIFOs.

 *

 * void (*pcs_enable)(struct fbnic_dev *fbd);

 *	Configure and enable PCS to enable link if not already enabled

 * void (*pcs_disable)(struct fbnic_dev *fbd);

 *	Shutdown the link if we are the only consumer of it.

 * bool (*get_link)(struct fbnic_dev *fbd);

 *	Check PCS link status

 * int (*get_link_event)(struct fbnic_dev *fbd)

 *	Get the current link event status, reports true if link has

 *	changed to either FBNIC_LINK_EVENT_DOWN or FBNIC_LINK_EVENT_UP

 * bool (*get_link)(struct fbnic_dev *fbd, u8 aui, u8 fec);

 *	Check link status

 *

 * void (*prepare)(struct fbnic_dev *fbd, u8 aui, u8 fec);

 *	Prepare PHY for init by fetching settings, disabling interrupts,

 *	and sending an updated PHY config to FW if needed.

 *

 * void (*link_down)(struct fbnic_dev *fbd);

 *	Configure MAC for link down event
@@ -74,10 +92,10 @@ enum fbnic_sensor_id { 
struct fbnic_mac {

	void (*init_regs)(struct fbnic_dev *fbd);



	int (*pcs_enable)(struct fbnic_dev *fbd);

	void (*pcs_disable)(struct fbnic_dev *fbd);

	bool (*get_link)(struct fbnic_dev *fbd);

	int (*get_link_event)(struct fbnic_dev *fbd);

	bool (*get_link)(struct fbnic_dev *fbd, u8 aui, u8 fec);



	void (*prepare)(struct fbnic_dev *fbd, u8 aui, u8 fec);



	void (*get_fec_stats)(struct fbnic_dev *fbd, bool reset,

			      struct fbnic_fec_stats *fec_stats);