ice: add Tx hang devlink health reporter (2a82874a) · Commits · git / linux-net

drivers/net/ethernet/intel/ice/Makefile

+1 −0

Original line number	Diff line number	Diff line
		@@ -32,6 +32,7 @@ ice-y := ice_main.o \
		ice_parser_rt.o \
		ice_idc.o \
		devlink/devlink.o \
		devlink/health.o \
		devlink/port.o \
		ice_sf_eth.o \
		ice_sf_vsi_vlan_ops.o \

drivers/net/ethernet/intel/ice/devlink/health.c

0 → 100644

+192 −0

Original line number	Diff line number	Diff line
		// SPDX-License-Identifier: GPL-2.0
		/* Copyright (c) 2024, Intel Corporation. */

		#include "health.h"
		#include "ice.h"

		#define ICE_DEVLINK_FMSG_PUT_FIELD(fmsg, obj, name) \
		devlink_fmsg_put(fmsg, #name, (obj)->name)

		/**
		* ice_devlink_health_report - boilerplate to call given @reporter
		*
		* @reporter: devlink health reporter to call, do nothing on NULL
		* @msg: message to pass up, "event name" is fine
		* @priv_ctx: typically some event struct
		*/
		static void ice_devlink_health_report(struct devlink_health_reporter *reporter,
		const char msg, void priv_ctx)
		{
		if (!reporter)
		return;

		/* We do not do auto recovering, so return value of the below function
		* will always be 0, thus we do ignore it.
		*/
		devlink_health_report(reporter, msg, priv_ctx);
		}

		/**
		* ice_fmsg_put_ptr - put hex value of pointer into fmsg
		*
		* @fmsg: devlink fmsg under construction
		* @name: name to pass
		* @ptr: 64 bit value to print as hex and put into fmsg
		*/
		static void ice_fmsg_put_ptr(struct devlink_fmsg fmsg, const char name,
		void *ptr)
		{
		char buf[sizeof(ptr) * 3];

		sprintf(buf, "%p", ptr);
		devlink_fmsg_put(fmsg, name, buf);
		}

		struct ice_tx_hang_event {
		u32 head;
		u32 intr;
		u16 vsi_num;
		u16 queue;
		u16 next_to_clean;
		u16 next_to_use;
		struct ice_tx_ring *tx_ring;
		};

		static int ice_tx_hang_reporter_dump(struct devlink_health_reporter *reporter,
		struct devlink_fmsg fmsg, void priv_ctx,
		struct netlink_ext_ack *extack)
		{
		struct ice_tx_hang_event *event = priv_ctx;
		struct sk_buff *skb;

		if (!event)
		return 0;

		skb = event->tx_ring->tx_buf->skb;
		devlink_fmsg_obj_nest_start(fmsg);
		ICE_DEVLINK_FMSG_PUT_FIELD(fmsg, event, head);
		ICE_DEVLINK_FMSG_PUT_FIELD(fmsg, event, intr);
		ICE_DEVLINK_FMSG_PUT_FIELD(fmsg, event, vsi_num);
		ICE_DEVLINK_FMSG_PUT_FIELD(fmsg, event, queue);
		ICE_DEVLINK_FMSG_PUT_FIELD(fmsg, event, next_to_clean);
		ICE_DEVLINK_FMSG_PUT_FIELD(fmsg, event, next_to_use);
		devlink_fmsg_put(fmsg, "irq-mapping", event->tx_ring->q_vector->name);
		ice_fmsg_put_ptr(fmsg, "desc-ptr", event->tx_ring->desc);
		ice_fmsg_put_ptr(fmsg, "dma-ptr", (void *)(long)event->tx_ring->dma);
		ice_fmsg_put_ptr(fmsg, "skb-ptr", skb);
		devlink_fmsg_binary_pair_put(fmsg, "desc", event->tx_ring->desc,
		event->tx_ring->count * sizeof(struct ice_tx_desc));
		devlink_fmsg_dump_skb(fmsg, skb);
		devlink_fmsg_obj_nest_end(fmsg);

		return 0;
		}

		void ice_prep_tx_hang_report(struct ice_pf pf, struct ice_tx_ring tx_ring,
		u16 vsi_num, u32 head, u32 intr)
		{
		struct ice_health_tx_hang_buf *buf = &pf->health_reporters.tx_hang_buf;

		buf->tx_ring = tx_ring;
		buf->vsi_num = vsi_num;
		buf->head = head;
		buf->intr = intr;
		}

		void ice_report_tx_hang(struct ice_pf *pf)
		{
		struct ice_health_tx_hang_buf *buf = &pf->health_reporters.tx_hang_buf;
		struct ice_tx_ring *tx_ring = buf->tx_ring;

		struct ice_tx_hang_event ev = {
		.head = buf->head,
		.intr = buf->intr,
		.vsi_num = buf->vsi_num,
		.queue = tx_ring->q_index,
		.next_to_clean = tx_ring->next_to_clean,
		.next_to_use = tx_ring->next_to_use,
		.tx_ring = tx_ring,
		};

		ice_devlink_health_report(pf->health_reporters.tx_hang, "Tx hang", &ev);
		}

		static struct devlink_health_reporter *
		ice_init_devlink_rep(struct ice_pf *pf,
		const struct devlink_health_reporter_ops *ops)
		{
		struct devlink *devlink = priv_to_devlink(pf);
		struct devlink_health_reporter *rep;
		const u64 graceful_period = 0;

		rep = devl_health_reporter_create(devlink, ops, graceful_period, pf);
		if (IS_ERR(rep)) {
		struct device *dev = ice_pf_to_dev(pf);

		dev_err(dev, "failed to create devlink %s health report er",
		ops->name);
		return NULL;
		}
		return rep;
		}

		#define ICE_DEFINE_HEALTH_REPORTER_OPS(_name) \
		static const struct devlink_health_reporter_ops ice_ ## _name ## _reporter_ops = { \
		.name = #_name, \
		.dump = ice_ ## _name ## _reporter_dump, \
		}

		ICE_DEFINE_HEALTH_REPORTER_OPS(tx_hang);

		/**
		* ice_health_init - allocate and init all ice devlink health reporters and
		* accompanied data
		*
		* @pf: PF struct
		*/
		void ice_health_init(struct ice_pf *pf)
		{
		struct ice_health *reps = &pf->health_reporters;

		reps->tx_hang = ice_init_devlink_rep(pf, &ice_tx_hang_reporter_ops);
		}

		/**
		* ice_deinit_devl_reporter - destroy given devlink health reporter
		* @reporter: reporter to destroy
		*/
		static void ice_deinit_devl_reporter(struct devlink_health_reporter *reporter)
		{
		if (reporter)
		devl_health_reporter_destroy(reporter);
		}

		/**
		* ice_health_deinit - deallocate all ice devlink health reporters and
		* accompanied data
		*
		* @pf: PF struct
		*/
		void ice_health_deinit(struct ice_pf *pf)
		{
		ice_deinit_devl_reporter(pf->health_reporters.tx_hang);
		}

		static
		void ice_health_assign_healthy_state(struct devlink_health_reporter *reporter)
		{
		if (reporter)
		devlink_health_reporter_state_update(reporter,
		DEVLINK_HEALTH_REPORTER_STATE_HEALTHY);
		}

		/**
		* ice_health_clear - clear devlink health issues after a reset
		* @pf: the PF device structure
		*
		* Mark the PF in healthy state again after a reset has completed.
		*/
		void ice_health_clear(struct ice_pf *pf)
		{
		ice_health_assign_healthy_state(pf->health_reporters.tx_hang);
		}

drivers/net/ethernet/intel/ice/devlink/health.h

0 → 100644

+47 −0

Original line number	Diff line number	Diff line
		/* SPDX-License-Identifier: GPL-2.0 */
		/* Copyright (c) 2024, Intel Corporation. */

		#ifndef _HEALTH_H_
		#define _HEALTH_H_

		#include <linux/types.h>

		/**
		* DOC: health.h
		*
		* This header file stores everything that is needed for broadly understood
		* devlink health mechanism for ice driver.
		*/

		struct ice_pf;
		struct ice_tx_ring;

		/**
		* struct ice_health - stores ice devlink health reporters and accompanied data
		* @tx_hang: devlink health reporter for tx_hang event
		* @tx_hang_buf: pre-allocated place to put info for Tx hang reporter from
		* non-sleeping context
		* @tx_ring: ring that the hang occurred on
		* @head: descriptor head
		* @intr: interrupt register value
		* @vsi_num: VSI owning the queue that the hang occurred on
		*/
		struct ice_health {
		struct devlink_health_reporter *tx_hang;
		struct_group_tagged(ice_health_tx_hang_buf, tx_hang_buf,
		struct ice_tx_ring *tx_ring;
		u32 head;
		u32 intr;
		u16 vsi_num;
		);
		};

		void ice_health_init(struct ice_pf *pf);
		void ice_health_deinit(struct ice_pf *pf);
		void ice_health_clear(struct ice_pf *pf);

		void ice_prep_tx_hang_report(struct ice_pf pf, struct ice_tx_ring tx_ring,
		u16 vsi_num, u32 head, u32 intr);
		void ice_report_tx_hang(struct ice_pf *pf);

		#endif /* _HEALTH_H_ */

drivers/net/ethernet/intel/ice/ice.h

+2 −0

Original line number	Diff line number	Diff line
		@@ -78,6 +78,7 @@
		#include "ice_irq.h"
		#include "ice_dpll.h"
		#include "ice_adapter.h"
		#include "devlink/health.h"

		#define ICE_BAR0 0
		#define ICE_REQ_DESC_MULTIPLE 32
		@@ -665,6 +666,7 @@ struct ice_pf {
		struct ice_agg_node vf_agg_node[ICE_MAX_VF_AGG_NODES];
		struct ice_dplls dplls;
		struct device *hwmon_dev;
		struct ice_health health_reporters;

		u8 num_quanta_prof_used;
		};

drivers/net/ethernet/intel/ice/ice_main.c

+13 −5

Original line number	Diff line number	Diff line
		@@ -2364,9 +2364,11 @@ static void ice_service_task(struct work_struct *work)
		struct ice_pf *pf = container_of(work, struct ice_pf, serv_task);
		unsigned long start_time = jiffies;

		/* subtasks */
		if (pf->health_reporters.tx_hang_buf.tx_ring) {
		ice_report_tx_hang(pf);
		pf->health_reporters.tx_hang_buf.tx_ring = NULL;
		}

		/* process reset requests first */
		ice_reset_subtask(pf);

		/* bail if a reset/recovery cycle is pending or rebuild failed */
		@@ -5087,6 +5089,7 @@ static int ice_init_devlink(struct ice_pf *pf)
		return err;

		ice_devlink_init_regions(pf);
		ice_health_init(pf);
		ice_devlink_register(pf);

		return 0;
		@@ -5095,6 +5098,7 @@ static int ice_init_devlink(struct ice_pf *pf)
		static void ice_deinit_devlink(struct ice_pf *pf)
		{
		ice_devlink_unregister(pf);
		ice_health_deinit(pf);
		ice_devlink_destroy_regions(pf);
		ice_devlink_unregister_params(pf);
		}
		@@ -7793,6 +7797,8 @@ static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type)
		/* if we get here, reset flow is successful */
		clear_bit(ICE_RESET_FAILED, pf->state);

		ice_health_clear(pf);

		ice_plug_aux_dev(pf);
		if (ice_is_feature_supported(pf, ICE_F_SRIOV_LAG))
		ice_lag_rebuild(pf);
		@@ -8283,16 +8289,18 @@ void ice_tx_timeout(struct net_device *netdev, unsigned int txqueue)

		if (tx_ring) {
		struct ice_hw *hw = &pf->hw;
		u32 head, val = 0;
		u32 head, intr = 0;

		head = FIELD_GET(QTX_COMM_HEAD_HEAD_M,
		rd32(hw, QTX_COMM_HEAD(vsi->txq_map[txqueue])));
		/* Read interrupt register */
		val = rd32(hw, GLINT_DYN_CTL(tx_ring->q_vector->reg_idx));
		intr = rd32(hw, GLINT_DYN_CTL(tx_ring->q_vector->reg_idx));

		netdev_info(netdev, "tx_timeout: VSI_num: %d, Q %u, NTC: 0x%x, HW_HEAD: 0x%x, NTU: 0x%x, INT: 0x%x\n",
		vsi->vsi_num, txqueue, tx_ring->next_to_clean,
		head, tx_ring->next_to_use, val);
		head, tx_ring->next_to_use, intr);

		ice_prep_tx_hang_report(pf, tx_ring, vsi->vsi_num, head, intr);
		}

		pf->tx_timeout_last_recovery = jiffies;