Commit 677bd4bf authored by Jakub Kicinski's avatar Jakub Kicinski
Browse files

Merge branch 'net-rds-rds-tcp-protocol-and-extension-improvements' 

Allison Henderson says:

====================
net/rds: RDS-TCP protocol and extension improvements

This is subset 3 of the larger RDS-TCP patch series I posted last
Oct.  The greater series aims to correct multiple rds-tcp issues that
can cause dropped or out of sequence messages.  I've broken it down into
smaller sets to make reviews more manageable.

In this set, we introduce extension headers for byte accounting
and fix several RDS/TCP protocol issues including message preservation
during connection transitions and multipath lane handling.

The entire set can be viewed in the rfc here:
https://lore.kernel.org/netdev/20251022191715.157755-1-achender@kernel.org/
====================

Link: https://patch.msgid.link/20260203055723.1085751-1-achender@kernel.org


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parents acd21dd2 9d27a0fb

net/rds/connection.c

+6 −1
Original line number Diff line number Diff line
@@ -442,16 +442,21 @@ void rds_conn_shutdown(struct rds_conn_path *cp) 
	 * to the conn hash, so we never trigger a reconnect on this

	 * conn - the reconnect is always triggered by the active peer. */

	cancel_delayed_work_sync(&cp->cp_conn_w);



	clear_bit(RDS_RECONNECT_PENDING, &cp->cp_flags);

	rcu_read_lock();

	if (!hlist_unhashed(&conn->c_hash_node)) {

		rcu_read_unlock();

		if (conn->c_trans->t_mp_capable &&

		    cp->cp_index == 0)

			rds_send_ping(conn, 0);

		rds_queue_reconnect(cp);

	} else {

		rcu_read_unlock();

	}



	if (conn->c_trans->conn_slots_available)

		conn->c_trans->conn_slots_available(conn);

		conn->c_trans->conn_slots_available(conn, false);

}



/* destroy a single rds_conn_path. rds_conn_destroy() iterates over

net/rds/ib_send.c

+33 −7
Original line number Diff line number Diff line
@@ -577,16 +577,42 @@ int rds_ib_xmit(struct rds_connection *conn, struct rds_message *rm, 
		/* If it has a RDMA op, tell the peer we did it. This is

		 * used by the peer to release use-once RDMA MRs. */

		if (rm->rdma.op_active) {

			struct rds_ext_header_rdma ext_hdr;

			struct rds_ext_header_rdma ext_hdr = {};

			struct rds_ext_header_rdma_bytes

				rdma_bytes_ext_hdr = {};



			ext_hdr.h_rdma_rkey = cpu_to_be32(rm->rdma.op_rkey);

			rds_message_add_extension(&rm->m_inc.i_hdr,

					RDS_EXTHDR_RDMA, &ext_hdr, sizeof(ext_hdr));

			if (rds_message_add_extension(&rm->m_inc.i_hdr,

						      RDS_EXTHDR_RDMA,

						      &ext_hdr)) {

				/* prepare the rdma bytes ext header */

				rdma_bytes_ext_hdr.h_rflags =

					rm->rdma.op_write ?

					RDS_FLAG_RDMA_WR_BYTES :

					RDS_FLAG_RDMA_RD_BYTES;

				rdma_bytes_ext_hdr.h_rdma_bytes =

					cpu_to_be32(rm->rdma.op_bytes);

			} else {

				rdsdebug("RDS_EXTHDR_RDMA dropped");

			}



			if (rds_message_add_extension(&rm->m_inc.i_hdr,

						      RDS_EXTHDR_RDMA_BYTES,

						      &rdma_bytes_ext_hdr)) {

				/* rdma bytes ext header was added successfully,

				 * notify the remote side via flag in header

				 */

				rm->m_inc.i_hdr.h_flags |=

					RDS_FLAG_EXTHDR_EXTENSION;

			} else {

				rdsdebug("RDS_EXTHDR_RDMA_BYTES dropped");

			}

		}

		if (rm->m_rdma_cookie) {

			rds_message_add_rdma_dest_extension(&rm->m_inc.i_hdr,

		if (rm->m_rdma_cookie &&

		    !rds_message_add_rdma_dest_extension(&rm->m_inc.i_hdr,

				rds_rdma_cookie_key(rm->m_rdma_cookie),

					rds_rdma_cookie_offset(rm->m_rdma_cookie));

				rds_rdma_cookie_offset(rm->m_rdma_cookie))) {

			rdsdebug("RDS_EXTHDR_RDMA_DEST dropped\n");

		}



		/* Note - rds_ib_piggyb_ack clears the ACK_REQUIRED bit, so

net/rds/message.c

+53 −13
Original line number Diff line number Diff line
@@ -44,8 +44,10 @@ static unsigned int rds_exthdr_size[__RDS_EXTHDR_MAX] = { 
[RDS_EXTHDR_VERSION]	= sizeof(struct rds_ext_header_version),

[RDS_EXTHDR_RDMA]	= sizeof(struct rds_ext_header_rdma),

[RDS_EXTHDR_RDMA_DEST]	= sizeof(struct rds_ext_header_rdma_dest),

[RDS_EXTHDR_RDMA_BYTES] = sizeof(struct rds_ext_header_rdma_bytes),

[RDS_EXTHDR_NPATHS]	= sizeof(__be16),

[RDS_EXTHDR_GEN_NUM]	= sizeof(__be32),

[RDS_EXTHDR_SPORT_IDX]	= 1,

};



void rds_message_addref(struct rds_message *rm)
@@ -191,31 +193,69 @@ void rds_message_populate_header(struct rds_header *hdr, __be16 sport, 
	hdr->h_sport = sport;

	hdr->h_dport = dport;

	hdr->h_sequence = cpu_to_be64(seq);

	hdr->h_exthdr[0] = RDS_EXTHDR_NONE;

	/* see rds_find_next_ext_space for reason why we memset the

	 * ext header

	 */

	memset(hdr->h_exthdr, RDS_EXTHDR_NONE, RDS_HEADER_EXT_SPACE);

}

EXPORT_SYMBOL_GPL(rds_message_populate_header);



int rds_message_add_extension(struct rds_header *hdr, unsigned int type,

			      const void *data, unsigned int len)

/*

 * Find the next place we can add an RDS header extension with

 * specific length. Extension headers are pushed one after the

 * other. In the following, the number after the colon is the number

 * of bytes:

 *

 * [ type1:1 dta1:len1 [ type2:1 dta2:len2 ] ... ] RDS_EXTHDR_NONE

 *

 * If the extension headers fill the complete extension header space

 * (16 bytes), the trailing RDS_EXTHDR_NONE is omitted.

 */

static int rds_find_next_ext_space(struct rds_header *hdr, unsigned int len,

				   u8 **ext_start)

{

	unsigned int ext_len = sizeof(u8) + len;

	unsigned char *dst;

	unsigned int ext_len;

	unsigned int type;

	int ind = 0;



	/* For now, refuse to add more than one extension header */

	if (hdr->h_exthdr[0] != RDS_EXTHDR_NONE)

	while ((ind + 1 + len) <= RDS_HEADER_EXT_SPACE) {

		if (hdr->h_exthdr[ind] == RDS_EXTHDR_NONE) {

			*ext_start = hdr->h_exthdr + ind;

			return 0;

		}



		type = hdr->h_exthdr[ind];



		ext_len = (type < __RDS_EXTHDR_MAX) ? rds_exthdr_size[type] : 0;

		WARN_ONCE(!ext_len, "Unknown ext hdr type %d\n", type);

		if (!ext_len)

			return -EINVAL;



		/* ind points to a valid ext hdr with known length */

		ind += 1 + ext_len;

	}



	/* no room for extension */

	return -ENOSPC;

}



/* The ext hdr space is prefilled with zero from the kzalloc() */

int rds_message_add_extension(struct rds_header *hdr,

			      unsigned int type, const void *data)

{

	unsigned char *dst;

	unsigned int len;



	if (type >= __RDS_EXTHDR_MAX || len != rds_exthdr_size[type])

	len = (type < __RDS_EXTHDR_MAX) ? rds_exthdr_size[type] : 0;

	if (!len)

		return 0;



	if (ext_len >= RDS_HEADER_EXT_SPACE)

	if (rds_find_next_ext_space(hdr, len, &dst))

		return 0;

	dst = hdr->h_exthdr;



	*dst++ = type;

	memcpy(dst, data, len);



	dst[len] = RDS_EXTHDR_NONE;

	return 1;

}

EXPORT_SYMBOL_GPL(rds_message_add_extension);
@@ -272,7 +312,7 @@ int rds_message_add_rdma_dest_extension(struct rds_header *hdr, u32 r_key, u32 o 


	ext_hdr.h_rdma_rkey = cpu_to_be32(r_key);

	ext_hdr.h_rdma_offset = cpu_to_be32(offset);

	return rds_message_add_extension(hdr, RDS_EXTHDR_RDMA_DEST, &ext_hdr, sizeof(ext_hdr));

	return rds_message_add_extension(hdr, RDS_EXTHDR_RDMA_DEST, &ext_hdr);

}

EXPORT_SYMBOL_GPL(rds_message_add_rdma_dest_extension);

net/rds/rds.h

+63 −42
Original line number Diff line number Diff line
@@ -147,6 +147,7 @@ struct rds_connection { 
				c_ping_triggered:1,

				c_pad_to_32:29;

	int			c_npaths;

	bool			c_with_sport_idx;

	struct rds_connection	*c_passive;

	struct rds_transport	*c_trans;
@@ -169,6 +170,8 @@ struct rds_connection { 


	u32			c_my_gen_num;

	u32			c_peer_gen_num;



	u64			c_cp0_mprds_catchup_tx_seq;

};



static inline
@@ -186,6 +189,7 @@ void rds_conn_net_set(struct rds_connection *conn, struct net *net) 
#define RDS_FLAG_CONG_BITMAP		0x01

#define RDS_FLAG_ACK_REQUIRED		0x02

#define RDS_FLAG_RETRANSMITTED		0x04

#define RDS_FLAG_EXTHDR_EXTENSION	0x20

#define RDS_MAX_ADV_CREDIT		255



/* RDS_FLAG_PROBE_PORT is the reserved sport used for sending a ping
@@ -258,13 +262,29 @@ struct rds_ext_header_rdma_dest { 
	__be32			h_rdma_offset;

};



/*

 * This extension header tells the peer about delivered RDMA byte count.

 */

#define RDS_EXTHDR_RDMA_BYTES	4



struct rds_ext_header_rdma_bytes {

	__be32		h_rdma_bytes;	/* byte count */

	u8		h_rflags;	/* direction of RDMA, write or read */

	u8		h_pad[3];

};



#define RDS_FLAG_RDMA_WR_BYTES	0x01

#define RDS_FLAG_RDMA_RD_BYTES	0x02



/* Extension header announcing number of paths.

 * Implicit length = 2 bytes.

 */

#define RDS_EXTHDR_NPATHS	5

#define RDS_EXTHDR_GEN_NUM	6

#define RDS_EXTHDR_SPORT_IDX    8



#define __RDS_EXTHDR_MAX	16 /* for now */



#define RDS_RX_MAX_TRACES	(RDS_MSG_RX_DGRAM_TRACE_MAX + 1)

#define	RDS_MSG_RX_HDR		0

#define	RDS_MSG_RX_START	1
@@ -529,7 +549,7 @@ struct rds_transport { 
	 * messages received on the new socket are not discarded when no

	 * connection path was available at the time.

	 */

	void (*conn_slots_available)(struct rds_connection *conn);

	void (*conn_slots_available)(struct rds_connection *conn, bool fan_out);

	int (*conn_path_connect)(struct rds_conn_path *cp);



	/*
@@ -695,42 +715,43 @@ static inline int rds_sk_rcvbuf(struct rds_sock *rs) 
}



struct rds_statistics {

	uint64_t	s_conn_reset;

	uint64_t	s_recv_drop_bad_checksum;

	uint64_t	s_recv_drop_old_seq;

	uint64_t	s_recv_drop_no_sock;

	uint64_t	s_recv_drop_dead_sock;

	uint64_t	s_recv_deliver_raced;

	uint64_t	s_recv_delivered;

	uint64_t	s_recv_queued;

	uint64_t	s_recv_immediate_retry;

	uint64_t	s_recv_delayed_retry;

	uint64_t	s_recv_ack_required;

	uint64_t	s_recv_rdma_bytes;

	uint64_t	s_recv_ping;

	uint64_t	s_send_queue_empty;

	uint64_t	s_send_queue_full;

	uint64_t	s_send_lock_contention;

	uint64_t	s_send_lock_queue_raced;

	uint64_t	s_send_immediate_retry;

	uint64_t	s_send_delayed_retry;

	uint64_t	s_send_drop_acked;

	uint64_t	s_send_ack_required;

	uint64_t	s_send_queued;

	uint64_t	s_send_rdma;

	uint64_t	s_send_rdma_bytes;

	uint64_t	s_send_pong;

	uint64_t	s_page_remainder_hit;

	uint64_t	s_page_remainder_miss;

	uint64_t	s_copy_to_user;

	uint64_t	s_copy_from_user;

	uint64_t	s_cong_update_queued;

	uint64_t	s_cong_update_received;

	uint64_t	s_cong_send_error;

	uint64_t	s_cong_send_blocked;

	uint64_t	s_recv_bytes_added_to_socket;

	uint64_t	s_recv_bytes_removed_from_socket;

	uint64_t	s_send_stuck_rm;

	u64	s_conn_reset;

	u64	s_recv_drop_bad_checksum;

	u64	s_recv_drop_old_seq;

	u64	s_recv_drop_no_sock;

	u64	s_recv_drop_dead_sock;

	u64	s_recv_deliver_raced;

	u64	s_recv_delivered;

	u64	s_recv_queued;

	u64	s_recv_immediate_retry;

	u64	s_recv_delayed_retry;

	u64	s_recv_ack_required;

	u64	s_recv_rdma_bytes;

	u64	s_recv_ping;

	u64	s_send_queue_empty;

	u64	s_send_queue_full;

	u64	s_send_lock_contention;

	u64	s_send_lock_queue_raced;

	u64	s_send_immediate_retry;

	u64	s_send_delayed_retry;

	u64	s_send_drop_acked;

	u64	s_send_ack_required;

	u64	s_send_queued;

	u64	s_send_rdma;

	u64	s_send_rdma_bytes;

	u64	s_send_pong;

	u64	s_page_remainder_hit;

	u64	s_page_remainder_miss;

	u64	s_copy_to_user;

	u64	s_copy_from_user;

	u64	s_cong_update_queued;

	u64	s_cong_update_received;

	u64	s_cong_send_error;

	u64	s_cong_send_blocked;

	u64	s_recv_bytes_added_to_socket;

	u64	s_recv_bytes_removed_from_socket;

	u64	s_send_stuck_rm;

	u64	s_mprds_catchup_tx0_retries;

};



/* af_rds.c */
@@ -871,7 +892,7 @@ struct rds_message *rds_message_map_pages(unsigned long *page_addrs, unsigned in 
void rds_message_populate_header(struct rds_header *hdr, __be16 sport,

				 __be16 dport, u64 seq);

int rds_message_add_extension(struct rds_header *hdr,

			      unsigned int type, const void *data, unsigned int len);

			      unsigned int type, const void *data);

int rds_message_next_extension(struct rds_header *hdr,

			       unsigned int *pos, void *buf, unsigned int *buflen);

int rds_message_add_rdma_dest_extension(struct rds_header *hdr, u32 r_key, u32 offset);

net/rds/recv.c

+33 −4
Original line number Diff line number Diff line
@@ -204,8 +204,14 @@ static void rds_recv_hs_exthdrs(struct rds_header *hdr, 
		struct rds_ext_header_version version;

		__be16 rds_npaths;

		__be32 rds_gen_num;

		u8 dummy;

	} buffer;

	bool new_with_sport_idx = false;

	u32 new_peer_gen_num = 0;

	int new_npaths;

	bool fan_out;



	new_npaths = conn->c_npaths;



	while (1) {

		len = sizeof(buffer);
@@ -215,25 +221,48 @@ static void rds_recv_hs_exthdrs(struct rds_header *hdr, 
		/* Process extension header here */

		switch (type) {

		case RDS_EXTHDR_NPATHS:

			conn->c_npaths = min_t(int, RDS_MPATH_WORKERS,

			new_npaths = min_t(int, RDS_MPATH_WORKERS,

					   be16_to_cpu(buffer.rds_npaths));

			break;

		case RDS_EXTHDR_GEN_NUM:

			new_peer_gen_num = be32_to_cpu(buffer.rds_gen_num);

			break;

		case RDS_EXTHDR_SPORT_IDX:

			new_with_sport_idx = true;

			break;

		default:

			pr_warn_ratelimited("ignoring unknown exthdr type "

					     "0x%x\n", type);

		}

	}



	conn->c_with_sport_idx = new_with_sport_idx;



	if (new_npaths > 1 && new_npaths != conn->c_npaths) {

		/* We're about to fan-out.

		 * Make sure that messages from cp_index#0

		 * are sent prior to handling other lanes.

		 */

		struct rds_conn_path *cp0 = conn->c_path;

		unsigned long flags;



		spin_lock_irqsave(&cp0->cp_lock, flags);

		conn->c_cp0_mprds_catchup_tx_seq = cp0->cp_next_tx_seq;

		spin_unlock_irqrestore(&cp0->cp_lock, flags);

		fan_out = true;

	} else {

		fan_out = false;

	}



	/* if RDS_EXTHDR_NPATHS was not found, default to a single-path */

	conn->c_npaths = max_t(int, conn->c_npaths, 1);

	conn->c_npaths = max_t(int, new_npaths, 1);



	conn->c_ping_triggered = 0;

	rds_conn_peer_gen_update(conn, new_peer_gen_num);



	if (conn->c_npaths > 1 &&

	    conn->c_trans->conn_slots_available)

		conn->c_trans->conn_slots_available(conn);

		conn->c_trans->conn_slots_available(conn, fan_out);

}



/* rds_start_mprds() will synchronously start multiple paths when appropriate.