Commit ba499a07 authored by Antonio Quartulli's avatar Antonio Quartulli
Browse files

ovpn: ensure sk is still valid during cleanup 



Removing a peer while userspace attempts to close its transport
socket triggers a race condition resulting in the following
crash:

Oops: general protection fault, probably for non-canonical address 0xdffffc0000000077: 0000 [#1] SMP KASAN
KASAN: null-ptr-deref in range [0x00000000000003b8-0x00000000000003bf]
CPU: 12 UID: 0 PID: 162 Comm: kworker/12:1 Tainted: G           O        6.15.0-rc2-00635-g521139ac3840 #272 PREEMPT(full)
Tainted: [O]=OOT_MODULE
Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.16.3-20240910_120124-localhost 04/01/2014
Workqueue: events ovpn_peer_keepalive_work [ovpn]
RIP: 0010:ovpn_socket_release+0x23c/0x500 [ovpn]
Code: ea 03 80 3c 02 00 0f 85 71 02 00 00 48 b8 00 00 00 00 00 fc ff df 4d 8b 64 24 18 49 8d bc 24 be 03 00 00 48 89 fa 48 c1 ea 03 <0f> b6 14 02 48 89 f8 83 e0 07 83 c0 01 38 d0 7c 08 84 d2 0f 85 30
RSP: 0018:ffffc90000c9fb18 EFLAGS: 00010217
RAX: dffffc0000000000 RBX: ffff8881148d7940 RCX: ffffffff817787bb
RDX: 0000000000000077 RSI: 0000000000000008 RDI: 00000000000003be
RBP: ffffc90000c9fb30 R08: 0000000000000000 R09: fffffbfff0d3e840
R10: ffffffff869f4207 R11: 0000000000000000 R12: 0000000000000000
R13: ffff888115eb9300 R14: ffffc90000c9fbc8 R15: 000000000000000c
FS:  0000000000000000(0000) GS:ffff8882b0151000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f37266b6114 CR3: 00000000054a8000 CR4: 0000000000750ef0
PKRU: 55555554
Call Trace:
 <TASK>
 unlock_ovpn+0x8b/0xe0 [ovpn]
 ovpn_peer_keepalive_work+0xe3/0x540 [ovpn]
 ? ovpn_peers_free+0x780/0x780 [ovpn]
 ? lock_acquire+0x56/0x70
 ? process_one_work+0x888/0x1740
 process_one_work+0x933/0x1740
 ? pwq_dec_nr_in_flight+0x10b0/0x10b0
 ? move_linked_works+0x12d/0x2c0
 ? assign_work+0x163/0x270
 worker_thread+0x4d6/0xd90
 ? preempt_count_sub+0x4c/0x70
 ? process_one_work+0x1740/0x1740
 kthread+0x36c/0x710
 ? trace_preempt_on+0x8c/0x1e0
 ? kthread_is_per_cpu+0xc0/0xc0
 ? preempt_count_sub+0x4c/0x70
 ? _raw_spin_unlock_irq+0x36/0x60
 ? calculate_sigpending+0x7b/0xa0
 ? kthread_is_per_cpu+0xc0/0xc0
 ret_from_fork+0x3a/0x80
 ? kthread_is_per_cpu+0xc0/0xc0
 ret_from_fork_asm+0x11/0x20
 </TASK>
Modules linked in: ovpn(O)

This happens because the peer deletion operation reaches
ovpn_socket_release() while ovpn_sock->sock (struct socket *)
and its sk member (struct sock *) are still both valid.
Here synchronize_rcu() is invoked, after which ovpn_sock->sock->sk
becomes NULL, due to the concurrent socket closing triggered
from userspace.

After having invoked synchronize_rcu(), ovpn_socket_release() will
attempt dereferencing ovpn_sock->sock->sk, triggering the crash
reported above.

The reason for accessing sk is that we need to retrieve its
protocol and continue the cleanup routine accordingly.

This crash can be easily produced by running openvpn userspace in
client mode with `--keepalive 10 20`, while entirely omitting this
option on the server side.
After 20 seconds ovpn will assume the peer (server) to be dead,
will start removing it and will notify userspace. The latter will
receive the notification and close the transport socket, thus
triggering the crash.

To fix the race condition for good, we need to refactor struct ovpn_socket.
Since ovpn is always only interested in the sock->sk member (struct sock *)
we can directly hold a reference to it, raher than accessing it via
its struct socket container.

This means changing "struct socket *ovpn_socket->sock" to
"struct sock *ovpn_socket->sk".

While acquiring a reference to sk, we can increase its refcounter
without affecting the socket close()/destroy() notification
(which we rely on when userspace closes a socket we are using).

By increasing sk's refcounter we know we can dereference it
in ovpn_socket_release() without incurring in any race condition
anymore.

ovpn_socket_release() will ultimately decrease the reference
counter.

Cc: Oleksandr Natalenko <oleksandr@natalenko.name>
Fixes: 11851cbd ("ovpn: implement TCP transport")
Reported-by: default avatarQingfang Deng <dqfext@gmail.com>
Closes: https://github.com/OpenVPN/ovpn-net-next/issues/1


Tested-by: default avatarGert Doering <gert@greenie.muc.de>
Link: https://www.mail-archive.com/openvpn-devel@lists.sourceforge.net/msg31575.html


Reviewed-by: default avatarMichal Swiatkowski <michal.swiatkowski@linux.intel.com>
Signed-off-by: default avatarAntonio Quartulli <antonio@openvpn.net>
parent 930faf1e

drivers/net/ovpn/io.c

+4 −4
Original line number Diff line number Diff line
@@ -134,7 +134,7 @@ void ovpn_decrypt_post(void *data, int ret) 


	rcu_read_lock();

	sock = rcu_dereference(peer->sock);

	if (sock && sock->sock->sk->sk_protocol == IPPROTO_UDP)

	if (sock && sock->sk->sk_protocol == IPPROTO_UDP)

		/* check if this peer changed local or remote endpoint */

		ovpn_peer_endpoints_update(peer, skb);

	rcu_read_unlock();
@@ -270,12 +270,12 @@ void ovpn_encrypt_post(void *data, int ret) 
	if (unlikely(!sock))

		goto err_unlock;



	switch (sock->sock->sk->sk_protocol) {

	switch (sock->sk->sk_protocol) {

	case IPPROTO_UDP:

		ovpn_udp_send_skb(peer, sock->sock, skb);

		ovpn_udp_send_skb(peer, sock->sk, skb);

		break;

	case IPPROTO_TCP:

		ovpn_tcp_send_skb(peer, sock->sock, skb);

		ovpn_tcp_send_skb(peer, sock->sk, skb);

		break;

	default:

		/* no transport configured yet */

drivers/net/ovpn/netlink.c

+8 −8
Original line number Diff line number Diff line
@@ -501,7 +501,7 @@ int ovpn_nl_peer_set_doit(struct sk_buff *skb, struct genl_info *info) 
	/* when using a TCP socket the remote IP is not expected */

	rcu_read_lock();

	sock = rcu_dereference(peer->sock);

	if (sock && sock->sock->sk->sk_protocol == IPPROTO_TCP &&

	if (sock && sock->sk->sk_protocol == IPPROTO_TCP &&

	    (attrs[OVPN_A_PEER_REMOTE_IPV4] ||

	     attrs[OVPN_A_PEER_REMOTE_IPV6])) {

		rcu_read_unlock();
@@ -559,14 +559,14 @@ static int ovpn_nl_send_peer(struct sk_buff *skb, const struct genl_info *info, 
		goto err_unlock;

	}



	if (!net_eq(genl_info_net(info), sock_net(sock->sock->sk))) {

	if (!net_eq(genl_info_net(info), sock_net(sock->sk))) {

		id = peernet2id_alloc(genl_info_net(info),

				      sock_net(sock->sock->sk),

				      sock_net(sock->sk),

				      GFP_ATOMIC);

		if (nla_put_s32(skb, OVPN_A_PEER_SOCKET_NETNSID, id))

			goto err_unlock;

	}

	local_port = inet_sk(sock->sock->sk)->inet_sport;

	local_port = inet_sk(sock->sk)->inet_sport;

	rcu_read_unlock();



	if (nla_put_u32(skb, OVPN_A_PEER_ID, peer->id))
@@ -1153,8 +1153,8 @@ int ovpn_nl_peer_del_notify(struct ovpn_peer *peer) 
		ret = -EINVAL;

		goto err_unlock;

	}

	genlmsg_multicast_netns(&ovpn_nl_family, sock_net(sock->sock->sk),

				msg, 0, OVPN_NLGRP_PEERS, GFP_ATOMIC);

	genlmsg_multicast_netns(&ovpn_nl_family, sock_net(sock->sk), msg, 0,

				OVPN_NLGRP_PEERS, GFP_ATOMIC);

	rcu_read_unlock();



	return 0;
@@ -1218,8 +1218,8 @@ int ovpn_nl_key_swap_notify(struct ovpn_peer *peer, u8 key_id) 
		ret = -EINVAL;

		goto err_unlock;

	}

	genlmsg_multicast_netns(&ovpn_nl_family, sock_net(sock->sock->sk),

				msg, 0, OVPN_NLGRP_PEERS, GFP_ATOMIC);

	genlmsg_multicast_netns(&ovpn_nl_family, sock_net(sock->sk), msg, 0,

				OVPN_NLGRP_PEERS, GFP_ATOMIC);

	rcu_read_unlock();



	return 0;

drivers/net/ovpn/peer.c

+2 −2
Original line number Diff line number Diff line
@@ -1145,7 +1145,7 @@ static void ovpn_peer_release_p2p(struct ovpn_priv *ovpn, struct sock *sk, 


	if (sk) {

		ovpn_sock = rcu_access_pointer(peer->sock);

		if (!ovpn_sock || ovpn_sock->sock->sk != sk) {

		if (!ovpn_sock || ovpn_sock->sk != sk) {

			spin_unlock_bh(&ovpn->lock);

			ovpn_peer_put(peer);

			return;
@@ -1175,7 +1175,7 @@ static void ovpn_peers_release_mp(struct ovpn_priv *ovpn, struct sock *sk, 
		if (sk) {

			rcu_read_lock();

			ovpn_sock = rcu_dereference(peer->sock);

			remove = ovpn_sock && ovpn_sock->sock->sk == sk;

			remove = ovpn_sock && ovpn_sock->sk == sk;

			rcu_read_unlock();

		}

drivers/net/ovpn/socket.c

+38 −30
Original line number Diff line number Diff line
@@ -24,9 +24,9 @@ static void ovpn_socket_release_kref(struct kref *kref) 
	struct ovpn_socket *sock = container_of(kref, struct ovpn_socket,

						refcount);



	if (sock->sock->sk->sk_protocol == IPPROTO_UDP)

	if (sock->sk->sk_protocol == IPPROTO_UDP)

		ovpn_udp_socket_detach(sock);

	else if (sock->sock->sk->sk_protocol == IPPROTO_TCP)

	else if (sock->sk->sk_protocol == IPPROTO_TCP)

		ovpn_tcp_socket_detach(sock);

}
@@ -75,14 +75,6 @@ void ovpn_socket_release(struct ovpn_peer *peer) 
	if (!sock)

		return;



	/* sanity check: we should not end up here if the socket

	 * was already closed

	 */

	if (!sock->sock->sk) {

		DEBUG_NET_WARN_ON_ONCE(1);

		return;

	}



	/* Drop the reference while holding the sock lock to avoid

	 * concurrent ovpn_socket_new call to mess up with a partially

	 * detached socket.
@@ -90,22 +82,24 @@ void ovpn_socket_release(struct ovpn_peer *peer) 
	 * Holding the lock ensures that a socket with refcnt 0 is fully

	 * detached before it can be picked by a concurrent reader.

	 */

	lock_sock(sock->sock->sk);

	lock_sock(sock->sk);

	released = ovpn_socket_put(peer, sock);

	release_sock(sock->sock->sk);

	release_sock(sock->sk);



	/* align all readers with sk_user_data being NULL */

	synchronize_rcu();



	/* following cleanup should happen with lock released */

	if (released) {

		if (sock->sock->sk->sk_protocol == IPPROTO_UDP) {

		if (sock->sk->sk_protocol == IPPROTO_UDP) {

			netdev_put(sock->ovpn->dev, &sock->dev_tracker);

		} else if (sock->sock->sk->sk_protocol == IPPROTO_TCP) {

		} else if (sock->sk->sk_protocol == IPPROTO_TCP) {

			/* wait for TCP jobs to terminate */

			ovpn_tcp_socket_wait_finish(sock);

			ovpn_peer_put(sock->peer);

		}

		/* drop reference acquired in ovpn_socket_new() */

		sock_put(sock->sk);

		/* we can call plain kfree() because we already waited one RCU

		 * period due to synchronize_rcu()

		 */
@@ -118,12 +112,14 @@ static bool ovpn_socket_hold(struct ovpn_socket *sock) 
	return kref_get_unless_zero(&sock->refcount);

}



static int ovpn_socket_attach(struct ovpn_socket *sock, struct ovpn_peer *peer)

static int ovpn_socket_attach(struct ovpn_socket *ovpn_sock,

			      struct socket *sock,

			      struct ovpn_peer *peer)

{

	if (sock->sock->sk->sk_protocol == IPPROTO_UDP)

		return ovpn_udp_socket_attach(sock, peer->ovpn);

	else if (sock->sock->sk->sk_protocol == IPPROTO_TCP)

		return ovpn_tcp_socket_attach(sock, peer);

	if (sock->sk->sk_protocol == IPPROTO_UDP)

		return ovpn_udp_socket_attach(ovpn_sock, sock, peer->ovpn);

	else if (sock->sk->sk_protocol == IPPROTO_TCP)

		return ovpn_tcp_socket_attach(ovpn_sock, peer);



	return -EOPNOTSUPP;

}
@@ -138,14 +134,15 @@ static int ovpn_socket_attach(struct ovpn_socket *sock, struct ovpn_peer *peer) 
struct ovpn_socket *ovpn_socket_new(struct socket *sock, struct ovpn_peer *peer)

{

	struct ovpn_socket *ovpn_sock;

	struct sock *sk = sock->sk;

	int ret;



	lock_sock(sock->sk);

	lock_sock(sk);



	/* a TCP socket can only be owned by a single peer, therefore there

	 * can't be any other user

	 */

	if (sock->sk->sk_protocol == IPPROTO_TCP && sock->sk->sk_user_data) {

	if (sk->sk_protocol == IPPROTO_TCP && sk->sk_user_data) {

		ovpn_sock = ERR_PTR(-EBUSY);

		goto sock_release;

	}
@@ -153,8 +150,8 @@ struct ovpn_socket *ovpn_socket_new(struct socket *sock, struct ovpn_peer *peer) 
	/* a UDP socket can be shared across multiple peers, but we must make

	 * sure it is not owned by something else

	 */

	if (sock->sk->sk_protocol == IPPROTO_UDP) {

		u8 type = READ_ONCE(udp_sk(sock->sk)->encap_type);

	if (sk->sk_protocol == IPPROTO_UDP) {

		u8 type = READ_ONCE(udp_sk(sk)->encap_type);



		/* socket owned by other encapsulation module */

		if (type && type != UDP_ENCAP_OVPNINUDP) {
@@ -163,7 +160,7 @@ struct ovpn_socket *ovpn_socket_new(struct socket *sock, struct ovpn_peer *peer) 
		}



		rcu_read_lock();

		ovpn_sock = rcu_dereference_sk_user_data(sock->sk);

		ovpn_sock = rcu_dereference_sk_user_data(sk);

		if (ovpn_sock) {

			/* socket owned by another ovpn instance, we can't use it */

			if (ovpn_sock->ovpn != peer->ovpn) {
@@ -200,11 +197,22 @@ struct ovpn_socket *ovpn_socket_new(struct socket *sock, struct ovpn_peer *peer) 
		goto sock_release;

	}



	ovpn_sock->sock = sock;

	ovpn_sock->sk = sk;

	kref_init(&ovpn_sock->refcount);



	ret = ovpn_socket_attach(ovpn_sock, peer);

	/* the newly created ovpn_socket is holding reference to sk,

	 * therefore we increase its refcounter.

	 *

	 * This ovpn_socket instance is referenced by all peers

	 * using the same socket.

	 *

	 * ovpn_socket_release() will take care of dropping the reference.

	 */

	sock_hold(sk);



	ret = ovpn_socket_attach(ovpn_sock, sock, peer);

	if (ret < 0) {

		sock_put(sk);

		kfree(ovpn_sock);

		ovpn_sock = ERR_PTR(ret);

		goto sock_release;
@@ -213,11 +221,11 @@ struct ovpn_socket *ovpn_socket_new(struct socket *sock, struct ovpn_peer *peer) 
	/* TCP sockets are per-peer, therefore they are linked to their unique

	 * peer

	 */

	if (sock->sk->sk_protocol == IPPROTO_TCP) {

	if (sk->sk_protocol == IPPROTO_TCP) {

		INIT_WORK(&ovpn_sock->tcp_tx_work, ovpn_tcp_tx_work);

		ovpn_sock->peer = peer;

		ovpn_peer_hold(peer);

	} else if (sock->sk->sk_protocol == IPPROTO_UDP) {

	} else if (sk->sk_protocol == IPPROTO_UDP) {

		/* in UDP we only link the ovpn instance since the socket is

		 * shared among multiple peers

		 */
@@ -226,8 +234,8 @@ struct ovpn_socket *ovpn_socket_new(struct socket *sock, struct ovpn_peer *peer) 
			    GFP_KERNEL);

	}



	rcu_assign_sk_user_data(sock->sk, ovpn_sock);

	rcu_assign_sk_user_data(sk, ovpn_sock);

sock_release:

	release_sock(sock->sk);

	release_sock(sk);

	return ovpn_sock;

}

drivers/net/ovpn/socket.h

+2 −2
Original line number Diff line number Diff line
@@ -22,7 +22,7 @@ struct ovpn_peer; 
 * @ovpn: ovpn instance owning this socket (UDP only)

 * @dev_tracker: reference tracker for associated dev (UDP only)

 * @peer: unique peer transmitting over this socket (TCP only)

 * @sock: the low level sock object

 * @sk: the low level sock object

 * @refcount: amount of contexts currently referencing this object

 * @work: member used to schedule release routine (it may block)

 * @tcp_tx_work: work for deferring outgoing packet processing (TCP only)
@@ -36,7 +36,7 @@ struct ovpn_socket { 
		struct ovpn_peer *peer;

	};



	struct socket *sock;

	struct sock *sk;

	struct kref refcount;

	struct work_struct work;

	struct work_struct tcp_tx_work;