1 files changed, 197 insertions, 192 deletions

diff --git a/net/rxrpc/conn_object.c b/net/rxrpc/conn_object.c
index c332722820c2..37340becb224 100644
--- a/net/rxrpc/conn_object.c
+++ b/net/rxrpc/conn_object.c
@@ -1,12 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
 /* RxRPC virtual connection handler, common bits.
  *
  * Copyright (C) 2007, 2016 Red Hat, Inc. All Rights Reserved.
  * Written by David Howells (dhowells@redhat.com)
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version
- * 2 of the License, or (at your option) any later version.
  */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
@@ -23,20 +19,41 @@
 unsigned int __read_mostly rxrpc_connection_expiry = 10 * 60;
 unsigned int __read_mostly rxrpc_closed_conn_expiry = 10;
 
-static void rxrpc_destroy_connection(struct rcu_head *);
+static void rxrpc_clean_up_connection(struct work_struct *work);
+static void rxrpc_set_service_reap_timer(struct rxrpc_net *rxnet,
+					 unsigned long reap_at);
+
+void rxrpc_poke_conn(struct rxrpc_connection *conn, enum rxrpc_conn_trace why)
+{
+	struct rxrpc_local *local = conn->local;
+	bool busy;
+
+	if (WARN_ON_ONCE(!local))
+		return;
+
+	spin_lock_irq(&local->lock);
+	busy = !list_empty(&conn->attend_link);
+	if (!busy) {
+		rxrpc_get_connection(conn, why);
+		list_add_tail(&conn->attend_link, &local->conn_attend_q);
+	}
+	spin_unlock_irq(&local->lock);
+	rxrpc_wake_up_io_thread(local);
+}
 
 static void rxrpc_connection_timer(struct timer_list *timer)
 {
 	struct rxrpc_connection *conn =
 		container_of(timer, struct rxrpc_connection, timer);
 
-	rxrpc_queue_conn(conn);
+	rxrpc_poke_conn(conn, rxrpc_conn_get_poke_timer);
 }
 
 /*
  * allocate a new connection
  */
-struct rxrpc_connection *rxrpc_alloc_connection(gfp_t gfp)
+struct rxrpc_connection *rxrpc_alloc_connection(struct rxrpc_net *rxnet,
+						gfp_t gfp)
 {
 	struct rxrpc_connection *conn;
 
@@ -45,17 +62,20 @@ struct rxrpc_connection *rxrpc_alloc_connection(gfp_t gfp)
 	conn = kzalloc(sizeof(struct rxrpc_connection), gfp);
 	if (conn) {
 		INIT_LIST_HEAD(&conn->cache_link);
-		spin_lock_init(&conn->channel_lock);
-		INIT_LIST_HEAD(&conn->waiting_calls);
 		timer_setup(&conn->timer, &rxrpc_connection_timer, 0);
-		INIT_WORK(&conn->processor, &rxrpc_process_connection);
+		INIT_WORK(&conn->processor, rxrpc_process_connection);
+		INIT_WORK(&conn->destructor, rxrpc_clean_up_connection);
 		INIT_LIST_HEAD(&conn->proc_link);
 		INIT_LIST_HEAD(&conn->link);
+		INIT_LIST_HEAD(&conn->attend_link);
+		mutex_init(&conn->security_lock);
+		mutex_init(&conn->tx_data_alloc_lock);
 		skb_queue_head_init(&conn->rx_queue);
+		conn->rxnet = rxnet;
 		conn->security = &rxrpc_no_security;
+		rwlock_init(&conn->security_use_lock);
 		spin_lock_init(&conn->state_lock);
 		conn->debug_id = atomic_inc_return(&rxrpc_debug_id);
-		conn->size_align = 4;
 		conn->idle_timestamp = jiffies;
 	}
 
@@ -74,89 +94,50 @@ struct rxrpc_connection *rxrpc_alloc_connection(gfp_t gfp)
  *
  * The caller must be holding the RCU read lock.
  */
-struct rxrpc_connection *rxrpc_find_connection_rcu(struct rxrpc_local *local,
-						   struct sk_buff *skb,
-						   struct rxrpc_peer **_peer)
+struct rxrpc_connection *rxrpc_find_client_connection_rcu(struct rxrpc_local *local,
+							  struct sockaddr_rxrpc *srx,
+							  struct sk_buff *skb)
 {
 	struct rxrpc_connection *conn;
-	struct rxrpc_conn_proto k;
 	struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
-	struct sockaddr_rxrpc srx;
 	struct rxrpc_peer *peer;
 
 	_enter(",%x", sp->hdr.cid & RXRPC_CIDMASK);
 
-	if (rxrpc_extract_addr_from_skb(&srx, skb) < 0)
-		goto not_found;
-
-	if (srx.transport.family != local->srx.transport.family &&
-	    (srx.transport.family == AF_INET &&
-	     local->srx.transport.family != AF_INET6)) {
-		pr_warn_ratelimited("AF_RXRPC: Protocol mismatch %u not %u\n",
-				    srx.transport.family,
-				    local->srx.transport.family);
+	/* Look up client connections by connection ID alone as their
+	 * IDs are unique for this machine.
+	 */
+	conn = idr_find(&local->conn_ids, sp->hdr.cid >> RXRPC_CIDSHIFT);
+	if (!conn || refcount_read(&conn->ref) == 0) {
+		_debug("no conn");
 		goto not_found;
 	}
 
-	k.epoch	= sp->hdr.epoch;
-	k.cid	= sp->hdr.cid & RXRPC_CIDMASK;
-
-	if (rxrpc_to_server(sp)) {
-		/* We need to look up service connections by the full protocol
-		 * parameter set.  We look up the peer first as an intermediate
-		 * step and then the connection from the peer's tree.
-		 */
-		peer = rxrpc_lookup_peer_rcu(local, &srx);
-		if (!peer)
-			goto not_found;
-		*_peer = peer;
-		conn = rxrpc_find_service_conn_rcu(peer, skb);
-		if (!conn || atomic_read(&conn->usage) == 0)
-			goto not_found;
-		_leave(" = %p", conn);
-		return conn;
-	} else {
-		/* Look up client connections by connection ID alone as their
-		 * IDs are unique for this machine.
-		 */
-		conn = idr_find(&rxrpc_client_conn_ids,
-				sp->hdr.cid >> RXRPC_CIDSHIFT);
-		if (!conn || atomic_read(&conn->usage) == 0) {
-			_debug("no conn");
-			goto not_found;
-		}
+	if (conn->proto.epoch != sp->hdr.epoch ||
+	    conn->local != local)
+		goto not_found;
 
-		if (conn->proto.epoch != k.epoch ||
-		    conn->params.local != local)
+	peer = conn->peer;
+	switch (srx->transport.family) {
+	case AF_INET:
+		if (peer->srx.transport.sin.sin_port !=
+		    srx->transport.sin.sin_port)
 			goto not_found;
-
-		peer = conn->params.peer;
-		switch (srx.transport.family) {
-		case AF_INET:
-			if (peer->srx.transport.sin.sin_port !=
-			    srx.transport.sin.sin_port ||
-			    peer->srx.transport.sin.sin_addr.s_addr !=
-			    srx.transport.sin.sin_addr.s_addr)
-				goto not_found;
-			break;
+		break;
 #ifdef CONFIG_AF_RXRPC_IPV6
-		case AF_INET6:
-			if (peer->srx.transport.sin6.sin6_port !=
-			    srx.transport.sin6.sin6_port ||
-			    memcmp(&peer->srx.transport.sin6.sin6_addr,
-				   &srx.transport.sin6.sin6_addr,
-				   sizeof(struct in6_addr)) != 0)
-				goto not_found;
-			break;
+	case AF_INET6:
+		if (peer->srx.transport.sin6.sin6_port !=
+		    srx->transport.sin6.sin6_port)
+			goto not_found;
+		break;
 #endif
-		default:
-			BUG();
-		}
-
-		_leave(" = %p", conn);
-		return conn;
+	default:
+		BUG();
 	}
 
+	_leave(" = %p", conn);
+	return conn;
+
 not_found:
 	_leave(" = NULL");
 	return NULL;
@@ -175,14 +156,14 @@ void __rxrpc_disconnect_call(struct rxrpc_connection *conn,
 
 	_enter("%d,%x", conn->debug_id, call->cid);
 
-	if (rcu_access_pointer(chan->call) == call) {
+	if (chan->call == call) {
 		/* Save the result of the call so that we can repeat it if necessary
 		 * through the channel, whilst disposing of the actual call record.
 		 */
 		trace_rxrpc_disconnect_call(call);
 		switch (call->completion) {
 		case RXRPC_CALL_SUCCEEDED:
-			chan->last_seq = call->rx_hard_ack;
+			chan->last_seq = call->rx_highest_seq;
 			chan->last_type = RXRPC_PACKET_TYPE_ACK;
 			break;
 		case RXRPC_CALL_LOCALLY_ABORTED:
@@ -190,17 +171,14 @@ void __rxrpc_disconnect_call(struct rxrpc_connection *conn,
 			chan->last_type = RXRPC_PACKET_TYPE_ABORT;
 			break;
 		default:
-			chan->last_abort = RX_USER_ABORT;
+			chan->last_abort = RX_CALL_DEAD;
 			chan->last_type = RXRPC_PACKET_TYPE_ABORT;
 			break;
 		}
 
-		/* Sync with rxrpc_conn_retransmit(). */
-		smp_wmb();
 		chan->last_call = chan->call_id;
 		chan->call_id = chan->call_counter;
-
-		rcu_assign_pointer(chan->call, NULL);
+		chan->call = NULL;
 	}
 
 	_leave("");
@@ -214,107 +192,79 @@ void rxrpc_disconnect_call(struct rxrpc_call *call)
 {
 	struct rxrpc_connection *conn = call->conn;
 
-	call->peer->cong_cwnd = call->cong_cwnd;
-
-	spin_lock_bh(&conn->params.peer->lock);
-	hlist_del_rcu(&call->error_link);
-	spin_unlock_bh(&conn->params.peer->lock);
-
-	if (rxrpc_is_client_call(call))
-		return rxrpc_disconnect_client_call(call);
+	set_bit(RXRPC_CALL_DISCONNECTED, &call->flags);
+	rxrpc_see_call(call, rxrpc_call_see_disconnected);
 
-	spin_lock(&conn->channel_lock);
-	__rxrpc_disconnect_call(conn, call);
-	spin_unlock(&conn->channel_lock);
+	call->peer->cong_ssthresh = call->cong_ssthresh;
 
-	call->conn = NULL;
-	conn->idle_timestamp = jiffies;
-	rxrpc_put_connection(conn);
-}
-
-/*
- * Kill off a connection.
- */
-void rxrpc_kill_connection(struct rxrpc_connection *conn)
-{
-	struct rxrpc_net *rxnet = conn->params.local->rxnet;
-
-	ASSERT(!rcu_access_pointer(conn->channels[0].call) &&
-	       !rcu_access_pointer(conn->channels[1].call) &&
-	       !rcu_access_pointer(conn->channels[2].call) &&
-	       !rcu_access_pointer(conn->channels[3].call));
-	ASSERT(list_empty(&conn->cache_link));
-
-	write_lock(&rxnet->conn_lock);
-	list_del_init(&conn->proc_link);
-	write_unlock(&rxnet->conn_lock);
+	if (!hlist_unhashed(&call->error_link)) {
+		spin_lock_irq(&call->peer->lock);
+		hlist_del_init(&call->error_link);
+		spin_unlock_irq(&call->peer->lock);
+	}
 
-	/* Drain the Rx queue.  Note that even though we've unpublished, an
-	 * incoming packet could still be being added to our Rx queue, so we
-	 * will need to drain it again in the RCU cleanup handler.
-	 */
-	rxrpc_purge_queue(&conn->rx_queue);
+	if (rxrpc_is_client_call(call)) {
+		rxrpc_disconnect_client_call(call->bundle, call);
+	} else {
+		__rxrpc_disconnect_call(conn, call);
+		conn->idle_timestamp = jiffies;
+		if (atomic_dec_and_test(&conn->active))
+			rxrpc_set_service_reap_timer(conn->rxnet,
+						     jiffies + rxrpc_connection_expiry * HZ);
+	}
 
-	/* Leave final destruction to RCU.  The connection processor work item
-	 * must carry a ref on the connection to prevent us getting here whilst
-	 * it is queued or running.
-	 */
-	call_rcu(&conn->rcu, rxrpc_destroy_connection);
+	rxrpc_put_call(call, rxrpc_call_put_io_thread);
 }
 
 /*
  * Queue a connection's work processor, getting a ref to pass to the work
  * queue.
  */
-bool rxrpc_queue_conn(struct rxrpc_connection *conn)
+void rxrpc_queue_conn(struct rxrpc_connection *conn, enum rxrpc_conn_trace why)
 {
-	const void *here = __builtin_return_address(0);
-	int n = atomic_fetch_add_unless(&conn->usage, 1, 0);
-	if (n == 0)
-		return false;
-	if (rxrpc_queue_work(&conn->processor))
-		trace_rxrpc_conn(conn, rxrpc_conn_queued, n + 1, here);
-	else
-		rxrpc_put_connection(conn);
-	return true;
+	if (atomic_read(&conn->active) >= 0 &&
+	    rxrpc_queue_work(&conn->processor))
+		rxrpc_see_connection(conn, why);
 }
 
 /*
  * Note the re-emergence of a connection.
  */
-void rxrpc_see_connection(struct rxrpc_connection *conn)
+void rxrpc_see_connection(struct rxrpc_connection *conn,
+			  enum rxrpc_conn_trace why)
 {
-	const void *here = __builtin_return_address(0);
 	if (conn) {
-		int n = atomic_read(&conn->usage);
+		int r = refcount_read(&conn->ref);
 
-		trace_rxrpc_conn(conn, rxrpc_conn_seen, n, here);
+		trace_rxrpc_conn(conn->debug_id, r, why);
 	}
 }
 
 /*
  * Get a ref on a connection.
  */
-void rxrpc_get_connection(struct rxrpc_connection *conn)
+struct rxrpc_connection *rxrpc_get_connection(struct rxrpc_connection *conn,
+					      enum rxrpc_conn_trace why)
 {
-	const void *here = __builtin_return_address(0);
-	int n = atomic_inc_return(&conn->usage);
+	int r;
 
-	trace_rxrpc_conn(conn, rxrpc_conn_got, n, here);
+	__refcount_inc(&conn->ref, &r);
+	trace_rxrpc_conn(conn->debug_id, r + 1, why);
+	return conn;
 }
 
 /*
  * Try to get a ref on a connection.
  */
 struct rxrpc_connection *
-rxrpc_get_connection_maybe(struct rxrpc_connection *conn)
+rxrpc_get_connection_maybe(struct rxrpc_connection *conn,
+			   enum rxrpc_conn_trace why)
 {
-	const void *here = __builtin_return_address(0);
+	int r;
 
 	if (conn) {
-		int n = atomic_fetch_add_unless(&conn->usage, 1, 0);
-		if (n > 0)
-			trace_rxrpc_conn(conn, rxrpc_conn_got, n + 1, here);
+		if (__refcount_inc_not_zero(&conn->ref, &r))
+			trace_rxrpc_conn(conn->debug_id, r + 1, why);
 		else
 			conn = NULL;
 	}
@@ -332,49 +282,103 @@ static void rxrpc_set_service_reap_timer(struct rxrpc_net *rxnet,
 }
 
 /*
- * Release a service connection
+ * destroy a virtual connection
  */
-void rxrpc_put_service_conn(struct rxrpc_connection *conn)
+static void rxrpc_rcu_free_connection(struct rcu_head *rcu)
 {
-	const void *here = __builtin_return_address(0);
-	int n;
-
-	n = atomic_dec_return(&conn->usage);
-	trace_rxrpc_conn(conn, rxrpc_conn_put_service, n, here);
-	ASSERTCMP(n, >=, 0);
-	if (n == 1)
-		rxrpc_set_service_reap_timer(conn->params.local->rxnet,
-					     jiffies + rxrpc_connection_expiry);
+	struct rxrpc_connection *conn =
+		container_of(rcu, struct rxrpc_connection, rcu);
+	struct rxrpc_net *rxnet = conn->rxnet;
+
+	_enter("{%d,u=%d}", conn->debug_id, refcount_read(&conn->ref));
+
+	trace_rxrpc_conn(conn->debug_id, refcount_read(&conn->ref),
+			 rxrpc_conn_free);
+	kfree(conn);
+
+	if (atomic_dec_and_test(&rxnet->nr_conns))
+		wake_up_var(&rxnet->nr_conns);
 }
 
 /*
- * destroy a virtual connection
+ * Clean up a dead connection.
  */
-static void rxrpc_destroy_connection(struct rcu_head *rcu)
+static void rxrpc_clean_up_connection(struct work_struct *work)
 {
 	struct rxrpc_connection *conn =
-		container_of(rcu, struct rxrpc_connection, rcu);
+		container_of(work, struct rxrpc_connection, destructor);
+	struct rxrpc_net *rxnet = conn->rxnet;
 
-	_enter("{%d,u=%d}", conn->debug_id, atomic_read(&conn->usage));
+	ASSERT(!conn->channels[0].call &&
+	       !conn->channels[1].call &&
+	       !conn->channels[2].call &&
+	       !conn->channels[3].call);
+	ASSERT(list_empty(&conn->cache_link));
+
+	timer_delete_sync(&conn->timer);
+	cancel_work_sync(&conn->processor); /* Processing may restart the timer */
+	timer_delete_sync(&conn->timer);
 
-	ASSERTCMP(atomic_read(&conn->usage), ==, 0);
+	write_lock(&rxnet->conn_lock);
+	list_del_init(&conn->proc_link);
+	write_unlock(&rxnet->conn_lock);
 
-	_net("DESTROY CONN %d", conn->debug_id);
+	if (conn->pmtud_probe) {
+		trace_rxrpc_pmtud_lost(conn, 0);
+		conn->peer->pmtud_probing = false;
+		conn->peer->pmtud_pending = true;
+	}
 
-	del_timer_sync(&conn->timer);
 	rxrpc_purge_queue(&conn->rx_queue);
+	rxrpc_free_skb(conn->tx_response, rxrpc_skb_put_response);
+
+	rxrpc_kill_client_conn(conn);
 
 	conn->security->clear(conn);
-	key_put(conn->params.key);
-	key_put(conn->server_key);
-	rxrpc_put_peer(conn->params.peer);
+	key_put(conn->key);
+	rxrpc_put_bundle(conn->bundle, rxrpc_bundle_put_conn);
+	rxrpc_put_peer(conn->peer, rxrpc_peer_put_conn);
+	rxrpc_put_local(conn->local, rxrpc_local_put_kill_conn);
 
-	if (atomic_dec_and_test(&conn->params.local->rxnet->nr_conns))
-		wake_up_var(&conn->params.local->rxnet->nr_conns);
-	rxrpc_put_local(conn->params.local);
+	/* Drain the Rx queue.  Note that even though we've unpublished, an
+	 * incoming packet could still be being added to our Rx queue, so we
+	 * will need to drain it again in the RCU cleanup handler.
+	 */
+	rxrpc_purge_queue(&conn->rx_queue);
 
-	kfree(conn);
-	_leave("");
+	page_frag_cache_drain(&conn->tx_data_alloc);
+	call_rcu(&conn->rcu, rxrpc_rcu_free_connection);
+}
+
+/*
+ * Drop a ref on a connection.
+ */
+void rxrpc_put_connection(struct rxrpc_connection *conn,
+			  enum rxrpc_conn_trace why)
+{
+	unsigned int debug_id;
+	bool dead;
+	int r;
+
+	if (!conn)
+		return;
+
+	debug_id = conn->debug_id;
+	dead = __refcount_dec_and_test(&conn->ref, &r);
+	trace_rxrpc_conn(debug_id, r - 1, why);
+	if (dead) {
+		timer_delete(&conn->timer);
+		cancel_work(&conn->processor);
+
+		if (in_softirq() || work_busy(&conn->processor) ||
+		    timer_pending(&conn->timer))
+			/* Can't use the rxrpc workqueue as we need to cancel/flush
+			 * something that may be running/waiting there.
+			 */
+			schedule_work(&conn->destructor);
+		else
+			rxrpc_clean_up_connection(&conn->destructor);
+	}
 }
 
 /*
@@ -386,6 +390,7 @@ void rxrpc_service_connection_reaper(struct work_struct *work)
 	struct rxrpc_net *rxnet =
 		container_of(work, struct rxrpc_net, service_conn_reaper);
 	unsigned long expire_at, earliest, idle_timestamp, now;
+	int active;
 
 	LIST_HEAD(graveyard);
 
@@ -396,20 +401,20 @@ void rxrpc_service_connection_reaper(struct work_struct *work)
 
 	write_lock(&rxnet->conn_lock);
 	list_for_each_entry_safe(conn, _p, &rxnet->service_conns, link) {
-		ASSERTCMP(atomic_read(&conn->usage), >, 0);
-		if (likely(atomic_read(&conn->usage) > 1))
+		ASSERTCMP(atomic_read(&conn->active), >=, 0);
+		if (likely(atomic_read(&conn->active) > 0))
 			continue;
 		if (conn->state == RXRPC_CONN_SERVICE_PREALLOC)
 			continue;
 
-		if (rxnet->live) {
+		if (rxnet->live && !conn->local->dead) {
 			idle_timestamp = READ_ONCE(conn->idle_timestamp);
 			expire_at = idle_timestamp + rxrpc_connection_expiry * HZ;
-			if (conn->params.local->service_closed)
+			if (conn->local->service_closed)
 				expire_at = idle_timestamp + rxrpc_closed_conn_expiry * HZ;
 
-			_debug("reap CONN %d { u=%d,t=%ld }",
-			       conn->debug_id, atomic_read(&conn->usage),
+			_debug("reap CONN %d { a=%d,t=%ld }",
+			       conn->debug_id, atomic_read(&conn->active),
 			       (long)expire_at - (long)now);
 
 			if (time_before(now, expire_at)) {
@@ -419,12 +424,13 @@ void rxrpc_service_connection_reaper(struct work_struct *work)
 			}
 		}
 
-		/* The usage count sits at 1 whilst the object is unused on the
-		 * list; we reduce that to 0 to make the object unavailable.
+		/* The activity count sits at 0 whilst the conn is unused on
+		 * the list; we reduce that to -1 to make the conn unavailable.
 		 */
-		if (atomic_cmpxchg(&conn->usage, 1, 0) != 1)
+		active = 0;
+		if (!atomic_try_cmpxchg(&conn->active, &active, -1))
 			continue;
-		trace_rxrpc_conn(conn, rxrpc_conn_reap_service, 0, NULL);
+		rxrpc_see_connection(conn, rxrpc_conn_see_reap_service);
 
 		if (rxrpc_conn_is_client(conn))
 			BUG();
@@ -446,8 +452,8 @@ void rxrpc_service_connection_reaper(struct work_struct *work)
 				  link);
 		list_del_init(&conn->link);
 
-		ASSERTCMP(atomic_read(&conn->usage), ==, 0);
-		rxrpc_kill_connection(conn);
+		ASSERTCMP(atomic_read(&conn->active), ==, -1);
+		rxrpc_put_connection(conn, rxrpc_conn_put_service_reaped);
 	}
 
 	_leave("");
@@ -465,16 +471,15 @@ void rxrpc_destroy_all_connections(struct rxrpc_net *rxnet)
 	_enter("");
 
 	atomic_dec(&rxnet->nr_conns);
-	rxrpc_destroy_all_client_connections(rxnet);
 
-	del_timer_sync(&rxnet->service_conn_reap_timer);
+	timer_delete_sync(&rxnet->service_conn_reap_timer);
 	rxrpc_queue_work(&rxnet->service_conn_reaper);
 	flush_workqueue(rxrpc_workqueue);
 
 	write_lock(&rxnet->conn_lock);
 	list_for_each_entry_safe(conn, _p, &rxnet->service_conns, link) {
 		pr_err("AF_RXRPC: Leaked conn %p {%d}\n",
-		       conn, atomic_read(&conn->usage));
+		       conn, refcount_read(&conn->ref));
 		leak = true;
 	}
 	write_unlock(&rxnet->conn_lock);