From 43a6c2239cd710230ef24ab4abdccdf8946f2ae9 Mon Sep 17 00:00:00 2001
From: Thomas Schoebel-Theuer <tst@1und1.de>
Date: Fri, 30 Nov 2012 11:52:21 +0100
Subject: [PATCH] net: fix release and re-open of sockets

---
 mars_net.c          | 41 ++++++++++++++++++++++-------------------
 mars_net.h          |  2 +-
 sy_old/mars_light.c | 24 +++++++++++++++++-------
 3 files changed, 40 insertions(+), 27 deletions(-)

diff --git a/mars_net.c b/mars_net.c
index aa8e6c40..e35ad885 100644
--- a/mars_net.c
+++ b/mars_net.c
@@ -241,7 +241,7 @@ EXPORT_SYMBOL_GPL(mars_accept_socket);
 
 bool mars_get_socket(struct mars_socket *msock)
 {
-	MARS_LOW("#%d get socket %p s_dead = %d s_count=%d\n", msock->s_debug_nr, msock->s_socket, msock->s_dead, atomic_read(&msock->s_count));
+	MARS_LOW("#%d get socket %p s_count=%d\n", msock->s_debug_nr, msock->s_socket, atomic_read(&msock->s_count));
 	if (unlikely(atomic_read(&msock->s_count) <= 0)) {
 		MARS_ERR("#%d bad nesting on msock = %p\n", msock->s_debug_nr, msock);
 		return false;
@@ -249,7 +249,7 @@ bool mars_get_socket(struct mars_socket *msock)
 
 	atomic_inc(&msock->s_count);
 
-	if (unlikely(!msock->s_socket || msock->s_dead)) {
+	if (unlikely(!msock->s_socket)) {
 		mars_put_socket(msock);
 		return false;
 	}
@@ -260,7 +260,7 @@ EXPORT_SYMBOL_GPL(mars_get_socket);
 
 void mars_put_socket(struct mars_socket *msock)
 {
-	MARS_LOW("#%d put socket %p s_dead = %d s_count=%d\n", msock->s_debug_nr, msock->s_socket, msock->s_dead, atomic_read(&msock->s_count));
+	MARS_LOW("#%d put socket %p s_count=%d\n", msock->s_debug_nr, msock->s_socket, atomic_read(&msock->s_count));
 	if (unlikely(atomic_read(&msock->s_count) <= 0)) {
 		MARS_ERR("#%d bad nesting on msock = %p sock = %p\n", msock->s_debug_nr, msock, msock->s_socket);
 	} else if (atomic_dec_and_test(&msock->s_count)) {
@@ -269,8 +269,13 @@ void mars_put_socket(struct mars_socket *msock)
 
 		MARS_DBG("#%d closing socket %p\n", msock->s_debug_nr, sock);
 		if (likely(sock)) {
+			msock->s_socket = NULL;
 			kernel_sock_shutdown(sock, SHUT_WR);
-			sock_release(sock);
+			msock->s_release_socket = sock;
+		}
+		if (likely(msock->s_release_socket)) {
+			MARS_DBG("#%d releasing socket %p\n", msock->s_debug_nr, msock->s_release_socket);
+			sock_release(msock->s_release_socket);
 		}
 		for (i = 0; i < MAX_DESC_CACHE; i++) {
 			if (msock->s_desc_send[i])
@@ -286,21 +291,21 @@ EXPORT_SYMBOL_GPL(mars_put_socket);
 
 void mars_shutdown_socket(struct mars_socket *msock)
 {
-	bool ok;
+	MARS_IO("#%d shutdown socket %p s_count=%d\n", msock->s_debug_nr, msock->s_socket, atomic_read(&msock->s_count));
 
-	MARS_IO("#%d shutdown socket %p s_dead = %d s_count=%d\n", msock->s_debug_nr, msock->s_socket, msock->s_dead, atomic_read(&msock->s_count));
-
-	ok = mars_get_socket(msock);
-	if (likely(ok)) {
-		struct socket *sock = msock->s_socket;
-		if (likely(sock)) {
-			if (!msock->s_dead) {
-				msock->s_dead = true;
-				MARS_DBG("shutdown socket #%d %p\n", msock->s_debug_nr, sock);
+	if (msock->s_socket) {
+		bool ok = mars_get_socket(msock);
+		if (likely(ok)) {
+			struct socket *sock = msock->s_socket;
+			if (likely(sock)) {
+				msock->s_socket = NULL;
+				MARS_DBG("#%d shutdown socket %p\n", msock->s_debug_nr, sock);
 				kernel_sock_shutdown(sock, SHUT_WR);
+				// delay sock_release() until the last reference has gone
+				msock->s_release_socket = sock;
 			}
+			mars_put_socket(msock);
 		}
-		mars_put_socket(msock);
 	}
 }
 EXPORT_SYMBOL_GPL(mars_shutdown_socket);
@@ -314,11 +319,9 @@ bool mars_socket_is_alive(struct mars_socket *msock)
 		MARS_ERR("#%d bad nesting on msock = %p sock = %p\n", msock->s_debug_nr, msock, msock->s_socket);
 		goto done;
 	}
-	if (msock->s_dead)
-		goto done;
 	res = true;
 done:
-	MARS_LOW("#%d %p s_count = %d s_dead = %d\n", msock->s_debug_nr, msock->s_socket, atomic_read(&msock->s_count), msock->s_dead);
+	MARS_LOW("#%d %p s_count = %d\n", msock->s_debug_nr, msock->s_socket, atomic_read(&msock->s_count));
 	return res;
 }
 EXPORT_SYMBOL_GPL(mars_socket_is_alive);
@@ -514,7 +517,7 @@ int mars_recv_raw(struct mars_socket *msock, void *buf, int minlen, int maxlen)
 #endif
 		};
 
-		if (unlikely(msock->s_dead)) {
+		if (unlikely(!msock->s_socket)) {
 			MARS_WRN("#%d socket has disappeared\n", msock->s_debug_nr);
 			status = -EIDRM;
 			goto err;
diff --git a/mars_net.h b/mars_net.h
index 7a13f0d0..8ea1e583 100644
--- a/mars_net.h
+++ b/mars_net.h
@@ -38,11 +38,11 @@ struct mars_desc_item {
  */
 struct mars_socket {
 	struct socket *s_socket;
+	struct socket *s_release_socket;
 	void *s_buffer;
 	atomic_t s_count;
 	int s_pos;
 	int s_debug_nr;
-	bool s_dead;
 	bool s_shutdown_on_err;
 	struct mars_desc_cache *s_desc_send[MAX_DESC_CACHE];
 	struct mars_desc_cache *s_desc_recv[MAX_DESC_CACHE];
diff --git a/sy_old/mars_light.c b/sy_old/mars_light.c
index 2b4b3b63..7e3252b8 100644
--- a/sy_old/mars_light.c
+++ b/sy_old/mars_light.c
@@ -1008,12 +1008,11 @@ static
 void _peer_cleanup(struct mars_peerinfo *peer)
 {
 	MARS_DBG("cleanup\n");
-	if (peer->socket.s_socket) {
+	if (mars_socket_is_alive(&peer->socket)) {
 		MARS_DBG("really shutdown socket\n");
 		mars_shutdown_socket(&peer->socket);
-		mars_put_socket(&peer->socket);
 	}
-
+	mars_put_socket(&peer->socket);
 }
 
 static DECLARE_WAIT_QUEUE_HEAD(remote_event);
@@ -1027,6 +1026,7 @@ int peer_thread(void *data)
 	char *real_peer;
 	struct sockaddr_storage sockaddr = {};
 	int pause_time = 0;
+	bool do_kill = false;
 	bool flip = false;
 	int status;
 
@@ -1055,7 +1055,8 @@ int peer_thread(void *data)
 		};
 
 		if (!mars_socket_is_alive(&peer->socket)) {
-			if (peer->socket.s_socket) {
+			if (do_kill) {
+				do_kill = false;
 				_peer_cleanup(peer);
 				brick_msleep(5000);
 				continue;
@@ -1071,6 +1072,7 @@ int peer_thread(void *data)
 				brick_msleep(5000);
 				continue;
 			}
+			do_kill = true;
 			peer->socket.s_shutdown_on_err = true;
 			MARS_DBG("successfully opened socket to '%s'\n", real_peer);
 			brick_msleep(100);
@@ -1091,7 +1093,10 @@ int peer_thread(void *data)
 		status = mars_send_struct(&peer->socket, &cmd, mars_cmd_meta);
 		if (unlikely(status < 0)) {
 			MARS_WRN("communication error on send, status = %d\n", status);
-			_peer_cleanup(peer);
+			if (do_kill) {
+				do_kill = false;
+				_peer_cleanup(peer);
+			}
 			brick_msleep(2000);
 			continue;
 		}
@@ -1106,7 +1111,10 @@ int peer_thread(void *data)
 		status = mars_recv_dent_list(&peer->socket, &tmp_list);
 		if (unlikely(status < 0)) {
 			MARS_WRN("communication error on receive, status = %d\n", status);
-			_peer_cleanup(peer);
+			if (do_kill) {
+				do_kill = false;
+				_peer_cleanup(peer);
+			}
 			mars_free_dent_all(NULL, &tmp_list);
 			brick_msleep(5000);
 			continue;
@@ -1138,7 +1146,9 @@ int peer_thread(void *data)
 
 	MARS_INF("-------- peer thread terminating\n");
 
-	_peer_cleanup(peer);
+	if (do_kill) {
+		_peer_cleanup(peer);
+	}
 
 done:
 	atomic_dec(&peer_thread_count);