MINOR: proxy: Don't close FDs if not our proxy.

When running with multiple process, if some proxies are just assigned
to some processes, the other processes will just close the file descriptors
for the listening sockets. However, we may still have to provide those
sockets when reloading, so instead we just try hard to pretend those proxies
are dead, while keeping the sockets opened.
A new global option, no-reused-socket", has been added, to restore the old
behavior of closing the sockets not bound to this process.

doc/configuration.txt

@@ -587,6 +587,7 @@ The following keywords are supported in the "global" section :
    - nosplice
    - nogetaddrinfo
    - noreuseport
+   - no-unused-socket
    - spread-checks
    - server-state-base
    - server-state-file
@@ -1250,6 +1251,12 @@ noreuseport
   Disables the use of SO_REUSEPORT - see socket(7). It is equivalent to the
   command line argument "-dR".
 
+no-unused-socket
+  By default, each haproxy process keeps all sockets opened, event those that
+  are only used by another processes, so that any process can provide all the
+  sockets, to make reloads seamless. This option disables this, and close all
+  unused sockets, to save some file descriptors.
+
 spread-checks <0..50, in percent>
   Sometimes it is desirable to avoid sending agent and health checks to
   servers at exact intervals, for instance when many logical servers are

include/proto/fd.h

@@ -41,6 +41,11 @@ extern int fd_nbupdt;               // number of updates in the list
  */
 void fd_delete(int fd);
 
+/* Deletes an FD from the fdsets, and recomputes the maxfd limit.
+ * The file descriptor is kept open.
+ */
+void fd_remove(int fd);
+
 /* disable the specified poller */
 void disable_poller(const char *poller_name);
 

include/proto/listener.h

@@ -88,6 +88,11 @@ void dequeue_all_listeners(struct list *list);
  */
 int unbind_listener(struct listener *listener);
 
+/* This function pretends the listener is dead, but keeps the FD opened, so
+ * that we can provide it, for conf reloading.
+ */
+int unbind_listener_no_close(struct listener *listener);
+
 /* This function closes all listening sockets bound to the protocol <proto>,
  * and the listeners end in LI_ASSIGNED state if they were higher. It does not
  * detach them from the protocol. It always returns ERR_NONE.

include/proto/proxy.h

@@ -42,6 +42,7 @@ void soft_stop(void);
 int pause_proxy(struct proxy *p);
 int resume_proxy(struct proxy *p);
 void stop_proxy(struct proxy *p);
+void zombify_proxy(struct proxy *p);
 void pause_proxies(void);
 void resume_proxies(void);
 int  stream_set_backend(struct stream *s, struct proxy *be);

include/types/global.h

@@ -62,6 +62,8 @@
 #define GTUNE_USE_REUSEPORT      (1<<6)
 #define GTUNE_RESOLVE_DONTFAIL   (1<<7)
 
+#define GTUNE_SOCKET_TRANSFER	 (1<<8)
+
 /* Access level for a stats socket */
 #define ACCESS_LVL_NONE     0
 #define ACCESS_LVL_USER     1

include/types/listener.h

@@ -47,6 +47,7 @@ enum li_state {
 	LI_INIT,        /* all parameters filled in, but not assigned yet */
 	LI_ASSIGNED,    /* assigned to the protocol, but not listening yet */
 	LI_PAUSED,      /* listener was paused, it's bound but not listening  */
+	LI_ZOMBIE,	/* The listener doesn't belong to the process, but is kept opened */
 	LI_LISTEN,      /* started, listening but not enabled */
 	LI_READY,       /* started, listening and enabled */
 	LI_FULL,        /* reached its connection limit */

src/cfgparse.c

@@ -659,6 +659,11 @@ int cfg_parse_global(const char *file, int linenum, char **args, int kwm)
 			goto out;
 		global.tune.options &= ~GTUNE_USE_REUSEPORT;
 	}
+	else if (!strcmp(args[0], "no-unused-socket")) {
+		if (alertif_too_many_args(0, file, linenum, args, &err_code))
+			goto out;
+		global.tune.options &= ~GTUNE_SOCKET_TRANSFER;
+	}
 	else if (!strcmp(args[0], "quiet")) {
 		if (alertif_too_many_args(0, file, linenum, args, &err_code))
 			goto out;

src/cli.c

@@ -1063,10 +1063,11 @@ static int _getsocks(char **args, struct appctx *appctx, void *private)
 		struct listener *l;
 
 		list_for_each_entry(l, &px->conf.listeners, by_fe) {
-			/* Only transfer IPv4/IPv6 sockets */
-			if (l->proto->sock_family == AF_INET ||
+			/* Only transfer IPv4/IPv6/UNIX sockets */
+			if (l->state >= LI_ZOMBIE &&
+			    (l->proto->sock_family == AF_INET ||
 			    l->proto->sock_family == AF_INET6 ||
-			    l->proto->sock_family == AF_UNIX)
+			    l->proto->sock_family == AF_UNIX))
 				tot_fd_nb++;
 		}
 		px = px->next;
@@ -1117,7 +1118,7 @@ static int _getsocks(char **args, struct appctx *appctx, void *private)
 		list_for_each_entry(l, &px->conf.listeners, by_fe) {
 			int ret;
 			/* Only transfer IPv4/IPv6 sockets */
-			if (l->state >= LI_LISTEN &&
+			if (l->state >= LI_ZOMBIE &&
 			    (l->proto->sock_family == AF_INET ||
 			    l->proto->sock_family == AF_INET6 ||
 			    l->proto->sock_family == AF_UNIX)) {

src/fd.c

@@ -175,7 +175,7 @@ int fd_nbupdt = 0;             // number of updates in the list
 /* Deletes an FD from the fdsets, and recomputes the maxfd limit.
  * The file descriptor is also closed.
  */
-void fd_delete(int fd)
+static void fd_dodelete(int fd, int do_close)
 {
 	if (fdtab[fd].linger_risk) {
 		/* this is generally set when connecting to servers */
@@ -190,7 +190,8 @@ void fd_delete(int fd)
 
 	port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port);
 	fdinfo[fd].port_range = NULL;
-	close(fd);
+	if (do_close)
+		close(fd);
 	fdtab[fd].owner = NULL;
 	fdtab[fd].new = 0;
 
@@ -198,6 +199,22 @@ void fd_delete(int fd)
 		maxfd--;
 }
 
+/* Deletes an FD from the fdsets, and recomputes the maxfd limit.
+ * The file descriptor is also closed.
+ */
+void fd_delete(int fd)
+{
+	fd_dodelete(fd, 1);
+}
+
+/* Deletes an FD from the fdsets, and recomputes the maxfd limit.
+ * The file descriptor is kept open.
+ */
+void fd_remove(int fd)
+{
+	fd_dodelete(fd, 0);
+}
+
 /* Scan and process the cached events. This should be called right after
  * the poller. The loop may cause new entries to be created, for example
  * if a listener causes an accept() to initiate a new incoming connection

src/haproxy.c

@@ -857,6 +857,7 @@ static void init(int argc, char **argv)
 #if defined(SO_REUSEPORT)
 	global.tune.options |= GTUNE_USE_REUSEPORT;
 #endif
+	global.tune.options |= GTUNE_SOCKET_TRANSFER;
 
 	pid = getpid();
 	progname = *argv;
@@ -1668,6 +1669,15 @@ void deinit(void)
 		}/* end while(s) */
 
 		list_for_each_entry_safe(l, l_next, &p->conf.listeners, by_fe) {
+			/*
+			 * Zombie proxy, the listener just pretend to be up
+			 * because they still hold an opened fd.
+			 * Close it and give the listener its real state.
+			 */
+			if (p->state == PR_STSTOPPED && l->state >= LI_ZOMBIE) {
+				close(l->fd);
+				l->state = LI_INIT;
+			}
 			unbind_listener(l);
 			delete_listener(l);
 			LIST_DEL(&l->by_fe);
@@ -2148,8 +2158,12 @@ int main(int argc, char **argv)
 		px = proxy;
 		while (px != NULL) {
 			if (px->bind_proc && px->state != PR_STSTOPPED) {
-				if (!(px->bind_proc & (1UL << proc)))
-					stop_proxy(px);
+				if (!(px->bind_proc & (1UL << proc))) {
+					if (global.tune.options & GTUNE_SOCKET_TRANSFER)
+						zombify_proxy(px);
+					else
+						stop_proxy(px);
+				}
 			}
 			px = px->next;
 		}

src/listener.c

@@ -59,7 +59,12 @@ void enable_listener(struct listener *listener)
 			/* we don't want to enable this listener and don't
 			 * want any fd event to reach it.
 			 */
-			unbind_listener(listener);
+			if (!(global.tune.options & GTUNE_SOCKET_TRANSFER))
+				unbind_listener(listener);
+			else {
+				unbind_listener_no_close(listener);
+				listener->state = LI_LISTEN;
+			}
 		}
 		else if (listener->nbconn < listener->maxconn) {
 			fd_want_recv(listener->fd);
@@ -95,7 +100,7 @@ void disable_listener(struct listener *listener)
  */
 int pause_listener(struct listener *l)
 {
-	if (l->state <= LI_PAUSED)
+	if (l->state <= LI_ZOMBIE)
 		return 1;
 
 	if (l->proto->pause) {
@@ -149,7 +154,7 @@ int resume_listener(struct listener *l)
 			return 0;
 	}
 
-	if (l->state < LI_PAUSED)
+	if (l->state < LI_PAUSED || l->state == LI_ZOMBIE)
 		return 0;
 
 	if (l->proto->sock_prot == IPPROTO_TCP &&
@@ -242,12 +247,7 @@ void dequeue_all_listeners(struct list *list)
 	}
 }
 
-/* This function closes the listening socket for the specified listener,
- * provided that it's already in a listening state. The listener enters the
- * LI_ASSIGNED state. It always returns ERR_NONE. This function is intended
- * to be used as a generic function for standard protocols.
- */
-int unbind_listener(struct listener *listener)
+static int do_unbind_listener(struct listener *listener, int do_close)
 {
 	if (listener->state == LI_READY)
 		fd_stop_recv(listener->fd);
@@ -256,13 +256,35 @@ int unbind_listener(struct listener *listener)
 		LIST_DEL(&listener->wait_queue);
 
 	if (listener->state >= LI_PAUSED) {
-		fd_delete(listener->fd);
-		listener->fd = -1;
+		if (do_close) {
+			fd_delete(listener->fd);
+			listener->fd = -1;
+		}
+		else
+			fd_remove(listener->fd);
 		listener->state = LI_ASSIGNED;
 	}
 	return ERR_NONE;
 }
 
+/* This function closes the listening socket for the specified listener,
+ * provided that it's already in a listening state. The listener enters the
+ * LI_ASSIGNED state. It always returns ERR_NONE. This function is intended
+ * to be used as a generic function for standard protocols.
+ */
+int unbind_listener(struct listener *listener)
+{
+	return do_unbind_listener(listener, 1);
+}
+
+/* This function pretends the listener is dead, but keeps the FD opened, so
+ * that we can provide it, for conf reloading.
+ */
+int unbind_listener_no_close(struct listener *listener)
+{
+	return do_unbind_listener(listener, 0);
+}
+
 /* This function closes all listening sockets bound to the protocol <proto>,
  * and the listeners end in LI_ASSIGNED state if they were higher. It does not
  * detach them from the protocol. It always returns ERR_NONE.

src/proxy.c

@@ -991,6 +991,19 @@ void soft_stop(void)
 	p = proxy;
 	tv_update_date(0,1); /* else, the old time before select will be used */
 	while (p) {
+		/* Zombie proxy, let's close the file descriptors */
+		if (p->state == PR_STSTOPPED &&
+		    !LIST_ISEMPTY(&p->conf.listeners) &&
+		    LIST_ELEM(p->conf.listeners.n,
+		    struct listener *, by_fe)->state >= LI_ZOMBIE) {
+			struct listener *l;
+			list_for_each_entry(l, &p->conf.listeners, by_fe) {
+				if (l->state >= LI_ZOMBIE)
+					close(l->fd);
+				l->state = LI_INIT;
+			}
+		}
+
 		if (p->state != PR_STSTOPPED) {
 			Warning("Stopping %s %s in %d ms.\n", proxy_cap_str(p->cap), p->id, p->grace);
 			send_log(p, LOG_WARNING, "Stopping %s %s in %d ms.\n", proxy_cap_str(p->cap), p->id, p->grace);
@@ -1051,6 +1064,45 @@ int pause_proxy(struct proxy *p)
 	return 1;
 }
 
+/* This function makes the proxy unusable, but keeps the listening sockets
+ * opened, so that if any process requests them, we are able to serve them.
+ * This should only be called early, before we started accepting requests.
+ */
+void zombify_proxy(struct proxy *p)
+{
+	struct listener *l;
+	struct listener *first_to_listen = NULL;
+
+	list_for_each_entry(l, &p->conf.listeners, by_fe) {
+		enum li_state oldstate = l->state;
+
+		unbind_listener_no_close(l);
+		if (l->state >= LI_ASSIGNED) {
+			delete_listener(l);
+			listeners--;
+			jobs--;
+		}
+		/*
+		 * Pretend we're still up and running so that the fd
+		 * will be sent if asked.
+		 */
+		l->state = LI_ZOMBIE;
+		if (!first_to_listen && oldstate >= LI_LISTEN)
+			first_to_listen = l;
+	}
+	/* Quick hack : at stop time, to know we have to close the sockets
+	 * despite the proxy being marked as stopped, make the first listener
+	 * of the listener list an active one, so that we don't have to
+	 * parse the whole list to be sure.
+	 */
+	if (first_to_listen && LIST_ELEM(p->conf.listeners.n,
+	    struct listener *, by_fe) != first_to_listen) {
+		LIST_DEL(&l->by_fe);
+		LIST_ADD(&p->conf.listeners, &l->by_fe);
+	}
+
+	p->state = PR_STSTOPPED;
+}
 
 /*
  * This function completely stops a proxy and releases its listeners. It has