MEDIUM: connection: Add private connections synchronously in session server list

When a connection is marked as private, it is now added in the session server
list. We don't wait a stream is detached from the mux to do so. When the
connection is created, this happens after the mux creation. Otherwise, it is
performed when the connection is marked as private.

To allow that, when a connection is created, the session is systematically set
as the connectin owner. Thus, a backend connection has always a owner during its
creation. And a private connection has always a owner until its death.

Note that outside the detach() callback, if the call to session_add_conn()
failed, the error is ignored. In this situation, we retry to add the connection
into the session server list in the detach() callback. If this fails at this
step, the multiplexer is destroyed and the connection is closed.

include/haproxy/session.h

@@ -90,11 +90,20 @@ static inline void session_unown_conn(struct session *sess, struct connection *c
 	}
 }
 
+/* Add the connection <conn> to the server list of the session <sess>. This
+ * function is called only if the connection is private. Nothing is performed if
+ * the connection is already in the session sever list or if the session does
+ * not own the connection.
+ */
 static inline int session_add_conn(struct session *sess, struct connection *conn, void *target)
 {
 	struct sess_srv_list *srv_list = NULL;
 	int found = 0;
 
+	/* Already attach to the session or not the connection owner */
+	if (!LIST_ISEMPTY(&conn->session_list) || conn->owner != sess)
+		return 1;
+
 	list_for_each_entry(srv_list, &sess->srv_list, srv_list) {
 		if (srv_list->target == target) {
 			found = 1;
@@ -114,11 +123,16 @@ static inline int session_add_conn(struct session *sess, struct connection *conn
 	return 1;
 }
 
-/* Returns 0 if the session can keep the idle conn, -1 if it was destroyed, or 1 if it was added to the server list */
+/* Returns 0 if the session can keep the idle conn, -1 if it was destroyed. The
+ * connection must be private.
+ */
 static inline int session_check_idle_conn(struct session *sess, struct connection *conn)
 {
-	if (!(conn->flags & CO_FL_PRIVATE) ||
+	/* Another session owns this connection */
-	    sess->idle_conns >= sess->fe->max_out_conns) {
+	if (conn->owner != sess)
+		return 0;
+
+	if (sess->idle_conns >= sess->fe->max_out_conns) {
 		session_unown_conn(sess, conn);
 		conn->owner = NULL;
 		conn->flags &= ~CO_FL_SESS_IDLE;

src/backend.c

@@ -1402,6 +1402,7 @@ int connect_server(struct stream *s)
 			srv_conn->target = s->target;
 		srv_cs = NULL;
 
+		srv_conn->owner = s->sess;
 		if ((s->be->options & PR_O_REUSE_MASK) == PR_O_REUSE_NEVR)
 			conn_set_private(srv_conn);
 	}
@@ -1466,10 +1467,7 @@ int connect_server(struct stream *s)
 		    srv->mux_proto || s->be->mode != PR_MODE_HTTP))
 #endif
 			init_mux = 1;
-#if defined(USE_OPENSSL) && defined(TLSEXT_TYPE_application_layer_protocol_negotiation)
+
-		else
-			srv_conn->owner = s->sess;
-#endif
 		/* process the case where the server requires the PROXY protocol to be sent */
 		srv_conn->send_proxy_ofs = 0;
 
@@ -1547,11 +1545,17 @@ int connect_server(struct stream *s)
 		}
 		/* If we're doing http-reuse always, and the connection is not
 		 * private with available streams (an http2 connection), add it
-		 * to the available list, so that others can use it right away.
+		 * to the available list, so that others can use it right
+		 * away. If the connection is private, add it in the session
+		 * server list.
 		 */
 		if (srv && ((s->be->options & PR_O_REUSE_MASK) == PR_O_REUSE_ALWS) &&
 		    !(srv_conn->flags & CO_FL_PRIVATE) && srv_conn->mux->avail_streams(srv_conn) > 0)
 			LIST_ADDQ(&srv->available_conns[tid], mt_list_to_list(&srv_conn->list));
+		else if (srv_conn->flags & CO_FL_PRIVATE) {
+			/* If it fail now, the same will be done in mux->detach() callack */
+			session_add_conn(srv_conn->owner, srv_conn, srv_conn->target);
+		}
 	}
 	/* The CO_FL_SEND_PROXY flag may have been set by the connect method,
 	 * if so, add our handshake pseudo-XPRT now.

src/connection.c

@@ -60,9 +60,20 @@ int conn_create_mux(struct connection *conn)
 		else if (conn_install_mux_be(conn, conn->ctx, conn->owner) < 0)
 			goto fail;
 		srv = objt_server(conn->target);
+
+		/* If we're doing http-reuse always, and the connection is not
+		 * private with available streams (an http2 connection), add it
+		 * to the available list, so that others can use it right
+		 * away. If the connection is private, add it in the session
+		 * server list.
+		 */
 		if (srv && ((srv->proxy->options & PR_O_REUSE_MASK) == PR_O_REUSE_ALWS) &&
 		    !(conn->flags & CO_FL_PRIVATE) && conn->mux->avail_streams(conn) > 0)
 			LIST_ADDQ(&srv->available_conns[tid], mt_list_to_list(&conn->list));
+		else if (conn->flags & CO_FL_PRIVATE) {
+			/* If it fail now, the same will be done in mux->detach() callack */
+			session_add_conn(conn->owner, conn, conn->target);
+		}
 		return 0;
 fail:
 		/* let the upper layer know the connection failed */

src/http_ana.c

@@ -1838,7 +1838,10 @@ int http_wait_for_response(struct stream *s, struct channel *rep, int an_bit)
 			if ((ctx.value.len >= 4 && strncasecmp(ctx.value.ptr, "Nego", 4) == 0) ||
 			    (ctx.value.len >= 4 && strncasecmp(ctx.value.ptr, "NTLM", 4) == 0)) {
 				sess->flags |= SESS_FL_PREFER_LAST;
+				conn_set_owner(srv_conn, sess, NULL);
 				conn_set_private(srv_conn);
+				/* If it fail now, the same will be done in mux->detach() callack */
+				session_add_conn(srv_conn->owner, srv_conn, srv_conn->target);
 				break;
 			}
 		}

src/mux_fcgi.c

@@ -3541,19 +3541,17 @@ static void fcgi_detach(struct conn_stream *cs)
 	if (!(fconn->conn->flags & (CO_FL_ERROR|CO_FL_SOCK_RD_SH|CO_FL_SOCK_WR_SH)) &&
 	    (fconn->flags & FCGI_CF_KEEP_CONN)) {
 		if (fconn->conn->flags & CO_FL_PRIVATE) {
-			if (!fconn->conn->owner) {
+			/* Add the connection in the session serverlist, if not already done */
-				fconn->conn->owner = sess;
+			if (!session_add_conn(sess, fconn->conn, fconn->conn->target)) {
-				if (!session_add_conn(sess, fconn->conn, fconn->conn->target)) {
+				fconn->conn->owner = NULL;
-					fconn->conn->owner = NULL;
+				if (eb_is_empty(&fconn->streams_by_id)) {
-					if (eb_is_empty(&fconn->streams_by_id)) {
+					/* let's kill the connection right away */
-						/* let's kill the connection right away */
+					fconn->conn->mux->destroy(fconn);
-						fconn->conn->mux->destroy(fconn);
+					TRACE_DEVEL("outgoing connection killed", FCGI_EV_STRM_END|FCGI_EV_FCONN_ERR);
-						TRACE_DEVEL("outgoing connection killed", FCGI_EV_STRM_END|FCGI_EV_FCONN_ERR);
+					return;
-						return;
-					}
 				}
 			}
-			if (eb_is_empty(&fconn->streams_by_id) && fconn->conn->owner == sess) {
+			if (eb_is_empty(&fconn->streams_by_id)) {
 				if (session_check_idle_conn(fconn->conn->owner, fconn->conn) != 0) {
 					/* The connection is destroyed, let's leave */
 					TRACE_DEVEL("outgoing connection killed", FCGI_EV_STRM_END|FCGI_EV_FCONN_ERR);

src/mux_h1.c

@@ -2453,20 +2453,21 @@ static void h1_detach(struct conn_stream *cs)
 			goto release;
 		}
 
-		if (!(h1c->conn->owner) && (h1c->conn->flags & CO_FL_PRIVATE)) {
+		if (h1c->conn->flags & CO_FL_PRIVATE) {
-			h1c->conn->owner = sess;
+			/* Add the connection in the session server list, if not already done */
 			if (!session_add_conn(sess, h1c->conn, h1c->conn->target)) {
 				h1c->conn->owner = NULL;
 				h1c->conn->mux->destroy(h1c);
 				goto end;
 			}
+			/* Always idle at this step */
 			if (session_check_idle_conn(sess, h1c->conn)) {
 				/* The connection got destroyed, let's leave */
 				TRACE_DEVEL("outgoing connection killed", H1_EV_STRM_END|H1_EV_H1C_END);
 				goto end;
 			}
 		}
-		if (!(h1c->conn->flags & CO_FL_PRIVATE)) {
+		else {
 			if (h1c->conn->owner == sess)
 				h1c->conn->owner = NULL;
 			h1c->conn->xprt->subscribe(h1c->conn, h1c->conn->xprt_ctx, SUB_RETRY_RECV, &h1c->wait_event);

src/mux_h2.c

@@ -3943,18 +3943,16 @@ static void h2_detach(struct conn_stream *cs)
 		if (!(h2c->conn->flags &
 		    (CO_FL_ERROR | CO_FL_SOCK_RD_SH | CO_FL_SOCK_WR_SH))) {
 			if (h2c->conn->flags & CO_FL_PRIVATE) {
-				if (!h2c->conn->owner) {
+				/* Add the connection in the session server list, if not already done */
-					h2c->conn->owner = sess;
+				if (!session_add_conn(sess, h2c->conn, h2c->conn->target)) {
-					if (!session_add_conn(sess, h2c->conn, h2c->conn->target)) {
+					h2c->conn->owner = NULL;
-						h2c->conn->owner = NULL;
+					if (eb_is_empty(&h2c->streams_by_id)) {
-						if (eb_is_empty(&h2c->streams_by_id)) {
+						h2c->conn->mux->destroy(h2c);
-							h2c->conn->mux->destroy(h2c);
+						TRACE_DEVEL("leaving on error after killing outgoing connection", H2_EV_STRM_END|H2_EV_H2C_ERR);
-							TRACE_DEVEL("leaving on error after killing outgoing connection", H2_EV_STRM_END|H2_EV_H2C_ERR);
+						return;
-							return;
-						}
 					}
 				}
-				if (eb_is_empty(&h2c->streams_by_id) && h2c->conn->owner == sess) {
+				if (eb_is_empty(&h2c->streams_by_id)) {
 					if (session_check_idle_conn(h2c->conn->owner, h2c->conn) != 0) {
 						/* At this point either the connection is destroyed, or it's been added to the server idle list, just stop */
 						TRACE_DEVEL("leaving without reusable idle connection", H2_EV_STRM_END);