diff --git a/include/proto/connection.h b/include/proto/connection.h
index 28e758091..79722d8c8 100644
--- a/include/proto/connection.h
+++ b/include/proto/connection.h
@@ -676,7 +676,8 @@ static inline void conn_free(struct connection *conn)
 	/* Remove ourself from the session's connections list, if any. */
 	if (!LIST_ISEMPTY(&conn->session_list)) {
 		struct session *sess = conn->owner;
-		sess->resp_conns--;
+		if (conn->flags & CO_FL_SESS_IDLE)
+			sess->idle_conns--;
 		session_unown_conn(sess, conn);
 	}
 
diff --git a/include/proto/session.h b/include/proto/session.h
index c34866d3d..d54e9454b 100644
--- a/include/proto/session.h
+++ b/include/proto/session.h
@@ -112,7 +112,6 @@ static inline int session_add_conn(struct session *sess, struct connection *conn
 		LIST_INIT(&srv_list->conn_list);
 		LIST_ADDQ(&sess->srv_list, &srv_list->srv_list);
 	}
-	sess->resp_conns++;
 	LIST_ADDQ(&srv_list->conn_list, &conn->session_list);
 	return 1;
 }
@@ -120,17 +119,19 @@ static inline int session_add_conn(struct session *sess, struct connection *conn
 /* Returns 0 if the session can keep the idle conn, -1 if it was destroyed, or 1 if it was added to the server list */
 static inline int session_check_idle_conn(struct session *sess, struct connection *conn)
 {
-	if (sess->resp_conns > sess->fe->max_out_conns) {
+	if (sess->idle_conns > sess->fe->max_out_conns) {
 		/* We can't keep the connection, let's try to add it to the server idle list */
 		session_unown_conn(sess, conn);
 		conn->owner = NULL;
-		sess->resp_conns--;
 		if (!srv_add_to_idle_list(objt_server(conn->target), conn)) {
 			/* The server doesn't want it, let's kill the connection right away */
 			conn->mux->destroy(conn);
 			return -1;
 		}
 		return 1;
+	} else {
+		conn->flags |= CO_FL_SESS_IDLE;
+		sess->idle_conns++;
 	}
 	return 0;
 }
diff --git a/include/types/connection.h b/include/types/connection.h
index f1fd7a09b..b273ece46 100644
--- a/include/types/connection.h
+++ b/include/types/connection.h
@@ -196,6 +196,8 @@ enum {
 	/* This flag is used to know that a PROXY protocol header was sent by the client */
 	CO_FL_RCVD_PROXY    = 0x20000000,
 
+	/* The connection is unused by its owner */
+	CO_FL_SESS_IDLE     = 0x40000000,
 	/* unused : 0x40000000 */
 
 	/* This last flag indicates that the transport layer is used (for instance
diff --git a/include/types/session.h b/include/types/session.h
index 1675f8f96..328372d6e 100644
--- a/include/types/session.h
+++ b/include/types/session.h
@@ -55,7 +55,7 @@ struct session {
 	struct vars vars;               /* list of variables for the session scope. */
 	struct task *task;              /* handshake timeout processing */
 	long t_handshake;               /* handshake duration, -1 = not completed */
-	int resp_conns;                 /* Number of connections we're currently responsible for */
+	int idle_conns;                 /* Number of connections we're currently responsible for that we are not using */
 	struct list srv_list;           /* List of servers and the connections the session is currently responsible for */
 };
 
diff --git a/src/backend.c b/src/backend.c
index b6f08d5e7..3c1620bbb 100644
--- a/src/backend.c
+++ b/src/backend.c
@@ -1240,6 +1240,13 @@ int connect_server(struct stream *s)
 		LIST_ADDQ(&srv->idle_conns[tid], &srv_conn->list);
 		if (LIST_ISEMPTY(&srv->idle_orphan_conns[tid]))
 			task_unlink_wq(srv->idle_task[tid]);
+	} else if (reuse) {
+		if (srv_conn->flags & CO_FL_SESS_IDLE) {
+			struct session *sess = srv_conn->owner;
+
+			srv_conn->flags &= ~CO_FL_SESS_IDLE;
+			sess->idle_conns--;
+		}
 	}
 
 	/* We're about to use another connection, let the mux know we're
@@ -1251,6 +1258,8 @@ int connect_server(struct stream *s)
 		if (sess) {
 			if (old_conn && !(old_conn->flags & CO_FL_PRIVATE) &&
 			    old_conn->mux != NULL) {
+				if (old_conn->flags & CO_FL_SESS_IDLE)
+					s->sess->idle_conns--;
 				session_unown_conn(s->sess, old_conn);
 				old_conn->owner = sess;
 				if (!session_add_conn(sess, old_conn, s->target)) {
diff --git a/src/session.c b/src/session.c
index 48a5d9a70..9e95b328a 100644
--- a/src/session.c
+++ b/src/session.c
@@ -62,7 +62,7 @@ struct session *session_new(struct proxy *fe, struct listener *li, enum obj_type
 		HA_ATOMIC_ADD(&totalconn, 1);
 		HA_ATOMIC_ADD(&jobs, 1);
 		LIST_INIT(&sess->srv_list);
-		sess->resp_conns = 0;
+		sess->idle_conns = 0;
 	}
 	return sess;
 }