[MEDIUM] stats: count server retries and redispatches

It is important to know how your installation performs. Haproxy masks
connection errors, which is extremely good for a client but it is bad for
an administrator (except people believing that "ignorance is a bless").

Attached patch adds retries and redispatches counters, so now haproxy:

1. For server:
 - counts retried connections (masked or not)

2. For backends:
 - counts retried connections (masked or not) that happened to
    a slave server
 - counts redispatched connections
 - does not count successfully redispatched connections as backend errors.
    Errors are increased only when client does not get a valid response,
    in other words: with failed redispatch or when this function is not
    enabled.

3. For statistics:
 - display Retr (retries) and Redis (redispatches) as a "Warning"
   information.

include/types/proxy.h

@@ -112,6 +112,7 @@ struct proxy {
 	unsigned int fullconn;			/* #conns on backend above which servers are used at full load */
 	struct in_addr except_net, except_mask; /* don't x-forward-for for this address. FIXME: should support IPv6 */
 	unsigned failed_conns, failed_resp;	/* failed connect() and responses */
+	unsigned retries, redispatches;		/* retried and redispatched connections */
 	unsigned denied_req, denied_resp;	/* blocked requests/responses because of security concerns */
 	unsigned failed_req;			/* failed requests (eg: invalid or timeout) */
 	long long bytes_in;			/* number of bytes transferred from the client to the server */

include/types/server.h

@@ -87,6 +87,7 @@ struct server {
 
 	unsigned failed_checks, down_trans;	/* failed checks and up-down transitions */
 	unsigned failed_conns, failed_resp;	/* failed connect() and responses */
+	unsigned retries;			/* retried connections */
 	unsigned failed_secu;			/* blocked responses because of security concerns */
 	unsigned cum_sess;			/* cumulated number of sessions really sent to this server */
 	long long bytes_in;			/* number of bytes transferred from the client to the server */

src/backend.c

@@ -574,6 +574,10 @@ int srv_count_retry_down(struct session *t, int conn_err)
 {
 	/* we are in front of a retryable error */
 	t->conn_retries--;
+	if (t->srv)
+		t->srv->retries++;
+	t->be->retries++;
+
 	if (t->conn_retries < 0) {
 		/* if not retryable anymore, let's abort */
 		tv_eternity(&t->req->cex);
@@ -644,7 +648,7 @@ int srv_retryable_connect(struct session *t)
 
 	if (t->srv)
 		t->srv->failed_conns++;
-	t->be->failed_conns++;
+	t->be->redispatches++;
 
 	t->flags &= ~(SN_DIRECT | SN_ASSIGNED | SN_ADDR_SET);
 	t->srv = NULL; /* it's left to the dispatcher to choose a server */

src/dumpstats.c

@@ -185,6 +185,7 @@ int stats_dump_raw(struct session *s, struct uri_auth *uri, int flags)
 			     "bin,bout,"
 			     "dreq,dresp,"
 			     "ereq,econ,eresp,"
+			     "wretr,wredis,"
 			     "weight,act,bck,"
 			     "chkfail,chkdown"
 			     "\n");
@@ -369,6 +370,7 @@ int stats_dump_http(struct session *s, struct uri_auth *uri, int flags)
 			     "bin,bout,"
 			     "dreq,dresp,"
 			     "ereq,econ,eresp,"
+			     "wretr,wredis,"
 			     "weight,act,bck,"
 			     "chkfail,chkdown"
 			     "\n");
@@ -589,13 +591,15 @@ int stats_dump_proxy(struct session *s, struct proxy *px, struct uri_auth *uri,
 				     "<th rowspan=2></th>"
 				     "<th colspan=2>Queue</th><th colspan=4>Sessions</th>"
 				     "<th colspan=2>Bytes</th><th colspan=2>Denied</th>"
-				     "<th colspan=3>Errors</th><th colspan=6>Server</th>"
+				     "<th colspan=3>Errors</th><th colspan=2>Warnings</th>"
+				     "<th colspan=6>Server</th>"
 				     "</tr>\n"
 				     "<tr align=\"center\" class=\"titre\">"
 				     "<th>Cur</th><th>Max</th><th>Cur</th><th>Max</th>"
 				     "<th>Limit</th><th>Cumul</th><th>In</th><th>Out</th>"
 				     "<th>Req</th><th>Resp</th><th>Req</th><th>Conn</th>"
-				     "<th>Resp</th><th>Status</th><th>Weight</th><th>Act</th>"
+				     "<th>Resp</th><th>Retr</th><th>Redis</th>"
+				     "<th>Status</th><th>Weight</th><th>Act</th>"
 				     "<th>Bck</th><th>Check</th><th>Down</th></tr>\n"
 				     "",
 				     px->id);
@@ -623,6 +627,8 @@ int stats_dump_proxy(struct session *s, struct proxy *px, struct uri_auth *uri,
 				     "<td align=right>%d</td><td align=right>%d</td>"
 				     /* errors : request, connect, response */
 				     "<td align=right>%d</td><td align=right></td><td align=right></td>"
+				     /* warnings: retries, redispatches */
+				     "<td align=right></td><td align=right></td>"
 				     /* server status : reflect frontend status */
 				     "<td align=center>%s</td>"
 				     /* rest of server: nothing */
@@ -646,6 +652,8 @@ int stats_dump_proxy(struct session *s, struct proxy *px, struct uri_auth *uri,
 				     "%d,%d,"
 				     /* errors : request, connect, response */
 				     "%d,,,"
+				     /* warnings: retries, redispatches */
+				     ",,"
 				     /* server status : reflect frontend status */
 				     "%s,"
 				     /* rest of server: nothing */
@@ -712,6 +720,8 @@ int stats_dump_proxy(struct session *s, struct proxy *px, struct uri_auth *uri,
 				     "<td align=right></td><td align=right>%d</td>"
 				     /* errors : request, connect, response */
 				     "<td align=right></td><td align=right>%d</td><td align=right>%d</td>\n"
+				     /* warnings: retries, redispatches */
+				     "<td align=right>%d</td><td align=right></td>"
 				     "",
 				     (sv->state & SRV_BACKUP) ? "backup" : "active",
 				     sv_state, sv->id,
@@ -719,7 +729,8 @@ int stats_dump_proxy(struct session *s, struct proxy *px, struct uri_auth *uri,
 				     sv->cur_sess, sv->cur_sess_max, sv->maxconn ? ultoa(sv->maxconn) : "-", sv->cum_sess,
 				     sv->bytes_in, sv->bytes_out,
 				     sv->failed_secu,
-				     sv->failed_conns, sv->failed_resp);
+				     sv->failed_conns, sv->failed_resp,
+				     sv->retries);
 				     
 				/* status */
 				chunk_printf(&msg, sizeof(trash), "<td nowrap>");
@@ -762,13 +773,16 @@ int stats_dump_proxy(struct session *s, struct proxy *px, struct uri_auth *uri,
 				     ",%d,"
 				     /* errors : request, connect, response */
 				     ",%d,%d,"
+				     /* warnings: retries, redispatches */
+				     ",%d,"
 				     "",
 				     px->id, sv->id,
 				     sv->nbpend, sv->nbpend_max,
 				     sv->cur_sess, sv->cur_sess_max, sv->maxconn ? ultoa(sv->maxconn) : "-", sv->cum_sess,
 				     sv->bytes_in, sv->bytes_out,
 				     sv->failed_secu,
-				     sv->failed_conns, sv->failed_resp);
+				     sv->failed_conns, sv->failed_resp,
+				     sv->retries);
 				     
 				/* status */
 				chunk_printf(&msg, sizeof(trash),
@@ -831,6 +845,8 @@ int stats_dump_proxy(struct session *s, struct proxy *px, struct uri_auth *uri,
 				     "<td align=right>%d</td><td align=right>%d</td>"
 				     /* errors : request, connect, response */
 				     "<td align=right></td><td align=right>%d</td><td align=right>%d</td>\n"
+				     /* warnings: retries, redispatches */
+				     "<td align=right>%d</td><td align=right>%d</td>"
 				     /* server status : reflect backend status (up/down) : we display UP
 				      * if the backend has known working servers or if it has no server at
 				      * all (eg: for stats). Tthen we display the total weight, number of
@@ -845,6 +861,7 @@ int stats_dump_proxy(struct session *s, struct proxy *px, struct uri_auth *uri,
 				     px->bytes_in, px->bytes_out,
 				     px->denied_req, px->denied_resp,
 				     px->failed_conns, px->failed_resp,
+				     px->retries, px->redispatches,
 				     (px->srv_map_sz > 0 || !px->srv) ? "UP" : "DOWN",
 				     px->srv_map_sz * gcd, px->srv_act, px->srv_bck);
 			} else {
@@ -861,6 +878,8 @@ int stats_dump_proxy(struct session *s, struct proxy *px, struct uri_auth *uri,
 				     "%d,%d,"
 				     /* errors : request, connect, response */
 				     ",%d,%d,"
+				     /* warnings: retries, redispatches */
+				     "%d,%d,"
 				     /* server status : reflect backend status (up/down) : we display UP
 				      * if the backend has known working servers or if it has no server at
 				      * all (eg: for stats). Tthen we display the total weight, number of
@@ -876,6 +895,7 @@ int stats_dump_proxy(struct session *s, struct proxy *px, struct uri_auth *uri,
 				     px->bytes_in, px->bytes_out,
 				     px->denied_req, px->denied_resp,
 				     px->failed_conns, px->failed_resp,
+				     px->retries, px->redispatches,
 				     (px->srv_map_sz > 0 || !px->srv) ? "UP" : "DOWN",
 				     px->srv_map_sz * gcd, px->srv_act, px->srv_bck);
 			}

src/proto_http.c

@@ -2493,7 +2493,7 @@ int process_srv(struct session *t)
 
 				if (t->srv)
 					t->srv->failed_conns++;
-				t->be->failed_conns++;
+				t->be->redispatches++;
 
 				t->flags &= ~(SN_DIRECT | SN_ASSIGNED | SN_ADDR_SET);
 				t->srv = NULL; /* it's left to the dispatcher to choose a server */