MEDIUM: backend: Allow redispatch on retry intervals

For backend load balancing it sometimes makes sense to redispatch rather
than retrying against the same server. For example, when machines or routers
fail you may not want to waste time retrying against a dead server and
would instead prefer to immediately redispatch against other servers.

This patch allows backend sections to specify that they want to
redispatch on a particular interval. If the interval N is positive the
redispatch occurs on every Nth retry, and if the interval N is negative then
the redispatch occurs on the Nth retry prior to the last retry (-1 is the
default and maintains backwards compatibility). In low latency environments
tuning this setting can save a few hundred milliseconds when backends fail.

doc/configuration.txt

@@ -5352,11 +5352,22 @@ no option prefer-last-server
 
 
 option redispatch
+option redispatch <interval>
 no option redispatch
   Enable or disable session redistribution in case of connection failure
   May be used in sections:    defaults | frontend | listen | backend
                                  yes   |    no    |   yes  |   yes
-  Arguments : none
+  Arguments :
+    <interval> The optional integer value that controls how often redispatches
+               occur when retrying connections. Positive value P indicates a
+               redispatch is desired on every Pth retry, and negative value
+               N indicate a redispath is desired on the Nth retry prior to the
+               last retry. For example, the default of -1 preserves the
+               historical behaviour of redispatching on the last retry, a
+               positive value of 1 would indicate a redispatch on every retry,
+               and a positive value of 3 would indicate a redispatch on every
+               third retry. You can disable redispatches with a value of 0.
+
 
   In HTTP mode, if a server designated by a cookie is down, clients may
   definitely stick to it because they cannot flush the cookie, so they will not
@@ -5365,7 +5376,7 @@ no option redispatch
   Specifying "option redispatch" will allow the proxy to break their
   persistence and redistribute them to a working server.
 
-  It also allows to retry last connection to another server in case of multiple
+  It also allows to retry connections to another server in case of multiple
   connection failures. Of course, it requires having "retries" set to a nonzero
   value.
 
@@ -6373,7 +6384,8 @@ retries <value>
   been established to a server, there will be no more retry.
 
   In order to avoid immediate reconnections to a server which is restarting,
-  a turn-around timer of 1 second is applied before a retry occurs.
+  a turn-around timer of min("timeout connect", one second) is applied before
+  a retry occurs.
 
   When "option redispatch" is set, the last retry may be performed on another
   server even if a cookie references a different server.

include/types/proxy.h

@@ -325,6 +325,7 @@ struct proxy {
 	char *server_id_hdr_name;                   /* the header to use to send the server id (name) */
 	int server_id_hdr_len;                      /* the length of the id (name) header... name */
 	int conn_retries;			/* maximum number of connect retries */
+	int redispatch_after;			/* number of retries before redispatch */
 	unsigned down_trans;			/* up-down transitions */
 	unsigned down_time;			/* total time the proxy was down */
 	time_t last_change;			/* last time, when the state was changed */

src/cfgparse.c

@@ -160,7 +160,6 @@ static const struct cfg_opt cfg_opts[] =
 	{ "logasap",      PR_O_LOGASAP,    PR_CAP_FE, 0, 0 },
 	{ "nolinger",     PR_O_TCP_NOLING, PR_CAP_FE | PR_CAP_BE, 0, 0 },
 	{ "persist",      PR_O_PERSIST,    PR_CAP_BE, 0, 0 },
-	{ "redispatch",   PR_O_REDISP,     PR_CAP_BE, 0, 0 },
 	{ "srvtcpka",     PR_O_TCP_SRV_KA, PR_CAP_BE, 0, 0 },
 #ifdef TPROXY
 	{ "transparent",  PR_O_TRANSP,     PR_CAP_BE, 0, 0 },
@@ -1617,6 +1616,7 @@ void init_default_instance()
 	defproxy.state = PR_STNEW;
 	defproxy.maxconn = cfg_maxpconn;
 	defproxy.conn_retries = CONN_RETRIES;
+	defproxy.redispatch_after = 0;
 
 	defproxy.defsrv.check.inter = DEF_CHKINTR;
 	defproxy.defsrv.check.fastinter = 0;
@@ -2242,6 +2242,7 @@ int cfg_parse_listen(const char *file, int linenum, char **args, int kwm)
 			curproxy->lbprm.algo = defproxy.lbprm.algo;
 			curproxy->fullconn = defproxy.fullconn;
 			curproxy->conn_retries = defproxy.conn_retries;
+			curproxy->redispatch_after = defproxy.redispatch_after;
 			curproxy->max_ka_queue = defproxy.max_ka_queue;
 
 			if (defproxy.check_req) {
@@ -4094,6 +4095,37 @@ stats_error_parsing:
 			}
 		}
 
+		/* Redispatch can take an integer argument that control when the
+		 * resispatch occurs. All values are relative to the retries option.
+		 * This can be cancelled using "no option xxx".
+		 */
+		if (strcmp(args[1], "redispatch") == 0) {
+			if (warnifnotcap(curproxy, PR_CAP_BE, file, linenum, args[1], NULL)) {
+				err_code |= ERR_WARN;
+				goto out;
+			}
+
+			curproxy->no_options &= ~PR_O_REDISP;
+			curproxy->options &= ~PR_O_REDISP;
+
+			switch (kwm) {
+			case KWM_STD:
+				curproxy->options |= PR_O_REDISP;
+				curproxy->redispatch_after = -1;
+				if(*args[2]) {
+					curproxy->redispatch_after = atol(args[2]);
+				}
+				break;
+			case KWM_NO:
+				curproxy->no_options |= PR_O_REDISP;
+				curproxy->redispatch_after = 0;
+				break;
+			case KWM_DEF: /* already cleared */
+				break;
+			}
+			goto out;
+		}
+
 		if (kwm != KWM_STD) {
 			Alert("parsing [%s:%d]: negation/default is not supported for option '%s'.\n",
 				file, linenum, args[1]);

src/stream.c

@@ -624,7 +624,9 @@ static int sess_update_st_cer(struct stream *s)
 	}
 
 	/* If the "redispatch" option is set on the backend, we are allowed to
-	 * retry on another server for the last retry. In order to achieve this,
+	 * retry on another server. By default this redispatch occurs on the
+	 * last retry, but if configured we allow redispatches to occur on
+	 * configurable intervals, e.g. on every retry. In order to achieve this,
 	 * we must mark the stream unassigned, and eventually clear the DIRECT
 	 * bit to ignore any persistence cookie. We won't count a retry nor a
 	 * redispatch yet, because this will depend on what server is selected.
@@ -634,10 +636,15 @@ static int sess_update_st_cer(struct stream *s)
 	 * we don't care about this particular server.
 	 */
 	if (objt_server(s->target) &&
-	    (si->conn_retries == 0 ||
+	    (s->be->options & PR_O_REDISP) && !(s->flags & SF_FORCE_PRST) &&
+	    ((((s->be->redispatch_after > 0) &&
+	       ((s->be->conn_retries - si->conn_retries) %
+	        s->be->redispatch_after == 0)) ||
+	      ((s->be->redispatch_after < 0) &&
+	       ((s->be->conn_retries - si->conn_retries) %
+	        (s->be->conn_retries + 1 + s->be->redispatch_after) == 0))) ||
 	     (!(s->flags & SF_DIRECT) && s->be->srv_act > 1 &&
-	      ((s->be->lbprm.algo & BE_LB_KIND) == BE_LB_KIND_RR))) &&
-	    s->be->options & PR_O_REDISP && !(s->flags & SF_FORCE_PRST)) {
+	      ((s->be->lbprm.algo & BE_LB_KIND) == BE_LB_KIND_RR)))) {
 		sess_change_server(s, NULL);
 		if (may_dequeue_tasks(objt_server(s->target), s->be))
 			process_srv_queue(objt_server(s->target));