From 726ab7145c069762116dd7c87537634073cd0abc Mon Sep 17 00:00:00 2001 From: Joseph Lynch Date: Mon, 11 May 2015 23:25:34 -0700 Subject: [PATCH] MEDIUM: backend: Allow redispatch on retry intervals For backend load balancing it sometimes makes sense to redispatch rather than retrying against the same server. For example, when machines or routers fail you may not want to waste time retrying against a dead server and would instead prefer to immediately redispatch against other servers. This patch allows backend sections to specify that they want to redispatch on a particular interval. If the interval N is positive the redispatch occurs on every Nth retry, and if the interval N is negative then the redispatch occurs on the Nth retry prior to the last retry (-1 is the default and maintains backwards compatibility). In low latency environments tuning this setting can save a few hundred milliseconds when backends fail. --- doc/configuration.txt | 18 +++++++++++++++--- include/types/proxy.h | 1 + src/cfgparse.c | 34 +++++++++++++++++++++++++++++++++- src/stream.c | 15 +++++++++++---- 4 files changed, 60 insertions(+), 8 deletions(-) diff --git a/doc/configuration.txt b/doc/configuration.txt index 0d9051c2f..15fd3fbc2 100644 --- a/doc/configuration.txt +++ b/doc/configuration.txt @@ -5352,11 +5352,22 @@ no option prefer-last-server option redispatch +option redispatch no option redispatch Enable or disable session redistribution in case of connection failure May be used in sections: defaults | frontend | listen | backend yes | no | yes | yes - Arguments : none + Arguments : + The optional integer value that controls how often redispatches + occur when retrying connections. Positive value P indicates a + redispatch is desired on every Pth retry, and negative value + N indicate a redispath is desired on the Nth retry prior to the + last retry. For example, the default of -1 preserves the + historical behaviour of redispatching on the last retry, a + positive value of 1 would indicate a redispatch on every retry, + and a positive value of 3 would indicate a redispatch on every + third retry. You can disable redispatches with a value of 0. + In HTTP mode, if a server designated by a cookie is down, clients may definitely stick to it because they cannot flush the cookie, so they will not @@ -5365,7 +5376,7 @@ no option redispatch Specifying "option redispatch" will allow the proxy to break their persistence and redistribute them to a working server. - It also allows to retry last connection to another server in case of multiple + It also allows to retry connections to another server in case of multiple connection failures. Of course, it requires having "retries" set to a nonzero value. @@ -6373,7 +6384,8 @@ retries been established to a server, there will be no more retry. In order to avoid immediate reconnections to a server which is restarting, - a turn-around timer of 1 second is applied before a retry occurs. + a turn-around timer of min("timeout connect", one second) is applied before + a retry occurs. When "option redispatch" is set, the last retry may be performed on another server even if a cookie references a different server. diff --git a/include/types/proxy.h b/include/types/proxy.h index 826a0f2b6..290b02d5c 100644 --- a/include/types/proxy.h +++ b/include/types/proxy.h @@ -325,6 +325,7 @@ struct proxy { char *server_id_hdr_name; /* the header to use to send the server id (name) */ int server_id_hdr_len; /* the length of the id (name) header... name */ int conn_retries; /* maximum number of connect retries */ + int redispatch_after; /* number of retries before redispatch */ unsigned down_trans; /* up-down transitions */ unsigned down_time; /* total time the proxy was down */ time_t last_change; /* last time, when the state was changed */ diff --git a/src/cfgparse.c b/src/cfgparse.c index b96a27199..e08b8750b 100644 --- a/src/cfgparse.c +++ b/src/cfgparse.c @@ -160,7 +160,6 @@ static const struct cfg_opt cfg_opts[] = { "logasap", PR_O_LOGASAP, PR_CAP_FE, 0, 0 }, { "nolinger", PR_O_TCP_NOLING, PR_CAP_FE | PR_CAP_BE, 0, 0 }, { "persist", PR_O_PERSIST, PR_CAP_BE, 0, 0 }, - { "redispatch", PR_O_REDISP, PR_CAP_BE, 0, 0 }, { "srvtcpka", PR_O_TCP_SRV_KA, PR_CAP_BE, 0, 0 }, #ifdef TPROXY { "transparent", PR_O_TRANSP, PR_CAP_BE, 0, 0 }, @@ -1617,6 +1616,7 @@ void init_default_instance() defproxy.state = PR_STNEW; defproxy.maxconn = cfg_maxpconn; defproxy.conn_retries = CONN_RETRIES; + defproxy.redispatch_after = 0; defproxy.defsrv.check.inter = DEF_CHKINTR; defproxy.defsrv.check.fastinter = 0; @@ -2242,6 +2242,7 @@ int cfg_parse_listen(const char *file, int linenum, char **args, int kwm) curproxy->lbprm.algo = defproxy.lbprm.algo; curproxy->fullconn = defproxy.fullconn; curproxy->conn_retries = defproxy.conn_retries; + curproxy->redispatch_after = defproxy.redispatch_after; curproxy->max_ka_queue = defproxy.max_ka_queue; if (defproxy.check_req) { @@ -4094,6 +4095,37 @@ stats_error_parsing: } } + /* Redispatch can take an integer argument that control when the + * resispatch occurs. All values are relative to the retries option. + * This can be cancelled using "no option xxx". + */ + if (strcmp(args[1], "redispatch") == 0) { + if (warnifnotcap(curproxy, PR_CAP_BE, file, linenum, args[1], NULL)) { + err_code |= ERR_WARN; + goto out; + } + + curproxy->no_options &= ~PR_O_REDISP; + curproxy->options &= ~PR_O_REDISP; + + switch (kwm) { + case KWM_STD: + curproxy->options |= PR_O_REDISP; + curproxy->redispatch_after = -1; + if(*args[2]) { + curproxy->redispatch_after = atol(args[2]); + } + break; + case KWM_NO: + curproxy->no_options |= PR_O_REDISP; + curproxy->redispatch_after = 0; + break; + case KWM_DEF: /* already cleared */ + break; + } + goto out; + } + if (kwm != KWM_STD) { Alert("parsing [%s:%d]: negation/default is not supported for option '%s'.\n", file, linenum, args[1]); diff --git a/src/stream.c b/src/stream.c index 23f7f5145..4d62c715f 100644 --- a/src/stream.c +++ b/src/stream.c @@ -624,7 +624,9 @@ static int sess_update_st_cer(struct stream *s) } /* If the "redispatch" option is set on the backend, we are allowed to - * retry on another server for the last retry. In order to achieve this, + * retry on another server. By default this redispatch occurs on the + * last retry, but if configured we allow redispatches to occur on + * configurable intervals, e.g. on every retry. In order to achieve this, * we must mark the stream unassigned, and eventually clear the DIRECT * bit to ignore any persistence cookie. We won't count a retry nor a * redispatch yet, because this will depend on what server is selected. @@ -634,10 +636,15 @@ static int sess_update_st_cer(struct stream *s) * we don't care about this particular server. */ if (objt_server(s->target) && - (si->conn_retries == 0 || + (s->be->options & PR_O_REDISP) && !(s->flags & SF_FORCE_PRST) && + ((((s->be->redispatch_after > 0) && + ((s->be->conn_retries - si->conn_retries) % + s->be->redispatch_after == 0)) || + ((s->be->redispatch_after < 0) && + ((s->be->conn_retries - si->conn_retries) % + (s->be->conn_retries + 1 + s->be->redispatch_after) == 0))) || (!(s->flags & SF_DIRECT) && s->be->srv_act > 1 && - ((s->be->lbprm.algo & BE_LB_KIND) == BE_LB_KIND_RR))) && - s->be->options & PR_O_REDISP && !(s->flags & SF_FORCE_PRST)) { + ((s->be->lbprm.algo & BE_LB_KIND) == BE_LB_KIND_RR)))) { sess_change_server(s, NULL); if (may_dequeue_tasks(objt_server(s->target), s->be)) process_srv_queue(objt_server(s->target));