[MEDIUM] Implement "track [<backend>/]<server>"

This patch implements ability to set the current state of one server
by tracking another one. It:
 - adds two variables: *tracknext, *tracked to struct server
 - implements findserver(), similar to findproxy()
 - adds "track" keyword accepting both "proxy/server" and "server" (assuming current proxy)
 - verifies if both checks and tracking is not enabled at the same time
 - changes set_server_down() to notify tracking server
 - creates set_server_up(), set_server_disabled(), set_server_enabled() by
   moving the code from process_chk() and adding notifications
 - changes stats to show a name of tracked server instead of Chk/Dwn/Dwntime(html)
   or by adding new variable (csv)

Changes from the previuos version:
 - it is possibile to track independently of the declaration order
 - one extra comma bug is fixed
 - new condition to check if there is no disable-on-404 inconsistency
This commit is contained in:
Krzysztof Piotr Oledzki 2008-02-18 01:26:35 +01:00 committed by Willy Tarreau
parent f14358bd1a
commit c8b16fc948
8 changed files with 332 additions and 114 deletions

3
TODO
View File

@ -174,9 +174,6 @@ TODO for 1.3
filters and backend, on which every entity could rely.
- implement 'on uri <uri> <proxy>', 'on host <host> <proxy>'
- remove the first now useless hop in hdr_idx
- implement "track XXX.YYY" for each server as an alternative to
health checks. This will automatically set the server state to
the same as server YYY of proxy XXX.
- balance on URI hash (specify length or depth)
- balance on any header hash (eg: host)
- balance with redirections to real servers

View File

@ -3893,6 +3893,13 @@ source <addr>[:<port>] [usesrc { <addr2>[:<port2>] | client | clientip } ]
as the backend "source" keyword, except that it only applies to the server
referencing it. Please consult the "source" keyword for details.
track [<proxy>/]<server>
This option enables ability to set the current state of the server by
tracking another one. Only a server with checks enabled can be tracked
so it is not possible for example to track a server that tracks another
one. If <proxy> is omitted the current one is used. If disable-on-404 is
used, it has to be enabled on both proxies.
weight <weight>
The "weight" parameter is used to adjust the server's weight relative to
other servers. All servers will receive a load proportional to their weight

View File

@ -36,6 +36,7 @@ void listen_proxies(void);
const char *proxy_cap_str(int cap);
const char *proxy_mode_str(int mode);
struct proxy *findproxy(const char *name, int mode, int cap);
struct server *findserver(const struct proxy *px, const char *name);
int proxy_parse_timeout(const char **args, struct proxy *proxy,
struct proxy *defpx, char *err, int errlen);

View File

@ -91,6 +91,8 @@ struct server {
struct sockaddr_in tproxy_addr; /* non-local address we want to bind to for connect() */
#endif
struct server *tracknext, *tracked; /* next server in a tracking list, tracked server */
char *trackit; /* temporary variable to make assignment deferrable */
struct sockaddr_in check_addr; /* the address to check, if different from <addr> */
short check_port; /* the port to use for the health checks */
int health; /* 0->rise-1 = bad; rise->rise+fall-1 = good */

View File

@ -1625,6 +1625,18 @@ int cfg_parse_listen(const char *file, int linenum, char **args, int inv)
newsrv->slowstart = (val + 999) / 1000;
cur_arg += 2;
}
else if (!strcmp(args[cur_arg], "track")) {
if (!*args[cur_arg + 1]) {
Alert("parsing [%s:%d]: 'track' expects [<proxy>/]<server> as argument.\n",
file, linenum);
return -1;
}
newsrv->trackit = strdup(args[cur_arg + 1]);
cur_arg += 2;
}
else if (!strcmp(args[cur_arg], "check")) {
global.maxsock++;
do_check = 1;
@ -1684,13 +1696,19 @@ int cfg_parse_listen(const char *file, int linenum, char **args, int inv)
return -1;
}
else {
Alert("parsing [%s:%d] : server %s only supports options 'backup', 'cookie', 'redir', 'check', 'inter', 'fastinter', 'downinter', 'rise', 'fall', 'addr', 'port', 'source', 'minconn', 'maxconn', 'maxqueue', 'slowstart' and 'weight'.\n",
Alert("parsing [%s:%d] : server %s only supports options 'backup', 'cookie', 'redir', 'check', 'track', 'inter', 'fastinter', 'downinter', 'rise', 'fall', 'addr', 'port', 'source', 'minconn', 'maxconn', 'maxqueue', 'slowstart' and 'weight'.\n",
file, linenum, newsrv->id);
return -1;
}
}
if (do_check) {
if (newsrv->trackit) {
Alert("parsing [%s:%d]: unable to enable checks and tracking at the same time!\n",
file, linenum);
return -1;
}
if (!newsrv->check_port && newsrv->check_addr.sin_port)
newsrv->check_port = newsrv->check_addr.sin_port;
@ -2913,6 +2931,7 @@ int readcfgfile(const char *file)
/*
* If this server supports a maxconn parameter, it needs a dedicated
* tasks to fill the emptied slots when a connection leaves.
* Also, resolve deferred tracking dependency if needed.
*/
newsrv = curproxy->srv;
while (newsrv != NULL) {
@ -2950,6 +2969,65 @@ int readcfgfile(const char *file)
tv_eternity(&t->expire);
task_queue(t);
}
if (newsrv->trackit) {
struct proxy *px;
struct server *srv;
char *pname, *sname;
pname = newsrv->trackit;
sname = strrchr(pname, '/');
if (sname)
*sname++ = '\0';
else {
sname = pname;
pname = NULL;
}
if (pname) {
px = findproxy(pname, curproxy->mode, PR_CAP_BE);
if (!px) {
Alert("parsing %s, %s '%s', server '%s': unable to find required proxy '%s' for tracking.\n",
file, proxy_type_str(curproxy), curproxy->id,
newsrv->id, pname);
return -1;
}
} else
px = curproxy;
srv = findserver(px, sname);
if (!srv) {
Alert("parsing %s, %s '%s', server '%s': unable to find required server '%s' for tracking.\n",
file, proxy_type_str(curproxy), curproxy->id,
newsrv->id, sname);
return -1;
}
if (!(srv->state & SRV_CHECKED)) {
Alert("parsing %s, %s '%s', server '%s': unable to use %s/%s for "
"tracing as it does not have checks enabled.\n",
file, proxy_type_str(curproxy), curproxy->id,
newsrv->id, px->id, srv->id);
return -1;
}
if (curproxy != px &&
(curproxy->options & PR_O_DISABLE404) != (px->options & PR_O_DISABLE404)) {
Alert("parsing %s, %s '%s', server '%s': unable to use %s/%s for"
"tracing: disable-on-404 option inconsistency.\n",
file, proxy_type_str(curproxy), curproxy->id,
newsrv->id, px->id, srv->id);
return -1;
}
newsrv->tracked = srv;
newsrv->tracknext = srv->tracknext;
srv->tracknext = newsrv;
free(newsrv->trackit);
}
newsrv = newsrv->next;
}

View File

@ -2,6 +2,7 @@
* Health-checks functions.
*
* Copyright 2000-2008 Willy Tarreau <w@1wt.eu>
* Copyright 2007-2008 Krzysztof Piotr Oledzki <ole@ans.pl>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
@ -34,6 +35,7 @@
#include <types/session.h>
#include <proto/backend.h>
#include <proto/buffers.h>
#include <proto/fd.h>
#include <proto/log.h>
#include <proto/queue.h>
@ -120,9 +122,11 @@ static int check_for_pending(struct server *s)
*/
static void set_server_down(struct server *s)
{
struct server *srv;
struct chunk msg;
int xferred;
if (s->health == s->rise) {
if (s->health == s->rise || s->tracked) {
int srv_was_paused = s->state & SRV_GOINGDOWN;
s->last_change = now.tv_sec;
@ -134,10 +138,21 @@ static void set_server_down(struct server *s)
* to another server or to the proxy itself.
*/
xferred = redistribute_pending(s);
sprintf(trash, "%sServer %s/%s is DOWN. %d active and %d backup servers left.%s"
msg.len = 0;
msg.str = trash;
chunk_printf(&msg, sizeof(trash),
"%sServer %s/%s is DOWN", s->state & SRV_BACKUP ? "Backup " : "",
s->proxy->id, s->id);
if (s->tracked)
chunk_printf(&msg, sizeof(trash), " via %s/%s",
s->tracked->proxy->id, s->tracked->id);
chunk_printf(&msg, sizeof(trash), ". %d active and %d backup servers left.%s"
" %d sessions active, %d requeued, %d remaining in queue.\n",
s->state & SRV_BACKUP ? "Backup " : "",
s->proxy->id, s->id, s->proxy->srv_act, s->proxy->srv_bck,
s->proxy->srv_act, s->proxy->srv_bck,
(s->proxy->srv_bck && !s->proxy->srv_act) ? " Running on backup." : "",
s->cur_sess, xferred, s->nbpend);
@ -153,10 +168,167 @@ static void set_server_down(struct server *s)
set_backend_down(s->proxy);
s->down_trans++;
if (s->state && SRV_CHECKED)
for(srv = s->tracknext; srv; srv = srv->tracknext)
set_server_down(srv);
}
s->health = 0; /* failure */
}
static void set_server_up(struct server *s) {
struct server *srv;
struct chunk msg;
int xferred;
if (s->health == s->rise || s->tracked) {
if (s->proxy->srv_bck == 0 && s->proxy->srv_act == 0) {
if (s->proxy->last_change < now.tv_sec) // ignore negative times
s->proxy->down_time += now.tv_sec - s->proxy->last_change;
s->proxy->last_change = now.tv_sec;
}
if (s->last_change < now.tv_sec) // ignore negative times
s->down_time += now.tv_sec - s->last_change;
s->last_change = now.tv_sec;
s->state |= SRV_RUNNING;
if (s->slowstart > 0) {
s->state |= SRV_WARMINGUP;
if (s->proxy->lbprm.algo & BE_LB_PROP_DYN) {
/* For dynamic algorithms, start at the first step of the weight,
* without multiplying by BE_WEIGHT_SCALE.
*/
s->eweight = s->uweight;
if (s->proxy->lbprm.update_server_eweight)
s->proxy->lbprm.update_server_eweight(s);
}
}
s->proxy->lbprm.set_server_status_up(s);
/* check if we can handle some connections queued at the proxy. We
* will take as many as we can handle.
*/
xferred = check_for_pending(s);
msg.len = 0;
msg.str = trash;
chunk_printf(&msg, sizeof(trash),
"%sServer %s/%s is UP", s->state & SRV_BACKUP ? "Backup " : "",
s->proxy->id, s->id);
if (s->tracked)
chunk_printf(&msg, sizeof(trash), " via %s/%s",
s->tracked->proxy->id, s->tracked->id);
chunk_printf(&msg, sizeof(trash), ". %d active and %d backup servers online.%s"
" %d sessions requeued, %d total in queue.\n",
s->proxy->srv_act, s->proxy->srv_bck,
(s->proxy->srv_bck && !s->proxy->srv_act) ? " Running on backup." : "",
s->cur_sess, xferred, s->nbpend);
Warning("%s", trash);
send_log(s->proxy, LOG_NOTICE, "%s", trash);
if (s->state && SRV_CHECKED)
for(srv = s->tracknext; srv; srv = srv->tracknext)
set_server_up(srv);
}
if (s->health >= s->rise)
s->health = s->rise + s->fall - 1; /* OK now */
}
static void set_server_disabled(struct server *s) {
struct server *srv;
struct chunk msg;
int xferred;
s->state |= SRV_GOINGDOWN;
s->proxy->lbprm.set_server_status_down(s);
/* we might have sessions queued on this server and waiting for
* a connection. Those which are redispatchable will be queued
* to another server or to the proxy itself.
*/
xferred = redistribute_pending(s);
msg.len = 0;
msg.str = trash;
chunk_printf(&msg, sizeof(trash),
"Load-balancing on %sServer %s/%s is disabled",
s->state & SRV_BACKUP ? "Backup " : "",
s->proxy->id, s->id);
if (s->tracked)
chunk_printf(&msg, sizeof(trash), " via %s/%s",
s->tracked->proxy->id, s->tracked->id);
chunk_printf(&msg, sizeof(trash),". %d active and %d backup servers online.%s"
" %d sessions requeued, %d total in queue.\n",
s->proxy->srv_act, s->proxy->srv_bck,
(s->proxy->srv_bck && !s->proxy->srv_act) ? " Running on backup." : "",
xferred, s->nbpend);
Warning("%s", trash);
send_log(s->proxy, LOG_NOTICE, "%s", trash);
if (!s->proxy->srv_bck && !s->proxy->srv_act)
set_backend_down(s->proxy);
if (s->state && SRV_CHECKED)
for(srv = s->tracknext; srv; srv = srv->tracknext)
set_server_disabled(srv);
}
static void set_server_enabled(struct server *s) {
struct server *srv;
struct chunk msg;
int xferred;
s->state &= ~SRV_GOINGDOWN;
s->proxy->lbprm.set_server_status_up(s);
/* check if we can handle some connections queued at the proxy. We
* will take as many as we can handle.
*/
xferred = check_for_pending(s);
msg.len = 0;
msg.str = trash;
chunk_printf(&msg, sizeof(trash),
"Load-balancing on %sServer %s/%s is enabled again",
s->state & SRV_BACKUP ? "Backup " : "",
s->proxy->id, s->id);
if (s->tracked)
chunk_printf(&msg, sizeof(trash), " via %s/%s",
s->tracked->proxy->id, s->tracked->id);
chunk_printf(&msg, sizeof(trash), ". %d active and %d backup servers online.%s"
" %d sessions requeued, %d total in queue.\n",
s->proxy->srv_act, s->proxy->srv_bck,
(s->proxy->srv_bck && !s->proxy->srv_act) ? " Running on backup." : "",
xferred, s->nbpend);
Warning("%s", trash);
send_log(s->proxy, LOG_NOTICE, "%s", trash);
if (s->state && SRV_CHECKED)
for(srv = s->tracknext; srv; srv = srv->tracknext)
set_server_enabled(srv);
}
/*
* This function is used only for server health-checks. It handles
@ -364,7 +536,6 @@ void process_chk(struct task *t, struct timeval *next)
__label__ new_chk, out;
struct server *s = t->context;
struct sockaddr_in sa;
int xferred;
int fd;
int rv;
@ -575,103 +746,18 @@ void process_chk(struct task *t, struct timeval *next)
/* we may have to add/remove this server from the LB group */
if ((s->state & SRV_RUNNING) && (s->proxy->options & PR_O_DISABLE404)) {
if ((s->state & SRV_GOINGDOWN) &&
((s->result & (SRV_CHK_RUNNING|SRV_CHK_DISABLE)) == SRV_CHK_RUNNING)) {
/* server enabled again */
s->state &= ~SRV_GOINGDOWN;
s->proxy->lbprm.set_server_status_up(s);
/* check if we can handle some connections queued at the proxy. We
* will take as many as we can handle.
*/
xferred = check_for_pending(s);
sprintf(trash,
"Load-balancing on %sServer %s/%s is enabled again. %d active and %d backup servers online.%s"
" %d sessions requeued, %d total in queue.\n",
s->state & SRV_BACKUP ? "Backup " : "",
s->proxy->id, s->id, s->proxy->srv_act, s->proxy->srv_bck,
(s->proxy->srv_bck && !s->proxy->srv_act) ? " Running on backup." : "",
xferred, s->nbpend);
Warning("%s", trash);
send_log(s->proxy, LOG_NOTICE, "%s", trash);
}
((s->result & (SRV_CHK_RUNNING|SRV_CHK_DISABLE)) == SRV_CHK_RUNNING))
set_server_enabled(s);
else if (!(s->state & SRV_GOINGDOWN) &&
((s->result & (SRV_CHK_RUNNING | SRV_CHK_DISABLE)) ==
(SRV_CHK_RUNNING | SRV_CHK_DISABLE))) {
/* server disabled */
s->state |= SRV_GOINGDOWN;
s->proxy->lbprm.set_server_status_down(s);
/* we might have sessions queued on this server and waiting for
* a connection. Those which are redispatchable will be queued
* to another server or to the proxy itself.
*/
xferred = redistribute_pending(s);
sprintf(trash,
"Load-balancing on %sServer %s/%s is disabled. %d active and %d backup servers online.%s"
" %d sessions requeued, %d total in queue.\n",
s->state & SRV_BACKUP ? "Backup " : "",
s->proxy->id, s->id, s->proxy->srv_act, s->proxy->srv_bck,
(s->proxy->srv_bck && !s->proxy->srv_act) ? " Running on backup." : "",
xferred, s->nbpend);
Warning("%s", trash);
send_log(s->proxy, LOG_NOTICE, "%s", trash);
if (!s->proxy->srv_bck && !s->proxy->srv_act)
set_backend_down(s->proxy);
}
(SRV_CHK_RUNNING | SRV_CHK_DISABLE)))
set_server_disabled(s);
}
if (s->health < s->rise + s->fall - 1) {
s->health++; /* was bad, stays for a while */
if (s->health == s->rise) {
if (s->proxy->srv_bck == 0 && s->proxy->srv_act == 0) {
if (s->proxy->last_change < now.tv_sec) // ignore negative times
s->proxy->down_time += now.tv_sec - s->proxy->last_change;
s->proxy->last_change = now.tv_sec;
}
if (s->last_change < now.tv_sec) // ignore negative times
s->down_time += now.tv_sec - s->last_change;
s->last_change = now.tv_sec;
s->state |= SRV_RUNNING;
if (s->slowstart > 0) {
s->state |= SRV_WARMINGUP;
if (s->proxy->lbprm.algo & BE_LB_PROP_DYN) {
/* For dynamic algorithms, start at the first step of the weight,
* without multiplying by BE_WEIGHT_SCALE.
*/
s->eweight = s->uweight;
if (s->proxy->lbprm.update_server_eweight)
s->proxy->lbprm.update_server_eweight(s);
}
}
s->proxy->lbprm.set_server_status_up(s);
/* check if we can handle some connections queued at the proxy. We
* will take as many as we can handle.
*/
xferred = check_for_pending(s);
sprintf(trash,
"%sServer %s/%s is UP. %d active and %d backup servers online.%s"
" %d sessions requeued, %d total in queue.\n",
s->state & SRV_BACKUP ? "Backup " : "",
s->proxy->id, s->id, s->proxy->srv_act, s->proxy->srv_bck,
(s->proxy->srv_bck && !s->proxy->srv_act) ? " Running on backup." : "",
xferred, s->nbpend);
Warning("%s", trash);
send_log(s->proxy, LOG_NOTICE, "%s", trash);
}
if (s->health >= s->rise)
s->health = s->rise + s->fall - 1; /* OK now */
set_server_up(s);
}
s->curfd = -1; /* no check running anymore */
fd_delete(fd);

View File

@ -171,7 +171,7 @@ int print_csv_header(struct chunk *msg, int size)
"wretr,wredis,"
"status,weight,act,bck,"
"chkfail,chkdown,lastchg,downtime,qlimit,"
"pid,iid,sid,throttle,lbtot,"
"pid,iid,sid,throttle,lbtot,tracked,"
"\n");
}
@ -587,7 +587,7 @@ int stats_dump_http(struct session *s, struct uri_auth *uri, int flags)
int stats_dump_proxy(struct session *s, struct proxy *px, struct uri_auth *uri, int flags)
{
struct buffer *rep = s->rep;
struct server *sv;
struct server *sv, *svs; /* server and server-state, server-state=server or server->tracked */
struct chunk msg;
msg.len = 0;
@ -706,8 +706,8 @@ int stats_dump_proxy(struct session *s, struct proxy *px, struct uri_auth *uri,
"%s,"
/* rest of server: nothing */
",,,,,,,,"
/* pid, iid, sid, throttle, lbtot, */
"%d,%d,0,,,"
/* pid, iid, sid, throttle, lbtot, tracked*/
"%d,%d,0,,,,"
"\n",
px->id,
px->feconn, px->feconn_max, px->maxconn, px->cum_feconn,
@ -734,20 +734,25 @@ int stats_dump_proxy(struct session *s, struct proxy *px, struct uri_auth *uri,
sv = s->data_ctx.stats.sv;
if (sv->tracked)
svs = sv->tracked;
else
svs = sv;
/* FIXME: produce some small strings for "UP/DOWN x/y &#xxxx;" */
if (!(sv->state & SRV_CHECKED))
if (!(svs->state & SRV_CHECKED))
sv_state = 6;
else if (sv->state & SRV_RUNNING) {
if (sv->health == sv->rise + sv->fall - 1)
else if (svs->state & SRV_RUNNING) {
if (svs->health == svs->rise + svs->fall - 1)
sv_state = 3; /* UP */
else
sv_state = 2; /* going down */
if (sv->state & SRV_GOINGDOWN)
if (svs->state & SRV_GOINGDOWN)
sv_state += 2;
}
else
if (sv->health)
if (svs->health)
sv_state = 1; /* going up */
else
sv_state = 0; /* DOWN */
@ -800,8 +805,8 @@ int stats_dump_proxy(struct session *s, struct proxy *px, struct uri_auth *uri,
chunk_printf(&msg, sizeof(trash),
srv_hlt_st[sv_state],
(sv->state & SRV_RUNNING) ? (sv->health - sv->rise + 1) : (sv->health),
(sv->state & SRV_RUNNING) ? (sv->fall) : (sv->rise));
(svs->state & SRV_RUNNING) ? (svs->health - svs->rise + 1) : (svs->health),
(svs->state & SRV_RUNNING) ? (svs->fall) : (svs->rise));
chunk_printf(&msg, sizeof(trash),
/* weight */
@ -819,8 +824,11 @@ int stats_dump_proxy(struct session *s, struct proxy *px, struct uri_auth *uri,
"<td align=right>%d</td><td align=right>%d</td>"
"<td nowrap align=right>%s</td>"
"",
sv->failed_checks, sv->down_trans,
svs->failed_checks, svs->down_trans,
human_time(srv_downtime(sv), 1));
else if (sv != svs)
chunk_printf(&msg, sizeof(trash),
"<td nowrap colspan=3>via %s/%s</td>", svs->proxy->id, svs->id );
else
chunk_printf(&msg, sizeof(trash),
"<td colspan=3></td>");
@ -908,6 +916,14 @@ int stats_dump_proxy(struct session *s, struct proxy *px, struct uri_auth *uri,
/* sessions: lbtot */
chunk_printf(&msg, sizeof(trash), ",%d", sv->cum_lbconn);
/* tracked */
if (sv->tracked)
chunk_printf(&msg, sizeof(trash), ",%s/%s",
sv->tracked->proxy->id, sv->tracked->id);
else
chunk_printf(&msg, sizeof(trash), ",");
/* ',' then EOL */
chunk_printf(&msg, sizeof(trash), ",\n");
}
@ -991,8 +1007,8 @@ int stats_dump_proxy(struct session *s, struct proxy *px, struct uri_auth *uri,
"%d,%d,%d,"
/* rest of backend: nothing, down transitions, last change, total downtime */
",%d,%d,%d,,"
/* pid, iid, sid, throttle, lbtot, */
"%d,%d,0,,%d,"
/* pid, iid, sid, throttle, lbtot, tracked,*/
"%d,%d,0,,%d,,"
"\n",
px->id,
px->nbpend /* or px->totpend ? */, px->nbpend_max,

View File

@ -176,7 +176,7 @@ int proxy_parse_timeout(const char **args, struct proxy *proxy,
struct proxy *findproxy(const char *name, int mode, int cap) {
struct proxy *curproxy, *target=NULL;
struct proxy *curproxy, *target = NULL;
for (curproxy = proxy; curproxy; curproxy = curproxy->next) {
if ((curproxy->cap & cap)!=cap || strcmp(curproxy->id, name))
@ -203,6 +203,37 @@ struct proxy *findproxy(const char *name, int mode, int cap) {
return target;
}
/*
* This function finds a server with matching name within selected proxy.
* It also checks if there are more matching servers with
* requested name as this often leads into unexpected situations.
*/
struct server *findserver(const struct proxy *px, const char *name) {
struct server *cursrv, *target = NULL;
if (!px)
return NULL;
for (cursrv = px->srv; cursrv; cursrv = cursrv->next) {
if (strcmp(cursrv->id, name))
continue;
if (!target) {
target = cursrv;
continue;
}
Alert("Refusing to use duplicated server '%s' fould in proxy: %s!\n",
name, px->id);
return NULL;
}
return target;
}
/*
* This function creates all proxy sockets. It should be done very early,
* typically before privileges are dropped. The sockets will be registered