From 2df1fbf8161a0dd8f9fbee6fd5c44a2dd8c67e44 Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Mon, 25 Apr 2022 18:02:03 +0200 Subject: [PATCH] MINOR: init: add global setting "fd-hard-limit" to bound system limits On some systems, the hard limit for ulimit -n may be huge, in the order of 1 billion, and using this to automatically compute maxconn doesn't work as it requires way too much memory. Users tend to hard-code maxconn but that's not convenient to manage deployments on heterogenous systems, nor when porting configs to developers' machines. The ulimit-n parameter doesn't work either because it forces the limit. What most users seem to want (and it makes sense) is to respect the system imposed limits up to a certain value and cap this value. This is exactly what fd-hard-limit does. This addresses github issue #1622. --- doc/configuration.txt | 30 ++++++++++++++++++++++++++++-- include/haproxy/global-t.h | 1 + src/cfgparse-global.c | 16 +++++++++++++++- src/haproxy.c | 15 ++++++++++++++- 4 files changed, 58 insertions(+), 4 deletions(-) diff --git a/doc/configuration.txt b/doc/configuration.txt index c1ba462ec..9461b9b90 100644 --- a/doc/configuration.txt +++ b/doc/configuration.txt @@ -1003,6 +1003,7 @@ The following keywords are supported in the "global" section : - deviceatlas-properties-cookie - expose-experimental-directives - external-check + - fd-hard-limit - gid - grace - group @@ -1334,6 +1335,26 @@ external-check See "option external-check", and "insecure-fork-wanted", and "insecure-setuid-wanted". +fd-hard-limit + Sets an upper bound to the maximum number of file descriptors that the + process will use, regardless of system limits. While "ulimit-n" and "maxconn" + may be used to enforce a value, when they are not set, the process will be + limited to the hard limit of the RLIMIT_NOFILE setting as reported by + "ulimit -n -H". But some modern operating systems are now allowing extremely + large values here (in the order of 1 billion), which will consume way too + much RAM for regular usage. The fd-hard-limit setting is provided to enforce + a possibly lower bound to this limit. This means that it will always respect + the system-imposed limits when they are below but the specified + value will be used if system-imposed limits are higher. In the example below, + no other setting is specified and the maxconn value will automatically adapt + to the lower of "fd-hard-limit" and the system-imposed limit: + + global + # use as many FDs as possible but no more than 50000 + fd-hard-limit 50000 + + See also: ulimit-n, maxconn + gid Changes the process's group ID to . It is recommended that the group ID is dedicated to HAProxy or to a small set of similar daemons. HAProxy must @@ -2121,12 +2142,15 @@ uid ulimit-n Sets the maximum number of per-process file-descriptors to . By default, it is automatically computed, so it is recommended not to use this - option. + option. If the intent is only to limit the number of file descriptors, better + use "fd-hard-limit" instead. Note that the dynamic servers are not taken into account in this automatic resource calculation. If using a large number of them, it may be needed to manually specify this value. + See also: fd-hard-limit, maxconn + unix-bind [ prefix ] [ mode ] [ user ] [ uid ] [ group ] [ gid ] @@ -2318,7 +2342,9 @@ maxconn "ulimit -n" command, possibly reduced to a lower value if a memory limit is enforced, based on the buffer size, memory allocated to compression, SSL cache size, and use or not of SSL and the associated maxsslconn (which can - also be automatic). + also be automatic). In any case, the fd-hard-limit applies if set. + + See also: fd-hard-limit, ulimit-n maxconnrate Sets the maximum per-process number of connections per second to . diff --git a/include/haproxy/global-t.h b/include/haproxy/global-t.h index 186968d45..c188cb38c 100644 --- a/include/haproxy/global-t.h +++ b/include/haproxy/global-t.h @@ -123,6 +123,7 @@ struct global { char *pidfile; char *node, *desc; /* node name & description */ int localpeer_cmdline; /* whether or not the commandline "-L" was set */ + int fd_hard_limit; /* hard limit on ulimit-n : 0=unset */ struct buffer log_tag; /* name for syslog */ struct list logsrvs; char *log_send_hostname; /* set hostname in syslog header */ diff --git a/src/cfgparse-global.c b/src/cfgparse-global.c index a9e1b9480..c9b7d6e01 100644 --- a/src/cfgparse-global.c +++ b/src/cfgparse-global.c @@ -711,7 +711,21 @@ int cfg_parse_global(const char *file, int linenum, char **args, int kwm) goto out; } } - + else if (strcmp(args[0], "fd-hard-limit") == 0) { + if (alertif_too_many_args(1, file, linenum, args, &err_code)) + goto out; + if (global.fd_hard_limit != 0) { + ha_alert("parsing [%s:%d] : '%s' already specified. Continuing.\n", file, linenum, args[0]); + err_code |= ERR_ALERT; + goto out; + } + if (*(args[1]) == 0) { + ha_alert("parsing [%s:%d] : '%s' expects an integer argument.\n", file, linenum, args[0]); + err_code |= ERR_ALERT | ERR_FATAL; + goto out; + } + global.fd_hard_limit = atol(args[1]); + } else if (strcmp(args[0], "ulimit-n") == 0) { if (alertif_too_many_args(1, file, linenum, args, &err_code)) goto out; diff --git a/src/haproxy.c b/src/haproxy.c index f61a3abf9..ddb23e2c8 100644 --- a/src/haproxy.c +++ b/src/haproxy.c @@ -1362,6 +1362,9 @@ static int compute_ideal_maxconn() * - two FDs per connection */ + if (global.fd_hard_limit && remain > global.fd_hard_limit) + remain = global.fd_hard_limit; + /* subtract listeners and checks */ remain -= global.maxsock; @@ -1439,6 +1442,9 @@ static int check_if_maxsock_permitted(int maxsock) struct rlimit orig_limit, test_limit; int ret; + if (global.fd_hard_limit && maxsock > global.fd_hard_limit) + return 0; + if (getrlimit(RLIMIT_NOFILE, &orig_limit) != 0) return 1; @@ -3049,8 +3055,12 @@ int main(int argc, char **argv) limit.rlim_cur = global.rlimit_nofile; limit.rlim_max = MAX(rlim_fd_max_at_boot, limit.rlim_cur); - if (setrlimit(RLIMIT_NOFILE, &limit) == -1) { + if ((global.fd_hard_limit && limit.rlim_cur > global.fd_hard_limit) || + setrlimit(RLIMIT_NOFILE, &limit) == -1) { getrlimit(RLIMIT_NOFILE, &limit); + if (global.fd_hard_limit && limit.rlim_cur > global.fd_hard_limit) + limit.rlim_cur = global.fd_hard_limit; + if (global.tune.options & GTUNE_STRICT_LIMITS) { ha_alert("[%s.main()] Cannot raise FD limit to %d, limit is %d.\n", argv[0], global.rlimit_nofile, (int)limit.rlim_cur); @@ -3059,6 +3069,9 @@ int main(int argc, char **argv) else { /* try to set it to the max possible at least */ limit.rlim_cur = limit.rlim_max; + if (global.fd_hard_limit && limit.rlim_cur > global.fd_hard_limit) + limit.rlim_cur = global.fd_hard_limit; + if (setrlimit(RLIMIT_NOFILE, &limit) != -1) getrlimit(RLIMIT_NOFILE, &limit);