MAJOR: threads: enable one thread per CPU by default

Threads have long matured by now, still for most users their usage is
not trivial. It's about time to enable them by default on platforms
where we know the number of CPUs bound. This patch does this, it counts
the number of CPUs the process is bound to upon startup, and enables as
many threads by default. Of course, "nbthread" still overrides this, but
if it's not set the default behaviour is to start one thread per CPU.

The default number of threads is reported in "haproxy -vv". Simply using
"taskset -c" is now enough to adjust this number of threads so that there
is no more need for playing with cpu-map. And thanks to the previous
patches on the listener, the vast majority of configurations will not
need to duplicate "bind" lines with the "process x/y" statement anymore
either, so a simple config will automatically adapt to the number of
processors available.

doc/configuration.txt

@@ -957,7 +957,8 @@ nbproc <number>
   Creates <number> processes when going daemon. This requires the "daemon"
   mode. By default, only one process is created, which is the recommended mode
   of operation. For systems limited to small sets of file descriptors per
-  process, it may be needed to fork multiple daemons. USING MULTIPLE PROCESSES
+  process, it may be needed to fork multiple daemons. When set to a value
+  larger than 1, threads are automatically disabled. USING MULTIPLE PROCESSES
   IS HARDER TO DEBUG AND IS REALLY DISCOURAGED. See also "daemon" and
   "nbthread".
 
@@ -968,7 +969,13 @@ nbthread <number>
   also involved a number of shortcomings related to the lack of synchronization
   between processes (health-checks, peers, stick-tables, stats, ...) which do
   not affect threads. As such, any modern configuration is strongly encouraged
-  to migrate away from "nbproc" to "nbthread". See also "nbproc".
+  to migrate away from "nbproc" to "nbthread". "nbthread" also works when
+  HAProxy is started in foreground. On some platforms supporting CPU affinity,
+  when nbproc is not used, the default "nbthread" value is automatically set to
+  the number of CPUs the process is bound to upon startup. This means that the
+  thread count can easily be adjusted from the calling process using commands
+  like "taskset" or "cpuset". Otherwise, this value defaults to 1. The default
+  value is reported in the output of "haproxy -vv". See also "nbproc".
 
 pidfile <pidfile>
   Writes PIDs of all daemons into file <pidfile>. This option is equivalent to

include/common/hathreads.h

@@ -984,11 +984,14 @@ void ha_rwlock_init(HA_RWLOCK_T *l);
 
 #endif /* USE_THREAD */
 
+extern int thread_cpus_enabled_at_boot;
+
 static inline void __ha_compiler_barrier(void)
 {
 	__asm __volatile("" ::: "memory");
 }
 
 int parse_nbthread(const char *arg, char **err);
+int thread_get_default_count();
 
 #endif /* _COMMON_HATHREADS_H */

src/cfgparse.c

@@ -2200,6 +2200,20 @@ int check_config_validity()
 	if (!global.tune.requri_len)
 		global.tune.requri_len = REQURI_LEN;
 
+	if (!global.nbthread) {
+		/* nbthread not set, thus automatic. In this case, and only if
+		 * running on a single process, we enable the same number of
+		 * threads as the number of CPUs the process is bound to. This
+		 * allows to easily control the number of threads using taskset.
+		 */
+		global.nbthread = 1;
+#if defined(USE_THREAD)
+		if (global.nbproc == 1)
+			global.nbthread = thread_cpus_enabled_at_boot;
+		all_threads_mask = nbits(global.nbthread);
+#endif
+	}
+
 	if (global.nbproc > 1 && global.nbthread > 1) {
 		ha_alert("config : cannot enable multiple processes if multiple threads are configured. Please use either nbproc or nbthread but not both.\n");
 		err_code |= ERR_ALERT | ERR_FATAL;

src/haproxy.c

@@ -136,7 +136,7 @@ volatile unsigned long sleeping_thread_mask; /* Threads that are about to sleep
 struct global global = {
 	.hard_stop_after = TICK_ETERNITY,
 	.nbproc = 1,
-	.nbthread = 1,
+	.nbthread = 0,
 	.req_count = 0,
 	.logsrvs = LIST_HEAD_INIT(global.logsrvs),
 	.maxzlibmem = 0,

src/hathreads.c

@@ -10,10 +10,15 @@
  *
  */
 
+#define _GNU_SOURCE
 #include <unistd.h>
 #include <stdlib.h>
 #include <fcntl.h>
 
+#ifdef USE_CPU_AFFINITY
+#include <sched.h>
+#endif
+
 #include <common/cfgparse.h>
 #include <common/hathreads.h>
 #include <common/standard.h>
@@ -28,6 +33,7 @@ volatile unsigned long threads_harmless_mask = 0;
 volatile unsigned long all_threads_mask  = 1; // nbthread 1 assumed by default
 THREAD_LOCAL unsigned int  tid           = 0;
 THREAD_LOCAL unsigned long tid_bit       = (1UL << 0);
+int thread_cpus_enabled_at_boot          = 1;
 
 
 #if defined(DEBUG_THREAD) || defined(DEBUG_FULL)
@@ -105,6 +111,24 @@ void ha_rwlock_init(HA_RWLOCK_T *l)
 	HA_RWLOCK_INIT(l);
 }
 
+/* returns the number of CPUs the current process is enabled to run on */
+static int thread_cpus_enabled()
+{
+	int ret = 1;
+
+#ifdef USE_CPU_AFFINITY
+#if defined(__linux__) && defined(CPU_COUNT)
+	cpu_set_t mask;
+
+	if (sched_getaffinity(0, sizeof(mask), &mask) == 0)
+		ret = CPU_COUNT(&mask);
+#endif
+#endif
+	ret = MAX(ret, 1);
+	ret = MIN(ret, MAX_THREADS);
+	return ret;
+}
+
 __attribute__((constructor))
 static void __hathreads_init(void)
 {
@@ -116,7 +140,11 @@ static void __hathreads_init(void)
 			 LONGBITS, MAX_THREADS);
 		exit(1);
 	}
-	memprintf(&ptr, "Built with multi-threading support (MAX_THREADS=%d).", MAX_THREADS);
+
+	thread_cpus_enabled_at_boot = thread_cpus_enabled();
+
+	memprintf(&ptr, "Built with multi-threading support (MAX_THREADS=%d, default=%d).",
+		  MAX_THREADS, thread_cpus_enabled_at_boot);
 	hap_register_build_opts(ptr, 1);
 
 #if defined(DEBUG_THREAD) || defined(DEBUG_FULL)