MEDIUM: global: add support for CPU binding on Linux ("cpu-map")

The new "cpu-map" directive allows one to assign the CPU sets that
a process is allowed to bind to. This is useful in combination with
the "nbproc" and "bind-process" directives.

The support is implicit on Linux 2.6.28 and above.

Makefile

@@ -30,6 +30,7 @@
 #   USE_ACCEPT4          : enable use of accept4() on linux. Automatic.
 #   USE_MY_ACCEPT4       : use own implemention of accept4() if glibc < 2.10.
 #   USE_ZLIB             : enable zlib library support.
+#   USE_CPU_AFFINITY     : enable pinning processes to CPU on Linux. Automatic.
 #
 # Options can be forced by specifying "USE_xxx=1" or can be disabled by using
 # "USE_xxx=" (empty string).
@@ -241,6 +242,7 @@ ifeq ($(TARGET),linux2628)
   USE_LINUX_TPROXY= implicit
   USE_ACCEPT4     = implicit
   USE_FUTEX       = implicit
+  USE_CPU_AFFINITY= implicit
 else
 ifeq ($(TARGET),solaris)
   # This is for Solaris 8
@@ -437,6 +439,11 @@ OPTIONS_CFLAGS += -DCONFIG_HAP_LINUX_VSYSCALL
 BUILD_OPTIONS  += $(call ignore_implicit,USE_VSYSCALL)
 endif
 
+ifneq ($(USE_CPU_AFFINITY),)
+OPTIONS_CFLAGS += -DUSE_CPU_AFFINITY
+BUILD_OPTIONS  += $(call ignore_implicit,USE_CPU_AFFINITY)
+endif
+
 ifneq ($(USE_MY_SPLICE),)
 OPTIONS_CFLAGS += -DUSE_MY_SPLICE
 BUILD_OPTIONS  += $(call ignore_implicit,USE_MY_SPLICE)

doc/configuration.txt

@@ -499,6 +499,21 @@ chroot <jail dir>
   with superuser privileges. It is important to ensure that <jail_dir> is both
   empty and unwritable to anyone.
 
+cpu-map <"all"|"odd"|"even"|process_num> <cpu-set>...
+  On Linux 2.6 and above, it is possible to bind a process to a specific CPU
+  set. This means that the process will never run on other CPUs. The "cpu-map"
+  directive specifies CPU sets for process sets. The first argument is the
+  process number to bind. This process must have a number between 1 and 32,
+  and any process IDs above nbproc are ignored. It is possible to specify all
+  processes at once using "all", only odd numbers using "odd" or even numbers
+  using "even", just like with the "bind-process" directive. The second and
+  forthcoming arguments are CPU sets. Each CPU set is either a unique number
+  between 0 and 31 or a range with two such numbers delimited by a dash ('-').
+  Multiple CPU numbers or ranges may be specified, and the processes will be
+  allowed to bind to all of them. Obviously, multiple "cpu-map" directives may
+  be specified. Each "cpu-map" directive will replace the previous ones when
+  they overlap.
+
 crt-base <dir>
   Assigns a default directory to fetch SSL certificates from when a relative
   path is used with "crtfile" directives. Absolute locations specified after

include/types/global.h

@@ -130,6 +130,9 @@ struct global {
 			int level;      /* access level (ACCESS_LVL_*) */
 		} ux;
 	} unix_bind;
+#ifdef USE_CPU_AFFINITY
+	unsigned long cpu_map[32];  /* list of CPU masks for the 32 first processes */
+#endif
 	struct proxy *stats_fe;     /* the frontend holding the stats settings */
 };
 

src/cfgparse.c

@@ -1160,6 +1160,75 @@ int cfg_parse_global(const char *file, int linenum, char **args, int kwm)
 			err_code |= ERR_ALERT | ERR_FATAL;
 		}
 	}
+	else if (strcmp(args[0], "cpu-map") == 0) {  /* map a process list to a CPU set */
+#ifdef USE_CPU_AFFINITY
+		int cur_arg, i;
+		unsigned int proc = 0;
+		unsigned long cpus = 0;
+
+		if (strcmp(args[1], "all") == 0)
+			proc = 0xFFFFFFFF;
+		else if (strcmp(args[1], "odd") == 0)
+			proc = 0x55555555;
+		else if (strcmp(args[1], "even") == 0)
+			proc = 0xAAAAAAAA;
+		else {
+			proc = atoi(args[1]);
+			if (proc >= 1 && proc <= 32)
+				proc = 1 << (proc - 1);
+		}
+
+		if (!proc || !*args[2]) {
+			Alert("parsing [%s:%d]: %s expects a process number including 'all', 'odd', 'even', or a number from 1 to 32, followed by a list of CPU ranges with numbers from 0 to 31.\n",
+			      file, linenum, args[0]);
+			err_code |= ERR_ALERT | ERR_FATAL;
+			goto out;
+		}
+
+		cur_arg = 2;
+		while (*args[cur_arg]) {
+			unsigned int low, high;
+
+			if (isdigit(*args[cur_arg])) {
+				char *dash = strchr(args[cur_arg], '-');
+
+				low = high = str2uic(args[cur_arg]);
+				if (dash)
+					high = str2uic(dash + 1);
+
+				if (high < low) {
+					unsigned int swap = low;
+					low = high;
+					high = swap;
+				}
+
+				if (low < 0 || high >= sizeof(long) * 8) {
+					Alert("parsing [%s:%d]: %s supports CPU numbers from 0 to %d.\n",
+					      file, linenum, args[0], (int)(sizeof(long) * 8 - 1));
+					err_code |= ERR_ALERT | ERR_FATAL;
+					goto out;
+				}
+
+				while (low <= high)
+					cpus |= 1UL << low++;
+			}
+			else {
+				Alert("parsing [%s:%d]: %s : '%s' is not a CPU range.\n",
+				      file, linenum, args[0], args[cur_arg]);
+				err_code |= ERR_ALERT | ERR_FATAL;
+				goto out;
+			}
+			cur_arg++;
+		}
+		for (i = 0; i < 32; i++)
+			if (proc & (1 << i))
+				global.cpu_map[i] = cpus;
+#else
+		Alert("parsing [%s:%d] : '%s' is not enabled, please check build options for USE_CPU_AFFINITY.\n", file, linenum, args[0]);
+		err_code |= ERR_ALERT | ERR_FATAL;
+		goto out;
+#endif
+	}
 	else {
 		struct cfg_kw_list *kwl;
 		int index;

src/haproxy.c

@@ -1,6 +1,6 @@
 /*
  * HA-Proxy : High Availability-enabled HTTP/TCP proxy
- * Copyright 2000-2011  Willy Tarreau <w@1wt.eu>.
+ * Copyright 2000-2012  Willy Tarreau <w@1wt.eu>.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -44,6 +44,11 @@
 #include <sys/resource.h>
 #include <time.h>
 #include <syslog.h>
+#ifdef USE_CPU_AFFINITY
+#define __USE_GNU
+#include <sched.h>
+#undef __USE_GNU
+#endif
 
 #ifdef DEBUG_FULL
 #include <assert.h>
@@ -1467,6 +1472,13 @@ int main(int argc, char **argv)
 			}
 			relative_pid++; /* each child will get a different one */
 		}
+
+#ifdef USE_CPU_AFFINITY
+		if (proc < global.nbproc &&  /* child */
+		    proc < 32 &&             /* only the first 32 processes may be pinned */
+		    global.cpu_map[proc])    /* only do this if the process has a CPU map */
+			sched_setaffinity(0, sizeof(unsigned long), (void *)&global.cpu_map[proc]);
+#endif
 		/* close the pidfile both in children and father */
 		if (pidfd >= 0) {
 			//lseek(pidfd, 0, SEEK_SET);  /* debug: emulate eglibc bug */