diff --git a/src/haproxy.c b/src/haproxy.c
index 993fa3939..6b8fdf158 100644
--- a/src/haproxy.c
+++ b/src/haproxy.c
@@ -3623,18 +3623,6 @@ int main(int argc, char **argv)
 			in_parent = 1;
 		}
 
-#if !defined(USE_THREAD) && defined(USE_CPU_AFFINITY)
-		if (!in_parent && ha_cpuset_count(&cpu_map[0].thread[0])) {   /* only do this if the process has a CPU map */
-
-#if defined(CPUSET_USE_CPUSET) || defined(__DragonFly__)
-			struct hap_cpuset *set = &cpu_map[0].thread[0];
-			sched_setaffinity(0, sizeof(set->cpuset), &set->cpuset);
-#elif defined(__FreeBSD__)
-			struct hap_cpuset *set = &cpu_map[0].thread[0];
-			ret = cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, -1, sizeof(set->cpuset), &set->cpuset);
-#endif
-		}
-#endif
 		/* close the pidfile both in children and father */
 		if (pidfd >= 0) {
 			//lseek(pidfd, 0, SEEK_SET);  /* debug: emulate eglibc bug */
@@ -3775,6 +3763,19 @@ int main(int argc, char **argv)
 		fork_poller();
 	}
 
+	/* Note that here we can't be in the parent/master anymore */
+#if !defined(USE_THREAD) && defined(USE_CPU_AFFINITY)
+	if (ha_cpuset_count(&cpu_map[0].thread[0])) {   /* only do this if the process has a CPU map */
+
+#if defined(CPUSET_USE_CPUSET) || defined(__DragonFly__)
+		struct hap_cpuset *set = &cpu_map[0].thread[0];
+		sched_setaffinity(0, sizeof(set->cpuset), &set->cpuset);
+#elif defined(__FreeBSD__)
+		struct hap_cpuset *set = &cpu_map[0].thread[0];
+		ret = cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, -1, sizeof(set->cpuset), &set->cpuset);
+#endif
+	}
+#endif
 	/* try our best to re-enable core dumps depending on system capabilities.
 	 * What is addressed here :
 	 *   - remove file size limits