Merge pull request #233 from ceph/wip-background

run something in the background

teuthology/task/background_exec.py

@@ -0,0 +1,75 @@
+"""
+Background task
+"""
+
+import contextlib
+import logging
+import os
+
+from teuthology import misc
+from ..orchestra import run
+
+log = logging.getLogger(__name__)
+
+@contextlib.contextmanager
+def task(ctx, config):
+    """
+    Run a background task.
+
+    Run the given command on a client, similar to exec.  However, when
+    we hit the finally because the subsequent task is ready to exit, kill
+    the child process.
+
+    We do not do any error code checking here since we are forcefully killing
+    off the child when we are done.
+
+    If the command a list, we simply join it with ;'s.
+
+    Example:
+
+       tasks:
+       - install:
+       - background_exec:
+           client.0: while true ; do date ; sleep 1 ; done
+           client.1:
+             - while true
+             - do id
+             - sleep 1
+             - done
+       - exec:
+           client.0:
+             - sleep 10
+
+    """
+    assert isinstance(config, dict), "task background got invalid config"
+
+    testdir = misc.get_testdir(ctx)
+
+    tasks = {}
+    for role, cmd in config.iteritems():
+        (remote,) = ctx.cluster.only(role).remotes.iterkeys()
+        log.info('Running background command on role %s host %s', role, remote.name)
+        if isinstance(cmd, list):
+            cmd = '; '.join(cmd)
+        cmd.replace('$TESTDIR', testdir)
+        tasks[remote.name] = remote.run(
+            args=[
+                'sudo',
+                'TESTDIR=%s' % testdir,
+                'daemon-helper', 'kill', '--kill-group',
+                'bash', '-c', cmd,
+            ],
+            wait=False,
+            stdin=run.PIPE,
+            check_status=False,
+            logger=log.getChild(remote.name)
+        )
+
+    try:
+        yield
+
+    finally:
+        for name, task in tasks.iteritems():
+            log.info('Stopping background command on %s', name)
+            task.stdin.close()
+        run.wait(tasks.itervalues())

teuthology/task/daemon-helper

@@ -12,6 +12,8 @@ then wait for EOF on stdin.
 When EOF is seen on stdin, the child process is killed.
 
 When the child process exits, this helper exits too.
+
+usage: daemon-helper <signal> [--kill-group] [nostdin] command...
 """
 
 import fcntl
@@ -28,6 +30,11 @@ if sys.argv[1] == "term":
 
 cmd_start = 2
 
+group = False
+if sys.argv[cmd_start] == "--kill-group":
+    group = True
+    cmd_start += 1
+
 nostdin = False
 if sys.argv[cmd_start] == "nostdin":
     nostdin = True
@@ -43,6 +50,7 @@ else:
         proc = subprocess.Popen(
             args=sys.argv[cmd_start:],
             stdin=devnull,
+            preexec_fn=os.setsid,
             )
 
 flags = fcntl.fcntl(0, fcntl.F_GETFL)
@@ -55,11 +63,18 @@ while True:
         data = os.read(0, 1)
         if not data:
             saw_eof = True
-            proc.send_signal(end_signal)
+            if not group:
+                proc.send_signal(end_signal)
+            else:
+                os.killpg(proc.pid, end_signal)
             break
         else:
             sig, = struct.unpack('!b', data)
-            proc.send_signal(sig)
+            if not group:
+                proc.send_signal(sig)
+            else:
+                os.killpg(proc.pid, end_signal)
+
 
     if proc.poll() is not None:
         # child exited