Add teuthology-suite, to run multiple tests in a batch.

setup.py

@@ -23,6 +23,7 @@ setup(
     entry_points={
         'console_scripts': [
             'teuthology = teuthology.run:main',
+            'teuthology-suite = teuthology.suite:main',
             ],
         },
 

teuthology/suite.py

@@ -0,0 +1,117 @@
+import argparse
+import itertools
+import logging
+import os
+import shutil
+import subprocess
+import sys
+import time
+
+log = logging.getLogger(__name__)
+
+def main():
+    parser = argparse.ArgumentParser(description="""
+Run a suite of ceph integration tests.
+
+A suite is a directory containing facets.
+
+A facet is a directory containing config snippets.
+
+Running the suite means running teuthology for every configuration
+combination generated by taking one config snippet from each facet.
+
+Any config files passed on the command line will be used for every
+combination, and will override anything in the suite. This is most
+useful for specifying actual machines to run on.
+""")
+    parser.add_argument(
+        '-v', '--verbose',
+        action='store_true', default=None,
+        help='be more verbose',
+        )
+    parser.add_argument(
+        '--suite',
+        metavar='DIR',
+        help='suite of tests to run',
+        required=True,
+        )
+    parser.add_argument(
+        '--archive-dir',
+        metavar='DIR',
+        help='path under which to archive results',
+        required=True,
+        )
+    parser.add_argument(
+        'config',
+        metavar='CONFFILE',
+        nargs='*',
+        default=[],
+        help='config file to read',
+        )
+
+    args = parser.parse_args()
+
+    loglevel = logging.INFO
+    if args.verbose:
+        loglevel = logging.DEBUG
+
+    logging.basicConfig(
+        level=loglevel,
+        )
+
+    if not os.path.isdir(args.archive_dir):
+        sys.exit("{prog}: archive directory must exist: {path}".format(
+                prog=os.path.basename(sys.argv[0]),
+                path=args.archive_dir,
+                ))
+
+    failed = False
+    facets = [
+        f for f in os.listdir(args.suite)
+        if not f.startswith('.')
+        and os.path.isdir(os.path.join(args.suite, f))
+        ]
+    facet_configs = (
+        [(f, name, os.path.join(args.suite, f, name)) for name in os.listdir(os.path.join(args.suite, f))
+         if not name.startswith('.')
+         and name.endswith('.yaml')
+         ]
+        for f in facets
+        )
+    for configs in itertools.product(*facet_configs):
+        log.info(
+            'Running teuthology with facets %s',
+            ' '.join('{facet}:{name}'.format(facet=facet, name=name)
+                     for facet, name, path in configs)
+            )
+        arg = [
+            os.path.join(os.path.dirname(sys.argv[0]), 'teuthology'),
+            ]
+
+        if args.verbose:
+            arg.append('-v')
+
+        while True:
+            archive = os.path.join(args.archive_dir, time.strftime('%Y-%m-%dT%H-%M-%S'))
+            if not os.path.exists(archive):
+                break
+            time.sleep(1)
+        arg.append('--archive={path}'.format(path=archive))
+
+        arg.append('--')
+
+        arg.extend(path for facet, name, path in configs)
+        arg.extend(args.config)
+        try:
+            subprocess.check_call(
+                args=arg,
+                close_fds=True,
+                )
+        except subprocess.CalledProcessError as e:
+            log.exception(e)
+            failed = True
+
+    if failed:
+        log.info('Failed.')
+        sys.exit(1)
+