From d8d2ef931c5883dd1a04e563ebb67b381af6f541 Mon Sep 17 00:00:00 2001
From: Zack Cerza <zack@cerza.org>
Date: Fri, 23 Aug 2013 11:39:02 -0500
Subject: [PATCH 01/19] Add teuthology.config, the start of a better system

---
 teuthology/config.py | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100644 teuthology/config.py

diff --git a/teuthology/config.py b/teuthology/config.py
new file mode 100644
index 00000000000..04e2aa299bd
--- /dev/null
+++ b/teuthology/config.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+import os
+import yaml
+import logging
+
+CONF_FILE = os.path.join(os.environ['HOME'], '.teuthology.yaml')
+
+log = logging.getLogger(__name__)
+
+
+class _Config(object):
+    def __init__(self):
+        self.__conf = {}
+        if not os.path.exists(CONF_FILE):
+            log.debug("%s not found", CONF_FILE)
+            return
+
+        with file(CONF_FILE) as f:
+            conf_obj = yaml.safe_load_all(f)
+            for item in conf_obj:
+                self.__conf.update(item)
+
+    @property
+    def lock_server(self):
+        return self.__conf.get('lock_server')
+
+    @property
+    def queue_host(self):
+        return self.__conf.get('queue_host')
+
+    @property
+    def queue_port(self):
+        return self.__conf.get('queue_port')
+
+    @property
+    def sentry_dsn(self):
+        return self.__conf.get('sentry_dsn')
+
+config = _Config()

From 609086366d90142771260a9b1739e457bf59b225 Mon Sep 17 00:00:00 2001
From: Zack Cerza <zack@cerza.org>
Date: Fri, 23 Aug 2013 11:40:22 -0500
Subject: [PATCH 02/19] Use teuthology.config

---
 teuthology/sentry.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/teuthology/sentry.py b/teuthology/sentry.py
index ecbd56fb236..1bf92f53b4a 100644
--- a/teuthology/sentry.py
+++ b/teuthology/sentry.py
@@ -1,13 +1,20 @@
+import logging
 from raven import Client
+from .config import config
+
+log = logging.getLogger(__name__)
 
 client = None
 
-def get_client(ctx):
+
+def get_client():
     global client
     if client:
+        log.debug("Found client, reusing")
         return client
-    dsn = ctx.teuthology_config.get('sentry_dsn')
+
+    log.debug("Getting sentry client")
+    dsn = config.sentry_dsn
     if dsn:
         client = Client(dsn=dsn)
         return client
-

From 7930f661a31f482b9f38694409e778e4df9e10a1 Mon Sep 17 00:00:00 2001
From: Zack Cerza <zack@cerza.org>
Date: Fri, 23 Aug 2013 11:40:44 -0500
Subject: [PATCH 03/19] Update call to get_sentry_client()

---
 teuthology/run_tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/teuthology/run_tasks.py b/teuthology/run_tasks.py
index 1e61a4c607f..dc21701697e 100644
--- a/teuthology/run_tasks.py
+++ b/teuthology/run_tasks.py
@@ -32,7 +32,7 @@ def run_tasks(tasks, ctx):
         if 'failure_reason' not in ctx.summary:
             ctx.summary['failure_reason'] = str(e)
         msg = 'Saw exception from tasks.'
-        sentry = get_sentry_client(ctx)
+        sentry = get_sentry_client()
         if sentry:
             exc_id = sentry.captureException()
             msg += " Sentry id %s" % exc_id

From 6c486ab0670717faba699560e06c6c2600a25450 Mon Sep 17 00:00:00 2001
From: Zack Cerza <zack@cerza.org>
Date: Fri, 23 Aug 2013 14:53:38 -0500
Subject: [PATCH 04/19] Tweak logging

---
 teuthology/run_tasks.py | 5 ++---
 teuthology/sentry.py    | 2 --
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/teuthology/run_tasks.py b/teuthology/run_tasks.py
index dc21701697e..c849d0fce64 100644
--- a/teuthology/run_tasks.py
+++ b/teuthology/run_tasks.py
@@ -31,12 +31,11 @@ def run_tasks(tasks, ctx):
         ctx.summary['success'] = False
         if 'failure_reason' not in ctx.summary:
             ctx.summary['failure_reason'] = str(e)
-        msg = 'Saw exception from tasks.'
+        log.exception('Saw exception from tasks.')
         sentry = get_sentry_client()
         if sentry:
             exc_id = sentry.captureException()
-            msg += " Sentry id %s" % exc_id
-        log.exception(msg)
+            log.exception(" Sentry id %s" % exc_id)
         if ctx.config.get('interactive-on-error'):
             from .task import interactive
             log.warning('Saw failure, going into interactive mode...')
diff --git a/teuthology/sentry.py b/teuthology/sentry.py
index 1bf92f53b4a..3d83a291361 100644
--- a/teuthology/sentry.py
+++ b/teuthology/sentry.py
@@ -10,10 +10,8 @@ client = None
 def get_client():
     global client
     if client:
-        log.debug("Found client, reusing")
         return client
 
-    log.debug("Getting sentry client")
     dsn = config.sentry_dsn
     if dsn:
         client = Client(dsn=dsn)

From 12cb686070abebeafd9c46bacfc95ea0f356494e Mon Sep 17 00:00:00 2001
From: Zack Cerza <zack@cerza.org>
Date: Mon, 26 Aug 2013 11:51:43 -0500
Subject: [PATCH 05/19] Rewrite email-generating code.

---
 teuthology/suite.py | 131 +++++++++++++++++++++++---------------------
 1 file changed, 68 insertions(+), 63 deletions(-)

diff --git a/teuthology/suite.py b/teuthology/suite.py
index bc30e4f8fb6..2d7e1c8a11a 100644
--- a/teuthology/suite.py
+++ b/teuthology/suite.py
@@ -337,13 +337,14 @@ def results():
         log.exception('error generating results')
         raise
 
+
 def _results(args):
     running_tests = [
         f for f in sorted(os.listdir(args.archive_dir))
         if not f.startswith('.')
         and os.path.isdir(os.path.join(args.archive_dir, f))
         and not os.path.exists(os.path.join(args.archive_dir, f, 'summary.yaml'))
-        ]
+    ]
     starttime = time.time()
     log.info('Waiting up to %d seconds for tests to finish...', args.timeout)
     while running_tests and args.timeout > 0:
@@ -359,67 +360,8 @@ def _results(args):
             time.sleep(10)
     log.info('Tests finished! gathering results...')
 
-    descriptions = []
-    failures = []
-    num_failures = 0
-    unfinished = []
-    passed = []
-    all_jobs = sorted(os.listdir(args.archive_dir))
-    for j in all_jobs:
-        job_dir = os.path.join(args.archive_dir, j)
-        if j.startswith('.') or not os.path.isdir(job_dir):
-            continue
-        summary_fn = os.path.join(job_dir, 'summary.yaml')
-        if not os.path.exists(summary_fn):
-            unfinished.append(j)
-            continue
-        summary = {}
-        with file(summary_fn) as f:
-            g = yaml.safe_load_all(f)
-            for new in g:
-                summary.update(new)
-        desc = '{test}: ({duration}s) {desc}'.format(
-            duration=int(summary.get('duration', 0)),
-            desc=summary['description'],
-            test=j,
-            )
-        descriptions.append(desc)
-        if summary['success']:
-            passed.append(desc)
-        else:
-            failures.append(desc)
-            num_failures += 1
-            if 'failure_reason' in summary:
-                failures.append('    {reason}'.format(
-                        reason=summary['failure_reason'],
-                        ))
-
-    if failures or unfinished:
-        subject = ('{num_failed} failed, {num_hung} hung, '
-                   '{num_passed} passed in {suite}'.format(
-                num_failed=num_failures,
-                num_hung=len(unfinished),
-                num_passed=len(passed),
-                suite=args.name,
-                ))
-        body = """
-The following tests failed:
-
-{failures}
-
-These tests may be hung (did not finish in {timeout} seconds after the last test in the suite):
-{unfinished}
-
-These tests passed:
-{passed}""".format(
-            failures='\n'.join(failures),
-            unfinished='\n'.join(unfinished),
-            passed='\n'.join(passed),
-            timeout=args.timeout,
-            )
-    else:
-        subject = '{num_passed} passed in {suite}'.format(suite=args.name, num_passed=len(passed))
-        body = '\n'.join(descriptions)
+    (subject, body) = build_email_body(args.name, args.archive_dir,
+                                       args.timeout)
 
     try:
         if args.email:
@@ -428,10 +370,69 @@ These tests passed:
                 from_=args.teuthology_config['results_sending_email'],
                 to=args.email,
                 body=body,
-                )
+            )
     finally:
         generate_coverage(args)
 
+
+def build_email_body(name, archive_dir, timeout):
+    failed = []
+    unfinished = []
+    passed = []
+    all_jobs = sorted(os.listdir(archive_dir))
+    for job in all_jobs:
+        job_dir = os.path.join(archive_dir, job)
+        if job.startswith('.') or not os.path.isdir(job_dir):
+            continue
+
+        summary_file = os.path.join(job_dir, 'summary.yaml')
+        # Unfinished jobs will have no summary.yaml
+        if not os.path.exists(summary_file):
+            unfinished.append(job)
+            continue
+
+        summary = {}
+        with file(summary_file) as f:
+            summary = yaml.safe_load(f)
+        long_desc = '{test}: ({duration}s) {desc}'.format(
+            duration=int(summary.get('duration', 0)),
+            desc=summary['description'],
+            test=job,
+        )
+        if summary['success']:
+            passed.append(long_desc)
+        else:
+            full_desc = long_desc
+            if 'failure_reason' in summary:
+                full_desc += '\n    %s' % summary['failure_reason']
+            failed.append(full_desc)
+
+    maybe_comma = lambda s: ', ' if s else ' '
+
+    subject = ''
+    body = ''
+    if failed:
+        subject += '{num_failed} failed{sep}'.format(
+            num_failed=len(failed),
+            sep=maybe_comma(unfinished or passed)
+        )
+        body += 'The following tests failed:\n%s\n\n\n' % '\n'.join(failed)
+    if unfinished:
+        subject += '{num_hung} hung{sep}'.format(
+            num_hung=len(unfinished),
+            sep=maybe_comma(passed)
+        )
+        body += 'These tests may be hung (did not finish in {timeout} seconds after the last test in the suite):\n{hung_jobs}\n\n\n'.format(
+            timeout=timeout,
+            hung_jobs='\n'.join(unfinished),
+        )
+    if passed:
+        subject += '%s passed ' % len(passed)
+        body += 'These tests passed:\n%s' % '\n'.join(passed)
+    subject += 'in {suite}'.format(suite=name)
+    return (subject.strip(), body.strip())
+
+
 def get_arch(config):
     for yamlfile in config:
         y = yaml.safe_load(file(yamlfile))
@@ -445,6 +446,7 @@ def get_arch(config):
                     return arch
     return None
 
+
 def get_os_type(configs):
     for config in configs:
         yamlfile = config[2]
@@ -454,6 +456,7 @@ def get_os_type(configs):
             return os_type
     return None
 
+
 def get_exclude_arch(configs):
     for config in configs:
         yamlfile = config[2]
@@ -463,6 +466,7 @@ def get_exclude_arch(configs):
             return os_type
     return None
 
+
 def get_exclude_os_type(configs):
     for config in configs:
         yamlfile = config[2]
@@ -472,6 +476,7 @@ def get_exclude_os_type(configs):
             return os_type
     return None
 
+
 def get_machine_type(config):
     for yamlfile in config:
         y = yaml.safe_load(file(yamlfile))

From fedc91c07f4e712f95903412ae7456f6b7a7e74f Mon Sep 17 00:00:00 2001
From: Zack Cerza <zack@cerza.org>
Date: Mon, 26 Aug 2013 12:36:01 -0500
Subject: [PATCH 06/19] Add a catch-all __getattr__(); add comments

---
 teuthology/config.py | 30 ++++++++++++++----------------
 1 file changed, 14 insertions(+), 16 deletions(-)

diff --git a/teuthology/config.py b/teuthology/config.py
index 04e2aa299bd..45b0b92add8 100644
--- a/teuthology/config.py
+++ b/teuthology/config.py
@@ -9,31 +9,29 @@ log = logging.getLogger(__name__)
 
 
 class _Config(object):
+    """
+    This class is intended to unify teuthology's many configuration files and
+    objects. Currently it serves as a convenient interface to
+    ~/.teuthology.yaml and nothing else.
+    """
     def __init__(self):
-        self.__conf = {}
         if not os.path.exists(CONF_FILE):
             log.debug("%s not found", CONF_FILE)
+            self.__conf = {}
             return
 
-        with file(CONF_FILE) as f:
-            conf_obj = yaml.safe_load_all(f)
-            for item in conf_obj:
-                self.__conf.update(item)
+        self.__conf = yaml.safe_load(file(CONF_FILE))
 
+    # This property declaration exists mainly as an example; it is not
+    # necessary unless you want to, say, define a set method and/or a
+    # docstring.
     @property
     def lock_server(self):
         return self.__conf.get('lock_server')
 
-    @property
-    def queue_host(self):
-        return self.__conf.get('queue_host')
-
-    @property
-    def queue_port(self):
-        return self.__conf.get('queue_port')
-
-    @property
-    def sentry_dsn(self):
-        return self.__conf.get('sentry_dsn')
+    # This takes care of any and all of the rest.
+    # If the parameter is defined, return it. Otherwise return None.
+    def __getattr__(self, name):
+        return self.__conf.get(name)
 
 config = _Config()

From 300374bb07cc886d145af0b7530846d50e35a0eb Mon Sep 17 00:00:00 2001
From: Zack Cerza <zack@cerza.org>
Date: Mon, 26 Aug 2013 12:37:04 -0500
Subject: [PATCH 07/19] For failures, add http links to log directories.

---
 teuthology/suite.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/teuthology/suite.py b/teuthology/suite.py
index 2d7e1c8a11a..dab53e2c99c 100644
--- a/teuthology/suite.py
+++ b/teuthology/suite.py
@@ -13,10 +13,12 @@ import subprocess
 import sys
 import time
 import yaml
+import urlparse
 
 from teuthology import misc as teuthology
 from teuthology import safepath
 from teuthology import lock as lock
+from teuthology.config import config
 
 log = logging.getLogger(__name__)
 
@@ -375,6 +377,14 @@ def _results(args):
         generate_coverage(args)
 
 
+def get_http_log_path(archive_dir, job_id):
+    http_base = config.archive_server
+    if not http_base:
+        return None
+    archive_subdir = os.path.split(archive_dir)[-1]
+    return urlparse.urljoin(http_base, archive_subdir, str(job_id))
+
+
 def build_email_body(name, archive_dir, timeout):
     failed = []
     unfinished = []
@@ -405,6 +415,9 @@ def build_email_body(name, archive_dir, timeout):
             full_desc = long_desc
             if 'failure_reason' in summary:
                 full_desc += '\n    %s' % summary['failure_reason']
+            http_log = get_http_log_path(archive_dir, job)
+            if http_log:
+                full_desc += '\n    %s' % http_log
             failed.append(full_desc)
 
     maybe_comma = lambda s: ', ' if s else ' '

From 6f00939053c97031728c6c0f5094cdc9f84001a9 Mon Sep 17 00:00:00 2001
From: Zack Cerza <zack@cerza.org>
Date: Mon, 26 Aug 2013 16:06:51 -0500
Subject: [PATCH 08/19] Use os.path.join, not urlparse.urljoin

---
 teuthology/suite.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/teuthology/suite.py b/teuthology/suite.py
index dab53e2c99c..391542e73d9 100644
--- a/teuthology/suite.py
+++ b/teuthology/suite.py
@@ -13,7 +13,6 @@ import subprocess
 import sys
 import time
 import yaml
-import urlparse
 
 from teuthology import misc as teuthology
 from teuthology import safepath
@@ -382,7 +381,7 @@ def get_http_log_path(archive_dir, job_id):
     if not http_base:
         return None
     archive_subdir = os.path.split(archive_dir)[-1]
-    return urlparse.urljoin(http_base, archive_subdir, str(job_id))
+    return os.path.join(http_base, archive_subdir, str(job_id))
 
 
 def build_email_body(name, archive_dir, timeout):

From 4007173278c43985fa6e0d45728e306ea46979ef Mon Sep 17 00:00:00 2001
From: Zack Cerza <zack@cerza.org>
Date: Mon, 26 Aug 2013 16:33:04 -0500
Subject: [PATCH 09/19] Add URL to Sentry event to traceback output.

---
 teuthology/run_tasks.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/teuthology/run_tasks.py b/teuthology/run_tasks.py
index c849d0fce64..3f7d7821afc 100644
--- a/teuthology/run_tasks.py
+++ b/teuthology/run_tasks.py
@@ -1,6 +1,7 @@
 import sys
 import logging
 from teuthology.sentry import get_client as get_sentry_client
+from .config import config
 
 log = logging.getLogger(__name__)
 
@@ -35,7 +36,9 @@ def run_tasks(tasks, ctx):
         sentry = get_sentry_client()
         if sentry:
             exc_id = sentry.captureException()
-            log.exception(" Sentry id %s" % exc_id)
+            log.exception(" Sentry event: {server}/search?q={id}".format(
+                server=config.sentry_server,
+                id=exc_id))
         if ctx.config.get('interactive-on-error'):
             from .task import interactive
             log.warning('Saw failure, going into interactive mode...')

From 489c1660f221d187257e97d677f3438ff5051c26 Mon Sep 17 00:00:00 2001
From: Zack Cerza <zack@cerza.org>
Date: Mon, 26 Aug 2013 17:11:51 -0500
Subject: [PATCH 10/19] Also leave a list of sentry events in the summary

---
 teuthology/run_tasks.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/teuthology/run_tasks.py b/teuthology/run_tasks.py
index 3f7d7821afc..e3a336d8aed 100644
--- a/teuthology/run_tasks.py
+++ b/teuthology/run_tasks.py
@@ -1,10 +1,11 @@
 import sys
 import logging
 from teuthology.sentry import get_client as get_sentry_client
-from .config import config
+from .config import config as teuth_config
 
 log = logging.getLogger(__name__)
 
+
 def run_one_task(taskname, **kwargs):
     submod = taskname
     subtask = 'task'
@@ -15,6 +16,7 @@ def run_one_task(taskname, **kwargs):
     fn = getattr(mod, subtask)
     return fn(**kwargs)
 
+
 def run_tasks(tasks, ctx):
     stack = []
     try:
@@ -35,10 +37,13 @@ def run_tasks(tasks, ctx):
         log.exception('Saw exception from tasks.')
         sentry = get_sentry_client()
         if sentry:
-            exc_id = sentry.captureException()
-            log.exception(" Sentry event: {server}/search?q={id}".format(
-                server=config.sentry_server,
-                id=exc_id))
+            exc_id = sentry.get_ident(sentry.captureException())
+            event_url = "{server}/search?q={id}".format(
+                server=teuth_config.sentry_server, id=exc_id)
+            log.exception(" Sentry event: %s" % event_url)
+            sentry_url_list = ctx.summary.get('sentry_events', [])
+            sentry_url_list.append(event_url)
+            ctx.summary['sentry_events'] = sentry_url_list
         if ctx.config.get('interactive-on-error'):
             from .task import interactive
             log.warning('Saw failure, going into interactive mode...')

From 050380311126bae1d75cef44e0acfbab382472a1 Mon Sep 17 00:00:00 2001
From: Zack Cerza <zack@cerza.org>
Date: Mon, 26 Aug 2013 17:21:48 -0500
Subject: [PATCH 11/19] Add sentry events to suite email.

---
 teuthology/suite.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/teuthology/suite.py b/teuthology/suite.py
index 391542e73d9..764543e41c6 100644
--- a/teuthology/suite.py
+++ b/teuthology/suite.py
@@ -417,6 +417,9 @@ def build_email_body(name, archive_dir, timeout):
             http_log = get_http_log_path(archive_dir, job)
             if http_log:
                 full_desc += '\n    %s' % http_log
+            sentry_events = summary.get('sentry_events')
+            if sentry_events:
+                full_desc += '\n    %s' % '\n    '.join(sentry_events)
             failed.append(full_desc)
 
     maybe_comma = lambda s: ', ' if s else ' '

From a1a261a88667c3066fd9e11e7af4673c1fca1b44 Mon Sep 17 00:00:00 2001
From: Zack Cerza <zack@cerza.org>
Date: Mon, 26 Aug 2013 17:38:23 -0500
Subject: [PATCH 12/19] Add tags! Task name and owner to start.

---
 teuthology/run_tasks.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/teuthology/run_tasks.py b/teuthology/run_tasks.py
index e3a336d8aed..7e6510d7dc1 100644
--- a/teuthology/run_tasks.py
+++ b/teuthology/run_tasks.py
@@ -37,7 +37,11 @@ def run_tasks(tasks, ctx):
         log.exception('Saw exception from tasks.')
         sentry = get_sentry_client()
         if sentry:
-            exc_id = sentry.get_ident(sentry.captureException())
+            tags = {
+                'task': taskname,
+                'owner': ctx.owner,
+            }
+            exc_id = sentry.get_ident(sentry.captureException(tags=tags))
             event_url = "{server}/search?q={id}".format(
                 server=teuth_config.sentry_server, id=exc_id)
             log.exception(" Sentry event: %s" % event_url)

From 81709ed13dbcb469bea03c50ed95c3f7c8f96f0e Mon Sep 17 00:00:00 2001
From: Zack Cerza <zack@cerza.org>
Date: Mon, 26 Aug 2013 18:12:20 -0500
Subject: [PATCH 13/19] Avoid double slashes in sentry event URL

---
 teuthology/run_tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/teuthology/run_tasks.py b/teuthology/run_tasks.py
index 7e6510d7dc1..4807191d9d5 100644
--- a/teuthology/run_tasks.py
+++ b/teuthology/run_tasks.py
@@ -43,7 +43,7 @@ def run_tasks(tasks, ctx):
             }
             exc_id = sentry.get_ident(sentry.captureException(tags=tags))
             event_url = "{server}/search?q={id}".format(
-                server=teuth_config.sentry_server, id=exc_id)
+                server=teuth_config.sentry_server.strip('/'), id=exc_id)
             log.exception(" Sentry event: %s" % event_url)
             sentry_url_list = ctx.summary.get('sentry_events', [])
             sentry_url_list.append(event_url)

From eb585d19a7e8481dc03492d06316ef7c319b2528 Mon Sep 17 00:00:00 2001
From: Zack Cerza <zack@cerza.org>
Date: Tue, 27 Aug 2013 14:51:27 -0500
Subject: [PATCH 14/19] Move job listing logic to get_jobs()

---
 teuthology/suite.py | 26 +++++++++++++++++---------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/teuthology/suite.py b/teuthology/suite.py
index 764543e41c6..34ae57c59c2 100644
--- a/teuthology/suite.py
+++ b/teuthology/suite.py
@@ -4,6 +4,7 @@ import errno
 import itertools
 import logging
 import os
+import re
 
 # this file is responsible for submitting tests into the queue
 # by generating combinations of facets found in
@@ -203,11 +204,8 @@ def ls():
         )
     args = parser.parse_args()
 
-    for j in sorted(os.listdir(args.archive_dir)):
+    for j in get_jobs(args.archive_dir):
         job_dir = os.path.join(args.archive_dir, j)
-        if j.startswith('.') or not os.path.isdir(job_dir):
-            continue
-
         summary = {}
         try:
             with file(os.path.join(job_dir, 'summary.yaml')) as f:
@@ -384,17 +382,27 @@ def get_http_log_path(archive_dir, job_id):
     return os.path.join(http_base, archive_subdir, str(job_id))
 
 
+def get_jobs(archive_dir):
+    dir_contents = os.listdir(archive_dir)
+
+    def is_job_dir(parent, subdir):
+        if os.path.isdir(os.path.join(parent, subdir)) and re.match('\d+$', subdir):
+            return True
+        return False
+
+    jobs = [job for job in dir_contents if is_job_dir(archive_dir, job)]
+    return sorted(jobs)
+
+
 def build_email_body(name, archive_dir, timeout):
     failed = []
     unfinished = []
     passed = []
-    all_jobs = sorted(os.listdir(archive_dir))
-    for job in all_jobs:
-        job_dir = os.path.join(archive_dir, job)
-        if job.startswith('.') or not os.path.isdir(job_dir):
-            continue
 
+    for job in get_jobs(archive_dir):
+        job_dir = os.path.join(archive_dir, job)
         summary_file = os.path.join(job_dir, 'summary.yaml')
+
         # Unfinished jobs will have no summary.yaml
         if not os.path.exists(summary_file):
             unfinished.append(job)

From dfdac24f27127c697baa703644c58c312c265360 Mon Sep 17 00:00:00 2001
From: Zack Cerza <zack@cerza.org>
Date: Tue, 27 Aug 2013 17:22:27 -0500
Subject: [PATCH 15/19] Make email formatting way, way nicer.

---
 teuthology/suite.py | 125 ++++++++++++++++++++++++++++++++++----------
 1 file changed, 96 insertions(+), 29 deletions(-)

diff --git a/teuthology/suite.py b/teuthology/suite.py
index 34ae57c59c2..ecbdcb49fc5 100644
--- a/teuthology/suite.py
+++ b/teuthology/suite.py
@@ -5,6 +5,7 @@ import itertools
 import logging
 import os
 import re
+from textwrap import dedent, fill
 
 # this file is responsible for submitting tests into the queue
 # by generating combinations of facets found in
@@ -394,10 +395,48 @@ def get_jobs(archive_dir):
     return sorted(jobs)
 
 
+email_templates = {
+    'body_templ': dedent("""\
+        Test Run
+        =================================================================
+        logs:   {log_root}
+        failed: {fail_count}
+        hung:   {hung_count}
+        passed: {pass_count}
+
+        {fail_sect}{hung_sect}{pass_sect}
+        """),
+    'sect_templ': dedent("""\
+        {title}
+        =================================================================
+        {jobs}
+        """),
+    'fail_templ': dedent("""\
+        [{job_id}]  {desc}
+        -----------------------------------------------------------------
+        time:   {time}{log_line}{sentry_line}
+
+        {reason}
+
+        """),
+    'fail_log_templ': "\nlog:    {log}",
+    'fail_sentry_templ': "\nsentry: {sentries}",
+    'hung_templ': dedent("""\
+        [{job_id}]
+        """),
+    'pass_templ': dedent("""\
+        [{job_id}] {desc}
+        time:    {time}
+
+        """),
+}
+
+
+
 def build_email_body(name, archive_dir, timeout):
-    failed = []
-    unfinished = []
-    passed = []
+    failed = {}
+    hung = {}
+    passed = {}
 
     for job in get_jobs(archive_dir):
         job_dir = os.path.join(archive_dir, job)
@@ -405,53 +444,81 @@ def build_email_body(name, archive_dir, timeout):
 
         # Unfinished jobs will have no summary.yaml
         if not os.path.exists(summary_file):
-            unfinished.append(job)
+            hung[job] = email_templates['hung_templ'].format(job_id=job)
             continue
 
-        summary = {}
         with file(summary_file) as f:
             summary = yaml.safe_load(f)
-        long_desc = '{test}: ({duration}s) {desc}'.format(
-            duration=int(summary.get('duration', 0)),
-            desc=summary['description'],
-            test=job,
-        )
+
         if summary['success']:
-            passed.append(long_desc)
+            passed[job] = email_templates['pass_templ'].format(
+                job_id=job,
+                desc=summary.get('description'),
+                time=summary.get('duration'),
+            )
         else:
-            full_desc = long_desc
-            if 'failure_reason' in summary:
-                full_desc += '\n    %s' % summary['failure_reason']
-            http_log = get_http_log_path(archive_dir, job)
-            if http_log:
-                full_desc += '\n    %s' % http_log
+            log = get_http_log_path(archive_dir, job)
+            if log:
+                log_line = email_templates['fail_log_templ'].format(log=log)
+            else:
+                log_line = ''
             sentry_events = summary.get('sentry_events')
             if sentry_events:
-                full_desc += '\n    %s' % '\n    '.join(sentry_events)
-            failed.append(full_desc)
+                sentry_line = email_templates['fail_sentry_templ'].format(
+                    sentries='\n        '.join(sentry_events))
+            else:
+                sentry_line = ''
+
+            failed[job] = email_templates['fail_templ'].format(
+                job_id=job,
+                desc=summary.get('description'),
+                time=summary.get('duration'),
+                reason=fill(summary.get('failure_reason'), 79),
+                log_line=log_line,
+                sentry_line=sentry_line,
+            )
 
     maybe_comma = lambda s: ', ' if s else ' '
 
     subject = ''
-    body = ''
+    fail_sect = ''
+    hung_sect = ''
+    pass_sect = ''
     if failed:
         subject += '{num_failed} failed{sep}'.format(
             num_failed=len(failed),
-            sep=maybe_comma(unfinished or passed)
+            sep=maybe_comma(hung or passed)
         )
-        body += 'The following tests failed:\n%s\n\n\n' % '\n'.join(failed)
-    if unfinished:
+        fail_sect = email_templates['sect_templ'].format(
+            title='Failed',
+            jobs=''.join(failed.values())
+        )
+    if hung:
         subject += '{num_hung} hung{sep}'.format(
-            num_hung=len(unfinished),
-            sep=maybe_comma(passed)
+            num_hung=len(hung),
+            sep=maybe_comma(passed),
         )
-        body += 'These tests may be hung (did not finish in {timeout} seconds after the last test in the suite):\n{hung_jobs}\n\n\n'.format(
-            timeout=timeout,
-            hung_jobs='\n'.join(unfinished),
+        hung_sect = email_templates['sect_templ'].format(
+            title='Hung',
+            jobs=''.join(hung.values()),
         )
     if passed:
         subject += '%s passed ' % len(passed)
-        body += 'These tests passed:\n%s' % '\n'.join(passed)
+        pass_sect = email_templates['sect_templ'].format(
+            title='Passed',
+            jobs=''.join(passed.values()),
+        )
+
+    body = email_templates['body_templ'].format(
+        log_root=get_http_log_path(archive_dir, ''),
+        fail_count=len(failed),
+        hung_count=len(hung),
+        pass_count=len(passed),
+        fail_sect=fail_sect,
+        hung_sect=hung_sect,
+        pass_sect=pass_sect,
+    )
+
     subject += 'in {suite}'.format(suite=name)
     return (subject.strip(), body.strip())
 

From 6afb238aba2c9b262f2acd295d599eb04ab78c1b Mon Sep 17 00:00:00 2001
From: Zack Cerza <zack@cerza.org>
Date: Tue, 27 Aug 2013 17:26:46 -0500
Subject: [PATCH 16/19] Time is an integer, in seconds.

---
 teuthology/suite.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/teuthology/suite.py b/teuthology/suite.py
index ecbdcb49fc5..8cb740172c5 100644
--- a/teuthology/suite.py
+++ b/teuthology/suite.py
@@ -414,7 +414,7 @@ email_templates = {
     'fail_templ': dedent("""\
         [{job_id}]  {desc}
         -----------------------------------------------------------------
-        time:   {time}{log_line}{sentry_line}
+        time:   {time}s{log_line}{sentry_line}
 
         {reason}
 
@@ -426,7 +426,7 @@ email_templates = {
         """),
     'pass_templ': dedent("""\
         [{job_id}] {desc}
-        time:    {time}
+        time:    {time}s
 
         """),
 }
@@ -454,7 +454,7 @@ def build_email_body(name, archive_dir, timeout):
             passed[job] = email_templates['pass_templ'].format(
                 job_id=job,
                 desc=summary.get('description'),
-                time=summary.get('duration'),
+                time=int(summary.get('duration')),
             )
         else:
             log = get_http_log_path(archive_dir, job)
@@ -472,7 +472,7 @@ def build_email_body(name, archive_dir, timeout):
             failed[job] = email_templates['fail_templ'].format(
                 job_id=job,
                 desc=summary.get('description'),
-                time=summary.get('duration'),
+                time=int(summary.get('duration')),
                 reason=fill(summary.get('failure_reason'), 79),
                 log_line=log_line,
                 sentry_line=sentry_line,

From 25defd40cc0126944b87d531fb644a96e1e1fd51 Mon Sep 17 00:00:00 2001
From: Zack Cerza <zack@cerza.org>
Date: Wed, 28 Aug 2013 11:12:10 -0500
Subject: [PATCH 17/19] Indent wrapped exceptions.

---
 teuthology/suite.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/teuthology/suite.py b/teuthology/suite.py
index 8cb740172c5..e15a25f687b 100644
--- a/teuthology/suite.py
+++ b/teuthology/suite.py
@@ -469,11 +469,18 @@ def build_email_body(name, archive_dir, timeout):
             else:
                 sentry_line = ''
 
+            # 'fill' is from the textwrap module and it collapses a given
+            # string into multiple lines of a maximum width as specified. We
+            # want 75 characters here so that when we indent by 4 on the next
+            # line, we have 79-character exception paragraphs.
+            reason = fill(summary.get('failure_reason'), 75)
+            reason = '\n'.join(('    ') + line for line in reason.splitlines())
+
             failed[job] = email_templates['fail_templ'].format(
                 job_id=job,
                 desc=summary.get('description'),
                 time=int(summary.get('duration')),
-                reason=fill(summary.get('failure_reason'), 79),
+                reason=reason,
                 log_line=log_line,
                 sentry_line=sentry_line,
             )

From 53cea02a0083742c515c03fe8470d914dbd5e1f8 Mon Sep 17 00:00:00 2001
From: Zack Cerza <zack@cerza.org>
Date: Wed, 28 Aug 2013 11:15:27 -0500
Subject: [PATCH 18/19] Add apology for non-public links

---
 teuthology/suite.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/teuthology/suite.py b/teuthology/suite.py
index e15a25f687b..1a3e13b229f 100644
--- a/teuthology/suite.py
+++ b/teuthology/suite.py
@@ -398,6 +398,7 @@ def get_jobs(archive_dir):
 email_templates = {
     'body_templ': dedent("""\
         Test Run
+        NOTE: Apologies for links inside the Inktank firewall; we are working to make them public.
         =================================================================
         logs:   {log_root}
         failed: {fail_count}

From 38a47ecef0518ba5711df78740cdff064a5bb870 Mon Sep 17 00:00:00 2001
From: Zack Cerza <zack@cerza.org>
Date: Wed, 28 Aug 2013 11:19:04 -0500
Subject: [PATCH 19/19] Don't return inside __init__

---
 teuthology/config.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/teuthology/config.py b/teuthology/config.py
index 45b0b92add8..24e466f5b8d 100644
--- a/teuthology/config.py
+++ b/teuthology/config.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 import os
 import yaml
 import logging
@@ -15,12 +14,11 @@ class _Config(object):
     ~/.teuthology.yaml and nothing else.
     """
     def __init__(self):
-        if not os.path.exists(CONF_FILE):
+        if os.path.exists(CONF_FILE):
+            self.__conf = yaml.safe_load(file(CONF_FILE))
+        else:
             log.debug("%s not found", CONF_FILE)
             self.__conf = {}
-            return
-
-        self.__conf = yaml.safe_load(file(CONF_FILE))
 
     # This property declaration exists mainly as an example; it is not
     # necessary unless you want to, say, define a set method and/or a