Merge pull request #275 from ceph/wip-replace-update-sh

Replace update.sh functionality

scripts/suite.py

@@ -4,8 +4,7 @@ import teuthology.suite
 
 doc = """
 usage: teuthology-suite [-h]
-       teuthology-suite --suite <suite> [options]
-       teuthology-suite  -s <suite> [options] [<config_yaml>...]
+       teuthology-suite  --suite <suite> [options] [<config_yaml>...]
 
 Run a suite of ceph integration tests. A suite is a directory containing
 facets. A facet is a directory containing config snippets. Running a suite
@@ -23,8 +22,6 @@ Miscellaneous arguments:
 
 Standard arguments:
   <config_yaml>               Optional extra job yaml to include
-  --base <base>               Base directory for the suite
-                              e.g. ~/src/ceph-qa-suite/suites
   -s <suite>, --suite <suite>
                               The suite to schedule
   -c <ceph>, --ceph <ceph>    The ceph branch to run against
@@ -44,6 +41,11 @@ Standard arguments:
   -d <distro>, --distro <distro>
                               Distribution to run against
                               [default: ubuntu]
+  --suite-branch <suite_branch>
+                              Use this suite branch instead of the ceph branch
+  --suite-dir <suite_dir>     Use this alternative directory as-is when
+                              assembling jobs from yaml fragments. This causes
+                              <suite_branch> to be ignored.
 
 Scheduler arguments:
   --owner <owner>             Job owner

teuthology/repo_utils.py

@@ -0,0 +1,130 @@
+import logging
+import os
+import shutil
+import subprocess
+import time
+
+log = logging.getLogger(__name__)
+
+
+def enforce_repo_state(repo_url, dest_path, branch):
+    """
+    Use git to either clone or update a given repo, forcing it to switch to the
+    specified branch.
+
+    :param repo_url:  The full URL to the repo (not including the branch)
+    :param dest_path: The full path to the destination directory
+    :param branch:    The branch.
+    :raises:          BranchNotFoundError if the branch is not found;
+                      RuntimeError for other errors
+    """
+    validate_branch(branch)
+    try:
+        if not os.path.isdir(dest_path):
+            clone_repo(repo_url, dest_path, branch)
+        elif time.time() - os.stat('/etc/passwd').st_mtime > 60:
+            # only do this at most once per minute
+            fetch_branch(dest_path, branch)
+            out = subprocess.check_output(('touch', dest_path))
+            if out:
+                log.info(out)
+        else:
+            log.info("%s was just updated; assuming it is current", branch)
+
+        reset_repo(repo_url, dest_path, branch)
+    except BranchNotFoundError:
+        shutil.rmtree(dest_path, ignore_errors=True)
+        raise
+
+
+def clone_repo(repo_url, dest_path, branch):
+    """
+    Clone a repo into a path
+
+    :param repo_url:  The full URL to the repo (not including the branch)
+    :param dest_path: The full path to the destination directory
+    :param branch:    The branch.
+    :raises:          BranchNotFoundError if the branch is not found;
+                      RuntimeError for other errors
+    """
+    validate_branch(branch)
+    log.info("Cloning %s %s from upstream", repo_url, branch)
+    proc = subprocess.Popen(
+        ('git', 'clone', '--branch', branch, repo_url, dest_path),
+        cwd=os.path.dirname(dest_path),
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT)
+    if proc.wait() != 0:
+        not_found_str = "Remote branch %s not found" % branch
+        out = proc.stdout.read()
+        log.error(out)
+        if not_found_str in out:
+            raise BranchNotFoundError(branch, repo_url)
+        else:
+            raise RuntimeError("git clone failed!")
+
+
+def fetch_branch(dest_path, branch):
+    """
+    Call "git fetch -p origin <branch>"
+
+    :param dest_path: The full path to the destination directory
+    :param branch:    The branch.
+    :raises:          BranchNotFoundError if the branch is not found;
+                      RuntimeError for other errors
+    """
+    validate_branch(branch)
+    log.info("Fetching %s from upstream", branch)
+    proc = subprocess.Popen(
+        ('git', 'fetch', '-p', 'origin', branch),
+        cwd=dest_path,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT)
+    if proc.wait() != 0:
+        not_found_str = "fatal: Couldn't find remote ref %s" % branch
+        out = proc.stdout.read()
+        log.error(out)
+        if not_found_str in out:
+            raise BranchNotFoundError(branch)
+        else:
+            raise RuntimeError("git fetch failed!")
+
+
+def reset_repo(repo_url, dest_path, branch):
+    """
+
+    :param repo_url:  The full URL to the repo (not including the branch)
+    :param dest_path: The full path to the destination directory
+    :param branch:    The branch.
+    :raises:          BranchNotFoundError if the branch is not found;
+                      RuntimeError for other errors
+    """
+    validate_branch(branch)
+    # This try/except block will notice if the requested branch doesn't
+    # exist, whether it was cloned or fetched.
+    try:
+        subprocess.check_output(
+            ('git', 'reset', '--hard', 'origin/%s' % branch),
+            cwd=dest_path,
+        )
+    except subprocess.CalledProcessError:
+        raise BranchNotFoundError(branch, repo_url)
+
+
+class BranchNotFoundError(ValueError):
+    def __init__(self, branch, repo=None):
+        self.branch = branch
+        self.repo = repo
+
+    def __str__(self):
+        if self.repo:
+            repo_str = " in repo: %s" % self.repo
+        else:
+            repo_str = ""
+        return "Branch {branch} not found{repo_str}!".format(
+            branch=self.branch, repo_str=repo_str)
+
+
+def validate_branch(branch):
+    if ' ' in branch:
+        raise ValueError("Illegal branch name: '%s'" % branch)

teuthology/suite.py

@@ -17,8 +17,9 @@ from email.mime.text import MIMEText
 from tempfile import NamedTemporaryFile
 
 import teuthology
-from teuthology import lock as lock
-from teuthology.config import config
+from . import lock
+from .config import config
+from .repo_utils import enforce_repo_state, BranchNotFoundError
 
 log = logging.getLogger(__name__)
 
@@ -30,9 +31,6 @@ def main(args):
     dry_run = args['--dry-run']
 
     base_yaml_paths = args['<config_yaml>']
-    base = os.path.expanduser(args['--base'])
-    if not os.path.exists(base):
-        schedule_fail("Base directory not found: {dir}".format(dir=base))
     suite = args['--suite']
     nice_suite = suite.replace('/', ':')
     ceph_branch = args['--ceph']
@@ -41,6 +39,8 @@ def main(args):
     teuthology_branch = args['--teuthology-branch']
     machine_type = args['--machine-type']
     distro = args['--distro']
+    suite_branch = args['--suite-branch'] or ceph_branch
+    suite_dir = args['--suite-dir']
 
     limit = int(args['--limit'])
     priority = int(args['--priority'])
@@ -54,6 +54,12 @@ def main(args):
 
     name = make_run_name(nice_suite, ceph_branch, kernel_branch, kernel_flavor,
                          machine_type)
+
+    if suite_dir:
+        suite_repo_path = suite_dir
+    else:
+        suite_repo_path = fetch_suite_repo(suite_branch, test_name=name)
+
     config_string = create_initial_config(nice_suite, ceph_branch,
                                           teuthology_branch, kernel_branch,
                                           kernel_flavor, distro, machine_type)
@@ -64,19 +70,19 @@ def main(args):
         base_yaml_path = base_yaml.name
     base_yaml_paths.insert(0, base_yaml_path)
     prepare_and_schedule(owner=owner,
-                            name=name,
-                            suite=suite,
-                            machine_type=machine_type,
-                            base=base,
-                            base_yaml_paths=base_yaml_paths,
-                            email=email,
-                            priority=priority,
-                            limit=limit,
-                            num=num,
-                            timeout=timeout,
-                            dry_run=dry_run,
-                            verbose=verbose,
-                            )
+                         name=name,
+                         suite=suite,
+                         machine_type=machine_type,
+                         suite_repo_path=suite_repo_path,
+                         base_yaml_paths=base_yaml_paths,
+                         email=email,
+                         priority=priority,
+                         limit=limit,
+                         num=num,
+                         timeout=timeout,
+                         dry_run=dry_run,
+                         verbose=verbose,
+                         )
     os.remove(base_yaml_path)
 
 
@@ -99,6 +105,39 @@ def make_run_name(suite, ceph_branch, kernel_branch, kernel_flavor,
     )
 
 
+def fetch_suite_repo(branch, test_name):
+    """
+    Fetch the suite repo (and also the teuthology repo) so that we can use it
+    to build jobs. Repos are stored in ~/src/.
+
+    The reason the teuthology repo is also fetched is that currently we use
+    subprocess to call teuthology-schedule to schedule jobs so we need to make
+    sure it is up-to-date. For that reason we always fetch the master branch
+    for test scheduling, regardless of what teuthology branch is requested for
+    testing.
+
+    :returns: The path to the repo on disk
+    """
+    src_base_path = os.path.expanduser('~/src')
+    if not os.path.exists(src_base_path):
+        os.mkdir(src_base_path)
+    suite_repo_path = os.path.join(src_base_path,
+                                   'ceph-qa-suite_' + branch)
+    try:
+        enforce_repo_state(
+            repo_url=os.path.join(config.ceph_git_base_url, 'teuthology.git'),
+            dest_path=os.path.join(src_base_path, 'teuthology'),
+            branch='master')
+        enforce_repo_state(
+            repo_url=os.path.join(config.ceph_git_base_url,
+                                  'ceph-qa-suite.git'),
+            dest_path=suite_repo_path,
+            branch=branch)
+    except BranchNotFoundError as exc:
+        schedule_fail(message=str(exc), name=test_name)
+    return suite_repo_path
+
+
 def create_initial_config(nice_suite, ceph_branch, teuthology_branch,
                           kernel_branch, kernel_flavor, distro, machine_type):
     """
@@ -119,12 +158,13 @@ def create_initial_config(nice_suite, ceph_branch, teuthology_branch,
         kernel_hash = get_hash('kernel', kernel_branch, kernel_flavor,
                                machine_type)
         if not kernel_hash:
-            schedule_fail(message="Kernel branch '{branch} not found".format(
+            schedule_fail(message="Kernel branch '{branch}' not found".format(
                 branch=kernel_branch))
     if kernel_hash:
         log.info("kernel sha1: {hash}".format(hash=kernel_hash))
         kernel_dict = dict(kernel=dict(kdb=True, sha1=kernel_hash))
-        kernel_stanza = yaml.dump(kernel_dict, default_flow_style=False).strip()
+        kernel_stanza = yaml.dump(kernel_dict,
+                                  default_flow_style=False).strip()
     else:
         kernel_stanza = ''
 
@@ -175,7 +215,7 @@ def create_initial_config(nice_suite, ceph_branch, teuthology_branch,
     return config_template.format(**config_input)
 
 
-def prepare_and_schedule(owner, name, suite, machine_type, base,
+def prepare_and_schedule(owner, name, suite, machine_type, suite_repo_path,
                          base_yaml_paths, email, priority, limit, num, timeout,
                          dry_run, verbose):
     """
@@ -199,7 +239,7 @@ def prepare_and_schedule(owner, name, suite, machine_type, base,
     if owner:
         base_args.extend(['--owner', owner])
 
-    suite_path = os.path.join(base, suite)
+    suite_path = os.path.join(suite_repo_path, 'suites', suite)
 
     num_jobs = schedule_suite(
         name=suite,
@@ -227,7 +267,7 @@ def prepare_and_schedule(owner, name, suite, machine_type, base,
             )
 
 
-def schedule_fail(message, name=None):
+def schedule_fail(message, name=''):
     """
     If an email address has been specified anywhere, send an alert there. Then
     raise a ScheduleFailError.
@@ -279,9 +319,9 @@ def get_hash(project='ceph', branch='master', flavor='basic',
     """
     # Alternate method for github-hosted projects - left here for informational
     # purposes
-    #resp = requests.get(
-    #    'https://api.github.com/repos/ceph/ceph/git/refs/heads/master')
-    #hash = .json()['object']['sha']
+    # resp = requests.get(
+    #     'https://api.github.com/repos/ceph/ceph/git/refs/heads/master')
+    # hash = .json()['object']['sha']
     (arch, release, pkg_type) = get_distro_defaults(distro, machine_type)
     base_url = get_gitbuilder_url(project, release, pkg_type, arch, flavor)
     url = os.path.join(base_url, 'ref', branch, 'sha1')

teuthology/test/test_repo_utils.py

@@ -0,0 +1,110 @@
+import logging
+import os.path
+from pytest import raises
+import shutil
+import subprocess
+
+from .. import repo_utils
+repo_utils.log.setLevel(logging.WARNING)
+
+
+class TestRepoUtils(object):
+    src_path = '/tmp/empty_src'
+    repo_url = 'file://' + src_path
+    dest_path = '/tmp/empty_dest'
+
+    def setup_method(self, method):
+        assert not os.path.exists(self.dest_path)
+        proc = subprocess.Popen(
+            ('git', 'init', self.src_path),
+            stdout=subprocess.PIPE,
+        )
+        assert proc.wait() == 0
+        proc = subprocess.Popen(
+            ('git', 'config', 'user.email', 'test@ceph.com'),
+            cwd=self.src_path,
+            stdout=subprocess.PIPE,
+        )
+        assert proc.wait() == 0
+        proc = subprocess.Popen(
+            ('git', 'config', 'user.name', 'Test User'),
+            cwd=self.src_path,
+            stdout=subprocess.PIPE,
+        )
+        assert proc.wait() == 0
+        proc = subprocess.Popen(
+            ('git', 'commit', '--allow-empty', '--allow-empty-message',
+             '--no-edit'),
+            cwd=self.src_path,
+            stdout=subprocess.PIPE,
+        )
+        assert proc.wait() == 0
+
+    def teardown_method(self, method):
+        shutil.rmtree(self.dest_path, ignore_errors=True)
+
+    def test_clone_repo_existing_branch(self):
+        repo_utils.clone_repo(self.repo_url, self.dest_path, 'master')
+        assert os.path.exists(self.dest_path)
+
+    def test_clone_repo_non_existing_branch(self):
+        with raises(repo_utils.BranchNotFoundError):
+            repo_utils.clone_repo(self.repo_url, self.dest_path, 'nobranch')
+        assert not os.path.exists(self.dest_path)
+
+    def test_fetch_branch_no_repo(self):
+        fake_dest_path = '/tmp/not_a_repo'
+        assert not os.path.exists(fake_dest_path)
+        with raises(OSError):
+            repo_utils.fetch_branch(fake_dest_path, 'master')
+        assert not os.path.exists(fake_dest_path)
+
+    def test_fetch_branch_fake_branch(self):
+        repo_utils.clone_repo(self.repo_url, self.dest_path, 'master')
+        with raises(repo_utils.BranchNotFoundError):
+            repo_utils.fetch_branch(self.dest_path, 'nobranch')
+
+    def test_enforce_existing_branch(self):
+        repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
+                                      'master')
+        assert os.path.exists(self.dest_path)
+
+    def test_enforce_non_existing_branch(self):
+        with raises(repo_utils.BranchNotFoundError):
+            repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
+                                          'blah')
+        assert not os.path.exists(self.dest_path)
+
+    def test_enforce_multiple_calls_same_branch(self):
+        repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
+                                      'master')
+        assert os.path.exists(self.dest_path)
+        repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
+                                      'master')
+        assert os.path.exists(self.dest_path)
+        repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
+                                      'master')
+        assert os.path.exists(self.dest_path)
+
+    def test_enforce_multiple_calls_different_branches(self):
+        with raises(repo_utils.BranchNotFoundError):
+            repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
+                                          'blah1')
+        assert not os.path.exists(self.dest_path)
+        repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
+                                      'master')
+        assert os.path.exists(self.dest_path)
+        repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
+                                      'master')
+        assert os.path.exists(self.dest_path)
+        with raises(repo_utils.BranchNotFoundError):
+            repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
+                                          'blah2')
+        assert not os.path.exists(self.dest_path)
+        repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
+                                      'master')
+        assert os.path.exists(self.dest_path)
+
+    def test_enforce_invalid_branch(self):
+        with raises(ValueError):
+            repo_utils.enforce_repo_state(self.repo_url, self.dest_path, 'a b')

teuthology/worker.py

@@ -2,7 +2,6 @@ import fcntl
 import logging
 import os
 import subprocess
-import shutil
 import sys
 import tempfile
 import time
@@ -18,6 +17,7 @@ from . import safepath
 from .config import config as teuth_config
 from .kill import kill_job
 from .misc import read_config
+from .repo_utils import enforce_repo_state, BranchNotFoundError
 
 log = logging.getLogger(__name__)
 start_time = datetime.utcnow()
@@ -75,70 +75,26 @@ class filelock(object):
         self.fd = None
 
 
-class BranchNotFoundError(ValueError):
-    def __init__(self, branch):
-        self.branch = branch
-
-    def __str__(self):
-        return "teuthology branch not found: '{0}'".format(self.branch)
-
-
-def fetch_teuthology_branch(path, branch='master'):
+def fetch_teuthology_branch(dest_path, branch='master'):
     """
     Make sure we have the correct teuthology branch checked out and up-to-date
     """
     # only let one worker create/update the checkout at a time
-    lock = filelock('%s.lock' % path)
+    lock = filelock('%s.lock' % dest_path)
     lock.acquire()
     try:
-        #if os.path.isdir(path):
-        #    p = subprocess.Popen('git status', shell=True, cwd=path)
-        #    if p.wait() == 128:
-        #        log.info("Repo at %s appears corrupt; removing",
-        #                 branch)
-        #        shutil.rmtree(path)
+        teuthology_git_upstream = teuth_config.ceph_git_base_url + \
+            'teuthology.git'
+        enforce_repo_state(teuthology_git_upstream, dest_path, branch)
 
-        if not os.path.isdir(path):
-            log.info("Cloning %s from upstream", branch)
-            teuthology_git_upstream = teuth_config.ceph_git_base_url + \
-                'teuthology.git'
-            log.info(
-                subprocess.check_output(('git', 'clone', '--branch', branch,
-                                         teuthology_git_upstream, path),
-                                        cwd=os.path.dirname(path))
-            )
-        elif time.time() - os.stat('/etc/passwd').st_mtime > 60:
-            # only do this at most once per minute
-            log.info("Fetching %s from upstream", branch)
-            log.info(
-                subprocess.check_output(('git', 'fetch', '-p', 'origin'),
-                                        cwd=path)
-            )
-            log.info(
-                subprocess.check_output(('touch', path))
-            )
-        else:
-            log.info("%s was just updated; assuming it is current", branch)
-
-        # This try/except block will notice if the requested branch doesn't
-        # exist, whether it was cloned or fetched.
-        try:
-            subprocess.check_output(
-                ('git', 'reset', '--hard', 'origin/%s' % branch),
-                cwd=path,
-            )
-        except subprocess.CalledProcessError:
-            shutil.rmtree(path)
-            raise BranchNotFoundError(branch)
-
-        log.debug("Bootstrapping %s", path)
+        log.debug("Bootstrapping %s", dest_path)
         # This magic makes the bootstrap script not attempt to clobber an
         # existing virtualenv. But the branch's bootstrap needs to actually
         # check for the NO_CLOBBER variable.
         env = os.environ.copy()
         env['NO_CLOBBER'] = '1'
         cmd = './bootstrap'
-        boot_proc = subprocess.Popen(cmd, shell=True, cwd=path, env=env,
+        boot_proc = subprocess.Popen(cmd, shell=True, cwd=dest_path, env=env,
                                      stdout=subprocess.PIPE,
                                      stderr=subprocess.STDOUT)
         returncode = boot_proc.wait()
@@ -214,7 +170,8 @@ def main(ctx):
                                   'teuthology-' + teuthology_branch)
 
         try:
-            fetch_teuthology_branch(path=teuth_path, branch=teuthology_branch)
+            fetch_teuthology_branch(dest_path=teuth_path,
+                                    branch=teuthology_branch)
         except BranchNotFoundError:
             log.exception(
                 "Branch not found; throwing job away")