From 5e3c13de60c2a82f2e9ecc710400329bc48c9a9a Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Wed, 25 Jun 2014 13:23:54 -0600 Subject: [PATCH 01/19] Use a more informative variable name for dest_path Signed-off-by: Zack Cerza --- teuthology/worker.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/teuthology/worker.py b/teuthology/worker.py index 8c9b2c38a68..47693113124 100644 --- a/teuthology/worker.py +++ b/teuthology/worker.py @@ -83,12 +83,12 @@ class BranchNotFoundError(ValueError): return "teuthology branch not found: '{0}'".format(self.branch) -def fetch_teuthology_branch(path, branch='master'): +def fetch_teuthology_branch(dest_path, branch='master'): """ Make sure we have the correct teuthology branch checked out and up-to-date """ # only let one worker create/update the checkout at a time - lock = filelock('%s.lock' % path) + lock = filelock('%s.lock' % dest_path) lock.acquire() try: #if os.path.isdir(path): @@ -98,24 +98,24 @@ def fetch_teuthology_branch(path, branch='master'): # branch) # shutil.rmtree(path) - if not os.path.isdir(path): + if not os.path.isdir(dest_path): log.info("Cloning %s from upstream", branch) teuthology_git_upstream = teuth_config.ceph_git_base_url + \ 'teuthology.git' log.info( subprocess.check_output(('git', 'clone', '--branch', branch, - teuthology_git_upstream, path), - cwd=os.path.dirname(path)) + teuthology_git_upstream, dest_path), + cwd=os.path.dirname(dest_path)) ) elif time.time() - os.stat('/etc/passwd').st_mtime > 60: # only do this at most once per minute log.info("Fetching %s from upstream", branch) log.info( subprocess.check_output(('git', 'fetch', '-p', 'origin'), - cwd=path) + cwd=dest_path) ) log.info( - subprocess.check_output(('touch', path)) + subprocess.check_output(('touch', dest_path)) ) else: log.info("%s was just updated; assuming it is current", branch) @@ -125,20 +125,20 @@ def fetch_teuthology_branch(path, branch='master'): try: subprocess.check_output( ('git', 'reset', '--hard', 'origin/%s' % branch), - cwd=path, + cwd=dest_path, ) except subprocess.CalledProcessError: - shutil.rmtree(path) + shutil.rmtree(dest_path) raise BranchNotFoundError(branch) - log.debug("Bootstrapping %s", path) + log.debug("Bootstrapping %s", dest_path) # This magic makes the bootstrap script not attempt to clobber an # existing virtualenv. But the branch's bootstrap needs to actually # check for the NO_CLOBBER variable. env = os.environ.copy() env['NO_CLOBBER'] = '1' cmd = './bootstrap' - boot_proc = subprocess.Popen(cmd, shell=True, cwd=path, env=env, + boot_proc = subprocess.Popen(cmd, shell=True, cwd=dest_path, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) returncode = boot_proc.wait() @@ -214,7 +214,8 @@ def main(ctx): 'teuthology-' + teuthology_branch) try: - fetch_teuthology_branch(path=teuth_path, branch=teuthology_branch) + fetch_teuthology_branch(dest_path=teuth_path, + branch=teuthology_branch) except BranchNotFoundError: log.exception( "Branch not found; throwing job away") From 93fd6b899ff3a8ce57b45bda36a694f6432ecf6f Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Wed, 25 Jun 2014 14:20:18 -0600 Subject: [PATCH 02/19] Move repo checkout code to new module repo_utils Signed-off-by: Zack Cerza --- teuthology/repo_utils.py | 55 ++++++++++++++++++++++++++++++++++++++++ teuthology/worker.py | 52 +++---------------------------------- 2 files changed, 59 insertions(+), 48 deletions(-) create mode 100644 teuthology/repo_utils.py diff --git a/teuthology/repo_utils.py b/teuthology/repo_utils.py new file mode 100644 index 00000000000..b34b15a5834 --- /dev/null +++ b/teuthology/repo_utils.py @@ -0,0 +1,55 @@ +import logging +import os +import shutil +import subprocess +import time + +log = logging.getLogger(__name__) + + +def checkout_repo(repo_url, dest_path, branch): + # if os.path.isdir(path): + # p = subprocess.Popen('git status', shell=True, cwd=path) + # if p.wait() == 128: + # log.info("Repo at %s appears corrupt; removing", + # branch) + # shutil.rmtree(path) + + if not os.path.isdir(dest_path): + log.info("Cloning %s %s from upstream", repo_url, branch) + log.info( + subprocess.check_output(('git', 'clone', '--branch', branch, + repo_url, dest_path), + cwd=os.path.dirname(dest_path)) + ) + elif time.time() - os.stat('/etc/passwd').st_mtime > 60: + # only do this at most once per minute + log.info("Fetching %s from upstream", branch) + log.info( + subprocess.check_output(('git', 'fetch', '-p', 'origin'), + cwd=dest_path) + ) + log.info( + subprocess.check_output(('touch', dest_path)) + ) + else: + log.info("%s was just updated; assuming it is current", branch) + + # This try/except block will notice if the requested branch doesn't + # exist, whether it was cloned or fetched. + try: + subprocess.check_output( + ('git', 'reset', '--hard', 'origin/%s' % branch), + cwd=dest_path, + ) + except subprocess.CalledProcessError: + shutil.rmtree(dest_path) + raise BranchNotFoundError(branch) + + +class BranchNotFoundError(ValueError): + def __init__(self, branch): + self.branch = branch + + def __str__(self): + return "teuthology branch not found: '{0}'".format(self.branch) diff --git a/teuthology/worker.py b/teuthology/worker.py index 47693113124..1a44480a5ce 100644 --- a/teuthology/worker.py +++ b/teuthology/worker.py @@ -2,7 +2,6 @@ import fcntl import logging import os import subprocess -import shutil import sys import tempfile import time @@ -18,6 +17,7 @@ from . import safepath from .config import config as teuth_config from .kill import kill_job from .misc import read_config +from .repo_utils import checkout_repo, BranchNotFoundError log = logging.getLogger(__name__) start_time = datetime.utcnow() @@ -75,14 +75,6 @@ class filelock(object): self.fd = None -class BranchNotFoundError(ValueError): - def __init__(self, branch): - self.branch = branch - - def __str__(self): - return "teuthology branch not found: '{0}'".format(self.branch) - - def fetch_teuthology_branch(dest_path, branch='master'): """ Make sure we have the correct teuthology branch checked out and up-to-date @@ -91,45 +83,9 @@ def fetch_teuthology_branch(dest_path, branch='master'): lock = filelock('%s.lock' % dest_path) lock.acquire() try: - #if os.path.isdir(path): - # p = subprocess.Popen('git status', shell=True, cwd=path) - # if p.wait() == 128: - # log.info("Repo at %s appears corrupt; removing", - # branch) - # shutil.rmtree(path) - - if not os.path.isdir(dest_path): - log.info("Cloning %s from upstream", branch) - teuthology_git_upstream = teuth_config.ceph_git_base_url + \ - 'teuthology.git' - log.info( - subprocess.check_output(('git', 'clone', '--branch', branch, - teuthology_git_upstream, dest_path), - cwd=os.path.dirname(dest_path)) - ) - elif time.time() - os.stat('/etc/passwd').st_mtime > 60: - # only do this at most once per minute - log.info("Fetching %s from upstream", branch) - log.info( - subprocess.check_output(('git', 'fetch', '-p', 'origin'), - cwd=dest_path) - ) - log.info( - subprocess.check_output(('touch', dest_path)) - ) - else: - log.info("%s was just updated; assuming it is current", branch) - - # This try/except block will notice if the requested branch doesn't - # exist, whether it was cloned or fetched. - try: - subprocess.check_output( - ('git', 'reset', '--hard', 'origin/%s' % branch), - cwd=dest_path, - ) - except subprocess.CalledProcessError: - shutil.rmtree(dest_path) - raise BranchNotFoundError(branch) + teuthology_git_upstream = teuth_config.ceph_git_base_url + \ + 'teuthology.git' + checkout_repo(teuthology_git_upstream, dest_path, branch) log.debug("Bootstrapping %s", dest_path) # This magic makes the bootstrap script not attempt to clobber an From 484693c2e927a5c8304e0d69937397557bdf363e Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Wed, 25 Jun 2014 14:24:19 -0600 Subject: [PATCH 03/19] Fix linter errors Signed-off-by: Zack Cerza --- teuthology/suite.py | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/teuthology/suite.py b/teuthology/suite.py index 2227d7e40a7..e5e2ed63469 100644 --- a/teuthology/suite.py +++ b/teuthology/suite.py @@ -64,19 +64,19 @@ def main(args): base_yaml_path = base_yaml.name base_yaml_paths.insert(0, base_yaml_path) prepare_and_schedule(owner=owner, - name=name, - suite=suite, - machine_type=machine_type, - base=base, - base_yaml_paths=base_yaml_paths, - email=email, - priority=priority, - limit=limit, - num=num, - timeout=timeout, - dry_run=dry_run, - verbose=verbose, - ) + name=name, + suite=suite, + machine_type=machine_type, + base=base, + base_yaml_paths=base_yaml_paths, + email=email, + priority=priority, + limit=limit, + num=num, + timeout=timeout, + dry_run=dry_run, + verbose=verbose, + ) os.remove(base_yaml_path) @@ -124,7 +124,8 @@ def create_initial_config(nice_suite, ceph_branch, teuthology_branch, if kernel_hash: log.info("kernel sha1: {hash}".format(hash=kernel_hash)) kernel_dict = dict(kernel=dict(kdb=True, sha1=kernel_hash)) - kernel_stanza = yaml.dump(kernel_dict, default_flow_style=False).strip() + kernel_stanza = yaml.dump(kernel_dict, + default_flow_style=False).strip() else: kernel_stanza = '' @@ -279,9 +280,9 @@ def get_hash(project='ceph', branch='master', flavor='basic', """ # Alternate method for github-hosted projects - left here for informational # purposes - #resp = requests.get( - # 'https://api.github.com/repos/ceph/ceph/git/refs/heads/master') - #hash = .json()['object']['sha'] + # resp = requests.get( + # 'https://api.github.com/repos/ceph/ceph/git/refs/heads/master') + # hash = .json()['object']['sha'] (arch, release, pkg_type) = get_distro_defaults(distro, machine_type) base_url = get_gitbuilder_url(project, release, pkg_type, arch, flavor) url = os.path.join(base_url, 'ref', branch, 'sha1') From 6e3e6693d157db9aa7b0ba5fd988ca11f90125a0 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Wed, 25 Jun 2014 14:40:44 -0600 Subject: [PATCH 04/19] Generalize error message Signed-off-by: Zack Cerza --- teuthology/repo_utils.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/teuthology/repo_utils.py b/teuthology/repo_utils.py index b34b15a5834..50d64236f4d 100644 --- a/teuthology/repo_utils.py +++ b/teuthology/repo_utils.py @@ -44,12 +44,14 @@ def checkout_repo(repo_url, dest_path, branch): ) except subprocess.CalledProcessError: shutil.rmtree(dest_path) - raise BranchNotFoundError(branch) + raise BranchNotFoundError(branch, repo_url) class BranchNotFoundError(ValueError): - def __init__(self, branch): + def __init__(self, branch, repo): self.branch = branch + self.repo = repo def __str__(self): - return "teuthology branch not found: '{0}'".format(self.branch) + return "Branch {branch} not found in repo: {repo}".format( + branch=self.branch, repo=self.repo) From 88d26e4ae4406853a73e50eb29999c2a8e59c306 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Wed, 25 Jun 2014 14:59:38 -0600 Subject: [PATCH 05/19] Handle ceph-qa-suite checkouts Previously we had an internally-maintained shell script that did this. Signed-off-by: Zack Cerza --- scripts/suite.py | 2 -- teuthology/suite.py | 42 ++++++++++++++++++++++++++++++++---------- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/scripts/suite.py b/scripts/suite.py index 1842890c5b4..a30a6e183d6 100644 --- a/scripts/suite.py +++ b/scripts/suite.py @@ -23,8 +23,6 @@ Miscellaneous arguments: Standard arguments: Optional extra job yaml to include - --base Base directory for the suite - e.g. ~/src/ceph-qa-suite/suites -s , --suite The suite to schedule -c , --ceph The ceph branch to run against diff --git a/teuthology/suite.py b/teuthology/suite.py index e5e2ed63469..62a8e31601d 100644 --- a/teuthology/suite.py +++ b/teuthology/suite.py @@ -17,8 +17,9 @@ from email.mime.text import MIMEText from tempfile import NamedTemporaryFile import teuthology -from teuthology import lock as lock -from teuthology.config import config +from . import lock +from .config import config +from .repo_utils import checkout_repo, BranchNotFoundError log = logging.getLogger(__name__) @@ -30,9 +31,6 @@ def main(args): dry_run = args['--dry-run'] base_yaml_paths = args[''] - base = os.path.expanduser(args['--base']) - if not os.path.exists(base): - schedule_fail("Base directory not found: {dir}".format(dir=base)) suite = args['--suite'] nice_suite = suite.replace('/', ':') ceph_branch = args['--ceph'] @@ -54,6 +52,9 @@ def main(args): name = make_run_name(nice_suite, ceph_branch, kernel_branch, kernel_flavor, machine_type) + + suite_repo_path = fetch_suite_repo(ceph_branch, test_name=name) + config_string = create_initial_config(nice_suite, ceph_branch, teuthology_branch, kernel_branch, kernel_flavor, distro, machine_type) @@ -67,7 +68,7 @@ def main(args): name=name, suite=suite, machine_type=machine_type, - base=base, + suite_repo_path=suite_repo_path, base_yaml_paths=base_yaml_paths, email=email, priority=priority, @@ -99,6 +100,27 @@ def make_run_name(suite, ceph_branch, kernel_branch, kernel_flavor, ) +def fetch_suite_repo(branch, test_name): + """ + Fetch the suite repo so we can use it to build jobs + + :returns: The path to the repo on disk + """ + src_base_path = os.path.expanduser('~/src') + if not os.path.exists(src_base_path): + os.mkdir(src_base_path) + suite_repo_path = os.path.join(src_base_path, + 'ceph-qa-suite_' + branch) + try: + checkout_repo( + repo_url=os.path.join(config.ceph_git_base_url, 'ceph-qa-suite'), + dest_path=suite_repo_path, + branch=branch) + except BranchNotFoundError as exc: + schedule_fail(message=str(exc), name=test_name) + return suite_repo_path + + def create_initial_config(nice_suite, ceph_branch, teuthology_branch, kernel_branch, kernel_flavor, distro, machine_type): """ @@ -119,7 +141,7 @@ def create_initial_config(nice_suite, ceph_branch, teuthology_branch, kernel_hash = get_hash('kernel', kernel_branch, kernel_flavor, machine_type) if not kernel_hash: - schedule_fail(message="Kernel branch '{branch} not found".format( + schedule_fail(message="Kernel branch '{branch}' not found".format( branch=kernel_branch)) if kernel_hash: log.info("kernel sha1: {hash}".format(hash=kernel_hash)) @@ -176,7 +198,7 @@ def create_initial_config(nice_suite, ceph_branch, teuthology_branch, return config_template.format(**config_input) -def prepare_and_schedule(owner, name, suite, machine_type, base, +def prepare_and_schedule(owner, name, suite, machine_type, suite_repo_path, base_yaml_paths, email, priority, limit, num, timeout, dry_run, verbose): """ @@ -200,7 +222,7 @@ def prepare_and_schedule(owner, name, suite, machine_type, base, if owner: base_args.extend(['--owner', owner]) - suite_path = os.path.join(base, suite) + suite_path = os.path.join(suite_repo_path, 'suites', suite) num_jobs = schedule_suite( name=suite, @@ -228,7 +250,7 @@ def prepare_and_schedule(owner, name, suite, machine_type, base, ) -def schedule_fail(message, name=None): +def schedule_fail(message, name=''): """ If an email address has been specified anywhere, send an alert there. Then raise a ScheduleFailError. From b16c48a8f8b8b10a510cb4f3ab5b818fd1cf2155 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Wed, 25 Jun 2014 15:10:37 -0600 Subject: [PATCH 06/19] Also handle teuthology repo checkouts Our shell script did this also. Signed-off-by: Zack Cerza --- teuthology/suite.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/teuthology/suite.py b/teuthology/suite.py index 62a8e31601d..e5b6f293400 100644 --- a/teuthology/suite.py +++ b/teuthology/suite.py @@ -102,7 +102,8 @@ def make_run_name(suite, ceph_branch, kernel_branch, kernel_flavor, def fetch_suite_repo(branch, test_name): """ - Fetch the suite repo so we can use it to build jobs + Fetch the suite repo (and also the teuthology repo) so that we can use it + to build jobs. Repos are stored in ~/src/. :returns: The path to the repo on disk """ @@ -112,6 +113,10 @@ def fetch_suite_repo(branch, test_name): suite_repo_path = os.path.join(src_base_path, 'ceph-qa-suite_' + branch) try: + checkout_repo( + repo_url=os.path.join(config.ceph_git_base_url, 'teuthology.git'), + dest_path=os.path.join(src_base_path, 'teuthology'), + branch='master') checkout_repo( repo_url=os.path.join(config.ceph_git_base_url, 'ceph-qa-suite'), dest_path=suite_repo_path, From 807d6fd7150ef56270234b030a09909413072b1d Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Wed, 25 Jun 2014 15:24:05 -0600 Subject: [PATCH 07/19] Use 'ceph-qa-suite.git' for the repo name Signed-off-by: Zack Cerza --- teuthology/suite.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/teuthology/suite.py b/teuthology/suite.py index e5b6f293400..ca409ed0d02 100644 --- a/teuthology/suite.py +++ b/teuthology/suite.py @@ -118,7 +118,8 @@ def fetch_suite_repo(branch, test_name): dest_path=os.path.join(src_base_path, 'teuthology'), branch='master') checkout_repo( - repo_url=os.path.join(config.ceph_git_base_url, 'ceph-qa-suite'), + repo_url=os.path.join(config.ceph_git_base_url, + 'ceph-qa-suite.git'), dest_path=suite_repo_path, branch=branch) except BranchNotFoundError as exc: From abd359086c9ad99becfe1f61c8e22212c76bcc58 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Thu, 26 Jun 2014 13:37:07 -0600 Subject: [PATCH 08/19] Add unit tests for repo_utils Signed-off-by: Zack Cerza --- teuthology/test/test_repo_utils.py | 49 ++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 teuthology/test/test_repo_utils.py diff --git a/teuthology/test/test_repo_utils.py b/teuthology/test/test_repo_utils.py new file mode 100644 index 00000000000..fdb82278343 --- /dev/null +++ b/teuthology/test/test_repo_utils.py @@ -0,0 +1,49 @@ +import logging +import os.path +from pytest import raises +import shutil + +from .. import repo_utils +repo_utils.log.setLevel(logging.WARNING) + + +class TestRepoUtils(object): + empty_repo = 'https://github.com/ceph/empty' + local_dir = '/tmp/empty' + + def setup(self): + assert not os.path.exists(self.local_dir) + + def teardown(self): + shutil.rmtree(self.local_dir, ignore_errors=True) + + def test_existing_branch(self): + repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'master') + assert os.path.exists(self.local_dir) + + def test_non_existing_branch(self): + with raises(repo_utils.BranchNotFoundError): + repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'blah') + assert not os.path.exists(self.local_dir) + + def test_multiple_calls_same_branch(self): + repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'master') + assert os.path.exists(self.local_dir) + repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'master') + assert os.path.exists(self.local_dir) + repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'master') + assert os.path.exists(self.local_dir) + + def test_multiple_calls_different_branches(self): + with raises(repo_utils.BranchNotFoundError): + repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'blah') + assert not os.path.exists(self.local_dir) + repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'master') + assert os.path.exists(self.local_dir) + repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'master') + assert os.path.exists(self.local_dir) + with raises(repo_utils.BranchNotFoundError): + repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'blah') + assert not os.path.exists(self.local_dir) + repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'master') + assert os.path.exists(self.local_dir) From f5bed55d4792728744d7d990de2f076732725766 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Thu, 26 Jun 2014 13:37:24 -0600 Subject: [PATCH 09/19] Improve missing branch detection and logging Signed-off-by: Zack Cerza --- teuthology/repo_utils.py | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/teuthology/repo_utils.py b/teuthology/repo_utils.py index 50d64236f4d..e085f52011e 100644 --- a/teuthology/repo_utils.py +++ b/teuthology/repo_utils.py @@ -17,21 +17,29 @@ def checkout_repo(repo_url, dest_path, branch): if not os.path.isdir(dest_path): log.info("Cloning %s %s from upstream", repo_url, branch) - log.info( - subprocess.check_output(('git', 'clone', '--branch', branch, - repo_url, dest_path), - cwd=os.path.dirname(dest_path)) - ) + proc = subprocess.Popen( + ('git', 'clone', '--branch', branch, repo_url, dest_path), + cwd=os.path.dirname(dest_path), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + not_found_str = "Remote branch %s not found" % branch + out = proc.stdout.read() + if proc.wait() != 0: + log.error(out) + if not_found_str in out: + raise BranchNotFoundError(branch, repo_url) + else: + raise RuntimeError("git clone failed!") elif time.time() - os.stat('/etc/passwd').st_mtime > 60: # only do this at most once per minute log.info("Fetching %s from upstream", branch) - log.info( - subprocess.check_output(('git', 'fetch', '-p', 'origin'), - cwd=dest_path) - ) - log.info( - subprocess.check_output(('touch', dest_path)) - ) + out = subprocess.check_output(('git', 'fetch', '-p', 'origin'), + cwd=dest_path) + if out: + log.info(out) + out = subprocess.check_output(('touch', dest_path)) + if out: + log.info(out) else: log.info("%s was just updated; assuming it is current", branch) From a4c3a02547d05f67f4e5d04ea6f02c5388238bf3 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Fri, 27 Jun 2014 10:12:45 -0600 Subject: [PATCH 10/19] Add a note about teuthology scheduling Specifically, that we always schedule using the master branch. Signed-off-by: Zack Cerza --- teuthology/suite.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/teuthology/suite.py b/teuthology/suite.py index ca409ed0d02..9d5f83f7223 100644 --- a/teuthology/suite.py +++ b/teuthology/suite.py @@ -105,6 +105,12 @@ def fetch_suite_repo(branch, test_name): Fetch the suite repo (and also the teuthology repo) so that we can use it to build jobs. Repos are stored in ~/src/. + The reason the teuthology repo is also fetched is that currently we use + subprocess to call teuthology-schedule to schedule jobs so we need to make + sure it is up-to-date. For that reason we always fetch the master branch + for test scheduling, regardless of what teuthology branch is requested for + testing. + :returns: The path to the repo on disk """ src_base_path = os.path.expanduser('~/src') From 25a40677cc835b2946365b363b8b4360e59c0654 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Fri, 27 Jun 2014 10:18:05 -0600 Subject: [PATCH 11/19] Drop a "fix" for a bug that probably never existed Signed-off-by: Zack Cerza --- teuthology/repo_utils.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/teuthology/repo_utils.py b/teuthology/repo_utils.py index e085f52011e..f9b34b6947e 100644 --- a/teuthology/repo_utils.py +++ b/teuthology/repo_utils.py @@ -8,13 +8,6 @@ log = logging.getLogger(__name__) def checkout_repo(repo_url, dest_path, branch): - # if os.path.isdir(path): - # p = subprocess.Popen('git status', shell=True, cwd=path) - # if p.wait() == 128: - # log.info("Repo at %s appears corrupt; removing", - # branch) - # shutil.rmtree(path) - if not os.path.isdir(dest_path): log.info("Cloning %s %s from upstream", repo_url, branch) proc = subprocess.Popen( From 55b65d62f0ff07fdc23624953a7abf9ca46df87e Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Fri, 27 Jun 2014 10:21:53 -0600 Subject: [PATCH 12/19] Document checkout_repo() Signed-off-by: Zack Cerza --- teuthology/repo_utils.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/teuthology/repo_utils.py b/teuthology/repo_utils.py index f9b34b6947e..1cebd695ffa 100644 --- a/teuthology/repo_utils.py +++ b/teuthology/repo_utils.py @@ -8,6 +8,16 @@ log = logging.getLogger(__name__) def checkout_repo(repo_url, dest_path, branch): + """ + Use git to either clone or update a given repo, forcing it to switch to the + specified branch. + + :param repo_url: The full URL to the repo (not including the branch) + :param dest_path: The full path to the destination directory + :param branch: The branch. + :raises: BranchNotFoundError if the branch is not found; + RuntimeError for other errors + """ if not os.path.isdir(dest_path): log.info("Cloning %s %s from upstream", repo_url, branch) proc = subprocess.Popen( From 5a61f449d1b63061159614d456542dba0dcde0e7 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Mon, 30 Jun 2014 10:10:31 -0600 Subject: [PATCH 13/19] Split up repo helper into separate functions Signed-off-by: Zack Cerza --- teuthology/repo_utils.py | 113 +++++++++++++++++++++-------- teuthology/suite.py | 6 +- teuthology/test/test_repo_utils.py | 30 +++++--- teuthology/worker.py | 4 +- 4 files changed, 108 insertions(+), 45 deletions(-) diff --git a/teuthology/repo_utils.py b/teuthology/repo_utils.py index 1cebd695ffa..7c5bd43cb80 100644 --- a/teuthology/repo_utils.py +++ b/teuthology/repo_utils.py @@ -7,7 +7,7 @@ import time log = logging.getLogger(__name__) -def checkout_repo(repo_url, dest_path, branch): +def enforce_repo_state(repo_url, dest_path, branch): """ Use git to either clone or update a given repo, forcing it to switch to the specified branch. @@ -18,34 +18,84 @@ def checkout_repo(repo_url, dest_path, branch): :raises: BranchNotFoundError if the branch is not found; RuntimeError for other errors """ - if not os.path.isdir(dest_path): - log.info("Cloning %s %s from upstream", repo_url, branch) - proc = subprocess.Popen( - ('git', 'clone', '--branch', branch, repo_url, dest_path), - cwd=os.path.dirname(dest_path), - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) + try: + if not os.path.isdir(dest_path): + clone_repo(repo_url, dest_path, branch) + elif time.time() - os.stat('/etc/passwd').st_mtime > 60: + # only do this at most once per minute + fetch_branch(dest_path, branch) + out = subprocess.check_output(('touch', dest_path)) + if out: + log.info(out) + else: + log.info("%s was just updated; assuming it is current", branch) + + reset_repo(repo_url, dest_path, branch) + except BranchNotFoundError: + shutil.rmtree(dest_path, ignore_errors=True) + raise + + +def clone_repo(repo_url, dest_path, branch): + """ + Clone a repo into a path + + :param repo_url: The full URL to the repo (not including the branch) + :param dest_path: The full path to the destination directory + :param branch: The branch. + :raises: BranchNotFoundError if the branch is not found; + RuntimeError for other errors + """ + log.info("Cloning %s %s from upstream", repo_url, branch) + proc = subprocess.Popen( + ('git', 'clone', '--branch', branch, repo_url, dest_path), + cwd=os.path.dirname(dest_path), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + if proc.wait() != 0: not_found_str = "Remote branch %s not found" % branch out = proc.stdout.read() - if proc.wait() != 0: - log.error(out) - if not_found_str in out: - raise BranchNotFoundError(branch, repo_url) - else: - raise RuntimeError("git clone failed!") - elif time.time() - os.stat('/etc/passwd').st_mtime > 60: - # only do this at most once per minute - log.info("Fetching %s from upstream", branch) - out = subprocess.check_output(('git', 'fetch', '-p', 'origin'), - cwd=dest_path) - if out: - log.info(out) - out = subprocess.check_output(('touch', dest_path)) - if out: - log.info(out) - else: - log.info("%s was just updated; assuming it is current", branch) + log.error(out) + if not_found_str in out: + raise BranchNotFoundError(branch, repo_url) + else: + raise RuntimeError("git clone failed!") + +def fetch_branch(dest_path, branch): + """ + Call "git fetch -p origin " + + :param dest_path: The full path to the destination directory + :param branch: The branch. + :raises: BranchNotFoundError if the branch is not found; + RuntimeError for other errors + """ + log.info("Fetching %s from upstream", branch) + proc = subprocess.Popen( + ('git', 'fetch', '-p', 'origin', branch), + cwd=dest_path, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + if proc.wait() != 0: + not_found_str = "fatal: Couldn't find remote ref %s" % branch + out = proc.stdout.read() + log.error(out) + if not_found_str in out: + raise BranchNotFoundError(branch) + else: + raise RuntimeError("git fetch failed!") + + +def reset_repo(repo_url, dest_path, branch): + """ + + :param repo_url: The full URL to the repo (not including the branch) + :param dest_path: The full path to the destination directory + :param branch: The branch. + :raises: BranchNotFoundError if the branch is not found; + RuntimeError for other errors + """ # This try/except block will notice if the requested branch doesn't # exist, whether it was cloned or fetched. try: @@ -54,15 +104,18 @@ def checkout_repo(repo_url, dest_path, branch): cwd=dest_path, ) except subprocess.CalledProcessError: - shutil.rmtree(dest_path) raise BranchNotFoundError(branch, repo_url) class BranchNotFoundError(ValueError): - def __init__(self, branch, repo): + def __init__(self, branch, repo=None): self.branch = branch self.repo = repo def __str__(self): - return "Branch {branch} not found in repo: {repo}".format( - branch=self.branch, repo=self.repo) + if self.repo: + repo_str = " in repo: %s" % self.repo + else: + repo_str = "" + return "Branch {branch} not found{repo_str}!".format( + branch=self.branch, repo_str=repo_str) diff --git a/teuthology/suite.py b/teuthology/suite.py index 9d5f83f7223..5fd22be12ba 100644 --- a/teuthology/suite.py +++ b/teuthology/suite.py @@ -19,7 +19,7 @@ from tempfile import NamedTemporaryFile import teuthology from . import lock from .config import config -from .repo_utils import checkout_repo, BranchNotFoundError +from .repo_utils import enforce_repo_state, BranchNotFoundError log = logging.getLogger(__name__) @@ -119,11 +119,11 @@ def fetch_suite_repo(branch, test_name): suite_repo_path = os.path.join(src_base_path, 'ceph-qa-suite_' + branch) try: - checkout_repo( + enforce_repo_state( repo_url=os.path.join(config.ceph_git_base_url, 'teuthology.git'), dest_path=os.path.join(src_base_path, 'teuthology'), branch='master') - checkout_repo( + enforce_repo_state( repo_url=os.path.join(config.ceph_git_base_url, 'ceph-qa-suite.git'), dest_path=suite_repo_path, diff --git a/teuthology/test/test_repo_utils.py b/teuthology/test/test_repo_utils.py index fdb82278343..9152b125b63 100644 --- a/teuthology/test/test_repo_utils.py +++ b/teuthology/test/test_repo_utils.py @@ -18,32 +18,42 @@ class TestRepoUtils(object): shutil.rmtree(self.local_dir, ignore_errors=True) def test_existing_branch(self): - repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'master') + repo_utils.enforce_repo_state(self.empty_repo, self.local_dir, + 'master') assert os.path.exists(self.local_dir) def test_non_existing_branch(self): with raises(repo_utils.BranchNotFoundError): - repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'blah') + repo_utils.enforce_repo_state(self.empty_repo, self.local_dir, + 'blah') assert not os.path.exists(self.local_dir) def test_multiple_calls_same_branch(self): - repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'master') + repo_utils.enforce_repo_state(self.empty_repo, self.local_dir, + 'master') assert os.path.exists(self.local_dir) - repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'master') + repo_utils.enforce_repo_state(self.empty_repo, self.local_dir, + 'master') assert os.path.exists(self.local_dir) - repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'master') + repo_utils.enforce_repo_state(self.empty_repo, self.local_dir, + 'master') assert os.path.exists(self.local_dir) def test_multiple_calls_different_branches(self): with raises(repo_utils.BranchNotFoundError): - repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'blah') + repo_utils.enforce_repo_state(self.empty_repo, self.local_dir, + 'blah1') assert not os.path.exists(self.local_dir) - repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'master') + repo_utils.enforce_repo_state(self.empty_repo, self.local_dir, + 'master') assert os.path.exists(self.local_dir) - repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'master') + repo_utils.enforce_repo_state(self.empty_repo, self.local_dir, + 'master') assert os.path.exists(self.local_dir) with raises(repo_utils.BranchNotFoundError): - repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'blah') + repo_utils.enforce_repo_state(self.empty_repo, self.local_dir, + 'blah2') assert not os.path.exists(self.local_dir) - repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'master') + repo_utils.enforce_repo_state(self.empty_repo, self.local_dir, + 'master') assert os.path.exists(self.local_dir) diff --git a/teuthology/worker.py b/teuthology/worker.py index 1a44480a5ce..53da3b7d1c6 100644 --- a/teuthology/worker.py +++ b/teuthology/worker.py @@ -17,7 +17,7 @@ from . import safepath from .config import config as teuth_config from .kill import kill_job from .misc import read_config -from .repo_utils import checkout_repo, BranchNotFoundError +from .repo_utils import enforce_repo_state, BranchNotFoundError log = logging.getLogger(__name__) start_time = datetime.utcnow() @@ -85,7 +85,7 @@ def fetch_teuthology_branch(dest_path, branch='master'): try: teuthology_git_upstream = teuth_config.ceph_git_base_url + \ 'teuthology.git' - checkout_repo(teuthology_git_upstream, dest_path, branch) + enforce_repo_state(teuthology_git_upstream, dest_path, branch) log.debug("Bootstrapping %s", dest_path) # This magic makes the bootstrap script not attempt to clobber an From bfd82261422f304bdc84617a485db05fa2efefeb Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Mon, 30 Jun 2014 17:23:20 -0600 Subject: [PATCH 14/19] Run unit tests offline Signed-off-by: Zack Cerza --- teuthology/test/test_repo_utils.py | 60 ++++++++++++++++++------------ 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/teuthology/test/test_repo_utils.py b/teuthology/test/test_repo_utils.py index 9152b125b63..e03831292ca 100644 --- a/teuthology/test/test_repo_utils.py +++ b/teuthology/test/test_repo_utils.py @@ -2,58 +2,70 @@ import logging import os.path from pytest import raises import shutil +import subprocess from .. import repo_utils repo_utils.log.setLevel(logging.WARNING) class TestRepoUtils(object): - empty_repo = 'https://github.com/ceph/empty' - local_dir = '/tmp/empty' + src_path = '/tmp/empty_src' + repo_url = 'file://' + src_path + dest_path = '/tmp/empty_dest' def setup(self): - assert not os.path.exists(self.local_dir) + assert not os.path.exists(self.dest_path) + proc = subprocess.Popen( + ('git', 'init', self.src_path), + ) + assert proc.wait() == 0 + proc = subprocess.Popen( + ('git', 'commit', '--allow-empty', '--allow-empty-message', + '--no-edit'), + cwd=self.src_path, + ) + assert proc.wait() == 0 def teardown(self): - shutil.rmtree(self.local_dir, ignore_errors=True) + shutil.rmtree(self.dest_path, ignore_errors=True) def test_existing_branch(self): - repo_utils.enforce_repo_state(self.empty_repo, self.local_dir, + repo_utils.enforce_repo_state(self.repo_url, self.dest_path, 'master') - assert os.path.exists(self.local_dir) + assert os.path.exists(self.dest_path) def test_non_existing_branch(self): with raises(repo_utils.BranchNotFoundError): - repo_utils.enforce_repo_state(self.empty_repo, self.local_dir, + repo_utils.enforce_repo_state(self.repo_url, self.dest_path, 'blah') - assert not os.path.exists(self.local_dir) + assert not os.path.exists(self.dest_path) def test_multiple_calls_same_branch(self): - repo_utils.enforce_repo_state(self.empty_repo, self.local_dir, + repo_utils.enforce_repo_state(self.repo_url, self.dest_path, 'master') - assert os.path.exists(self.local_dir) - repo_utils.enforce_repo_state(self.empty_repo, self.local_dir, + assert os.path.exists(self.dest_path) + repo_utils.enforce_repo_state(self.repo_url, self.dest_path, 'master') - assert os.path.exists(self.local_dir) - repo_utils.enforce_repo_state(self.empty_repo, self.local_dir, + assert os.path.exists(self.dest_path) + repo_utils.enforce_repo_state(self.repo_url, self.dest_path, 'master') - assert os.path.exists(self.local_dir) + assert os.path.exists(self.dest_path) def test_multiple_calls_different_branches(self): with raises(repo_utils.BranchNotFoundError): - repo_utils.enforce_repo_state(self.empty_repo, self.local_dir, + repo_utils.enforce_repo_state(self.repo_url, self.dest_path, 'blah1') - assert not os.path.exists(self.local_dir) - repo_utils.enforce_repo_state(self.empty_repo, self.local_dir, + assert not os.path.exists(self.dest_path) + repo_utils.enforce_repo_state(self.repo_url, self.dest_path, 'master') - assert os.path.exists(self.local_dir) - repo_utils.enforce_repo_state(self.empty_repo, self.local_dir, + assert os.path.exists(self.dest_path) + repo_utils.enforce_repo_state(self.repo_url, self.dest_path, 'master') - assert os.path.exists(self.local_dir) + assert os.path.exists(self.dest_path) with raises(repo_utils.BranchNotFoundError): - repo_utils.enforce_repo_state(self.empty_repo, self.local_dir, + repo_utils.enforce_repo_state(self.repo_url, self.dest_path, 'blah2') - assert not os.path.exists(self.local_dir) - repo_utils.enforce_repo_state(self.empty_repo, self.local_dir, + assert not os.path.exists(self.dest_path) + repo_utils.enforce_repo_state(self.repo_url, self.dest_path, 'master') - assert os.path.exists(self.local_dir) + assert os.path.exists(self.dest_path) From e431abd4aa30a533c3734acff59e21efcea0ddaa Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Mon, 30 Jun 2014 17:35:11 -0600 Subject: [PATCH 15/19] Add a basic validation of the branch value Signed-off-by: Zack Cerza --- teuthology/repo_utils.py | 9 +++++++++ teuthology/test/test_repo_utils.py | 10 ++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/teuthology/repo_utils.py b/teuthology/repo_utils.py index 7c5bd43cb80..bf5a28cbbb9 100644 --- a/teuthology/repo_utils.py +++ b/teuthology/repo_utils.py @@ -18,6 +18,7 @@ def enforce_repo_state(repo_url, dest_path, branch): :raises: BranchNotFoundError if the branch is not found; RuntimeError for other errors """ + validate_branch(branch) try: if not os.path.isdir(dest_path): clone_repo(repo_url, dest_path, branch) @@ -46,6 +47,7 @@ def clone_repo(repo_url, dest_path, branch): :raises: BranchNotFoundError if the branch is not found; RuntimeError for other errors """ + validate_branch(branch) log.info("Cloning %s %s from upstream", repo_url, branch) proc = subprocess.Popen( ('git', 'clone', '--branch', branch, repo_url, dest_path), @@ -71,6 +73,7 @@ def fetch_branch(dest_path, branch): :raises: BranchNotFoundError if the branch is not found; RuntimeError for other errors """ + validate_branch(branch) log.info("Fetching %s from upstream", branch) proc = subprocess.Popen( ('git', 'fetch', '-p', 'origin', branch), @@ -96,6 +99,7 @@ def reset_repo(repo_url, dest_path, branch): :raises: BranchNotFoundError if the branch is not found; RuntimeError for other errors """ + validate_branch(branch) # This try/except block will notice if the requested branch doesn't # exist, whether it was cloned or fetched. try: @@ -119,3 +123,8 @@ class BranchNotFoundError(ValueError): repo_str = "" return "Branch {branch} not found{repo_str}!".format( branch=self.branch, repo_str=repo_str) + + +def validate_branch(branch): + if ' ' in branch: + raise ValueError("Illegal branch name: '%s'" % branch) diff --git a/teuthology/test/test_repo_utils.py b/teuthology/test/test_repo_utils.py index e03831292ca..0fc685e86e6 100644 --- a/teuthology/test/test_repo_utils.py +++ b/teuthology/test/test_repo_utils.py @@ -13,20 +13,22 @@ class TestRepoUtils(object): repo_url = 'file://' + src_path dest_path = '/tmp/empty_dest' - def setup(self): + def setup_method(self, method): assert not os.path.exists(self.dest_path) proc = subprocess.Popen( ('git', 'init', self.src_path), + stdout=subprocess.PIPE, ) assert proc.wait() == 0 proc = subprocess.Popen( ('git', 'commit', '--allow-empty', '--allow-empty-message', '--no-edit'), cwd=self.src_path, + stdout=subprocess.PIPE, ) assert proc.wait() == 0 - def teardown(self): + def teardown_method(self, method): shutil.rmtree(self.dest_path, ignore_errors=True) def test_existing_branch(self): @@ -69,3 +71,7 @@ class TestRepoUtils(object): repo_utils.enforce_repo_state(self.repo_url, self.dest_path, 'master') assert os.path.exists(self.dest_path) + + def test_invalid_branch(self): + with raises(ValueError): + repo_utils.enforce_repo_state(self.repo_url, self.dest_path, 'a b') From d6f1752897fb0f5e17b8b3a70871a50e13d49de0 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Tue, 1 Jul 2014 10:42:16 -0600 Subject: [PATCH 16/19] Add more unit tests Signed-off-by: Zack Cerza --- teuthology/test/test_repo_utils.py | 31 +++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/teuthology/test/test_repo_utils.py b/teuthology/test/test_repo_utils.py index 0fc685e86e6..224011203ec 100644 --- a/teuthology/test/test_repo_utils.py +++ b/teuthology/test/test_repo_utils.py @@ -31,18 +31,39 @@ class TestRepoUtils(object): def teardown_method(self, method): shutil.rmtree(self.dest_path, ignore_errors=True) - def test_existing_branch(self): + def test_clone_repo_existing_branch(self): + repo_utils.clone_repo(self.repo_url, self.dest_path, 'master') + assert os.path.exists(self.dest_path) + + def test_clone_repo_non_existing_branch(self): + with raises(repo_utils.BranchNotFoundError): + repo_utils.clone_repo(self.repo_url, self.dest_path, 'nobranch') + assert not os.path.exists(self.dest_path) + + def test_fetch_branch_no_repo(self): + fake_dest_path = '/tmp/not_a_repo' + assert not os.path.exists(fake_dest_path) + with raises(OSError): + repo_utils.fetch_branch(fake_dest_path, 'master') + assert not os.path.exists(fake_dest_path) + + def test_fetch_branch_fake_branch(self): + repo_utils.clone_repo(self.repo_url, self.dest_path, 'master') + with raises(repo_utils.BranchNotFoundError): + repo_utils.fetch_branch(self.dest_path, 'nobranch') + + def test_enforce_existing_branch(self): repo_utils.enforce_repo_state(self.repo_url, self.dest_path, 'master') assert os.path.exists(self.dest_path) - def test_non_existing_branch(self): + def test_enforce_non_existing_branch(self): with raises(repo_utils.BranchNotFoundError): repo_utils.enforce_repo_state(self.repo_url, self.dest_path, 'blah') assert not os.path.exists(self.dest_path) - def test_multiple_calls_same_branch(self): + def test_enforce_multiple_calls_same_branch(self): repo_utils.enforce_repo_state(self.repo_url, self.dest_path, 'master') assert os.path.exists(self.dest_path) @@ -53,7 +74,7 @@ class TestRepoUtils(object): 'master') assert os.path.exists(self.dest_path) - def test_multiple_calls_different_branches(self): + def test_enforce_multiple_calls_different_branches(self): with raises(repo_utils.BranchNotFoundError): repo_utils.enforce_repo_state(self.repo_url, self.dest_path, 'blah1') @@ -72,6 +93,6 @@ class TestRepoUtils(object): 'master') assert os.path.exists(self.dest_path) - def test_invalid_branch(self): + def test_enforce_invalid_branch(self): with raises(ValueError): repo_utils.enforce_repo_state(self.repo_url, self.dest_path, 'a b') From 42a73500c063cb1b1b8fd2b0a1e96cd418b2bf42 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Tue, 1 Jul 2014 10:56:41 -0600 Subject: [PATCH 17/19] Use a test-specific name/email for the git calls Signed-off-by: Zack Cerza --- teuthology/test/test_repo_utils.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/teuthology/test/test_repo_utils.py b/teuthology/test/test_repo_utils.py index 224011203ec..9e94c1af0f6 100644 --- a/teuthology/test/test_repo_utils.py +++ b/teuthology/test/test_repo_utils.py @@ -20,6 +20,18 @@ class TestRepoUtils(object): stdout=subprocess.PIPE, ) assert proc.wait() == 0 + proc = subprocess.Popen( + ('git', 'config', 'user.email', 'test@ceph.com'), + cwd=self.src_path, + stdout=subprocess.PIPE, + ) + assert proc.wait() == 0 + proc = subprocess.Popen( + ('git', 'config', 'user.name', 'Test User'), + cwd=self.src_path, + stdout=subprocess.PIPE, + ) + assert proc.wait() == 0 proc = subprocess.Popen( ('git', 'commit', '--allow-empty', '--allow-empty-message', '--no-edit'), From 434348aa49a5f0b64313d76b28de2fabd14b946f Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Tue, 1 Jul 2014 11:45:10 -0600 Subject: [PATCH 18/19] Add --suite-branch and --suite-base The former lets you specify a specific ceph-qa-suite branch to use for testing. The latter lets you specify an as-is directory to use, for example if you want ro run tests that you don't want to commit yet. Signed-off-by: Zack Cerza --- scripts/suite.py | 8 ++++++-- teuthology/suite.py | 7 ++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/scripts/suite.py b/scripts/suite.py index a30a6e183d6..2a9676d75b6 100644 --- a/scripts/suite.py +++ b/scripts/suite.py @@ -4,8 +4,7 @@ import teuthology.suite doc = """ usage: teuthology-suite [-h] - teuthology-suite --suite [options] - teuthology-suite -s [options] [...] + teuthology-suite --suite [options] [...] Run a suite of ceph integration tests. A suite is a directory containing facets. A facet is a directory containing config snippets. Running a suite @@ -42,6 +41,11 @@ Standard arguments: -d , --distro Distribution to run against [default: ubuntu] + --suite-branch + Use this suite branch instead of the ceph branch + --suite-base Use this alternative directory as-is when + assembling jobs from yaml fragments. This causes + to be ignored. Scheduler arguments: --owner Job owner diff --git a/teuthology/suite.py b/teuthology/suite.py index 5fd22be12ba..dfb137508bc 100644 --- a/teuthology/suite.py +++ b/teuthology/suite.py @@ -39,6 +39,8 @@ def main(args): teuthology_branch = args['--teuthology-branch'] machine_type = args['--machine-type'] distro = args['--distro'] + suite_branch = args['--suite-branch'] or ceph_branch + suite_base = args['--suite-base'] limit = int(args['--limit']) priority = int(args['--priority']) @@ -53,7 +55,10 @@ def main(args): name = make_run_name(nice_suite, ceph_branch, kernel_branch, kernel_flavor, machine_type) - suite_repo_path = fetch_suite_repo(ceph_branch, test_name=name) + if suite_base: + suite_repo_path = suite_base + else: + suite_repo_path = fetch_suite_repo(suite_branch, test_name=name) config_string = create_initial_config(nice_suite, ceph_branch, teuthology_branch, kernel_branch, From ec95ee25c38b9bfbfca995a27424a861cd95d52a Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Tue, 1 Jul 2014 12:21:45 -0600 Subject: [PATCH 19/19] Rename suite_base to suite_dir Signed-off-by: Zack Cerza --- scripts/suite.py | 2 +- teuthology/suite.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/suite.py b/scripts/suite.py index 2a9676d75b6..5646363247e 100644 --- a/scripts/suite.py +++ b/scripts/suite.py @@ -43,7 +43,7 @@ Standard arguments: [default: ubuntu] --suite-branch Use this suite branch instead of the ceph branch - --suite-base Use this alternative directory as-is when + --suite-dir Use this alternative directory as-is when assembling jobs from yaml fragments. This causes to be ignored. diff --git a/teuthology/suite.py b/teuthology/suite.py index dfb137508bc..7dc59d4cbe5 100644 --- a/teuthology/suite.py +++ b/teuthology/suite.py @@ -40,7 +40,7 @@ def main(args): machine_type = args['--machine-type'] distro = args['--distro'] suite_branch = args['--suite-branch'] or ceph_branch - suite_base = args['--suite-base'] + suite_dir = args['--suite-dir'] limit = int(args['--limit']) priority = int(args['--priority']) @@ -55,8 +55,8 @@ def main(args): name = make_run_name(nice_suite, ceph_branch, kernel_branch, kernel_flavor, machine_type) - if suite_base: - suite_repo_path = suite_base + if suite_dir: + suite_repo_path = suite_dir else: suite_repo_path = fetch_suite_repo(suite_branch, test_name=name)