Merge pull request #132 from ceph/wip-refactor-scripts

Fix imports; fix keyscan_check()

scripts/nuke.py

@@ -2,6 +2,7 @@ import argparse
 from argparse import RawTextHelpFormatter
 import textwrap
 
+import teuthology.misc
 import teuthology.nuke
 
 
@@ -10,9 +11,6 @@ def main():
 
 
 def parse_args():
-    from teuthology.run import config_file
-    from teuthology.run import MergeConfig
-
     parser = argparse.ArgumentParser(
         description='Reset test machines',
         epilog=textwrap.dedent('''
@@ -29,8 +27,8 @@ def parse_args():
     parser.add_argument(
         '-t', '--targets',
         nargs='+',
-        type=config_file,
-        action=MergeConfig,
+        type=teuthology.misc.config_file,
+        action=teuthology.misc.MergeConfig,
         default={},
         dest='config',
         help='yaml config containing machines to nuke',

scripts/run.py

@@ -1,5 +1,6 @@
 import argparse
 
+import teuthology.misc
 import teuthology.run
 
 
@@ -18,8 +19,8 @@ def parse_args():
         'config',
         metavar='CONFFILE',
         nargs='+',
-        type=teuthology.run.config_file,
-        action=teuthology.run.MergeConfig,
+        type=teuthology.misc.config_file,
+        action=teuthology.misc.MergeConfig,
         default={},
         help='config file to read',
     )

scripts/schedule.py

@@ -1,6 +1,6 @@
 import argparse
 
-import teuthology.run
+import teuthology.misc
 import teuthology.schedule
 
 
@@ -15,8 +15,8 @@ def parse_args():
         'config',
         metavar='CONFFILE',
         nargs='*',
-        type=teuthology.run.config_file,
-        action=teuthology.run.MergeConfig,
+        type=teuthology.misc.config_file,
+        action=teuthology.misc.MergeConfig,
         default={},
         help='config file to read',
     )

scripts/test/test_updatekeys.py

@@ -1,11 +1,16 @@
 from script import Script
 import subprocess
 from pytest import raises
+from pytest import skip
 
 
 class TestUpdatekeys(Script):
     script_name = 'teuthology-updatekeys'
 
+    def test_invalid(self):
+        skip("teuthology.lock needs to be partially refactored to allow" +
+             "teuthology-updatekeys to return nonzero in all erorr cases")
+
     def test_all_and_targets(self):
         args = (self.script_name, '-a', '-t', 'foo')
         with raises(subprocess.CalledProcessError):

scripts/updatekeys.py

@@ -5,7 +5,8 @@ import teuthology.lock
 
 
 def main():
-    teuthology.lock.updatekeys(parse_args())
+    status = teuthology.lock.updatekeys(parse_args())
+    sys.exit(status)
 
 
 def parse_args():

teuthology/lock.py

@@ -348,7 +348,11 @@ def keyscan_check(ctx, machines):
             _, machines[i] = machine.rsplit('@')
     args = ['ssh-keyscan']
     args.extend(machines)
-    out = subprocess.check_output(args)
+    p = subprocess.Popen(
+        args=args,
+        stdout=subprocess.PIPE,
+    )
+    out, err = p.communicate()
     return (out, current_locks)
 
 

teuthology/misc.py

@@ -1,5 +1,6 @@
 from cStringIO import StringIO
 
+import argparse
 import os
 import logging
 import configobj
@@ -26,6 +27,25 @@ is_vm = lambda x: x.startswith('vpm') or x.startswith('ubuntu@vpm')
 is_arm = lambda x: x.startswith('tala') or x.startswith('ubuntu@tala') or x.startswith('saya') or x.startswith('ubuntu@saya')
 
 
+def config_file(string):
+    config_dict = {}
+    try:
+        with file(string) as f:
+            g = yaml.safe_load_all(f)
+            for new in g:
+                config_dict.update(new)
+    except IOError as e:
+        raise argparse.ArgumentTypeError(str(e))
+    return config_dict
+
+
+class MergeConfig(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        config_dict = getattr(namespace, self.dest)
+        for new in values:
+            deep_merge(config_dict, new)
+
+
 def get_testdir(ctx):
     if 'test_path' in ctx.teuthology_config:
         return ctx.teuthology_config['test_path']

teuthology/nuke.py

@@ -1,9 +1,26 @@
 import argparse
+import logging
+import os
+import subprocess
+import time
 import yaml
 
+from . import orchestra
+from .orchestra import run
+from .lock import list_locks
+from .lock import unlock_one
+from .misc import config_file
+from .misc import get_testdir
+from .misc import get_user
+from .misc import read_config
+from .misc import reconnect
+from .parallel import parallel
+from .task import install as install_task
+from .task.internal import check_lock
+from .task.internal import connect
+
 
 def shutdown_daemons(ctx, log):
-    from .orchestra import run
     nodes = {}
     for remote in ctx.cluster.remotes.iterkeys():
         proc = remote.run(
@@ -41,7 +58,6 @@ def shutdown_daemons(ctx, log):
 
 
 def find_kernel_mounts(ctx, log):
-    from .orchestra import run
     nodes = {}
     log.info('Looking for kernel mounts to handle...')
     for remote in ctx.cluster.remotes.iterkeys():
@@ -71,7 +87,6 @@ def remove_kernel_mounts(ctx, kernel_mounts, log):
     properly we should be able to just do a forced unmount,
     but that doesn't seem to be working, so you should reboot instead
     """
-    from .orchestra import run
     nodes = {}
     for remote in kernel_mounts:
         log.info('clearing kernel mount from %s', remote.name)
@@ -96,7 +111,6 @@ def remove_osd_mounts(ctx, log):
     """
     unmount any osd data mounts (scratch disks)
     """
-    from .orchestra import run
     ctx.cluster.run(
         args=[
             'grep',
@@ -115,7 +129,6 @@ def remove_osd_tmpfs(ctx, log):
     """
     unmount tmpfs mounts
     """
-    from .orchestra import run
     ctx.cluster.run(
         args=[
             'egrep', 'tmpfs\s+/mnt', '/etc/mtab', run.Raw('|'),
@@ -128,8 +141,6 @@ def remove_osd_tmpfs(ctx, log):
 
 
 def reboot(ctx, remotes, log):
-    from .orchestra import run
-    import time
     nodes = {}
     for remote in remotes:
         log.info('rebooting %s', remote.name)
@@ -146,7 +157,6 @@ def reboot(ctx, remotes, log):
         # send anything back to the ssh client!
         # for remote, proc in nodes.iteritems():
         # proc.exitstatus.get()
-    from teuthology.misc import reconnect
     if remotes:
         log.info('waiting for nodes to reboot')
         time.sleep(5)  # if we try and reconnect too quickly, it succeeds!
@@ -154,7 +164,6 @@ def reboot(ctx, remotes, log):
 
 
 def reset_syslog_dir(ctx, log):
-    from .orchestra import run
     nodes = {}
     for remote in ctx.cluster.remotes.iterkeys():
         proc = remote.run(
@@ -179,7 +188,6 @@ def reset_syslog_dir(ctx, log):
 
 
 def dpkg_configure(ctx, log):
-    from .orchestra import run
     nodes = {}
     for remote in ctx.cluster.remotes.iterkeys():
         proc = remote.run(
@@ -202,7 +210,6 @@ def dpkg_configure(ctx, log):
 
 
 def remove_installed_packages(ctx, log):
-    from teuthology.task import install as install_task
 
     dpkg_configure(ctx, log)
     config = {'project': 'ceph'}
@@ -216,8 +223,6 @@ def remove_installed_packages(ctx, log):
 
 
 def remove_testing_tree(ctx, log):
-    from teuthology.misc import get_testdir
-    from .orchestra import run
     nodes = {}
     for remote in ctx.cluster.remotes.iterkeys():
         proc = remote.run(
@@ -241,7 +246,6 @@ def remove_testing_tree(ctx, log):
 
 
 def synch_clocks(remotes, log):
-    from .orchestra import run
     nodes = {}
     for remote in remotes:
         proc = remote.run(
@@ -265,11 +269,6 @@ def synch_clocks(remotes, log):
 
 
 def main(ctx):
-    from teuthology.run import config_file
-    import os
-
-    import logging
-
     log = logging.getLogger(__name__)
 
     loglevel = logging.INFO
@@ -296,7 +295,6 @@ def main(ctx):
             if not ctx.owner:
                 ctx.owner = open(ctx.archive + '/owner').read().rstrip('\n')
 
-    from teuthology.misc import read_config
     read_config(ctx)
 
     log.info(
@@ -306,7 +304,6 @@ def main(ctx):
                 default_flow_style=False).splitlines()))
 
     if ctx.owner is None:
-        from teuthology.misc import get_user
         ctx.owner = get_user()
 
     if ctx.pid:
@@ -317,7 +314,6 @@ def main(ctx):
                 ctx.pid,
                 ctx.pid))
         else:
-            import subprocess
             subprocess.check_call(["kill", "-9", str(ctx.pid)])
 
     nuke(ctx, log, ctx.unlock, ctx.synch_clocks, ctx.reboot_all, ctx.noipmi)
@@ -325,8 +321,6 @@ def main(ctx):
 
 def nuke(ctx, log, should_unlock, sync_clocks=True, reboot_all=True,
          noipmi=False):
-    from teuthology.parallel import parallel
-    from teuthology.lock import list_locks
     total_unnuked = {}
     targets = dict(ctx.config['targets'])
     if ctx.name:
@@ -367,7 +361,6 @@ def nuke(ctx, log, should_unlock, sync_clocks=True, reboot_all=True,
 
 def nuke_one(ctx, targets, log, should_unlock, synch_clocks, reboot_all,
              check_locks, noipmi):
-    from teuthology.lock import unlock_one
     ret = None
     ctx = argparse.Namespace(
         config=dict(targets=targets),
@@ -394,7 +387,6 @@ def nuke_one(ctx, targets, log, should_unlock, synch_clocks, reboot_all,
 
 def nuke_helper(ctx, log):
     # ensure node is up with ipmi
-    from teuthology.orchestra import remote
 
     (target,) = ctx.config['targets'].keys()
     host = target.split('@')[-1]
@@ -404,7 +396,7 @@ def nuke_helper(ctx, log):
     log.debug('shortname: %s' % shortname)
     log.debug('{ctx}'.format(ctx=ctx))
     if not ctx.noipmi and 'ipmi_user' in ctx.teuthology_config:
-        console = remote.getRemoteConsole(
+        console = orchestra.remote.getRemoteConsole(
             name=host,
             ipmiuser=ctx.teuthology_config['ipmi_user'],
             ipmipass=ctx.teuthology_config['ipmi_password'],
@@ -425,7 +417,6 @@ def nuke_helper(ctx, log):
         else:
             log.info('console ready on %s' % cname)
 
-    from teuthology.task.internal import check_lock, connect
     if ctx.check_locks:
         check_lock(ctx, None)
     connect(ctx, None)

teuthology/queue.py

@@ -10,8 +10,9 @@ import yaml
 
 import beanstalkc
 
-from .config import config as teuth_config
 from . import safepath
+from .config import config as teuth_config
+from .misc import read_config
 
 log = logging.getLogger(__name__)
 
@@ -118,7 +119,6 @@ def worker(ctx):
             path=ctx.archive_dir,
         ))
 
-    from teuthology.misc import read_config
     read_config(ctx)
 
     beanstalk = connect(ctx)

teuthology/run.py

@@ -1,36 +1,21 @@
-import argparse
 import os
 import yaml
 import StringIO
 import contextlib
 import sys
+import logging
 from traceback import format_tb
 
-
-def config_file(string):
-    config = {}
-    try:
-        with file(string) as f:
-            g = yaml.safe_load_all(f)
-            for new in g:
-                config.update(new)
-    except IOError as e:
-        raise argparse.ArgumentTypeError(str(e))
-    return config
-
-
-class MergeConfig(argparse.Action):
-
-    def __call__(self, parser, namespace, values, option_string=None):
-        config = getattr(namespace, self.dest)
-        from teuthology.misc import deep_merge
-        for new in values:
-            deep_merge(config, new)
+from . import report
+from .misc import get_distro
+from .misc import get_user
+from .misc import read_config
+from .nuke import nuke
+from .run_tasks import run_tasks
+from .results import email_results
 
 
 def set_up_logging(ctx):
-    import logging
-
     loglevel = logging.INFO
     if ctx.verbose:
         loglevel = logging.DEBUG
@@ -54,8 +39,6 @@ def set_up_logging(ctx):
 
 def install_except_hook():
     def log_exception(exception_class, exception, traceback):
-        import logging
-
         logging.critical(''.join(format_tb(traceback)))
         if not exception.message:
             logging.critical(exception_class.__name__)
@@ -91,14 +74,10 @@ def write_initial_metadata(ctx):
 
 
 def main(ctx):
-    import logging
     set_up_logging(ctx)
     log = logging.getLogger(__name__)
 
-    from . import report
-
     if ctx.owner is None:
-        from teuthology.misc import get_user
         ctx.owner = get_user()
 
     # Older versions of teuthology stored job_id as an int. Convert it to a str
@@ -127,7 +106,6 @@ def main(ctx):
         assert ctx.lock, \
             'the --block option is only supported with the --lock option'
 
-    from teuthology.misc import read_config
     read_config(ctx)
 
     log.debug('\n  '.join(['Config:', ] + yaml.safe_dump(
@@ -162,7 +140,6 @@ def main(ctx):
         {'internal.vm_setup': None},
     ])
     if 'kernel' in ctx.config:
-        from teuthology.misc import get_distro
         distro = get_distro(ctx)
         if distro == 'ubuntu':
             init_tasks.append({'kernel': ctx.config['kernel']})
@@ -177,12 +154,10 @@ def main(ctx):
 
     ctx.config['tasks'][:0] = init_tasks
 
-    from teuthology.run_tasks import run_tasks
     try:
         run_tasks(tasks=ctx.config['tasks'], ctx=ctx)
     finally:
         if not ctx.summary.get('success') and ctx.config.get('nuke-on-error'):
-            from teuthology.nuke import nuke
             # only unlock if we locked them in the first place
             nuke(ctx, log, ctx.lock)
         if ctx.archive is not None:
@@ -199,7 +174,6 @@ def main(ctx):
                 emsg = f.getvalue()
                 subject = "Teuthology error -- %s" % ctx.summary[
                     'failure_reason']
-                from teuthology.suite import email_results
                 email_results(subject, "Teuthology", ctx.config[
                               'email-on-error'], emsg)
 
@@ -209,5 +183,4 @@ def main(ctx):
             log.info('pass')
         else:
             log.info('FAIL')
-            import sys
             sys.exit(1)

teuthology/schedule.py

@@ -1,7 +1,8 @@
 import yaml
 
-from teuthology.misc import read_config, get_user
 import teuthology.queue
+from teuthology.misc import get_user
+from teuthology.misc import read_config
 
 
 def main(ctx):