Merge pull request #203 from ceph/wip-devstack

Add devstack task

requirements.txt

@@ -16,6 +16,7 @@ raven
 web.py
 docopt
 psutil
+configparser
 
 # Test Dependencies
 # nose >=1.0.0

teuthology/lock.py

@@ -452,15 +452,12 @@ def create_if_vm(ctx, machine_name):
                 if lcnfg.keys() == ['downburst']:
                     lcnfg = lcnfg['downburst']
         except (TypeError, AttributeError):
-            try:
-                lcnfg = {}
-                for tdict in ctx.config['downburst']:
-                    for key in tdict:
-                        lcnfg[key] = tdict[key]
-            except (KeyError, AttributeError):
+            if hasattr(ctx, 'config') and ctx.config is not None:
+                lcnfg = ctx.config.get('downburst', dict())
+            else:
                 lcnfg = {}
         except IOError:
-            print "Error reading %s" % lfile 
+            print "Error reading %s" % lfile
             return False
 
         distro = lcnfg.get('distro', os_type.lower())

teuthology/misc.py

@@ -35,7 +35,7 @@ is_arm = lambda x: x.startswith('tala') or x.startswith(
 
 def config_file(string):
     """
-    Create a config file  
+    Create a config file
 
     :param string: name of yaml file used for config.
     :returns: Dictionary of configuration information.
@@ -97,7 +97,7 @@ def get_archive_dir(ctx):
 def get_http_log_path(archive_dir, job_id=None):
     """
     :param archive_dir: directory to be searched
-    :param job_id: id of job that terminates the name of the log path 
+    :param job_id: id of job that terminates the name of the log path
     :returns: http log path
     """
     http_base = config.archive_server
@@ -267,7 +267,7 @@ def skeleton_config(ctx, roles, ips):
 def roles_of_type(roles_for_host, type_):
     """
     Generator of roles.
- 
+
     Each call returns the next possible role of the type specified.
     :param roles_for host: list of roles possible
     :param type_: type of role
@@ -326,9 +326,9 @@ def is_type(type_):
 def num_instances_of_type(cluster, type_):
     """
     Total the number of instances of the role type specified in all remotes.
-    
+
     :param cluster: Cluster extracted from ctx.
-    :param type_: role 
+    :param type_: role
     """
     remotes_and_roles = cluster.remotes.items()
     roles = [roles for (remote, roles) in remotes_and_roles]
@@ -351,7 +351,7 @@ def create_simple_monmap(ctx, remote, conf):
     def gen_addresses():
         """
         Monitor address generator.
-    
+
         Each invocation returns the next monitor address
         """
         for section, data in conf.iteritems():
@@ -411,7 +411,7 @@ def write_file(remote, path, data):
         )
 
 
-def sudo_write_file(remote, path, data, perms=None):
+def sudo_write_file(remote, path, data, perms=None, owner=None):
     """
     Write data to a remote file as super user
 
@@ -419,10 +419,16 @@ def sudo_write_file(remote, path, data, perms=None):
     :param path: Path on the remote being written to.
     :param data: Data to be written.
     :param perms: Permissions on the file being written
+    :param owner: Owner for the file being written
+
+    Both perms and owner are passed directly to chmod.
     """
     permargs = []
     if perms:
         permargs = [run.Raw('&&'), 'sudo', 'chmod', perms, path]
+    owner_args = []
+    if owner:
+        owner_args = [run.Raw('&&'), 'sudo', 'chown', owner, path]
     remote.run(
         args=[
             'sudo',
@@ -430,11 +436,23 @@ def sudo_write_file(remote, path, data, perms=None):
             '-c',
             'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))',
             path,
-            ] + permargs,
+            ] + owner_args + permargs,
         stdin=data,
         )
 
 
+def copy_file(from_remote, from_path, to_remote, to_path=None):
+    """
+    Copies a file from one remote to another.
+    """
+    if to_path is None:
+        to_path = from_path
+    from_remote.run(args=[
+        'sudo', 'scp', '-v', from_path, "{host}:{file}".format(
+            host=to_remote.name, file=to_path)
+    ])
+
+
 def move_file(remote, from_path, to_path, sudo=False):
     """
     Move a file from one path to another on a remote site
@@ -708,7 +726,7 @@ def get_wwn_id_map(remote, devs):
 
     Sample dev information:    /dev/sdb: /dev/disk/by-id/wwn-0xf00bad
 
-    :returns: map of devices to device id links 
+    :returns: map of devices to device id links
     """
     stdout = None
     try:
@@ -873,6 +891,31 @@ def wait_until_fuse_mounted(remote, fuse, mountpoint):
     log.info('ceph-fuse is mounted on %s', mountpoint)
 
 
+def reboot(node, timeout=300, interval=30):
+    """
+    Reboots a given system, then waits for it to come back up and
+    re-establishes the ssh connection.
+
+    :param node: The teuthology.orchestra.remote.Remote object of the node
+    :param timeout: The amount of time, in seconds, after which to give up
+                    waiting for the node to return
+    :param interval: The amount of time, in seconds, to wait between attempts
+                     to re-establish with the node. This should not be set to
+                     less than maybe 10, to make sure the node actually goes
+                     down first.
+    """
+    log.info("Rebooting {host}...".format(host=node.hostname))
+    node.run(args=['sudo', 'shutdown', '-r', 'now'])
+    reboot_start_time = time.time()
+    while time.time() - reboot_start_time < timeout:
+        time.sleep(interval)
+        if node.is_online or node.reconnect():
+            return
+    raise RuntimeError(
+        "{host} did not come up after reboot within {time}s".format(
+            host=node.hostname, time=timeout))
+
+
 def reconnect(ctx, timeout, remotes=None):
     """
     Connect to all the machines in ctx.cluster.
@@ -895,23 +938,14 @@ def reconnect(ctx, timeout, remotes=None):
     else:
         need_reconnect = ctx.cluster.remotes.keys()
 
-    for r in need_reconnect:
-        r.ssh.close()
-
     while need_reconnect:
         for remote in need_reconnect:
-            try:
-                log.info('trying to connect to %s', remote.name)
-                key = ctx.config['targets'][remote.name]
-                from .orchestra import connection
-                remote.ssh = connection.connect(
-                    user_at_host=remote.name,
-                    host_key=key,
-                    keep_alive=True,
-                    )
-            except Exception:
+            log.info('trying to connect to %s', remote.name)
+            success = remote.reconnect()
+            if not success:
                 if time.time() - starttime > timeout:
-                    raise
+                    raise RuntimeError("Could not reconnect to %s" %
+                                       remote.name)
             else:
                 need_reconnect.remove(remote)
 

teuthology/orchestra/remote.py

@@ -2,6 +2,7 @@
 Support for paramiko remote objects.
 """
 from . import run
+import connection
 from teuthology import misc
 import time
 import pexpect
@@ -18,6 +19,7 @@ log = logging.getLogger(__name__)
 
 
 class Remote(object):
+
     """
     A connection to a remote host.
 
@@ -27,11 +29,33 @@ class Remote(object):
     # for unit tests to hook into
     _runner = staticmethod(run.run)
 
-    def __init__(self, name, ssh, shortname=None, console=None):
+    def __init__(self, name, ssh=None, shortname=None, console=None,
+                 host_key=None, keep_alive=True):
         self.name = name
         self._shortname = shortname
-        self.ssh = ssh
+        self.host_key = host_key
+        self.keep_alive = keep_alive
         self.console = console
+        self.ssh = ssh or self.connect()
+
+    def connect(self):
+        self.ssh = connection.connect(user_at_host=self.name,
+                                      host_key=self.host_key,
+                                      keep_alive=self.keep_alive)
+        return self.ssh
+
+    def reconnect(self):
+        """
+        Attempts to re-establish connection. Returns True for success; False
+        for failure.
+        """
+        self.ssh.close()
+        try:
+            self.ssh = self.connect()
+            return self.is_online
+        except Exception as e:
+            log.debug(e)
+            return False
 
     @property
     def shortname(self):
@@ -43,6 +67,20 @@ class Remote(object):
             name = self.name
         return name
 
+    @property
+    def hostname(self):
+        return self.name.split('@')[1]
+
+    @property
+    def is_online(self):
+        if self.ssh is None:
+            return False
+        try:
+            self.run(args="echo online")
+        except Exception:
+            return False
+        return self.ssh.get_transport().is_active()
+
     @property
     def system_type(self):
         """
@@ -78,11 +116,13 @@ def getShortName(name):
     p = re.compile('([^.]+)\.?.*')
     return p.match(hn).groups()[0]
 
+
 class PhysicalConsole():
     """
     Physical Console (set from getRemoteConsole)
     """
-    def __init__(self, name, ipmiuser, ipmipass, ipmidomain, logfile=None, timeout=20):
+    def __init__(self, name, ipmiuser, ipmipass, ipmidomain, logfile=None,
+                 timeout=20):
         self.name = name
         self.shortname = getShortName(name)
         self.timeout = timeout
@@ -96,20 +136,20 @@ class PhysicalConsole():
         Run the cmd specified using ipmitool.
         """
         if not self.ipmiuser or not self.ipmipass or not self.ipmidomain:
-            log.error('Must set ipmi_user, ipmi_password, and ipmi_domain in .teuthology.yaml')
-        log.debug('pexpect command: ipmitool -H {s}.{dn} -I lanplus -U {ipmiuser} -P {ipmipass} {cmd}'.format(
-                                   cmd=cmd,
-                                   s=self.shortname,
-                                   dn=self.ipmidomain,
-                                   ipmiuser=self.ipmiuser,
-                                   ipmipass=self.ipmipass))
+            log.error('Must set ipmi_user, ipmi_password, and ipmi_domain in .teuthology.yaml')  # noqa
+        log.debug('pexpect command: ipmitool -H {s}.{dn} -I lanplus -U {ipmiuser} -P {ipmipass} {cmd}'.format(  # noqa
+                  cmd=cmd,
+                  s=self.shortname,
+                  dn=self.ipmidomain,
+                  ipmiuser=self.ipmiuser,
+                  ipmipass=self.ipmipass))
 
-        child = pexpect.spawn ('ipmitool -H {s}.{dn} -I lanplus -U {ipmiuser} -P {ipmipass} {cmd}'.format(
-                                   cmd=cmd,
-                                   s=self.shortname,
-                                   dn=self.ipmidomain,
-                                   ipmiuser=self.ipmiuser,
-                                   ipmipass=self.ipmipass))
+        child = pexpect.spawn('ipmitool -H {s}.{dn} -I lanplus -U {ipmiuser} -P {ipmipass} {cmd}'.format(  # noqa
+                              cmd=cmd,
+                              s=self.shortname,
+                              dn=self.ipmidomain,
+                              ipmiuser=self.ipmiuser,
+                              ipmipass=self.ipmipass))
         if self.logfile:
             child.logfile = self.logfile
         return child
@@ -119,7 +159,8 @@ class PhysicalConsole():
         t = timeout
         if not t:
             t = self.timeout
-        r = child.expect(['terminated ipmitool', pexpect.TIMEOUT, pexpect.EOF], timeout=t)
+        r = child.expect(
+            ['terminated ipmitool', pexpect.TIMEOUT, pexpect.EOF], timeout=t)
         if r != 0:
             self._exec('sol deactivate')
 
@@ -138,13 +179,18 @@ class PhysicalConsole():
                 child = self._exec('sol activate')
                 child.send('\n')
                 log.debug('expect: {s} login'.format(s=self.shortname))
-                r = child.expect(['{s} login: '.format(s=self.shortname), pexpect.TIMEOUT, pexpect.EOF], timeout=(t - (time.time() - start)))
+                r = child.expect(
+                    ['{s} login: '.format(s=self.shortname),
+                     pexpect.TIMEOUT,
+                     pexpect.EOF],
+                    timeout=(t - (time.time() - start)))
                 log.debug('expect before: {b}'.format(b=child.before))
                 log.debug('expect after: {a}'.format(a=child.after))
 
                 self._exit_session(child)
                 if r == 0:
                     return
+
     def check_power(self, state, timeout=None):
         """
         Check power.  Retry if EOF encountered on power check read.
@@ -157,7 +203,8 @@ class PhysicalConsole():
         ta = time.time()
         while total < total_timeout:
             c = self._exec('power status')
-            r = c.expect(['Chassis Power is {s}'.format(s=state), pexpect.EOF, pexpect.TIMEOUT], timeout=t)
+            r = c.expect(['Chassis Power is {s}'.format(
+                s=state), pexpect.EOF, pexpect.TIMEOUT], timeout=t)
             tb = time.time()
             if r == 0:
                 return True
@@ -176,12 +223,13 @@ class PhysicalConsole():
         """
         Check status.  Returns True if console is at login prompt
         """
-        try :
+        try:
             # check for login prompt at console
             self._wait_for_login(timeout)
             return True
         except Exception as e:
-            log.info('Failed to get ipmi console status for {s}: {e}'.format(s=self.shortname, e=e))
+            log.info('Failed to get ipmi console status for {s}: {e}'.format(
+                s=self.shortname, e=e))
             return False
 
     def power_cycle(self):
@@ -203,7 +251,8 @@ class PhysicalConsole():
         start = time.time()
         while time.time() - start < self.timeout:
             child = self._exec('power reset')
-            r = child.expect(['Chassis Power Control: Reset', pexpect.EOF], timeout=self.timeout)
+            r = child.expect(['Chassis Power Control: Reset', pexpect.EOF],
+                             timeout=self.timeout)
             if r == 0:
                 break
         self._wait_for_login()
@@ -217,7 +266,8 @@ class PhysicalConsole():
         start = time.time()
         while time.time() - start < self.timeout:
             child = self._exec('power on')
-            r = child.expect(['Chassis Power Control: Up/On', pexpect.EOF], timeout=self.timeout)
+            r = child.expect(['Chassis Power Control: Up/On', pexpect.EOF],
+                             timeout=self.timeout)
             if r == 0:
                 break
         if not self.check_power('on'):
@@ -232,7 +282,8 @@ class PhysicalConsole():
         start = time.time()
         while time.time() - start < self.timeout:
             child = self._exec('power off')
-            r = child.expect(['Chassis Power Control: Down/Off', pexpect.EOF], timeout=self.timeout)
+            r = child.expect(['Chassis Power Control: Down/Off', pexpect.EOF],
+                             timeout=self.timeout)
             if r == 0:
                 break
         if not self.check_power('off', 60):
@@ -242,10 +293,11 @@ class PhysicalConsole():
     def power_off_for_interval(self, interval=30):
         """
         Physical power off for an interval. Wait for login when complete.
-       
+
         :param interval: Length of power-off period.
         """
-        log.info('Power off {s} for {i} seconds'.format(s=self.shortname, i=interval))
+        log.info('Power off {s} for {i} seconds'.format(
+            s=self.shortname, i=interval))
         child = self._exec('power off')
         child.expect('Chassis Power Control: Down/Off', timeout=self.timeout)
 
@@ -254,13 +306,16 @@ class PhysicalConsole():
         child = self._exec('power on')
         child.expect('Chassis Power Control: Up/On', timeout=self.timeout)
         self._wait_for_login()
-        log.info('Power off for {i} seconds completed'.format(s=self.shortname, i=interval))
+        log.info('Power off for {i} seconds completed'.format(
+            s=self.shortname, i=interval))
+
 
 class VirtualConsole():
     """
     Virtual Console (set from getRemoteConsole)
     """
-    def __init__(self, name, ipmiuser, ipmipass, ipmidomain, logfile=None, timeout=20):
+    def __init__(self, name, ipmiuser, ipmipass, ipmidomain, logfile=None,
+                 timeout=20):
         if libvirt is None:
             raise RuntimeError("libvirt not found")
 
@@ -282,14 +337,15 @@ class VirtualConsole():
         """
         Return true if vm domain state indicates power is on.
         """
-        return self.vm_domain.info[0] in [libvirt.VIR_DOMAIN_RUNNING, libvirt.VIR_DOMAIN_BLOCKED,
-                libvirt.VIR_DOMAIN_PAUSED]
+        return self.vm_domain.info[0] in [libvirt.VIR_DOMAIN_RUNNING,
+                                          libvirt.VIR_DOMAIN_BLOCKED,
+                                          libvirt.VIR_DOMAIN_PAUSED]
 
     def check_status(self, timeout=None):
         """
         Return true if running.
         """
-        return self.vm_domain.info()[0]  == libvirt.VIR_DOMAIN_RUNNING
+        return self.vm_domain.info()[0] == libvirt.VIR_DOMAIN_RUNNING
 
     def power_cycle(self):
         """
@@ -320,16 +376,22 @@ class VirtualConsole():
         """
         Simiulate power off for an interval.
         """
-        log.info('Power off {s} for {i} seconds'.format(s=self.shortname, i=interval))
+        log.info('Power off {s} for {i} seconds'.format(
+            s=self.shortname, i=interval))
         self.vm_domain.info().destroy()
         time.sleep(interval)
         self.vm_domain.info().create()
-        log.info('Power off for {i} seconds completed'.format(s=self.shortname, i=interval))
+        log.info('Power off for {i} seconds completed'.format(
+            s=self.shortname, i=interval))
 
-def getRemoteConsole(name, ipmiuser, ipmipass, ipmidomain, logfile=None, timeout=20):
+
+def getRemoteConsole(name, ipmiuser, ipmipass, ipmidomain, logfile=None,
+                     timeout=20):
     """
     Return either VirtualConsole or PhysicalConsole depending on name.
     """
     if misc.is_vm(name):
-        return VirtualConsole(name, ipmiuser, ipmipass, ipmidomain, logfile, timeout)
-    return PhysicalConsole(name, ipmiuser, ipmipass, ipmidomain, logfile, timeout)
+        return VirtualConsole(name, ipmiuser, ipmipass, ipmidomain, logfile,
+                              timeout)
+    return PhysicalConsole(name, ipmiuser, ipmipass, ipmidomain, logfile,
+                           timeout)

teuthology/orchestra/run.py

@@ -259,7 +259,7 @@ def run(
     :param stdout: What to do with standard output. Either a file-like object, a `logging.Logger`, `PIPE`, or `None` for copying to default log. `PIPE` means caller is responsible for reading, or command may never exit.
     :param stderr: What to do with standard error. See `stdout`.
     :param logger: If logging, write stdout/stderr to "out" and "err" children of this logger. Defaults to logger named after this module.
-    :param check_status: Whether to raise CalledProcessError on non-zero exit status, and . Defaults to True. All signals and connection loss are made to look like SIGHUP.
+    :param check_status: Whether to raise CommandFailedError on non-zero exit status, and . Defaults to True. All signals and connection loss are made to look like SIGHUP.
     :param wait: Whether to wait for process to exit. If False, returned ``r.exitstatus`` s a `gevent.event.AsyncResult`, and the actual status is available via ``.get()``.
     """
     r = execute(client, args)

teuthology/task/devstack.py

@@ -0,0 +1,382 @@
+#!/usr/bin/env python
+import contextlib
+import logging
+from cStringIO import StringIO
+import textwrap
+from configparser import ConfigParser
+import time
+
+from ..orchestra import run
+from .. import misc
+from ..contextutil import nested
+
+log = logging.getLogger(__name__)
+
+DEVSTACK_GIT_REPO = 'https://github.com/openstack-dev/devstack.git'
+DS_STABLE_BRANCHES = ("havana", "grizzly")
+
+is_devstack_node = lambda role: role.startswith('devstack')
+is_osd_node = lambda role: role.startswith('osd')
+
+
+@contextlib.contextmanager
+def task(ctx, config):
+    if config is None:
+        config = {}
+    if not isinstance(config, dict):
+        raise TypeError("config must be a dict")
+    with nested(lambda: install(ctx=ctx, config=config),
+                lambda: smoke(ctx=ctx, config=config),
+                ):
+        yield
+
+
+@contextlib.contextmanager
+def install(ctx, config):
+    """
+    Install OpenStack DevStack and configure it to use a Ceph cluster for
+    Glance and Cinder.
+
+    Requires one node with a role 'devstack'
+
+    Since devstack runs rampant on the system it's used on, typically you will
+    want to reprovision that machine after using devstack on it.
+
+    Also, the default 2GB of RAM that is given to vps nodes is insufficient. I
+    recommend 4GB. Downburst can be instructed to give 4GB to a vps node by
+    adding this to the yaml:
+
+    downburst:
+        ram: 4G
+
+    This was created using documentation found here:
+        https://github.com/openstack-dev/devstack/blob/master/README.md
+        http://ceph.com/docs/master/rbd/rbd-openstack/
+    """
+    if config is None:
+        config = {}
+    if not isinstance(config, dict):
+        raise TypeError("config must be a dict")
+
+    devstack_node = ctx.cluster.only(is_devstack_node).remotes.keys()[0]
+    an_osd_node = ctx.cluster.only(is_osd_node).remotes.keys()[0]
+
+    devstack_branch = config.get("branch", "master")
+    install_devstack(devstack_node, devstack_branch)
+    try:
+        configure_devstack_and_ceph(ctx, config, devstack_node, an_osd_node)
+        yield
+    finally:
+        pass
+
+
+def install_devstack(devstack_node, branch="master"):
+    log.info("Cloning DevStack repo...")
+
+    args = ['git', 'clone', DEVSTACK_GIT_REPO]
+    devstack_node.run(args=args)
+
+    if branch != "master":
+        if branch in DS_STABLE_BRANCHES and not branch.startswith("stable"):
+            branch = "stable/" + branch
+        log.info("Checking out {branch} branch...".format(branch=branch))
+        cmd = "cd devstack && git checkout " + branch
+        devstack_node.run(args=cmd)
+
+    log.info("Installing DevStack...")
+    args = ['cd', 'devstack', run.Raw('&&'), './stack.sh']
+    devstack_node.run(args=args)
+
+
+def configure_devstack_and_ceph(ctx, config, devstack_node, ceph_node):
+    pool_size = config.get('pool_size', '128')
+    create_pools(ceph_node, pool_size)
+    distribute_ceph_conf(devstack_node, ceph_node)
+    # This is where we would install python-ceph and ceph-common but it appears
+    # the ceph task does that for us.
+    generate_ceph_keys(ceph_node)
+    distribute_ceph_keys(devstack_node, ceph_node)
+    secret_uuid = set_libvirt_secret(devstack_node, ceph_node)
+    update_devstack_config_files(devstack_node, secret_uuid)
+    set_apache_servername(devstack_node)
+    # Rebooting is the most-often-used method of restarting devstack services
+    misc.reboot(devstack_node)
+    start_devstack(devstack_node)
+    restart_apache(devstack_node)
+
+
+def create_pools(ceph_node, pool_size):
+    log.info("Creating pools on Ceph cluster...")
+
+    for pool_name in ['volumes', 'images', 'backups']:
+        args = ['ceph', 'osd', 'pool', 'create', pool_name, pool_size]
+        ceph_node.run(args=args)
+
+
+def distribute_ceph_conf(devstack_node, ceph_node):
+    log.info("Copying ceph.conf to DevStack node...")
+
+    ceph_conf_path = '/etc/ceph/ceph.conf'
+    ceph_conf = misc.get_file(ceph_node, ceph_conf_path, sudo=True)
+    misc.sudo_write_file(devstack_node, ceph_conf_path, ceph_conf)
+
+
+def generate_ceph_keys(ceph_node):
+    log.info("Generating Ceph keys...")
+
+    ceph_auth_cmds = [
+        ['ceph', 'auth', 'get-or-create', 'client.cinder', 'mon',
+            'allow r', 'osd', 'allow class-read object_prefix rbd_children, allow rwx pool=volumes, allow rx pool=images'],  # noqa
+        ['ceph', 'auth', 'get-or-create', 'client.glance', 'mon',
+            'allow r', 'osd', 'allow class-read object_prefix rbd_children, allow rwx pool=images'],  # noqa
+        ['ceph', 'auth', 'get-or-create', 'client.cinder-backup', 'mon',
+            'allow r', 'osd', 'allow class-read object_prefix rbd_children, allow rwx pool=backups'],  # noqa
+    ]
+    for cmd in ceph_auth_cmds:
+        ceph_node.run(args=cmd)
+
+
+def distribute_ceph_keys(devstack_node, ceph_node):
+    log.info("Copying Ceph keys to DevStack node...")
+
+    def copy_key(from_remote, key_name, to_remote, dest_path, owner):
+        key_stringio = StringIO()
+        from_remote.run(
+            args=['ceph', 'auth', 'get-or-create', key_name],
+            stdout=key_stringio)
+        key_stringio.seek(0)
+        misc.sudo_write_file(to_remote, dest_path,
+                             key_stringio, owner=owner)
+    keys = [
+        dict(name='client.glance',
+             path='/etc/ceph/ceph.client.glance.keyring',
+             # devstack appears to just want root:root
+             #owner='glance:glance',
+             ),
+        dict(name='client.cinder',
+             path='/etc/ceph/ceph.client.cinder.keyring',
+             # devstack appears to just want root:root
+             #owner='cinder:cinder',
+             ),
+        dict(name='client.cinder-backup',
+             path='/etc/ceph/ceph.client.cinder-backup.keyring',
+             # devstack appears to just want root:root
+             #owner='cinder:cinder',
+             ),
+    ]
+    for key_dict in keys:
+        copy_key(ceph_node, key_dict['name'], devstack_node,
+                 key_dict['path'], key_dict.get('owner'))
+
+
+def set_libvirt_secret(devstack_node, ceph_node):
+    log.info("Setting libvirt secret...")
+
+    cinder_key_stringio = StringIO()
+    ceph_node.run(args=['ceph', 'auth', 'get-key', 'client.cinder'],
+                  stdout=cinder_key_stringio)
+    cinder_key = cinder_key_stringio.getvalue().strip()
+
+    uuid_stringio = StringIO()
+    devstack_node.run(args=['uuidgen'], stdout=uuid_stringio)
+    uuid = uuid_stringio.getvalue().strip()
+
+    secret_path = '/tmp/secret.xml'
+    secret_template = textwrap.dedent("""
+    <secret ephemeral='no' private='no'>
+        <uuid>{uuid}</uuid>
+        <usage type='ceph'>
+            <name>client.cinder secret</name>
+        </usage>
+    </secret>""")
+    misc.sudo_write_file(devstack_node, secret_path,
+                         secret_template.format(uuid=uuid))
+    devstack_node.run(args=['sudo', 'virsh', 'secret-define', '--file',
+                            secret_path])
+    devstack_node.run(args=['sudo', 'virsh', 'secret-set-value', '--secret',
+                            uuid, '--base64', cinder_key])
+    return uuid
+
+
+def update_devstack_config_files(devstack_node, secret_uuid):
+    log.info("Updating DevStack config files to use Ceph...")
+
+    def backup_config(node, file_name, backup_ext='.orig.teuth'):
+        node.run(args=['cp', '-f', file_name, file_name + backup_ext])
+
+    def update_config(config_name, config_stream, update_dict,
+                      section='DEFAULT'):
+        parser = ConfigParser()
+        parser.read_file(config_stream)
+        for (key, value) in update_dict.items():
+            parser.set(section, key, value)
+        out_stream = StringIO()
+        parser.write(out_stream)
+        out_stream.seek(0)
+        return out_stream
+
+    updates = [
+        dict(name='/etc/glance/glance-api.conf', options=dict(
+            default_store='rbd',
+            rbd_store_user='glance',
+            rbd_store_pool='images',
+            show_image_direct_url='True',)),
+        dict(name='/etc/cinder/cinder.conf', options=dict(
+            volume_driver='cinder.volume.drivers.rbd.RBDDriver',
+            rbd_pool='volumes',
+            rbd_ceph_conf='/etc/ceph/ceph.conf',
+            rbd_flatten_volume_from_snapshot='false',
+            rbd_max_clone_depth='5',
+            glance_api_version='2',
+            rbd_user='cinder',
+            rbd_secret_uuid=secret_uuid,
+            backup_driver='cinder.backup.drivers.ceph',
+            backup_ceph_conf='/etc/ceph/ceph.conf',
+            backup_ceph_user='cinder-backup',
+            backup_ceph_chunk_size='134217728',
+            backup_ceph_pool='backups',
+            backup_ceph_stripe_unit='0',
+            backup_ceph_stripe_count='0',
+            restore_discard_excess_bytes='true',
+            )),
+        dict(name='/etc/nova/nova.conf', options=dict(
+            libvirt_images_type='rbd',
+            libvirt_images_rbd_pool='volumes',
+            libvirt_images_rbd_ceph_conf='/etc/ceph/ceph.conf',
+            rbd_user='cinder',
+            rbd_secret_uuid=secret_uuid,
+            libvirt_inject_password='false',
+            libvirt_inject_key='false',
+            libvirt_inject_partition='-2',
+            )),
+    ]
+
+    for update in updates:
+        file_name = update['name']
+        options = update['options']
+        config_str = misc.get_file(devstack_node, file_name, sudo=True)
+        config_stream = StringIO(config_str)
+        backup_config(devstack_node, file_name)
+        new_config_stream = update_config(file_name, config_stream, options)
+        misc.sudo_write_file(devstack_node, file_name, new_config_stream)
+
+
+def set_apache_servername(node):
+    # Apache complains: "Could not reliably determine the server's fully
+    # qualified domain name, using 127.0.0.1 for ServerName"
+    # So, let's make sure it knows its name.
+    log.info("Setting Apache ServerName...")
+
+    hostname = node.hostname
+    config_file = '/etc/apache2/conf.d/servername'
+    misc.sudo_write_file(node, config_file,
+                         "ServerName {name}".format(name=hostname))
+
+
+def start_devstack(devstack_node):
+    log.info("Patching devstack start script...")
+    # This causes screen to start headless - otherwise rejoin-stack.sh fails
+    # because there is no terminal attached.
+    cmd = "cd devstack && sed -ie 's/screen -c/screen -dm -c/' rejoin-stack.sh"
+    devstack_node.run(args=cmd)
+
+    log.info("Starting devstack...")
+    cmd = "cd devstack && ./rejoin-stack.sh"
+    devstack_node.run(args=cmd)
+
+    # This was added because I was getting timeouts on Cinder requests - which
+    # were trying to access Keystone on port 5000. A more robust way to handle
+    # this would be to introduce a wait-loop on devstack_node that checks to
+    # see if a service is listening on port 5000.
+    log.info("Waiting 30s for devstack to start...")
+    time.sleep(30)
+
+
+def restart_apache(node):
+    node.run(args=['sudo', '/etc/init.d/apache2', 'restart'], wait=True)
+
+
+@contextlib.contextmanager
+def exercise(ctx, config):
+    log.info("Running devstack exercises...")
+
+    if config is None:
+        config = {}
+    if not isinstance(config, dict):
+        raise TypeError("config must be a dict")
+
+    devstack_node = ctx.cluster.only(is_devstack_node).remotes.keys()[0]
+
+    # TODO: save the log *and* preserve failures
+    #devstack_archive_dir = create_devstack_archive(ctx, devstack_node)
+
+    try:
+        #cmd = "cd devstack && ./exercise.sh 2>&1 | tee {dir}/exercise.log".format(  # noqa
+        #    dir=devstack_archive_dir)
+        cmd = "cd devstack && ./exercise.sh"
+        devstack_node.run(args=cmd, wait=True)
+        yield
+    finally:
+        pass
+
+
+def create_devstack_archive(ctx, devstack_node):
+    test_dir = misc.get_testdir(ctx)
+    devstack_archive_dir = "{test_dir}/archive/devstack".format(
+        test_dir=test_dir)
+    devstack_node.run(args="mkdir -p " + devstack_archive_dir)
+    return devstack_archive_dir
+
+
+@contextlib.contextmanager
+def smoke(ctx, config):
+    log.info("Running a basic smoketest...")
+
+    devstack_node = ctx.cluster.only(is_devstack_node).remotes.keys()[0]
+    an_osd_node = ctx.cluster.only(is_osd_node).remotes.keys()[0]
+
+    try:
+        create_volume(devstack_node, an_osd_node, 'smoke0', 1)
+        yield
+    finally:
+        pass
+
+
+def create_volume(devstack_node, ceph_node, vol_name, size):
+    """
+    :param size: The size of the volume, in GB
+    """
+    size = str(size)
+    log.info("Creating a {size}GB volume named {name}...".format(
+        name=vol_name,
+        size=size))
+    args = ['source', 'devstack/openrc', run.Raw('&&'), 'cinder', 'create',
+            '--display-name', vol_name, size]
+    out_stream = StringIO()
+    devstack_node.run(args=args, stdout=out_stream, wait=True)
+    vol_info = parse_os_table(out_stream.getvalue())
+    log.debug("Volume info: %s", str(vol_info))
+
+    out_stream = StringIO()
+    try:
+        ceph_node.run(args="rbd --id cinder ls -l volumes", stdout=out_stream,
+                      wait=True)
+    except run.CommandFailedError:
+        log.debug("Original rbd call failed; retrying without '--id cinder'")
+        ceph_node.run(args="rbd ls -l volumes", stdout=out_stream,
+                      wait=True)
+
+    assert vol_info['id'] in out_stream.getvalue(), \
+        "Volume not found on Ceph cluster"
+    assert vol_info['size'] == size, \
+        "Volume size on Ceph cluster is different than specified"
+    return vol_info['id']
+
+
+def parse_os_table(table_str):
+    out_dict = dict()
+    for line in table_str.split('\n'):
+        if line.startswith('|'):
+            items = line.split()
+            out_dict[items[1]] = items[3]
+    return out_dict

teuthology/task/internal.py

@@ -202,7 +202,7 @@ def connect(ctx, config):
     Open a connection to a remote host.
     """
     log.info('Opening connections...')
-    from ..orchestra import connection, remote
+    from ..orchestra import remote
     from ..orchestra import cluster
     remotes = []
     machs = []
@@ -219,11 +219,7 @@ def connect(ctx, config):
             if teuthology.is_vm(t):
                 key = None
         remotes.append(
-            remote.Remote(name=t,
-                          ssh=connection.connect(user_at_host=t,
-                                                 host_key=key,
-                                                 keep_alive=True),
-                          console=None))
+            remote.Remote(name=t, host_key=key, keep_alive=True, console=None))
     ctx.cluster = cluster.Cluster()
     if 'roles' in ctx.config:
         for rem, roles in zip(remotes, ctx.config['roles']):
@@ -237,7 +233,7 @@ def connect(ctx, config):
 
 def check_ceph_data(ctx, config):
     """
-    Check for old /var/lib/ceph directories and detect staleness. 
+    Check for old /var/lib/ceph directories and detect staleness.
     """
     log.info('Checking for old /var/lib/ceph...')
     processes = ctx.cluster.run(

teuthology/task/test/test_devstack.py

@@ -0,0 +1,48 @@
+from textwrap import dedent
+
+from .. import devstack
+
+
+class TestDevstack(object):
+    def test_parse_os_table(self):
+        table_str = dedent("""
+            +---------------------+--------------------------------------+
+            |       Property      |                Value                 |
+            +---------------------+--------------------------------------+
+            |     attachments     |                  []                  |
+            |  availability_zone  |                 nova                 |
+            |       bootable      |                false                 |
+            |      created_at     |      2014-02-21T17:14:47.548361      |
+            | display_description |                 None                 |
+            |     display_name    |                 NAME                 |
+            |          id         | ffdbd1bb-60dc-4d95-acfe-88774c09ad3e |
+            |       metadata      |                  {}                  |
+            |         size        |                  1                   |
+            |     snapshot_id     |                 None                 |
+            |     source_volid    |                 None                 |
+            |        status       |               creating               |
+            |     volume_type     |                 None                 |
+            +---------------------+--------------------------------------+
+            """).strip()
+        expected = {
+            'Property': 'Value',
+            'attachments': '[]',
+            'availability_zone': 'nova',
+            'bootable': 'false',
+            'created_at': '2014-02-21T17:14:47.548361',
+            'display_description': 'None',
+            'display_name': 'NAME',
+            'id': 'ffdbd1bb-60dc-4d95-acfe-88774c09ad3e',
+            'metadata': '{}',
+            'size': '1',
+            'snapshot_id': 'None',
+            'source_volid': 'None',
+            'status': 'creating',
+            'volume_type': 'None'}
+
+        vol_info = devstack.parse_os_table(table_str)
+        assert vol_info == expected
+
+
+
+