diff --git a/teuthology/misc.py b/teuthology/misc.py index 33f9b516973..bc025ee02c0 100644 --- a/teuthology/misc.py +++ b/teuthology/misc.py @@ -5,6 +5,7 @@ import logging import configobj import getpass import socket +import tarfile import time import urllib2 import urlparse @@ -12,6 +13,7 @@ import yaml import json import subprocess +from teuthology import safepath from .orchestra import run log = logging.getLogger(__name__) @@ -239,6 +241,51 @@ def get_file(remote, path): data = proc.stdout.getvalue() return data +def pull_directory(remote, remotedir, localdir): + """ + Copy a remote directory to a local directory. + """ + os.mkdir(localdir) + log.debug('Transferring archived files from %s:%s to %s', + remote.shortname, remotedir, localdir) + proc = remote.run( + args=[ + 'tar', + 'c', + '-f', '-', + '-C', remotedir, + '--', + '.', + ], + stdout=run.PIPE, + wait=False, + ) + tar = tarfile.open(mode='r|', fileobj=proc.stdout) + while True: + ti = tar.next() + if ti is None: + break + + if ti.isdir(): + # ignore silently; easier to just create leading dirs below + pass + elif ti.isfile(): + sub = safepath.munge(ti.name) + safepath.makedirs(root=localdir, path=os.path.dirname(sub)) + tar.makefile(ti, targetpath=os.path.join(localdir, sub)) + else: + if ti.isdev(): + type_ = 'device' + elif ti.issym(): + type_ = 'symlink' + elif ti.islnk(): + type_ = 'hard link' + else: + type_ = 'unknown' + log.info('Ignoring tar entry: %r type %r', ti.name, type_) + continue + proc.exitstatus.get() + def get_scratch_devices(remote): """ Read the scratch disk list from remote host diff --git a/teuthology/task/internal.py b/teuthology/task/internal.py index 0ad0b03caad..3ff0a20063e 100644 --- a/teuthology/task/internal.py +++ b/teuthology/task/internal.py @@ -3,13 +3,11 @@ import contextlib import gevent import logging import os -import tarfile import time import yaml from teuthology import lock from teuthology import misc as teuthology -from teuthology import safepath from ..orchestra import run log = logging.getLogger(__name__) @@ -190,51 +188,12 @@ def archive(ctx, config): yield finally: if ctx.archive is not None: - log.info('Transferring archived files...') logdir = os.path.join(ctx.archive, 'remote') os.mkdir(logdir) for remote in ctx.cluster.remotes.iterkeys(): path = os.path.join(logdir, remote.shortname) - os.mkdir(path) - log.debug('Transferring archived files from %s to %s', remote.shortname, path) - proc = remote.run( - args=[ - 'tar', - 'c', - '-f', '-', - '-C', '/tmp/cephtest/archive', - '--', - '.', - ], - stdout=run.PIPE, - wait=False, - ) - tar = tarfile.open(mode='r|', fileobj=proc.stdout) - while True: - ti = tar.next() - if ti is None: - break - - if ti.isdir(): - # ignore silently; easier to just create leading dirs below - pass - elif ti.isfile(): - sub = safepath.munge(ti.name) - safepath.makedirs(root=path, path=os.path.dirname(sub)) - tar.makefile(ti, targetpath=os.path.join(path, sub)) - else: - if ti.isdev(): - type_ = 'device' - elif ti.issym(): - type_ = 'symlink' - elif ti.islnk(): - type_ = 'hard link' - else: - type_ = 'unknown' - log.info('Ignoring tar entry: %r type %r', ti.name, type_) - continue - proc.exitstatus.get() + teuthology.pull_directory(remote, '/tmp/cephtest/archive', path) log.info('Removing archive directory...') run.wait(