RepoMirrors/ceph
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2025-02-24 11:37:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

internal: move pulling archive w/ tar to helper

Browse Source
This commit is contained in:
Sage Weil 2012-07-11 09:22:50 -07:00
parent 9ea22133b7
commit cff2cfa217
2 changed files with 48 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
teuthology/misc.py
View File

@ -5,6 +5,7 @@ import logging
import configobj
import getpass
import socket
import tarfile
import time
import urllib2
import urlparse
@ -12,6 +13,7 @@ import yaml
import json
import subprocess
from teuthology import safepath
from .orchestra import run
log = logging.getLogger(__name__)
@ -239,6 +241,51 @@ def get_file(remote, path):
data = proc.stdout.getvalue()
return data
def pull_directory(remote, remotedir, localdir):
"""
Copy a remote directory to a local directory.
"""
os.mkdir(localdir)
log.debug('Transferring archived files from %s:%s to %s',
remote.shortname, remotedir, localdir)
proc = remote.run(
args=[
'tar',
'c',
'-f', '-',
'-C', remotedir,
'--',
'.',
],
stdout=run.PIPE,
wait=False,
)
tar = tarfile.open(mode='r|', fileobj=proc.stdout)
while True:
ti = tar.next()
if ti is None:
break
if ti.isdir():
# ignore silently; easier to just create leading dirs below
pass
elif ti.isfile():
sub = safepath.munge(ti.name)
safepath.makedirs(root=localdir, path=os.path.dirname(sub))
tar.makefile(ti, targetpath=os.path.join(localdir, sub))
else:
if ti.isdev():
type_ = 'device'
elif ti.issym():
type_ = 'symlink'
elif ti.islnk():
type_ = 'hard link'
else:
type_ = 'unknown'
log.info('Ignoring tar entry: %r type %r', ti.name, type_)
continue
proc.exitstatus.get()
def get_scratch_devices(remote):
"""
Read the scratch disk list from remote host

43
teuthology/task/internal.py
View File

@ -3,13 +3,11 @@ import contextlib
import gevent
import logging
import os
import tarfile
import time
import yaml
from teuthology import lock
from teuthology import misc as teuthology
from teuthology import safepath
from ..orchestra import run
log = logging.getLogger(__name__)
@ -190,51 +188,12 @@ def archive(ctx, config):
yield
finally:
if ctx.archive is not None:
log.info('Transferring archived files...')
logdir = os.path.join(ctx.archive, 'remote')
os.mkdir(logdir)
for remote in ctx.cluster.remotes.iterkeys():
path = os.path.join(logdir, remote.shortname)
os.mkdir(path)
log.debug('Transferring archived files from %s to %s', remote.shortname, path)
proc = remote.run(
args=[
'tar',
'c',
'-f', '-',
'-C', '/tmp/cephtest/archive',
'--',
'.',
],
stdout=run.PIPE,
wait=False,
)
tar = tarfile.open(mode='r|', fileobj=proc.stdout)
while True:
ti = tar.next()
if ti is None:
break
if ti.isdir():
# ignore silently; easier to just create leading dirs below
pass
elif ti.isfile():
sub = safepath.munge(ti.name)
safepath.makedirs(root=path, path=os.path.dirname(sub))
tar.makefile(ti, targetpath=os.path.join(path, sub))
else:
if ti.isdev():
type_ = 'device'
elif ti.issym():
type_ = 'symlink'
elif ti.islnk():
type_ = 'hard link'
else:
type_ = 'unknown'
log.info('Ignoring tar entry: %r type %r', ti.name, type_)
continue
proc.exitstatus.get()
teuthology.pull_directory(remote, '/tmp/cephtest/archive', path)
log.info('Removing archive directory...')
run.wait(