diff --git a/teuthology/misc.py b/teuthology/misc.py index 27e9b8b2e20..c075e56d6dd 100644 --- a/teuthology/misc.py +++ b/teuthology/misc.py @@ -18,8 +18,6 @@ import urlparse import yaml import json import re -import tempfile -import paramiko from teuthology import safepath from .orchestra import run @@ -556,7 +554,7 @@ def remove_lines_from_file(remote, path, line_is_valid_test, # get a temp file path on the remote host to write to, # we don't want to blow away the remote file and then have the # network drop out - temp_file_path = remote.remote_mktemp() + temp_file_path = remote_mktemp(remote) # write out the data to a temp file write_file(remote, temp_file_path, out_data) @@ -572,7 +570,7 @@ def append_lines_to_file(remote, path, lines, sudo=False): Remove_lines_from_list. """ - temp_file_path = remote.remote_mktemp() + temp_file_path = remote_mktemp(remote) data = get_file(remote, path, sudo) @@ -587,6 +585,26 @@ def append_lines_to_file(remote, path, lines, sudo=False): move_file(remote, temp_file_path, path) +def remote_mktemp(remote, sudo=False): + """ + Make a temporary file on a remote system + """ + args = [] + if sudo: + args.append('sudo') + args.extend([ + 'python', + '-c', + 'import os; import tempfile; (fd,fname) = tempfile.mkstemp(); os.close(fd); print fname.rstrip()' + ]) + proc = remote.run( + args=args, + stdout=StringIO(), + ) + data = proc.stdout.getvalue() + return data + + def create_file(remote, path, data="", permissions=str(644), sudo=False): """ Create a file on the remote host. @@ -612,35 +630,25 @@ def create_file(remote, path, data="", permissions=str(644), sudo=False): append_lines_to_file(remote, path, data, sudo) -def do_remote_sftp(remote, tempf, sudo=False): - """ - Make sure file is aways readble if root, and use SFTPClient to - copy across data. - """ - if sudo: - args = [] - args.extend([ - 'sudo', - 'chmod', - '0666', - tempf, - ]) - remote.run( - args=args, - stdout=StringIO(), - ) - conn = remote.connect() - transport = conn.get_transport() - sftp = paramiko.SFTPClient.from_transport(transport) - with sftp.open(tempf, 'rb') as file_sftp: - result = file_sftp.read() - return result - def get_file(remote, path, sudo=False): """ - Copy_remote wrapper. + Read a file from remote host into memory. """ - return remote.copy_remote(path, sudo) + args = [] + if sudo: + args.append('sudo') + args.extend([ + 'cat', + '--', + path, + ]) + proc = remote.run( + args=args, + stdout=StringIO(), + ) + data = proc.stdout.getvalue() + return data + def pull_directory(remote, remotedir, localdir): """ @@ -650,36 +658,44 @@ def pull_directory(remote, remotedir, localdir): remote.shortname, remotedir, localdir) if not os.path.exists(localdir): os.mkdir(localdir) - result = remote.tar_remote(remotedir, sudo=True) - _, local_tarfile = tempfile.mkstemp() - with open(local_tarfile, 'r+') as fb1: - fb1.write(result) - fb1.seek(0) - tar = tarfile.open(mode='r|', fileobj=fb1) - while True: - ti = tar.next() - if ti is None: - break + proc = remote.run( + args=[ + 'sudo', + 'tar', + 'c', + '-f', '-', + '-C', remotedir, + '--', + '.', + ], + stdout=run.PIPE, + wait=False, + ) + tar = tarfile.open(mode='r|', fileobj=proc.stdout) + while True: + ti = tar.next() + if ti is None: + break - if ti.isdir(): - # ignore silently; easier to just create leading dirs below - pass - elif ti.isfile(): - sub = safepath.munge(ti.name) - safepath.makedirs(root=localdir, path=os.path.dirname(sub)) - tar.makefile(ti, targetpath=os.path.join(localdir, sub)) + if ti.isdir(): + # ignore silently; easier to just create leading dirs below + pass + elif ti.isfile(): + sub = safepath.munge(ti.name) + safepath.makedirs(root=localdir, path=os.path.dirname(sub)) + tar.makefile(ti, targetpath=os.path.join(localdir, sub)) + else: + if ti.isdev(): + type_ = 'device' + elif ti.issym(): + type_ = 'symlink' + elif ti.islnk(): + type_ = 'hard link' else: - if ti.isdev(): - type_ = 'device' - elif ti.issym(): - type_ = 'symlink' - elif ti.islnk(): - type_ = 'hard link' - else: - type_ = 'unknown' - log.info('Ignoring tar entry: %r type %r', ti.name, type_) - continue - os.remove(local_tarfile) + type_ = 'unknown' + log.info('Ignoring tar entry: %r type %r', ti.name, type_) + continue + proc.exitstatus.get() def pull_directory_tarball(remote, remotedir, localfile): @@ -688,9 +704,21 @@ def pull_directory_tarball(remote, remotedir, localfile): """ log.debug('Transferring archived files from %s:%s to %s', remote.shortname, remotedir, localfile) - tardata = remote.tar_remote(remotedir, zip_flag=True, sudo=True) - with open(localfile, 'w') as out: - out.write(tardata) + out = open(localfile, 'w') + proc = remote.run( + args=[ + 'sudo', + 'tar', + 'cz', + '-f', '-', + '-C', remotedir, + '--', + '.', + ], + stdout=out, + wait=False, + ) + proc.exitstatus.get() def get_wwn_id_map(remote, devs): diff --git a/teuthology/orchestra/remote.py b/teuthology/orchestra/remote.py index 905fc6f88ab..79dbc2d413b 100644 --- a/teuthology/orchestra/remote.py +++ b/teuthology/orchestra/remote.py @@ -8,9 +8,7 @@ import time import pexpect import re import logging -from cStringIO import StringIO from teuthology import lockstatus as ls -import paramiko try: import libvirt @@ -109,82 +107,6 @@ class Remote(object): r.remote = self return r - def remote_mktemp(self, sudo=False): - """ - Make a remote temporary file - """ - args = [] - if sudo: - args.append('sudo') - args.extend([ - 'python', - '-c', - 'import os; import tempfile; import sys; (fd,fname) = tempfile.mkstemp(); os.close(fd); sys.stdout.write(fname.rstrip()); sys.stdout.flush()' - ]) - proc = self.run( - args=args, - stdout=StringIO(), - ) - data = proc.stdout.getvalue() - return data - - def _set_remote_perms(self, tempf, perms): - args = [] - args.extend([ - 'sudo', - 'chmod', - perms, - tempf, - ]) - self.run( - args=args, - stdout=StringIO(), - ) - - def _do_sftp_cmd(self, args, tempf, sudo=False): - self.run( - args=args, - stdout=StringIO(), - ) - if sudo: - self._set_remote_perms(tempf, '0666') - conn = self.connect() - transport = conn.get_transport() - sftp = paramiko.SFTPClient.from_transport(transport) - with sftp.open(tempf, 'rb') as file_sftp: - result = file_sftp.read() - return result - - def copy_remote(self, path, sudo=False): - """ - Read a file from the remote host into memory. - """ - tempf = self.remote_mktemp() - args = [ - 'sudo', - 'cp', - path, - tempf, - ] - return self._do_sftp_cmd(args, tempf, sudo) - - def tar_remote(self, path, sudo=False, zip_flag=False): - """ - Tar a remote file. - """ - zip_fld = lambda x: 'cz' if x else 'c' - tempf = self.remote_mktemp() - args = [ - 'sudo', - 'tar', - zip_fld(zip_flag), - '-f', tempf, - '-C', path, - '--', - '.', - ] - return self._do_sftp_cmd(args, tempf, sudo) - def getShortName(name): """ diff --git a/teuthology/task/kernel.py b/teuthology/task/kernel.py index d54e79ed9da..7d63190dcc0 100644 --- a/teuthology/task/kernel.py +++ b/teuthology/task/kernel.py @@ -649,7 +649,7 @@ def update_grub_rpm(remote, newversion): newgrub = generate_legacy_grub_entry(remote, newversion) for line in newgrub: data += line + '\n' - temp_file_path = remote.remote_mktemp() + temp_file_path = teuthology.remote_mktemp(remote) teuthology.sudo_write_file(remote, temp_file_path, StringIO(data), '755') teuthology.move_file(remote, temp_file_path, '/boot/grub/grub.conf', True) else: