mirror of
https://github.com/ceph/ceph
synced 2024-12-27 14:03:25 +00:00
500 lines
15 KiB
Python
500 lines
15 KiB
Python
"""
|
|
Support for paramiko remote objects.
|
|
"""
|
|
from . import run
|
|
import connection
|
|
from teuthology import misc
|
|
import time
|
|
import pexpect
|
|
import re
|
|
import logging
|
|
from cStringIO import StringIO
|
|
from teuthology import lockstatus as ls
|
|
import os
|
|
import tempfile
|
|
|
|
try:
|
|
import libvirt
|
|
except ImportError:
|
|
libvirt = None
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
class Remote(object):
|
|
|
|
"""
|
|
A connection to a remote host.
|
|
|
|
This is a higher-level wrapper around Paramiko's `SSHClient`.
|
|
"""
|
|
|
|
# for unit tests to hook into
|
|
_runner = staticmethod(run.run)
|
|
|
|
def __init__(self, name, ssh=None, shortname=None, console=None,
|
|
host_key=None, keep_alive=True):
|
|
self.name = name
|
|
self.hostname = name.split('@')[-1]
|
|
self._shortname = shortname
|
|
self.host_key = host_key
|
|
self.keep_alive = keep_alive
|
|
self.console = console
|
|
self.ssh = ssh or self.connect()
|
|
|
|
def connect(self):
|
|
self.ssh = connection.connect(user_at_host=self.name,
|
|
host_key=self.host_key,
|
|
keep_alive=self.keep_alive)
|
|
return self.ssh
|
|
|
|
def reconnect(self):
|
|
"""
|
|
Attempts to re-establish connection. Returns True for success; False
|
|
for failure.
|
|
"""
|
|
self.ssh.close()
|
|
try:
|
|
self.ssh = self.connect()
|
|
return self.is_online
|
|
except Exception as e:
|
|
log.debug(e)
|
|
return False
|
|
|
|
@property
|
|
def shortname(self):
|
|
name = self._shortname
|
|
if name is None:
|
|
name = self.hostname.split('.')[0]
|
|
return name
|
|
|
|
@property
|
|
def is_online(self):
|
|
if self.ssh is None:
|
|
return False
|
|
try:
|
|
self.run(args="true")
|
|
except Exception:
|
|
return False
|
|
return self.ssh.get_transport().is_active()
|
|
|
|
def ensure_online(self):
|
|
if not self.is_online:
|
|
return self.connect()
|
|
|
|
@property
|
|
def system_type(self):
|
|
"""
|
|
System type decorator
|
|
"""
|
|
return misc.get_system_type(self)
|
|
|
|
def __str__(self):
|
|
return self.name
|
|
|
|
def __repr__(self):
|
|
return '{classname}(name={name!r})'.format(
|
|
classname=self.__class__.__name__,
|
|
name=self.name,
|
|
)
|
|
|
|
def run(self, **kwargs):
|
|
"""
|
|
This calls `orchestra.run.run` with our SSH client.
|
|
|
|
TODO refactor to move run.run here?
|
|
"""
|
|
r = self._runner(client=self.ssh, name=self.hostname, **kwargs)
|
|
r.remote = self
|
|
return r
|
|
|
|
def mktemp(self):
|
|
"""
|
|
Make a remote temporary file
|
|
|
|
Returns: the name of the temp file created using
|
|
tempfile.mkstemp
|
|
"""
|
|
py_cmd = "import os; import tempfile; import sys;" + \
|
|
"(fd,fname) = tempfile.mkstemp();" + \
|
|
"os.close(fd);" + \
|
|
"sys.stdout.write(fname.rstrip());" + \
|
|
"sys.stdout.flush()"
|
|
args = [
|
|
'python',
|
|
'-c',
|
|
py_cmd,
|
|
]
|
|
proc = self.run(
|
|
args=args,
|
|
stdout=StringIO(),
|
|
)
|
|
data = proc.stdout.getvalue()
|
|
return data
|
|
|
|
def chmod(self, file_path, permissions):
|
|
"""
|
|
As super-user, set permissions on the remote file specified.
|
|
"""
|
|
args = [
|
|
'sudo',
|
|
'chmod',
|
|
permissions,
|
|
file_path,
|
|
]
|
|
self.run(
|
|
args=args,
|
|
)
|
|
|
|
def _sftp_get_file(self, remote_path, local_path):
|
|
"""
|
|
Use the paramiko.SFTPClient to get a file. Returns the local filename.
|
|
"""
|
|
sftp = self.ssh.open_sftp()
|
|
sftp.get(remote_path, local_path)
|
|
return local_path
|
|
|
|
def _sftp_open_file(self, remote_path):
|
|
"""
|
|
Use the paramiko.SFTPClient to open a file. Returns a
|
|
paramiko.SFTPFile object.
|
|
"""
|
|
sftp = self.ssh.open_sftp()
|
|
return sftp.open(remote_path)
|
|
|
|
def remove(self, path):
|
|
self.run(args=['rm', '-fr', path])
|
|
|
|
def get_file(self, path, sudo=False, dest_dir='/tmp'):
|
|
"""
|
|
Fetch a remote file, and return its local filename.
|
|
"""
|
|
if sudo:
|
|
orig_path = path
|
|
path = self.mktemp()
|
|
args = [
|
|
'sudo',
|
|
'cp',
|
|
orig_path,
|
|
path,
|
|
]
|
|
self.run(args=args)
|
|
self.chmod(path, '0666')
|
|
(fd, local_temp_path) = tempfile.mkstemp(dir=dest_dir)
|
|
os.close(fd)
|
|
self._sftp_get_file(path, local_temp_path)
|
|
if sudo:
|
|
self.remove(path)
|
|
return local_temp_path
|
|
|
|
def get_tar(self, path, to_path, sudo=False):
|
|
"""
|
|
Tar a remote directory and copy it locally
|
|
"""
|
|
remote_temp_path = self.mktemp()
|
|
args = []
|
|
if sudo:
|
|
args.append('sudo')
|
|
args.extend([
|
|
'tar',
|
|
'cz',
|
|
'-f', remote_temp_path,
|
|
'-C', path,
|
|
'--',
|
|
'.',
|
|
])
|
|
self.run(args=args)
|
|
if sudo:
|
|
self.chmod(remote_temp_path, '0666')
|
|
self._sftp_get_file(remote_temp_path, to_path)
|
|
self.remove(remote_temp_path)
|
|
|
|
|
|
def getShortName(name):
|
|
"""
|
|
Extract the name portion from remote name strings.
|
|
"""
|
|
hn = name.split('@')[-1]
|
|
p = re.compile('([^.]+)\.?.*')
|
|
return p.match(hn).groups()[0]
|
|
|
|
|
|
class PhysicalConsole():
|
|
"""
|
|
Physical Console (set from getRemoteConsole)
|
|
"""
|
|
def __init__(self, name, ipmiuser, ipmipass, ipmidomain, logfile=None,
|
|
timeout=20):
|
|
self.name = name
|
|
self.shortname = getShortName(name)
|
|
self.timeout = timeout
|
|
self.logfile = None
|
|
self.ipmiuser = ipmiuser
|
|
self.ipmipass = ipmipass
|
|
self.ipmidomain = ipmidomain
|
|
|
|
def _exec(self, cmd):
|
|
"""
|
|
Run the cmd specified using ipmitool.
|
|
"""
|
|
if not self.ipmiuser or not self.ipmipass or not self.ipmidomain:
|
|
log.error('Must set ipmi_user, ipmi_password, and ipmi_domain in .teuthology.yaml') # noqa
|
|
log.debug('pexpect command: ipmitool -H {s}.{dn} -I lanplus -U {ipmiuser} -P {ipmipass} {cmd}'.format( # noqa
|
|
cmd=cmd,
|
|
s=self.shortname,
|
|
dn=self.ipmidomain,
|
|
ipmiuser=self.ipmiuser,
|
|
ipmipass=self.ipmipass))
|
|
|
|
child = pexpect.spawn('ipmitool -H {s}.{dn} -I lanplus -U {ipmiuser} -P {ipmipass} {cmd}'.format( # noqa
|
|
cmd=cmd,
|
|
s=self.shortname,
|
|
dn=self.ipmidomain,
|
|
ipmiuser=self.ipmiuser,
|
|
ipmipass=self.ipmipass))
|
|
if self.logfile:
|
|
child.logfile = self.logfile
|
|
return child
|
|
|
|
def _exit_session(self, child, timeout=None):
|
|
child.send('~.')
|
|
t = timeout
|
|
if not t:
|
|
t = self.timeout
|
|
r = child.expect(
|
|
['terminated ipmitool', pexpect.TIMEOUT, pexpect.EOF], timeout=t)
|
|
if r != 0:
|
|
self._exec('sol deactivate')
|
|
|
|
def _wait_for_login(self, timeout=None, attempts=6):
|
|
"""
|
|
Wait for login. Retry if timeouts occur on commands.
|
|
"""
|
|
log.debug('Waiting for login prompt on {s}'.format(s=self.shortname))
|
|
# wait for login prompt to indicate boot completed
|
|
t = timeout
|
|
if not t:
|
|
t = self.timeout
|
|
for i in range(0, attempts):
|
|
start = time.time()
|
|
while time.time() - start < t:
|
|
child = self._exec('sol activate')
|
|
child.send('\n')
|
|
log.debug('expect: {s} login'.format(s=self.shortname))
|
|
r = child.expect(
|
|
['{s} login: '.format(s=self.shortname),
|
|
pexpect.TIMEOUT,
|
|
pexpect.EOF],
|
|
timeout=(t - (time.time() - start)))
|
|
log.debug('expect before: {b}'.format(b=child.before))
|
|
log.debug('expect after: {a}'.format(a=child.after))
|
|
|
|
self._exit_session(child)
|
|
if r == 0:
|
|
return
|
|
|
|
def check_power(self, state, timeout=None):
|
|
"""
|
|
Check power. Retry if EOF encountered on power check read.
|
|
"""
|
|
total_timeout = timeout
|
|
if not total_timeout:
|
|
total_timeout = self.timeout
|
|
t = 1
|
|
total = t
|
|
ta = time.time()
|
|
while total < total_timeout:
|
|
c = self._exec('power status')
|
|
r = c.expect(['Chassis Power is {s}'.format(
|
|
s=state), pexpect.EOF, pexpect.TIMEOUT], timeout=t)
|
|
tb = time.time()
|
|
if r == 0:
|
|
return True
|
|
elif r == 1:
|
|
# keep trying if EOF is reached, first sleep for remaining
|
|
# timeout interval
|
|
if tb - ta < t:
|
|
time.sleep(t - (tb - ta))
|
|
# go around again if EOF or TIMEOUT
|
|
ta = tb
|
|
t *= 2
|
|
total += t
|
|
return False
|
|
|
|
def check_status(self, timeout=None):
|
|
"""
|
|
Check status. Returns True if console is at login prompt
|
|
"""
|
|
try:
|
|
# check for login prompt at console
|
|
self._wait_for_login(timeout)
|
|
return True
|
|
except Exception as e:
|
|
log.info('Failed to get ipmi console status for {s}: {e}'.format(
|
|
s=self.shortname, e=e))
|
|
return False
|
|
|
|
def power_cycle(self):
|
|
"""
|
|
Power cycle and wait for login.
|
|
"""
|
|
log.info('Power cycling {s}'.format(s=self.shortname))
|
|
child = self._exec('power cycle')
|
|
child.expect('Chassis Power Control: Cycle', timeout=self.timeout)
|
|
self._wait_for_login()
|
|
log.info('Power cycle for {s} completed'.format(s=self.shortname))
|
|
|
|
def hard_reset(self):
|
|
"""
|
|
Perform physical hard reset. Retry if EOF returned from read
|
|
and wait for login when complete.
|
|
"""
|
|
log.info('Performing hard reset of {s}'.format(s=self.shortname))
|
|
start = time.time()
|
|
while time.time() - start < self.timeout:
|
|
child = self._exec('power reset')
|
|
r = child.expect(['Chassis Power Control: Reset', pexpect.EOF],
|
|
timeout=self.timeout)
|
|
if r == 0:
|
|
break
|
|
self._wait_for_login()
|
|
log.info('Hard reset for {s} completed'.format(s=self.shortname))
|
|
|
|
def power_on(self):
|
|
"""
|
|
Physical power on. Loop checking cmd return.
|
|
"""
|
|
log.info('Power on {s}'.format(s=self.shortname))
|
|
start = time.time()
|
|
while time.time() - start < self.timeout:
|
|
child = self._exec('power on')
|
|
r = child.expect(['Chassis Power Control: Up/On', pexpect.EOF],
|
|
timeout=self.timeout)
|
|
if r == 0:
|
|
break
|
|
if not self.check_power('on'):
|
|
log.error('Failed to power on {s}'.format(s=self.shortname))
|
|
log.info('Power on for {s} completed'.format(s=self.shortname))
|
|
|
|
def power_off(self):
|
|
"""
|
|
Physical power off. Loop checking cmd return.
|
|
"""
|
|
log.info('Power off {s}'.format(s=self.shortname))
|
|
start = time.time()
|
|
while time.time() - start < self.timeout:
|
|
child = self._exec('power off')
|
|
r = child.expect(['Chassis Power Control: Down/Off', pexpect.EOF],
|
|
timeout=self.timeout)
|
|
if r == 0:
|
|
break
|
|
if not self.check_power('off', 60):
|
|
log.error('Failed to power off {s}'.format(s=self.shortname))
|
|
log.info('Power off for {s} completed'.format(s=self.shortname))
|
|
|
|
def power_off_for_interval(self, interval=30):
|
|
"""
|
|
Physical power off for an interval. Wait for login when complete.
|
|
|
|
:param interval: Length of power-off period.
|
|
"""
|
|
log.info('Power off {s} for {i} seconds'.format(
|
|
s=self.shortname, i=interval))
|
|
child = self._exec('power off')
|
|
child.expect('Chassis Power Control: Down/Off', timeout=self.timeout)
|
|
|
|
time.sleep(interval)
|
|
|
|
child = self._exec('power on')
|
|
child.expect('Chassis Power Control: Up/On', timeout=self.timeout)
|
|
self._wait_for_login()
|
|
log.info('Power off for {i} seconds completed'.format(
|
|
s=self.shortname, i=interval))
|
|
|
|
|
|
class VirtualConsole():
|
|
"""
|
|
Virtual Console (set from getRemoteConsole)
|
|
"""
|
|
def __init__(self, name, ipmiuser, ipmipass, ipmidomain, logfile=None,
|
|
timeout=20):
|
|
if libvirt is None:
|
|
raise RuntimeError("libvirt not found")
|
|
|
|
self.shortname = getShortName(name)
|
|
status_info = ls.get_status('', self.shortname)
|
|
try:
|
|
phys_host = status_info['vpshost']
|
|
except TypeError:
|
|
return
|
|
self.connection = libvirt.open(phys_host)
|
|
for i in self.connection.listDomainsID():
|
|
d = self.connection.lookupByID(i)
|
|
if d.name() == self.shortname:
|
|
self.vm_domain = d
|
|
break
|
|
return
|
|
|
|
def check_power(self, state, timeout=None):
|
|
"""
|
|
Return true if vm domain state indicates power is on.
|
|
"""
|
|
return self.vm_domain.info[0] in [libvirt.VIR_DOMAIN_RUNNING,
|
|
libvirt.VIR_DOMAIN_BLOCKED,
|
|
libvirt.VIR_DOMAIN_PAUSED]
|
|
|
|
def check_status(self, timeout=None):
|
|
"""
|
|
Return true if running.
|
|
"""
|
|
return self.vm_domain.info()[0] == libvirt.VIR_DOMAIN_RUNNING
|
|
|
|
def power_cycle(self):
|
|
"""
|
|
Simiulate virtual machine power cycle
|
|
"""
|
|
self.vm_domain.info().destroy()
|
|
self.vm_domain.info().create()
|
|
|
|
def hard_reset(self):
|
|
"""
|
|
Simiulate hard reset
|
|
"""
|
|
self.vm_domain.info().destroy()
|
|
|
|
def power_on(self):
|
|
"""
|
|
Simiulate power on
|
|
"""
|
|
self.vm_domain.info().create()
|
|
|
|
def power_off(self):
|
|
"""
|
|
Simiulate power off
|
|
"""
|
|
self.vm_domain.info().destroy()
|
|
|
|
def power_off_for_interval(self, interval=30):
|
|
"""
|
|
Simiulate power off for an interval.
|
|
"""
|
|
log.info('Power off {s} for {i} seconds'.format(
|
|
s=self.shortname, i=interval))
|
|
self.vm_domain.info().destroy()
|
|
time.sleep(interval)
|
|
self.vm_domain.info().create()
|
|
log.info('Power off for {i} seconds completed'.format(
|
|
s=self.shortname, i=interval))
|
|
|
|
|
|
def getRemoteConsole(name, ipmiuser, ipmipass, ipmidomain, logfile=None,
|
|
timeout=20):
|
|
"""
|
|
Return either VirtualConsole or PhysicalConsole depending on name.
|
|
"""
|
|
if misc.is_vm(name):
|
|
return VirtualConsole(name, ipmiuser, ipmipass, ipmidomain, logfile,
|
|
timeout)
|
|
return PhysicalConsole(name, ipmiuser, ipmipass, ipmidomain, logfile,
|
|
timeout)
|