RepoMirrors/ceph
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2025-01-20 10:01:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make install.upgrade work with CentOS/RHEL/Fedora.

Browse Source
This commit is contained in:
Zack Cerza 2013-08-09 17:01:36 -05:00
parent 2a7a61cd17
commit b78a6c779b
1 changed files with 213 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

274
teuthology/task/install.py
View File

@ -8,20 +8,69 @@ from teuthology import misc as teuthology
from teuthology import contextutil
from teuthology.parallel import parallel
from ..orchestra import run
from ..orchestra.run import CommandFailedError
log = logging.getLogger(__name__)
# Should the RELEASE value get extracted from somewhere?
RELEASE = "1-0"
# This is intended to be a complete listing of ceph packages. If we're going
# to hardcode this stuff, I don't want to do it in more than once place.
PACKAGES = {}
PACKAGES['ceph'] = {}
PACKAGES['ceph']['deb'] = [
'ceph',
'ceph-dbg',
'ceph-mds',
'ceph-mds-dbg',
'ceph-common',
'ceph-common-dbg',
'ceph-fuse',
'ceph-fuse-dbg',
'ceph-test',
'ceph-test-dbg',
'radosgw',
'radosgw-dbg',
'python-ceph',
'libcephfs1',
'libcephfs1-dbg',
'libcephfs-java',
'librados2',
'librados2-dbg',
'librbd1',
'librbd1-dbg',
]
PACKAGES['ceph']['rpm'] = [
'ceph-debug',
'ceph-radosgw',
'ceph-test',
'ceph-devel',
'ceph',
'ceph-fuse',
'rest-bench',
'libcephfs_jni1',
'libcephfs1',
'python-ceph',
]
def _run_and_log_error_if_fails(remote, args):
response = StringIO()
try:
remote.run(
args=args,
stdout=response,
stderr=response,
)
except CommandFailedError:
log.error(response.getvalue().strip())
raise
def _get_baseurlinfo_and_dist(ctx, remote, config):
retval = {}
relval = None
r = remote.run(
args=['lsb_release', '-sc'],
stdout=StringIO(),
)
retval['dist'] = r.stdout.getvalue().strip()
r = remote.run(
args=['arch'],
stdout=StringIO(),
@ -38,18 +87,24 @@ def _get_baseurlinfo_and_dist(ctx, remote, config):
retval['relval'] = r.stdout.getvalue().strip()
dist_name = None
if ((retval['distro'] == 'CentOS') | (retval['distro'] == 'RedHatEnterpriseServer')):
distri = 'centos'
dist_name = 'el'
relval = retval['relval']
relval = relval[0:relval.find('.')]
retval['distro_release'] = '%s%s' %(distri, relval)
retval['dist_release'] = '%s%s' %(dist_name, relval)
distri = 'centos'
retval['distro_release'] = '%s%s' % (distri, relval)
retval['dist'] = retval['distro_release']
dist_name = 'el'
retval['dist_release'] = '%s%s' % (dist_name, relval)
elif retval['distro'] == 'Fedora':
distri = retval['distro']
dist_name = 'fc'
retval['distro_release'] = '%s%s' %(dist_name, retval['relval'])
retval['dist_release'] = '%s%s' %(dist_name, retval['relval'])
retval['distro_release'] = '%s%s' % (dist_name, retval['relval'])
retval['dist_release'] = retval['distro_release']
else:
r = remote.run(
args=['lsb_release', '-sc'],
stdout=StringIO(),
)
retval['dist'] = r.stdout.getvalue().strip()
retval['distro_release'] = None
retval['dist_release'] = None
@ -70,6 +125,51 @@ def _get_baseurlinfo_and_dist(ctx, remote, config):
return retval
def _get_baseurl(ctx, remote, config):
# get distro name and arch
baseparms = _get_baseurlinfo_and_dist(ctx, remote, config)
base_url = 'http://{host}/{proj}-{pkg_type}-{dist}-{arch}-{flavor}/{uri}'.format(
host=ctx.teuthology_config.get('gitbuilder_host',
'gitbuilder.ceph.com'),
proj=config.get('project', 'ceph'),
pkg_type=remote.system_type,
**baseparms
)
return base_url
class VersionNotFoundError(Exception):
def __init__(self, url):
self.url = url
def __str__(self):
return "Failed to fetch package version from %s" % self.url
def _block_looking_for_package_version(remote, base_url, wait=False):
"""
Look for, and parse, a file called 'version' in base_url.
wait -- wait forever for the file to show up. (default False)
"""
while True:
r = remote.run(
args=['wget', '-q', '-O-', base_url + '/version'],
stdout=StringIO(),
check_status=False,
)
if r.exitstatus != 0:
if wait:
log.info('Package not there yet, waiting...')
time.sleep(15)
continue
raise VersionNotFoundError(base_url)
break
version = r.stdout.getvalue().strip()
return version
def _update_deb_package_list_and_install(ctx, remote, debs, config):
"""
updates the package list so that apt-get can
@ -95,17 +195,12 @@ def _update_deb_package_list_and_install(ctx, remote, debs, config):
stdout=StringIO(),
)
# get distro name and arch
baseparms = _get_baseurlinfo_and_dist(ctx, remote, config)
log.info("Installing packages: {pkglist} on remote deb {arch}".format(
pkglist=", ".join(debs), arch=baseparms['arch']))
dist = baseparms['dist']
base_url = 'http://{host}/{proj}-deb-{dist}-{arch}-{flavor}/{uri}'.format(
host=ctx.teuthology_config.get('gitbuilder_host',
'gitbuilder.ceph.com'),
proj=config.get('project', 'ceph'),
**baseparms
)
pkglist=", ".join(debs), arch=baseparms['arch'])
)
# get baseurl
base_url = _get_baseurl(ctx, remote, config)
log.info('Pulling from %s', base_url)
# get package version string
@ -130,7 +225,7 @@ def _update_deb_package_list_and_install(ctx, remote, debs, config):
remote.run(
args=[
'echo', 'deb', base_url, dist, 'main',
'echo', 'deb', base_url, baseparms['dist'], 'main',
run.Raw('|'),
'sudo', 'tee', '/etc/apt/sources.list.d/{proj}.list'.format(proj=config.get('project', 'ceph')),
],
@ -146,6 +241,19 @@ def _update_deb_package_list_and_install(ctx, remote, debs, config):
stdout=StringIO(),
)
def _yum_fix_repo_priority(remote, project):
remote.run(
args=[
'sudo',
'sed',
'-i',
run.Raw('\':a;N;$!ba;s/enabled=1\\ngpg/enabled=1\\npriority=1\\ngpg/g\''),
'/etc/yum.repos.d/%s.repo' % project,
]
)
def _update_rpm_package_list_and_install(ctx, remote, rpm, config):
baseparms = _get_baseurlinfo_and_dist(ctx, remote, config)
log.info("Installing packages: {pkglist} on remote rpm {arch}".format(
@ -176,10 +284,7 @@ def _update_rpm_package_list_and_install(ctx, remote, rpm, config):
remote.run(args=['rm', '-f', rpm_name])
#Fix Repo Priority
remote.run(
args=[
'sudo', 'sed', '-i', run.Raw('\':a;N;$!ba;s/enabled=1\\ngpg/enabled=1\\npriority=1\\ngpg/g\''), '/etc/yum.repos.d/ceph.repo',
])
_yum_fix_repo_priority(remote, config.get('project', 'ceph'))
remote.run(
args=[
@ -474,7 +579,8 @@ def install(ctx, config):
if project == 'ceph':
purge_data(ctx)
def _upgrade_packages(ctx, config, remote, debs, branch):
def _upgrade_deb_packages(ctx, config, remote, debs, branch):
"""
upgrade all packages
"""
@ -482,7 +588,7 @@ def _upgrade_packages(ctx, config, remote, debs, branch):
r = remote.run(
args=[
'sudo', 'apt-key', 'list', run.Raw('|'), 'grep', 'Ceph',
],
],
stdout=StringIO(),
)
if r.stdout.getvalue().find('Ceph automated package') == -1:
@ -558,6 +664,69 @@ def _upgrade_packages(ctx, config, remote, debs, branch):
stdout=StringIO(),
)
def _upgrade_rpm_packages(ctx, config, remote, pkgs, branch):
"""
Upgrade RPM packages.
"""
distinfo = _get_baseurlinfo_and_dist(ctx, remote, config)
log.info(
"Host {host} is: {distro} {ver} {arch}".format(
host=remote.shortname,
distro=distinfo['distro'],
ver=distinfo['relval'],
arch=distinfo['arch'],)
)
base_url = _get_baseurl(ctx, remote, config)
log.info('Repo base URL: %s', base_url)
version = _block_looking_for_package_version(
remote,
base_url,
config.get('wait_for_package', False))
# FIXME: 'version' as retreived from the repo is actually the RPM version
# PLUS *part* of the release. Example:
# Right now, ceph master is given the following version in the repo file:
# v0.67-rc3.164.gd5aa3a9 - whereas in reality the RPM version is 0.61.7
# and the release is 37.g1243c97.el6 (for centos6).
# Point being, I have to mangle a little here.
if version[0] == 'v':
version = version[1:]
version = "{repover}.{therest}".format(
repover=version,
therest=distinfo['dist_release'],
)
project = config.get('project', 'ceph')
# Remove the -release package before upgrading it
args = ['sudo', 'rpm', '-ev', '%s-release' % project]
_run_and_log_error_if_fails(remote, args)
# Build the new -release package path
release_rpm = "{base}/noarch/{proj}-release-{release}.{dist_release}.noarch.rpm".format(
base=base_url,
proj=project,
release=RELEASE,
dist_release=distinfo['dist_release'],
)
# Upgrade the -release package
args = ['sudo', 'rpm', '-Uv', release_rpm]
_run_and_log_error_if_fails(remote, args)
_yum_fix_repo_priority(remote, project)
# Build a space-separated string consisting of $PKG-$VER for yum
pkgs_with_vers = ["%s-%s" % (pkg, version) for pkg in pkgs]
# Actually upgrade the project packages
# FIXME: This currently outputs nothing until the command is finished
# executing. That sucks; fix it.
args = ['sudo', 'yum', '-y', 'install']
args += pkgs_with_vers
_run_and_log_error_if_fails(remote, args)
@contextlib.contextmanager
def upgrade(ctx, config):
"""
@ -585,41 +754,11 @@ def upgrade(ctx, config):
branch = None
proj_debs = {}
proj_debs['ceph'] = [
'ceph',
'ceph-dbg',
'ceph-mds',
'ceph-mds-dbg',
'ceph-common',
'ceph-common-dbg',
'ceph-fuse',
'ceph-fuse-dbg',
'ceph-test',
'ceph-test-dbg',
'radosgw',
'radosgw-dbg',
'python-ceph',
'libcephfs1',
'libcephfs1-dbg',
'libcephfs-java',
'librados2',
'librados2-dbg',
'librbd1',
'librbd1-dbg',
]
project = config.get('project', 'ceph')
debs = proj_debs.get(project, [])
# FIXME: extra_pkgs is not distro-agnostic
extra_pkgs = config.get('extra_packages', [])
log.info('extra packages: {packages}'.format(packages=extra_pkgs))
debs += extra_pkgs
log.info("Upgrading {proj} debian packages: {debs}".format(
proj=project, debs=', '.join(debs)))
if config.get('all') is not None:
node = config.get('all')
@ -627,7 +766,17 @@ def upgrade(ctx, config):
if var == 'branch' or var == 'tag' or var == 'sha1':
branch = branch_val
for remote in ctx.cluster.remotes.iterkeys():
_upgrade_packages(ctx, config, remote, debs, branch)
system_type = teuthology.get_system_type(remote)
assert system_type in ('deb', 'rpm')
pkgs = PACKAGES[project][system_type]
log.info("Upgrading {proj} {system_type} packages: {pkgs}".format(
proj=project, system_type=system_type, pkgs=', '.join(pkgs)))
# FIXME: again, make extra_pkgs distro-agnostic
pkgs += extra_pkgs
if system_type == 'deb':
_upgrade_deb_packages(ctx, config, remote, pkgs, branch)
elif system_type == 'rpm':
_upgrade_rpm_packages(ctx, config, remote, pkgs, branch)
else:
list_roles = []
for role in config.keys():
@ -639,7 +788,10 @@ def upgrade(ctx, config):
for var, branch_val in kkeys.iteritems():
if var == 'branch' or var == 'tag' or var == 'sha1':
branch = branch_val
_upgrade_packages(ctx, config, remote, debs, branch)
system_type = teuthology.get_system_type(remote)
assert system_type in ('deb', 'rpm')
pkgs = PACKAGES[project][system_type]
_upgrade_deb_packages(ctx, config, remote, pkgs, branch)
list_roles.append(remote)
yield
@ -697,7 +849,7 @@ def task(ctx, config):
flavor = 'gcov'
ctx.summary['flavor'] = flavor
with contextutil.nested(
lambda: install(ctx=ctx, config=dict(
branch=config.get('branch'),