mirror of
https://github.com/ceph/ceph
synced 2025-03-22 10:17:23 +00:00
253 lines
7.8 KiB
Python
Executable File
253 lines
7.8 KiB
Python
Executable File
#!/usr/bin/python3
|
|
#
|
|
# in default mode:
|
|
# make a combined "manifest-list" container out of two arch-specific containers
|
|
# searches for latest tags on HOST/{AMD,ARM}64_REPO, makes sure they refer
|
|
# to the same Ceph SHA1, and creates a manifest-list ("fat") image on
|
|
# MANIFEST_HOST/MANIFEST_REPO with the 'standard' set of tags:
|
|
# v<major>
|
|
# v<major>.<minor>
|
|
# v<major>.<minor>.<micro>
|
|
# v<major>.<minor>.<micro>-<YYYYMMDD>
|
|
#
|
|
# uses scratch local manifest LOCALMANIFEST, defined here; will be destroyed if present
|
|
#
|
|
# in promote mode (by adding the --promote argument):
|
|
# instead of building the manifest-list container, copy it
|
|
# (and all of its tags) from the prerelease repo to the release repo
|
|
#
|
|
# Assumes valid logins to the necessary hosts/repos with permission to write images
|
|
#
|
|
# Environment variables to set:
|
|
# ARCH_SPECIFIC_HOST (default 'quay.ceph.io'): host of prerelease repos
|
|
# AMD64_REPO (default 'ceph/prerelease-amd64') prerelease amd64 repo
|
|
# ARM64_REPO (default 'ceph/prerelease-arm64') prerelease arm64 repo
|
|
# MANIFEST_HOST (default 'quay.ceph.io') prerelease manifest-list host
|
|
# MANIFEST_REPO (default 'ceph/prerelease') prerelease manifest-list repo
|
|
# RELEASE_MANIFEST_HOST (default 'quay.io') release host
|
|
# RELEASE_MANIFEST_REPO (default 'ceph/ceph') release repo
|
|
|
|
|
|
import argparse
|
|
from datetime import datetime
|
|
import functools
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
|
|
# Manifest image. Will be destroyed if already present.
|
|
LOCALMANIFEST = 'localhost/m'
|
|
|
|
|
|
def dump_vars(names, vardict):
|
|
for name in names:
|
|
print(f'{name}: {vardict[name]}', file=sys.stderr)
|
|
|
|
|
|
def run_command(args):
|
|
print(f'running {args}', file=sys.stderr)
|
|
if not isinstance(args, list):
|
|
args = args.split()
|
|
try:
|
|
result = subprocess.run(
|
|
args,
|
|
capture_output=True,
|
|
text=True,
|
|
check=True)
|
|
return True, result.stdout, result.stderr
|
|
|
|
except subprocess.CalledProcessError as e:
|
|
return False, e.output, e.stderr
|
|
|
|
|
|
def get_command_output(args):
|
|
success, stdout, stderr = run_command(args)
|
|
return (stdout if success else None)
|
|
|
|
|
|
def run_command_show_failure(args):
|
|
success, stdout, stderr = run_command(args)
|
|
if not success:
|
|
print(f'{args} failed:', file=sys.stderr)
|
|
print(f'stdout:\n{stdout}')
|
|
print(f'stderr:\n{stderr}')
|
|
return success
|
|
|
|
|
|
@functools.lru_cache
|
|
def get_tags(path):
|
|
cmdout = get_command_output(f'skopeo list-tags docker://{path}')
|
|
return json.loads(cmdout)['Tags']
|
|
|
|
|
|
def get_latest_tag(path):
|
|
try:
|
|
latest_tag = get_tags(path)[-1]
|
|
except IndexError:
|
|
return None
|
|
return latest_tag
|
|
|
|
|
|
@functools.lru_cache
|
|
def get_image_inspect(path):
|
|
info = json.loads(
|
|
get_command_output(f'skopeo inspect docker://{path}')
|
|
)
|
|
return info
|
|
|
|
|
|
def get_sha1(info):
|
|
labels = info.get('Labels', None)
|
|
if not labels:
|
|
return None
|
|
return labels.get('CEPH_SHA1', None)
|
|
|
|
|
|
@functools.lru_cache
|
|
def get_all_matching_digest_tags(path, tag):
|
|
|
|
matching_tags = list()
|
|
digest = get_image_inspect(f'{path}:{tag}')['Digest']
|
|
|
|
for t in get_tags(path):
|
|
this_digest = get_image_inspect(f'{path}:{t}')['Digest']
|
|
if this_digest == digest:
|
|
matching_tags.append(t)
|
|
|
|
return matching_tags
|
|
|
|
|
|
def parse_args():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument('-n', '--dry-run', action='store_true', help='do all local manipulations but do not push final containers to MANIFEST_HOST, or in --promote, calculate but do not copy images to release host')
|
|
ap.add_argument('-P', '--promote', action='store_true', help='promote newest prerelease manifest container to released (move from MANIFEST_HOST to RELEASE_MANIFEST_HOST')
|
|
args = ap.parse_args()
|
|
return args
|
|
|
|
def build_prerelease(sysargs):
|
|
global args
|
|
|
|
arch_specific_host = os.environ.get('ARCH_SPECIFIC_HOST', 'quay.ceph.io')
|
|
amd64_repo = os.environ.get('AMD64_REPO', 'ceph/prerelease-amd64')
|
|
arm64_repo = os.environ.get('ARM64_REPO', 'ceph/prerelease-arm64')
|
|
manifest_host = os.environ.get('MANIFEST_HOST', 'quay.ceph.io')
|
|
manifest_repo = os.environ.get('MANIFEST_REPO', 'ceph/prerelease')
|
|
|
|
dump_vars(
|
|
('arch_specific_host',
|
|
'amd64_repo',
|
|
'arm64_repo',
|
|
'manifest_host',
|
|
'manifest_repo',
|
|
),
|
|
locals())
|
|
repopaths = (
|
|
f'{arch_specific_host}/{amd64_repo}',
|
|
f'{arch_specific_host}/{arm64_repo}',
|
|
)
|
|
tags = [get_latest_tag(p) for p in repopaths]
|
|
print(f'latest tags: amd64:{tags[0]} arm64:{tags[1]}')
|
|
|
|
# check that version of latest tag matches
|
|
version_re = \
|
|
r'v(?P<major>\d+)\.(?P<minor>\d+)\.(?P<micro>\d+)-(?P<date>\d+)'
|
|
versions = list()
|
|
for tag in tags:
|
|
mo = re.match(version_re, tag)
|
|
ver = f'{mo.group("major")}.{mo.group("minor")}.{mo.group("micro")}'
|
|
versions.append(ver)
|
|
if versions[0] != versions[1]:
|
|
print(
|
|
f'version mismatch: amd64:{versions[0]} arm64:{versions[1]}',
|
|
file=sys.stderr,
|
|
)
|
|
return(1)
|
|
|
|
major, minor, micro = mo.group(1), mo.group(2), mo.group(3)
|
|
print(f'Ceph version: {major}.{minor}.{micro}', file=sys.stderr)
|
|
|
|
# check that ceph sha1 of two arch images matches
|
|
paths_with_tags = [f'{p}:{t}' for (p, t) in zip(repopaths, tags)]
|
|
info = [get_image_inspect(p) for p in paths_with_tags]
|
|
sha1s = [get_sha1(i) for i in info]
|
|
if sha1s[0] != sha1s[1]:
|
|
print(
|
|
f'sha1 mismatch: amd64: {sha1s[0]} arm64: {sha1s[1]}',
|
|
file=sys.stderr,
|
|
)
|
|
builddate = [i['Created'] for i in info]
|
|
print(
|
|
f'Build dates: amd64: {builddate[0]} arm64: {builddate[1]}',
|
|
file=sys.stderr,
|
|
)
|
|
return(1)
|
|
|
|
# create manifest list image with the standard list of tags
|
|
# ignore failure on manifest rm
|
|
run_command(f'podman manifest rm {LOCALMANIFEST}')
|
|
run_command_show_failure(f'podman manifest create {LOCALMANIFEST}')
|
|
for p in paths_with_tags:
|
|
run_command_show_failure(f'podman manifest add m {p}')
|
|
base = f'{manifest_host}/{manifest_repo}'
|
|
for t in (
|
|
f'v{major}',
|
|
f'v{major}.{minor}',
|
|
f'v{major}.{minor}.{micro}',
|
|
f'v{major}.{minor}.{micro}-{datetime.today().strftime("%Y%m%d")}',
|
|
):
|
|
if sysargs.dry_run:
|
|
print(f'skipping podman manifest push {LOCALMANIFEST} {base}:{t}')
|
|
else:
|
|
run_command_show_failure(
|
|
f'podman manifest push {LOCALMANIFEST} {base}:{t}')
|
|
|
|
def promote(sysargs):
|
|
manifest_host = os.environ.get('MANIFEST_HOST', 'quay.ceph.io')
|
|
manifest_repo = os.environ.get('MANIFEST_REPO', 'ceph/prerelease')
|
|
release_manifest_host = os.environ.get('RELEASE_MANIFEST_HOST', 'quay.io')
|
|
release_manifest_repo = os.environ.get('RELEASE_MANIFEST_REPO', 'ceph/ceph')
|
|
dump_vars(
|
|
('manifest_host',
|
|
'manifest_repo',
|
|
'release_manifest_host',
|
|
'release_manifest_repo',
|
|
),
|
|
locals())
|
|
|
|
manifest_path = f'{manifest_host}/{manifest_repo}'
|
|
release_path = f'{release_manifest_host}/{release_manifest_repo}'
|
|
latest_tag = get_latest_tag(manifest_path)
|
|
all_tags = get_all_matching_digest_tags(manifest_path, latest_tag)
|
|
|
|
copypaths = list()
|
|
for t in all_tags:
|
|
from_path = f'{manifest_path}:{t}'
|
|
to_path = f'{release_path}:{t}'
|
|
copypaths.append((from_path, to_path))
|
|
|
|
if sysargs.dry_run:
|
|
for f, t in copypaths:
|
|
print(f'dry-run: Would copy: {f} -> {t}')
|
|
return(0)
|
|
|
|
for f, t in copypaths:
|
|
print(f'Will copy: {f} -> {t}')
|
|
|
|
for f, t in copypaths:
|
|
run_command_show_failure(f'skopeo copy --multi-arch=all docker://{f} docker://{t}')
|
|
|
|
|
|
def main():
|
|
args = parse_args()
|
|
|
|
if args.promote:
|
|
promote(args)
|
|
else:
|
|
build_prerelease(args)
|
|
|
|
|
|
if (__name__ == '__main__'):
|
|
sys.exit(main())
|