ceph/qa/tasks/util/chacra.py

#!/usr/bin/env python3

import argparse
import logging
import requests
import sys

from pathlib import Path
from urllib.parse import urlparse

log = logging.getLogger(__name__)

SHAMAN_SEARCH_URL = 'https://shaman.ceph.com/api/search'

PROJECT = 'ceph'
DISTRO = 'ubuntu'
RELEASE = 'focal'
ARCH='x86_64'
BRANCH = 'main'
SHA1 = 'latest'
FLAVOR = 'default'
FILENAME = 'cephadm'


def search(*args, **kwargs):
    '''
    Query shaman for a build result
    '''
    resp = requests.get(SHAMAN_SEARCH_URL, params=kwargs)
    resp.raise_for_status()
    return resp

def _get_distros(distro, release, arch=None):
    ret = f'{distro}/{release}'
    if arch:
        ret = f'{ret}/{arch}'
    return ret

def _get_binary_url(host, project, ref, sha1, distro, release, arch, flavor, filename):
    return f'https://{host}/binaries/{project}/{ref}/{sha1}/{distro}/{release}/{arch}/flavors/{flavor}/{filename}'

def get_binary_url(
    filename,
    project=None,
    distro=None,
    release=None,
    arch=None,
    flavor=None,
    branch=None,
    sha1=None
):
    '''
    Return the chacra url for a build result
    '''
    # query shaman for the built binary
    s = {}
    if project:
        s['project'] = project
    if distro:
        s['distros'] = _get_distros(distro, release, arch)
    if flavor:
        s['flavor'] = flavor
    if branch:
        s['ref'] = branch
    if sha1:
        s['sha1'] = sha1

    resp = search(**s)
    result = resp.json()

    if len(result) == 0:
        raise RuntimeError(f'no results found at {resp.url}')

    # TODO: filter the result down to the correct arch etc.?
    result = result[0]

    status = result['status']
    if status != 'ready':
        raise RuntimeError(f'cannot pull file with status: {status}')

    # build the chacra url
    chacra_host = urlparse(result['url']).netloc
    chacra_ref = result['ref']
    chacra_sha1 = result['sha1']
    log.info(f'got chacra host {chacra_host}, ref {chacra_ref}, sha1 {chacra_sha1} from {resp.url}')

    # prefer codename if a release is not specified
    if result.get('distro_codename'):
        release = result.get('distro_codename')
    elif result.get('distro_version'):
        release = result.get('distro_version')
    elif not release:
        raise RuntimeError('cannot determine distro release!')

    if not arch:
        if ARCH in result['archs']:
            arch = ARCH
        elif len(result['archs']) > 0:
            arch = result['archs'][0]
        else:
            raise RuntimeError('cannot determine the arch type!')

    # build the url to the binary
    url = _get_binary_url(
        chacra_host,
        result['project'],
        chacra_ref,
        chacra_sha1,
        result['distro'],
        release,
        arch,
        result['flavor'],
        filename,
    )

    return url

def pull(
    filename,
    project=None,
    distro=None,
    release=None,
    arch=None,
    flavor=None,
    branch=None,
    sha1=None
):
    '''
    Pull a build result from chacra
    '''
    url = get_binary_url(
            filename,
            project=project,
            distro=distro,
            release=release,
            arch=arch,
            flavor=flavor,
            branch=branch,
            sha1=sha1
    )
    resp = requests.get(url, stream=True)
    resp.raise_for_status()
    log.info(f'got file from {resp.url}')

    return resp

def main():
    handler = logging.StreamHandler(sys.stdout)
    log.addHandler(handler)
    log.setLevel(logging.INFO)

    parser = argparse.ArgumentParser()
    parser.add_argument('--project', default=PROJECT)
    parser.add_argument('--distro', default=DISTRO)
    parser.add_argument('--release', default=RELEASE)
    parser.add_argument('--arch', default=ARCH)
    parser.add_argument('--branch', default=BRANCH)
    parser.add_argument('--sha1', default=SHA1)
    parser.add_argument('--flavor', default=FLAVOR)
    parser.add_argument('--src', default=FILENAME)
    parser.add_argument('--dest', default=FILENAME)
    args = parser.parse_args()

    resp = pull(
        args.src,
        project=args.project,
        distro=args.distro,
        release=args.release,
        arch=args.arch,
        flavor=args.flavor,
        branch=args.branch,
        sha1=args.sha1
    )

    dest = Path(args.dest).absolute()
    with open(dest, 'wb') as f:
        for chunk in resp.iter_content(chunk_size=None, decode_unicode=True):
            log.info('.',)
            f.write(chunk)
    log.info(f'wrote binary file: {dest}')

    return 0


if __name__ == '__main__':
   sys.exit(main())