ceph/qa/tasks/kubeadm.py

"""
Kubernetes cluster task, deployed via kubeadm
"""
import argparse
import contextlib
import ipaddress
import json
import logging
import random
import yaml
from io import BytesIO

from teuthology import misc as teuthology
from teuthology import contextutil
from teuthology.config import config as teuth_config
from teuthology.orchestra import run

log = logging.getLogger(__name__)


def _kubectl(ctx, config, args, **kwargs):
    cluster_name = config['cluster']
    ctx.kubeadm[cluster_name].bootstrap_remote.run(
        args=['kubectl'] + args,
        **kwargs,
    )


def kubectl(ctx, config):
    if isinstance(config, str):
        config = [config]
    assert isinstance(config, list)
    for c in config:
        if isinstance(c, str):
            _kubectl(ctx, config, c.split(' '))
        else:
            _kubectl(ctx, config, c)


@contextlib.contextmanager
def preflight(ctx, config):
    run.wait(
        ctx.cluster.run(
            args=[
                'sudo', 'modprobe', 'br_netfilter',
                run.Raw('&&'),
                'sudo', 'sysctl', 'net.bridge.bridge-nf-call-ip6tables=1',
                run.Raw('&&'),
                'sudo', 'sysctl', 'net.bridge.bridge-nf-call-iptables=1',
                run.Raw('&&'),
                'sudo', 'sysctl', 'net.ipv4.ip_forward=1',
                run.Raw('&&'),
                'sudo', 'swapoff', '-a',
            ],
            wait=False,
        )
    )

    # set docker cgroup driver = systemd
    #  see https://kubernetes.io/docs/setup/production-environment/container-runtimes/#docker
    #  see https://github.com/kubernetes/kubeadm/issues/2066
    for remote in ctx.cluster.remotes.keys():
        try:
            orig = remote.read_file('/etc/docker/daemon.json', sudo=True)
            j = json.loads(orig)
        except Exception as e:
            log.info(f'Failed to pull old daemon.json: {e}')
            j = {}
        j["exec-opts"] = ["native.cgroupdriver=systemd"]
        j["log-driver"] = "json-file"
        j["log-opts"] = {"max-size": "100m"}
        j["storage-driver"] = "overlay2"
        remote.write_file('/etc/docker/daemon.json', json.dumps(j), sudo=True)
    run.wait(
        ctx.cluster.run(
            args=[
                'sudo', 'systemctl', 'restart', 'docker',
                run.Raw('||'),
                'true',
            ],
            wait=False,
        )
    )
    yield


@contextlib.contextmanager
def kubeadm_install(ctx, config):
    version = config.get('version', '1.21')

    os_type = teuthology.get_distro(ctx)
    os_version = teuthology.get_distro_version(ctx)

    try:
        if os_type in ['centos', 'rhel']:
            os = f"CentOS_{os_version.split('.')[0]}"
            log.info('Installing cri-o')
            run.wait(
                ctx.cluster.run(
                    args=[
                        'sudo',
                        'curl', '-L', '-o',
                        '/etc/yum.repos.d/devel:kubic:libcontainers:stable.repo',
                        f'https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/{os}/devel:kubic:libcontainers:stable.repo',
                        run.Raw('&&'),
                        'sudo',
                        'curl', '-L', '-o',
                        f'/etc/yum.repos.d/devel:kubic:libcontainers:stable:cri-o:{version}.repo',
                        f'https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/{version}/{os}/devel:kubic:libcontainers:stable:cri-o:{version}.repo',
                        run.Raw('&&'),
                        'sudo', 'dnf', 'install', '-y', 'cri-o',
                    ],
                    wait=False,
                )
            )

            log.info('Installing kube{adm,ctl,let}')
            repo = """[kubernetes]
name=Kubernetes
baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-$basearch
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
"""
            for remote in ctx.cluster.remotes.keys():
                remote.write_file(
                    '/etc/yum.repos.d/kubernetes.repo',
                    repo,
                    sudo=True,
                )
            run.wait(
                ctx.cluster.run(
                    args=[
                        'sudo', 'dnf', 'install', '-y',
                        'kubelet', 'kubeadm', 'kubectl',
                        'iproute-tc', 'bridge-utils',
                    ],
                    wait=False,
                )
            )

            # fix cni config
            for remote in ctx.cluster.remotes.keys():
                conf = """# from https://github.com/cri-o/cri-o/blob/master/tutorials/kubernetes.md#flannel-network
{
    "name": "crio",
    "type": "flannel"
}
"""
                remote.write_file('/etc/cni/net.d/10-crio-flannel.conf', conf, sudo=True)
                remote.run(args=[
                    'sudo', 'rm', '-f',
                    '/etc/cni/net.d/87-podman-bridge.conflist',
                    '/etc/cni/net.d/100-crio-bridge.conf',
                ])

            # start crio
            run.wait(
                ctx.cluster.run(
                    args=[
                        'sudo', 'systemctl', 'daemon-reload',
                        run.Raw('&&'),
                        'sudo', 'systemctl', 'enable', 'crio', '--now',
                    ],
                    wait=False,
                )
            )

        elif os_type == 'ubuntu':
            os = f"xUbuntu_{os_version}"
            log.info('Installing kube{adm,ctl,let}')
            run.wait(
                ctx.cluster.run(
                    args=[
                        'sudo', 'apt', 'update',
                        run.Raw('&&'),
                        'sudo', 'apt', 'install', '-y',
                        'apt-transport-https', 'ca-certificates', 'curl',
                        run.Raw('&&'),
                        'sudo', 'curl', '-fsSLo',
                        '/usr/share/keyrings/kubernetes-archive-keyring.gpg',
                        'https://packages.cloud.google.com/apt/doc/apt-key.gpg',
                        run.Raw('&&'),
                        'echo', 'deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main',
                        run.Raw('|'),
                        'sudo', 'tee', '/etc/apt/sources.list.d/kubernetes.list',
                        run.Raw('&&'),
                        'sudo', 'apt', 'update',
                        run.Raw('&&'),
                        'sudo', 'apt', 'install', '-y',
                        'kubelet', 'kubeadm', 'kubectl',
                        'bridge-utils',
                    ],
                    wait=False,
                )
            )

        else:
            raise RuntimeError(f'unsupported distro {os_type} for cri-o')

        run.wait(
            ctx.cluster.run(
                args=[
                    'sudo', 'systemctl', 'enable', '--now', 'kubelet',
                    run.Raw('&&'),
                    'sudo', 'kubeadm', 'config', 'images', 'pull',
                ],
                wait=False,
            )
        )

        yield

    finally:
        if config.get('uninstall', True):
            log.info('Uninstalling kube{adm,let,ctl}')
            if os_type in ['centos', 'rhel']:
                run.wait(
                    ctx.cluster.run(
                        args=[
                            'sudo', 'rm', '-f',
                            '/etc/yum.repos.d/kubernetes.repo',
                            run.Raw('&&'),
                            'sudo', 'dnf', 'remove', '-y',
                            'kubeadm', 'kubelet', 'kubectl', 'cri-o',
                        ],
                        wait=False
                    )
                )
            elif os_type == 'ubuntu' and False:
                run.wait(
                    ctx.cluster.run(
                        args=[
                            'sudo', 'rm', '-f',
                            '/etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list',
                            f'/etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:{version}.list',
                            '/etc/apt/trusted.gpg.d/libcontainers-cri-o.gpg',
                            run.Raw('&&'),
                            'sudo', 'apt', 'remove', '-y',
                            'kkubeadm', 'kubelet', 'kubectl', 'cri-o', 'cri-o-runc',
                        ],
                        wait=False,
                    )
                )


@contextlib.contextmanager
def kubeadm_init_join(ctx, config):
    cluster_name = config['cluster']

    bootstrap_remote = None
    remotes = {}      # remote -> ip
    for remote, roles in ctx.cluster.remotes.items():
        for role in roles:
            if role.startswith('host.'):
                if not bootstrap_remote:
                    bootstrap_remote = remote
                if remote not in remotes:
                    remotes[remote] = remote.ssh.get_transport().getpeername()[0]
    if not bootstrap_remote:
        raise RuntimeError('must define at least one host.something role')
    ctx.kubeadm[cluster_name].bootstrap_remote = bootstrap_remote
    ctx.kubeadm[cluster_name].remotes = remotes
    ctx.kubeadm[cluster_name].token = 'abcdef.' + ''.join([
        random.choice('0123456789abcdefghijklmnopqrstuvwxyz') for _ in range(16)
    ])
    log.info(f'Token: {ctx.kubeadm[cluster_name].token}')
    log.info(f'Remotes: {ctx.kubeadm[cluster_name].remotes}')

    try:
        # init
        cmd = [
            'sudo', 'kubeadm', 'init',
            '--node-name', ctx.kubeadm[cluster_name].bootstrap_remote.shortname,
            '--token', ctx.kubeadm[cluster_name].token,
            '--pod-network-cidr', str(ctx.kubeadm[cluster_name].pod_subnet),
        ]
        bootstrap_remote.run(args=cmd)

        # join additional nodes
        joins = []
        for remote, ip in ctx.kubeadm[cluster_name].remotes.items():
            if remote == bootstrap_remote:
                continue
            cmd = [
                'sudo', 'kubeadm', 'join',
                ctx.kubeadm[cluster_name].remotes[ctx.kubeadm[cluster_name].bootstrap_remote] + ':6443',
                '--node-name', remote.shortname,
                '--token', ctx.kubeadm[cluster_name].token,
                '--discovery-token-unsafe-skip-ca-verification',
            ]
            joins.append(remote.run(args=cmd, wait=False))
        run.wait(joins)
        yield

    except Exception as e:
        log.exception(e)
        raise

    finally:
        log.info('Cleaning up node')
        run.wait(
            ctx.cluster.run(
                args=['sudo', 'kubeadm', 'reset', 'cleanup-node', '-f'],
                wait=False,
            )
        )


@contextlib.contextmanager
def kubectl_config(ctx, config):
    cluster_name = config['cluster']
    bootstrap_remote = ctx.kubeadm[cluster_name].bootstrap_remote

    ctx.kubeadm[cluster_name].admin_conf = \
        bootstrap_remote.read_file('/etc/kubernetes/admin.conf', sudo=True)

    log.info('Setting up kubectl')
    try:
        ctx.cluster.run(args=[
            'mkdir', '-p', '.kube',
            run.Raw('&&'),
            'sudo', 'mkdir', '-p', '/root/.kube',
        ])
        for remote in ctx.kubeadm[cluster_name].remotes.keys():
            remote.write_file('.kube/config', ctx.kubeadm[cluster_name].admin_conf)
            remote.sudo_write_file('/root/.kube/config',
                                   ctx.kubeadm[cluster_name].admin_conf)
        yield

    except Exception as e:
        log.exception(e)
        raise

    finally:
        log.info('Deconfiguring kubectl')
        ctx.cluster.run(args=[
            'rm', '-rf', '.kube',
            run.Raw('&&'),
            'sudo', 'rm', '-rf', '/root/.kube',
        ])


def map_vnet(mip):
    for mapping in teuth_config.get('vnet', []):
        mnet = ipaddress.ip_network(mapping['machine_subnet'])
        vnet = ipaddress.ip_network(mapping['virtual_subnet'])
        if vnet.prefixlen >= mnet.prefixlen:
            log.error(f"virtual_subnet {vnet} prefix >= machine_subnet {mnet} prefix")
            return None
        if mip in mnet:
            pos = list(mnet.hosts()).index(mip)
            log.info(f"{mip} is in {mnet} at pos {pos}")
            sub = list(vnet.subnets(32 - mnet.prefixlen))[pos]
            return sub
    return None


@contextlib.contextmanager
def allocate_pod_subnet(ctx, config):
    """
    Allocate a private subnet that will not collide with other test machines/clusters
    """
    cluster_name = config['cluster']
    assert cluster_name == 'kubeadm', 'multiple subnets not yet implemented'

    log.info('Identifying pod subnet')
    remote = list(ctx.cluster.remotes.keys())[0]
    ip = remote.ssh.get_transport().getpeername()[0]
    mip = ipaddress.ip_address(ip)
    vnet = map_vnet(mip)
    assert vnet
    log.info(f'Pod subnet: {vnet}')
    ctx.kubeadm[cluster_name].pod_subnet = vnet
    yield


@contextlib.contextmanager
def pod_network(ctx, config):
    cluster_name = config['cluster']
    pnet = config.get('pod_network', 'calico')
    if pnet == 'flannel':
        r = ctx.kubeadm[cluster_name].bootstrap_remote.run(
            args=[
                'curl',
                'https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml',
            ],
            stdout=BytesIO(),
        )
        assert r.exitstatus == 0
        flannel = list(yaml.load_all(r.stdout.getvalue(), Loader=yaml.FullLoader))
        for o in flannel:
            if o.get('data', {}).get('net-conf.json'):
                log.info(f'Updating {o}')
                o['data']['net-conf.json'] = o['data']['net-conf.json'].replace(
                    '10.244.0.0/16',
                    str(ctx.kubeadm[cluster_name].pod_subnet)
                )
                log.info(f'Now {o}')
        flannel_yaml = yaml.dump_all(flannel)
        log.debug(f'Flannel:\n{flannel_yaml}')
        _kubectl(ctx, config, ['apply', '-f', '-'], stdin=flannel_yaml)

    elif pnet == 'calico':
        _kubectl(ctx, config, [
            'create', '-f',
            'https://docs.projectcalico.org/manifests/tigera-operator.yaml'
        ])
        cr = {
            'apiVersion': 'operator.tigera.io/v1',
            'kind': 'Installation',
            'metadata': {'name': 'default'},
            'spec': {
                'calicoNetwork': {
                    'ipPools': [
                        {
                            'blockSize': 26,
                            'cidr': str(ctx.kubeadm[cluster_name].pod_subnet),
                            'encapsulation': 'IPIPCrossSubnet',
                            'natOutgoing': 'Enabled',
                            'nodeSelector': 'all()',
                        }
                    ]
                }
            }
        }
        _kubectl(ctx, config, ['create', '-f', '-'], stdin=yaml.dump(cr))

    else:
        raise RuntimeError(f'unrecognized pod_network {pnet}')

    try:
        yield

    finally:
        if pnet == 'flannel':
            _kubectl(ctx, config, [
                'delete', '-f',
                'https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml',
            ])

        elif pnet == 'calico':
            _kubectl(ctx, config, ['delete', 'installation', 'default'])
            _kubectl(ctx, config, [
                'delete', '-f',
                'https://docs.projectcalico.org/manifests/tigera-operator.yaml'
            ])


@contextlib.contextmanager
def setup_pvs(ctx, config):
    """
    Create PVs for all scratch LVs and set up a trivial provisioner
    """
    log.info('Scanning for scratch devices')
    crs = []
    for remote in ctx.cluster.remotes.keys():
        ls = remote.read_file('/scratch_devs').decode('utf-8').strip().splitlines()
        log.info(f'Scratch devices on {remote.shortname}: {ls}')
        for dev in ls:
            devname = dev.split('/')[-1].replace("_", "-")
            crs.append({
                'apiVersion': 'v1',
                'kind': 'PersistentVolume',
                'metadata': {'name': f'{remote.shortname}-{devname}'},
                'spec': {
                    'volumeMode': 'Block',
                    'accessModes': ['ReadWriteOnce'],
                    'capacity': {'storage': '100Gi'},  # doesn't matter?
                    'persistentVolumeReclaimPolicy': 'Retain',
                    'storageClassName': 'scratch',
                    'local': {'path': dev},
                    'nodeAffinity': {
                        'required': {
                            'nodeSelectorTerms': [
                                {
                                    'matchExpressions': [
                                        {
                                            'key': 'kubernetes.io/hostname',
                                            'operator': 'In',
                                            'values': [remote.shortname]
                                        }
                                    ]
                                }
                            ]
                        }
                    }
                }
            })
            # overwriting first few MB is enough to make k8s happy
            remote.run(args=[
                'sudo', 'dd', 'if=/dev/zero', f'of={dev}', 'bs=1M', 'count=10'
            ])
    crs.append({
        'kind': 'StorageClass',
        'apiVersion': 'storage.k8s.io/v1',
        'metadata': {'name': 'scratch'},
        'provisioner': 'kubernetes.io/no-provisioner',
        'volumeBindingMode': 'WaitForFirstConsumer',
    })
    y = yaml.dump_all(crs)
    log.info('Creating PVs + StorageClass')
    log.debug(y)
    _kubectl(ctx, config, ['create', '-f', '-'], stdin=y)

    yield


@contextlib.contextmanager
def final(ctx, config):
    cluster_name = config['cluster']

    # remove master node taint
    _kubectl(ctx, config, [
        'taint', 'node',
        ctx.kubeadm[cluster_name].bootstrap_remote.shortname,
        'node-role.kubernetes.io/master-',
        run.Raw('||'),
        'true',
    ])

    yield


@contextlib.contextmanager
def task(ctx, config):
    if not config:
        config = {}
    assert isinstance(config, dict), \
        "task only supports a dictionary for configuration"

    log.info('Kubeadm start')

    overrides = ctx.config.get('overrides', {})
    teuthology.deep_merge(config, overrides.get('kubeadm', {}))
    log.info('Config: ' + str(config))

    # set up cluster context
    if not hasattr(ctx, 'kubeadm'):
        ctx.kubeadm = {}
    if 'cluster' not in config:
        config['cluster'] = 'kubeadm'
    cluster_name = config['cluster']
    if cluster_name not in ctx.kubeadm:
        ctx.kubeadm[cluster_name] = argparse.Namespace()

    with contextutil.nested(
            lambda: preflight(ctx, config),
            lambda: allocate_pod_subnet(ctx, config),
            lambda: kubeadm_install(ctx, config),
            lambda: kubeadm_init_join(ctx, config),
            lambda: kubectl_config(ctx, config),
            lambda: pod_network(ctx, config),
            lambda: setup_pvs(ctx, config),
            lambda: final(ctx, config),
    ):
        try:
            log.info('Kubeadm complete, yielding')
            yield

        finally:
            log.info('Tearing down kubeadm')