mirror of
https://github.com/ceph/ceph
synced 2024-12-29 15:03:33 +00:00
280d735847
This commit adds testing for the drive_group_loop in the Rook orchestrator that reapplies drive groups that were applied previously. This test removes an OSD, zaps the underlying device then waits for the OSD to be re-created by the drive_group_loop. This commit also updates the rook test suite to test v1.7.2 instead of 1.7.0 since `orch device zap` is only supported from v1.7.2 onwards. Fixes: https://tracker.ceph.com/issues/53501 Signed-off-by: Joseph Sawaya <jsawaya@redhat.com>
564 lines
19 KiB
Python
564 lines
19 KiB
Python
"""
|
|
Kubernetes cluster task, deployed via kubeadm
|
|
"""
|
|
import argparse
|
|
import contextlib
|
|
import ipaddress
|
|
import json
|
|
import logging
|
|
import random
|
|
import yaml
|
|
from io import BytesIO
|
|
|
|
from teuthology import misc as teuthology
|
|
from teuthology import contextutil
|
|
from teuthology.config import config as teuth_config
|
|
from teuthology.orchestra import run
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
def _kubectl(ctx, config, args, **kwargs):
|
|
cluster_name = config['cluster']
|
|
ctx.kubeadm[cluster_name].bootstrap_remote.run(
|
|
args=['kubectl'] + args,
|
|
**kwargs,
|
|
)
|
|
|
|
|
|
def kubectl(ctx, config):
|
|
if isinstance(config, str):
|
|
config = [config]
|
|
assert isinstance(config, list)
|
|
for c in config:
|
|
if isinstance(c, str):
|
|
_kubectl(ctx, config, c.split(' '))
|
|
else:
|
|
_kubectl(ctx, config, c)
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def preflight(ctx, config):
|
|
run.wait(
|
|
ctx.cluster.run(
|
|
args=[
|
|
'sudo', 'modprobe', 'br_netfilter',
|
|
run.Raw('&&'),
|
|
'sudo', 'sysctl', 'net.bridge.bridge-nf-call-ip6tables=1',
|
|
run.Raw('&&'),
|
|
'sudo', 'sysctl', 'net.bridge.bridge-nf-call-iptables=1',
|
|
run.Raw('&&'),
|
|
'sudo', 'sysctl', 'net.ipv4.ip_forward=1',
|
|
run.Raw('&&'),
|
|
'sudo', 'swapoff', '-a',
|
|
],
|
|
wait=False,
|
|
)
|
|
)
|
|
|
|
# set docker cgroup driver = systemd
|
|
# see https://kubernetes.io/docs/setup/production-environment/container-runtimes/#docker
|
|
# see https://github.com/kubernetes/kubeadm/issues/2066
|
|
for remote in ctx.cluster.remotes.keys():
|
|
try:
|
|
orig = remote.read_file('/etc/docker/daemon.json', sudo=True)
|
|
j = json.loads(orig)
|
|
except Exception as e:
|
|
log.info(f'Failed to pull old daemon.json: {e}')
|
|
j = {}
|
|
j["exec-opts"] = ["native.cgroupdriver=systemd"]
|
|
j["log-driver"] = "json-file"
|
|
j["log-opts"] = {"max-size": "100m"}
|
|
j["storage-driver"] = "overlay2"
|
|
remote.write_file('/etc/docker/daemon.json', json.dumps(j), sudo=True)
|
|
run.wait(
|
|
ctx.cluster.run(
|
|
args=[
|
|
'sudo', 'systemctl', 'restart', 'docker',
|
|
run.Raw('||'),
|
|
'true',
|
|
],
|
|
wait=False,
|
|
)
|
|
)
|
|
yield
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def kubeadm_install(ctx, config):
|
|
version = config.get('version', '1.21')
|
|
|
|
os_type = teuthology.get_distro(ctx)
|
|
os_version = teuthology.get_distro_version(ctx)
|
|
|
|
try:
|
|
if os_type in ['centos', 'rhel']:
|
|
os = f"CentOS_{os_version.split('.')[0]}"
|
|
log.info('Installing cri-o')
|
|
run.wait(
|
|
ctx.cluster.run(
|
|
args=[
|
|
'sudo',
|
|
'curl', '-L', '-o',
|
|
'/etc/yum.repos.d/devel:kubic:libcontainers:stable.repo',
|
|
f'https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/{os}/devel:kubic:libcontainers:stable.repo',
|
|
run.Raw('&&'),
|
|
'sudo',
|
|
'curl', '-L', '-o',
|
|
f'/etc/yum.repos.d/devel:kubic:libcontainers:stable:cri-o:{version}.repo',
|
|
f'https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/{version}/{os}/devel:kubic:libcontainers:stable:cri-o:{version}.repo',
|
|
run.Raw('&&'),
|
|
'sudo', 'dnf', 'install', '-y', 'cri-o',
|
|
],
|
|
wait=False,
|
|
)
|
|
)
|
|
|
|
log.info('Installing kube{adm,ctl,let}')
|
|
repo = """[kubernetes]
|
|
name=Kubernetes
|
|
baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-$basearch
|
|
enabled=1
|
|
gpgcheck=1
|
|
repo_gpgcheck=1
|
|
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
|
|
"""
|
|
for remote in ctx.cluster.remotes.keys():
|
|
remote.write_file(
|
|
'/etc/yum.repos.d/kubernetes.repo',
|
|
repo,
|
|
sudo=True,
|
|
)
|
|
run.wait(
|
|
ctx.cluster.run(
|
|
args=[
|
|
'sudo', 'dnf', 'install', '-y',
|
|
'kubelet', 'kubeadm', 'kubectl',
|
|
'iproute-tc', 'bridge-utils',
|
|
],
|
|
wait=False,
|
|
)
|
|
)
|
|
|
|
# fix cni config
|
|
for remote in ctx.cluster.remotes.keys():
|
|
conf = """# from https://github.com/cri-o/cri-o/blob/master/tutorials/kubernetes.md#flannel-network
|
|
{
|
|
"name": "crio",
|
|
"type": "flannel"
|
|
}
|
|
"""
|
|
remote.write_file('/etc/cni/net.d/10-crio-flannel.conf', conf, sudo=True)
|
|
remote.run(args=[
|
|
'sudo', 'rm', '-f',
|
|
'/etc/cni/net.d/87-podman-bridge.conflist',
|
|
'/etc/cni/net.d/100-crio-bridge.conf',
|
|
])
|
|
|
|
# start crio
|
|
run.wait(
|
|
ctx.cluster.run(
|
|
args=[
|
|
'sudo', 'systemctl', 'daemon-reload',
|
|
run.Raw('&&'),
|
|
'sudo', 'systemctl', 'enable', 'crio', '--now',
|
|
],
|
|
wait=False,
|
|
)
|
|
)
|
|
|
|
elif os_type == 'ubuntu':
|
|
os = f"xUbuntu_{os_version}"
|
|
log.info('Installing kube{adm,ctl,let}')
|
|
run.wait(
|
|
ctx.cluster.run(
|
|
args=[
|
|
'sudo', 'apt', 'update',
|
|
run.Raw('&&'),
|
|
'sudo', 'apt', 'install', '-y',
|
|
'apt-transport-https', 'ca-certificates', 'curl',
|
|
run.Raw('&&'),
|
|
'sudo', 'curl', '-fsSLo',
|
|
'/usr/share/keyrings/kubernetes-archive-keyring.gpg',
|
|
'https://packages.cloud.google.com/apt/doc/apt-key.gpg',
|
|
run.Raw('&&'),
|
|
'echo', 'deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main',
|
|
run.Raw('|'),
|
|
'sudo', 'tee', '/etc/apt/sources.list.d/kubernetes.list',
|
|
run.Raw('&&'),
|
|
'sudo', 'apt', 'update',
|
|
run.Raw('&&'),
|
|
'sudo', 'apt', 'install', '-y',
|
|
'kubelet', 'kubeadm', 'kubectl',
|
|
'bridge-utils',
|
|
],
|
|
wait=False,
|
|
)
|
|
)
|
|
|
|
else:
|
|
raise RuntimeError(f'unsupported distro {os_type} for cri-o')
|
|
|
|
run.wait(
|
|
ctx.cluster.run(
|
|
args=[
|
|
'sudo', 'systemctl', 'enable', '--now', 'kubelet',
|
|
run.Raw('&&'),
|
|
'sudo', 'kubeadm', 'config', 'images', 'pull',
|
|
],
|
|
wait=False,
|
|
)
|
|
)
|
|
|
|
yield
|
|
|
|
finally:
|
|
if config.get('uninstall', True):
|
|
log.info('Uninstalling kube{adm,let,ctl}')
|
|
if os_type in ['centos', 'rhel']:
|
|
run.wait(
|
|
ctx.cluster.run(
|
|
args=[
|
|
'sudo', 'rm', '-f',
|
|
'/etc/yum.repos.d/kubernetes.repo',
|
|
run.Raw('&&'),
|
|
'sudo', 'dnf', 'remove', '-y',
|
|
'kubeadm', 'kubelet', 'kubectl', 'cri-o',
|
|
],
|
|
wait=False
|
|
)
|
|
)
|
|
elif os_type == 'ubuntu' and False:
|
|
run.wait(
|
|
ctx.cluster.run(
|
|
args=[
|
|
'sudo', 'rm', '-f',
|
|
'/etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list',
|
|
f'/etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:{version}.list',
|
|
'/etc/apt/trusted.gpg.d/libcontainers-cri-o.gpg',
|
|
run.Raw('&&'),
|
|
'sudo', 'apt', 'remove', '-y',
|
|
'kkubeadm', 'kubelet', 'kubectl', 'cri-o', 'cri-o-runc',
|
|
],
|
|
wait=False,
|
|
)
|
|
)
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def kubeadm_init_join(ctx, config):
|
|
cluster_name = config['cluster']
|
|
|
|
bootstrap_remote = None
|
|
remotes = {} # remote -> ip
|
|
for remote, roles in ctx.cluster.remotes.items():
|
|
for role in roles:
|
|
if role.startswith('host.'):
|
|
if not bootstrap_remote:
|
|
bootstrap_remote = remote
|
|
if remote not in remotes:
|
|
remotes[remote] = remote.ssh.get_transport().getpeername()[0]
|
|
if not bootstrap_remote:
|
|
raise RuntimeError('must define at least one host.something role')
|
|
ctx.kubeadm[cluster_name].bootstrap_remote = bootstrap_remote
|
|
ctx.kubeadm[cluster_name].remotes = remotes
|
|
ctx.kubeadm[cluster_name].token = 'abcdef.' + ''.join([
|
|
random.choice('0123456789abcdefghijklmnopqrstuvwxyz') for _ in range(16)
|
|
])
|
|
log.info(f'Token: {ctx.kubeadm[cluster_name].token}')
|
|
log.info(f'Remotes: {ctx.kubeadm[cluster_name].remotes}')
|
|
|
|
try:
|
|
# init
|
|
cmd = [
|
|
'sudo', 'kubeadm', 'init',
|
|
'--node-name', ctx.kubeadm[cluster_name].bootstrap_remote.shortname,
|
|
'--token', ctx.kubeadm[cluster_name].token,
|
|
'--pod-network-cidr', str(ctx.kubeadm[cluster_name].pod_subnet),
|
|
]
|
|
bootstrap_remote.run(args=cmd)
|
|
|
|
# join additional nodes
|
|
joins = []
|
|
for remote, ip in ctx.kubeadm[cluster_name].remotes.items():
|
|
if remote == bootstrap_remote:
|
|
continue
|
|
cmd = [
|
|
'sudo', 'kubeadm', 'join',
|
|
ctx.kubeadm[cluster_name].remotes[ctx.kubeadm[cluster_name].bootstrap_remote] + ':6443',
|
|
'--node-name', remote.shortname,
|
|
'--token', ctx.kubeadm[cluster_name].token,
|
|
'--discovery-token-unsafe-skip-ca-verification',
|
|
]
|
|
joins.append(remote.run(args=cmd, wait=False))
|
|
run.wait(joins)
|
|
yield
|
|
|
|
except Exception as e:
|
|
log.exception(e)
|
|
raise
|
|
|
|
finally:
|
|
log.info('Cleaning up node')
|
|
run.wait(
|
|
ctx.cluster.run(
|
|
args=['sudo', 'kubeadm', 'reset', 'cleanup-node', '-f'],
|
|
wait=False,
|
|
)
|
|
)
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def kubectl_config(ctx, config):
|
|
cluster_name = config['cluster']
|
|
bootstrap_remote = ctx.kubeadm[cluster_name].bootstrap_remote
|
|
|
|
ctx.kubeadm[cluster_name].admin_conf = \
|
|
bootstrap_remote.read_file('/etc/kubernetes/admin.conf', sudo=True)
|
|
|
|
log.info('Setting up kubectl')
|
|
try:
|
|
ctx.cluster.run(args=[
|
|
'mkdir', '-p', '.kube',
|
|
run.Raw('&&'),
|
|
'sudo', 'mkdir', '-p', '/root/.kube',
|
|
])
|
|
for remote in ctx.kubeadm[cluster_name].remotes.keys():
|
|
remote.write_file('.kube/config', ctx.kubeadm[cluster_name].admin_conf)
|
|
remote.sudo_write_file('/root/.kube/config',
|
|
ctx.kubeadm[cluster_name].admin_conf)
|
|
yield
|
|
|
|
except Exception as e:
|
|
log.exception(e)
|
|
raise
|
|
|
|
finally:
|
|
log.info('Deconfiguring kubectl')
|
|
ctx.cluster.run(args=[
|
|
'rm', '-rf', '.kube',
|
|
run.Raw('&&'),
|
|
'sudo', 'rm', '-rf', '/root/.kube',
|
|
])
|
|
|
|
|
|
def map_vnet(mip):
|
|
for mapping in teuth_config.get('vnet', []):
|
|
mnet = ipaddress.ip_network(mapping['machine_subnet'])
|
|
vnet = ipaddress.ip_network(mapping['virtual_subnet'])
|
|
if vnet.prefixlen >= mnet.prefixlen:
|
|
log.error(f"virtual_subnet {vnet} prefix >= machine_subnet {mnet} prefix")
|
|
return None
|
|
if mip in mnet:
|
|
pos = list(mnet.hosts()).index(mip)
|
|
log.info(f"{mip} is in {mnet} at pos {pos}")
|
|
sub = list(vnet.subnets(32 - mnet.prefixlen))[pos]
|
|
return sub
|
|
return None
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def allocate_pod_subnet(ctx, config):
|
|
"""
|
|
Allocate a private subnet that will not collide with other test machines/clusters
|
|
"""
|
|
cluster_name = config['cluster']
|
|
assert cluster_name == 'kubeadm', 'multiple subnets not yet implemented'
|
|
|
|
log.info('Identifying pod subnet')
|
|
remote = list(ctx.cluster.remotes.keys())[0]
|
|
ip = remote.ssh.get_transport().getpeername()[0]
|
|
mip = ipaddress.ip_address(ip)
|
|
vnet = map_vnet(mip)
|
|
assert vnet
|
|
log.info(f'Pod subnet: {vnet}')
|
|
ctx.kubeadm[cluster_name].pod_subnet = vnet
|
|
yield
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def pod_network(ctx, config):
|
|
cluster_name = config['cluster']
|
|
pnet = config.get('pod_network', 'calico')
|
|
if pnet == 'flannel':
|
|
r = ctx.kubeadm[cluster_name].bootstrap_remote.run(
|
|
args=[
|
|
'curl',
|
|
'https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml',
|
|
],
|
|
stdout=BytesIO(),
|
|
)
|
|
assert r.exitstatus == 0
|
|
flannel = list(yaml.load_all(r.stdout.getvalue(), Loader=yaml.FullLoader))
|
|
for o in flannel:
|
|
if o.get('data', {}).get('net-conf.json'):
|
|
log.info(f'Updating {o}')
|
|
o['data']['net-conf.json'] = o['data']['net-conf.json'].replace(
|
|
'10.244.0.0/16',
|
|
str(ctx.kubeadm[cluster_name].pod_subnet)
|
|
)
|
|
log.info(f'Now {o}')
|
|
flannel_yaml = yaml.dump_all(flannel)
|
|
log.debug(f'Flannel:\n{flannel_yaml}')
|
|
_kubectl(ctx, config, ['apply', '-f', '-'], stdin=flannel_yaml)
|
|
|
|
elif pnet == 'calico':
|
|
_kubectl(ctx, config, [
|
|
'apply', '-f',
|
|
'https://docs.projectcalico.org/manifests/tigera-operator.yaml'
|
|
])
|
|
cr = {
|
|
'apiVersion': 'operator.tigera.io/v1',
|
|
'kind': 'Installation',
|
|
'metadata': {'name': 'default'},
|
|
'spec': {
|
|
'calicoNetwork': {
|
|
'ipPools': [
|
|
{
|
|
'blockSize': 26,
|
|
'cidr': str(ctx.kubeadm[cluster_name].pod_subnet),
|
|
'encapsulation': 'IPIPCrossSubnet',
|
|
'natOutgoing': 'Enabled',
|
|
'nodeSelector': 'all()',
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
_kubectl(ctx, config, ['create', '-f', '-'], stdin=yaml.dump(cr))
|
|
|
|
else:
|
|
raise RuntimeError(f'unrecognized pod_network {pnet}')
|
|
|
|
try:
|
|
yield
|
|
|
|
finally:
|
|
if pnet == 'flannel':
|
|
_kubectl(ctx, config, [
|
|
'delete', '-f',
|
|
'https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml',
|
|
])
|
|
|
|
elif pnet == 'calico':
|
|
_kubectl(ctx, config, ['delete', 'installation', 'default'])
|
|
_kubectl(ctx, config, [
|
|
'delete', '-f',
|
|
'https://docs.projectcalico.org/manifests/tigera-operator.yaml'
|
|
])
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def setup_pvs(ctx, config):
|
|
"""
|
|
Create PVs for all scratch LVs and set up a trivial provisioner
|
|
"""
|
|
log.info('Scanning for scratch devices')
|
|
crs = []
|
|
for remote in ctx.cluster.remotes.keys():
|
|
ls = remote.read_file('/scratch_devs').decode('utf-8').strip().splitlines()
|
|
log.info(f'Scratch devices on {remote.shortname}: {ls}')
|
|
for dev in ls:
|
|
devname = dev.split('/')[-1].replace("_", "-")
|
|
crs.append({
|
|
'apiVersion': 'v1',
|
|
'kind': 'PersistentVolume',
|
|
'metadata': {'name': f'{remote.shortname}-{devname}'},
|
|
'spec': {
|
|
'volumeMode': 'Block',
|
|
'accessModes': ['ReadWriteOnce'],
|
|
'capacity': {'storage': '100Gi'}, # doesn't matter?
|
|
'persistentVolumeReclaimPolicy': 'Retain',
|
|
'storageClassName': 'scratch',
|
|
'local': {'path': dev},
|
|
'nodeAffinity': {
|
|
'required': {
|
|
'nodeSelectorTerms': [
|
|
{
|
|
'matchExpressions': [
|
|
{
|
|
'key': 'kubernetes.io/hostname',
|
|
'operator': 'In',
|
|
'values': [remote.shortname]
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
})
|
|
# overwriting first few MB is enough to make k8s happy
|
|
remote.run(args=[
|
|
'sudo', 'dd', 'if=/dev/zero', f'of={dev}', 'bs=1M', 'count=10'
|
|
])
|
|
crs.append({
|
|
'kind': 'StorageClass',
|
|
'apiVersion': 'storage.k8s.io/v1',
|
|
'metadata': {'name': 'scratch'},
|
|
'provisioner': 'kubernetes.io/no-provisioner',
|
|
'volumeBindingMode': 'WaitForFirstConsumer',
|
|
})
|
|
y = yaml.dump_all(crs)
|
|
log.info('Creating PVs + StorageClass')
|
|
log.debug(y)
|
|
_kubectl(ctx, config, ['create', '-f', '-'], stdin=y)
|
|
|
|
yield
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def final(ctx, config):
|
|
cluster_name = config['cluster']
|
|
|
|
# remove master node taint
|
|
_kubectl(ctx, config, [
|
|
'taint', 'node',
|
|
ctx.kubeadm[cluster_name].bootstrap_remote.shortname,
|
|
'node-role.kubernetes.io/master-',
|
|
run.Raw('||'),
|
|
'true',
|
|
])
|
|
|
|
yield
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def task(ctx, config):
|
|
if not config:
|
|
config = {}
|
|
assert isinstance(config, dict), \
|
|
"task only supports a dictionary for configuration"
|
|
|
|
log.info('Kubeadm start')
|
|
|
|
overrides = ctx.config.get('overrides', {})
|
|
teuthology.deep_merge(config, overrides.get('kubeadm', {}))
|
|
log.info('Config: ' + str(config))
|
|
|
|
# set up cluster context
|
|
if not hasattr(ctx, 'kubeadm'):
|
|
ctx.kubeadm = {}
|
|
if 'cluster' not in config:
|
|
config['cluster'] = 'kubeadm'
|
|
cluster_name = config['cluster']
|
|
if cluster_name not in ctx.kubeadm:
|
|
ctx.kubeadm[cluster_name] = argparse.Namespace()
|
|
|
|
with contextutil.nested(
|
|
lambda: preflight(ctx, config),
|
|
lambda: allocate_pod_subnet(ctx, config),
|
|
lambda: kubeadm_install(ctx, config),
|
|
lambda: kubeadm_init_join(ctx, config),
|
|
lambda: kubectl_config(ctx, config),
|
|
lambda: pod_network(ctx, config),
|
|
lambda: setup_pvs(ctx, config),
|
|
lambda: final(ctx, config),
|
|
):
|
|
try:
|
|
log.info('Kubeadm complete, yielding')
|
|
yield
|
|
|
|
finally:
|
|
log.info('Tearing down kubeadm')
|