Merge pull request #1008 from ceph/wip-systemd-tests

Wip systemd tests

suites/smoke/systemd/distro/centos_7.2.yaml

@@ -0,0 +1 @@
+../../../../distros/supported/centos_7.2.yaml

suites/smoke/systemd/distro/ubuntu_16.04.yaml

@@ -0,0 +1,2 @@
+os_type: ubuntu
+os_version: "16.04"

suites/smoke/systemd/tasks/systemd.yaml

@@ -0,0 +1,13 @@
+roles:
+- [mon.a, osd.0]
+- [osd.1, osd.2]
+- [mds.a, osd.3]
+- [mon.b, client.0]
+tasks:
+- ssh-keys:
+- ceph-deploy:
+- systemd:
+- workunit:
+    clients:
+      all:
+      - rados/load-gen-mix.sh

tasks/systemd.py

@@ -0,0 +1,113 @@
+"""
+Systemd test
+"""
+import contextlib
+import logging
+import re
+import time
+
+from cStringIO import StringIO
+from teuthology.orchestra import run
+
+log = logging.getLogger(__name__)
+
+
+@contextlib.contextmanager
+def task(ctx, config):
+    """
+      - tasks:
+          ceph-deploy:
+          systemd:
+
+    Test ceph systemd services can start, stop and restart and
+    check for any failed services and report back errors
+    """
+    for remote, roles in ctx.cluster.remotes.iteritems():
+        remote.run(args=['sudo', 'ps', '-eaf', run.Raw('|'),
+                         'grep', 'ceph'])
+        r = remote.run(args=['sudo', 'systemctl', 'list-units', run.Raw('|'),
+                             'grep', 'ceph'], stdout=StringIO(),
+                       check_status=False)
+        log.info(r.stdout.getvalue())
+        if r.stdout.getvalue().find('failed'):
+            log.info("Ceph services in failed state")
+
+        # test overall service stop and start using ceph.target
+        # ceph.target tests are meant for ceph systemd tests
+        # and not actual process testing using 'ps'
+        log.info("Stopping all Ceph services")
+        remote.run(args=['sudo', 'systemctl', 'stop', 'ceph.target'])
+        r = remote.run(args=['sudo', 'systemctl', 'status', 'ceph.target'],
+                       stdout=StringIO(), check_status=False)
+        log.info(r.stdout.getvalue())
+        log.info("Checking process status")
+        r = remote.run(args=['sudo', 'ps', '-eaf', run.Raw('|'),
+                             'grep', 'ceph'], stdout=StringIO())
+        if r.stdout.getvalue().find('Active: inactive'):
+            log.info("Sucessfully stopped all ceph services")
+        else:
+            log.info("Failed to stop ceph services")
+
+        log.info("Starting all Ceph services")
+        remote.run(args=['sudo', 'systemctl', 'start', 'ceph.target'])
+        r = remote.run(args=['sudo', 'systemctl', 'status', 'ceph.target'],
+                       stdout=StringIO())
+        log.info(r.stdout.getvalue())
+        if r.stdout.getvalue().find('Active: active'):
+            log.info("Sucessfully started all Ceph services")
+        else:
+            log.info("info", "Failed to start Ceph services")
+        r = remote.run(args=['sudo', 'ps', '-eaf', run.Raw('|'),
+                             'grep', 'ceph'], stdout=StringIO())
+        log.info(r.stdout.getvalue())
+        time.sleep(4)
+
+        # test individual services start stop
+        name = remote.shortname
+        mon_name = 'ceph-mon@' + name + '.service'
+        mds_name = 'ceph-mds@' + name + '.service'
+        mon_role_name = 'mon.' + name
+        mds_role_name = 'mds.' + name
+        m_osd = re.search('--id (\d+) --setuser ceph', r.stdout.getvalue())
+        if m_osd:
+            osd_service = 'ceph-osd@{m}.service'.format(m=m_osd.group(1))
+            remote.run(args=['sudo', 'systemctl', 'status',
+                             osd_service])
+            remote.run(args=['sudo', 'systemctl', 'stop',
+                             osd_service])
+            time.sleep(4)  # immediate check will result in deactivating state
+            r = remote.run(args=['sudo', 'systemctl', 'status', osd_service],
+                           stdout=StringIO(), check_status=False)
+            log.info(r.stdout.getvalue())
+            if r.stdout.getvalue().find('Active: inactive'):
+                log.info("Sucessfully stopped single osd ceph service")
+            else:
+                log.info("Failed to stop ceph osd services")
+            remote.run(args=['sudo', 'systemctl', 'start',
+                             osd_service])
+            time.sleep(4)
+        if mon_role_name in roles:
+            remote.run(args=['sudo', 'systemctl', 'status', mon_name])
+            remote.run(args=['sudo', 'systemctl', 'stop', mon_name])
+            time.sleep(4)  # immediate check will result in deactivating state
+            r = remote.run(args=['sudo', 'systemctl', 'status', mon_name],
+                           stdout=StringIO(), check_status=False)
+            if r.stdout.getvalue().find('Active: inactive'):
+                log.info("Sucessfully stopped single mon ceph service")
+            else:
+                log.info("Failed to stop ceph mon service")
+            remote.run(args=['sudo', 'systemctl', 'start', mon_name])
+            time.sleep(4)
+        if mds_role_name in roles:
+            remote.run(args=['sudo', 'systemctl', 'status', mds_name])
+            remote.run(args=['sudo', 'systemctl', 'stop', mds_name])
+            time.sleep(4)  # immediate check will result in deactivating state
+            r = remote.run(args=['sudo', 'systemctl', 'status', mds_name],
+                           stdout=StringIO(), check_status=False)
+            if r.stdout.getvalue().find('Active: inactive'):
+                log.info("Sucessfully stopped single ceph mds service")
+            else:
+                log.info("Failed to stop ceph mds service")
+            remote.run(args=['sudo', 'systemctl', 'start', mds_name])
+            time.sleep(4)
+    yield