From d88c834ea44bd67cfde0bd11ec4ded079b76d11a Mon Sep 17 00:00:00 2001 From: Wong Hoi Sing Edison Date: Tue, 25 Aug 2020 12:16:54 +0800 Subject: [PATCH] systemd: Support Graceful Reboot for AIO Node Ceph AIO installation with single/multiple node is not friendly for loopback mount, especially always get deadlock issue during graceful system reboot. We already have `rbdmap.service` with graceful system reboot friendly as below: [Unit] After=network-online.target Before=remote-fs-pre.target Wants=network-online.target remote-fs-pre.target [Service] ExecStart=/usr/bin/rbdmap map ExecReload=/usr/bin/rbdmap map ExecStop=/usr/bin/rbdmap unmap-all This PR introduce: - `ceph-mon.target`: Ensure startup after `network-online.target` and before `remote-fs-pre.target` - `ceph-*.target`: Ensure startup after `ceph-mon.target` and before `remote-fs-pre.target` - `rbdmap.service`: Once all `_netdev` get unmount by `remote-fs.target`, ensure unmap all RBD BEFORE any Ceph components under `ceph.target` get stopped during shutdown The logic is concept proof by ; also works as expected with Ceph + Kubernetes deployment by . No more deadlock happened during graceful system reboot, both AIO single/multiple no de with loopback mount. Also see: - - - - Fixes: https://tracker.ceph.com/issues/47528 Signed-off-by: Wong Hoi Sing Edison --- systemd/ceph-fuse.target | 1 + systemd/ceph-immutable-object-cache.target | 1 + systemd/ceph-mds.target | 3 +++ systemd/ceph-mds@.service.in | 5 +++-- systemd/ceph-mgr.target | 3 +++ systemd/ceph-mgr@.service.in | 7 +++---- systemd/ceph-mon.target | 2 ++ systemd/ceph-mon@.service.in | 6 +++--- systemd/ceph-osd.target | 3 +++ systemd/ceph-osd@.service.in | 5 +++-- systemd/ceph-radosgw.target | 3 +++ systemd/ceph-radosgw@.service.in | 5 +++-- systemd/ceph-rbd-mirror.target | 1 + systemd/ceph.target | 1 + systemd/rbdmap.service.in | 5 ++--- 15 files changed, 35 insertions(+), 16 deletions(-) diff --git a/systemd/ceph-fuse.target b/systemd/ceph-fuse.target index 70f5cb6e16b..c31fdfa8dce 100644 --- a/systemd/ceph-fuse.target +++ b/systemd/ceph-fuse.target @@ -2,5 +2,6 @@ Description=ceph target allowing to start/stop all ceph-fuse@.service instances at once PartOf=ceph.target Before=ceph.target + [Install] WantedBy=remote-fs.target ceph.target diff --git a/systemd/ceph-immutable-object-cache.target b/systemd/ceph-immutable-object-cache.target index ed512953b0b..f041bad6013 100644 --- a/systemd/ceph-immutable-object-cache.target +++ b/systemd/ceph-immutable-object-cache.target @@ -2,5 +2,6 @@ Description=ceph target allowing to start/stop all ceph-immutable-object-cache@.service instances at once PartOf=ceph.target Before=ceph.target + [Install] WantedBy=multi-user.target ceph.target diff --git a/systemd/ceph-mds.target b/systemd/ceph-mds.target index 238f3ab90c7..1101f21f699 100644 --- a/systemd/ceph-mds.target +++ b/systemd/ceph-mds.target @@ -1,6 +1,9 @@ [Unit] Description=ceph target allowing to start/stop all ceph-mds@.service instances at once PartOf=ceph.target +After=ceph-mon.target Before=ceph.target +Wants=ceph.target ceph-mon.target + [Install] WantedBy=multi-user.target ceph.target diff --git a/systemd/ceph-mds@.service.in b/systemd/ceph-mds@.service.in index 6a235ec1dfa..2884f587f97 100644 --- a/systemd/ceph-mds@.service.in +++ b/systemd/ceph-mds@.service.in @@ -1,8 +1,9 @@ [Unit] Description=Ceph metadata server daemon -After=network-online.target local-fs.target time-sync.target -Wants=network-online.target local-fs.target time-sync.target PartOf=ceph-mds.target +After=network-online.target local-fs.target time-sync.target +Before=remote-fs-pre.target ceph-mds.target +Wants=network-online.target local-fs.target time-sync.target remote-fs-pre.target ceph-mds.target [Service] Environment=CLUSTER=ceph diff --git a/systemd/ceph-mgr.target b/systemd/ceph-mgr.target index f25e494b1d3..288888b0d8d 100644 --- a/systemd/ceph-mgr.target +++ b/systemd/ceph-mgr.target @@ -1,6 +1,9 @@ [Unit] Description=ceph target allowing to start/stop all ceph-mgr@.service instances at once PartOf=ceph.target +After=ceph-mon.target Before=ceph.target +Wants=ceph.target ceph-mon.target + [Install] WantedBy=multi-user.target ceph.target diff --git a/systemd/ceph-mgr@.service.in b/systemd/ceph-mgr@.service.in index f0f3fefc87f..1ee28285209 100644 --- a/systemd/ceph-mgr@.service.in +++ b/systemd/ceph-mgr@.service.in @@ -1,8 +1,9 @@ [Unit] Description=Ceph cluster manager daemon -After=network-online.target local-fs.target time-sync.target -Wants=network-online.target local-fs.target time-sync.target PartOf=ceph-mgr.target +After=network-online.target local-fs.target time-sync.target +Before=remote-fs-pre.target ceph-mgr.target +Wants=network-online.target local-fs.target time-sync.target remote-fs-pre.target ceph-mgr.target [Service] Environment=CLUSTER=ceph @@ -28,11 +29,9 @@ RestartSec=10 RestrictSUIDSGID=true StartLimitBurst=3 StartLimitInterval=30min - # We need to disable this protection as some python libraries generate # dynamic code, like python-cffi, and require mmap calls to succeed MemoryDenyWriteExecute=false - [Install] WantedBy=ceph-mgr.target diff --git a/systemd/ceph-mon.target b/systemd/ceph-mon.target index 097c83ba3be..4325bac7afe 100644 --- a/systemd/ceph-mon.target +++ b/systemd/ceph-mon.target @@ -2,5 +2,7 @@ Description=ceph target allowing to start/stop all ceph-mon@.service instances at once PartOf=ceph.target Before=ceph.target +Wants=ceph.target + [Install] WantedBy=multi-user.target ceph.target diff --git a/systemd/ceph-mon@.service.in b/systemd/ceph-mon@.service.in index 6fbdabec2ca..994cdfd2869 100644 --- a/systemd/ceph-mon@.service.in +++ b/systemd/ceph-mon@.service.in @@ -1,13 +1,13 @@ [Unit] Description=Ceph cluster monitor daemon - +PartOf=ceph-mon.target # According to: # http://www.freedesktop.org/wiki/Software/systemd/NetworkTarget # these can be removed once ceph-mon will dynamically change network # configuration. After=network-online.target local-fs.target time-sync.target -Wants=network-online.target local-fs.target time-sync.target -PartOf=ceph-mon.target +Before=remote-fs-pre.target ceph-mon.target +Wants=network-online.target local-fs.target time-sync.target remote-fs-pre.target ceph-mon.target [Service] Environment=CLUSTER=ceph diff --git a/systemd/ceph-osd.target b/systemd/ceph-osd.target index 7f677f54d5e..e4d1b9f07fc 100644 --- a/systemd/ceph-osd.target +++ b/systemd/ceph-osd.target @@ -1,6 +1,9 @@ [Unit] Description=ceph target allowing to start/stop all ceph-osd@.service instances at once PartOf=ceph.target +After=ceph-mon.target Before=ceph.target +Wants=ceph.target ceph-mon.target + [Install] WantedBy=multi-user.target ceph.target diff --git a/systemd/ceph-osd@.service.in b/systemd/ceph-osd@.service.in index 4a2254e9947..4981417d620 100644 --- a/systemd/ceph-osd@.service.in +++ b/systemd/ceph-osd@.service.in @@ -1,8 +1,9 @@ [Unit] Description=Ceph object storage daemon osd.%i -After=network-online.target local-fs.target time-sync.target ceph-mon.target -Wants=network-online.target local-fs.target time-sync.target PartOf=ceph-osd.target +After=network-online.target local-fs.target time-sync.target +Before=remote-fs-pre.target ceph-osd.target +Wants=network-online.target local-fs.target time-sync.target remote-fs-pre.target ceph-osd.target [Service] Environment=CLUSTER=ceph diff --git a/systemd/ceph-radosgw.target b/systemd/ceph-radosgw.target index 1799e29e84d..8ea707a0bbd 100644 --- a/systemd/ceph-radosgw.target +++ b/systemd/ceph-radosgw.target @@ -1,6 +1,9 @@ [Unit] Description=ceph target allowing to start/stop all ceph-radosgw@.service instances at once PartOf=ceph.target +After=ceph-mon.target Before=ceph.target +Wants=ceph.target ceph-mon.target + [Install] WantedBy=multi-user.target ceph.target diff --git a/systemd/ceph-radosgw@.service.in b/systemd/ceph-radosgw@.service.in index 040a9ccdafa..cfff60c18b8 100644 --- a/systemd/ceph-radosgw@.service.in +++ b/systemd/ceph-radosgw@.service.in @@ -1,8 +1,9 @@ [Unit] Description=Ceph rados gateway -After=network-online.target local-fs.target time-sync.target -Wants=network-online.target local-fs.target time-sync.target PartOf=ceph-radosgw.target +After=network-online.target local-fs.target time-sync.target +Before=remote-fs-pre.target ceph-radosgw.target +Wants=network-online.target local-fs.target time-sync.target remote-fs-pre.target ceph-radosgw.target [Service] Environment=CLUSTER=ceph diff --git a/systemd/ceph-rbd-mirror.target b/systemd/ceph-rbd-mirror.target index 43e9a4c8164..57ea09f1dc5 100644 --- a/systemd/ceph-rbd-mirror.target +++ b/systemd/ceph-rbd-mirror.target @@ -2,5 +2,6 @@ Description=ceph target allowing to start/stop all ceph-rbd-mirror@.service instances at once PartOf=ceph.target Before=ceph.target + [Install] WantedBy=multi-user.target ceph.target diff --git a/systemd/ceph.target b/systemd/ceph.target index 60734baff68..67a982c5b44 100644 --- a/systemd/ceph.target +++ b/systemd/ceph.target @@ -1,4 +1,5 @@ [Unit] Description=ceph target allowing to start/stop all ceph*@.service instances at once + [Install] WantedBy=multi-user.target diff --git a/systemd/rbdmap.service.in b/systemd/rbdmap.service.in index 4757ee6ccb2..6644508cf0d 100644 --- a/systemd/rbdmap.service.in +++ b/systemd/rbdmap.service.in @@ -1,9 +1,8 @@ [Unit] Description=Map RBD devices - -After=network-online.target +After=network-online.target ceph.target Before=remote-fs-pre.target -Wants=network-online.target remote-fs-pre.target +Wants=network-online.target remote-fs-pre.target ceph.target [Service] EnvironmentFile=-@SYSTEMD_ENV_FILE@