systemd: updated demo templates

systemd-testing/SYSTEMD-load-occupy.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+# This is a trivial script for testing.
+#
+# Beside some trivial dd write load generator,
+# it may occupy the mount point for some time,
+# simulating a badly terminating process so that umount
+# cannot work immediately, but needs to be restarted
+# somewhat later.
+#
+# When this script is killed from "systemctl stop", the
+# "sleep" sub-process will _not_ be killed, but terminate
+# a little bit later. This leads to a race condition with lsof,
+# similar to some observed practical daemon behaviour.
+
+# Important: set pwd to occupy the mountpoint, also for sub-processes
+cd "$1" || exit $?
+
+# End writing after a while => allow for reaching UpToDate
+nr_load="${2:-5}"
+
+# 2-edge handshake protocol
+start_flag=run-the-load.flag
+running_flag=load-is-running.flag
+
+if [[ -e $start_flag ]]; then
+    exit 0
+fi
+
+: > $start_flag
+: > $running_flag
+
+{
+    testfile=dummy-testfile.zero
+
+    while [[ -e $start_flag ]] && [[ -e $running_flag ]]; do
+    # do not write too much, for reasonable testing of KASAN kernels
+	if (( nr_load-- > 0 )); then
+	    dd if=/dev/zero of=$testfile bs=1k count=1001
+	else
+	    touch $testfile
+	fi
+	sleep $(( RANDOM * 5 / 32767 )) &
+	wait
+    done
+
+    rm -f $running_flag
+} &
+
+exit 0

systemd-testing/SYSTEMD-load-stop.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# This is a trivial script for testing.
+
+# Important: set pwd to occupy the mountpoint.
+cd "$1" || exit $?
+
+# 2-edge handshake protocol
+start_flag=run-the-load.flag
+running_flag=load-is-running.flag
+
+if ! [[ -e $start_flag ]]; then
+    exit 0
+fi
+
+rm -f $start_flag
+
+while [[ -e $running_flag ]]; do
+    sleep 1
+done
+
+exit 0

systemd-testing/^{mntname}-@escvar{res}-delay.service

@@ -0,0 +1,34 @@
+@eval{%let{mntpath}{%subst{%{mntname}}{-}{/}}}
+[Unit]
+Description=MARS TESTING (u)mount delay for /@{mntpath}/@{res}
+Documentation=https://github.com/schoebel/mars/docu/mars-user-manual.pdf
+
+## No markers for the template processor are necessary here, because
+## this unit is indirectly referenced from other units.
+## The template engine will follow all known dependencies and instantiate
+## this one when necessary.
+
+## HINT: in addition to the combination of BindsTo= with After= as
+## documented in "man systemd.unit" to get an _ordering_ dependency for
+## startup, it looks like we also need PartOf= to get _another_ ordering
+## dependency for _stopping_ in the right order.
+## The docs were not particularly clear about this, I found this by testing.
+##
+BindsTo=@{mntname}-@escvar{res}.mount
+After=@{mntname}-@escvar{res}.mount
+PartOf=@{mntname}-@escvar{res}.mount
+
+ConditionPathIsMountPoint=/@{mntpath}/@{res}
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+# Important for avoiding failures: do not send any signals
+KillMode=none
+
+## Simulate typical runtime behaviour by random delays
+ExecStart=/bin/bash -c 'sleep $(( RANDOM * 3 / 32767 ))'
+ExecStop=/bin/bash  -c 'sleep $(( RANDOM * 4 / 32767 ))'
+
+[Install]
+WantedBy=mars.service

systemd-testing/daemon-reload.service

@@ -0,0 +1,24 @@
+[Unit]
+Description=MARS global single-threaded daemon-reloead
+Documentation=https://github.com/schoebel/mars/docu/mars-manual.pdf
+
+## The trick is to "misuse" this service (which exists only once)
+## for inhibiting startup of "systemctl daemon-real" in parallel to
+## itself.
+## In addition, Before= and After= depencencies may be used for
+## runtime exclusion of parallelism from/with other services.
+## This appears to necessary in some places, where races have
+## between daemon-reload and startup of other service have been observed.
+Before=mars-trigger.service
+
+[Service]
+# Important for blocking parallelism with itself
+Type=oneshot
+RemainAfterExit=no
+# Important for avoiding failures: do not send any signals
+KillMode=none
+
+ExecStart=/usr/bin/systemctl daemon-reload
+
+[Install]
+WantedBy=mars.service

systemd-testing/mars-@escvar{res}-trigger.path

@@ -0,0 +1,17 @@
+[Unit]
+Description=MARS per-resource watcher for /mars/resource-@{res}/systemd-trigger
+Documentation=https://github.com/schoebel/mars/docu/mars-manual.pdf
+Requires=mars-trigger.path
+After=mars-trigger.path
+
+## Markers for the template processor
+# ALWAYS_START
+
+[Path]
+PathExists=/mars/resource-@{res}/systemd-trigger
+PathChanged=/mars/resource-@{res}/systemd-trigger
+PathModified=/mars/resource-@{res}/systemd-trigger
+Unit=mars-@escvar{res}-trigger.service
+
+[Install]
+WantedBy=mars.service

systemd-testing/mars-@escvar{res}-trigger.service

@@ -0,0 +1,36 @@
+[Unit]
+Description=MARS per-resource trigger from /mars/resource-@{res}/systemd-trigger
+Documentation=https://github.com/schoebel/mars/docu/mars-manual.pdf
+
+## Prohibit parallelism with any daemon-reload
+After=daemon-reload.service
+
+## The per-resource trigger is the _only_ one which is allowed
+## to remotely start / stop any units dealing with resources.
+## Otherwise, races or conflicting start / stop operations might
+## occur, leading to deadlocks / failures / etc.
+##
+## General rule: each *.path watcher can only start its corresponding
+## *-trigger.service at most _once_ in parallel (provided that the actions
+## caused by "marsadm systemd-trigger-extern" are not forking into background).
+## So there is never any parallelism of _any_ unit with exactly _itself_
+## (although _different_ units are allowed to run in parallel to each other,
+## of course).
+
+## Do not start both the global and any of the per-resource triggers
+## in parallel.
+## It could lead to various races between global and per-resource units.
+## In contrast, different per-resource triggers are not mutually exclusive.
+After=mars-trigger.service
+
+[Service]
+# Important for blocking parallelism with itself
+Type=oneshot
+RemainAfterExit=no
+# Important for avoiding failures: do not send any signals
+KillMode=none
+# Important: the resource argument will keep remote triggers disjoint
+ExecStart=/usr/bin/marsadm systemd-trigger-extern @{res}
+
+[Install]
+WantedBy=mars.service

systemd-testing/mars-emergency.path

@@ -1,5 +1,5 @@
 [Unit]
-Description=MARS systemd trigger from /mars/userspace/emergency-trigger
+Description=MARS trigger from /mars/userspace/emergency-trigger
 Documentation=https://github.com/schoebel/mars/docu/mars-user-manual.pdf
 Requires=mars.service
 After=mars.service

systemd-testing/mars-emergency.service

@@ -5,6 +5,7 @@ Documentation=https://github.com/schoebel/mars/docu/mars-user-manual.pdf
 [Service]
 Type=oneshot
 RemainAfterExit=no
+# Important for avoiding failures: do not send any signals
 KillMode=none
 ExecStart=/usr/bin/marsadm cron
 

systemd-testing/mars-trigger.path

@@ -1,11 +1,17 @@
 [Unit]
-Description=MARS systemd trigger from /mars/userspace/systemd-trigger
+Description=MARS global watcher /mars/userspace/systemd-trigger
 Documentation=https://github.com/schoebel/mars/docu/mars-user-manual.pdf
-Requires=mars.service
+
+## Global path watcher for calling mars-trigger.service
+
+## Markers for the template processor
+# ALWAYS_START
+
+BindsTo=mars.service
 After=mars.service
 
 [Path]
-#PathExists=/mars/userspace/systemd-trigger
+PathExists=/mars/userspace/systemd-trigger
 PathChanged=/mars/userspace/systemd-trigger
 PathModified=/mars/userspace/systemd-trigger
 Unit=mars-trigger.service

systemd-testing/mars-trigger.service

@@ -0,0 +1,26 @@
+[Unit]
+Description=MARS global trigger from /mars/userspace/systemd-trigger
+Documentation=https://github.com/schoebel/mars/docu/mars-manual.pdf
+
+## Prohibit parallelism with any daemon-reload
+After=daemon-reload.service
+
+## The global trigger calls "marsadm systemd-trigger-extern" _without_
+## resource argument.
+## The global variant will only generate any new templates, but will not
+## activate/deactivate them (except when they are new).
+## This is necessary for race avoidance with per-resource triggers.
+## Only the per-resource triggers (see mars-@{res}-trigger.service)
+## are allowed to actually start/stop any units dealing with resources.
+
+[Service]
+# Important for blocking parallelism with itself
+Type=oneshot
+RemainAfterExit=no
+# Important for avoiding failures: do not send any signals
+KillMode=none
+# Important: no resource argument must be given here
+ExecStart=/usr/bin/marsadm systemd-trigger-extern
+
+[Install]
+WantedBy=mars.service

systemd-testing/mars.path

@@ -2,6 +2,9 @@
 Description=MARS block layer HA (activation by appearance of /mars/uuid)
 Documentation=https://github.com/schoebel/mars/docu/mars-user-manual.pdf
 
+## Markers for the template processor
+# ALWAYS_START
+
 [Path]
 PathExists=/mars/uuid
 Unit=mars.service

systemd-testing/mars.service

@@ -1,17 +1,24 @@
 [Unit]
 Description=MARS block layer HA (kernel module)
 Documentation=https://github.com/schoebel/mars/docu/mars-user-manual.pdf
+
+## Markers for the template processor
+# ALWAYS_START
+
+BindsTo=mars.mount
+After=mars.mount
 RequiresMountsFor=/mars/uuid
+
+Wants=network-online.target
+
 StopWhenUnneeded=false
 IgnoreOnIsolate=true
 
-ConditionPathIsSymbolicLink=/mars/uuid
-
 [Service]
 Type=oneshot
+# Important for avoiding failures: do not send any signals
 KillMode=none
 RemainAfterExit=yes
-#ExecStartPre=if ! mountpoint /mars; then mount /dev/*/mars /mars; fi
 ExecStart=/sbin/modprobe mars
 ExecStop=/sbin/rmmod mars
 

systemd-testing/mnt-test-@escvar{res}-testload.script

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+# Markers for the template processor
+# DEFAULT_START
+# DEFAULT_STOP
+#
+# ALSO=mars-@escvar{res}-trigger.path
+# ALSO=mars-@escvar{res}-trigger.service
+
+op="$1"
+
+# Ensure that pure trigger does not change anything.
+
+mars_dev=/dev/mars/@{res}
+mnt=/mnt/test/@{res}
+
+case "$op" in
+start)
+    # Assumption: start and vmstart seem to be idempotent already
+    if ! [[ -b $mars_dev ]]; then
+	echo "ignoring, $mars_dev is not present" 
+	exit 0
+    fi
+    if mountpoint $mnt; then
+	echo "ignoring, $mnt is already mounted" 
+	exit 0
+    fi
+    mkdir -p $mnt
+    mount $mars_dev $mnt
+    mountpoint $mnt
+    rc=$?
+    # cleanup any old flags
+    rm -f $mnt/*.flag
+    # start load
+    if (( !rc )); then
+	/etc/marsadm/systemd-templates/SYSTEMD-load-occupy.sh $mnt
+    fi
+    exit $rc
+    ;;
+
+stop)
+    if ! mountpoint $mnt; then
+	# Idempotence
+	exit 0
+    fi
+
+    /etc/marsadm/systemd-templates/SYSTEMD-load-stop.sh $mnt
+
+    # for additional safety
+    sleep 1
+
+    # umount, idempotent for increased robustness
+    if mountpoint $mnt; then
+	umount $mnt
+    fi
+
+    # status
+    if mountpoint $mnt; then
+	exit 1
+    fi
+    exit 0
+    ;;
+
+*)
+    # Ignore all other ops, like enable / disable / etc
+    echo "Ignore '$op'"
+    exit 0
+esac
+
+exit 0

systemd-testing/mnt-test-@escvar{res}-testload.service

@@ -0,0 +1,49 @@
+[Unit]
+Description=MARS TESTING dummy process running on /mnt/test/@{res}
+Documentation=https://github.com/schoebel/mars/docu/mars-user-manual.pdf
+
+## Markers for the template processor
+# TESTING_START
+
+## Prohibit parallelism with any daemon-reload
+After=daemon-reload.service
+
+## HINT: in addition to the combination of BindsTo= with After= as
+## documented in "man systemd.unit" to get an _ordering_ dependency for
+## startup, it looks like we also need PartOf= to get _another_ ordering
+## dependency for _stopping_ in the right order.
+## The docs were not particularly clear about this, I found this by testing.
+
+## Comment / uncomment the following, depending on test behaviour:
+
+## This version uses an intermediate "delay" service for testing.
+## Notice: the pathname part of the delay service is generated by
+## pattern matching from ^{mnt} . Look at the generated units.
+## This setup also tests transitive systemd dependencies, where
+## BindsTo+After+PartOf is crucial for starting and stopping any
+## intermediate units in the right order.
+#BindsTo=mnt-test-@escvar{res}-delay.service
+#After=mnt-test-@escvar{res}-delay.service
+#PartOf=mnt-test-@escvar{res}-delay.service
+
+## Direct version without delay. Use in place of the delay service.
+BindsTo=mnt-test-@escvar{res}.mount
+After=mnt-test-@escvar{res}.mount
+PartOf=mnt-test-@escvar{res}.mount
+
+ConditionPathIsMountPoint=/mnt/test/@{res}
+
+[Service]
+Type=simple
+ExecStart=/etc/marsadm/systemd-templates/SYSTEMD-load-occupy.sh /mnt/test/@{res}
+ExecStop=/etc/marsadm/systemd-templates/SYSTEMD-load-stop.sh /mnt/test/@{res}
+# Important for avoiding failures: do not send any signals
+KillMode=none
+
+## Following is an alternative which does not work due to races.
+## Only enable this if you want to trigger such problems for testing.
+#ExecStop=/usr/bin/killall -r MARS-load
+#KillMode=mixed
+
+[Install]
+WantedBy=mars.service

systemd-testing/mnt-test-@escvar{res}.mount

@@ -0,0 +1,44 @@
+[Unit]
+Description=MARS TESTING local mount on /mnt/test/@{res}
+Documentation=https://github.com/schoebel/mars/docu/mars-user-manual.pdf
+
+## Markers for the template processor
+## Examples:
+
+## marsdm set-systemd-unit all TESTIN ==> mount + may produce some data
+## (depending on another unit containing marker "TESTING_START")
+# TESTING_STOP
+
+## marsadm set-systemd-unit all MOUNTED ==> only mount, nothing else
+## This may be necessary for reaching UpToDate without inherent races
+## on data production.
+# MOUNTED_START
+# MOUNTED_STOP
+
+## The following could be used for transitively stopping everything
+## upon "systemctl stop mars.service".
+## Not recommended, since it may cause unnecessary customer downtime
+## when executed "by accident".
+#BindsTo=mars.service
+
+Requires=mars.service
+After=mars.service
+
+## Prohibit parallelism with any daemon-reload
+After=daemon-reload.service
+
+# Only start when marsadm is controlling the target prosumer
+ConditionPathIsSymbolicLink=/mars/resource-@{res}/systemd-want
+
+# Try to achieve idempotence for starting
+ConditionPathIsMountPoint=!/mnt/test/@{res}
+
+[Mount]
+What=/dev/mars/@{res}
+Where=/mnt/test/@{res}
+Options=noauto
+# Important for avoiding failures: do not send any signals
+KillMode=none
+
+[Install]
+WantedBy=mars.service

systemd-testing/trigger-wrapper-for-testing.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+pid=$BASHPID
+ppid=$PPID
+
+echo -n L $* $(pstree $pid) > /proc/sys/mars/trigger
+
+exec /usr/local/bin/marsadm systemd-trigger >> /mars/trigger.log 2>&1

systemd/MARS-load-occupy.sh

@@ -1,19 +0,0 @@
-#!/bin/bash
-
-# This is a trivial script for testing.
-#
-# It just occupies the mount point for some time,
-# simulating a badly terminating process so that umount
-# cannot work immediately, but needs to be restarted
-# somewhat later.
-#
-# When this script is killed from "systemctl stop", the
-# "sleep" sub-process will _not_ be killed, but terminate
-# a little bit later. This leads to a race condition with lsof,
-# similar to some observed practical daemon behaviour.
-
-cd "$1" || exit $?
-
-while true; do
-	sleep 1
-done

systemd/^{mntname}-@{res}-medium-01.service

@@ -1,14 +0,0 @@
-@eval{%let{mntpath}{%subst{%{mntname}}{-}{/}}}
-[Unit]
-Description=MARS dummy process running on /@{mntpath}/@{res}
-Documentation=https://github.com/schoebel/mars/docu/mars-user-manual.pdf
-BindsTo=@{mntname}-@escvar{res}.mount
-After=@{mntname}-@escvar{res}.mount
-
-[Service]
-Type=simple
-ExecStart=/etc/marsadm/systemd-templates/MARS-load-occupy.sh /@{mntpath}/@{res}
-KillMode=process
-
-[Install]
-WantedBy=mars.service

systemd/^{mntname}-@{res}-medium-02.service

@@ -1,14 +0,0 @@
-@eval{%let{mntpath}{%subst{%{mntname}}{-}{/}}}
-[Unit]
-Description=MARS dummy process running on /@{mntpath}/@{res}
-Documentation=https://github.com/schoebel/mars/docu/mars-user-manual.pdf
-BindsTo=@{mntname}-@escvar{res}-medium-01.service
-After=@{mntname}-@escvar{res}-medium-01.service
-
-[Service]
-Type=simple
-ExecStart=/etc/marsadm/systemd-templates/MARS-load-occupy.sh /@{mntpath}/@{res}
-KillMode=process
-
-[Install]
-WantedBy=mars.service

systemd/^{mntname}-@{res}-testload.service

@@ -1,14 +0,0 @@
-@eval{%let{mntpath}{%subst{%{mntname}}{-}{/}}}
-[Unit]
-Description=MARS dummy process running on /@{mntpath}/@{res}
-Documentation=https://github.com/schoebel/mars/docu/mars-user-manual.pdf
-BindsTo=@{mntname}-@escvar{res}-medium-02.service
-After=@{mntname}-@escvar{res}-medium-02.service
-
-[Service]
-Type=simple
-ExecStart=/etc/marsadm/systemd-templates/MARS-load-occupy.sh /@{mntpath}/@{res}
-KillMode=process
-
-[Install]
-WantedBy=mars.service

systemd/^{mntname}-@{res}.mount

@@ -1,18 +0,0 @@
-@eval{%let{mntpath}{%subst{%{mntname}}{-}{/}}}
-[Unit]
-Description=MARS local mount on /@{mntpath}/@{res}
-Documentation=https://github.com/schoebel/mars/docu/mars-user-manual.pdf
-Requires=mars.service
-After=mars.service
-
-ConditionPathIsSymbolicLink=/mars/resource-@{res}/systemd-want
-ConditionPathExists=/mars/resource-@{res}/userspace/systemd-want-@{host}
-ConditionPathExists=/dev/mars/@{res}
-ConditionPathIsDirectory=/@{mntpath}/@{res}
-
-[Mount]
-What=/dev/mars/@{res}
-Where=/@{mntpath}/@{res}
-
-[Install]
-WantedBy=mars.service

systemd/dev-mars-@{res}.path

@@ -1,12 +0,0 @@
-[Unit]
-Description=MARS auto-activation of primary device /dev/mars/@{res}
-Documentation=https://github.com/schoebel/mars/docu/mars-user-manual.pdf
-Requires=mars.service
-After=mars.service
-
-[Path]
-PathExists=/dev/mars/@{res}
-Unit=mnt-test-@escvar{res}.mount
-
-[Install]
-WantedBy=mars.service

systemd/mars-@{res}-trigger.path

@@ -1,13 +0,0 @@
-[Unit]
-Description=MARS auto-activation of primary device /dev/mars/@{res}
-Documentation=https://github.com/schoebel/mars/docu/mars-manual.pdf
-Requires=mars.service
-After=mars.service
-
-[Path]
-PathChanged=/mars/resource-@{res}/systemd-want
-PathModified=/mars/resource-@{res}/systemd-want
-Unit=mars-trigger.service
-
-[Install]
-WantedBy=mars.service

systemd/mars-trigger.service

@@ -1,12 +0,0 @@
-[Unit]
-Description=MARS systemd trigger service from /mars/userspace/systemd-trigger
-Documentation=https://github.com/schoebel/mars/docu/mars-manual.pdf
-
-[Service]
-Type=oneshot
-RemainAfterExit=no
-KillMode=none
-ExecStart=/usr/local/bin/marsadm systemd-trigger
-
-[Install]
-WantedBy=mars.service