From 74794182a7a9cb736afd5b3e41dcb96258ab51f1 Mon Sep 17 00:00:00 2001
From: Vitaly Zhuravlev <zhuravlev.vitaly@gmail.com>
Date: Tue, 28 Mar 2023 03:35:41 +0800
Subject: [PATCH] Add failed systemd service alert

Signed-off-by: Vitaly Zhuravlev <v-zhuravlev@users.noreply.github.com>
---
 docs/node-mixin/alerts/alerts.libsonnet | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/docs/node-mixin/alerts/alerts.libsonnet b/docs/node-mixin/alerts/alerts.libsonnet
index 4f732058..c1f98c9c 100644
--- a/docs/node-mixin/alerts/alerts.libsonnet
+++ b/docs/node-mixin/alerts/alerts.libsonnet
@@ -337,6 +337,20 @@
               description: 'Memory is filling up on {{ $labels.instance }}, has been above 90% for the last 15 minutes, is currently at {{ printf "%.2f" $value }}%.',
             },
           },
+          {
+            alert: 'NodeSystemdServiceFailed',
+            expr: |||
+              node_systemd_unit_state{%(nodeExporterSelector)s, state="failed"} == 1
+            ||| % $._config,
+            'for': '5m',
+            labels: {
+              severity: 'critical',
+            },
+            annotations: {
+              summary: 'Systemd service has entered failed state.',
+              description: 'Systemd service {{ $labels.name }} has entered failed state on {{ $labels.instance }}',
+            },
+          },
         ],
       },
     ],