Merge pull request #47804 from p-se/wip-7294-pse-custom-prometheus-alerts

mgr/dashboard: enable addition of custom Prometheus alerts

Reviewed-by: Adam King <adking@redhat.com>
Reviewed-by: Anthony D'Atri <anthonyeleven@users.noreply.github.com>

doc/cephadm/services/monitoring.rst

@@ -231,6 +231,7 @@ set``:
 - ``services/grafana/ceph-dashboard.yml``
 - ``services/grafana/grafana.ini``
 - ``services/prometheus/prometheus.yml``
+- ``services/prometheus/alerting/custom_alerts.yml``
 - ``services/loki.yml``
 - ``services/promtail.yml``
 
@@ -280,6 +281,15 @@ Example
   # reconfig the prometheus service
   ceph orch reconfig prometheus
 
+.. code-block:: bash
+
+  # set additional custom alerting rules for Prometheus
+  ceph config-key set mgr/cephadm/services/prometheus/alerting/custom_alerts.yml \
+    -i $PWD/custom_alerts.yml
+
+  # Note that custom alerting rules are not parsed by Jinja and hence escaping
+  # will not be an issue.
+
 Deploying monitoring without cephadm
 ------------------------------------
 

src/pybind/mgr/cephadm/services/monitoring.py

@@ -319,6 +319,23 @@ class PrometheusService(CephadmService):
                 alerts = f.read()
             r['files']['/etc/prometheus/alerting/ceph_alerts.yml'] = alerts
 
+        # Include custom alerts if present in key value store. This enables the
+        # users to add custom alerts. Write the file in any case, so that if the
+        # content of the key value store changed, that file is overwritten
+        # (emptied in case they value has been removed from the key value
+        # store). This prevents the necessity to adapt `cephadm` binary to
+        # remove the file.
+        #
+        # Don't use the template engine for it as
+        #
+        #   1. the alerts are always static and
+        #   2. they are a template themselves for the Go template engine, which
+        #      use curly braces and escaping that is cumbersome and unnecessary
+        #      for the user.
+        #
+        r['files']['/etc/prometheus/alerting/custom_alerts.yml'] = \
+            self.mgr.get_store('services/prometheus/alerting/custom_alerts.yml', '')
+
         return r, sorted(self.calculate_deps())
 
     def calculate_deps(self) -> List[str]:

src/pybind/mgr/cephadm/tests/test_services.py

@@ -430,7 +430,8 @@ class TestMonitoring:
                         '--config-json', '-',
                         '--tcp-ports', '9095'
                     ],
-                    stdin=json.dumps({"files": {"prometheus.yml": y, "root_cert.pem": ''}}),
+                    stdin=json.dumps({"files": {"prometheus.yml": y, "root_cert.pem": '',
+                                                "/etc/prometheus/alerting/custom_alerts.yml": ""}}),
                     image='')
 
     @patch("cephadm.serve.CephadmServe._run_cephadm")

src/pybind/mgr/requirements-required.txt

@@ -5,7 +5,7 @@ cryptography
 jsonpatch
 Jinja2
 pecan
-prettytable
+prettytable==3.3.0
 pyfakefs
 pyOpenSSL
 pytest-cov==2.7.1