diff --git a/documentation/prometheus-mixin/dashboards.libsonnet b/documentation/prometheus-mixin/dashboards.libsonnet index 1e64af4ca..b3a2ba2f7 100644 --- a/documentation/prometheus-mixin/dashboards.libsonnet +++ b/documentation/prometheus-mixin/dashboards.libsonnet @@ -91,6 +91,60 @@ local g = import 'grafana-builder/grafana.libsonnet'; { yaxes: g.yaxes('ms') } + g.stack, ) - ) - }, + ), + # Remote write specific dashboard. + 'prometheus-remote-write.json': + g.dashboard('Prometheus Remote Write') + .addMultiTemplate('instance', 'prometheus_build_info', 'instance') + .addMultiTemplate('cluster', 'kube_pod_container_info{image=~".*prometheus.*"}', 'cluster') + .addRow( + g.row('Timestamps') + .addPanel( + g.panel('Highest Timestamp In vs. Highest Timestamp Sent') + + g.queryPanel('prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~"$cluster", instance=~"$instance"} - ignoring(queue) group_right(instance) prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~"$cluster", instance=~"$instance"}', '{{cluster}}:{{instance}}-{{queue}}') + + { yaxes: g.yaxes('s') } + ) + .addPanel( + g.panel('Rate[5m]') + + g.queryPanel('rate(prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~"$cluster", instance=~"$instance"}[5m]) - ignoring (queue) group_right(instance) rate(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~"$cluster", instance=~"$instance"}[5m])', '{{cluster}}:{{instance}}-{{queue}}') + ) + ) + .addRow( + g.row('Samples') + .addPanel( + g.panel('Rate, in vs. succeeded or dropped [5m]') + + g.queryPanel('rate(prometheus_remote_storage_samples_in_total{cluster=~"$cluster", instance=~"$instance"}[5m])- ignoring(queue) group_right(instance) rate(prometheus_remote_storage_succeeded_samples_total{cluster=~"$cluster", instance=~"$instance"}[5m]) - rate(prometheus_remote_storage_dropped_samples_total{cluster=~"$cluster", instance=~"$instance"}[5m])', '{{cluster}}:{{instance}}-{{queue}}') + ) + ) + .addRow( + g.row('Shards') + .addPanel( + g.panel("Num. Shards") + + g.queryPanel('prometheus_remote_storage_shards{cluster=~"$cluster", instance=~"$instance"}','{{cluster}}:{{instance}}-{{queue}}') + ) + .addPanel( + g.panel("Capacity") + + g.queryPanel('prometheus_remote_storage_shard_capacity{cluster=~"$cluster", instance=~"$instance"}','{{cluster}}:{{instance}}-{{queue}}') + ) + ) + .addRow( + g.row('Misc Rates.') + .addPanel( + g.panel("Dropped Samples") + + g.queryPanel('rate(prometheus_remote_storage_dropped_samples_total{cluster=~"$cluster", instance=~"$instance"}[5m])','{{cluster}}:{{instance}}-{{queue}}') + ) + .addPanel( + g.panel("Failed Samples") + + g.queryPanel('rate(prometheus_remote_storage_failed_samples_total{cluster=~"$cluster", instance=~"$instance"}[5m])','{{cluster}}:{{instance}}-{{queue}}') + ) + .addPanel( + g.panel("Retried Samples") + + g.queryPanel('rate(prometheus_remote_storage_retried_samples_total{cluster=~"$cluster", instance=~"$instance"}[5m])','{{cluster}}:{{instance}}-{{queue}}') + ) + .addPanel( + g.panel("Enqueue Retries") + + g.queryPanel('rate(prometheus_remote_storage_enqueue_retries_total{cluster=~"$cluster", instance=~"$instance"}[5m])','{{cluster}}:{{instance}}-{{queue}}') + ) + ) + } }