exposing softirq metrics (#2294)

Signed-off-by: abbeywoodyear <abbey.woodyear@thehutgroup.com>

README.md

@@ -197,6 +197,7 @@ perf | Exposes perf based metrics (Warning: Metrics are dependent on kernel conf
 processes | Exposes aggregate process statistics from `/proc`. | Linux
 qdisc | Exposes [queuing discipline](https://en.wikipedia.org/wiki/Network_scheduler#Linux_kernel) statistics | Linux
 slabinfo | Exposes slab statistics from `/proc/slabinfo`. Note that permission of `/proc/slabinfo` is usually 0400, so set it appropriately. | Linux
+softirqs | Exposes detailed softirq statistics from `/proc/softirqs`. | Linux
 sysctl | Expose sysctl values from `/proc/sys`. Use `--collector.sysctl.include(-info)` to configure. | Linux
 systemd | Exposes service and system status from [systemd](http://www.freedesktop.org/wiki/Software/systemd/). | Linux
 tcpstat | Exposes TCP connection status information from `/proc/net/tcp` and `/proc/net/tcp6`. (Warning: the current version has potential performance issues in high load situations.) | Linux

collector/fixtures/e2e-64k-page-output.txt

@@ -2922,6 +2922,7 @@ node_scrape_collector_success{collector="schedstat"} 1
 node_scrape_collector_success{collector="selinux"} 1
 node_scrape_collector_success{collector="slabinfo"} 1
 node_scrape_collector_success{collector="sockstat"} 1
+node_scrape_collector_success{collector="softirqs"} 1
 node_scrape_collector_success{collector="softnet"} 1
 node_scrape_collector_success{collector="stat"} 1
 node_scrape_collector_success{collector="sysctl"} 1
@@ -3028,6 +3029,28 @@ node_sockstat_UDP_mem_bytes 0
 # HELP node_sockstat_sockets_used Number of IPv4 sockets in use.
 # TYPE node_sockstat_sockets_used gauge
 node_sockstat_sockets_used 229
+# HELP node_softirqs_functions_total Softirq counts per CPU.
+# TYPE node_softirqs_functions_total counter
+node_softirqs_functions_total{cpu="0",type="BLOCK"} 23776
+node_softirqs_functions_total{cpu="0",type="HI"} 7
+node_softirqs_functions_total{cpu="0",type="HRTIMER"} 40
+node_softirqs_functions_total{cpu="0",type="IRQ_POLL"} 0
+node_softirqs_functions_total{cpu="0",type="NET_RX"} 43066
+node_softirqs_functions_total{cpu="0",type="NET_TX"} 2301
+node_softirqs_functions_total{cpu="0",type="RCU"} 155929
+node_softirqs_functions_total{cpu="0",type="SCHED"} 378895
+node_softirqs_functions_total{cpu="0",type="TASKLET"} 372
+node_softirqs_functions_total{cpu="0",type="TIMER"} 424191
+node_softirqs_functions_total{cpu="1",type="BLOCK"} 24115
+node_softirqs_functions_total{cpu="1",type="HI"} 1
+node_softirqs_functions_total{cpu="1",type="HRTIMER"} 346
+node_softirqs_functions_total{cpu="1",type="IRQ_POLL"} 0
+node_softirqs_functions_total{cpu="1",type="NET_RX"} 104508
+node_softirqs_functions_total{cpu="1",type="NET_TX"} 2430
+node_softirqs_functions_total{cpu="1",type="RCU"} 146631
+node_softirqs_functions_total{cpu="1",type="SCHED"} 152852
+node_softirqs_functions_total{cpu="1",type="TASKLET"} 1899
+node_softirqs_functions_total{cpu="1",type="TIMER"} 108342
 # HELP node_softirqs_total Number of softirq calls.
 # TYPE node_softirqs_total counter
 node_softirqs_total{vector="block"} 186066

collector/fixtures/e2e-output.txt

@@ -2944,6 +2944,7 @@ node_scrape_collector_success{collector="schedstat"} 1
 node_scrape_collector_success{collector="selinux"} 1
 node_scrape_collector_success{collector="slabinfo"} 1
 node_scrape_collector_success{collector="sockstat"} 1
+node_scrape_collector_success{collector="softirqs"} 1
 node_scrape_collector_success{collector="softnet"} 1
 node_scrape_collector_success{collector="stat"} 1
 node_scrape_collector_success{collector="sysctl"} 1
@@ -3050,6 +3051,28 @@ node_sockstat_UDP_mem_bytes 0
 # HELP node_sockstat_sockets_used Number of IPv4 sockets in use.
 # TYPE node_sockstat_sockets_used gauge
 node_sockstat_sockets_used 229
+# HELP node_softirqs_functions_total Softirq counts per CPU.
+# TYPE node_softirqs_functions_total counter
+node_softirqs_functions_total{cpu="0",type="BLOCK"} 23776
+node_softirqs_functions_total{cpu="0",type="HI"} 7
+node_softirqs_functions_total{cpu="0",type="HRTIMER"} 40
+node_softirqs_functions_total{cpu="0",type="IRQ_POLL"} 0
+node_softirqs_functions_total{cpu="0",type="NET_RX"} 43066
+node_softirqs_functions_total{cpu="0",type="NET_TX"} 2301
+node_softirqs_functions_total{cpu="0",type="RCU"} 155929
+node_softirqs_functions_total{cpu="0",type="SCHED"} 378895
+node_softirqs_functions_total{cpu="0",type="TASKLET"} 372
+node_softirqs_functions_total{cpu="0",type="TIMER"} 424191
+node_softirqs_functions_total{cpu="1",type="BLOCK"} 24115
+node_softirqs_functions_total{cpu="1",type="HI"} 1
+node_softirqs_functions_total{cpu="1",type="HRTIMER"} 346
+node_softirqs_functions_total{cpu="1",type="IRQ_POLL"} 0
+node_softirqs_functions_total{cpu="1",type="NET_RX"} 104508
+node_softirqs_functions_total{cpu="1",type="NET_TX"} 2430
+node_softirqs_functions_total{cpu="1",type="RCU"} 146631
+node_softirqs_functions_total{cpu="1",type="SCHED"} 152852
+node_softirqs_functions_total{cpu="1",type="TASKLET"} 1899
+node_softirqs_functions_total{cpu="1",type="TIMER"} 108342
 # HELP node_softirqs_total Number of softirq calls.
 # TYPE node_softirqs_total counter
 node_softirqs_total{vector="block"} 186066

collector/fixtures/proc/softirqs

@@ -0,0 +1,11 @@
+                    CPU0       CPU1       
+          HI:          7          1
+       TIMER:     424191     108342
+      NET_TX:       2301       2430
+      NET_RX:      43066     104508
+       BLOCK:      23776      24115
+    IRQ_POLL:          0          0
+     TASKLET:        372       1899
+       SCHED:     378895     152852
+     HRTIMER:         40        346
+         RCU:     155929     146631

collector/softirq_linux.go

@@ -0,0 +1,68 @@
+// Copyright 2023 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !nosoftirqs
+// +build !nosoftirqs
+
+package collector
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	softirqLabelNames = []string{"cpu", "type"}
+)
+
+func (c *softirqsCollector) Update(ch chan<- prometheus.Metric) (err error) {
+	softirqs, err := c.fs.Softirqs()
+	if err != nil {
+		return fmt.Errorf("couldn't get softirqs: %w", err)
+	}
+
+	for cpuNo, value := range softirqs.Hi {
+		ch <- c.desc.mustNewConstMetric(float64(value), strconv.Itoa(cpuNo), "HI")
+	}
+	for cpuNo, value := range softirqs.Timer {
+		ch <- c.desc.mustNewConstMetric(float64(value), strconv.Itoa(cpuNo), "TIMER")
+	}
+	for cpuNo, value := range softirqs.NetTx {
+		ch <- c.desc.mustNewConstMetric(float64(value), strconv.Itoa(cpuNo), "NET_TX")
+	}
+	for cpuNo, value := range softirqs.NetRx {
+		ch <- c.desc.mustNewConstMetric(float64(value), strconv.Itoa(cpuNo), "NET_RX")
+	}
+	for cpuNo, value := range softirqs.Block {
+		ch <- c.desc.mustNewConstMetric(float64(value), strconv.Itoa(cpuNo), "BLOCK")
+	}
+	for cpuNo, value := range softirqs.IRQPoll {
+		ch <- c.desc.mustNewConstMetric(float64(value), strconv.Itoa(cpuNo), "IRQ_POLL")
+	}
+	for cpuNo, value := range softirqs.Tasklet {
+		ch <- c.desc.mustNewConstMetric(float64(value), strconv.Itoa(cpuNo), "TASKLET")
+	}
+	for cpuNo, value := range softirqs.Sched {
+		ch <- c.desc.mustNewConstMetric(float64(value), strconv.Itoa(cpuNo), "SCHED")
+	}
+	for cpuNo, value := range softirqs.HRTimer {
+		ch <- c.desc.mustNewConstMetric(float64(value), strconv.Itoa(cpuNo), "HRTIMER")
+	}
+	for cpuNo, value := range softirqs.RCU {
+		ch <- c.desc.mustNewConstMetric(float64(value), strconv.Itoa(cpuNo), "RCU")
+	}
+
+	return err
+}

collector/softirqs_common.go

@@ -0,0 +1,50 @@
+// Copyright 2023 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux && !nosoftirqs
+// +build linux,!nosoftirqs
+
+package collector
+
+import (
+	"fmt"
+	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/procfs"
+)
+
+type softirqsCollector struct {
+	fs     procfs.FS
+	desc   typedDesc
+	logger log.Logger
+}
+
+func init() {
+	registerCollector("softirqs", defaultDisabled, NewSoftirqsCollector)
+}
+
+// NewSoftirqsCollector returns a new Collector exposing softirq stats.
+func NewSoftirqsCollector(logger log.Logger) (Collector, error) {
+	desc := typedDesc{prometheus.NewDesc(
+		namespace+"_softirqs_functions_total",
+		"Softirq counts per CPU.",
+		softirqLabelNames, nil,
+	), prometheus.CounterValue}
+
+	fs, err := procfs.NewFS(*procPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open procfs: %w", err)
+	}
+
+	return &softirqsCollector{fs, desc, logger}, nil
+}

end-to-end-test.sh

@@ -42,6 +42,7 @@ enabled_collectors=$(cat << COLLECTORS
   selinux
   slabinfo
   sockstat
+  softirqs
   stat
   sysctl
   textfile