diff --git a/README.md b/README.md index 38affb73..97a9bdc2 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ Name | Description bonding | Exposes the number of configured and active slaves of Linux bonding interfaces. gmond | Exposes statistics from Ganglia. interrupts | Exposes detailed interrupts statistics from `/proc/interrupts`. +ipvs | Exposes IPVS status from `/proc/net/ip_vs` and stats from `/proc/net/ip_vs_stats`. lastlogin | Exposes the last time there was a login. megacli | Exposes RAID statistics from MegaCLI. ntp | Exposes time drift from an NTP server. diff --git a/collector/fixtures/net/ip_vs b/collector/fixtures/net/ip_vs new file mode 100644 index 00000000..6a6a97d7 --- /dev/null +++ b/collector/fixtures/net/ip_vs @@ -0,0 +1,14 @@ +IP Virtual Server version 1.2.1 (size=4096) +Prot LocalAddress:Port Scheduler Flags + -> RemoteAddress:Port Forward Weight ActiveConn InActConn +TCP C0A80016:0CEA wlc + -> C0A85216:0CEA Tunnel 100 248 2 + -> C0A85318:0CEA Tunnel 100 248 2 + -> C0A85315:0CEA Tunnel 100 248 1 +TCP C0A80039:0CEA wlc + -> C0A85416:0CEA Tunnel 0 0 0 + -> C0A85215:0CEA Tunnel 100 1499 0 + -> C0A83215:0CEA Tunnel 100 1498 0 +TCP C0A80037:0CEA wlc + -> C0A8321A:0CEA Tunnel 0 0 0 + -> C0A83120:0CEA Tunnel 100 0 0 diff --git a/collector/fixtures/net/ip_vs_result.txt b/collector/fixtures/net/ip_vs_result.txt new file mode 100644 index 00000000..2202dc8a --- /dev/null +++ b/collector/fixtures/net/ip_vs_result.txt @@ -0,0 +1,45 @@ +# HELP node_ipvs_backend_connections_active The current active connections by local and remote address. +# TYPE node_ipvs_backend_connections_active gauge +node_ipvs_backend_connections_active{local_address="192.168.0.22",local_port="3306",proto="TCP",remote_address="192.168.82.22",remote_port="3306"} 248 +node_ipvs_backend_connections_active{local_address="192.168.0.22",local_port="3306",proto="TCP",remote_address="192.168.83.21",remote_port="3306"} 248 +node_ipvs_backend_connections_active{local_address="192.168.0.22",local_port="3306",proto="TCP",remote_address="192.168.83.24",remote_port="3306"} 248 +node_ipvs_backend_connections_active{local_address="192.168.0.55",local_port="3306",proto="TCP",remote_address="192.168.49.32",remote_port="3306"} 0 +node_ipvs_backend_connections_active{local_address="192.168.0.55",local_port="3306",proto="TCP",remote_address="192.168.50.26",remote_port="3306"} 0 +node_ipvs_backend_connections_active{local_address="192.168.0.57",local_port="3306",proto="TCP",remote_address="192.168.50.21",remote_port="3306"} 1498 +node_ipvs_backend_connections_active{local_address="192.168.0.57",local_port="3306",proto="TCP",remote_address="192.168.82.21",remote_port="3306"} 1499 +node_ipvs_backend_connections_active{local_address="192.168.0.57",local_port="3306",proto="TCP",remote_address="192.168.84.22",remote_port="3306"} 0 +# HELP node_ipvs_backend_connections_inactive The current inactive connections by local and remote address. +# TYPE node_ipvs_backend_connections_inactive gauge +node_ipvs_backend_connections_inactive{local_address="192.168.0.22",local_port="3306",proto="TCP",remote_address="192.168.82.22",remote_port="3306"} 2 +node_ipvs_backend_connections_inactive{local_address="192.168.0.22",local_port="3306",proto="TCP",remote_address="192.168.83.21",remote_port="3306"} 1 +node_ipvs_backend_connections_inactive{local_address="192.168.0.22",local_port="3306",proto="TCP",remote_address="192.168.83.24",remote_port="3306"} 2 +node_ipvs_backend_connections_inactive{local_address="192.168.0.55",local_port="3306",proto="TCP",remote_address="192.168.49.32",remote_port="3306"} 0 +node_ipvs_backend_connections_inactive{local_address="192.168.0.55",local_port="3306",proto="TCP",remote_address="192.168.50.26",remote_port="3306"} 0 +node_ipvs_backend_connections_inactive{local_address="192.168.0.57",local_port="3306",proto="TCP",remote_address="192.168.50.21",remote_port="3306"} 0 +node_ipvs_backend_connections_inactive{local_address="192.168.0.57",local_port="3306",proto="TCP",remote_address="192.168.82.21",remote_port="3306"} 0 +node_ipvs_backend_connections_inactive{local_address="192.168.0.57",local_port="3306",proto="TCP",remote_address="192.168.84.22",remote_port="3306"} 0 +# HELP node_ipvs_backend_weight The current backend weight by local and remote address. +# TYPE node_ipvs_backend_weight gauge +node_ipvs_backend_weight{local_address="192.168.0.22",local_port="3306",proto="TCP",remote_address="192.168.82.22",remote_port="3306"} 100 +node_ipvs_backend_weight{local_address="192.168.0.22",local_port="3306",proto="TCP",remote_address="192.168.83.21",remote_port="3306"} 100 +node_ipvs_backend_weight{local_address="192.168.0.22",local_port="3306",proto="TCP",remote_address="192.168.83.24",remote_port="3306"} 100 +node_ipvs_backend_weight{local_address="192.168.0.55",local_port="3306",proto="TCP",remote_address="192.168.49.32",remote_port="3306"} 100 +node_ipvs_backend_weight{local_address="192.168.0.55",local_port="3306",proto="TCP",remote_address="192.168.50.26",remote_port="3306"} 0 +node_ipvs_backend_weight{local_address="192.168.0.57",local_port="3306",proto="TCP",remote_address="192.168.50.21",remote_port="3306"} 100 +node_ipvs_backend_weight{local_address="192.168.0.57",local_port="3306",proto="TCP",remote_address="192.168.82.21",remote_port="3306"} 100 +node_ipvs_backend_weight{local_address="192.168.0.57",local_port="3306",proto="TCP",remote_address="192.168.84.22",remote_port="3306"} 0 +# HELP node_ipvs_connections_total The total number of connections made. +# TYPE node_ipvs_connections_total counter +node_ipvs_connections_total 2.3765872e+07 +# HELP node_ipvs_incoming_bytes_total The total amount of incoming data. +# TYPE node_ipvs_incoming_bytes_total counter +node_ipvs_incoming_bytes_total 8.9991519156915e+13 +# HELP node_ipvs_incoming_packets_total The total number of incoming packets. +# TYPE node_ipvs_incoming_packets_total counter +node_ipvs_incoming_packets_total 3.811989221e+09 +# HELP node_ipvs_outgoing_bytes_total The total amount of outgoing data. +# TYPE node_ipvs_outgoing_bytes_total counter +node_ipvs_outgoing_bytes_total 0 +# HELP node_ipvs_outgoing_packets_total The total number of outgoing packets. +# TYPE node_ipvs_outgoing_packets_total counter +node_ipvs_outgoing_packets_total 0 diff --git a/collector/fixtures/net/ip_vs_stats b/collector/fixtures/net/ip_vs_stats new file mode 100644 index 00000000..c00724e0 --- /dev/null +++ b/collector/fixtures/net/ip_vs_stats @@ -0,0 +1,6 @@ + Total Incoming Outgoing Incoming Outgoing + Conns Packets Packets Bytes Bytes + 16AA370 E33656E5 0 51D8C8883AB3 0 + + Conns/s Pkts/s Pkts/s Bytes/s Bytes/s + 4 1FB3C 0 1282A8F 0 diff --git a/collector/ipvs.go b/collector/ipvs.go new file mode 100644 index 00000000..ac377069 --- /dev/null +++ b/collector/ipvs.go @@ -0,0 +1,171 @@ +// +build !noipvs + +package collector + +import ( + "fmt" + "strconv" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/procfs" +) + +type ipvsCollector struct { + Collector + fs procfs.FS + backendConnectionsActive, backendConnectionsInact, backendWeight *prometheus.GaugeVec + connections, incomingPackets, outgoingPackets, incomingBytes, outgoingBytes prometheus.Counter +} + +func init() { + Factories["ipvs"] = NewIPVSCollector +} + +// NewIPVSCollector sets up a new collector for IPVS metrics. It accepts the +// "procfs" config parameter to override the default proc location (/proc). +func NewIPVSCollector(config Config) (Collector, error) { + return newIPVSCollector(config) +} + +func newIPVSCollector(config Config) (*ipvsCollector, error) { + var ( + ipvsBackendLabelNames = []string{ + "local_address", + "local_port", + "remote_address", + "remote_port", + "proto", + } + c ipvsCollector + subsystem string + err error + ) + + if p, ok := config.Config["procfs"]; !ok { + c.fs, err = procfs.NewFS(procfs.DefaultMountPoint) + } else { + c.fs, err = procfs.NewFS(p) + } + if err != nil { + return nil, err + } + if s, ok := config.Config["ipvs_subsystem"]; ok { + subsystem = s + } else { + subsystem = "ipvs" + } + + c.connections = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: subsystem, + Name: "connections_total", + Help: "The total number of connections made.", + }, + ) + c.incomingPackets = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: subsystem, + Name: "incoming_packets_total", + Help: "The total number of incoming packets.", + }, + ) + c.outgoingPackets = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: subsystem, + Name: "outgoing_packets_total", + Help: "The total number of outgoing packets.", + }, + ) + c.incomingBytes = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: subsystem, + Name: "incoming_bytes_total", + Help: "The total amount of incoming data.", + }, + ) + c.outgoingBytes = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: subsystem, + Name: "outgoing_bytes_total", + Help: "The total amount of outgoing data.", + }, + ) + + c.backendConnectionsActive = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: subsystem, + Name: "backend_connections_active", + Help: "The current active connections by local and remote address.", + }, + ipvsBackendLabelNames, + ) + c.backendConnectionsInact = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: subsystem, + Name: "backend_connections_inactive", + Help: "The current inactive connections by local and remote address.", + }, + ipvsBackendLabelNames, + ) + c.backendWeight = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: subsystem, + Name: "backend_weight", + Help: "The current backend weight by local and remote address.", + }, + ipvsBackendLabelNames, + ) + + return &c, nil +} + +func (c *ipvsCollector) Update(ch chan<- prometheus.Metric) error { + ipvsStats, err := c.fs.NewIPVSStats() + if err != nil { + return fmt.Errorf("could not get IPVS stats: %s", err) + } + + c.connections.Set(float64(ipvsStats.Connections)) + c.incomingPackets.Set(float64(ipvsStats.IncomingPackets)) + c.outgoingPackets.Set(float64(ipvsStats.OutgoingPackets)) + c.incomingBytes.Set(float64(ipvsStats.IncomingBytes)) + c.outgoingBytes.Set(float64(ipvsStats.OutgoingBytes)) + + c.connections.Collect(ch) + c.incomingPackets.Collect(ch) + c.outgoingPackets.Collect(ch) + c.incomingBytes.Collect(ch) + c.outgoingBytes.Collect(ch) + + backendStats, err := c.fs.NewIPVSBackendStatus() + if err != nil { + return fmt.Errorf("could not get backend status: %s", err) + } + + for _, backend := range backendStats { + labelValues := []string{ + backend.LocalAddress.String(), + strconv.FormatUint(uint64(backend.LocalPort), 10), + backend.RemoteAddress.String(), + strconv.FormatUint(uint64(backend.RemotePort), 10), + backend.Proto, + } + c.backendConnectionsActive.WithLabelValues(labelValues...).Set(float64(backend.ActiveConn)) + c.backendConnectionsInact.WithLabelValues(labelValues...).Set(float64(backend.InactConn)) + c.backendWeight.WithLabelValues(labelValues...).Set(float64(backend.Weight)) + } + + c.backendConnectionsActive.Collect(ch) + c.backendConnectionsInact.Collect(ch) + c.backendWeight.Collect(ch) + + return nil +} diff --git a/collector/ipvs_test.go b/collector/ipvs_test.go new file mode 100644 index 00000000..65fd4e12 --- /dev/null +++ b/collector/ipvs_test.go @@ -0,0 +1,205 @@ +package collector + +import ( + "io/ioutil" + "net" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/procfs" +) + +var ( + expectedIPVSStats = procfs.IPVSStats{ + Connections: 23765872, + IncomingPackets: 3811989221, + OutgoingPackets: 0, + IncomingBytes: 89991519156915, + OutgoingBytes: 0, + } + expectedIPVSBackendStatuses = []procfs.IPVSBackendStatus{ + procfs.IPVSBackendStatus{ + LocalAddress: net.ParseIP("192.168.0.22"), + LocalPort: 3306, + RemoteAddress: net.ParseIP("192.168.82.22"), + RemotePort: 3306, + Proto: "TCP", + Weight: 100, + ActiveConn: 248, + InactConn: 2, + }, + procfs.IPVSBackendStatus{ + LocalAddress: net.ParseIP("192.168.0.22"), + LocalPort: 3306, + RemoteAddress: net.ParseIP("192.168.83.24"), + RemotePort: 3306, + Proto: "TCP", + Weight: 100, + ActiveConn: 248, + InactConn: 2, + }, + procfs.IPVSBackendStatus{ + LocalAddress: net.ParseIP("192.168.0.22"), + LocalPort: 3306, + RemoteAddress: net.ParseIP("192.168.83.21"), + RemotePort: 3306, + Proto: "TCP", + Weight: 100, + ActiveConn: 248, + InactConn: 1, + }, + procfs.IPVSBackendStatus{ + LocalAddress: net.ParseIP("192.168.0.57"), + LocalPort: 3306, + RemoteAddress: net.ParseIP("192.168.84.22"), + RemotePort: 3306, + Proto: "TCP", + Weight: 0, + ActiveConn: 0, + InactConn: 0, + }, + procfs.IPVSBackendStatus{ + LocalAddress: net.ParseIP("192.168.0.57"), + LocalPort: 3306, + RemoteAddress: net.ParseIP("192.168.82.21"), + RemotePort: 3306, + Proto: "TCP", + Weight: 100, + ActiveConn: 1499, + InactConn: 0, + }, + procfs.IPVSBackendStatus{ + LocalAddress: net.ParseIP("192.168.0.57"), + LocalPort: 3306, + RemoteAddress: net.ParseIP("192.168.50.21"), + RemotePort: 3306, + Proto: "TCP", + Weight: 100, + ActiveConn: 1498, + InactConn: 0, + }, + procfs.IPVSBackendStatus{ + LocalAddress: net.ParseIP("192.168.0.55"), + LocalPort: 3306, + RemoteAddress: net.ParseIP("192.168.50.26"), + RemotePort: 3306, + Proto: "TCP", + Weight: 0, + ActiveConn: 0, + InactConn: 0, + }, + procfs.IPVSBackendStatus{ + LocalAddress: net.ParseIP("192.168.0.55"), + LocalPort: 3306, + RemoteAddress: net.ParseIP("192.168.49.32"), + RemotePort: 3306, + Proto: "TCP", + Weight: 100, + ActiveConn: 0, + InactConn: 0, + }, + } +) + +func TestIPVSCollector(t *testing.T) { + collector, err := newIPVSCollector(Config{Config: map[string]string{"procfs": "fixtures"}}) + if err != nil { + t.Fatal(err) + } + sink := make(chan prometheus.Metric) + go func() { + for { + <-sink + } + }() + + err = collector.Update(sink) + if err != nil { + t.Fatal(err) + } + + for _, expect := range expectedIPVSBackendStatuses { + labels := prometheus.Labels{ + "local_address": expect.LocalAddress.String(), + "local_port": strconv.FormatUint(uint64(expect.LocalPort), 10), + "remote_address": expect.RemoteAddress.String(), + "remote_port": strconv.FormatUint(uint64(expect.RemotePort), 10), + "proto": expect.Proto, + } + // TODO: Pending prometheus/client_golang#58, check the actual numbers + _, err = collector.backendConnectionsActive.GetMetricWith(labels) + if err != nil { + t.Errorf("Missing active connections metric for label combination: %+v", labels) + } + _, err = collector.backendConnectionsInact.GetMetricWith(labels) + if err != nil { + t.Errorf("Missing inactive connections metric for label combination: %+v", labels) + } + _, err = collector.backendWeight.GetMetricWith(labels) + if err != nil { + t.Errorf("Missing weight metric for label combination: %+v", labels) + } + } +} + +// mock collector +type miniCollector struct { + c Collector +} + +func (c miniCollector) Collect(ch chan<- prometheus.Metric) { + c.c.Update(ch) +} + +func (c miniCollector) Describe(ch chan<- *prometheus.Desc) { + prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "fake", + Subsystem: "fake", + Name: "fake", + Help: "fake", + }).Describe(ch) +} + +func TestIPVSCollectorResponse(t *testing.T) { + collector, err := NewIPVSCollector(Config{Config: map[string]string{"procfs": "fixtures"}}) + if err != nil { + t.Fatal(err) + } + prometheus.MustRegister(miniCollector{c: collector}) + + rw := httptest.NewRecorder() + prometheus.Handler().ServeHTTP(rw, &http.Request{}) + + metricsFile := "fixtures/net/ip_vs_result.txt" + wantMetrics, err := ioutil.ReadFile(metricsFile) + if err != nil { + t.Fatalf("unable to read input test file %s: %s", metricsFile, err) + } + + wantLines := strings.Split(string(wantMetrics), "\n") + gotLines := strings.Split(string(rw.Body.String()), "\n") + gotLinesIdx := 0 + + // Until the Prometheus Go client library offers better testability + // (https://github.com/prometheus/client_golang/issues/58), we simply compare + // verbatim text-format metrics outputs, but ignore any lines we don't have + // in the fixture. Put differently, we are only testing that each line from + // the fixture is present, in the order given. +wantLoop: + for _, want := range wantLines { + for _, got := range gotLines[gotLinesIdx:] { + if want == got { + // this is a line we are interested in, and it is correct + continue wantLoop + } else { + gotLinesIdx++ + } + } + // if this point is reached, the line we want was missing + t.Fatalf("Missing expected output line(s), first missing line is %s", want) + } +}