Add IPVS stats collector.
This collector exposes IPVS status from `/proc/net/ip_vs` and stats from `/proc/net/ip_vs_stats`. Note that it is _not_ tested with IPv6, YMMV.
This commit is contained in:
Matthias Rampke
parent
d1822e4749
commit
37c7e5e60f
|
|
@ -50,6 +50,7 @@ Name | Description
|
|||
bonding | Exposes the number of configured and active slaves of Linux bonding interfaces.
|
||||
gmond | Exposes statistics from Ganglia.
|
||||
interrupts | Exposes detailed interrupts statistics from `/proc/interrupts`.
|
||||
ipvs | Exposes IPVS status from `/proc/net/ip_vs` and stats from `/proc/net/ip_vs_stats`.
|
||||
lastlogin | Exposes the last time there was a login.
|
||||
megacli | Exposes RAID statistics from MegaCLI.
|
||||
ntp | Exposes time drift from an NTP server.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,14 @@
|
|||
IP Virtual Server version 1.2.1 (size=4096)
|
||||
Prot LocalAddress:Port Scheduler Flags
|
||||
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
|
||||
TCP C0A80016:0CEA wlc
|
||||
-> C0A85216:0CEA Tunnel 100 248 2
|
||||
-> C0A85318:0CEA Tunnel 100 248 2
|
||||
-> C0A85315:0CEA Tunnel 100 248 1
|
||||
TCP C0A80039:0CEA wlc
|
||||
-> C0A85416:0CEA Tunnel 0 0 0
|
||||
-> C0A85215:0CEA Tunnel 100 1499 0
|
||||
-> C0A83215:0CEA Tunnel 100 1498 0
|
||||
TCP C0A80037:0CEA wlc
|
||||
-> C0A8321A:0CEA Tunnel 0 0 0
|
||||
-> C0A83120:0CEA Tunnel 100 0 0
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
# HELP node_ipvs_backend_connections_active The current active connections by local and remote address.
|
||||
# TYPE node_ipvs_backend_connections_active gauge
|
||||
node_ipvs_backend_connections_active{local_address="192.168.0.22",local_port="3306",proto="TCP",remote_address="192.168.82.22",remote_port="3306"} 248
|
||||
node_ipvs_backend_connections_active{local_address="192.168.0.22",local_port="3306",proto="TCP",remote_address="192.168.83.21",remote_port="3306"} 248
|
||||
node_ipvs_backend_connections_active{local_address="192.168.0.22",local_port="3306",proto="TCP",remote_address="192.168.83.24",remote_port="3306"} 248
|
||||
node_ipvs_backend_connections_active{local_address="192.168.0.55",local_port="3306",proto="TCP",remote_address="192.168.49.32",remote_port="3306"} 0
|
||||
node_ipvs_backend_connections_active{local_address="192.168.0.55",local_port="3306",proto="TCP",remote_address="192.168.50.26",remote_port="3306"} 0
|
||||
node_ipvs_backend_connections_active{local_address="192.168.0.57",local_port="3306",proto="TCP",remote_address="192.168.50.21",remote_port="3306"} 1498
|
||||
node_ipvs_backend_connections_active{local_address="192.168.0.57",local_port="3306",proto="TCP",remote_address="192.168.82.21",remote_port="3306"} 1499
|
||||
node_ipvs_backend_connections_active{local_address="192.168.0.57",local_port="3306",proto="TCP",remote_address="192.168.84.22",remote_port="3306"} 0
|
||||
# HELP node_ipvs_backend_connections_inactive The current inactive connections by local and remote address.
|
||||
# TYPE node_ipvs_backend_connections_inactive gauge
|
||||
node_ipvs_backend_connections_inactive{local_address="192.168.0.22",local_port="3306",proto="TCP",remote_address="192.168.82.22",remote_port="3306"} 2
|
||||
node_ipvs_backend_connections_inactive{local_address="192.168.0.22",local_port="3306",proto="TCP",remote_address="192.168.83.21",remote_port="3306"} 1
|
||||
node_ipvs_backend_connections_inactive{local_address="192.168.0.22",local_port="3306",proto="TCP",remote_address="192.168.83.24",remote_port="3306"} 2
|
||||
node_ipvs_backend_connections_inactive{local_address="192.168.0.55",local_port="3306",proto="TCP",remote_address="192.168.49.32",remote_port="3306"} 0
|
||||
node_ipvs_backend_connections_inactive{local_address="192.168.0.55",local_port="3306",proto="TCP",remote_address="192.168.50.26",remote_port="3306"} 0
|
||||
node_ipvs_backend_connections_inactive{local_address="192.168.0.57",local_port="3306",proto="TCP",remote_address="192.168.50.21",remote_port="3306"} 0
|
||||
node_ipvs_backend_connections_inactive{local_address="192.168.0.57",local_port="3306",proto="TCP",remote_address="192.168.82.21",remote_port="3306"} 0
|
||||
node_ipvs_backend_connections_inactive{local_address="192.168.0.57",local_port="3306",proto="TCP",remote_address="192.168.84.22",remote_port="3306"} 0
|
||||
# HELP node_ipvs_backend_weight The current backend weight by local and remote address.
|
||||
# TYPE node_ipvs_backend_weight gauge
|
||||
node_ipvs_backend_weight{local_address="192.168.0.22",local_port="3306",proto="TCP",remote_address="192.168.82.22",remote_port="3306"} 100
|
||||
node_ipvs_backend_weight{local_address="192.168.0.22",local_port="3306",proto="TCP",remote_address="192.168.83.21",remote_port="3306"} 100
|
||||
node_ipvs_backend_weight{local_address="192.168.0.22",local_port="3306",proto="TCP",remote_address="192.168.83.24",remote_port="3306"} 100
|
||||
node_ipvs_backend_weight{local_address="192.168.0.55",local_port="3306",proto="TCP",remote_address="192.168.49.32",remote_port="3306"} 100
|
||||
node_ipvs_backend_weight{local_address="192.168.0.55",local_port="3306",proto="TCP",remote_address="192.168.50.26",remote_port="3306"} 0
|
||||
node_ipvs_backend_weight{local_address="192.168.0.57",local_port="3306",proto="TCP",remote_address="192.168.50.21",remote_port="3306"} 100
|
||||
node_ipvs_backend_weight{local_address="192.168.0.57",local_port="3306",proto="TCP",remote_address="192.168.82.21",remote_port="3306"} 100
|
||||
node_ipvs_backend_weight{local_address="192.168.0.57",local_port="3306",proto="TCP",remote_address="192.168.84.22",remote_port="3306"} 0
|
||||
# HELP node_ipvs_connections_total The total number of connections made.
|
||||
# TYPE node_ipvs_connections_total counter
|
||||
node_ipvs_connections_total 2.3765872e+07
|
||||
# HELP node_ipvs_incoming_bytes_total The total amount of incoming data.
|
||||
# TYPE node_ipvs_incoming_bytes_total counter
|
||||
node_ipvs_incoming_bytes_total 8.9991519156915e+13
|
||||
# HELP node_ipvs_incoming_packets_total The total number of incoming packets.
|
||||
# TYPE node_ipvs_incoming_packets_total counter
|
||||
node_ipvs_incoming_packets_total 3.811989221e+09
|
||||
# HELP node_ipvs_outgoing_bytes_total The total amount of outgoing data.
|
||||
# TYPE node_ipvs_outgoing_bytes_total counter
|
||||
node_ipvs_outgoing_bytes_total 0
|
||||
# HELP node_ipvs_outgoing_packets_total The total number of outgoing packets.
|
||||
# TYPE node_ipvs_outgoing_packets_total counter
|
||||
node_ipvs_outgoing_packets_total 0
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
Total Incoming Outgoing Incoming Outgoing
|
||||
Conns Packets Packets Bytes Bytes
|
||||
16AA370 E33656E5 0 51D8C8883AB3 0
|
||||
|
||||
Conns/s Pkts/s Pkts/s Bytes/s Bytes/s
|
||||
4 1FB3C 0 1282A8F 0
|
||||
|
|
@ -0,0 +1,171 @@
|
|||
// +build !noipvs
|
||||
|
||||
package collector
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/procfs"
|
||||
)
|
||||
|
||||
type ipvsCollector struct {
|
||||
Collector
|
||||
fs procfs.FS
|
||||
backendConnectionsActive, backendConnectionsInact, backendWeight *prometheus.GaugeVec
|
||||
connections, incomingPackets, outgoingPackets, incomingBytes, outgoingBytes prometheus.Counter
|
||||
}
|
||||
|
||||
func init() {
|
||||
Factories["ipvs"] = NewIPVSCollector
|
||||
}
|
||||
|
||||
// NewIPVSCollector sets up a new collector for IPVS metrics. It accepts the
|
||||
// "procfs" config parameter to override the default proc location (/proc).
|
||||
func NewIPVSCollector(config Config) (Collector, error) {
|
||||
return newIPVSCollector(config)
|
||||
}
|
||||
|
||||
func newIPVSCollector(config Config) (*ipvsCollector, error) {
|
||||
var (
|
||||
ipvsBackendLabelNames = []string{
|
||||
"local_address",
|
||||
"local_port",
|
||||
"remote_address",
|
||||
"remote_port",
|
||||
"proto",
|
||||
}
|
||||
c ipvsCollector
|
||||
subsystem string
|
||||
err error
|
||||
)
|
||||
|
||||
if p, ok := config.Config["procfs"]; !ok {
|
||||
c.fs, err = procfs.NewFS(procfs.DefaultMountPoint)
|
||||
} else {
|
||||
c.fs, err = procfs.NewFS(p)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if s, ok := config.Config["ipvs_subsystem"]; ok {
|
||||
subsystem = s
|
||||
} else {
|
||||
subsystem = "ipvs"
|
||||
}
|
||||
|
||||
c.connections = prometheus.NewCounter(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: subsystem,
|
||||
Name: "connections_total",
|
||||
Help: "The total number of connections made.",
|
||||
},
|
||||
)
|
||||
c.incomingPackets = prometheus.NewCounter(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: subsystem,
|
||||
Name: "incoming_packets_total",
|
||||
Help: "The total number of incoming packets.",
|
||||
},
|
||||
)
|
||||
c.outgoingPackets = prometheus.NewCounter(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: subsystem,
|
||||
Name: "outgoing_packets_total",
|
||||
Help: "The total number of outgoing packets.",
|
||||
},
|
||||
)
|
||||
c.incomingBytes = prometheus.NewCounter(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: subsystem,
|
||||
Name: "incoming_bytes_total",
|
||||
Help: "The total amount of incoming data.",
|
||||
},
|
||||
)
|
||||
c.outgoingBytes = prometheus.NewCounter(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: subsystem,
|
||||
Name: "outgoing_bytes_total",
|
||||
Help: "The total amount of outgoing data.",
|
||||
},
|
||||
)
|
||||
|
||||
c.backendConnectionsActive = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: subsystem,
|
||||
Name: "backend_connections_active",
|
||||
Help: "The current active connections by local and remote address.",
|
||||
},
|
||||
ipvsBackendLabelNames,
|
||||
)
|
||||
c.backendConnectionsInact = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: subsystem,
|
||||
Name: "backend_connections_inactive",
|
||||
Help: "The current inactive connections by local and remote address.",
|
||||
},
|
||||
ipvsBackendLabelNames,
|
||||
)
|
||||
c.backendWeight = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: subsystem,
|
||||
Name: "backend_weight",
|
||||
Help: "The current backend weight by local and remote address.",
|
||||
},
|
||||
ipvsBackendLabelNames,
|
||||
)
|
||||
|
||||
return &c, nil
|
||||
}
|
||||
|
||||
func (c *ipvsCollector) Update(ch chan<- prometheus.Metric) error {
|
||||
ipvsStats, err := c.fs.NewIPVSStats()
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not get IPVS stats: %s", err)
|
||||
}
|
||||
|
||||
c.connections.Set(float64(ipvsStats.Connections))
|
||||
c.incomingPackets.Set(float64(ipvsStats.IncomingPackets))
|
||||
c.outgoingPackets.Set(float64(ipvsStats.OutgoingPackets))
|
||||
c.incomingBytes.Set(float64(ipvsStats.IncomingBytes))
|
||||
c.outgoingBytes.Set(float64(ipvsStats.OutgoingBytes))
|
||||
|
||||
c.connections.Collect(ch)
|
||||
c.incomingPackets.Collect(ch)
|
||||
c.outgoingPackets.Collect(ch)
|
||||
c.incomingBytes.Collect(ch)
|
||||
c.outgoingBytes.Collect(ch)
|
||||
|
||||
backendStats, err := c.fs.NewIPVSBackendStatus()
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not get backend status: %s", err)
|
||||
}
|
||||
|
||||
for _, backend := range backendStats {
|
||||
labelValues := []string{
|
||||
backend.LocalAddress.String(),
|
||||
strconv.FormatUint(uint64(backend.LocalPort), 10),
|
||||
backend.RemoteAddress.String(),
|
||||
strconv.FormatUint(uint64(backend.RemotePort), 10),
|
||||
backend.Proto,
|
||||
}
|
||||
c.backendConnectionsActive.WithLabelValues(labelValues...).Set(float64(backend.ActiveConn))
|
||||
c.backendConnectionsInact.WithLabelValues(labelValues...).Set(float64(backend.InactConn))
|
||||
c.backendWeight.WithLabelValues(labelValues...).Set(float64(backend.Weight))
|
||||
}
|
||||
|
||||
c.backendConnectionsActive.Collect(ch)
|
||||
c.backendConnectionsInact.Collect(ch)
|
||||
c.backendWeight.Collect(ch)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,205 @@
|
|||
package collector
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/procfs"
|
||||
)
|
||||
|
||||
var (
|
||||
expectedIPVSStats = procfs.IPVSStats{
|
||||
Connections: 23765872,
|
||||
IncomingPackets: 3811989221,
|
||||
OutgoingPackets: 0,
|
||||
IncomingBytes: 89991519156915,
|
||||
OutgoingBytes: 0,
|
||||
}
|
||||
expectedIPVSBackendStatuses = []procfs.IPVSBackendStatus{
|
||||
procfs.IPVSBackendStatus{
|
||||
LocalAddress: net.ParseIP("192.168.0.22"),
|
||||
LocalPort: 3306,
|
||||
RemoteAddress: net.ParseIP("192.168.82.22"),
|
||||
RemotePort: 3306,
|
||||
Proto: "TCP",
|
||||
Weight: 100,
|
||||
ActiveConn: 248,
|
||||
InactConn: 2,
|
||||
},
|
||||
procfs.IPVSBackendStatus{
|
||||
LocalAddress: net.ParseIP("192.168.0.22"),
|
||||
LocalPort: 3306,
|
||||
RemoteAddress: net.ParseIP("192.168.83.24"),
|
||||
RemotePort: 3306,
|
||||
Proto: "TCP",
|
||||
Weight: 100,
|
||||
ActiveConn: 248,
|
||||
InactConn: 2,
|
||||
},
|
||||
procfs.IPVSBackendStatus{
|
||||
LocalAddress: net.ParseIP("192.168.0.22"),
|
||||
LocalPort: 3306,
|
||||
RemoteAddress: net.ParseIP("192.168.83.21"),
|
||||
RemotePort: 3306,
|
||||
Proto: "TCP",
|
||||
Weight: 100,
|
||||
ActiveConn: 248,
|
||||
InactConn: 1,
|
||||
},
|
||||
procfs.IPVSBackendStatus{
|
||||
LocalAddress: net.ParseIP("192.168.0.57"),
|
||||
LocalPort: 3306,
|
||||
RemoteAddress: net.ParseIP("192.168.84.22"),
|
||||
RemotePort: 3306,
|
||||
Proto: "TCP",
|
||||
Weight: 0,
|
||||
ActiveConn: 0,
|
||||
InactConn: 0,
|
||||
},
|
||||
procfs.IPVSBackendStatus{
|
||||
LocalAddress: net.ParseIP("192.168.0.57"),
|
||||
LocalPort: 3306,
|
||||
RemoteAddress: net.ParseIP("192.168.82.21"),
|
||||
RemotePort: 3306,
|
||||
Proto: "TCP",
|
||||
Weight: 100,
|
||||
ActiveConn: 1499,
|
||||
InactConn: 0,
|
||||
},
|
||||
procfs.IPVSBackendStatus{
|
||||
LocalAddress: net.ParseIP("192.168.0.57"),
|
||||
LocalPort: 3306,
|
||||
RemoteAddress: net.ParseIP("192.168.50.21"),
|
||||
RemotePort: 3306,
|
||||
Proto: "TCP",
|
||||
Weight: 100,
|
||||
ActiveConn: 1498,
|
||||
InactConn: 0,
|
||||
},
|
||||
procfs.IPVSBackendStatus{
|
||||
LocalAddress: net.ParseIP("192.168.0.55"),
|
||||
LocalPort: 3306,
|
||||
RemoteAddress: net.ParseIP("192.168.50.26"),
|
||||
RemotePort: 3306,
|
||||
Proto: "TCP",
|
||||
Weight: 0,
|
||||
ActiveConn: 0,
|
||||
InactConn: 0,
|
||||
},
|
||||
procfs.IPVSBackendStatus{
|
||||
LocalAddress: net.ParseIP("192.168.0.55"),
|
||||
LocalPort: 3306,
|
||||
RemoteAddress: net.ParseIP("192.168.49.32"),
|
||||
RemotePort: 3306,
|
||||
Proto: "TCP",
|
||||
Weight: 100,
|
||||
ActiveConn: 0,
|
||||
InactConn: 0,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
func TestIPVSCollector(t *testing.T) {
|
||||
collector, err := newIPVSCollector(Config{Config: map[string]string{"procfs": "fixtures"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sink := make(chan prometheus.Metric)
|
||||
go func() {
|
||||
for {
|
||||
<-sink
|
||||
}
|
||||
}()
|
||||
|
||||
err = collector.Update(sink)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for _, expect := range expectedIPVSBackendStatuses {
|
||||
labels := prometheus.Labels{
|
||||
"local_address": expect.LocalAddress.String(),
|
||||
"local_port": strconv.FormatUint(uint64(expect.LocalPort), 10),
|
||||
"remote_address": expect.RemoteAddress.String(),
|
||||
"remote_port": strconv.FormatUint(uint64(expect.RemotePort), 10),
|
||||
"proto": expect.Proto,
|
||||
}
|
||||
// TODO: Pending prometheus/client_golang#58, check the actual numbers
|
||||
_, err = collector.backendConnectionsActive.GetMetricWith(labels)
|
||||
if err != nil {
|
||||
t.Errorf("Missing active connections metric for label combination: %+v", labels)
|
||||
}
|
||||
_, err = collector.backendConnectionsInact.GetMetricWith(labels)
|
||||
if err != nil {
|
||||
t.Errorf("Missing inactive connections metric for label combination: %+v", labels)
|
||||
}
|
||||
_, err = collector.backendWeight.GetMetricWith(labels)
|
||||
if err != nil {
|
||||
t.Errorf("Missing weight metric for label combination: %+v", labels)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// mock collector
|
||||
type miniCollector struct {
|
||||
c Collector
|
||||
}
|
||||
|
||||
func (c miniCollector) Collect(ch chan<- prometheus.Metric) {
|
||||
c.c.Update(ch)
|
||||
}
|
||||
|
||||
func (c miniCollector) Describe(ch chan<- *prometheus.Desc) {
|
||||
prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: "fake",
|
||||
Subsystem: "fake",
|
||||
Name: "fake",
|
||||
Help: "fake",
|
||||
}).Describe(ch)
|
||||
}
|
||||
|
||||
func TestIPVSCollectorResponse(t *testing.T) {
|
||||
collector, err := NewIPVSCollector(Config{Config: map[string]string{"procfs": "fixtures"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prometheus.MustRegister(miniCollector{c: collector})
|
||||
|
||||
rw := httptest.NewRecorder()
|
||||
prometheus.Handler().ServeHTTP(rw, &http.Request{})
|
||||
|
||||
metricsFile := "fixtures/net/ip_vs_result.txt"
|
||||
wantMetrics, err := ioutil.ReadFile(metricsFile)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to read input test file %s: %s", metricsFile, err)
|
||||
}
|
||||
|
||||
wantLines := strings.Split(string(wantMetrics), "\n")
|
||||
gotLines := strings.Split(string(rw.Body.String()), "\n")
|
||||
gotLinesIdx := 0
|
||||
|
||||
// Until the Prometheus Go client library offers better testability
|
||||
// (https://github.com/prometheus/client_golang/issues/58), we simply compare
|
||||
// verbatim text-format metrics outputs, but ignore any lines we don't have
|
||||
// in the fixture. Put differently, we are only testing that each line from
|
||||
// the fixture is present, in the order given.
|
||||
wantLoop:
|
||||
for _, want := range wantLines {
|
||||
for _, got := range gotLines[gotLinesIdx:] {
|
||||
if want == got {
|
||||
// this is a line we are interested in, and it is correct
|
||||
continue wantLoop
|
||||
} else {
|
||||
gotLinesIdx++
|
||||
}
|
||||
}
|
||||
// if this point is reached, the line we want was missing
|
||||
t.Fatalf("Missing expected output line(s), first missing line is %s", want)
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue