From cc479e019a96df50f86ce5cf33f88d04c25ea9a8 Mon Sep 17 00:00:00 2001 From: Yue Zhu Date: Mon, 23 Sep 2019 17:51:50 -0400 Subject: [PATCH] Add metric ceph_pg_objects_recovered_total --- collectors/conn.go | 84 ++++- collectors/osd.go | 48 ++- collectors/osd_test.go | 809 +++++++++++++++++++++++------------------ 3 files changed, 563 insertions(+), 378 deletions(-) diff --git a/collectors/conn.go b/collectors/conn.go index 6dfeb5d..c32b4ed 100644 --- a/collectors/conn.go +++ b/collectors/conn.go @@ -14,7 +14,12 @@ package collectors -import "github.com/ceph/go-ceph/rados" +import ( + "encoding/json" + "fmt" + + "github.com/ceph/go-ceph/rados" +) // Conn interface implements only necessary methods that are used // in this repository of *rados.Conn. This keeps rest of the implementation @@ -36,16 +41,27 @@ var _ Conn = &rados.Conn{} // each individual collectors becomes a lot easier after that. type NoopConn struct { output string + cmdOut map[string]string } // The stub we use for testing should also satisfy the interface properties. var _ Conn = &NoopConn{} -// NewNoopConn returns an instance of *NoopConn. The string that we want -// outputted at the end of the command we issue to ceph, should be -// specified in the only input parameter. +// NewNoopConn returns an instance of *NoopConn. The string that we want output +// at the end of the command we issue to Ceph is fixed and should be specified +// in the only input parameter. func NewNoopConn(output string) *NoopConn { - return &NoopConn{output} + return &NoopConn{ + output: output, + cmdOut: make(map[string]string), + } +} + +// NewNoopConnWithCmdOut returns an instance of *NoopConn. The string that we +// want output at the end of the command we issue to Ceph can be various and +// should be specified by the map in the only input parameter. +func NewNoopConnWithCmdOut(cmdOut map[string]string) *NoopConn { + return &NoopConn{cmdOut: cmdOut} } // ReadDefaultConfigFile does not need to return an error. It satisfies @@ -65,12 +81,66 @@ func (n *NoopConn) Shutdown() {} // MonCommand returns the provided output string to NoopConn as is, making // it seem like it actually ran something and produced that string as a result. -func (n *NoopConn) MonCommand(_ []byte) ([]byte, string, error) { +func (n *NoopConn) MonCommand(args []byte) ([]byte, string, error) { + // Unmarshal the input command and see if we need to intercept + cmd := map[string]interface{}{} + err := json.Unmarshal(args, &cmd) + if err != nil { + return []byte(n.output), "", err + } + + // Intercept and mock the output + switch prefix := cmd["prefix"]; prefix { + case "pg dump": + dc, ok := cmd["dumpcontents"].([]interface{}) + if !ok || len(dc) == 0 { + break + } + + switch dc[0] { + case "pgs_brief": + return []byte(n.cmdOut["ceph pg dump pgs_brief"]), "", nil + } + + case "osd tree": + st, ok := cmd["states"].([]interface{}) + if !ok || len(st) == 0 { + break + } + + switch st[0] { + case "down": + return []byte(n.cmdOut["ceph osd tree down"]), "", nil + } + + case "osd df": + return []byte(n.cmdOut["ceph osd df"]), "", nil + + case "osd perf": + return []byte(n.cmdOut["ceph osd perf"]), "", nil + + case "osd dump": + return []byte(n.cmdOut["ceph osd dump"]), "", nil + } + return []byte(n.output), "", nil } // PGCommand returns the provided output string to NoopConn as is, making // it seem like it actually ran something and produced that string as a result. -func (n *NoopConn) PGCommand(_, _ []byte) ([]byte, string, error) { +func (n *NoopConn) PGCommand(pgid, args []byte) ([]byte, string, error) { + // Unmarshal the input command and see if we need to intercept + cmd := map[string]interface{}{} + err := json.Unmarshal(args, &cmd) + if err != nil { + return []byte(n.output), "", err + } + + // Intercept and mock the output + switch prefix := cmd["prefix"]; prefix { + case "query": + return []byte(n.cmdOut[fmt.Sprintf("ceph tell %s query", string(pgid))]), "", nil + } + return []byte(n.output), "", nil } diff --git a/collectors/osd.go b/collectors/osd.go index ee5cf5d..c363ca2 100644 --- a/collectors/osd.go +++ b/collectors/osd.go @@ -31,7 +31,6 @@ type cephPGDumpBrief []struct { type cephPGQuery struct { State string `json:"state"` Info struct { - PGID string `json:"pgid"` Stats struct { StatSum struct { NumObjectsRecovered int64 `json:"num_objects_recovered"` @@ -128,6 +127,9 @@ type OSDCollector struct { // ScrubbingStateDesc depicts if an osd is being scrubbed // labelled by OSD ScrubbingStateDesc *prometheus.Desc + + // PGObjectsRecoveredDesc displays total number of objects recovered in a PG + PGObjectsRecoveredDesc *prometheus.Desc } //NewOSDCollector creates an instance of the OSDCollector and instantiates @@ -348,6 +350,13 @@ func NewOSDCollector(conn Conn, cluster string) *OSDCollector { []string{"osd"}, labels, ), + + PGObjectsRecoveredDesc: prometheus.NewDesc( + fmt.Sprintf("%s_pg_objects_recovered_total", cephNamespace), + "Number of objects recovered in a PG", + []string{"pgid"}, + labels, + ), } } @@ -729,6 +738,25 @@ func (o *OSDCollector) collectOSDScrubState(ch chan<- prometheus.Metric) error { return nil } +func (o *OSDCollector) collectPGRecoveryState(ch chan<- prometheus.Metric) error { + for _, pg := range o.pgDumpBrief { + if strings.Contains(pg.State, "recovering") { + query, err := o.performPGQuery(pg.PGID) + if err != nil { + continue + } + + ch <- prometheus.MustNewConstMetric( + o.PGObjectsRecoveredDesc, + prometheus.CounterValue, + float64(query.Info.Stats.StatSum.NumObjectsRecovered), + pg.PGID, + ) + } + } + return nil +} + func (o *OSDCollector) cephOSDDump() []byte { cmd, err := json.Marshal(map[string]interface{}{ "prefix": "osd dump", @@ -806,7 +834,9 @@ func (o *OSDCollector) Describe(ch chan<- *prometheus.Desc) { for _, metric := range o.collectorList() { metric.Describe(ch) } + ch <- o.OSDDownDesc ch <- o.ScrubbingStateDesc + ch <- o.PGObjectsRecoveredDesc } // Collect sends all the collected metrics to the provided prometheus channel. @@ -829,19 +859,19 @@ func (o *OSDCollector) Collect(ch chan<- prometheus.Metric) { o.OSDUp.Reset() if err := o.collectOSDPerf(); err != nil { - log.Println("failed collecting osd perf stats:", err) + log.Println("failed collecting OSD perf stats:", err) } if err := o.collectOSDDump(); err != nil { - log.Println("failed collecting osd dump:", err) + log.Println("failed collecting OSD dump:", err) } if err := o.collectOSDDF(); err != nil { - log.Println("failed collecting osd metrics:", err) + log.Println("failed collecting OSD metrics:", err) } if err := o.collectOSDTreeDown(ch); err != nil { - log.Println("failed collecting osd metrics:", err) + log.Println("failed collecting OSD metrics:", err) } for _, metric := range o.collectorList() { @@ -849,10 +879,14 @@ func (o *OSDCollector) Collect(ch chan<- prometheus.Metric) { } if err := o.performPGDumpBrief(); err != nil { - log.Println("failed performing pg dump brief:", err) + log.Println("failed performing PG dump brief:", err) } if err := o.collectOSDScrubState(ch); err != nil { - log.Println("failed collecting osd scrub state:", err) + log.Println("failed collecting OSD scrub state:", err) + } + + if err := o.collectPGRecoveryState(ch); err != nil { + log.Println("failed collecting PG recovery state:", err) } } diff --git a/collectors/osd_test.go b/collectors/osd_test.go index 87e173a..eb76e28 100644 --- a/collectors/osd_test.go +++ b/collectors/osd_test.go @@ -12,100 +12,102 @@ import ( func TestOSDCollector(t *testing.T) { for _, tt := range []struct { - input string + cmdOut map[string]string regexes []*regexp.Regexp }{ { - input: ` + cmdOut: map[string]string{ + "ceph osd df": ` { - "nodes": [ - { - "id": 0, - "name": "osd.0", - "type": "osd", - "type_id": 0, - "crush_weight": 0.010391, - "depth": 2, - "reweight": 1.000000, - "kb": 11150316, - "kb_used": 40772, - "kb_avail": 11109544, - "utilization": 0.365658, - "var": 1.053676, - "pgs": 283 - }, - { - "id": 2, - "name": "osd.2", - "type": "osd", - "type_id": 0, - "crush_weight": 0.010391, - "depth": 2, - "reweight": 1.000000, - "kb": 11150316, - "kb_used": 36712, - "kb_avail": 11113604, - "utilization": 0.329246, - "var": 0.948753, - "pgs": 162 - }, - { - "id": 1, - "name": "osd.1", - "type": "osd", - "type_id": 0, - "crush_weight": 0.010391, - "depth": 2, - "reweight": 1.000000, - "kb": 11150316, - "kb_used": 40512, - "kb_avail": 11109804, - "utilization": 0.363326, - "var": 1.046957, - "pgs": 279 - }, - { - "id": 3, - "name": "osd.3", - "type": "osd", - "type_id": 0, - "crush_weight": 0.010391, - "depth": 2, - "reweight": 1.000000, - "kb": 11150316, - "kb_used": 36784, - "kb_avail": 11113532, - "utilization": 0.329892, - "var": 0.950614, - "pgs": 164 - }, - { - "id": 4, - "name": "osd.4", - "type": "osd", - "type_id": 0, - "crush_weight": 0.010391, - "depth": 2, - "reweight": 0, - "kb": 0, - "kb_used": 0, - "kb_avail": 0, - "utilization": -nan, - "var": -nan, - "pgs": 0 - } - ], - "stray": [], - "summary": { - "total_kb": 44601264, - "total_kb_used": 154780, - "total_kb_avail": 44446484, - "average_utilization": 0.347031, - "min_var": 0.948753, - "max_var": 1.053676, - "dev": 0.017482 + "nodes": [ + { + "id": 0, + "name": "osd.0", + "type": "osd", + "type_id": 0, + "crush_weight": 0.010391, + "depth": 2, + "reweight": 1.000000, + "kb": 11150316, + "kb_used": 40772, + "kb_avail": 11109544, + "utilization": 0.365658, + "var": 1.053676, + "pgs": 283 + }, + { + "id": 2, + "name": "osd.2", + "type": "osd", + "type_id": 0, + "crush_weight": 0.010391, + "depth": 2, + "reweight": 1.000000, + "kb": 11150316, + "kb_used": 36712, + "kb_avail": 11113604, + "utilization": 0.329246, + "var": 0.948753, + "pgs": 162 + }, + { + "id": 1, + "name": "osd.1", + "type": "osd", + "type_id": 0, + "crush_weight": 0.010391, + "depth": 2, + "reweight": 1.000000, + "kb": 11150316, + "kb_used": 40512, + "kb_avail": 11109804, + "utilization": 0.363326, + "var": 1.046957, + "pgs": 279 + }, + { + "id": 3, + "name": "osd.3", + "type": "osd", + "type_id": 0, + "crush_weight": 0.010391, + "depth": 2, + "reweight": 1.000000, + "kb": 11150316, + "kb_used": 36784, + "kb_avail": 11113532, + "utilization": 0.329892, + "var": 0.950614, + "pgs": 164 + }, + { + "id": 4, + "name": "osd.4", + "type": "osd", + "type_id": 0, + "crush_weight": 0.010391, + "depth": 2, + "reweight": 0, + "kb": 0, + "kb_used": 0, + "kb_avail": 0, + "utilization": -nan, + "var": -nan, + "pgs": 0 } + ], + "stray": [], + "summary": { + "total_kb": 44601264, + "total_kb_used": 154780, + "total_kb_avail": 44446484, + "average_utilization": 0.347031, + "min_var": 0.948753, + "max_var": 1.053676, + "dev": 0.017482 + } }`, + }, regexes: []*regexp.Regexp{ regexp.MustCompile(`ceph_osd_crush_weight{cluster="ceph",osd="osd.0"} 0.010391`), regexp.MustCompile(`ceph_osd_crush_weight{cluster="ceph",osd="osd.1"} 0.010391`), @@ -159,46 +161,48 @@ func TestOSDCollector(t *testing.T) { }, }, { - input: ` + cmdOut: map[string]string{ + "ceph osd perf": ` { - "osd_perf_infos": [ - { - "id": 4, - "perf_stats": { - "commit_latency_ms": 0, - "apply_latency_ms": 0 - } - }, - { - "id": 3, - "perf_stats": { - "commit_latency_ms": 1, - "apply_latency_ms": 64 - } - }, - { - "id": 2, - "perf_stats": { - "commit_latency_ms": 2, - "apply_latency_ms": 79 - } - }, - { - "id": 1, - "perf_stats": { - "commit_latency_ms": 2, - "apply_latency_ms": 39 - } - }, - { - "id": 0, - "perf_stats": { - "commit_latency_ms": 2, - "apply_latency_ms": 31 - } - } - ] + "osd_perf_infos": [ + { + "id": 4, + "perf_stats": { + "commit_latency_ms": 0, + "apply_latency_ms": 0 + } + }, + { + "id": 3, + "perf_stats": { + "commit_latency_ms": 1, + "apply_latency_ms": 64 + } + }, + { + "id": 2, + "perf_stats": { + "commit_latency_ms": 2, + "apply_latency_ms": 79 + } + }, + { + "id": 1, + "perf_stats": { + "commit_latency_ms": 2, + "apply_latency_ms": 39 + } + }, + { + "id": 0, + "perf_stats": { + "commit_latency_ms": 2, + "apply_latency_ms": 31 + } + } + ] }`, + }, regexes: []*regexp.Regexp{ regexp.MustCompile(`ceph_osd_perf_commit_latency_seconds{cluster="ceph",osd="osd.0"} 0.002`), regexp.MustCompile(`ceph_osd_perf_commit_latency_seconds{cluster="ceph",osd="osd.1"} 0.002`), @@ -213,58 +217,59 @@ func TestOSDCollector(t *testing.T) { }, }, { - input: ` + cmdOut: map[string]string{ + "ceph osd dump": ` { - "osds": [ - { - "osd": 0, - "uuid": "135b53c3", - "up": 1, - "in": 1 - }, - { - "osd": 1, - "uuid": "370a33f2", - "up": 1, - "in": 1 - }, - { - "osd": 2, - "uuid": "ca9ab3de", - "up": 1, - "in": 1, - "state": [ - "nearfull", - "exists", - "up" - ] - }, - { - "osd": 3, - "uuid": "bef98b10", - "up": 1, - "in": 1, - "state": [ - "full", - "backfillfull", - "exists", - "up" - ] - }, - { - "osd": 4, - "uuid": "5936c9e8", - "up": 0, - "in": 0, - "state": [ - "backfillfull", - "exists", - "up" - ] - } - ] -} -`, + "osds": [ + { + "osd": 0, + "uuid": "135b53c3", + "up": 1, + "in": 1 + }, + { + "osd": 1, + "uuid": "370a33f2", + "up": 1, + "in": 1 + }, + { + "osd": 2, + "uuid": "ca9ab3de", + "up": 1, + "in": 1, + "state": [ + "nearfull", + "exists", + "up" + ] + }, + { + "osd": 3, + "uuid": "bef98b10", + "up": 1, + "in": 1, + "state": [ + "full", + "backfillfull", + "exists", + "up" + ] + }, + { + "osd": 4, + "uuid": "5936c9e8", + "up": 0, + "in": 0, + "state": [ + "backfillfull", + "exists", + "up" + ] + } + ] +}`, + }, regexes: []*regexp.Regexp{ regexp.MustCompile(`ceph_osd_in{cluster="ceph",osd="osd.0"} 1`), regexp.MustCompile(`ceph_osd_in{cluster="ceph",osd="osd.1"} 1`), @@ -294,42 +299,44 @@ func TestOSDCollector(t *testing.T) { }, }, { - input: ` + cmdOut: map[string]string{ + "ceph pg dump pgs_brief": ` [ - { - "acting": [ - 1, - 2, - 3, - 4 - ], - "acting_primary": 1, - "pgid": "81.1fff", - "state": "active+clean" - }, - { - "acting": [ - 10, - 11, - 12, - 13 - ], - "acting_primary": 10, - "pgid": "82.1fff", - "state": "active+clean+scrubbing" - }, - { - "acting": [ - 20, - 21, - 22, - 23 - ], - "acting_primary": 20, - "pgid": "83.1fff", - "state": "active+clean+scrubbing+deep" - } + { + "acting": [ + 1, + 2, + 3, + 4 + ], + "acting_primary": 1, + "pgid": "81.1fff", + "state": "active+clean" + }, + { + "acting": [ + 10, + 11, + 12, + 13 + ], + "acting_primary": 10, + "pgid": "82.1fff", + "state": "active+clean+scrubbing" + }, + { + "acting": [ + 20, + 21, + 22, + 23 + ], + "acting_primary": 20, + "pgid": "83.1fff", + "state": "active+clean+scrubbing+deep" + } ]`, + }, regexes: []*regexp.Regexp{ regexp.MustCompile(`ceph_osd_scrub_state{cluster="ceph",osd="osd.10"} 1`), regexp.MustCompile(`ceph_osd_scrub_state{cluster="ceph",osd="osd.11"} 1`), @@ -342,183 +349,258 @@ func TestOSDCollector(t *testing.T) { }, }, { - input: ` - { - "nodes": [], - "stray": [ - { - "id": 524, - "name": "osd.524", - "type": "osd", - "type_id": 0, - "crush_weight": 0.000000, - "depth": 0, - "exists": 1, - "status": "destroyed", - "reweight": 0.000000, - "primary_affinity": 1.000000 - } - ] - }`, + cmdOut: map[string]string{ + "ceph osd tree down": ` +{ + "nodes": [], + "stray": [ + { + "id": 524, + "name": "osd.524", + "type": "osd", + "type_id": 0, + "crush_weight": 0.000000, + "depth": 0, + "exists": 1, + "status": "destroyed", + "reweight": 0.000000, + "primary_affinity": 1.000000 + } + ] +}`, + }, regexes: []*regexp.Regexp{ regexp.MustCompile(`ceph_osd_down{cluster="ceph",osd="osd.524",status="destroyed"} 1`), }, }, { - input: ` - { - "nodes": [], - "stray": [ - { - "id": 524, - "name": "osd.524", - "type": "osd", - "type_id": 0, - "crush_weight": 0.000000, - "depth": 0, - "exists": 1, - "status": "down", - "reweight": 0.000000, - "primary_affinity": 1.000000 - } - ] - }`, + cmdOut: map[string]string{ + "ceph osd tree down": ` +{ + "nodes": [], + "stray": [ + { + "id": 524, + "name": "osd.524", + "type": "osd", + "type_id": 0, + "crush_weight": 0.000000, + "depth": 0, + "exists": 1, + "status": "down", + "reweight": 0.000000, + "primary_affinity": 1.000000 + } + ] +}`, + }, regexes: []*regexp.Regexp{ regexp.MustCompile(`ceph_osd_down{cluster="ceph",osd="osd.524",status="down"} 1`), }, }, { - input: ` - { - "nodes": [ - { - "id": -18, - "name": "data", - "type": "root", - "type_id": 10, - "children": [ - -20 - ] - }, - { - "id": -20, - "name": "R1-data", - "type": "rack", - "type_id": 3, - "pool_weights": {}, - "children": [ - -8 - ] - }, - { - "id": -8, - "name": "test-data03-object01", - "type": "host", - "type_id": 1, - "pool_weights": {}, - "children": [ - 97 - ] - }, - { - "id": 524, - "device_class": "hdd", - "name": "osd.524", - "type": "osd", - "type_id": 0, - "crush_weight": 7.265991, - "depth": 3, - "pool_weights": {}, - "exists": 1, - "status": "destroyed", - "reweight": 0.000000, - "primary_affinity": 1.000000 - } - ], - "stray": [] - }`, + cmdOut: map[string]string{ + "ceph osd tree down": ` +{ + "nodes": [ + { + "id": -18, + "name": "data", + "type": "root", + "type_id": 10, + "children": [ + -20 + ] + }, + { + "id": -20, + "name": "R1-data", + "type": "rack", + "type_id": 3, + "pool_weights": {}, + "children": [ + -8 + ] + }, + { + "id": -8, + "name": "test-data03-object01", + "type": "host", + "type_id": 1, + "pool_weights": {}, + "children": [ + 97 + ] + }, + { + "id": 524, + "device_class": "hdd", + "name": "osd.524", + "type": "osd", + "type_id": 0, + "crush_weight": 7.265991, + "depth": 3, + "pool_weights": {}, + "exists": 1, + "status": "destroyed", + "reweight": 0.000000, + "primary_affinity": 1.000000 + } + ], + "stray": [] +}`, + }, regexes: []*regexp.Regexp{ regexp.MustCompile(`ceph_osd_down{cluster="ceph",osd="osd.524",status="destroyed"} 1`), }, }, { - input: ` - { - "nodes": [ - { - "id": -18, - "name": "data", - "type": "root", - "type_id": 10, - "children": [ - -20 - ] - }, - { - "id": -20, - "name": "R1-data", - "type": "rack", - "type_id": 3, - "pool_weights": {}, - "children": [ - -8 - ] - }, - { - "id": -8, - "name": "test-data03-object01", - "type": "host", - "type_id": 1, - "pool_weights": {}, - "children": [ - 97 - ] - }, - { - "id": 524, - "device_class": "hdd", - "name": "osd.524", - "type": "osd", - "type_id": 0, - "crush_weight": 7.265991, - "depth": 3, - "pool_weights": {}, - "exists": 1, - "status": "destroyed", - "reweight": 0.000000, - "primary_affinity": 1.000000 - } - ], - "stray": [ - { - "id": 525, - "name": "osd.525", - "type": "osd", - "type_id": 0, - "crush_weight": 0.000000, - "depth": 0, - "exists": 1, - "status": "down", - "reweight": 0.000000, - "primary_affinity": 1.000000 - } - ] - }`, + cmdOut: map[string]string{ + "ceph osd tree down": ` +{ + "nodes": [ + { + "id": -18, + "name": "data", + "type": "root", + "type_id": 10, + "children": [ + -20 + ] + }, + { + "id": -20, + "name": "R1-data", + "type": "rack", + "type_id": 3, + "pool_weights": {}, + "children": [ + -8 + ] + }, + { + "id": -8, + "name": "test-data03-object01", + "type": "host", + "type_id": 1, + "pool_weights": {}, + "children": [ + 97 + ] + }, + { + "id": 524, + "device_class": "hdd", + "name": "osd.524", + "type": "osd", + "type_id": 0, + "crush_weight": 7.265991, + "depth": 3, + "pool_weights": {}, + "exists": 1, + "status": "destroyed", + "reweight": 0.000000, + "primary_affinity": 1.000000 + } + ], + "stray": [ + { + "id": 525, + "name": "osd.525", + "type": "osd", + "type_id": 0, + "crush_weight": 0.000000, + "depth": 0, + "exists": 1, + "status": "down", + "reweight": 0.000000, + "primary_affinity": 1.000000 + } + ] +}`, + }, regexes: []*regexp.Regexp{ regexp.MustCompile(`ceph_osd_down{cluster="ceph",osd="osd.524",status="destroyed"} 1`), regexp.MustCompile(`ceph_osd_down{cluster="ceph",osd="osd.525",status="down"} 1`), }, }, { - input: ` - { - "nodes": []}} - }`, + cmdOut: map[string]string{ + "ceph osd tree down": ` +{ + "nodes": []}} +}`, + }, regexes: []*regexp.Regexp{}, }, + { + cmdOut: map[string]string{ + "ceph pg dump pgs_brief": ` +[ + { + "acting": [ + 1, + 2, + 3, + 4 + ], + "acting_primary": 1, + "pgid": "81.1fff", + "state": "active+clean" + }, + { + "acting": [ + 10, + 11, + 12, + 13 + ], + "acting_primary": 10, + "pgid": "82.1fff", + "state": "active+clean+scrubbing" + }, + { + "acting": [ + 20, + 21, + 22, + 23 + ], + "acting_primary": 20, + "pgid": "83.1fff", + "state": "active+clean+scrubbing+deep" + }, + { + "acting": [ + 30, + 31, + 32, + 33 + ], + "acting_primary": 30, + "pgid": "84.1fff", + "state": "active+recovering+degraded" + } +]`, + "ceph tell 84.1fff query": ` +{ + "state": "active+recovering+degraded", + "info": { + "stats": { + "stat_sum": { + "num_objects_recovered": 123 + } + } + } +}`, + }, + regexes: []*regexp.Regexp{ + regexp.MustCompile(`ceph_pg_objects_recovered_total{cluster="ceph",pgid="84.1fff"} 123`), + }, + }, } { func() { - collector := NewOSDCollector(NewNoopConn(tt.input), "ceph") + collector := NewOSDCollector(NewNoopConnWithCmdOut(tt.cmdOut), "ceph") if err := prometheus.Register(collector); err != nil { t.Fatalf("collector failed to register: %s", err) } @@ -537,7 +619,6 @@ func TestOSDCollector(t *testing.T) { if err != nil { t.Fatalf("failed reading server response: %s", err) } - for _, re := range tt.regexes { if !re.Match(buf) { t.Errorf("failed matching: %q", re)