diff --git a/ceph/cluster_usage_test.go b/ceph/cluster_usage_test.go index a37e9ed..0192c62 100644 --- a/ceph/cluster_usage_test.go +++ b/ceph/cluster_usage_test.go @@ -15,12 +15,14 @@ package ceph import ( + "encoding/json" "io/ioutil" "net/http" "net/http/httptest" "regexp" "testing" + "github.com/google/go-cmp/cmp" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" @@ -31,6 +33,7 @@ import ( func TestClusterUsage(t *testing.T) { for _, tt := range []struct { input string + version string reMatch, reUnmatch []*regexp.Regexp }{ { @@ -42,6 +45,7 @@ func TestClusterUsage(t *testing.T) { "total_avail_bytes": 4 } }`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`ceph_cluster_capacity_bytes{cluster="ceph"} 10`), regexp.MustCompile(`ceph_cluster_used_bytes{cluster="ceph"} 6`), @@ -57,6 +61,7 @@ func TestClusterUsage(t *testing.T) { "total_avail_bytes": 4 } }`, + version: `{"version":"ceph version 16.2.11-98-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`ceph_cluster_capacity_bytes{cluster="ceph"} 0`), regexp.MustCompile(`ceph_cluster_used_bytes{cluster="ceph"} 6`), @@ -72,6 +77,7 @@ func TestClusterUsage(t *testing.T) { "total_avail_bytes": 4 } }`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`ceph_cluster_capacity_bytes{cluster="ceph"} 10`), regexp.MustCompile(`ceph_cluster_used_bytes{cluster="ceph"} 0`), @@ -87,6 +93,7 @@ func TestClusterUsage(t *testing.T) { "total_used_bytes": 6 } }`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`ceph_cluster_capacity_bytes{cluster="ceph"} 10`), regexp.MustCompile(`ceph_cluster_used_bytes{cluster="ceph"} 6`), @@ -103,6 +110,7 @@ func TestClusterUsage(t *testing.T) { "total_avail_bytes": 4 } }`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`ceph_cluster_capacity_bytes{cluster="ceph"} 10`), regexp.MustCompile(`ceph_cluster_used_bytes{cluster="ceph"} 6`), @@ -119,6 +127,7 @@ func TestClusterUsage(t *testing.T) { "total_avail_bytes": 4 } }`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{}, reUnmatch: []*regexp.Regexp{ regexp.MustCompile(`ceph_cluster_capacity_bytes{cluster="ceph"}`), @@ -129,14 +138,42 @@ func TestClusterUsage(t *testing.T) { } { func() { conn := &MockConn{} + conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool { + v := map[string]interface{}{} + + err := json.Unmarshal(in.([]byte), &v) + require.NoError(t, err) + + return cmp.Equal(v, map[string]interface{}{ + "prefix": "version", + "format": "json", + }) + })).Return([]byte(tt.version), "", nil) + + // versions is only used to check if rbd mirror is present + conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool { + v := map[string]interface{}{} + + err := json.Unmarshal(in.([]byte), &v) + require.NoError(t, err) + + return cmp.Equal(v, map[string]interface{}{ + "prefix": "versions", + "format": "json", + }) + })).Return([]byte(`{}`), "", nil) + conn.On("MonCommand", mock.Anything).Return( []byte(tt.input), "", nil, ) - collector := NewClusterUsageCollector(&Exporter{Conn: conn, Cluster: "ceph", Logger: logrus.New()}) - err := prometheus.Register(collector) + e := &Exporter{Conn: conn, Cluster: "ceph", Logger: logrus.New()} + e.cc = map[string]interface{}{ + "clusterUsage": NewClusterUsageCollector(e), + } + err := prometheus.Register(e) require.NoError(t, err) - defer prometheus.Unregister(collector) + defer prometheus.Unregister(e) server := httptest.NewServer(promhttp.Handler()) defer server.Close() diff --git a/ceph/crashes_test.go b/ceph/crashes_test.go index 286e140..b98d007 100644 --- a/ceph/crashes_test.go +++ b/ceph/crashes_test.go @@ -15,12 +15,14 @@ package ceph import ( + "encoding/json" "io/ioutil" "net/http" "net/http/httptest" "regexp" "testing" + "github.com/google/go-cmp/cmp" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" @@ -33,6 +35,7 @@ func TestCrashesCollector(t *testing.T) { for _, tt := range []struct { name string input string + version string reMatch []*regexp.Regexp }{ { @@ -85,6 +88,7 @@ func TestCrashesCollector(t *testing.T) { "assert_func": "void ConfigProxy::call_gate_enter(ConfigProxy::md_config_obs_t*)" } ]`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`crash_reports{cluster="ceph",entity="client.admin",hostname="test-ceph-server.company.example",status="new"} 1`), }, @@ -102,6 +106,7 @@ func TestCrashesCollector(t *testing.T) { } ] `, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`crash_reports{cluster="ceph",entity="client.admin",hostname="test-ceph-server.company.example",status="archived"} 1`), }, @@ -123,6 +128,7 @@ func TestCrashesCollector(t *testing.T) { "crash_id": "2022-02-03_04:05:45.419226Z_11c639af-5eb2-4a29-91aa-20120218891a" } ]`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`crash_reports{cluster="ceph",entity="osd.0",hostname="test-ceph-server.company.example",status="new"} 2`), }, @@ -145,6 +151,7 @@ func TestCrashesCollector(t *testing.T) { "crash_id": "2022-02-03_04:05:45.419226Z_11c639af-5eb2-4a29-91aa-20120218891a" } ]`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`crash_reports{cluster="ceph",entity="osd.0",hostname="test-ceph-server.company.example",status="new"} 1`), regexp.MustCompile(`crash_reports{cluster="ceph",entity="osd.0",hostname="test-ceph-server.company.example",status="archived"} 1`), @@ -167,6 +174,7 @@ func TestCrashesCollector(t *testing.T) { "crash_id": "2022-02-03_04:05:45.419226Z_11c639af-5eb2-4a29-91aa-20120218891a" } ]`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`crash_reports{cluster="ceph",entity="mgr.mgr-node-01",hostname="test-ceph-server.company.example",status="new"} 1`), regexp.MustCompile(`crash_reports{cluster="ceph",entity="client.admin",hostname="test-ceph-server.company.example",status="new"} 1`), @@ -176,6 +184,7 @@ func TestCrashesCollector(t *testing.T) { // At least code shouldn't panic name: "no crashes", input: `[]`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{}, }, } { @@ -183,14 +192,42 @@ func TestCrashesCollector(t *testing.T) { tt.name, func(t *testing.T) { conn := &MockConn{} + conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool { + v := map[string]interface{}{} + + err := json.Unmarshal(in.([]byte), &v) + require.NoError(t, err) + + return cmp.Equal(v, map[string]interface{}{ + "prefix": "version", + "format": "json", + }) + })).Return([]byte(tt.version), "", nil) + + // versions is only used to check if rbd mirror is present + conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool { + v := map[string]interface{}{} + + err := json.Unmarshal(in.([]byte), &v) + require.NoError(t, err) + + return cmp.Equal(v, map[string]interface{}{ + "prefix": "versions", + "format": "json", + }) + })).Return([]byte(`{}`), "", nil) + conn.On("MonCommand", mock.Anything).Return( []byte(tt.input), "", nil, ) - collector := NewCrashesCollector(&Exporter{Conn: conn, Cluster: "ceph", Logger: logrus.New(), Version: Pacific}) - err := prometheus.Register(collector) + e := &Exporter{Conn: conn, Cluster: "ceph", Logger: logrus.New()} + e.cc = map[string]interface{}{ + "crashes": NewCrashesCollector(e), + } + err := prometheus.Register(e) require.NoError(t, err) - defer prometheus.Unregister(collector) + defer prometheus.Unregister(e) server := httptest.NewServer(promhttp.Handler()) defer server.Close() diff --git a/ceph/health_test.go b/ceph/health_test.go index 4cb669b..e68a4e7 100644 --- a/ceph/health_test.go +++ b/ceph/health_test.go @@ -15,12 +15,14 @@ package ceph import ( + "encoding/json" "io/ioutil" "net/http" "net/http/httptest" "regexp" "testing" + "github.com/google/go-cmp/cmp" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" @@ -29,14 +31,11 @@ import ( ) func TestClusterHealthCollector(t *testing.T) { - allVersions := []*Version{Nautilus, Octopus, Pacific} - nautilusOnly := []*Version{Nautilus} - octopusPlus := []*Version{Octopus, Pacific} for _, tt := range []struct { - name string - versions []*Version // Defaults to allVersions if not provided. - input string - reMatch []*regexp.Regexp + name string + version string + input string + reMatch []*regexp.Regexp }{ { name: "15 pgs stuck degraded", @@ -44,6 +43,7 @@ func TestClusterHealthCollector(t *testing.T) { { "health": {"summary": [{"severity": "HEALTH_WARN", "summary": "15 pgs stuck degraded"}]} }`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`stuck_degraded_pgs{cluster="ceph"} 15`), }, @@ -54,6 +54,7 @@ func TestClusterHealthCollector(t *testing.T) { { "health": {"summary": [{"severity": "HEALTH_WARN", "summary": "16 pgs stuck unclean"}]} }`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`stuck_unclean_pgs{cluster="ceph"} 16`), }, @@ -64,6 +65,7 @@ func TestClusterHealthCollector(t *testing.T) { { "health": {"summary": [{"severity": "HEALTH_WARN", "summary": "17 pgs stuck undersized"}]} }`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`stuck_undersized_pgs{cluster="ceph"} 17`), }, @@ -74,6 +76,7 @@ func TestClusterHealthCollector(t *testing.T) { { "health": {"summary": [{"severity": "HEALTH_WARN", "summary": "18 pgs stuck stale"}]} }`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`stuck_stale_pgs{cluster="ceph"} 18`), }, @@ -84,6 +87,7 @@ func TestClusterHealthCollector(t *testing.T) { { "pgmap": { "degraded_objects": 10 } }`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`degraded_objects{cluster="ceph"} 10`), }, @@ -94,13 +98,14 @@ func TestClusterHealthCollector(t *testing.T) { { "pgmap": { "misplaced_objects": 20 } }`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`misplaced_objects{cluster="ceph"} 20`), }, }, { - name: "10 down osds", - versions: nautilusOnly, + name: "10 down osds", + version: `{"version":"ceph version 14.2.9-12-zasd (1337) pacific (stable)"}`, input: ` { "osdmap": { @@ -117,8 +122,8 @@ func TestClusterHealthCollector(t *testing.T) { }, }, { - name: "normal osdmap", - versions: nautilusOnly, + name: "normal osdmap", + version: `{"version":"ceph version 14.2.9-12-zasd (1337) pacific (stable)"}`, input: ` { "osdmap": { @@ -139,8 +144,8 @@ func TestClusterHealthCollector(t *testing.T) { }, }, { - name: "10 down osds", - versions: octopusPlus, + name: "10 down osds", + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, input: ` { "osdmap": { @@ -155,8 +160,8 @@ func TestClusterHealthCollector(t *testing.T) { }, }, { - name: "normal osdmap", - versions: octopusPlus, + name: "normal osdmap", + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, input: ` { "osdmap": { @@ -175,39 +180,44 @@ func TestClusterHealthCollector(t *testing.T) { }, }, { - name: "health ok", - input: `{"health": { "status": "HEALTH_OK" } }`, + name: "health ok", + input: `{"health": { "status": "HEALTH_OK" } }`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`health_status{cluster="ceph"} 0`), }, }, { - name: "health warn", - input: `{"health": { "status": "HEALTH_OK" } }`, + name: "health warn", + input: `{"health": { "status": "HEALTH_OK" } }`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`health_status{cluster="ceph"} 0`), regexp.MustCompile(`health_status_interp{cluster="ceph"} 0`), }, }, { - name: "health ok 2", - input: `{"health": { "status": "HEALTH_OK" } }`, + name: "health ok 2", + input: `{"health": { "status": "HEALTH_OK" } }`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`health_status{cluster="ceph"} 0`), regexp.MustCompile(`health_status_interp{cluster="ceph"} 0`), }, }, { - name: "health warn 2", - input: `{"health": { "status": "HEALTH_WARN" } }`, + name: "health warn 2", + input: `{"health": { "status": "HEALTH_WARN" } }`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`health_status{cluster="ceph"} 1`), regexp.MustCompile(`health_status_interp{cluster="ceph"} 2`), }, }, { - name: "health err", - input: `{"health": { "status": "HEALTH_ERR" } }`, + name: "health err", + input: `{"health": { "status": "HEALTH_ERR" } }`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`health_status{cluster="ceph"} 2`), regexp.MustCompile(`health_status_interp{cluster="ceph"} 3`), @@ -223,6 +233,7 @@ $ sudo ceph -s recovery io 5779 MB/s, 4 keys/s, 1522 objects/s client io 4273 kB/s rd, 2740 MB/s wr, 2863 op/s `, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`recovery_io_bytes{cluster="ceph"} 5.779e`), regexp.MustCompile(`recovery_io_keys{cluster="ceph"} 4`), @@ -243,6 +254,7 @@ $ sudo ceph -s client io 2863 op/s rd, 5847 op/s wr cache io 251 MB/s flush, 6646 kB/s evict, 55 op/s promote `, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`recovery_io_bytes{cluster="ceph"} 5.779e`), regexp.MustCompile(`recovery_io_keys{cluster="ceph"} 4`), @@ -262,6 +274,7 @@ $ sudo ceph -s "pgmap": { "num_pgs": 52000, "num_objects": 13156 }, "health": {"summary": [{"severity": "HEALTH_WARN", "summary": "7 pgs undersized"}]} }`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`total_pgs{cluster="ceph"} 52000`), regexp.MustCompile(`cluster_objects{cluster="ceph"} 13156`), @@ -303,6 +316,7 @@ $ sudo ceph -s }, "health": {"summary": [{"severity": "HEALTH_WARN", "summary": "7 pgs undersized"}]} }`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`active_pgs{cluster="ceph"} 49`), regexp.MustCompile(`scrubbing_pgs{cluster="ceph"} 2`), @@ -329,6 +343,7 @@ $ sudo ceph -s } } }`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`mons_down{cluster="ceph"} 1`), }, @@ -346,6 +361,7 @@ $ sudo ceph -s ] } }`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`slow_requests{cluster="ceph"} 3`), }, @@ -365,6 +381,7 @@ $ sudo ceph -s } } }`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`slow_requests{cluster="ceph"} 3`), }, @@ -384,6 +401,7 @@ $ sudo ceph -s } } }`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`slow_requests{cluster="ceph"} 18`), }, @@ -404,6 +422,7 @@ $ sudo ceph -s }, "pgmap": { "degraded_objects": 154443937 } }`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`degraded_objects{cluster="ceph"} 1.54443937e\+08`), regexp.MustCompile(`health_status_interp{cluster="ceph"} 1`), @@ -424,6 +443,7 @@ $ sudo ceph -s } } }`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`new_crash_reports{cluster="ceph"} 2`), regexp.MustCompile(`health_status_interp{cluster="ceph"} 1`), @@ -444,6 +464,7 @@ $ sudo ceph -s } } }`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`osds_too_many_repair{cluster="ceph"} 25`), regexp.MustCompile(`health_status_interp{cluster="ceph"} 1`), @@ -464,6 +485,7 @@ $ sudo ceph -s } } }`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`health_status_interp{cluster="ceph"} 2`), }, @@ -483,6 +505,7 @@ $ sudo ceph -s } } }`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`osdmap_flag_full{cluster="ceph"} 0`), regexp.MustCompile(`osdmap_flag_pauserd{cluster="ceph"} 1`), @@ -515,6 +538,7 @@ $ sudo ceph -s } } }`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`osd_map_flags{cluster="ceph",flag="pauserd"} 1`), regexp.MustCompile(`osd_map_flags{cluster="ceph",flag="pausewr"} 1`), @@ -628,6 +652,7 @@ $ sudo ceph -s "bytes_total": 2537720565469184 } }`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`active_pgs{cluster="ceph"} 44`), regexp.MustCompile(`degraded_pgs{cluster="ceph"} 40`), @@ -674,8 +699,8 @@ $ sudo ceph -s }, }, { - name: "manager map", - versions: nautilusOnly, + name: "manager map", + version: `{"version":"ceph version 14.2.9-12-zasd (1337) pacific (stable)"}`, input: ` { "mgrmap": { @@ -725,8 +750,8 @@ $ sudo ceph -s }, }, { - name: "manager map", - versions: octopusPlus, + name: "manager map", + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, input: ` { "mgrmap": { @@ -784,6 +809,7 @@ $ sudo ceph -s } } }`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`rbd_mirror_up{cluster="ceph",\s*name="prod-mon01-block01"} 1`), regexp.MustCompile(`rbd_mirror_up{cluster="ceph",\s*name="prod-mon02-block01"} 1`), @@ -791,38 +817,58 @@ $ sudo ceph -s }, } { t.Run(tt.name, func(t *testing.T) { - versions := allVersions - if len(tt.versions) > 0 { - versions = tt.versions - } - for _, version := range versions { - t.Run(version.String(), func(t *testing.T) { - conn := &MockConn{} - conn.On("MonCommand", mock.Anything).Return( - []byte(tt.input), "", nil, - ) + conn := &MockConn{} + conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool { + v := map[string]interface{}{} - collector := NewClusterHealthCollector(&Exporter{Conn: conn, Cluster: "ceph", Logger: logrus.New(), Version: version}) - err := prometheus.Register(collector) - require.NoError(t, err) - defer prometheus.Unregister(collector) + err := json.Unmarshal(in.([]byte), &v) + require.NoError(t, err) - server := httptest.NewServer(promhttp.Handler()) - defer server.Close() - - resp, err := http.Get(server.URL) - require.NoError(t, err) - defer resp.Body.Close() - - buf, err := ioutil.ReadAll(resp.Body) - require.NoError(t, err) - - for _, re := range tt.reMatch { - if !re.Match(buf) { - t.Errorf("expected %s to match\n", re.String()) - } - } + return cmp.Equal(v, map[string]interface{}{ + "prefix": "version", + "format": "json", }) + })).Return([]byte(tt.version), "", nil) + + // versions is only used to check if rbd mirror is present + conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool { + v := map[string]interface{}{} + + err := json.Unmarshal(in.([]byte), &v) + require.NoError(t, err) + + return cmp.Equal(v, map[string]interface{}{ + "prefix": "versions", + "format": "json", + }) + })).Return([]byte(`{}`), "", nil) + + conn.On("MonCommand", mock.Anything).Return( + []byte(tt.input), "", nil, + ) + e := &Exporter{Conn: conn, Cluster: "ceph", Logger: logrus.New()} + e.cc = map[string]interface{}{ + "clusterHealth": NewClusterHealthCollector(e), + } + + err := prometheus.Register(e) + require.NoError(t, err) + defer prometheus.Unregister(e) + + server := httptest.NewServer(promhttp.Handler()) + defer server.Close() + + resp, err := http.Get(server.URL) + require.NoError(t, err) + defer resp.Body.Close() + + buf, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + + for _, re := range tt.reMatch { + if !re.Match(buf) { + t.Errorf("expected %s to match\n", re.String()) + } } }) } diff --git a/ceph/monitors_test.go b/ceph/monitors_test.go index 8ccbfbc..cc9e392 100644 --- a/ceph/monitors_test.go +++ b/ceph/monitors_test.go @@ -15,12 +15,14 @@ package ceph import ( + "encoding/json" "io/ioutil" "net/http" "net/http/httptest" "regexp" "testing" + "github.com/google/go-cmp/cmp" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" @@ -31,10 +33,11 @@ import ( func TestMonitorCollector(t *testing.T) { for _, tt := range []struct { input string + version string regexes []*regexp.Regexp }{ { - ` + input: ` { "health": { "health": { @@ -210,7 +213,8 @@ func TestMonitorCollector(t *testing.T) { } } `, - []*regexp.Regexp{ + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, + regexes: []*regexp.Regexp{ regexp.MustCompile(`ceph_monitor_avail_bytes{cluster="ceph",monitor="test-mon01"} 3.9927552e`), regexp.MustCompile(`ceph_monitor_avail_bytes{cluster="ceph",monitor="test-mon02"} 3.99211569152e`), regexp.MustCompile(`ceph_monitor_avail_bytes{cluster="ceph",monitor="test-mon03"} 3.98986235904e`), @@ -267,14 +271,42 @@ func TestMonitorCollector(t *testing.T) { } { func() { conn := &MockConn{} + conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool { + v := map[string]interface{}{} + + err := json.Unmarshal(in.([]byte), &v) + require.NoError(t, err) + + return cmp.Equal(v, map[string]interface{}{ + "prefix": "version", + "format": "json", + }) + })).Return([]byte(tt.version), "", nil) + + // versions is only used to check if rbd mirror is present + conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool { + v := map[string]interface{}{} + + err := json.Unmarshal(in.([]byte), &v) + require.NoError(t, err) + + return cmp.Equal(v, map[string]interface{}{ + "prefix": "versions", + "format": "json", + }) + })).Return([]byte(`{}`), "", nil) + conn.On("MonCommand", mock.Anything).Return( []byte(tt.input), "", nil, ) - collector := NewMonitorCollector(&Exporter{Conn: conn, Cluster: "ceph", Logger: logrus.New()}) - err := prometheus.Register(collector) + e := &Exporter{Conn: conn, Cluster: "ceph", Logger: logrus.New()} + e.cc = map[string]interface{}{ + "mon": NewMonitorCollector(e), + } + err := prometheus.Register(e) require.NoError(t, err) - defer prometheus.Unregister(collector) + defer prometheus.Unregister(e) server := httptest.NewServer(promhttp.Handler()) defer server.Close() @@ -296,9 +328,11 @@ func TestMonitorCollector(t *testing.T) { func TestMonitorTimeSyncStats(t *testing.T) { for _, tt := range []struct { input string + version string reMatch []*regexp.Regexp }{ - {` + { + input: ` { "time_skew_status": { "test-mon01": { @@ -334,7 +368,8 @@ func TestMonitorTimeSyncStats(t *testing.T) { } } `, - []*regexp.Regexp{ + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, + reMatch: []*regexp.Regexp{ regexp.MustCompile(`ceph_monitor_clock_skew_seconds{cluster="ceph",monitor="test-mon01"} 2.2e\-05`), regexp.MustCompile(`ceph_monitor_clock_skew_seconds{cluster="ceph",monitor="test-mon02"} 0.001051`), regexp.MustCompile(`ceph_monitor_clock_skew_seconds{cluster="ceph",monitor="test-mon03"} 0.003029`), @@ -347,7 +382,8 @@ func TestMonitorTimeSyncStats(t *testing.T) { regexp.MustCompile(`ceph_monitor_latency_seconds{cluster="ceph",monitor="test-mon05"} 0.000667`), }, }, - {` + { + input: ` { "time_skew_status": { "test-mon01": { @@ -357,9 +393,11 @@ func TestMonitorTimeSyncStats(t *testing.T) { } } `, - []*regexp.Regexp{}, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, + reMatch: []*regexp.Regexp{}, }, - {` + { + input: ` { "time_skew_status": { "test-mon01": { @@ -369,9 +407,11 @@ func TestMonitorTimeSyncStats(t *testing.T) { } } `, - []*regexp.Regexp{}, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, + reMatch: []*regexp.Regexp{}, }, - {` + { + input: ` { "time_skew_status": { "test-mon01": { @@ -381,19 +421,48 @@ func TestMonitorTimeSyncStats(t *testing.T) { } } `, - []*regexp.Regexp{}, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, + reMatch: []*regexp.Regexp{}, }, } { func() { conn := &MockConn{} + conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool { + v := map[string]interface{}{} + + err := json.Unmarshal(in.([]byte), &v) + require.NoError(t, err) + + return cmp.Equal(v, map[string]interface{}{ + "prefix": "version", + "format": "json", + }) + })).Return([]byte(tt.version), "", nil) + + // versions is only used to check if rbd mirror is present + conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool { + v := map[string]interface{}{} + + err := json.Unmarshal(in.([]byte), &v) + require.NoError(t, err) + + return cmp.Equal(v, map[string]interface{}{ + "prefix": "versions", + "format": "json", + }) + })).Return([]byte(`{}`), "", nil) + conn.On("MonCommand", mock.Anything).Return( []byte(tt.input), "", nil, ) - collector := NewMonitorCollector(&Exporter{Conn: conn, Cluster: "ceph", Logger: logrus.New()}) - err := prometheus.Register(collector) + e := &Exporter{Conn: conn, Cluster: "ceph", Logger: logrus.New()} + e.cc = map[string]interface{}{ + "mon": NewMonitorCollector(e), + } + err := prometheus.Register(e) require.NoError(t, err) - defer prometheus.Unregister(collector) + defer prometheus.Unregister(e) server := httptest.NewServer(promhttp.Handler()) defer server.Close() @@ -415,9 +484,11 @@ func TestMonitorTimeSyncStats(t *testing.T) { func TestMonitorCephVersions(t *testing.T) { for _, tt := range []struct { input string + version string reMatch []*regexp.Regexp }{ - {` + { + input: ` { "mon": { "ceph version 12.2.13 (584a20eb0237c657dc0567da126be145106aa47e) luminous (stable)": 5 @@ -439,7 +510,8 @@ func TestMonitorCephVersions(t *testing.T) { } } `, - []*regexp.Regexp{ + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, + reMatch: []*regexp.Regexp{ regexp.MustCompile(`ceph_versions{cluster="ceph",daemon="mon",release_name="luminous",sha1="584a20eb0237c657dc0567da126be145106aa47e",version_tag="12.2.13"} 5`), regexp.MustCompile(`ceph_versions{cluster="ceph",daemon="rgw",release_name="luminous",sha1="58a2283da6a62d2cc1600d4a9928a0799d63c7c9",version_tag="12.2.5-8-g58a2283"} 4`), }, @@ -447,14 +519,29 @@ func TestMonitorCephVersions(t *testing.T) { } { func() { conn := &MockConn{} + conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool { + v := map[string]interface{}{} + + err := json.Unmarshal(in.([]byte), &v) + require.NoError(t, err) + + return cmp.Equal(v, map[string]interface{}{ + "prefix": "version", + "format": "json", + }) + })).Return([]byte(tt.version), "", nil) + conn.On("MonCommand", mock.Anything).Return( []byte(tt.input), "", nil, ) - collector := NewMonitorCollector(&Exporter{Conn: conn, Cluster: "ceph", Logger: logrus.New()}) - err := prometheus.Register(collector) + e := &Exporter{Conn: conn, Cluster: "ceph", Logger: logrus.New()} + e.cc = map[string]interface{}{ + "mon": NewMonitorCollector(e), + } + err := prometheus.Register(e) require.NoError(t, err) - defer prometheus.Unregister(collector) + defer prometheus.Unregister(e) server := httptest.NewServer(promhttp.Handler()) defer server.Close() @@ -476,9 +563,11 @@ func TestMonitorCephVersions(t *testing.T) { func TestMonitorCephFeatures(t *testing.T) { for _, tt := range []struct { input string + version string reMatch []*regexp.Regexp }{ - {` + { + input: ` { "mon": [ { @@ -520,21 +609,50 @@ func TestMonitorCephFeatures(t *testing.T) { ] } `, - []*regexp.Regexp{ + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, + reMatch: []*regexp.Regexp{ regexp.MustCompile(`ceph_features{cluster="ceph",daemon="client",features="0x3ffddff8ffacffff",release="luminous"} 53`), }, }, } { func() { conn := &MockConn{} + conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool { + v := map[string]interface{}{} + + err := json.Unmarshal(in.([]byte), &v) + require.NoError(t, err) + + return cmp.Equal(v, map[string]interface{}{ + "prefix": "version", + "format": "json", + }) + })).Return([]byte(tt.version), "", nil) + + // versions is only used to check if rbd mirror is present + conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool { + v := map[string]interface{}{} + + err := json.Unmarshal(in.([]byte), &v) + require.NoError(t, err) + + return cmp.Equal(v, map[string]interface{}{ + "prefix": "versions", + "format": "json", + }) + })).Return([]byte(`{}`), "", nil) + conn.On("MonCommand", mock.Anything).Return( []byte(tt.input), "", nil, ) - collector := NewMonitorCollector(&Exporter{Conn: conn, Cluster: "ceph", Logger: logrus.New()}) - err := prometheus.Register(collector) + e := &Exporter{Conn: conn, Cluster: "ceph", Logger: logrus.New()} + e.cc = map[string]interface{}{ + "mon": NewMonitorCollector(e), + } + err := prometheus.Register(e) require.NoError(t, err) - defer prometheus.Unregister(collector) + defer prometheus.Unregister(e) server := httptest.NewServer(promhttp.Handler()) defer server.Close() diff --git a/ceph/osd_test.go b/ceph/osd_test.go index 345337b..5828d95 100644 --- a/ceph/osd_test.go +++ b/ceph/osd_test.go @@ -447,28 +447,33 @@ func TestOSDCollector(t *testing.T) { for _, tt := range []struct { test string + version string reMatch []*regexp.Regexp }{ { - test: "1", + test: "1", + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`ceph_osd_down{cluster="ceph",device_class="ssd",host="prod-data02-block01",osd="osd.524",rack="default",root="A8R2",status="destroyed"} 1`), }, }, { - test: "2", + test: "2", + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`ceph_osd_down{cluster="ceph",device_class="ssd",host="prod-data02-block01",osd="osd.524",rack="default",root="A8R2",status="down"} 1`), }, }, { - test: "3", + test: "3", + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`ceph_osd_down{cluster="ceph",device_class="ssd",host="prod-data02-block01",osd="osd.524",rack="default",root="A8R2",status="destroyed"} 1`), }, }, { - test: "4", + test: "4", + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`ceph_osd_down{cluster="ceph",device_class="ssd",host="prod-data02-block01",osd="osd.524",rack="default",root="A8R2",status="destroyed"} 1`), regexp.MustCompile(`ceph_osd_down{cluster="ceph",device_class="ssd",host="prod-data02-block01",osd="osd.525",rack="default",root="A8R2",status="down"} 1`), @@ -476,6 +481,7 @@ func TestOSDCollector(t *testing.T) { }, { test: "5", + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{}, }, } { @@ -965,11 +971,38 @@ func TestOSDCollector(t *testing.T) { ] } }`), "", nil) + conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool { + v := map[string]interface{}{} - collector := NewOSDCollector(&Exporter{Conn: conn, Cluster: "ceph", Logger: logrus.New()}) - err := prometheus.Register(collector) + err := json.Unmarshal(in.([]byte), &v) + require.NoError(t, err) + + return cmp.Equal(v, map[string]interface{}{ + "prefix": "version", + "format": "json", + }) + })).Return([]byte(tt.version), "", nil) + + // versions is only used to check if rbd mirror is present + conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool { + v := map[string]interface{}{} + + err := json.Unmarshal(in.([]byte), &v) + require.NoError(t, err) + + return cmp.Equal(v, map[string]interface{}{ + "prefix": "versions", + "format": "json", + }) + })).Return([]byte(`{}`), "", nil) + + e := &Exporter{Conn: conn, Cluster: "ceph", Logger: logrus.New()} + e.cc = map[string]interface{}{ + "osd": NewOSDCollector(e), + } + err := prometheus.Register(e) require.NoError(t, err) - defer prometheus.Unregister(collector) + defer prometheus.Unregister(e) server := httptest.NewServer(promhttp.Handler()) defer server.Close() diff --git a/ceph/pool_test.go b/ceph/pool_test.go index b48edcb..b2a9e02 100644 --- a/ceph/pool_test.go +++ b/ceph/pool_test.go @@ -33,9 +33,11 @@ import ( func TestPoolInfoCollector(t *testing.T) { for _, tt := range []struct { + version string reMatch, reUnmatch []*regexp.Regexp }{ { + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`pool_size{cluster="ceph",pool="rbd",profile="ec-4-2",root="non-default-root"} 6`), regexp.MustCompile(`pool_min_size{cluster="ceph",pool="rbd",profile="ec-4-2",root="non-default-root"} 4`), @@ -60,6 +62,32 @@ func TestPoolInfoCollector(t *testing.T) { } { func() { conn := &MockConn{} + + conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool { + v := map[string]interface{}{} + + err := json.Unmarshal(in.([]byte), &v) + require.NoError(t, err) + + return cmp.Equal(v, map[string]interface{}{ + "prefix": "version", + "format": "json", + }) + })).Return([]byte(tt.version), "", nil) + + // versions is only used to check if rbd mirror is present + conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool { + v := map[string]interface{}{} + + err := json.Unmarshal(in.([]byte), &v) + require.NoError(t, err) + + return cmp.Equal(v, map[string]interface{}{ + "prefix": "versions", + "format": "json", + }) + })).Return([]byte(`{}`), "", nil) + conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool { v := map[string]interface{}{} @@ -181,11 +209,13 @@ func TestPoolInfoCollector(t *testing.T) { }) })).Return([]byte(""), "", fmt.Errorf("unknown erasure code profile")) - collector := NewPoolInfoCollector(&Exporter{Conn: conn, Cluster: "ceph", Logger: logrus.New()}) - - err := prometheus.Register(collector) + e := &Exporter{Conn: conn, Cluster: "ceph", Logger: logrus.New()} + e.cc = map[string]interface{}{ + "poolInfo": NewPoolInfoCollector(e), + } + err := prometheus.Register(e) require.NoError(t, err) - defer prometheus.Unregister(collector) + defer prometheus.Unregister(e) server := httptest.NewServer(promhttp.Handler()) defer server.Close() diff --git a/ceph/pool_usage_test.go b/ceph/pool_usage_test.go index 56b8998..7d06193 100644 --- a/ceph/pool_usage_test.go +++ b/ceph/pool_usage_test.go @@ -15,6 +15,7 @@ package ceph import ( + "encoding/json" "fmt" "io/ioutil" "net/http" @@ -22,6 +23,7 @@ import ( "regexp" "testing" + "github.com/google/go-cmp/cmp" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" @@ -32,6 +34,7 @@ import ( func TestPoolUsageCollector(t *testing.T) { for _, tt := range []struct { input string + version string reMatch, reUnmatch []*regexp.Regexp }{ { @@ -39,6 +42,7 @@ func TestPoolUsageCollector(t *testing.T) { {"pools": [ {"name": "rbd", "id": 11, "stats": {"stored": 20, "objects": 5, "rd": 4, "wr": 6}} ]}`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`pool_used_bytes{cluster="ceph",pool="rbd"} 20`), regexp.MustCompile(`pool_objects_total{cluster="ceph",pool="rbd"} 5`), @@ -52,6 +56,7 @@ func TestPoolUsageCollector(t *testing.T) { {"pools": [ {"name": "rbd", "id": 11, "stats": {"objects": 5, "rd": 4, "wr": 6}} ]}`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`pool_used_bytes{cluster="ceph",pool="rbd"} 0`), regexp.MustCompile(`pool_objects_total{cluster="ceph",pool="rbd"} 5`), @@ -65,6 +70,7 @@ func TestPoolUsageCollector(t *testing.T) { {"pools": [ {"name": "rbd", "id": 11, "stats": {"stored": 20, "rd": 4, "wr": 6}} ]}`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`pool_used_bytes{cluster="ceph",pool="rbd"} 20`), regexp.MustCompile(`pool_objects_total{cluster="ceph",pool="rbd"} 0`), @@ -78,6 +84,7 @@ func TestPoolUsageCollector(t *testing.T) { {"pools": [ {"name": "rbd", "id": 11, "stats": {"stored": 20, "objects": 5, "wr": 6}} ]}`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`pool_used_bytes{cluster="ceph",pool="rbd"} 20`), regexp.MustCompile(`pool_objects_total{cluster="ceph",pool="rbd"} 5`), @@ -91,6 +98,7 @@ func TestPoolUsageCollector(t *testing.T) { {"pools": [ {"name": "rbd", "id": 11, "stats": {"stored": 20, "objects": 5, "rd": 4}} ]}`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`pool_used_bytes{cluster="ceph",pool="rbd"} 20`), regexp.MustCompile(`pool_objects_total{cluster="ceph",pool="rbd"} 5`), @@ -104,6 +112,7 @@ func TestPoolUsageCollector(t *testing.T) { {"pools": [ {{{{"name": "rbd", "id": 11, "stats": {"stored": 20, "objects": 5, "rd": 4, "wr": 6}} ]}`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{}, reUnmatch: []*regexp.Regexp{ regexp.MustCompile(`pool_used_bytes{cluster="ceph"}`), @@ -118,6 +127,7 @@ func TestPoolUsageCollector(t *testing.T) { {"name": "rbd", "id": 11, "stats": {"stored": 20, "objects": 5, "rd": 4, "wr": 6}}, {"name": "rbd-new", "id": 12, "stats": {"stored": 50, "objects": 20, "rd": 10, "wr": 30}} ]}`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`pool_used_bytes{cluster="ceph",pool="rbd"} 20`), regexp.MustCompile(`pool_objects_total{cluster="ceph",pool="rbd"} 5`), @@ -135,6 +145,7 @@ func TestPoolUsageCollector(t *testing.T) { {"pools": [ {"name": "ssd", "id": 11, "stats": {"max_avail": 4618201748262, "objects": 5, "rd": 4, "wr": 6}} ]}`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`pool_available_bytes{cluster="ceph",pool="ssd"} 4.618201748262e\+12`), }, @@ -144,6 +155,7 @@ func TestPoolUsageCollector(t *testing.T) { {"pools": [ {"name": "ssd", "id": 11, "stats": {"percent_used": 1.3390908861765638e-06, "objects": 5, "rd": 4, "wr": 6}} ]}`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`pool_percent_used{cluster="ceph",pool="ssd"} 1.3390908861765638e\-06`), }, @@ -154,6 +166,7 @@ func TestPoolUsageCollector(t *testing.T) { {"id": 32, "name": "cinder_sas", "stats": { "stored": 71525351713, "dirty": 17124, "kb_used": 69848977, "max_avail": 6038098673664, "objects": 17124, "quota_bytes": 0, "quota_objects": 0, "stored_raw": 214576054272, "rd": 348986643, "rd_bytes": 3288983853056, "wr": 45792703, "wr_bytes": 272268791808 }}, {"id": 33, "name": "cinder_ssd", "stats": { "stored": 68865564849, "dirty": 16461, "kb_used": 67251529, "max_avail": 186205372416, "objects": 16461, "quota_bytes": 0, "quota_objects": 0, "stored_raw": 206596702208, "rd": 347, "rd_bytes": 12899328, "wr": 26721, "wr_bytes": 68882356224 }} ]}`, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`ceph_pool_available_bytes{cluster="ceph",pool="cinder_sas"} 6.038098673664e\+12`), regexp.MustCompile(`ceph_pool_dirty_objects_total{cluster="ceph",pool="cinder_sas"} 17124`), @@ -178,6 +191,32 @@ func TestPoolUsageCollector(t *testing.T) { } { func() { conn := &MockConn{} + + conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool { + v := map[string]interface{}{} + + err := json.Unmarshal(in.([]byte), &v) + require.NoError(t, err) + + return cmp.Equal(v, map[string]interface{}{ + "prefix": "version", + "format": "json", + }) + })).Return([]byte(tt.version), "", nil) + + // versions is only used to check if rbd mirror is present + conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool { + v := map[string]interface{}{} + + err := json.Unmarshal(in.([]byte), &v) + require.NoError(t, err) + + return cmp.Equal(v, map[string]interface{}{ + "prefix": "versions", + "format": "json", + }) + })).Return([]byte(`{}`), "", nil) + conn.On("MonCommand", mock.Anything).Return( []byte(tt.input), "", nil, ) @@ -186,10 +225,13 @@ func TestPoolUsageCollector(t *testing.T) { nil, fmt.Errorf("not implemented"), ) - collector := NewPoolUsageCollector(&Exporter{Conn: conn, Cluster: "ceph", Logger: logrus.New()}) - err := prometheus.Register(collector) + e := &Exporter{Conn: conn, Cluster: "ceph", Logger: logrus.New()} + e.cc = map[string]interface{}{ + "poolUsage": NewPoolUsageCollector(e), + } + err := prometheus.Register(e) require.NoError(t, err) - defer prometheus.Unregister(collector) + defer prometheus.Unregister(e) server := httptest.NewServer(promhttp.Handler()) defer server.Close() diff --git a/ceph/rbd_mirror_status_test.go b/ceph/rbd_mirror_status_test.go index 8448174..1ffae91 100644 --- a/ceph/rbd_mirror_status_test.go +++ b/ceph/rbd_mirror_status_test.go @@ -22,9 +22,11 @@ import ( "regexp" "testing" + "github.com/google/go-cmp/cmp" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -37,8 +39,10 @@ func setStatus(b []byte) { func TestRbdMirrorStatusCollector(t *testing.T) { for _, tt := range []struct { - input []byte - reMatch []*regexp.Regexp + input []byte + version string + versions string + reMatch []*regexp.Regexp }{ { input: []byte(` @@ -52,6 +56,8 @@ func TestRbdMirrorStatusCollector(t *testing.T) { } } }`), + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, + versions: `{"rbd-mirror":{"ceph version 16.2.11-98-g1984a8c (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)":3}}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`ceph_rbd_mirror_pool_status{cluster="ceph"} 1`), regexp.MustCompile(`ceph_rbd_mirror_pool_image_status{cluster="ceph"} 1`), @@ -68,6 +74,8 @@ func TestRbdMirrorStatusCollector(t *testing.T) { "states": {} } }`), + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, + versions: `{"rbd-mirror":{"ceph version 16.2.11-98-g1984a8c (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)":3}}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`ceph_rbd_mirror_pool_status{cluster="ceph"} 1`), regexp.MustCompile(`ceph_rbd_mirror_pool_daemon_status{cluster="ceph"} 1`), @@ -84,6 +92,8 @@ func TestRbdMirrorStatusCollector(t *testing.T) { "states": {} } }`), + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, + versions: `{"rbd-mirror":{"ceph version 16.2.11-98-g1984a8c (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)":3}}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`ceph_rbd_mirror_pool_status{cluster="ceph"} 0`), regexp.MustCompile(`ceph_rbd_mirror_pool_daemon_status{cluster="ceph"} 0`), @@ -100,23 +110,68 @@ func TestRbdMirrorStatusCollector(t *testing.T) { "states": {} } }`), + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, + versions: `{"rbd-mirror":{"ceph version 16.2.11-98-g1984a8c (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)":3}}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`ceph_rbd_mirror_pool_status{cluster="ceph"} 2`), regexp.MustCompile(`ceph_rbd_mirror_pool_daemon_status{cluster="ceph"} 0`), regexp.MustCompile(`ceph_rbd_mirror_pool_image_status{cluster="ceph"} 2`), }, }, + { + input: []byte(` + { + "summary": { + "health": "OK", + "daemon_health": "OK", + "image_health": "ERROR" + } + }`), + version: `{"version":"ceph version 14.2.9-12-zasd (1337) pacific (stable)"}`, + versions: `{"rbd-mirror":{"ceph version 14.2.9-12-zasd (1337) pacific (stable)":3}}`, + reMatch: []*regexp.Regexp{ + regexp.MustCompile(`ceph_rbd_mirror_pool_status{cluster="ceph"} 0`), + }, + }, } { func() { - collector := NewRbdMirrorStatusCollector(&Exporter{Cluster: "ceph", Version: Pacific, Logger: logrus.New()}) + conn := &MockConn{} + conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool { + v := map[string]interface{}{} - var rbdStatus rbdMirrorPoolStatus - setStatus(tt.input) - _ = json.Unmarshal([]byte(tt.input), &rbdStatus) + err := json.Unmarshal(in.([]byte), &v) + require.NoError(t, err) - err := prometheus.Register(collector) + return cmp.Equal(v, map[string]interface{}{ + "prefix": "version", + "format": "json", + }) + })).Return([]byte(tt.version), "", nil) + + // versions is only used to check if rbd mirror is present + conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool { + v := map[string]interface{}{} + + err := json.Unmarshal(in.([]byte), &v) + require.NoError(t, err) + + return cmp.Equal(v, map[string]interface{}{ + "prefix": "versions", + "format": "json", + }) + })).Return([]byte(tt.versions), "", nil) + + e := &Exporter{Conn: conn, Cluster: "ceph", Logger: logrus.New()} + // We do not create the rbdCollector since it will + // be automatically initiated from the output of `ceph versions` + // if the rbd-mirror key is present + e.cc = map[string]interface{}{} + + err := prometheus.Register(e) require.NoError(t, err) - defer prometheus.Unregister(collector) + defer prometheus.Unregister(e) + + setStatus(tt.input) server := httptest.NewServer(promhttp.Handler()) defer server.Close() diff --git a/ceph/rgw_test.go b/ceph/rgw_test.go index 07f07b7..9d0fea9 100644 --- a/ceph/rgw_test.go +++ b/ceph/rgw_test.go @@ -15,6 +15,7 @@ package ceph import ( + "encoding/json" "errors" "io/ioutil" "net/http" @@ -22,15 +23,18 @@ import ( "regexp" "testing" + "github.com/google/go-cmp/cmp" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) func TestRGWCollector(t *testing.T) { for _, tt := range []struct { input []byte + version string reMatch []*regexp.Regexp reUnmatch []*regexp.Regexp }{ @@ -99,6 +103,7 @@ func TestRGWCollector(t *testing.T) { } ] `), + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`ceph_rgw_gc_active_tasks{cluster="ceph"} 2`), regexp.MustCompile(`ceph_rgw_gc_active_objects{cluster="ceph"} 4`), @@ -107,7 +112,8 @@ func TestRGWCollector(t *testing.T) { }, }, { - input: []byte(`[]`), + input: []byte(`[]`), + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reMatch: []*regexp.Regexp{ regexp.MustCompile(`ceph_rgw_gc_active_tasks{cluster="ceph"} 0`), regexp.MustCompile(`ceph_rgw_gc_active_objects{cluster="ceph"} 0`), @@ -117,31 +123,62 @@ func TestRGWCollector(t *testing.T) { }, { // force an error return json deserialization - input: []byte(`[ { "bad-object": 17,,, ]`), + input: []byte(`[ { "bad-object": 17,,, ]`), + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reUnmatch: []*regexp.Regexp{ regexp.MustCompile(`ceph_rgw_gc`), }, }, { // force an error return from getRGWGCTaskList - input: nil, + input: nil, + version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`, reUnmatch: []*regexp.Regexp{ regexp.MustCompile(`ceph_rgw_gc`), }, }, } { func() { - collector := NewRGWCollector(&Exporter{Cluster: "ceph", Logger: logrus.New()}, false) // run in foreground for testing - collector.getRGWGCTaskList = func(cluster string, user string) ([]byte, error) { + conn := &MockConn{} + conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool { + v := map[string]interface{}{} + + err := json.Unmarshal(in.([]byte), &v) + require.NoError(t, err) + + return cmp.Equal(v, map[string]interface{}{ + "prefix": "version", + "format": "json", + }) + })).Return([]byte(tt.version), "", nil) + + // versions is only used to check if rbd mirror is present + conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool { + v := map[string]interface{}{} + + err := json.Unmarshal(in.([]byte), &v) + require.NoError(t, err) + + return cmp.Equal(v, map[string]interface{}{ + "prefix": "versions", + "format": "json", + }) + })).Return([]byte(`{}`), "", nil) + e := &Exporter{Conn: conn, Cluster: "ceph", Logger: logrus.New()} + e.cc = map[string]interface{}{ + "rgw": NewRGWCollector(e, false), + } + + e.cc["rgw"].(*RGWCollector).getRGWGCTaskList = func(cluster string, user string) ([]byte, error) { if tt.input != nil { return tt.input, nil } return nil, errors.New("fake error") } - err := prometheus.Register(collector) + err := prometheus.Register(e) require.NoError(t, err) - defer prometheus.Unregister(collector) + defer prometheus.Unregister(e) server := httptest.NewServer(promhttp.Handler()) defer server.Close()