ceph_exporter/ceph/osd_test.go

990 lines
31 KiB
Go

// Copyright 2022 DigitalOcean
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package ceph
import (
"encoding/json"
"io/ioutil"
"net/http"
"net/http/httptest"
"regexp"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)
const (
testOSDTreeOutput = `
{
"stray": [],
"nodes": [
{
"children": [
-14,
-15
],
"type_id": 10,
"type": "root",
"name": "default",
"id": -1
},
{
"children": [
-2
],
"pool_weights": {},
"type_id": 3,
"type": "rack",
"name": "A8R1",
"id": -14
},
{
"children": [
-3
],
"pool_weights": {},
"type_id": 3,
"type": "rack",
"name": "A8R2",
"id": -15
},
{
"children": [
45,
44,
23,
22,
21,
20,
13,
12,
11,
10,
4,
3,
2,
1,
0
],
"pool_weights": {},
"type_id": 1,
"type": "host",
"name": "prod-data01-block01",
"id": -2
},
{
"children": [
525,
524
],
"pool_weights": {},
"type_id": 1,
"type": "host",
"name": "prod-data02-block01",
"id": -3
},
{
"primary_affinity": 1,
"reweight": 1,
"status": "up",
"exists": 1,
"id": 0,
"device_class": "hdd",
"name": "osd.0",
"type": "osd",
"type_id": 0,
"crush_weight": 3.481995,
"depth": 3,
"pool_weights": {}
},
{
"primary_affinity": 1,
"reweight": 0.950027,
"status": "up",
"exists": 1,
"id": 1,
"device_class": "ssd",
"name": "osd.1",
"type": "osd",
"type_id": 0,
"crush_weight": 3.481995,
"depth": 3,
"pool_weights": {}
},
{
"primary_affinity": 1,
"reweight": 1,
"status": "up",
"exists": 1,
"id": 2,
"device_class": "ssd",
"name": "osd.2",
"type": "osd",
"type_id": 0,
"crush_weight": 3.481995,
"depth": 3,
"pool_weights": {}
},
{
"primary_affinity": 1,
"reweight": 0.980011,
"status": "up",
"exists": 1,
"id": 3,
"device_class": "ssd",
"name": "osd.3",
"type": "osd",
"type_id": 0,
"crush_weight": 3.481995,
"depth": 3,
"pool_weights": {}
},
{
"primary_affinity": 1,
"reweight": 0.980011,
"status": "up",
"exists": 1,
"id": 4,
"device_class": "ssd",
"name": "osd.4",
"type": "osd",
"type_id": 0,
"crush_weight": 3.481995,
"depth": 3,
"pool_weights": {}
},
{
"primary_affinity": 1,
"reweight": 0.980011,
"status": "up",
"exists": 1,
"id": 10,
"device_class": "ssd",
"name": "osd.10",
"type": "osd",
"type_id": 0,
"crush_weight": 3.481995,
"depth": 3,
"pool_weights": {}
},
{
"primary_affinity": 1,
"reweight": 0.980011,
"status": "up",
"exists": 1,
"id": 11,
"device_class": "ssd",
"name": "osd.11",
"type": "osd",
"type_id": 0,
"crush_weight": 3.481995,
"depth": 3,
"pool_weights": {}
},
{
"primary_affinity": 1,
"reweight": 0.980011,
"status": "up",
"exists": 1,
"id": 12,
"device_class": "ssd",
"name": "osd.12",
"type": "osd",
"type_id": 0,
"crush_weight": 3.481995,
"depth": 3,
"pool_weights": {}
},
{
"primary_affinity": 1,
"reweight": 0.980011,
"status": "up",
"exists": 1,
"id": 13,
"device_class": "ssd",
"name": "osd.13",
"type": "osd",
"type_id": 0,
"crush_weight": 3.481995,
"depth": 3,
"pool_weights": {}
},
{
"primary_affinity": 1,
"reweight": 0.980011,
"status": "up",
"exists": 1,
"id": 20,
"device_class": "ssd",
"name": "osd.20",
"type": "osd",
"type_id": 0,
"crush_weight": 3.481995,
"depth": 3,
"pool_weights": {}
},
{
"primary_affinity": 1,
"reweight": 0.980011,
"status": "up",
"exists": 1,
"id": 21,
"device_class": "ssd",
"name": "osd.21",
"type": "osd",
"type_id": 0,
"crush_weight": 3.481995,
"depth": 3,
"pool_weights": {}
},
{
"primary_affinity": 1,
"reweight": 0.980011,
"status": "up",
"exists": 1,
"id": 22,
"device_class": "ssd",
"name": "osd.22",
"type": "osd",
"type_id": 0,
"crush_weight": 3.481995,
"depth": 3,
"pool_weights": {}
},
{
"primary_affinity": 1,
"reweight": 0.980011,
"status": "up",
"exists": 1,
"id": 23,
"device_class": "ssd",
"name": "osd.23",
"type": "osd",
"type_id": 0,
"crush_weight": 3.481995,
"depth": 3,
"pool_weights": {}
},
{
"primary_affinity": 1,
"reweight": 0.980011,
"status": "up",
"exists": 1,
"id": 44,
"device_class": "ssd",
"name": "osd.44",
"type": "osd",
"type_id": 0,
"crush_weight": 3.481995,
"depth": 3,
"pool_weights": {}
},
{
"primary_affinity": 1,
"reweight": 0.980011,
"status": "up",
"exists": 1,
"id": 45,
"device_class": "ssd",
"name": "osd.45",
"type": "osd",
"type_id": 0,
"crush_weight": 3.481995,
"depth": 3,
"pool_weights": {}
},
{
"id": 524,
"device_class": "ssd",
"name": "osd.524",
"type": "osd",
"type_id": 0,
"crush_weight": 0.000000,
"depth": 0,
"exists": 1,
"status": "destroyed",
"reweight": 0.000000,
"primary_affinity": 1.000000
},
{
"id": 525,
"device_class": "ssd",
"name": "osd.525",
"type": "osd",
"type_id": 0,
"crush_weight": 0.000000,
"depth": 0,
"exists": 1,
"status": "destroyed",
"reweight": 0.000000,
"primary_affinity": 1.000000
}
]
}
`
)
func TestOSDLabelBuilder(t *testing.T) {
osds, err := buildOSDLabels([]byte(testOSDTreeOutput))
require.NoError(t, err)
osd, ok := osds[0]
require.Truef(t, ok, "expect to find node in map")
require.Equalf(t, "osd", osd.Type, "expect to be an OSD")
require.Equalf(t, "A8R1", osd.Rack, "expect to be in the rack")
require.Equalf(t, "prod-data01-block01", osd.Host, "expect to be in the host")
require.Equalf(t, "default", osd.Root, "expect root to be default")
require.Equalf(t, "hdd", osd.DeviceClass, "expect to be an HDD")
}
func TestOSDCollector(t *testing.T) {
reMatch := []*regexp.Regexp{
regexp.MustCompile(`ceph_osd_crush_weight{cluster="ceph",device_class="hdd",host="prod-data01-block01",osd="osd.0",rack="A8R1",root="default"} 0.010391`),
regexp.MustCompile(`ceph_osd_crush_weight{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.1",rack="A8R1",root="default"} 0.010391`),
regexp.MustCompile(`ceph_osd_crush_weight{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.2",rack="A8R1",root="default"} 0.010391`),
regexp.MustCompile(`ceph_osd_crush_weight{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.3",rack="A8R1",root="default"} 0.010391`),
regexp.MustCompile(`ceph_osd_crush_weight{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.4",rack="A8R1",root="default"} 0.010391`),
regexp.MustCompile(`ceph_osd_depth{cluster="ceph",device_class="hdd",host="prod-data01-block01",osd="osd.0",rack="A8R1",root="default"} 2`),
regexp.MustCompile(`ceph_osd_depth{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.1",rack="A8R1",root="default"} 2`),
regexp.MustCompile(`ceph_osd_depth{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.2",rack="A8R1",root="default"} 2`),
regexp.MustCompile(`ceph_osd_depth{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.3",rack="A8R1",root="default"} 2`),
regexp.MustCompile(`ceph_osd_depth{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.4",rack="A8R1",root="default"} 2`),
regexp.MustCompile(`ceph_osd_reweight{cluster="ceph",device_class="hdd",host="prod-data01-block01",osd="osd.0",rack="A8R1",root="default"} 1`),
regexp.MustCompile(`ceph_osd_reweight{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.1",rack="A8R1",root="default"} 1`),
regexp.MustCompile(`ceph_osd_reweight{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.2",rack="A8R1",root="default"} 1`),
regexp.MustCompile(`ceph_osd_reweight{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.3",rack="A8R1",root="default"} 1`),
regexp.MustCompile(`ceph_osd_reweight{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.4",rack="A8R1",root="default"} 0`),
regexp.MustCompile(`ceph_osd_bytes{cluster="ceph",device_class="hdd",host="prod-data01-block01",osd="osd.0",rack="A8R1",root="default"} 1.1417923584e`),
regexp.MustCompile(`ceph_osd_bytes{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.1",rack="A8R1",root="default"} 1.1417923584e`),
regexp.MustCompile(`ceph_osd_bytes{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.2",rack="A8R1",root="default"} 1.1417923584e`),
regexp.MustCompile(`ceph_osd_bytes{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.3",rack="A8R1",root="default"} 1.1417923584e`),
regexp.MustCompile(`ceph_osd_bytes{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.4",rack="A8R1",root="default"} 0`),
regexp.MustCompile(`ceph_osd_used_bytes{cluster="ceph",device_class="hdd",host="prod-data01-block01",osd="osd.0",rack="A8R1",root="default"} 4.1750528e`),
regexp.MustCompile(`ceph_osd_used_bytes{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.1",rack="A8R1",root="default"} 4.1484288e`),
regexp.MustCompile(`ceph_osd_used_bytes{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.2",rack="A8R1",root="default"} 3.7593088e`),
regexp.MustCompile(`ceph_osd_used_bytes{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.3",rack="A8R1",root="default"} 3.7666816e`),
regexp.MustCompile(`ceph_osd_used_bytes{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.4",rack="A8R1",root="default"} 0`),
regexp.MustCompile(`ceph_osd_avail_bytes{cluster="ceph",device_class="hdd",host="prod-data01-block01",osd="osd.0",rack="A8R1",root="default"} 1.1376173056e`),
regexp.MustCompile(`ceph_osd_avail_bytes{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.1",rack="A8R1",root="default"} 1.1376439296e`),
regexp.MustCompile(`ceph_osd_avail_bytes{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.2",rack="A8R1",root="default"} 1.1380330496e`),
regexp.MustCompile(`ceph_osd_avail_bytes{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.3",rack="A8R1",root="default"} 1.1380256768e`),
regexp.MustCompile(`ceph_osd_avail_bytes{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.4",rack="A8R1",root="default"} 0`),
regexp.MustCompile(`ceph_osd_utilization{cluster="ceph",device_class="hdd",host="prod-data01-block01",osd="osd.0",rack="A8R1",root="default"} 0.365658`),
regexp.MustCompile(`ceph_osd_utilization{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.1",rack="A8R1",root="default"} 0.363326`),
regexp.MustCompile(`ceph_osd_utilization{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.2",rack="A8R1",root="default"} 0.329246`),
regexp.MustCompile(`ceph_osd_utilization{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.3",rack="A8R1",root="default"} 0.329892`),
regexp.MustCompile(`ceph_osd_utilization{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.4",rack="A8R1",root="default"} 0`),
regexp.MustCompile(`ceph_osd_variance{cluster="ceph",device_class="hdd",host="prod-data01-block01",osd="osd.0",rack="A8R1",root="default"} 1.053676`),
regexp.MustCompile(`ceph_osd_variance{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.1",rack="A8R1",root="default"} 1.046957`),
regexp.MustCompile(`ceph_osd_variance{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.2",rack="A8R1",root="default"} 0.948753`),
regexp.MustCompile(`ceph_osd_variance{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.3",rack="A8R1",root="default"} 0.950614`),
regexp.MustCompile(`ceph_osd_variance{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.4",rack="A8R1",root="default"} 0`),
regexp.MustCompile(`ceph_osd_pgs{cluster="ceph",device_class="hdd",host="prod-data01-block01",osd="osd.0",rack="A8R1",root="default"} 283`),
regexp.MustCompile(`ceph_osd_pgs{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.1",rack="A8R1",root="default"} 279`),
regexp.MustCompile(`ceph_osd_pgs{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.2",rack="A8R1",root="default"} 162`),
regexp.MustCompile(`ceph_osd_pgs{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.3",rack="A8R1",root="default"} 164`),
regexp.MustCompile(`ceph_osd_pgs{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.4",rack="A8R1",root="default"} 0`),
regexp.MustCompile(`ceph_osd_pg_upmap_items_total{cluster="ceph"} 2`),
regexp.MustCompile(`ceph_osd_total_bytes{cluster="ceph"} 4.5671694336e`),
regexp.MustCompile(`ceph_osd_total_used_bytes{cluster="ceph"} 1.5849472e`),
regexp.MustCompile(`ceph_osd_total_avail_bytes{cluster="ceph"} 4.5513199616e`),
regexp.MustCompile(`ceph_osd_average_utilization{cluster="ceph"} 0.347031`),
regexp.MustCompile(`ceph_osd_near_full_ratio{cluster="ceph"} 0.7`),
regexp.MustCompile(`ceph_osd_backfill_full_ratio{cluster="ceph"} 0.8`),
regexp.MustCompile(`ceph_osd_full_ratio{cluster="ceph"} 0.9`),
regexp.MustCompile(`ceph_osd_in{cluster="ceph",device_class="hdd",host="prod-data01-block01",osd="osd.0",rack="A8R1",root="default"} 1`),
regexp.MustCompile(`ceph_osd_in{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.1",rack="A8R1",root="default"} 1`),
regexp.MustCompile(`ceph_osd_in{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.2",rack="A8R1",root="default"} 1`),
regexp.MustCompile(`ceph_osd_in{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.3",rack="A8R1",root="default"} 1`),
regexp.MustCompile(`ceph_osd_in{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.4",rack="A8R1",root="default"} 0`),
regexp.MustCompile(`ceph_osd_up{cluster="ceph",device_class="hdd",host="prod-data01-block01",osd="osd.0",rack="A8R1",root="default"} 1`),
regexp.MustCompile(`ceph_osd_up{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.1",rack="A8R1",root="default"} 1`),
regexp.MustCompile(`ceph_osd_up{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.2",rack="A8R1",root="default"} 1`),
regexp.MustCompile(`ceph_osd_up{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.3",rack="A8R1",root="default"} 1`),
regexp.MustCompile(`ceph_osd_up{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.4",rack="A8R1",root="default"} 0`),
regexp.MustCompile(`ceph_osd_full{cluster="ceph",device_class="hdd",host="prod-data01-block01",osd="osd.0",rack="A8R1",root="default"} 0`),
regexp.MustCompile(`ceph_osd_full{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.1",rack="A8R1",root="default"} 0`),
regexp.MustCompile(`ceph_osd_full{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.2",rack="A8R1",root="default"} 0`),
regexp.MustCompile(`ceph_osd_full{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.3",rack="A8R1",root="default"} 1`),
regexp.MustCompile(`ceph_osd_full{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.4",rack="A8R1",root="default"} 0`),
regexp.MustCompile(`ceph_osd_near_full{cluster="ceph",device_class="hdd",host="prod-data01-block01",osd="osd.0",rack="A8R1",root="default"} 0`),
regexp.MustCompile(`ceph_osd_near_full{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.1",rack="A8R1",root="default"} 0`),
regexp.MustCompile(`ceph_osd_near_full{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.2",rack="A8R1",root="default"} 1`),
regexp.MustCompile(`ceph_osd_near_full{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.3",rack="A8R1",root="default"} 0`),
regexp.MustCompile(`ceph_osd_near_full{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.4",rack="A8R1",root="default"} 0`),
regexp.MustCompile(`ceph_osd_backfill_full{cluster="ceph",device_class="hdd",host="prod-data01-block01",osd="osd.0",rack="A8R1",root="default"} 0`),
regexp.MustCompile(`ceph_osd_backfill_full{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.1",rack="A8R1",root="default"} 0`),
regexp.MustCompile(`ceph_osd_backfill_full{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.2",rack="A8R1",root="default"} 0`),
regexp.MustCompile(`ceph_osd_backfill_full{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.3",rack="A8R1",root="default"} 1`),
regexp.MustCompile(`ceph_osd_backfill_full{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.4",rack="A8R1",root="default"} 1`),
regexp.MustCompile(`ceph_osd_scrub_state{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.10",rack="default",root="default"} 1`),
regexp.MustCompile(`ceph_osd_scrub_state{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.11",rack="default",root="default"} 1`),
regexp.MustCompile(`ceph_osd_scrub_state{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.12",rack="default",root="default"} 1`),
regexp.MustCompile(`ceph_osd_scrub_state{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.13",rack="default",root="default"} 1`),
regexp.MustCompile(`ceph_osd_scrub_state{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.20",rack="default",root="default"} 2`),
regexp.MustCompile(`ceph_osd_scrub_state{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.21",rack="default",root="default"} 2`),
regexp.MustCompile(`ceph_osd_scrub_state{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.22",rack="default",root="default"} 2`),
regexp.MustCompile(`ceph_osd_scrub_state{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.23",rack="default",root="default"} 2`),
}
for _, tt := range []struct {
test string
reMatch []*regexp.Regexp
}{
{
test: "1",
reMatch: []*regexp.Regexp{
regexp.MustCompile(`ceph_osd_down{cluster="ceph",device_class="ssd",host="prod-data02-block01",osd="osd.524",rack="default",root="A8R2",status="destroyed"} 1`),
},
},
{
test: "2",
reMatch: []*regexp.Regexp{
regexp.MustCompile(`ceph_osd_down{cluster="ceph",device_class="ssd",host="prod-data02-block01",osd="osd.524",rack="default",root="A8R2",status="down"} 1`),
},
},
{
test: "3",
reMatch: []*regexp.Regexp{
regexp.MustCompile(`ceph_osd_down{cluster="ceph",device_class="ssd",host="prod-data02-block01",osd="osd.524",rack="default",root="A8R2",status="destroyed"} 1`),
},
},
{
test: "4",
reMatch: []*regexp.Regexp{
regexp.MustCompile(`ceph_osd_down{cluster="ceph",device_class="ssd",host="prod-data02-block01",osd="osd.524",rack="default",root="A8R2",status="destroyed"} 1`),
regexp.MustCompile(`ceph_osd_down{cluster="ceph",device_class="ssd",host="prod-data02-block01",osd="osd.525",rack="default",root="A8R2",status="down"} 1`),
},
},
{
test: "5",
reMatch: []*regexp.Regexp{},
},
} {
func() {
conn := &MockConn{}
conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool {
v := map[string]interface{}{}
err := json.Unmarshal(in.([]byte), &v)
require.NoError(t, err)
return cmp.Equal(v, map[string]interface{}{
"prefix": "osd tree",
"format": "json",
})
})).Return([]byte(testOSDTreeOutput), "", nil)
conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool {
v := map[string]interface{}{}
err := json.Unmarshal(in.([]byte), &v)
require.NoError(t, err)
return cmp.Equal(v, map[string]interface{}{
"prefix": "osd tree",
"states": []interface{}{"down"},
"format": "json",
})
})).Return([]byte(map[string]string{
"1": `
{
"nodes": [],
"stray": [
{
"id": 524,
"name": "osd.524",
"type": "osd",
"type_id": 0,
"crush_weight": 0.000000,
"depth": 0,
"exists": 1,
"status": "destroyed",
"reweight": 0.000000,
"primary_affinity": 1.000000
}
]
}`,
"2": `
{
"nodes": [],
"stray": [
{
"id": 524,
"name": "osd.524",
"type": "osd",
"type_id": 0,
"crush_weight": 0.000000,
"depth": 0,
"exists": 1,
"status": "down",
"reweight": 0.000000,
"primary_affinity": 1.000000
}
]
}`,
"3": `
{
"nodes": [
{
"id": -18,
"name": "data",
"type": "root",
"type_id": 10,
"children": [
-20
]
},
{
"id": -20,
"name": "R1-data",
"type": "rack",
"type_id": 3,
"pool_weights": {},
"children": [
-8
]
},
{
"id": -8,
"name": "test-data03-object01",
"type": "host",
"type_id": 1,
"pool_weights": {},
"children": [
97
]
},
{
"id": 524,
"device_class": "hdd",
"name": "osd.524",
"type": "osd",
"type_id": 0,
"crush_weight": 7.265991,
"depth": 3,
"pool_weights": {},
"exists": 1,
"status": "destroyed",
"reweight": 0.000000,
"primary_affinity": 1.000000
}
],
"stray": []
}`,
"4": `
{
"nodes": [
{
"id": -18,
"name": "data",
"type": "root",
"type_id": 10,
"children": [
-20
]
},
{
"id": -20,
"name": "R1-data",
"type": "rack",
"type_id": 3,
"pool_weights": {},
"children": [
-8
]
},
{
"id": -8,
"name": "test-data03-object01",
"type": "host",
"type_id": 1,
"pool_weights": {},
"children": [
97
]
},
{
"id": 524,
"device_class": "hdd",
"name": "osd.524",
"type": "osd",
"type_id": 0,
"crush_weight": 7.265991,
"depth": 3,
"pool_weights": {},
"exists": 1,
"status": "destroyed",
"reweight": 0.000000,
"primary_affinity": 1.000000
}
],
"stray": [
{
"id": 525,
"name": "osd.525",
"type": "osd",
"type_id": 0,
"crush_weight": 0.000000,
"depth": 0,
"exists": 1,
"status": "down",
"reweight": 0.000000,
"primary_affinity": 1.000000
}
]
}`,
"5": `
{
"nodes": []}}
}`,
}[tt.test]), "", nil)
conn.On("MgrCommand", mock.MatchedBy(func(in interface{}) bool {
v := map[string]interface{}{}
uv, ok := in.([][]byte)
require.True(t, ok)
require.Len(t, uv, 1)
err := json.Unmarshal(uv[0], &v)
require.NoError(t, err)
return cmp.Equal(v, map[string]interface{}{
"prefix": "pg dump",
"dumpcontents": []interface{}{"pgs_brief"},
"format": "json",
})
})).Return([]byte(`
{
"pg_ready": true,
"pg_stats": [
{
"acting": [
1,
2,
3,
4
],
"acting_primary": 1,
"pgid": "81.1fff",
"state": "active+clean"
},
{
"acting": [
10,
11,
12,
13
],
"acting_primary": 10,
"pgid": "82.1fff",
"state": "active+clean+scrubbing"
},
{
"acting": [
20,
21,
22,
23
],
"acting_primary": 20,
"pgid": "83.1fff",
"state": "active+clean+scrubbing+deep"
}
]
}`), "", nil)
conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool {
v := map[string]interface{}{}
err := json.Unmarshal(in.([]byte), &v)
require.NoError(t, err)
return cmp.Equal(v, map[string]interface{}{
"prefix": "osd dump",
"format": "json",
})
})).Return([]byte(`
{
"full_ratio": 0.9,
"backfillfull_ratio": 0.8,
"nearfull_ratio": 0.7,
"osds": [
{
"osd": 0,
"uuid": "135b53c3",
"up": 1,
"in": 1
},
{
"osd": 1,
"uuid": "370a33f2",
"up": 1,
"in": 1
},
{
"osd": 2,
"uuid": "ca9ab3de",
"up": 1,
"in": 1,
"state": [
"nearfull",
"exists",
"up"
]
},
{
"osd": 3,
"uuid": "bef98b10",
"up": 1,
"in": 1,
"state": [
"full",
"backfillfull",
"exists",
"up"
]
},
{
"osd": 4,
"uuid": "5936c9e8",
"up": 0,
"in": 0,
"state": [
"backfillfull",
"exists",
"up"
]
}
],
"pg_upmap_items": [
{
"pgid": "1.8f",
"mappings": [
{
"from": 37,
"to": 36
}
]
},
{
"pgid": "1.90",
"mappings": [
{
"from": 37,
"to": 36
},
{
"from": 31,
"to": 30
}
]
}
]
}`), "", nil)
conn.On("MgrCommand", mock.MatchedBy(func(in interface{}) bool {
v := map[string]interface{}{}
uv, ok := in.([][]byte)
require.True(t, ok)
require.Len(t, uv, 1)
err := json.Unmarshal(uv[0], &v)
require.NoError(t, err)
return cmp.Equal(v, map[string]interface{}{
"prefix": "osd df",
"format": "json",
})
})).Return([]byte(`
{
"nodes": [
{
"id": 0,
"name": "osd.0",
"type": "osd",
"type_id": 0,
"crush_weight": 0.010391,
"depth": 2,
"reweight": 1.000000,
"kb": 11150316,
"kb_used": 40772,
"kb_avail": 11109544,
"utilization": 0.365658,
"var": 1.053676,
"pgs": 283
},
{
"id": 2,
"name": "osd.2",
"type": "osd",
"type_id": 0,
"crush_weight": 0.010391,
"depth": 2,
"reweight": 1.000000,
"kb": 11150316,
"kb_used": 36712,
"kb_avail": 11113604,
"utilization": 0.329246,
"var": 0.948753,
"pgs": 162
},
{
"id": 1,
"name": "osd.1",
"type": "osd",
"type_id": 0,
"crush_weight": 0.010391,
"depth": 2,
"reweight": 1.000000,
"kb": 11150316,
"kb_used": 40512,
"kb_avail": 11109804,
"utilization": 0.363326,
"var": 1.046957,
"pgs": 279
},
{
"id": 3,
"name": "osd.3",
"type": "osd",
"type_id": 0,
"crush_weight": 0.010391,
"depth": 2,
"reweight": 1.000000,
"kb": 11150316,
"kb_used": 36784,
"kb_avail": 11113532,
"utilization": 0.329892,
"var": 0.950614,
"pgs": 164
},
{
"id": 4,
"name": "osd.4",
"type": "osd",
"type_id": 0,
"crush_weight": 0.010391,
"depth": 2,
"reweight": 0,
"kb": 0,
"kb_used": 0,
"kb_avail": 0,
"utilization": -nan,
"var": -nan,
"pgs": 0
}
],
"stray": [],
"summary": {
"total_kb": 44601264,
"total_kb_used": 154780,
"total_kb_avail": 44446484,
"average_utilization": 0.347031,
"min_var": 0.948753,
"max_var": 1.053676,
"dev": 0.017482
}
}`), "", nil)
conn.On("MgrCommand", mock.MatchedBy(func(in interface{}) bool {
v := map[string]interface{}{}
uv, ok := in.([][]byte)
require.True(t, ok)
require.Len(t, uv, 1)
err := json.Unmarshal(uv[0], &v)
require.NoError(t, err)
return cmp.Equal(v, map[string]interface{}{
"prefix": "osd perf",
"format": "json",
})
})).Return([]byte(`
{
"osdstats": {
"osd_perf_infos": [
{
"id": 4,
"perf_stats": {
"commit_latency_ms": 0,
"apply_latency_ms": 0
}
},
{
"id": 3,
"perf_stats": {
"commit_latency_ms": 1,
"apply_latency_ms": 64
}
},
{
"id": 2,
"perf_stats": {
"commit_latency_ms": 2,
"apply_latency_ms": 79
}
},
{
"id": 1,
"perf_stats": {
"commit_latency_ms": 2,
"apply_latency_ms": 39
}
},
{
"id": 0,
"perf_stats": {
"commit_latency_ms": 2,
"apply_latency_ms": 31
}
}
]
}
}`), "", nil)
collector := NewOSDCollector(&Exporter{Conn: conn, Cluster: "ceph", Logger: logrus.New()})
err := prometheus.Register(collector)
require.NoError(t, err)
defer prometheus.Unregister(collector)
server := httptest.NewServer(promhttp.Handler())
defer server.Close()
resp, err := http.Get(server.URL)
require.NoError(t, err)
defer resp.Body.Close()
buf, err := ioutil.ReadAll(resp.Body)
require.NoError(t, err)
for _, re := range append(reMatch, tt.reMatch...) {
require.True(t, re.Match(buf))
}
}()
}
}