1048 lines
34 KiB
Go
1048 lines
34 KiB
Go
// Copyright 2022 DigitalOcean
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package ceph
|
|
|
|
import (
|
|
"encoding/json"
|
|
"io/ioutil"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"regexp"
|
|
"testing"
|
|
|
|
"github.com/google/go-cmp/cmp"
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
|
"github.com/sirupsen/logrus"
|
|
"github.com/stretchr/testify/mock"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
const (
|
|
testOSDTreeOutput = `
|
|
{
|
|
"stray": [],
|
|
"nodes": [
|
|
{
|
|
"children": [
|
|
-14,
|
|
-15
|
|
],
|
|
"type_id": 10,
|
|
"type": "root",
|
|
"name": "default",
|
|
"id": -1
|
|
},
|
|
{
|
|
"children": [
|
|
-2
|
|
],
|
|
"pool_weights": {},
|
|
"type_id": 3,
|
|
"type": "rack",
|
|
"name": "A8R1",
|
|
"id": -14
|
|
},
|
|
{
|
|
"children": [
|
|
-3
|
|
],
|
|
"pool_weights": {},
|
|
"type_id": 3,
|
|
"type": "rack",
|
|
"name": "A8R2",
|
|
"id": -15
|
|
},
|
|
{
|
|
"children": [
|
|
45,
|
|
44,
|
|
23,
|
|
22,
|
|
21,
|
|
20,
|
|
13,
|
|
12,
|
|
11,
|
|
10,
|
|
4,
|
|
3,
|
|
2,
|
|
1,
|
|
0
|
|
],
|
|
"pool_weights": {},
|
|
"type_id": 1,
|
|
"type": "host",
|
|
"name": "prod-data01-block01",
|
|
"id": -2
|
|
},
|
|
{
|
|
"children": [
|
|
525,
|
|
524
|
|
],
|
|
"pool_weights": {},
|
|
"type_id": 1,
|
|
"type": "host",
|
|
"name": "prod-data02-block01",
|
|
"id": -3
|
|
},
|
|
{
|
|
"primary_affinity": 1,
|
|
"reweight": 1,
|
|
"status": "up",
|
|
"exists": 1,
|
|
"id": 0,
|
|
"device_class": "hdd",
|
|
"name": "osd.0",
|
|
"type": "osd",
|
|
"type_id": 0,
|
|
"crush_weight": 3.481995,
|
|
"depth": 3,
|
|
"pool_weights": {}
|
|
},
|
|
{
|
|
"primary_affinity": 1,
|
|
"reweight": 0.950027,
|
|
"status": "up",
|
|
"exists": 1,
|
|
"id": 1,
|
|
"device_class": "ssd",
|
|
"name": "osd.1",
|
|
"type": "osd",
|
|
"type_id": 0,
|
|
"crush_weight": 3.481995,
|
|
"depth": 3,
|
|
"pool_weights": {}
|
|
},
|
|
{
|
|
"primary_affinity": 1,
|
|
"reweight": 1,
|
|
"status": "up",
|
|
"exists": 1,
|
|
"id": 2,
|
|
"device_class": "ssd",
|
|
"name": "osd.2",
|
|
"type": "osd",
|
|
"type_id": 0,
|
|
"crush_weight": 3.481995,
|
|
"depth": 3,
|
|
"pool_weights": {}
|
|
},
|
|
{
|
|
"primary_affinity": 1,
|
|
"reweight": 0.980011,
|
|
"status": "up",
|
|
"exists": 1,
|
|
"id": 3,
|
|
"device_class": "ssd",
|
|
"name": "osd.3",
|
|
"type": "osd",
|
|
"type_id": 0,
|
|
"crush_weight": 3.481995,
|
|
"depth": 3,
|
|
"pool_weights": {}
|
|
},
|
|
{
|
|
"primary_affinity": 1,
|
|
"reweight": 0.980011,
|
|
"status": "up",
|
|
"exists": 1,
|
|
"id": 4,
|
|
"device_class": "ssd",
|
|
"name": "osd.4",
|
|
"type": "osd",
|
|
"type_id": 0,
|
|
"crush_weight": 3.481995,
|
|
"depth": 3,
|
|
"pool_weights": {}
|
|
},
|
|
{
|
|
"primary_affinity": 1,
|
|
"reweight": 0.980011,
|
|
"status": "up",
|
|
"exists": 1,
|
|
"id": 10,
|
|
"device_class": "ssd",
|
|
"name": "osd.10",
|
|
"type": "osd",
|
|
"type_id": 0,
|
|
"crush_weight": 3.481995,
|
|
"depth": 3,
|
|
"pool_weights": {}
|
|
},
|
|
{
|
|
"primary_affinity": 1,
|
|
"reweight": 0.980011,
|
|
"status": "up",
|
|
"exists": 1,
|
|
"id": 11,
|
|
"device_class": "ssd",
|
|
"name": "osd.11",
|
|
"type": "osd",
|
|
"type_id": 0,
|
|
"crush_weight": 3.481995,
|
|
"depth": 3,
|
|
"pool_weights": {}
|
|
},
|
|
{
|
|
"primary_affinity": 1,
|
|
"reweight": 0.980011,
|
|
"status": "up",
|
|
"exists": 1,
|
|
"id": 12,
|
|
"device_class": "ssd",
|
|
"name": "osd.12",
|
|
"type": "osd",
|
|
"type_id": 0,
|
|
"crush_weight": 3.481995,
|
|
"depth": 3,
|
|
"pool_weights": {}
|
|
},
|
|
{
|
|
"primary_affinity": 1,
|
|
"reweight": 0.980011,
|
|
"status": "up",
|
|
"exists": 1,
|
|
"id": 13,
|
|
"device_class": "ssd",
|
|
"name": "osd.13",
|
|
"type": "osd",
|
|
"type_id": 0,
|
|
"crush_weight": 3.481995,
|
|
"depth": 3,
|
|
"pool_weights": {}
|
|
},
|
|
{
|
|
"primary_affinity": 1,
|
|
"reweight": 0.980011,
|
|
"status": "up",
|
|
"exists": 1,
|
|
"id": 20,
|
|
"device_class": "ssd",
|
|
"name": "osd.20",
|
|
"type": "osd",
|
|
"type_id": 0,
|
|
"crush_weight": 3.481995,
|
|
"depth": 3,
|
|
"pool_weights": {}
|
|
},
|
|
{
|
|
"primary_affinity": 1,
|
|
"reweight": 0.980011,
|
|
"status": "up",
|
|
"exists": 1,
|
|
"id": 21,
|
|
"device_class": "ssd",
|
|
"name": "osd.21",
|
|
"type": "osd",
|
|
"type_id": 0,
|
|
"crush_weight": 3.481995,
|
|
"depth": 3,
|
|
"pool_weights": {}
|
|
},
|
|
{
|
|
"primary_affinity": 1,
|
|
"reweight": 0.980011,
|
|
"status": "up",
|
|
"exists": 1,
|
|
"id": 22,
|
|
"device_class": "ssd",
|
|
"name": "osd.22",
|
|
"type": "osd",
|
|
"type_id": 0,
|
|
"crush_weight": 3.481995,
|
|
"depth": 3,
|
|
"pool_weights": {}
|
|
},
|
|
{
|
|
"primary_affinity": 1,
|
|
"reweight": 0.980011,
|
|
"status": "up",
|
|
"exists": 1,
|
|
"id": 23,
|
|
"device_class": "ssd",
|
|
"name": "osd.23",
|
|
"type": "osd",
|
|
"type_id": 0,
|
|
"crush_weight": 3.481995,
|
|
"depth": 3,
|
|
"pool_weights": {}
|
|
},
|
|
{
|
|
"primary_affinity": 1,
|
|
"reweight": 0.980011,
|
|
"status": "up",
|
|
"exists": 1,
|
|
"id": 44,
|
|
"device_class": "ssd",
|
|
"name": "osd.44",
|
|
"type": "osd",
|
|
"type_id": 0,
|
|
"crush_weight": 3.481995,
|
|
"depth": 3,
|
|
"pool_weights": {}
|
|
},
|
|
{
|
|
"primary_affinity": 1,
|
|
"reweight": 0.980011,
|
|
"status": "up",
|
|
"exists": 1,
|
|
"id": 45,
|
|
"device_class": "ssd",
|
|
"name": "osd.45",
|
|
"type": "osd",
|
|
"type_id": 0,
|
|
"crush_weight": 3.481995,
|
|
"depth": 3,
|
|
"pool_weights": {}
|
|
},
|
|
{
|
|
"id": 524,
|
|
"device_class": "ssd",
|
|
"name": "osd.524",
|
|
"type": "osd",
|
|
"type_id": 0,
|
|
"crush_weight": 0.000000,
|
|
"depth": 0,
|
|
"exists": 1,
|
|
"status": "destroyed",
|
|
"reweight": 0.000000,
|
|
"primary_affinity": 1.000000
|
|
},
|
|
{
|
|
"id": 525,
|
|
"device_class": "ssd",
|
|
"name": "osd.525",
|
|
"type": "osd",
|
|
"type_id": 0,
|
|
"crush_weight": 0.000000,
|
|
"depth": 0,
|
|
"exists": 1,
|
|
"status": "destroyed",
|
|
"reweight": 0.000000,
|
|
"primary_affinity": 1.000000
|
|
}
|
|
]
|
|
}
|
|
`
|
|
)
|
|
|
|
func TestOSDLabelBuilder(t *testing.T) {
|
|
osds, err := buildOSDLabels([]byte(testOSDTreeOutput))
|
|
require.NoError(t, err)
|
|
|
|
osd, ok := osds[0]
|
|
require.Truef(t, ok, "expect to find node in map")
|
|
require.Equalf(t, "osd", osd.Type, "expect to be an OSD")
|
|
require.Equalf(t, "A8R1", osd.Rack, "expect to be in the rack")
|
|
require.Equalf(t, "prod-data01-block01", osd.Host, "expect to be in the host")
|
|
require.Equalf(t, "default", osd.Root, "expect root to be default")
|
|
require.Equalf(t, "hdd", osd.DeviceClass, "expect to be an HDD")
|
|
}
|
|
|
|
func TestOSDCollector(t *testing.T) {
|
|
reMatch := []*regexp.Regexp{
|
|
regexp.MustCompile(`ceph_osd_crush_weight{cluster="ceph",device_class="hdd",host="prod-data01-block01",osd="osd.0",rack="A8R1",root="default"} 0.010391`),
|
|
regexp.MustCompile(`ceph_osd_crush_weight{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.1",rack="A8R1",root="default"} 0.010391`),
|
|
regexp.MustCompile(`ceph_osd_crush_weight{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.2",rack="A8R1",root="default"} 0.010391`),
|
|
regexp.MustCompile(`ceph_osd_crush_weight{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.3",rack="A8R1",root="default"} 0.010391`),
|
|
regexp.MustCompile(`ceph_osd_crush_weight{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.4",rack="A8R1",root="default"} 0.010391`),
|
|
regexp.MustCompile(`ceph_osd_depth{cluster="ceph",device_class="hdd",host="prod-data01-block01",osd="osd.0",rack="A8R1",root="default"} 2`),
|
|
regexp.MustCompile(`ceph_osd_depth{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.1",rack="A8R1",root="default"} 2`),
|
|
regexp.MustCompile(`ceph_osd_depth{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.2",rack="A8R1",root="default"} 2`),
|
|
regexp.MustCompile(`ceph_osd_depth{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.3",rack="A8R1",root="default"} 2`),
|
|
regexp.MustCompile(`ceph_osd_depth{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.4",rack="A8R1",root="default"} 2`),
|
|
regexp.MustCompile(`ceph_osd_reweight{cluster="ceph",device_class="hdd",host="prod-data01-block01",osd="osd.0",rack="A8R1",root="default"} 1`),
|
|
regexp.MustCompile(`ceph_osd_reweight{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.1",rack="A8R1",root="default"} 1`),
|
|
regexp.MustCompile(`ceph_osd_reweight{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.2",rack="A8R1",root="default"} 1`),
|
|
regexp.MustCompile(`ceph_osd_reweight{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.3",rack="A8R1",root="default"} 1`),
|
|
regexp.MustCompile(`ceph_osd_reweight{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.4",rack="A8R1",root="default"} 0`),
|
|
regexp.MustCompile(`ceph_osd_bytes{cluster="ceph",device_class="hdd",host="prod-data01-block01",osd="osd.0",rack="A8R1",root="default"} 1.1417923584e`),
|
|
regexp.MustCompile(`ceph_osd_bytes{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.1",rack="A8R1",root="default"} 1.1417923584e`),
|
|
regexp.MustCompile(`ceph_osd_bytes{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.2",rack="A8R1",root="default"} 1.1417923584e`),
|
|
regexp.MustCompile(`ceph_osd_bytes{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.3",rack="A8R1",root="default"} 1.1417923584e`),
|
|
regexp.MustCompile(`ceph_osd_bytes{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.4",rack="A8R1",root="default"} 0`),
|
|
regexp.MustCompile(`ceph_osd_used_bytes{cluster="ceph",device_class="hdd",host="prod-data01-block01",osd="osd.0",rack="A8R1",root="default"} 4.1750528e`),
|
|
regexp.MustCompile(`ceph_osd_used_bytes{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.1",rack="A8R1",root="default"} 4.1484288e`),
|
|
regexp.MustCompile(`ceph_osd_used_bytes{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.2",rack="A8R1",root="default"} 3.7593088e`),
|
|
regexp.MustCompile(`ceph_osd_used_bytes{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.3",rack="A8R1",root="default"} 3.7666816e`),
|
|
regexp.MustCompile(`ceph_osd_used_bytes{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.4",rack="A8R1",root="default"} 0`),
|
|
regexp.MustCompile(`ceph_osd_avail_bytes{cluster="ceph",device_class="hdd",host="prod-data01-block01",osd="osd.0",rack="A8R1",root="default"} 1.1376173056e`),
|
|
regexp.MustCompile(`ceph_osd_avail_bytes{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.1",rack="A8R1",root="default"} 1.1376439296e`),
|
|
regexp.MustCompile(`ceph_osd_avail_bytes{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.2",rack="A8R1",root="default"} 1.1380330496e`),
|
|
regexp.MustCompile(`ceph_osd_avail_bytes{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.3",rack="A8R1",root="default"} 1.1380256768e`),
|
|
regexp.MustCompile(`ceph_osd_avail_bytes{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.4",rack="A8R1",root="default"} 0`),
|
|
regexp.MustCompile(`ceph_osd_utilization{cluster="ceph",device_class="hdd",host="prod-data01-block01",osd="osd.0",rack="A8R1",root="default"} 0.365658`),
|
|
regexp.MustCompile(`ceph_osd_utilization{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.1",rack="A8R1",root="default"} 0.363326`),
|
|
regexp.MustCompile(`ceph_osd_utilization{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.2",rack="A8R1",root="default"} 0.329246`),
|
|
regexp.MustCompile(`ceph_osd_utilization{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.3",rack="A8R1",root="default"} 0.329892`),
|
|
regexp.MustCompile(`ceph_osd_utilization{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.4",rack="A8R1",root="default"} 0`),
|
|
regexp.MustCompile(`ceph_osd_variance{cluster="ceph",device_class="hdd",host="prod-data01-block01",osd="osd.0",rack="A8R1",root="default"} 1.053676`),
|
|
regexp.MustCompile(`ceph_osd_variance{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.1",rack="A8R1",root="default"} 1.046957`),
|
|
regexp.MustCompile(`ceph_osd_variance{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.2",rack="A8R1",root="default"} 0.948753`),
|
|
regexp.MustCompile(`ceph_osd_variance{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.3",rack="A8R1",root="default"} 0.950614`),
|
|
regexp.MustCompile(`ceph_osd_variance{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.4",rack="A8R1",root="default"} 0`),
|
|
regexp.MustCompile(`ceph_osd_pgs{cluster="ceph",device_class="hdd",host="prod-data01-block01",osd="osd.0",rack="A8R1",root="default"} 283`),
|
|
regexp.MustCompile(`ceph_osd_pgs{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.1",rack="A8R1",root="default"} 279`),
|
|
regexp.MustCompile(`ceph_osd_pgs{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.2",rack="A8R1",root="default"} 162`),
|
|
regexp.MustCompile(`ceph_osd_pgs{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.3",rack="A8R1",root="default"} 164`),
|
|
regexp.MustCompile(`ceph_osd_pgs{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.4",rack="A8R1",root="default"} 0`),
|
|
regexp.MustCompile(`ceph_osd_pg_upmap_items_total{cluster="ceph"} 2`),
|
|
regexp.MustCompile(`ceph_osd_total_bytes{cluster="ceph"} 4.5671694336e`),
|
|
regexp.MustCompile(`ceph_osd_total_used_bytes{cluster="ceph"} 1.5849472e`),
|
|
regexp.MustCompile(`ceph_osd_total_avail_bytes{cluster="ceph"} 4.5513199616e`),
|
|
regexp.MustCompile(`ceph_osd_average_utilization{cluster="ceph"} 0.347031`),
|
|
regexp.MustCompile(`ceph_osd_near_full_ratio{cluster="ceph"} 0.7`),
|
|
regexp.MustCompile(`ceph_osd_backfill_full_ratio{cluster="ceph"} 0.8`),
|
|
regexp.MustCompile(`ceph_osd_full_ratio{cluster="ceph"} 0.9`),
|
|
regexp.MustCompile(`ceph_osd_in{cluster="ceph",device_class="hdd",host="prod-data01-block01",osd="osd.0",rack="A8R1",root="default"} 1`),
|
|
regexp.MustCompile(`ceph_osd_in{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.1",rack="A8R1",root="default"} 1`),
|
|
regexp.MustCompile(`ceph_osd_in{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.2",rack="A8R1",root="default"} 1`),
|
|
regexp.MustCompile(`ceph_osd_in{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.3",rack="A8R1",root="default"} 1`),
|
|
regexp.MustCompile(`ceph_osd_in{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.4",rack="A8R1",root="default"} 0`),
|
|
regexp.MustCompile(`ceph_osd_metadata{ceph_version_when_created="ceph version 16.2.11-119-g6e981ce \(6e981ceb1084ad7628ea32a6a0a23ce09bc5cf8b\) pacific \(stable\)",cluster="ceph",created_at="2023-03-24T20:25:57.763728Z",objectstore="bluestore",osd="0"} 1`),
|
|
regexp.MustCompile(`ceph_osd_metadata{ceph_version_when_created="",cluster="ceph",created_at="",objectstore="filestore",osd="1"} 1`),
|
|
regexp.MustCompile(`ceph_osd_metadata{ceph_version_when_created="ceph version 16.2.11-119-g6e981ce \(6e981ceb1084ad7628ea32a6a0a23ce09bc5cf8b\) pacific \(stable\)",cluster="ceph",created_at="2023-03-24T20:25:57.763728Z",objectstore="bluestore",osd="2"} 1`),
|
|
regexp.MustCompile(`ceph_osd_metadata{ceph_version_when_created="",cluster="ceph",created_at="",objectstore="filestore",osd="3"} 1`),
|
|
regexp.MustCompile(`ceph_osd_metadata{ceph_version_when_created="",cluster="ceph",created_at="",objectstore="filestore",osd="4"} 1`),
|
|
regexp.MustCompile(`ceph_osd_up{cluster="ceph",device_class="hdd",host="prod-data01-block01",osd="osd.0",rack="A8R1",root="default"} 1`),
|
|
regexp.MustCompile(`ceph_osd_up{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.1",rack="A8R1",root="default"} 1`),
|
|
regexp.MustCompile(`ceph_osd_up{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.2",rack="A8R1",root="default"} 1`),
|
|
regexp.MustCompile(`ceph_osd_up{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.3",rack="A8R1",root="default"} 1`),
|
|
regexp.MustCompile(`ceph_osd_up{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.4",rack="A8R1",root="default"} 0`),
|
|
regexp.MustCompile(`ceph_osd_full{cluster="ceph",device_class="hdd",host="prod-data01-block01",osd="osd.0",rack="A8R1",root="default"} 0`),
|
|
regexp.MustCompile(`ceph_osd_full{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.1",rack="A8R1",root="default"} 0`),
|
|
regexp.MustCompile(`ceph_osd_full{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.2",rack="A8R1",root="default"} 0`),
|
|
regexp.MustCompile(`ceph_osd_full{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.3",rack="A8R1",root="default"} 1`),
|
|
regexp.MustCompile(`ceph_osd_full{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.4",rack="A8R1",root="default"} 0`),
|
|
regexp.MustCompile(`ceph_osd_near_full{cluster="ceph",device_class="hdd",host="prod-data01-block01",osd="osd.0",rack="A8R1",root="default"} 0`),
|
|
regexp.MustCompile(`ceph_osd_near_full{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.1",rack="A8R1",root="default"} 0`),
|
|
regexp.MustCompile(`ceph_osd_near_full{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.2",rack="A8R1",root="default"} 1`),
|
|
regexp.MustCompile(`ceph_osd_near_full{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.3",rack="A8R1",root="default"} 0`),
|
|
regexp.MustCompile(`ceph_osd_near_full{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.4",rack="A8R1",root="default"} 0`),
|
|
regexp.MustCompile(`ceph_osd_backfill_full{cluster="ceph",device_class="hdd",host="prod-data01-block01",osd="osd.0",rack="A8R1",root="default"} 0`),
|
|
regexp.MustCompile(`ceph_osd_backfill_full{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.1",rack="A8R1",root="default"} 0`),
|
|
regexp.MustCompile(`ceph_osd_backfill_full{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.2",rack="A8R1",root="default"} 0`),
|
|
regexp.MustCompile(`ceph_osd_backfill_full{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.3",rack="A8R1",root="default"} 1`),
|
|
regexp.MustCompile(`ceph_osd_backfill_full{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.4",rack="A8R1",root="default"} 1`),
|
|
|
|
regexp.MustCompile(`ceph_osd_scrub_state{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.10",rack="A8R1",root="default"} 1`),
|
|
regexp.MustCompile(`ceph_osd_scrub_state{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.11",rack="A8R1",root="default"} 1`),
|
|
regexp.MustCompile(`ceph_osd_scrub_state{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.12",rack="A8R1",root="default"} 1`),
|
|
regexp.MustCompile(`ceph_osd_scrub_state{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.13",rack="A8R1",root="default"} 1`),
|
|
regexp.MustCompile(`ceph_osd_scrub_state{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.20",rack="A8R1",root="default"} 2`),
|
|
regexp.MustCompile(`ceph_osd_scrub_state{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.21",rack="A8R1",root="default"} 2`),
|
|
regexp.MustCompile(`ceph_osd_scrub_state{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.22",rack="A8R1",root="default"} 2`),
|
|
regexp.MustCompile(`ceph_osd_scrub_state{cluster="ceph",device_class="ssd",host="prod-data01-block01",osd="osd.23",rack="A8R1",root="default"} 2`),
|
|
}
|
|
|
|
for _, tt := range []struct {
|
|
test string
|
|
version string
|
|
reMatch []*regexp.Regexp
|
|
}{
|
|
{
|
|
test: "1",
|
|
version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`,
|
|
reMatch: []*regexp.Regexp{
|
|
regexp.MustCompile(`ceph_osd_down{cluster="ceph",device_class="ssd",host="prod-data02-block01",osd="osd.524",rack="A8R2",root="default",status="destroyed"} 1`),
|
|
},
|
|
},
|
|
{
|
|
test: "2",
|
|
version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`,
|
|
reMatch: []*regexp.Regexp{
|
|
regexp.MustCompile(`ceph_osd_down{cluster="ceph",device_class="ssd",host="prod-data02-block01",osd="osd.524",rack="A8R2",root="default",status="down"} 1`),
|
|
},
|
|
},
|
|
{
|
|
test: "3",
|
|
version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`,
|
|
reMatch: []*regexp.Regexp{
|
|
regexp.MustCompile(`ceph_osd_down{cluster="ceph",device_class="ssd",host="prod-data02-block01",osd="osd.524",rack="A8R2",root="default",status="destroyed"} 1`),
|
|
},
|
|
},
|
|
{
|
|
test: "4",
|
|
version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`,
|
|
reMatch: []*regexp.Regexp{
|
|
regexp.MustCompile(`ceph_osd_down{cluster="ceph",device_class="ssd",host="prod-data02-block01",osd="osd.524",rack="A8R2",root="default",status="destroyed"} 1`),
|
|
regexp.MustCompile(`ceph_osd_down{cluster="ceph",device_class="ssd",host="prod-data02-block01",osd="osd.525",rack="A8R2",root="default",status="down"} 1`),
|
|
},
|
|
},
|
|
{
|
|
test: "5",
|
|
version: `{"version":"ceph version 16.2.11-22-wasd (1984a8c33225d70559cdf27dbab81e3ce153f6ac) pacific (stable)"}`,
|
|
reMatch: []*regexp.Regexp{},
|
|
},
|
|
} {
|
|
func() {
|
|
conn := setupVersionMocks(tt.version, "{}")
|
|
|
|
conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool {
|
|
v := map[string]interface{}{}
|
|
|
|
err := json.Unmarshal(in.([]byte), &v)
|
|
require.NoError(t, err)
|
|
|
|
return cmp.Equal(v, map[string]interface{}{
|
|
"prefix": "osd tree",
|
|
"format": "json",
|
|
})
|
|
})).Return([]byte(testOSDTreeOutput), "", nil)
|
|
|
|
conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool {
|
|
v := map[string]interface{}{}
|
|
|
|
err := json.Unmarshal(in.([]byte), &v)
|
|
require.NoError(t, err)
|
|
|
|
return cmp.Equal(v, map[string]interface{}{
|
|
"prefix": "osd tree",
|
|
"states": []interface{}{"down"},
|
|
"format": "json",
|
|
})
|
|
})).Return([]byte(map[string]string{
|
|
"1": `
|
|
{
|
|
"nodes": [],
|
|
"stray": [
|
|
{
|
|
"id": 524,
|
|
"name": "osd.524",
|
|
"type": "osd",
|
|
"type_id": 0,
|
|
"crush_weight": 0.000000,
|
|
"depth": 0,
|
|
"exists": 1,
|
|
"status": "destroyed",
|
|
"reweight": 0.000000,
|
|
"primary_affinity": 1.000000
|
|
}
|
|
]
|
|
}`,
|
|
|
|
"2": `
|
|
{
|
|
"nodes": [],
|
|
"stray": [
|
|
{
|
|
"id": 524,
|
|
"name": "osd.524",
|
|
"type": "osd",
|
|
"type_id": 0,
|
|
"crush_weight": 0.000000,
|
|
"depth": 0,
|
|
"exists": 1,
|
|
"status": "down",
|
|
"reweight": 0.000000,
|
|
"primary_affinity": 1.000000
|
|
}
|
|
]
|
|
}`,
|
|
"3": `
|
|
{
|
|
"nodes": [
|
|
{
|
|
"id": -18,
|
|
"name": "data",
|
|
"type": "root",
|
|
"type_id": 10,
|
|
"children": [
|
|
-20
|
|
]
|
|
},
|
|
{
|
|
"id": -20,
|
|
"name": "R1-data",
|
|
"type": "rack",
|
|
"type_id": 3,
|
|
"pool_weights": {},
|
|
"children": [
|
|
-8
|
|
]
|
|
},
|
|
{
|
|
"id": -8,
|
|
"name": "test-data03-object01",
|
|
"type": "host",
|
|
"type_id": 1,
|
|
"pool_weights": {},
|
|
"children": [
|
|
97
|
|
]
|
|
},
|
|
{
|
|
"id": 524,
|
|
"device_class": "hdd",
|
|
"name": "osd.524",
|
|
"type": "osd",
|
|
"type_id": 0,
|
|
"crush_weight": 7.265991,
|
|
"depth": 3,
|
|
"pool_weights": {},
|
|
"exists": 1,
|
|
"status": "destroyed",
|
|
"reweight": 0.000000,
|
|
"primary_affinity": 1.000000
|
|
}
|
|
],
|
|
"stray": []
|
|
}`,
|
|
"4": `
|
|
{
|
|
"nodes": [
|
|
{
|
|
"id": -18,
|
|
"name": "data",
|
|
"type": "root",
|
|
"type_id": 10,
|
|
"children": [
|
|
-20
|
|
]
|
|
},
|
|
{
|
|
"id": -20,
|
|
"name": "R1-data",
|
|
"type": "rack",
|
|
"type_id": 3,
|
|
"pool_weights": {},
|
|
"children": [
|
|
-8
|
|
]
|
|
},
|
|
{
|
|
"id": -8,
|
|
"name": "test-data03-object01",
|
|
"type": "host",
|
|
"type_id": 1,
|
|
"pool_weights": {},
|
|
"children": [
|
|
97
|
|
]
|
|
},
|
|
{
|
|
"id": 524,
|
|
"device_class": "hdd",
|
|
"name": "osd.524",
|
|
"type": "osd",
|
|
"type_id": 0,
|
|
"crush_weight": 7.265991,
|
|
"depth": 3,
|
|
"pool_weights": {},
|
|
"exists": 1,
|
|
"status": "destroyed",
|
|
"reweight": 0.000000,
|
|
"primary_affinity": 1.000000
|
|
}
|
|
],
|
|
"stray": [
|
|
{
|
|
"id": 525,
|
|
"name": "osd.525",
|
|
"type": "osd",
|
|
"type_id": 0,
|
|
"crush_weight": 0.000000,
|
|
"depth": 0,
|
|
"exists": 1,
|
|
"status": "down",
|
|
"reweight": 0.000000,
|
|
"primary_affinity": 1.000000
|
|
}
|
|
]
|
|
}`,
|
|
"5": `
|
|
{
|
|
"nodes": []}}
|
|
}`,
|
|
}[tt.test]), "", nil)
|
|
|
|
conn.On("MgrCommand", mock.MatchedBy(func(in interface{}) bool {
|
|
v := map[string]interface{}{}
|
|
|
|
uv, ok := in.([][]byte)
|
|
require.True(t, ok)
|
|
require.Len(t, uv, 1)
|
|
|
|
err := json.Unmarshal(uv[0], &v)
|
|
require.NoError(t, err)
|
|
|
|
return cmp.Equal(v, map[string]interface{}{
|
|
"prefix": "pg dump",
|
|
"dumpcontents": []interface{}{"pgs_brief"},
|
|
"format": "json",
|
|
})
|
|
})).Return([]byte(`
|
|
{
|
|
"pg_ready": true,
|
|
"pg_stats": [
|
|
{
|
|
"acting": [
|
|
1,
|
|
2,
|
|
3,
|
|
4
|
|
],
|
|
"acting_primary": 1,
|
|
"pgid": "81.1fff",
|
|
"state": "active+clean"
|
|
},
|
|
{
|
|
"acting": [
|
|
10,
|
|
11,
|
|
12,
|
|
13
|
|
],
|
|
"acting_primary": 10,
|
|
"pgid": "82.1fff",
|
|
"state": "active+clean+scrubbing"
|
|
},
|
|
{
|
|
"acting": [
|
|
20,
|
|
21,
|
|
22,
|
|
23
|
|
],
|
|
"acting_primary": 20,
|
|
"pgid": "83.1fff",
|
|
"state": "active+clean+scrubbing+deep"
|
|
}
|
|
]
|
|
}`), "", nil)
|
|
|
|
conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool {
|
|
v := map[string]interface{}{}
|
|
|
|
err := json.Unmarshal(in.([]byte), &v)
|
|
require.NoError(t, err)
|
|
|
|
return cmp.Equal(v, map[string]interface{}{
|
|
"prefix": "osd dump",
|
|
"format": "json",
|
|
})
|
|
})).Return([]byte(`
|
|
{
|
|
"full_ratio": 0.9,
|
|
"backfillfull_ratio": 0.8,
|
|
"nearfull_ratio": 0.7,
|
|
"osds": [
|
|
{
|
|
"osd": 0,
|
|
"uuid": "135b53c3",
|
|
"up": 1,
|
|
"in": 1
|
|
},
|
|
{
|
|
"osd": 1,
|
|
"uuid": "370a33f2",
|
|
"up": 1,
|
|
"in": 1
|
|
},
|
|
{
|
|
"osd": 2,
|
|
"uuid": "ca9ab3de",
|
|
"up": 1,
|
|
"in": 1,
|
|
"state": [
|
|
"nearfull",
|
|
"exists",
|
|
"up"
|
|
]
|
|
},
|
|
{
|
|
"osd": 3,
|
|
"uuid": "bef98b10",
|
|
"up": 1,
|
|
"in": 1,
|
|
"state": [
|
|
"full",
|
|
"backfillfull",
|
|
"exists",
|
|
"up"
|
|
]
|
|
},
|
|
{
|
|
"osd": 4,
|
|
"uuid": "5936c9e8",
|
|
"up": 0,
|
|
"in": 0,
|
|
"state": [
|
|
"backfillfull",
|
|
"exists",
|
|
"up"
|
|
]
|
|
}
|
|
],
|
|
"pg_upmap_items": [
|
|
{
|
|
"pgid": "1.8f",
|
|
"mappings": [
|
|
{
|
|
"from": 37,
|
|
"to": 36
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"pgid": "1.90",
|
|
"mappings": [
|
|
{
|
|
"from": 37,
|
|
"to": 36
|
|
},
|
|
{
|
|
"from": 31,
|
|
"to": 30
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}`), "", nil)
|
|
|
|
conn.On("MonCommand", mock.MatchedBy(func(in interface{}) bool {
|
|
v := map[string]interface{}{}
|
|
|
|
err := json.Unmarshal(in.([]byte), &v)
|
|
require.NoError(t, err)
|
|
|
|
return cmp.Equal(v, map[string]interface{}{
|
|
"prefix": "osd metadata",
|
|
"format": "json",
|
|
})
|
|
})).Return([]byte(`
|
|
[
|
|
{
|
|
"id": 0,
|
|
"osd_objectstore": "bluestore",
|
|
"ceph_version_when_created": "ceph version 16.2.11-119-g6e981ce (6e981ceb1084ad7628ea32a6a0a23ce09bc5cf8b) pacific (stable)",
|
|
"created_at": "2023-03-24T20:25:57.763728Z"
|
|
},
|
|
{
|
|
"id": 1,
|
|
"osd_objectstore": "filestore",
|
|
"ceph_version_when_created": "",
|
|
"created_at": ""
|
|
},
|
|
{
|
|
"id": 2,
|
|
"osd_objectstore": "bluestore",
|
|
"ceph_version_when_created": "ceph version 16.2.11-119-g6e981ce (6e981ceb1084ad7628ea32a6a0a23ce09bc5cf8b) pacific (stable)",
|
|
"created_at": "2023-03-24T20:25:57.763728Z"
|
|
},
|
|
{
|
|
"id": 3,
|
|
"osd_objectstore": "filestore",
|
|
"ceph_version_when_created": "",
|
|
"created_at": ""
|
|
},
|
|
{
|
|
"id": 4,
|
|
"osd_objectstore": "filestore",
|
|
"ceph_version_when_created": "",
|
|
"created_at": ""
|
|
}
|
|
]`), "", nil)
|
|
|
|
conn.On("MgrCommand", mock.MatchedBy(func(in interface{}) bool {
|
|
v := map[string]interface{}{}
|
|
|
|
uv, ok := in.([][]byte)
|
|
require.True(t, ok)
|
|
require.Len(t, uv, 1)
|
|
|
|
err := json.Unmarshal(uv[0], &v)
|
|
require.NoError(t, err)
|
|
|
|
return cmp.Equal(v, map[string]interface{}{
|
|
"prefix": "osd df",
|
|
"format": "json",
|
|
})
|
|
})).Return([]byte(`
|
|
{
|
|
"nodes": [
|
|
{
|
|
"id": 0,
|
|
"name": "osd.0",
|
|
"type": "osd",
|
|
"type_id": 0,
|
|
"crush_weight": 0.010391,
|
|
"depth": 2,
|
|
"reweight": 1.000000,
|
|
"kb": 11150316,
|
|
"kb_used": 40772,
|
|
"kb_avail": 11109544,
|
|
"utilization": 0.365658,
|
|
"var": 1.053676,
|
|
"pgs": 283
|
|
},
|
|
{
|
|
"id": 2,
|
|
"name": "osd.2",
|
|
"type": "osd",
|
|
"type_id": 0,
|
|
"crush_weight": 0.010391,
|
|
"depth": 2,
|
|
"reweight": 1.000000,
|
|
"kb": 11150316,
|
|
"kb_used": 36712,
|
|
"kb_avail": 11113604,
|
|
"utilization": 0.329246,
|
|
"var": 0.948753,
|
|
"pgs": 162
|
|
},
|
|
{
|
|
"id": 1,
|
|
"name": "osd.1",
|
|
"type": "osd",
|
|
"type_id": 0,
|
|
"crush_weight": 0.010391,
|
|
"depth": 2,
|
|
"reweight": 1.000000,
|
|
"kb": 11150316,
|
|
"kb_used": 40512,
|
|
"kb_avail": 11109804,
|
|
"utilization": 0.363326,
|
|
"var": 1.046957,
|
|
"pgs": 279
|
|
},
|
|
{
|
|
"id": 3,
|
|
"name": "osd.3",
|
|
"type": "osd",
|
|
"type_id": 0,
|
|
"crush_weight": 0.010391,
|
|
"depth": 2,
|
|
"reweight": 1.000000,
|
|
"kb": 11150316,
|
|
"kb_used": 36784,
|
|
"kb_avail": 11113532,
|
|
"utilization": 0.329892,
|
|
"var": 0.950614,
|
|
"pgs": 164
|
|
},
|
|
{
|
|
"id": 4,
|
|
"name": "osd.4",
|
|
"type": "osd",
|
|
"type_id": 0,
|
|
"crush_weight": 0.010391,
|
|
"depth": 2,
|
|
"reweight": 0,
|
|
"kb": 0,
|
|
"kb_used": 0,
|
|
"kb_avail": 0,
|
|
"utilization": -nan,
|
|
"var": -nan,
|
|
"pgs": 0
|
|
}
|
|
],
|
|
"stray": [],
|
|
"summary": {
|
|
"total_kb": 44601264,
|
|
"total_kb_used": 154780,
|
|
"total_kb_avail": 44446484,
|
|
"average_utilization": 0.347031,
|
|
"min_var": 0.948753,
|
|
"max_var": 1.053676,
|
|
"dev": 0.017482
|
|
}
|
|
}`), "", nil)
|
|
|
|
conn.On("MgrCommand", mock.MatchedBy(func(in interface{}) bool {
|
|
v := map[string]interface{}{}
|
|
|
|
uv, ok := in.([][]byte)
|
|
require.True(t, ok)
|
|
require.Len(t, uv, 1)
|
|
|
|
err := json.Unmarshal(uv[0], &v)
|
|
require.NoError(t, err)
|
|
|
|
return cmp.Equal(v, map[string]interface{}{
|
|
"prefix": "osd perf",
|
|
"format": "json",
|
|
})
|
|
})).Return([]byte(`
|
|
{
|
|
"osdstats": {
|
|
"osd_perf_infos": [
|
|
{
|
|
"id": 4,
|
|
"perf_stats": {
|
|
"commit_latency_ms": 0,
|
|
"apply_latency_ms": 0
|
|
}
|
|
},
|
|
{
|
|
"id": 3,
|
|
"perf_stats": {
|
|
"commit_latency_ms": 1,
|
|
"apply_latency_ms": 64
|
|
}
|
|
},
|
|
{
|
|
"id": 2,
|
|
"perf_stats": {
|
|
"commit_latency_ms": 2,
|
|
"apply_latency_ms": 79
|
|
}
|
|
},
|
|
{
|
|
"id": 1,
|
|
"perf_stats": {
|
|
"commit_latency_ms": 2,
|
|
"apply_latency_ms": 39
|
|
}
|
|
},
|
|
{
|
|
"id": 0,
|
|
"perf_stats": {
|
|
"commit_latency_ms": 2,
|
|
"apply_latency_ms": 31
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}`), "", nil)
|
|
|
|
e := &Exporter{Conn: conn, Cluster: "ceph", Logger: logrus.New()}
|
|
e.cc = map[string]versionedCollector{
|
|
"osd": NewOSDCollector(e),
|
|
}
|
|
err := prometheus.Register(e)
|
|
require.NoError(t, err)
|
|
defer prometheus.Unregister(e)
|
|
|
|
server := httptest.NewServer(promhttp.Handler())
|
|
defer server.Close()
|
|
|
|
resp, err := http.Get(server.URL)
|
|
require.NoError(t, err)
|
|
defer resp.Body.Close()
|
|
|
|
buf, err := ioutil.ReadAll(resp.Body)
|
|
require.NoError(t, err)
|
|
|
|
for _, re := range append(reMatch, tt.reMatch...) {
|
|
require.True(t, re.Match(buf))
|
|
}
|
|
}()
|
|
}
|
|
}
|