api v1 alerts/rules json endpoint
Signed-off-by: mg03 <mgeng03@gmail.com>
This commit is contained in:
parent
ffb7836c14
commit
31f8ca0dfb
|
@ -131,6 +131,41 @@ func (r *AlertingRule) Name() string {
|
||||||
return r.name
|
return r.name
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Query returns the query expression of the alert.
|
||||||
|
func (r *AlertingRule) Query() promql.Expr {
|
||||||
|
return r.vector
|
||||||
|
}
|
||||||
|
|
||||||
|
// Duration returns the hold duration of the alert.
|
||||||
|
func (r *AlertingRule) Duration() time.Duration {
|
||||||
|
return r.holdDuration
|
||||||
|
}
|
||||||
|
|
||||||
|
// Labels returns the labels of the alert.
|
||||||
|
func (r *AlertingRule) Labels() labels.Labels {
|
||||||
|
return r.labels
|
||||||
|
}
|
||||||
|
|
||||||
|
// Annotations returns the annotations of the alert.
|
||||||
|
func (r *AlertingRule) Annotations() labels.Labels {
|
||||||
|
return r.annotations
|
||||||
|
}
|
||||||
|
|
||||||
|
// Alertinfo return an array of alerts
|
||||||
|
func (r *AlertingRule) Alertinfo() []*Alert {
|
||||||
|
activealerts := &r.active
|
||||||
|
alertsarr := make([]*Alert, 0)
|
||||||
|
if len(*activealerts) > 0 {
|
||||||
|
for _, a := range *activealerts {
|
||||||
|
if a.ResolvedAt.IsZero() {
|
||||||
|
alertsarr = append(alertsarr, a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return alertsarr
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (r *AlertingRule) equal(o *AlertingRule) bool {
|
func (r *AlertingRule) equal(o *AlertingRule) bool {
|
||||||
return r.name == o.name && labels.Equal(r.labels, o.labels)
|
return r.name == o.name && labels.Equal(r.labels, o.labels)
|
||||||
}
|
}
|
||||||
|
|
|
@ -41,6 +41,7 @@ import (
|
||||||
"github.com/prometheus/prometheus/pkg/timestamp"
|
"github.com/prometheus/prometheus/pkg/timestamp"
|
||||||
"github.com/prometheus/prometheus/prompb"
|
"github.com/prometheus/prometheus/prompb"
|
||||||
"github.com/prometheus/prometheus/promql"
|
"github.com/prometheus/prometheus/promql"
|
||||||
|
"github.com/prometheus/prometheus/rules"
|
||||||
"github.com/prometheus/prometheus/scrape"
|
"github.com/prometheus/prometheus/scrape"
|
||||||
"github.com/prometheus/prometheus/storage"
|
"github.com/prometheus/prometheus/storage"
|
||||||
"github.com/prometheus/prometheus/storage/remote"
|
"github.com/prometheus/prometheus/storage/remote"
|
||||||
|
@ -95,6 +96,19 @@ type alertmanagerRetriever interface {
|
||||||
DroppedAlertmanagers() []*url.URL
|
DroppedAlertmanagers() []*url.URL
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type alertRetreiver interface {
|
||||||
|
AlertingRules() []*rules.AlertingRule
|
||||||
|
}
|
||||||
|
|
||||||
|
type rulesRetreiver interface {
|
||||||
|
RuleGroups() []*rules.Group
|
||||||
|
}
|
||||||
|
|
||||||
|
type alertsrulesRetreiver interface {
|
||||||
|
alertRetreiver
|
||||||
|
rulesRetreiver
|
||||||
|
}
|
||||||
|
|
||||||
type response struct {
|
type response struct {
|
||||||
Status status `json:"status"`
|
Status status `json:"status"`
|
||||||
Data interface{} `json:"data,omitempty"`
|
Data interface{} `json:"data,omitempty"`
|
||||||
|
@ -119,11 +133,11 @@ type API struct {
|
||||||
|
|
||||||
targetRetriever targetRetriever
|
targetRetriever targetRetriever
|
||||||
alertmanagerRetriever alertmanagerRetriever
|
alertmanagerRetriever alertmanagerRetriever
|
||||||
|
alertsrulesRetreiver alertsrulesRetreiver
|
||||||
now func() time.Time
|
now func() time.Time
|
||||||
config func() config.Config
|
config func() config.Config
|
||||||
flagsMap map[string]string
|
flagsMap map[string]string
|
||||||
ready func(http.HandlerFunc) http.HandlerFunc
|
ready func(http.HandlerFunc) http.HandlerFunc
|
||||||
|
|
||||||
db func() *tsdb.DB
|
db func() *tsdb.DB
|
||||||
enableAdmin bool
|
enableAdmin bool
|
||||||
|
@ -142,18 +156,20 @@ func NewAPI(
|
||||||
db func() *tsdb.DB,
|
db func() *tsdb.DB,
|
||||||
enableAdmin bool,
|
enableAdmin bool,
|
||||||
logger log.Logger,
|
logger log.Logger,
|
||||||
|
al alertsrulesRetreiver,
|
||||||
) *API {
|
) *API {
|
||||||
return &API{
|
return &API{
|
||||||
QueryEngine: qe,
|
QueryEngine: qe,
|
||||||
Queryable: q,
|
Queryable: q,
|
||||||
targetRetriever: tr,
|
targetRetriever: tr,
|
||||||
alertmanagerRetriever: ar,
|
alertmanagerRetriever: ar,
|
||||||
now: time.Now,
|
now: time.Now,
|
||||||
config: configFunc,
|
config: configFunc,
|
||||||
flagsMap: flagsMap,
|
flagsMap: flagsMap,
|
||||||
ready: readyFunc,
|
ready: readyFunc,
|
||||||
db: db,
|
db: db,
|
||||||
enableAdmin: enableAdmin,
|
enableAdmin: enableAdmin,
|
||||||
|
alertsrulesRetreiver: al,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -199,6 +215,9 @@ func (api *API) Register(r *route.Router) {
|
||||||
r.Get("/status/flags", wrap(api.serveFlags))
|
r.Get("/status/flags", wrap(api.serveFlags))
|
||||||
r.Post("/read", api.ready(http.HandlerFunc(api.remoteRead)))
|
r.Post("/read", api.ready(http.HandlerFunc(api.remoteRead)))
|
||||||
|
|
||||||
|
r.Get("/alerts", wrap(api.alerts))
|
||||||
|
r.Get("/rules", wrap(api.rules))
|
||||||
|
|
||||||
// Admin APIs
|
// Admin APIs
|
||||||
r.Post("/admin/tsdb/delete_series", wrap(api.deleteSeries))
|
r.Post("/admin/tsdb/delete_series", wrap(api.deleteSeries))
|
||||||
r.Post("/admin/tsdb/clean_tombstones", wrap(api.cleanTombstones))
|
r.Post("/admin/tsdb/clean_tombstones", wrap(api.cleanTombstones))
|
||||||
|
@ -578,6 +597,94 @@ func (api *API) alertmanagers(r *http.Request) (interface{}, *apiError, func())
|
||||||
return ams, nil, nil
|
return ams, nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// AlertDiscovery has info for all alerts
|
||||||
|
type AlertDiscovery struct {
|
||||||
|
Alertgrps []*Alertgrp `json:"alertgrp"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Alert has info for a alert
|
||||||
|
type Alert struct {
|
||||||
|
Labels labels.Labels `json:"labels"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
Activesince *time.Time `json:"activesince,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Alertgrp has info for alerts part of a group
|
||||||
|
type Alertgrp struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Query string `json:"query"`
|
||||||
|
Duration string `json:"duration"`
|
||||||
|
Annotations labels.Labels `json:"annotations,omitempty"`
|
||||||
|
Alerts []*Alert `json:"alerts"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (api *API) alerts(r *http.Request) (interface{}, *apiError) {
|
||||||
|
alertingrules := api.alertsrulesRetreiver.AlertingRules()
|
||||||
|
var alertgrps []*Alertgrp
|
||||||
|
res := &AlertDiscovery{Alertgrps: alertgrps}
|
||||||
|
for _, activerule := range alertingrules {
|
||||||
|
t := &Alertgrp{
|
||||||
|
Name: activerule.Name(),
|
||||||
|
Query: fmt.Sprintf("%v", activerule.Query()),
|
||||||
|
Duration: activerule.Duration().String(),
|
||||||
|
Annotations: activerule.Annotations(),
|
||||||
|
}
|
||||||
|
alerts := activerule.Alertinfo()
|
||||||
|
var activealerts []*Alert
|
||||||
|
for _, alert := range alerts {
|
||||||
|
q := &Alert{
|
||||||
|
Labels: alert.Labels,
|
||||||
|
Status: alert.State.String(),
|
||||||
|
Activesince: &alert.ActiveAt,
|
||||||
|
}
|
||||||
|
|
||||||
|
activealerts = append(activealerts, q)
|
||||||
|
}
|
||||||
|
t.Alerts = activealerts
|
||||||
|
res.Alertgrps = append(res.Alertgrps, t)
|
||||||
|
}
|
||||||
|
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GroupDiscovery has info for all rules
|
||||||
|
type GroupDiscovery struct {
|
||||||
|
Rulegrps []*Rulegrp `json:"groups"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rulegrp has info for rules which are part of a group
|
||||||
|
type Rulegrp struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
File string `json:"file"`
|
||||||
|
Rules []*Ruleinfo `json:"rules"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ruleinfo has rule in human readable format using \n as line separators
|
||||||
|
type Ruleinfo struct {
|
||||||
|
Rule string `json:"rule"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (api *API) rules(r *http.Request) (interface{}, *apiError) {
|
||||||
|
grps := api.alertsrulesRetreiver.RuleGroups()
|
||||||
|
res := &GroupDiscovery{Rulegrps: make([]*Rulegrp, len(grps))}
|
||||||
|
for i, grp := range grps {
|
||||||
|
t := &Rulegrp{
|
||||||
|
Name: grp.Name(),
|
||||||
|
File: grp.File(),
|
||||||
|
}
|
||||||
|
var rulearr []*Ruleinfo
|
||||||
|
for _, rule := range grp.Rules() {
|
||||||
|
q := &Ruleinfo{
|
||||||
|
Rule: rule.String(),
|
||||||
|
}
|
||||||
|
rulearr = append(rulearr, q)
|
||||||
|
}
|
||||||
|
t.Rules = rulearr
|
||||||
|
res.Rulegrps[i] = t
|
||||||
|
}
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
type prometheusConfig struct {
|
type prometheusConfig struct {
|
||||||
YAML string `json:"yaml"`
|
YAML string `json:"yaml"`
|
||||||
}
|
}
|
||||||
|
|
|
@ -19,7 +19,9 @@ import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"github.com/go-kit/kit/log"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
|
stdlog "log"
|
||||||
"math"
|
"math"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
|
@ -41,9 +43,11 @@ import (
|
||||||
"github.com/prometheus/prometheus/pkg/timestamp"
|
"github.com/prometheus/prometheus/pkg/timestamp"
|
||||||
"github.com/prometheus/prometheus/prompb"
|
"github.com/prometheus/prometheus/prompb"
|
||||||
"github.com/prometheus/prometheus/promql"
|
"github.com/prometheus/prometheus/promql"
|
||||||
|
"github.com/prometheus/prometheus/rules"
|
||||||
"github.com/prometheus/prometheus/scrape"
|
"github.com/prometheus/prometheus/scrape"
|
||||||
"github.com/prometheus/prometheus/storage"
|
"github.com/prometheus/prometheus/storage"
|
||||||
"github.com/prometheus/prometheus/storage/remote"
|
"github.com/prometheus/prometheus/storage/remote"
|
||||||
|
"github.com/prometheus/prometheus/util/testutil"
|
||||||
)
|
)
|
||||||
|
|
||||||
type testTargetRetriever struct{}
|
type testTargetRetriever struct{}
|
||||||
|
@ -98,6 +102,68 @@ func (t testAlertmanagerRetriever) DroppedAlertmanagers() []*url.URL {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type testalertsrulesfunc struct {
|
||||||
|
test *testing.T
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t testalertsrulesfunc) AlertingRules() []*rules.AlertingRule {
|
||||||
|
expr1, err := promql.ParseExpr(`absent(test_metric3) != 1`)
|
||||||
|
if err != nil {
|
||||||
|
stdlog.Fatalf("Unable to parse alert expression: %s", err)
|
||||||
|
}
|
||||||
|
expr2, err := promql.ParseExpr(`up == 1`)
|
||||||
|
if err != nil {
|
||||||
|
stdlog.Fatalf("Unable to parse alert expression: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
rule1 := rules.NewAlertingRule(
|
||||||
|
"test_metric3",
|
||||||
|
expr1,
|
||||||
|
time.Second,
|
||||||
|
labels.Labels{},
|
||||||
|
labels.Labels{},
|
||||||
|
log.NewNopLogger(),
|
||||||
|
)
|
||||||
|
rule2 := rules.NewAlertingRule(
|
||||||
|
"test_metric4",
|
||||||
|
expr2,
|
||||||
|
time.Second,
|
||||||
|
labels.Labels{},
|
||||||
|
labels.Labels{},
|
||||||
|
log.NewNopLogger(),
|
||||||
|
)
|
||||||
|
var r []*rules.AlertingRule
|
||||||
|
r = append(r, rule1)
|
||||||
|
r = append(r, rule2)
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t testalertsrulesfunc) RuleGroups() []*rules.Group {
|
||||||
|
var ar testalertsrulesfunc
|
||||||
|
arules := ar.AlertingRules()
|
||||||
|
storage := testutil.NewStorage(t.test)
|
||||||
|
defer storage.Close()
|
||||||
|
|
||||||
|
engine := promql.NewEngine(nil, nil, 10, 10*time.Second)
|
||||||
|
opts := &rules.ManagerOptions{
|
||||||
|
QueryFunc: rules.EngineQueryFunc(engine, storage),
|
||||||
|
Appendable: storage,
|
||||||
|
Context: context.Background(),
|
||||||
|
Logger: log.NewNopLogger(),
|
||||||
|
}
|
||||||
|
|
||||||
|
var r []rules.Rule
|
||||||
|
|
||||||
|
for _, alertrule := range arules {
|
||||||
|
r = append(r, alertrule)
|
||||||
|
}
|
||||||
|
|
||||||
|
group := rules.NewGroup("grp", "/path/to/file", time.Second, r, opts)
|
||||||
|
fmt.Println(group)
|
||||||
|
return []*rules.Group{group}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
var samplePrometheusCfg = config.Config{
|
var samplePrometheusCfg = config.Config{
|
||||||
GlobalConfig: config.GlobalConfig{},
|
GlobalConfig: config.GlobalConfig{},
|
||||||
AlertingConfig: config.AlertingConfig{},
|
AlertingConfig: config.AlertingConfig{},
|
||||||
|
@ -131,15 +197,24 @@ func TestEndpoints(t *testing.T) {
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
|
|
||||||
t.Run("local", func(t *testing.T) {
|
t.Run("local", func(t *testing.T) {
|
||||||
|
|
||||||
|
var algr testalertsrulesfunc
|
||||||
|
algr.test = t
|
||||||
|
|
||||||
|
algr.AlertingRules()
|
||||||
|
|
||||||
|
algr.RuleGroups()
|
||||||
|
|
||||||
api := &API{
|
api := &API{
|
||||||
Queryable: suite.Storage(),
|
Queryable: suite.Storage(),
|
||||||
QueryEngine: suite.QueryEngine(),
|
QueryEngine: suite.QueryEngine(),
|
||||||
targetRetriever: testTargetRetriever{},
|
targetRetriever: testTargetRetriever{},
|
||||||
alertmanagerRetriever: testAlertmanagerRetriever{},
|
alertmanagerRetriever: testAlertmanagerRetriever{},
|
||||||
now: func() time.Time { return now },
|
now: func() time.Time { return now },
|
||||||
config: func() config.Config { return samplePrometheusCfg },
|
config: func() config.Config { return samplePrometheusCfg },
|
||||||
flagsMap: sampleFlagMap,
|
flagsMap: sampleFlagMap,
|
||||||
ready: func(f http.HandlerFunc) http.HandlerFunc { return f },
|
ready: func(f http.HandlerFunc) http.HandlerFunc { return f },
|
||||||
|
alertsrulesRetreiver: algr,
|
||||||
}
|
}
|
||||||
|
|
||||||
testEndpoints(t, api, true)
|
testEndpoints(t, api, true)
|
||||||
|
@ -176,15 +251,23 @@ func TestEndpoints(t *testing.T) {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var algr testalertsrulesfunc
|
||||||
|
algr.test = t
|
||||||
|
|
||||||
|
algr.AlertingRules()
|
||||||
|
|
||||||
|
algr.RuleGroups()
|
||||||
|
|
||||||
api := &API{
|
api := &API{
|
||||||
Queryable: remote,
|
Queryable: remote,
|
||||||
QueryEngine: suite.QueryEngine(),
|
QueryEngine: suite.QueryEngine(),
|
||||||
targetRetriever: testTargetRetriever{},
|
targetRetriever: testTargetRetriever{},
|
||||||
alertmanagerRetriever: testAlertmanagerRetriever{},
|
alertmanagerRetriever: testAlertmanagerRetriever{},
|
||||||
now: func() time.Time { return now },
|
now: func() time.Time { return now },
|
||||||
config: func() config.Config { return samplePrometheusCfg },
|
config: func() config.Config { return samplePrometheusCfg },
|
||||||
flagsMap: sampleFlagMap,
|
flagsMap: sampleFlagMap,
|
||||||
ready: func(f http.HandlerFunc) http.HandlerFunc { return f },
|
ready: func(f http.HandlerFunc) http.HandlerFunc { return f },
|
||||||
|
alertsrulesRetreiver: algr,
|
||||||
}
|
}
|
||||||
|
|
||||||
testEndpoints(t, api, false)
|
testEndpoints(t, api, false)
|
||||||
|
@ -237,7 +320,6 @@ func setupRemote(s storage.Storage) *httptest.Server {
|
||||||
}
|
}
|
||||||
|
|
||||||
func testEndpoints(t *testing.T, api *API, testLabelAPI bool) {
|
func testEndpoints(t *testing.T, api *API, testLabelAPI bool) {
|
||||||
|
|
||||||
start := time.Unix(0, 0)
|
start := time.Unix(0, 0)
|
||||||
|
|
||||||
type test struct {
|
type test struct {
|
||||||
|
@ -567,6 +649,46 @@ func testEndpoints(t *testing.T, api *API, testLabelAPI bool) {
|
||||||
endpoint: api.serveFlags,
|
endpoint: api.serveFlags,
|
||||||
response: sampleFlagMap,
|
response: sampleFlagMap,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
endpoint: api.alerts,
|
||||||
|
response: &AlertDiscovery{
|
||||||
|
Alertgrps: []*Alertgrp{
|
||||||
|
{
|
||||||
|
Name: "test_metric3",
|
||||||
|
Query: "absent(test_metric3) != 1",
|
||||||
|
Duration: "1s",
|
||||||
|
Alerts: nil,
|
||||||
|
Annotations: labels.Labels{},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Name: "test_metric4",
|
||||||
|
Query: "up == 1",
|
||||||
|
Duration: "1s",
|
||||||
|
Alerts: nil,
|
||||||
|
Annotations: labels.Labels{},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
endpoint: api.rules,
|
||||||
|
response: &GroupDiscovery{
|
||||||
|
Rulegrps: []*Rulegrp{
|
||||||
|
{
|
||||||
|
Name: "grp",
|
||||||
|
File: "/path/to/file",
|
||||||
|
Rules: []*Ruleinfo{
|
||||||
|
{
|
||||||
|
Rule: "alert: test_metric3\nexpr: absent(test_metric3) != 1\nfor: 1s\n",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Rule: "alert: test_metric4\nexpr: up == 1\nfor: 1s\n",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
if testLabelAPI {
|
if testLabelAPI {
|
||||||
|
|
|
@ -228,6 +228,7 @@ func New(logger log.Logger, o *Options) *Handler {
|
||||||
h.options.TSDB,
|
h.options.TSDB,
|
||||||
h.options.EnableAdminAPI,
|
h.options.EnableAdminAPI,
|
||||||
logger,
|
logger,
|
||||||
|
h.ruleManager,
|
||||||
)
|
)
|
||||||
|
|
||||||
if o.RoutePrefix != "/" {
|
if o.RoutePrefix != "/" {
|
||||||
|
|
Loading…
Reference in New Issue