diff --git a/docs/querying/api.md b/docs/querying/api.md index 8548cfebec..f53027a56a 100644 --- a/docs/querying/api.md +++ b/docs/querying/api.md @@ -315,9 +315,6 @@ String results are returned as result type `string`. The corresponding ## Targets -> This API is experimental as it is intended to be extended with targets -> dropped due to relabelling in the future. - The following endpoint returns an overview of the current state of the Prometheus target discovery: @@ -325,7 +322,9 @@ Prometheus target discovery: GET /api/v1/targets ``` -Currently only the active targets are part of the response. +Both the active and dropped targets are part of the response. +`labels` represents the label set after relabelling has occurred. +`discoveredLabels` represent the unmodified labels retrieved during service discovery before relabelling has occurred. ```json $ curl http://localhost:9090/api/v1/targets @@ -349,6 +348,16 @@ $ curl http://localhost:9090/api/v1/targets "lastScrape": "2017-01-17T15:07:44.723715405+01:00", "health": "up" } + ], + "droppedTargets": [ + { + "discoveredLabels": { + "__address__": "127.0.0.1:9100", + "__metrics_path__": "/metrics", + "__scheme__": "http", + "job": "node" + }, + } ] } } diff --git a/scrape/manager.go b/scrape/manager.go index b498ea0b40..af27ccf640 100644 --- a/scrape/manager.go +++ b/scrape/manager.go @@ -132,6 +132,21 @@ func (m *Manager) Targets() []*Target { return targets } +// DroppedTargets returns the targets dropped during relabelling. +func (m *Manager) DroppedTargets() []*Target { + m.mtx.Lock() + defer m.mtx.Unlock() + var droppedTargets []*Target + for _, p := range m.scrapePools { + p.mtx.RLock() + for _, tt := range p.droppedTargets { + droppedTargets = append(droppedTargets, tt) + } + p.mtx.RUnlock() + } + return droppedTargets +} + func (m *Manager) reload(t map[string][]*targetgroup.Group) { for tsetName, tgroup := range t { scrapeConfig, ok := m.scrapeConfigs[tsetName] diff --git a/web/api/v1/api.go b/web/api/v1/api.go index 5d3fb2cb97..ea48ea47c4 100644 --- a/web/api/v1/api.go +++ b/web/api/v1/api.go @@ -83,6 +83,7 @@ func (e *apiError) Error() string { type targetRetriever interface { Targets() []*scrape.Target + DroppedTargets() []*scrape.Target } type alertmanagerRetriever interface { @@ -438,14 +439,22 @@ type Target struct { Health scrape.TargetHealth `json:"health"` } +// DroppedTarget has the information for one target that was dropped during relabelling. +type DroppedTarget struct { + // Labels before any processing. + DiscoveredLabels map[string]string `json:"discoveredLabels"` +} + // TargetDiscovery has all the active targets. type TargetDiscovery struct { - ActiveTargets []*Target `json:"activeTargets"` + ActiveTargets []*Target `json:"activeTargets"` + DroppedTargets []*DroppedTarget `json:"droppedTargets"` } func (api *API) targets(r *http.Request) (interface{}, *apiError) { targets := api.targetRetriever.Targets() - res := &TargetDiscovery{ActiveTargets: make([]*Target, len(targets))} + droppedTargets := api.targetRetriever.DroppedTargets() + res := &TargetDiscovery{ActiveTargets: make([]*Target, len(targets)), DroppedTargets: make([]*DroppedTarget, len(droppedTargets))} for i, t := range targets { lastErrStr := "" @@ -464,6 +473,12 @@ func (api *API) targets(r *http.Request) (interface{}, *apiError) { } } + for i, t := range droppedTargets { + res.DroppedTargets[i] = &DroppedTarget{ + DiscoveredLabels: t.DiscoveredLabels().Map(), + } + } + return res, nil } diff --git a/web/api/v1/api_test.go b/web/api/v1/api_test.go index f919188528..e4128b1b45 100644 --- a/web/api/v1/api_test.go +++ b/web/api/v1/api_test.go @@ -42,10 +42,34 @@ import ( "github.com/prometheus/prometheus/storage/remote" ) -type targetRetrieverFunc func() []*scrape.Target +type testTargetRetriever struct{} -func (f targetRetrieverFunc) Targets() []*scrape.Target { - return f() +func (t testTargetRetriever) Targets() []*scrape.Target { + return []*scrape.Target{ + scrape.NewTarget( + labels.FromMap(map[string]string{ + model.SchemeLabel: "http", + model.AddressLabel: "example.com:8080", + model.MetricsPathLabel: "/metrics", + }), + nil, + url.Values{}, + ), + } +} +func (t testTargetRetriever) DroppedTargets() []*scrape.Target { + return []*scrape.Target{ + scrape.NewTarget( + nil, + labels.FromMap(map[string]string{ + model.AddressLabel: "http://dropped.example.com:9115", + model.MetricsPathLabel: "/probe", + model.SchemeLabel: "http", + model.JobLabel: "blackbox", + }), + url.Values{}, + ), + } } type testAlertmanagerRetriever struct{} @@ -102,19 +126,7 @@ func TestEndpoints(t *testing.T) { now := time.Now() - tr := targetRetrieverFunc(func() []*scrape.Target { - return []*scrape.Target{ - scrape.NewTarget( - labels.FromMap(map[string]string{ - model.SchemeLabel: "http", - model.AddressLabel: "example.com:8080", - model.MetricsPathLabel: "/metrics", - }), - nil, - url.Values{}, - ), - } - }) + var tr testTargetRetriever var ar testAlertmanagerRetriever @@ -447,6 +459,16 @@ func TestEndpoints(t *testing.T) { Health: "unknown", }, }, + DroppedTargets: []*DroppedTarget{ + { + DiscoveredLabels: map[string]string{ + "__address__": "http://dropped.example.com:9115", + "__metrics_path__": "/probe", + "__scheme__": "http", + "job": "blackbox", + }, + }, + }, }, }, {