Merge pull request #4458 from FUSAKLA/k8s-sd-add-metrics

feat: added more k8s SD metrics
This commit is contained in:
Frederic Branczyk 2018-09-21 13:10:48 +02:00 committed by GitHub
commit b75ec7e6ef
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 264 additions and 4 deletions

View File

@ -0,0 +1,246 @@
// Copyright 2018 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package kubernetes
import (
"net/url"
"time"
"github.com/prometheus/client_golang/prometheus"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/metrics"
"k8s.io/client-go/util/workqueue"
)
const (
cacheMetricsNamespace = metricsNamespace + "_cache"
workqueueMetricsNamespace = metricsNamespace + "_workqueue"
)
var (
// Metrics for client-go's HTTP requests.
clientGoRequestResultMetricVec = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: metricsNamespace,
Name: "http_request_total",
Help: "Total number of HTTP requests to the Kubernetes API by status code.",
},
[]string{"status_code"},
)
clientGoRequestLatencyMetricVec = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Namespace: metricsNamespace,
Name: "http_request_duration_seconds",
Help: "Summary of latencies for HTTP requests to the Kubernetes API by endpoint.",
Objectives: map[float64]float64{},
},
[]string{"endpoint"},
)
// Definition of metrics for client-go cache metrics provider.
clientGoCacheListTotalMetric = prometheus.NewCounter(
prometheus.CounterOpts{
Namespace: cacheMetricsNamespace,
Name: "list_total",
Help: "Total number of list operations.",
},
)
clientGoCacheListDurationMetric = prometheus.NewSummary(
prometheus.SummaryOpts{
Namespace: cacheMetricsNamespace,
Name: "list_duration_seconds",
Help: "Duration of a Kubernetes API call in seconds.",
Objectives: map[float64]float64{},
},
)
clientGoCacheItemsInListCountMetric = prometheus.NewSummary(
prometheus.SummaryOpts{
Namespace: cacheMetricsNamespace,
Name: "list_items",
Help: "Count of items in a list from the Kubernetes API.",
Objectives: map[float64]float64{},
},
)
clientGoCacheWatchesCountMetric = prometheus.NewCounter(
prometheus.CounterOpts{
Namespace: cacheMetricsNamespace,
Name: "watches_total",
Help: "Total number of watch operations.",
},
)
clientGoCacheShortWatchesCountMetric = prometheus.NewCounter(
prometheus.CounterOpts{
Namespace: cacheMetricsNamespace,
Name: "short_watches_total",
Help: "Total number of short watch operations.",
},
)
clientGoCacheWatchesDurationMetric = prometheus.NewSummary(
prometheus.SummaryOpts{
Namespace: cacheMetricsNamespace,
Name: "watch_duration_seconds",
Help: "Duration of watches on the Kubernetes API.",
Objectives: map[float64]float64{},
},
)
clientGoCacheItemsInWatchesCountMetric = prometheus.NewSummary(
prometheus.SummaryOpts{
Namespace: cacheMetricsNamespace,
Name: "watch_events",
Help: "Number of items in watches on the Kubernetes API.",
Objectives: map[float64]float64{},
},
)
clientGoCacheLastResourceVersionMetric = prometheus.NewGauge(
prometheus.GaugeOpts{
Namespace: cacheMetricsNamespace,
Name: "last_resource_version",
Help: "Last resource version from the Kubernetes API.",
},
)
// Definition of metrics for client-go workflow metrics provider
clientGoWorkqueueDepthMetricVec = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: workqueueMetricsNamespace,
Name: "depth",
Help: "Current depth of the work queue.",
},
[]string{"queue_name"},
)
clientGoWorkqueueAddsMetricVec = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: workqueueMetricsNamespace,
Name: "items_total",
Help: "Total number of items added to the work queue.",
},
[]string{"queue_name"},
)
clientGoWorkqueueLatencyMetricVec = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Namespace: workqueueMetricsNamespace,
Name: "latency_seconds",
Help: "How long an item stays in the work queue.",
Objectives: map[float64]float64{},
},
[]string{"queue_name"},
)
clientGoWorkqueueWorkDurationMetricVec = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Namespace: workqueueMetricsNamespace,
Name: "work_duration_seconds",
Help: "How long processing an item from the work queue takes.",
Objectives: map[float64]float64{},
},
[]string{"queue_name"},
)
)
// Definition of dummy metric used as a placeholder if we don't want to observe some data.
type noopMetric struct{}
func (noopMetric) Inc() {}
func (noopMetric) Dec() {}
func (noopMetric) Observe(float64) {}
// Definition of client-go metrics adapters for HTTP requests observation
type clientGoRequestMetricAdapter struct{}
func (f *clientGoRequestMetricAdapter) Register(registerer prometheus.Registerer) {
metrics.Register(f, f)
registerer.MustRegister(clientGoRequestResultMetricVec)
registerer.MustRegister(clientGoRequestLatencyMetricVec)
}
func (clientGoRequestMetricAdapter) Increment(code string, method string, host string) {
clientGoRequestResultMetricVec.WithLabelValues(code).Inc()
}
func (clientGoRequestMetricAdapter) Observe(verb string, u url.URL, latency time.Duration) {
clientGoRequestLatencyMetricVec.WithLabelValues(u.EscapedPath()).Observe(latency.Seconds())
}
// Definition of client-go cache metrics provider definition
type clientGoCacheMetricsProvider struct{}
func (f *clientGoCacheMetricsProvider) Register(registerer prometheus.Registerer) {
cache.SetReflectorMetricsProvider(f)
registerer.MustRegister(clientGoCacheWatchesDurationMetric)
registerer.MustRegister(clientGoCacheWatchesCountMetric)
registerer.MustRegister(clientGoCacheListDurationMetric)
registerer.MustRegister(clientGoCacheListTotalMetric)
registerer.MustRegister(clientGoCacheLastResourceVersionMetric)
registerer.MustRegister(clientGoCacheShortWatchesCountMetric)
registerer.MustRegister(clientGoCacheItemsInWatchesCountMetric)
registerer.MustRegister(clientGoCacheItemsInListCountMetric)
}
func (clientGoCacheMetricsProvider) NewListsMetric(name string) cache.CounterMetric {
return clientGoCacheListTotalMetric
}
func (clientGoCacheMetricsProvider) NewListDurationMetric(name string) cache.SummaryMetric {
return clientGoCacheListDurationMetric
}
func (clientGoCacheMetricsProvider) NewItemsInListMetric(name string) cache.SummaryMetric {
return clientGoCacheItemsInListCountMetric
}
func (clientGoCacheMetricsProvider) NewWatchesMetric(name string) cache.CounterMetric {
return clientGoCacheWatchesCountMetric
}
func (clientGoCacheMetricsProvider) NewShortWatchesMetric(name string) cache.CounterMetric {
return clientGoCacheShortWatchesCountMetric
}
func (clientGoCacheMetricsProvider) NewWatchDurationMetric(name string) cache.SummaryMetric {
return clientGoCacheWatchesDurationMetric
}
func (clientGoCacheMetricsProvider) NewItemsInWatchMetric(name string) cache.SummaryMetric {
return clientGoCacheItemsInWatchesCountMetric
}
func (clientGoCacheMetricsProvider) NewLastResourceVersionMetric(name string) cache.GaugeMetric {
return clientGoCacheLastResourceVersionMetric
}
// Definition of client-go workqueue metrics provider definition
type clientGoWorkqueueMetricsProvider struct{}
func (f *clientGoWorkqueueMetricsProvider) Register(registerer prometheus.Registerer) {
workqueue.SetProvider(f)
registerer.MustRegister(clientGoWorkqueueDepthMetricVec)
registerer.MustRegister(clientGoWorkqueueAddsMetricVec)
registerer.MustRegister(clientGoWorkqueueLatencyMetricVec)
registerer.MustRegister(clientGoWorkqueueWorkDurationMetricVec)
}
func (f *clientGoWorkqueueMetricsProvider) NewDepthMetric(name string) workqueue.GaugeMetric {
return clientGoWorkqueueDepthMetricVec.WithLabelValues(name)
}
func (f *clientGoWorkqueueMetricsProvider) NewAddsMetric(name string) workqueue.CounterMetric {
return clientGoWorkqueueAddsMetricVec.WithLabelValues(name)
}
func (f *clientGoWorkqueueMetricsProvider) NewLatencyMetric(name string) workqueue.SummaryMetric {
metric := clientGoWorkqueueLatencyMetricVec.WithLabelValues(name)
// Convert microseconds to seconds for consistency across metrics.
return prometheus.ObserverFunc(func(v float64) {
metric.Observe(v / 1e6)
})
}
func (f *clientGoWorkqueueMetricsProvider) NewWorkDurationMetric(name string) workqueue.SummaryMetric {
metric := clientGoWorkqueueWorkDurationMetricVec.WithLabelValues(name)
// Convert microseconds to seconds for consistency across metrics.
return prometheus.ObserverFunc(func(v float64) {
metric.Observe(v / 1e6)
})
}
func (clientGoWorkqueueMetricsProvider) NewRetriesMetric(name string) workqueue.CounterMetric {
// Retries are not used so the metric is ommited.
return noopMetric{}
}

View File

@ -40,15 +40,18 @@ import (
const (
// kubernetesMetaLabelPrefix is the meta prefix used for all meta labels.
// in this discovery.
metaLabelPrefix = model.MetaLabelPrefix + "kubernetes_"
namespaceLabel = metaLabelPrefix + "namespace"
metaLabelPrefix = model.MetaLabelPrefix + "kubernetes_"
namespaceLabel = metaLabelPrefix + "namespace"
metricsNamespace = "prometheus_sd_kubernetes"
)
var (
// Custom events metric
eventCount = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "prometheus_sd_kubernetes_events_total",
Help: "The number of Kubernetes events handled.",
Namespace: metricsNamespace,
Name: "events_total",
Help: "The number of Kubernetes events handled.",
},
[]string{"role", "event"},
)
@ -139,6 +142,17 @@ func init() {
eventCount.WithLabelValues(role, evt)
}
}
var (
clientGoRequestMetricAdapterInstance = clientGoRequestMetricAdapter{}
clientGoCacheMetricsProviderInstance = clientGoCacheMetricsProvider{}
clientGoWorkqueueMetricsProviderInstance = clientGoWorkqueueMetricsProvider{}
)
clientGoRequestMetricAdapterInstance.Register(prometheus.DefaultRegisterer)
clientGoCacheMetricsProviderInstance.Register(prometheus.DefaultRegisterer)
clientGoWorkqueueMetricsProviderInstance.Register(prometheus.DefaultRegisterer)
}
// This is only for internal use.