diff --git a/documentation/examples/prometheus-kubernetes.yml b/documentation/examples/prometheus-kubernetes.yml index 8a283f5e6..a82c34bb6 100644 --- a/documentation/examples/prometheus-kubernetes.yml +++ b/documentation/examples/prometheus-kubernetes.yml @@ -1,22 +1,62 @@ -# A scrape configuration for running prometheus in cluster on kubernetes. It -# will create endpoints for node and master roles, as well as any service -# which is annotated with `prometheus_io_scrape=true` +# A scrape configuration for running Prometheus on a Kubernetes cluster. +# This uses separate scrape configs for cluster components (i.e. master, node) +# and services to allow each to use different authentication configs. +# +# Kubernetes labels will be added as Prometheus labels on metrics via the +# `labelmap` relabeling action. +# Scrape config for cluster components. scrape_configs: -- job_name: 'kubernetes' +- job_name: 'kubernetes-cluster' + + # This TLS & bearer token file config is used to connect to the actual scrape + # endpoints for cluster components. This is separate to discovery auth + # configuration (`in_cluster` below) because discovery & scraping are two + # separate concerns in Prometheus. + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token kubernetes_sd_configs: - masters: - 'https://kubernetes.default.svc' in_cluster: true - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token relabel_configs: - - source_labels: [__meta_kubernetes_role, __meta_kubernetes_service_annotation_prometheus_io_scrape] + - source_labels: [__meta_kubernetes_role] action: keep - regex: ^(?:(?:master|node);.*|.*;true)$ + regex: ^(?:master|node)$ + - action: labelmap + regex: ^__meta_kubernetes_node_label_(.+)$ + replacement: $1 + - source_labels: [__meta_kubernetes_role] + action: replace + regex: ^(.+)$ + target_label: kubernetes_role + replacement: $1 + +# Scrape config for services. +# +# The relabeling allows the actual service scrape endpoint to be configured +# via the following annotations: +# +# * `prometheus.io/scrape`: Only scrape services that have a value of `true` +# * `prometheus.io/scheme`: If the metrics endpoint is secured then you will need +# to set this to `https` & most likely set the `tls_config` of the scrape config. +# * `prometheus.io/path`: If the metrics path is not `/metrics` override this. +# * `prometheus.io/port`: If the metrics are exposed on a different port to the +# service then set this appropriately. +- job_name: 'kubernetes-services' + + kubernetes_sd_configs: + - masters: + - 'https://kubernetes.default.svc' + in_cluster: true + + relabel_configs: + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] + action: keep + regex: ^true$ - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] action: replace target_label: __scheme__ @@ -32,3 +72,11 @@ scrape_configs: target_label: __address__ regex: ^(.+)(?::\d+);(\d+)$ replacement: $1:$2 + - action: labelmap + regex: ^__meta_kubernetes_service_label_(.+)$ + replacement: $1 + - source_labels: [__meta_kubernetes_role] + action: replace + regex: ^(.+)$ + target_label: kubernetes_role + replacement: $1 diff --git a/retrieval/discovery/kubernetes/discovery.go b/retrieval/discovery/kubernetes/discovery.go index 6fbe0ac80..a66675388 100644 --- a/retrieval/discovery/kubernetes/discovery.go +++ b/retrieval/discovery/kubernetes/discovery.go @@ -584,7 +584,7 @@ func newKubernetesHTTPClient(conf *config.KubernetesSDConfig) (*http.Client, err bearerTokenFile = serviceAccountToken } if len(caFile) == 0 { - // With recent versions, the CA certificate is provided as a token + // With recent versions, the CA certificate is mounted as a secret // but we need to handle older versions too. In this case, don't // set the CAFile & the configuration will have to use InsecureSkipVerify. if _, err := os.Stat(serviceAccountCACert); err == nil {