From 7ff5cc66ea4571e4c48ca8dad3aff049b3b5d554 Mon Sep 17 00:00:00 2001 From: Jimmi Dyson Date: Fri, 23 Oct 2015 16:47:10 +0100 Subject: [PATCH 1/3] Kubernetes SD authentication options cleanup --- config/config.go | 38 +++++++++++-------- config/config_test.go | 14 +++++-- config/testdata/conf.good.yml | 6 ++- .../testdata/kubernetes_bearertoken.bad.yml | 10 +++++ .../kubernetes_bearertoken_basicauth.bad.yml | 12 ++++++ retrieval/discovery/kubernetes/discovery.go | 22 +++++++---- 6 files changed, 73 insertions(+), 29 deletions(-) create mode 100644 config/testdata/kubernetes_bearertoken.bad.yml create mode 100644 config/testdata/kubernetes_bearertoken_basicauth.bad.yml diff --git a/config/config.go b/config/config.go index 2be3524d1..eeb2c72d5 100644 --- a/config/config.go +++ b/config/config.go @@ -626,22 +626,6 @@ type MarathonSDConfig struct { XXX map[string]interface{} `yaml:",inline"` } -// KubernetesSDConfig is the configuration for Kubernetes service discovery. -type KubernetesSDConfig struct { - Masters []URL `yaml:"masters"` - KubeletPort int `yaml:"kubelet_port,omitempty"` - InCluster bool `yaml:"in_cluster,omitempty"` - BearerTokenFile string `yaml:"bearer_token_file,omitempty"` - Username string `yaml:"username,omitempty"` - Password string `yaml:"password,omitempty"` - RetryInterval Duration `yaml:"retry_interval,omitempty"` - RequestTimeout Duration `yaml:"request_timeout,omitempty"` - TLSConfig TLSConfig `yaml:"tls_config,omitempty"` - - // Catches all undefined fields and must be empty after parsing. - XXX map[string]interface{} `yaml:",inline"` -} - // UnmarshalYAML implements the yaml.Unmarshaler interface. func (c *MarathonSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { *c = DefaultMarathonSDConfig @@ -657,6 +641,22 @@ func (c *MarathonSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) erro return checkOverflow(c.XXX, "marathon_sd_config") } +// KubernetesSDConfig is the configuration for Kubernetes service discovery. +type KubernetesSDConfig struct { + Masters []URL `yaml:"masters"` + KubeletPort int `yaml:"kubelet_port,omitempty"` + InCluster bool `yaml:"in_cluster,omitempty"` + BasicAuth *BasicAuth `yaml:"basic_auth,omitempty"` + BearerToken string `yaml:"bearer_token,omitempty"` + BearerTokenFile string `yaml:"bearer_token_file,omitempty"` + RetryInterval Duration `yaml:"retry_interval,omitempty"` + RequestTimeout Duration `yaml:"request_timeout,omitempty"` + TLSConfig TLSConfig `yaml:"tls_config,omitempty"` + + // Catches all undefined fields and must be empty after parsing. + XXX map[string]interface{} `yaml:",inline"` +} + // UnmarshalYAML implements the yaml.Unmarshaler interface. func (c *KubernetesSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { *c = DefaultKubernetesSDConfig @@ -668,6 +668,12 @@ func (c *KubernetesSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) er if len(c.Masters) == 0 { return fmt.Errorf("Kubernetes SD configuration requires at least one Kubernetes master") } + if len(c.BearerToken) > 0 && len(c.BearerTokenFile) > 0 { + return fmt.Errorf("at most one of bearer_token & bearer_token_file must be configured") + } + if c.BasicAuth != nil && (len(c.BearerToken) > 0 || len(c.BearerTokenFile) > 0) { + return fmt.Errorf("at most one of basic_auth, bearer_token & bearer_token_file must be configured") + } return checkOverflow(c.XXX, "kubernetes_sd_config") } diff --git a/config/config_test.go b/config/config_test.go index 28324c97a..a6a03b42a 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -203,9 +203,11 @@ var expectedConf = &Config{ KubernetesSDConfigs: []*KubernetesSDConfig{ { - Masters: []URL{kubernetesSDHostURL()}, - Username: "myusername", - Password: "mypassword", + Masters: []URL{kubernetesSDHostURL()}, + BasicAuth: &BasicAuth{ + Username: "myusername", + Password: "mypassword", + }, KubeletPort: 10255, RequestTimeout: Duration(10 * time.Second), RetryInterval: Duration(1 * time.Second), @@ -324,6 +326,12 @@ var expectedErrors = []struct { }, { filename: "bearertoken_basicauth.bad.yml", errMsg: "at most one of basic_auth, bearer_token & bearer_token_file must be configured", + }, { + filename: "kubernetes_bearertoken.bad.yml", + errMsg: "at most one of bearer_token & bearer_token_file must be configured", + }, { + filename: "kubernetes_bearertoken_basicauth.bad.yml", + errMsg: "at most one of basic_auth, bearer_token & bearer_token_file must be configured", }, { filename: "marathon_no_servers.bad.yml", errMsg: "Marathon SD config must contain at least one Marathon server", diff --git a/config/testdata/conf.good.yml b/config/testdata/conf.good.yml index 783c75393..259b011c8 100644 --- a/config/testdata/conf.good.yml +++ b/config/testdata/conf.good.yml @@ -107,8 +107,10 @@ scrape_configs: kubernetes_sd_configs: - masters: - 'https://localhost:1234' - username: 'myusername' - password: 'mypassword' + + basic_auth: + username: 'myusername' + password: 'mypassword' - job_name: service-marathon marathon_sd_configs: diff --git a/config/testdata/kubernetes_bearertoken.bad.yml b/config/testdata/kubernetes_bearertoken.bad.yml new file mode 100644 index 000000000..15aa7aad2 --- /dev/null +++ b/config/testdata/kubernetes_bearertoken.bad.yml @@ -0,0 +1,10 @@ +scrape_configs: + - job_name: prometheus + + kubernetes_sd_configs: + - masters: + - 'https://localhost:1234' + + bearer_token: 1234 + bearer_token_file: somefile + diff --git a/config/testdata/kubernetes_bearertoken_basicauth.bad.yml b/config/testdata/kubernetes_bearertoken_basicauth.bad.yml new file mode 100644 index 000000000..f138d9e50 --- /dev/null +++ b/config/testdata/kubernetes_bearertoken_basicauth.bad.yml @@ -0,0 +1,12 @@ +scrape_configs: + - job_name: prometheus + + kubernetes_sd_configs: + - masters: + - 'https://localhost:1234' + + bearer_token: 1234 + basic_auth: + username: user + password: password + diff --git a/retrieval/discovery/kubernetes/discovery.go b/retrieval/discovery/kubernetes/discovery.go index c981c8ee8..903478f96 100644 --- a/retrieval/discovery/kubernetes/discovery.go +++ b/retrieval/discovery/kubernetes/discovery.go @@ -624,16 +624,22 @@ func newKubernetesHTTPClient(conf *config.KubernetesSDConfig) (*http.Client, err TLSClientConfig: tlsConfig, } - bearerToken, err := ioutil.ReadFile(bearerTokenFile) - if err != nil { - return nil, err + // If a bearer token is provided, create a round tripper that will set the + // Authorization header correctly on each request. + bearerToken := conf.BearerToken + if len(bearerToken) == 0 && len(bearerTokenFile) > 0 { + b, err := ioutil.ReadFile(bearerTokenFile) + if err != nil { + return nil, fmt.Errorf("unable to read bearer token file %s: %s", bearerTokenFile, err) + } + bearerToken = string(b) + } + if len(bearerToken) > 0 { + rt = httputil.NewBearerAuthRoundTripper(bearerToken, rt) } - if len(bearerToken) > 0 { - rt = httputil.NewBearerAuthRoundTripper(string(bearerToken), rt) - } - if len(conf.Username) > 0 && len(conf.Password) > 0 { - rt = httputil.NewBasicAuthRoundTripper(conf.Username, conf.Password, rt) + if conf.BasicAuth != nil { + rt = httputil.NewBasicAuthRoundTripper(conf.BasicAuth.Username, conf.BasicAuth.Password, rt) } return &http.Client{ From 87940ec213c72067b3546441219f3e41a62ebf7f Mon Sep 17 00:00:00 2001 From: Jimmi Dyson Date: Sat, 24 Oct 2015 14:41:14 +0100 Subject: [PATCH 2/3] Kubernetes SD: Rename `masters` to `api_servers` in config --- config/config.go | 6 +- config/config_test.go | 2 +- config/testdata/conf.good.yml | 2 +- .../testdata/kubernetes_bearertoken.bad.yml | 2 +- .../kubernetes_bearertoken_basicauth.bad.yml | 2 +- .../examples/prometheus-kubernetes.yml | 8 +- retrieval/discovery/kubernetes/discovery.go | 82 ++++++++++--------- 7 files changed, 53 insertions(+), 51 deletions(-) diff --git a/config/config.go b/config/config.go index eeb2c72d5..dd5864e23 100644 --- a/config/config.go +++ b/config/config.go @@ -643,7 +643,7 @@ func (c *MarathonSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) erro // KubernetesSDConfig is the configuration for Kubernetes service discovery. type KubernetesSDConfig struct { - Masters []URL `yaml:"masters"` + APIServers []URL `yaml:"api_servers"` KubeletPort int `yaml:"kubelet_port,omitempty"` InCluster bool `yaml:"in_cluster,omitempty"` BasicAuth *BasicAuth `yaml:"basic_auth,omitempty"` @@ -665,8 +665,8 @@ func (c *KubernetesSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) er if err != nil { return err } - if len(c.Masters) == 0 { - return fmt.Errorf("Kubernetes SD configuration requires at least one Kubernetes master") + if len(c.APIServers) == 0 { + return fmt.Errorf("Kubernetes SD configuration requires at least one Kubernetes API server") } if len(c.BearerToken) > 0 && len(c.BearerTokenFile) > 0 { return fmt.Errorf("at most one of bearer_token & bearer_token_file must be configured") diff --git a/config/config_test.go b/config/config_test.go index a6a03b42a..b1d19f27b 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -203,7 +203,7 @@ var expectedConf = &Config{ KubernetesSDConfigs: []*KubernetesSDConfig{ { - Masters: []URL{kubernetesSDHostURL()}, + APIServers: []URL{kubernetesSDHostURL()}, BasicAuth: &BasicAuth{ Username: "myusername", Password: "mypassword", diff --git a/config/testdata/conf.good.yml b/config/testdata/conf.good.yml index 259b011c8..9732b682f 100644 --- a/config/testdata/conf.good.yml +++ b/config/testdata/conf.good.yml @@ -105,7 +105,7 @@ scrape_configs: - job_name: service-kubernetes kubernetes_sd_configs: - - masters: + - api_servers: - 'https://localhost:1234' basic_auth: diff --git a/config/testdata/kubernetes_bearertoken.bad.yml b/config/testdata/kubernetes_bearertoken.bad.yml index 15aa7aad2..756644be6 100644 --- a/config/testdata/kubernetes_bearertoken.bad.yml +++ b/config/testdata/kubernetes_bearertoken.bad.yml @@ -2,7 +2,7 @@ scrape_configs: - job_name: prometheus kubernetes_sd_configs: - - masters: + - api_servers: - 'https://localhost:1234' bearer_token: 1234 diff --git a/config/testdata/kubernetes_bearertoken_basicauth.bad.yml b/config/testdata/kubernetes_bearertoken_basicauth.bad.yml index f138d9e50..9ef16ae4b 100644 --- a/config/testdata/kubernetes_bearertoken_basicauth.bad.yml +++ b/config/testdata/kubernetes_bearertoken_basicauth.bad.yml @@ -2,7 +2,7 @@ scrape_configs: - job_name: prometheus kubernetes_sd_configs: - - masters: + - api_servers: - 'https://localhost:1234' bearer_token: 1234 diff --git a/documentation/examples/prometheus-kubernetes.yml b/documentation/examples/prometheus-kubernetes.yml index 115e86b88..776864a13 100644 --- a/documentation/examples/prometheus-kubernetes.yml +++ b/documentation/examples/prometheus-kubernetes.yml @@ -1,5 +1,5 @@ # A scrape configuration for running Prometheus on a Kubernetes cluster. -# This uses separate scrape configs for cluster components (i.e. master, node) +# This uses separate scrape configs for cluster components (i.e. API server, node) # and services to allow each to use different authentication configs. # # Kubernetes labels will be added as Prometheus labels on metrics via the @@ -18,14 +18,14 @@ scrape_configs: bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token kubernetes_sd_configs: - - masters: + - api_servers: - 'https://kubernetes.default.svc' in_cluster: true relabel_configs: - source_labels: [__meta_kubernetes_role] action: keep - regex: ^(?:master|node)$ + regex: ^(?:apiserver|node)$ - action: labelmap regex: ^__meta_kubernetes_node_label_(.+)$ replacement: $1 @@ -49,7 +49,7 @@ scrape_configs: - job_name: 'kubernetes-services' kubernetes_sd_configs: - - masters: + - api_servers: - 'https://kubernetes.default.svc' in_cluster: true diff --git a/retrieval/discovery/kubernetes/discovery.go b/retrieval/discovery/kubernetes/discovery.go index 903478f96..e45c4a48c 100644 --- a/retrieval/discovery/kubernetes/discovery.go +++ b/retrieval/discovery/kubernetes/discovery.go @@ -49,8 +49,8 @@ const ( serviceAnnotationPrefix = metaLabelPrefix + "service_annotation_" // nodesTargetGroupName is the name given to the target group for nodes. nodesTargetGroupName = "nodes" - // mastersTargetGroupName is the name given to the target group for masters. - mastersTargetGroupName = "masters" + // apiServersTargetGroupName is the name given to the target group for API servers. + apiServersTargetGroupName = "apiServers" // roleLabel is the name for the label containing a target's role. roleLabel = metaLabelPrefix + "role" @@ -70,8 +70,8 @@ type Discovery struct { client *http.Client Conf *config.KubernetesSDConfig - masters []config.URL - mastersMu sync.RWMutex + apiServers []config.URL + apiServersMu sync.RWMutex nodesResourceVersion string servicesResourceVersion string endpointsResourceVersion string @@ -90,7 +90,7 @@ func (kd *Discovery) Initialize() error { return err } - kd.masters = kd.Conf.Masters + kd.apiServers = kd.Conf.APIServers kd.client = client kd.nodes = map[string]*Node{} kd.services = map[string]map[string]*Service{} @@ -101,12 +101,12 @@ func (kd *Discovery) Initialize() error { // Sources implements the TargetProvider interface. func (kd *Discovery) Sources() []string { - sourceNames := make([]string, 0, len(kd.masters)) - for _, master := range kd.masters { - sourceNames = append(sourceNames, mastersTargetGroupName+":"+master.Host) + sourceNames := make([]string, 0, len(kd.apiServers)) + for _, apiServer := range kd.apiServers { + sourceNames = append(sourceNames, apiServersTargetGroupName+":"+apiServer.Host) } - res, err := kd.queryMasterPath(nodesURL) + res, err := kd.queryAPIServerPath(nodesURL) if err != nil { // If we can't list nodes then we can't watch them. Assume this is a misconfiguration // & log & return empty. @@ -135,7 +135,7 @@ func (kd *Discovery) Sources() []string { kd.nodes[node.ObjectMeta.Name] = &nodes.Items[idx] } - res, err = kd.queryMasterPath(servicesURL) + res, err = kd.queryAPIServerPath(servicesURL) if err != nil { // If we can't list services then we can't watch them. Assume this is a misconfiguration // & log & return empty. @@ -174,7 +174,7 @@ func (kd *Discovery) Sources() []string { func (kd *Discovery) Run(ch chan<- config.TargetGroup, done <-chan struct{}) { defer close(ch) - if tg := kd.updateMastersTargetGroup(); tg != nil { + if tg := kd.updateAPIServersTargetGroup(); tg != nil { select { case ch <- *tg: case <-done: @@ -243,61 +243,63 @@ func (kd *Discovery) Run(ch chan<- config.TargetGroup, done <-chan struct{}) { } } -func (kd *Discovery) queryMasterPath(path string) (*http.Response, error) { +func (kd *Discovery) queryAPIServerPath(path string) (*http.Response, error) { req, err := http.NewRequest("GET", path, nil) if err != nil { return nil, err } - return kd.queryMasterReq(req) + return kd.queryAPIServerReq(req) } -func (kd *Discovery) queryMasterReq(req *http.Request) (*http.Response, error) { - // Lock in case we need to rotate masters to request. - kd.mastersMu.Lock() - defer kd.mastersMu.Unlock() - for i := 0; i < len(kd.masters); i++ { +func (kd *Discovery) queryAPIServerReq(req *http.Request) (*http.Response, error) { + // Lock in case we need to rotate API servers to request. + kd.apiServersMu.Lock() + defer kd.apiServersMu.Unlock() + var lastErr error + for i := 0; i < len(kd.apiServers); i++ { cloneReq := *req - cloneReq.URL.Host = kd.masters[0].Host - cloneReq.URL.Scheme = kd.masters[0].Scheme + cloneReq.URL.Host = kd.apiServers[0].Host + cloneReq.URL.Scheme = kd.apiServers[0].Scheme res, err := kd.client.Do(&cloneReq) if err == nil { return res, nil } - kd.rotateMasters() + lastErr = err + kd.rotateAPIServers() } - return nil, fmt.Errorf("Unable to query any masters") + return nil, fmt.Errorf("Unable to query any API servers: %v", lastErr) } -func (kd *Discovery) rotateMasters() { - if len(kd.masters) > 1 { - kd.masters = append(kd.masters[1:], kd.masters[0]) +func (kd *Discovery) rotateAPIServers() { + if len(kd.apiServers) > 1 { + kd.apiServers = append(kd.apiServers[1:], kd.apiServers[0]) } } -func (kd *Discovery) updateMastersTargetGroup() *config.TargetGroup { +func (kd *Discovery) updateAPIServersTargetGroup() *config.TargetGroup { tg := &config.TargetGroup{ - Source: mastersTargetGroupName, + Source: apiServersTargetGroupName, Labels: model.LabelSet{ - roleLabel: model.LabelValue("master"), + roleLabel: model.LabelValue("apiserver"), }, } - for _, master := range kd.masters { - masterAddress := master.Host - _, _, err := net.SplitHostPort(masterAddress) + for _, apiServer := range kd.apiServers { + apiServerAddress := apiServer.Host + _, _, err := net.SplitHostPort(apiServerAddress) // If error then no port is specified - use default for scheme. if err != nil { - switch master.Scheme { + switch apiServer.Scheme { case "http": - masterAddress = net.JoinHostPort(masterAddress, "80") + apiServerAddress = net.JoinHostPort(apiServerAddress, "80") case "https": - masterAddress = net.JoinHostPort(masterAddress, "443") + apiServerAddress = net.JoinHostPort(apiServerAddress, "443") } } t := model.LabelSet{ - model.AddressLabel: model.LabelValue(masterAddress), - model.SchemeLabel: model.LabelValue(master.Scheme), + model.AddressLabel: model.LabelValue(apiServerAddress), + model.SchemeLabel: model.LabelValue(apiServer.Scheme), } tg.Targets = append(tg.Targets, t) } @@ -360,7 +362,7 @@ func (kd *Discovery) watchNodes(events chan interface{}, done <-chan struct{}, r values.Add("watch", "true") values.Add("resourceVersion", kd.nodesResourceVersion) req.URL.RawQuery = values.Encode() - res, err := kd.queryMasterReq(req) + res, err := kd.queryAPIServerReq(req) if err != nil { log.Errorf("Failed to watch nodes: %s", err) return @@ -402,7 +404,7 @@ func (kd *Discovery) watchServices(events chan interface{}, done <-chan struct{} values.Add("resourceVersion", kd.servicesResourceVersion) req.URL.RawQuery = values.Encode() - res, err := kd.queryMasterReq(req) + res, err := kd.queryAPIServerReq(req) if err != nil { log.Errorf("Failed to watch services: %s", err) return @@ -473,7 +475,7 @@ func (kd *Discovery) addService(service *Service) *config.TargetGroup { namespace[service.ObjectMeta.Name] = service endpointURL := fmt.Sprintf(serviceEndpointsURL, service.ObjectMeta.Namespace, service.ObjectMeta.Name) - res, err := kd.queryMasterPath(endpointURL) + res, err := kd.queryAPIServerPath(endpointURL) if err != nil { log.Errorf("Error getting service endpoints: %s", err) return nil @@ -546,7 +548,7 @@ func (kd *Discovery) watchServiceEndpoints(events chan interface{}, done <-chan values.Add("resourceVersion", kd.servicesResourceVersion) req.URL.RawQuery = values.Encode() - res, err := kd.queryMasterReq(req) + res, err := kd.queryAPIServerReq(req) if err != nil { log.Errorf("Failed to watch service endpoints: %s", err) return From 104803e74efd7eb10d64345c1c01e209f21c580f Mon Sep 17 00:00:00 2001 From: Jimmi Dyson Date: Sat, 24 Oct 2015 19:29:58 +0100 Subject: [PATCH 3/3] Kubernetes SD: Cleanup example config label regexps --- .../examples/prometheus-kubernetes.yml | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/documentation/examples/prometheus-kubernetes.yml b/documentation/examples/prometheus-kubernetes.yml index 776864a13..e0f6df447 100644 --- a/documentation/examples/prometheus-kubernetes.yml +++ b/documentation/examples/prometheus-kubernetes.yml @@ -25,13 +25,13 @@ scrape_configs: relabel_configs: - source_labels: [__meta_kubernetes_role] action: keep - regex: ^(?:apiserver|node)$ + regex: (?:apiserver|node) - action: labelmap - regex: ^__meta_kubernetes_node_label_(.+)$ + regex: __meta_kubernetes_node_label_(.+) replacement: $1 - source_labels: [__meta_kubernetes_role] action: replace - regex: ^(.+)$ + regex: (.+) target_label: kubernetes_role replacement: $1 @@ -56,37 +56,37 @@ scrape_configs: relabel_configs: - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] action: keep - regex: ^true$ + regex: true - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] action: replace target_label: __scheme__ - regex: ^(https?)$ + regex: (https?) replacement: $1 - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] action: replace target_label: __metrics_path__ - regex: ^(.+)$ + regex: (.+) replacement: $1 - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] action: replace target_label: __address__ - regex: ^(.+)(?::\d+);(\d+)$ + regex: (.+)(?::\d+);(\d+) replacement: $1:$2 - action: labelmap - regex: ^__meta_kubernetes_service_label_(.+)$ + regex: __meta_kubernetes_service_label_(.+) replacement: $1 - source_labels: [__meta_kubernetes_role] action: replace - regex: ^(.+)$ + regex: (.+) target_label: kubernetes_role replacement: $1 - source_labels: [__meta_kubernetes_service_namespace] action: replace - regex: ^(.+)$ + regex: (.+) target_label: kubernetes_namespace replacement: $1 - source_labels: [__meta_kubernetes_service_name] action: replace - regex: ^(.+)$ + regex: (.+) target_label: kubernetes_name replacement: $1