From c85efa02d90b8ef5a01b9343bf405df8b242f993 Mon Sep 17 00:00:00 2001 From: Georg Gadinger Date: Tue, 1 Feb 2022 10:20:03 +0100 Subject: [PATCH 1/3] Fix k8s target discovery when not running inside k8s When using Kubernetes service discovery on a Prometheus instance that's not running inside Kubernetes, the creation of the service discovery fails with a "no such file or directory" error as the special `/var/run/secrets/kubernetes.io/serviceaccount/namespace` file is not there. This commit moves the code that reads this file into the if-branch where no `APIServer.URL` is given (that one basically makes Prometheus assume it is running inside of a Kubernetes cluster). Signed-off-by: Georg Gadinger --- discovery/kubernetes/kubernetes.go | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/discovery/kubernetes/kubernetes.go b/discovery/kubernetes/kubernetes.go index 5109b86ae..7c2ab5ed4 100644 --- a/discovery/kubernetes/kubernetes.go +++ b/discovery/kubernetes/kubernetes.go @@ -263,7 +263,7 @@ func (d *Discovery) getNamespaces() []string { return []string{apiv1.NamespaceAll} } - if includeOwnNamespace { + if includeOwnNamespace && d.ownNamespace != "" { return append(namespaces, d.ownNamespace) } @@ -276,8 +276,9 @@ func New(l log.Logger, conf *SDConfig) (*Discovery, error) { l = log.NewNopLogger() } var ( - kcfg *rest.Config - err error + kcfg *rest.Config + err error + ownNamespace string ) if conf.KubeConfig != "" { kcfg, err = clientcmd.BuildConfigFromFlags("", conf.KubeConfig) @@ -291,6 +292,13 @@ func New(l log.Logger, conf *SDConfig) (*Discovery, error) { if err != nil { return nil, err } + + ownNamespaceContents, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace") + if err != nil { + return nil, fmt.Errorf("could not determine the pod's namespace: %w", err) + } + ownNamespace = string(ownNamespaceContents) + level.Info(l).Log("msg", "Using pod service account via in-cluster config") } else { rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "kubernetes_sd") @@ -310,11 +318,6 @@ func New(l log.Logger, conf *SDConfig) (*Discovery, error) { return nil, err } - ownNamespace, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace") - if err != nil { - return nil, fmt.Errorf("could not determine the pod's namespace: %w", err) - } - return &Discovery{ client: c, logger: l, @@ -322,7 +325,7 @@ func New(l log.Logger, conf *SDConfig) (*Discovery, error) { namespaceDiscovery: &conf.NamespaceDiscovery, discoverers: make([]discovery.Discoverer, 0), selectors: mapSelector(conf.Selectors), - ownNamespace: string(ownNamespace), + ownNamespace: ownNamespace, }, nil } From 9d63502204fe3119850141cdbde68969736fead2 Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Tue, 1 Feb 2022 14:59:09 +0100 Subject: [PATCH 2/3] k8s: improve 'own_namespace' Fail configuration unmarshalling if kubeconfig or api url are set with "own namespace" Only read namespace file if needed. Signed-off-by: Julien Pivotto --- config/config_test.go | 8 +++++++ ...etes_api_server_with_own_namespace.bad.yml | 7 +++++++ ...etes_kubeconfig_with_own_namespace.bad.yml | 7 +++++++ discovery/kubernetes/kubernetes.go | 21 ++++++++++++++----- 4 files changed, 38 insertions(+), 5 deletions(-) create mode 100644 config/testdata/kubernetes_api_server_with_own_namespace.bad.yml create mode 100644 config/testdata/kubernetes_kubeconfig_with_own_namespace.bad.yml diff --git a/config/config_test.go b/config/config_test.go index 695c0188a..198c67252 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1181,6 +1181,14 @@ var expectedErrors = []struct { filename: "kubernetes_http_config_without_api_server.bad.yml", errMsg: "to use custom HTTP client configuration please provide the 'api_server' URL explicitly", }, + { + filename: "kubernetes_kubeconfig_with_own_namespace.bad.yml", + errMsg: "cannot use 'kubeconfig_file' and 'namespaces.own_namespace' simultaneously", + }, + { + filename: "kubernetes_api_server_with_own_namespace.bad.yml", + errMsg: "cannot use 'api_server' and 'namespaces.own_namespace' simultaneously", + }, { filename: "kubernetes_kubeconfig_with_apiserver.bad.yml", errMsg: "cannot use 'kubeconfig_file' and 'api_server' simultaneously", diff --git a/config/testdata/kubernetes_api_server_with_own_namespace.bad.yml b/config/testdata/kubernetes_api_server_with_own_namespace.bad.yml new file mode 100644 index 000000000..8a7f49b4b --- /dev/null +++ b/config/testdata/kubernetes_api_server_with_own_namespace.bad.yml @@ -0,0 +1,7 @@ +scrape_configs: + - job_name: prometheus + kubernetes_sd_configs: + - role: endpoints + api_server: 'https://localhost:1234' + namespaces: + own_namespace: true diff --git a/config/testdata/kubernetes_kubeconfig_with_own_namespace.bad.yml b/config/testdata/kubernetes_kubeconfig_with_own_namespace.bad.yml new file mode 100644 index 000000000..d6999fcc6 --- /dev/null +++ b/config/testdata/kubernetes_kubeconfig_with_own_namespace.bad.yml @@ -0,0 +1,7 @@ +scrape_configs: + - job_name: prometheus + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: /home/User1/.kubeconfig + namespaces: + own_namespace: true diff --git a/discovery/kubernetes/kubernetes.go b/discovery/kubernetes/kubernetes.go index 7c2ab5ed4..f935e9400 100644 --- a/discovery/kubernetes/kubernetes.go +++ b/discovery/kubernetes/kubernetes.go @@ -184,6 +184,12 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { if c.APIServer.URL == nil && !reflect.DeepEqual(c.HTTPClientConfig, config.DefaultHTTPClientConfig) { return errors.Errorf("to use custom HTTP client configuration please provide the 'api_server' URL explicitly") } + if c.APIServer.URL != nil && c.NamespaceDiscovery.IncludeOwnNamespace { + return errors.Errorf("cannot use 'api_server' and 'namespaces.own_namespace' simultaneously") + } + if c.KubeConfig != "" && c.NamespaceDiscovery.IncludeOwnNamespace { + return errors.Errorf("cannot use 'kubeconfig_file' and 'namespaces.own_namespace' simultaneously") + } foundSelectorRoles := make(map[Role]struct{}) allowedSelectors := map[Role][]string{ @@ -263,7 +269,7 @@ func (d *Discovery) getNamespaces() []string { return []string{apiv1.NamespaceAll} } - if includeOwnNamespace && d.ownNamespace != "" { + if includeOwnNamespace { return append(namespaces, d.ownNamespace) } @@ -293,11 +299,16 @@ func New(l log.Logger, conf *SDConfig) (*Discovery, error) { return nil, err } - ownNamespaceContents, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace") - if err != nil { - return nil, fmt.Errorf("could not determine the pod's namespace: %w", err) + if conf.NamespaceDiscovery.IncludeOwnNamespace { + ownNamespaceContents, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace") + if err != nil { + return nil, fmt.Errorf("could not determine the pod's namespace: %w", err) + } + if len(ownNamespaceContents) == 0 { + return nil, errors.New("could not read own namespace name (empty file)") + } + ownNamespace = string(ownNamespaceContents) } - ownNamespace = string(ownNamespaceContents) level.Info(l).Log("msg", "Using pod service account via in-cluster config") } else { From 7f57110fd05a5e4cd7fd2b328d6c68f462cb176f Mon Sep 17 00:00:00 2001 From: beorn7 Date: Wed, 2 Feb 2022 14:38:12 +0100 Subject: [PATCH 3/3] Cut v2.33.1 Signed-off-by: beorn7 --- CHANGELOG.md | 4 ++++ VERSION | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bb316df3..064d4b5f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 2.33.1 / 2022-02-02 + +* [BUGFIX] SD: Fix _no such file or directory_ in K8s SD when not running inside K8s. #10235 + ## 2.33.0 / 2022-01-29 * [CHANGE] PromQL: Promote negative offset and `@` modifer to stable features. #10121 diff --git a/VERSION b/VERSION index 3afbaeb2b..ba13d3caf 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.33.0 +2.33.1