From b450a501032136ffa5b4f22de90d656c83d9c9d8 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Thu, 30 Jul 2020 17:36:58 -0600 Subject: [PATCH] Add HostProcess Container Configuration for k8s Co-authored-by: Brian Redmond Signed-off-by: Brian Redmond Signed-off-by: James Sturtevant --- .github/workflows/ci.yml | 28 ++++++- Dockerfile | 9 ++ Makefile | 31 +++++++ README.md | 5 ++ exporter.go | 11 +++ kubernetes/kubernetes.md | 91 +++++++++++++++++++++ kubernetes/windows-exporter-daemonset.yaml | 61 ++++++++++++++ kubernetes/windows-exporter-podmonitor.yaml | 15 ++++ 8 files changed, 248 insertions(+), 3 deletions(-) create mode 100644 Dockerfile create mode 100644 kubernetes/kubernetes.md create mode 100644 kubernetes/windows-exporter-daemonset.yaml create mode 100644 kubernetes/windows-exporter-podmonitor.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 33e2e22b..4738c23c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,10 @@ on: - published - edited +permissions: + contents: read + packages: write + jobs: test: runs-on: windows-2019 @@ -34,7 +38,7 @@ jobs: run: make e2e-test lint: - runs-on: windows-2019 + runs-on: windows-2022 steps: # `gofmt` linter run by golangci-lint fails on CRLF line endings (the default for Windows) - name: Set git to use LF @@ -73,7 +77,7 @@ jobs: ignore_words_list: calle build: - runs-on: windows-2019 + runs-on: windows-2022 needs: - test - lint @@ -90,6 +94,7 @@ jobs: - name: Install Build deps run: | + dotnet tool install --global GitVersion.Tool --version 5.* go get github.com/prometheus/promu@v0.11.1 go get github.com/josephspurrier/goversioninfo/cmd/goversioninfo@v1.2.0 # GOPATH\bin dir must be added to PATH else the `promu` and `goversioninfo` commands won't be found @@ -99,13 +104,14 @@ jobs: run: | $ErrorActionPreference = "Stop" - gitversion /output json /showvariable FullSemVer | Set-Content VERSION -PassThru + dotnet-gitversion /output json /showvariable FullSemVer | Set-Content VERSION -PassThru $Version = Get-Content VERSION # Windows versioninfo resources need the file version by parts (but product version is free text) $VersionParts = ($Version -replace '^v?([0-9\.]+).*$','$1').Split(".") goversioninfo.exe -ver-major $VersionParts[0] -ver-minor $VersionParts[1] -ver-patch $VersionParts[2] -product-version $Version -platform-specific make crossbuild + make build-all # GH requires all files to have different names, so add version/arch to differentiate foreach($Arch in "amd64","386") { Move-Item output\$Arch\windows_exporter.exe output\windows_exporter-$Version-$Arch.exe @@ -133,6 +139,21 @@ jobs: promu checksum output\ + - name: Login to GitHub container registry + if: startsWith(github.ref, 'refs/tags/') + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Push Latest image + if: ${{ github.event_name != 'pull_request' }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + VERSION=latest make push-all + - name: Release if: startsWith(github.ref, 'refs/tags/') env: @@ -140,3 +161,4 @@ jobs: run: | $TagName = $env:GITHUB_REF -replace 'refs/tags/', '' Get-ChildItem -Path output\* -Include @('windows_exporter*.msi', 'windows_exporter*.exe', 'sha256sums.txt') | Foreach-Object {gh release upload $TagName $_} + make push-all diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..a59d913e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,9 @@ +# Note this image doesn't really matter for hostprocess but it is good to build per OS version +# the files in the image are copied to $env:CONTAINER_SANDBOX_MOUNT_POINT on the host +# but the file system is the Host NOT the container +ARG BASE="mcr.microsoft.com/windows/nanoserver:1809" +FROM $BASE + +ENV PATH="C:\Windows\system32;C:\Windows;" +COPY output/amd64/windows_exporter.exe /windows_exporter.exe +ENTRYPOINT ["windows_exporter.exe"] diff --git a/Makefile b/Makefile index b20cb9cb..94c06d6f 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,15 @@ export GOOS=windows +export DOCKER_IMAGE_NAME ?= windows-exporter +export DOCKER_REPO ?= ghcr.io/prometheus-community + +VERSION?=$(shell cat VERSION) +DOCKER?=docker + +# Image Variables for Hostprocess Container +# Windows image build is heavily influenced by https://github.com/kubernetes/kubernetes/blob/master/cluster/images/etcd/Makefile +OS=1809 +ALL_OS:= 1809 ltsc2022 +BASE_IMAGE=mcr.microsoft.com/windows/nanoserver .PHONY: build build: windows_exporter.exe @@ -26,3 +37,23 @@ crossbuild: # on Windows, so for now, we'll just build twice GOARCH=amd64 promu build --prefix=output/amd64 GOARCH=386 promu build --prefix=output/386 + +build-image: crossbuild + $(DOCKER) build --build-arg=BASE=$(BASE_IMAGE):$(OS) -f Dockerfile -t $(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(VERSION)-$(OS) . + +sub-build-%: + $(MAKE) OS=$* build-image + +build-all: $(addprefix sub-build-,$(ALL_OS)) + +push: + set -x; \ + for osversion in ${ALL_OS}; do \ + $(DOCKER) push $(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(VERSION)-$${osversion}; \ + $(DOCKER) manifest create --amend $(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(VERSION) $(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(VERSION)-$${osversion}; \ + full_version=`$(DOCKER) manifest inspect $(BASE_IMAGE):$${osversion} | grep "os.version" | head -n 1 | awk -F\" '{print $$4}'` || true; \ + $(DOCKER) manifest annotate --os windows --arch amd64 --os-version $${full_version} $(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(VERSION) $(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(VERSION)-$${osversion}; \ + done + $(DOCKER) manifest push --purge $(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(VERSION) + +push-all: build-all push diff --git a/README.md b/README.md index ffac4836..af6871f1 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,11 @@ On some older versions of Windows you may need to surround parameter values with msiexec /i C:\Users\Administrator\Downloads\windows_exporter.msi ENABLED_COLLECTORS="ad,iis,logon,memory,process,tcp,thermalzone" TEXTFILE_DIR="C:\custom_metrics\" ``` + +## Kubernetes Implementation + +See detailed steps to install on Windows Kubernetes [here](./kubernetes/kubernetes.md). + ## Supported versions windows_exporter supports Windows Server versions 2008R2 and later, and desktop Windows version 7 and later. diff --git a/exporter.go b/exporter.go index fc085118..4e87b494 100644 --- a/exporter.go +++ b/exporter.go @@ -9,6 +9,7 @@ import ( "net/http" _ "net/http/pprof" "os" + "os/user" "sort" "strconv" "strings" @@ -345,6 +346,16 @@ func main() { log.Fatalf("Couldn't load collectors: %s", err) } + u, err := user.Current() + if err != nil { + log.Fatalf(err.Error()) + } + + log.Infof("Running as %v", u.Username) + if strings.Contains(u.Username, "ContainerAdministrator") || strings.Contains(u.Username, "ContainerUser") { + log.Warnf("Running as a preconfigured Windows Container user. This may mean you do not have Windows HostProcess containers configured correctly and some functionality will not work as expected.") + } + log.Infof("Enabled collectors: %v", strings.Join(keys(collectors), ", ")) h := &metricsHandler{ diff --git a/kubernetes/kubernetes.md b/kubernetes/kubernetes.md new file mode 100644 index 00000000..be454b87 --- /dev/null +++ b/kubernetes/kubernetes.md @@ -0,0 +1,91 @@ +# windows_exporter on Kubernetes + +With Kubernetes supporting HostProcess containers on Windows nodes (as of [v1.22](https://kubernetes.io/blog/2021/08/16/windows-hostprocess-containers/), it is useful to run the `windows_exporter` as a container on Windows to export metrics for your Prometheus implementation. Read the [Kubernetes HostProcess documentation](https://kubernetes.io/docs/tasks/configure-pod-container/create-hostprocess-pod/) for more information. + +Requirements: + +- Kubernetes 1.22+ +- containerd 1.6 Beta+ +- WindowsHostProcessContainers feature-gate turned on for `kube-apiserver` and `kubelet` + +> IMPORTANT: This does not work unless you are specifically targeting Host Process Containers with Containerd (Docker doesn't have support). The image will build but will **not** be able to access the host. + +## Container Image + +The image is multi arch image (WS 2019, WS 2022) built on Windows. To build the images: + +``` +DOCKER_REPO= make push-all +``` + +If you don't have a version of `make` on your Windows machine, You can use WSL to build the image with Windows Containers by creating a symbolic link to the docker cli and then override the docker command in the `Makefile`: + +On Windows: +``` +Item -ItemType SymbolicLink -Path "c:\docker" -Target "C:\Program Files\Docker\Docker\resources\bin\docker.exe" + +In WSL: +``` +DOCKER_REPO= DOCKER=/mnt/c/docker make push-all +``` + +## Kubernetes Quick Start + +Before beginning you need to deploy the [prometheus operator](https://github.com/prometheus-operator/prometheus-operator) to your cluster. As a quick start, you can use a project like https://github.com/prometheus-operator/kube-prometheus. The export itself doesn't have any dependency on prometheus operator and the exporter image can be used in manual configurations. + +### Windows Exporter DaemonSet + +This create a deployment on every node. A config map is created for to handle the configuration of the Windows exporter with [configuration file](../README.md#using-a-configuration-file). Adjust the configuration file for the collectors you are interested in. + +```bash +kubectl apply -f kubernetes/windows-exporter-daemonset.yaml +``` + +> Note: This example manifest deploys the latest bleeding edge image `ghcr.io/prometheus-community/windows-exporter:latest` built from the main branch. You should update this to use a released version which you can find at https://github.com/prometheus-community/windows_exporter/releases + +#### Configuring the firewall +The firewall on the node needs to be configured to allow connections on the node: `New-NetFirewallRule -DisplayName 'windows-exporter' -Direction inbound -Profile Any -Action Allow -LocalPort 9182 -Protocol TCP` + +You could do this by adding an init container but if you remove the deployment at a later date you will need to remove the firewall rule manually. The following could be added to the `windows-exporter-daemonset.yaml`: + +``` +apiVersion: apps/v1 +kind: DaemonSet +spec: + template: + spec: + initContainers: + - name: configure-firewall + image: mcr.microsoft.com/windows/nanoserver:1809 + command: ["powershell"] + args: ["New-NetFirewallRule", "-DisplayName", "'windows-exporter'", "-Direction", "inbound", "-Profile", "Any", "-Action", "Allow", "-LocalPort", "9182", "-Protocol", "TCP"] +``` + +### Prometheus PodMonitor + +Create the [Pod Monitor](https://prometheus-operator.dev/docs/operator/design/#podmonitor) to configure the scraping: + +```bash +kubectl apply -f windows-exporter-podmonitor.yaml +``` + +### View Metrics + +Open Prometheus with + +``` +kubectl --namespace monitoring port-forward svc/prometheus-k8s 9091:9090 +``` + +Navigate to prometheus UI and add a query to see node cpu (replacing with your ip address) + +``` +sum by (mode) (irate(windows_cpu_time_total{instance="10.1.0.5:9182"}[5m])) +``` + +![windows cpu total time graph in prometheus ui](https://user-images.githubusercontent.com/648372/140547130-b535c766-6479-47d3-b2d3-cd8a551647df.png) + + +## Configuring TLS + +It is possible to configure TLS of the solution using `--web.config.file`. Read more at https://github.com/prometheus/exporter-toolkit/blob/master/docs/web-configuration.md \ No newline at end of file diff --git a/kubernetes/windows-exporter-daemonset.yaml b/kubernetes/windows-exporter-daemonset.yaml new file mode 100644 index 00000000..a87a09f7 --- /dev/null +++ b/kubernetes/windows-exporter-daemonset.yaml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app: windows-exporter + name: windows-exporter + namespace: monitoring +spec: + selector: + matchLabels: + app: windows-exporter + template: + metadata: + labels: + app: windows-exporter + spec: + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + hostNetwork: true + initContainers: + - name: configure-firewall + image: mcr.microsoft.com/windows/nanoserver:1809 + command: ["powershell"] + args: ["New-NetFirewallRule", "-DisplayName", "'windows-exporter'", "-Direction", "inbound", "-Profile", "Any", "-Action", "Allow", "-LocalPort", "9182", "-Protocol", "TCP"] + containers: + - args: + - --config.file=%CONTAINER_SANDBOX_MOUNT_POINT%/config.yml + name: windows-exporter + image: ghcr.io/prometheus-community/windows-exporter:latest + imagePullPolicy: Always + ports: + - containerPort: 9182 + hostPort: 9182 + name: http + volumeMounts: + - name: windows-exporter-config + mountPath: /config.yml + subPath: config.yml + nodeSelector: + kubernetes.io/os: windows + volumes: + - name: windows-exporter-config + configMap: + name: windows-exporter-config +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: windows-exporter-config + namespace: monitoring + labels: + app: windows-exporter +data: + config.yml: | + collectors: + enabled: '[defaults],container' + collector: + service: + services-where: "Name='containerd' or Name='kubelet'" diff --git a/kubernetes/windows-exporter-podmonitor.yaml b/kubernetes/windows-exporter-podmonitor.yaml new file mode 100644 index 00000000..777314ac --- /dev/null +++ b/kubernetes/windows-exporter-podmonitor.yaml @@ -0,0 +1,15 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + labels: + app: windows-exporter + name: windows-exporter + namespace: monitoring +spec: + jobLabel: windows-exporter + selector: + matchLabels: + app: windows-exporter + podMetricsEndpoints: + - port: http + scheme: http