Add HostProcess Container Configuration for k8s
Co-authored-by: Brian Redmond <brianisrunning@gmail.com> Signed-off-by: Brian Redmond <brianisrunning@gmail.com> Signed-off-by: James Sturtevant <jstur@microsoft.com>
This commit is contained in:
James Sturtevant
James Sturtevant
parent
e07b2053af
commit
b450a50103
|
|
@ -11,6 +11,10 @@ on:
|
|||
- published
|
||||
- edited
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: windows-2019
|
||||
|
|
@ -34,7 +38,7 @@ jobs:
|
|||
run: make e2e-test
|
||||
|
||||
lint:
|
||||
runs-on: windows-2019
|
||||
runs-on: windows-2022
|
||||
steps:
|
||||
# `gofmt` linter run by golangci-lint fails on CRLF line endings (the default for Windows)
|
||||
- name: Set git to use LF
|
||||
|
|
@ -73,7 +77,7 @@ jobs:
|
|||
ignore_words_list: calle
|
||||
|
||||
build:
|
||||
runs-on: windows-2019
|
||||
runs-on: windows-2022
|
||||
needs:
|
||||
- test
|
||||
- lint
|
||||
|
|
@ -90,6 +94,7 @@ jobs:
|
|||
|
||||
- name: Install Build deps
|
||||
run: |
|
||||
dotnet tool install --global GitVersion.Tool --version 5.*
|
||||
go get github.com/prometheus/promu@v0.11.1
|
||||
go get github.com/josephspurrier/goversioninfo/cmd/goversioninfo@v1.2.0
|
||||
# GOPATH\bin dir must be added to PATH else the `promu` and `goversioninfo` commands won't be found
|
||||
|
|
@ -99,13 +104,14 @@ jobs:
|
|||
run: |
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
gitversion /output json /showvariable FullSemVer | Set-Content VERSION -PassThru
|
||||
dotnet-gitversion /output json /showvariable FullSemVer | Set-Content VERSION -PassThru
|
||||
$Version = Get-Content VERSION
|
||||
# Windows versioninfo resources need the file version by parts (but product version is free text)
|
||||
$VersionParts = ($Version -replace '^v?([0-9\.]+).*$','$1').Split(".")
|
||||
goversioninfo.exe -ver-major $VersionParts[0] -ver-minor $VersionParts[1] -ver-patch $VersionParts[2] -product-version $Version -platform-specific
|
||||
|
||||
make crossbuild
|
||||
make build-all
|
||||
# GH requires all files to have different names, so add version/arch to differentiate
|
||||
foreach($Arch in "amd64","386") {
|
||||
Move-Item output\$Arch\windows_exporter.exe output\windows_exporter-$Version-$Arch.exe
|
||||
|
|
@ -133,6 +139,21 @@ jobs:
|
|||
|
||||
promu checksum output\
|
||||
|
||||
- name: Login to GitHub container registry
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: docker/login-action@v1
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Push Latest image
|
||||
if: ${{ github.event_name != 'pull_request' }}
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
VERSION=latest make push-all
|
||||
|
||||
- name: Release
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
env:
|
||||
|
|
@ -140,3 +161,4 @@ jobs:
|
|||
run: |
|
||||
$TagName = $env:GITHUB_REF -replace 'refs/tags/', ''
|
||||
Get-ChildItem -Path output\* -Include @('windows_exporter*.msi', 'windows_exporter*.exe', 'sha256sums.txt') | Foreach-Object {gh release upload $TagName $_}
|
||||
make push-all
|
||||
|
|
|
|||
|
|
@ -0,0 +1,9 @@
|
|||
# Note this image doesn't really matter for hostprocess but it is good to build per OS version
|
||||
# the files in the image are copied to $env:CONTAINER_SANDBOX_MOUNT_POINT on the host
|
||||
# but the file system is the Host NOT the container
|
||||
ARG BASE="mcr.microsoft.com/windows/nanoserver:1809"
|
||||
FROM $BASE
|
||||
|
||||
ENV PATH="C:\Windows\system32;C:\Windows;"
|
||||
COPY output/amd64/windows_exporter.exe /windows_exporter.exe
|
||||
ENTRYPOINT ["windows_exporter.exe"]
|
||||
31
Makefile
31
Makefile
|
|
@ -1,4 +1,15 @@
|
|||
export GOOS=windows
|
||||
export DOCKER_IMAGE_NAME ?= windows-exporter
|
||||
export DOCKER_REPO ?= ghcr.io/prometheus-community
|
||||
|
||||
VERSION?=$(shell cat VERSION)
|
||||
DOCKER?=docker
|
||||
|
||||
# Image Variables for Hostprocess Container
|
||||
# Windows image build is heavily influenced by https://github.com/kubernetes/kubernetes/blob/master/cluster/images/etcd/Makefile
|
||||
OS=1809
|
||||
ALL_OS:= 1809 ltsc2022
|
||||
BASE_IMAGE=mcr.microsoft.com/windows/nanoserver
|
||||
|
||||
.PHONY: build
|
||||
build: windows_exporter.exe
|
||||
|
|
@ -26,3 +37,23 @@ crossbuild:
|
|||
# on Windows, so for now, we'll just build twice
|
||||
GOARCH=amd64 promu build --prefix=output/amd64
|
||||
GOARCH=386 promu build --prefix=output/386
|
||||
|
||||
build-image: crossbuild
|
||||
$(DOCKER) build --build-arg=BASE=$(BASE_IMAGE):$(OS) -f Dockerfile -t $(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(VERSION)-$(OS) .
|
||||
|
||||
sub-build-%:
|
||||
$(MAKE) OS=$* build-image
|
||||
|
||||
build-all: $(addprefix sub-build-,$(ALL_OS))
|
||||
|
||||
push:
|
||||
set -x; \
|
||||
for osversion in ${ALL_OS}; do \
|
||||
$(DOCKER) push $(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(VERSION)-$${osversion}; \
|
||||
$(DOCKER) manifest create --amend $(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(VERSION) $(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(VERSION)-$${osversion}; \
|
||||
full_version=`$(DOCKER) manifest inspect $(BASE_IMAGE):$${osversion} | grep "os.version" | head -n 1 | awk -F\" '{print $$4}'` || true; \
|
||||
$(DOCKER) manifest annotate --os windows --arch amd64 --os-version $${full_version} $(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(VERSION) $(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(VERSION)-$${osversion}; \
|
||||
done
|
||||
$(DOCKER) manifest push --purge $(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(VERSION)
|
||||
|
||||
push-all: build-all push
|
||||
|
|
|
|||
|
|
@ -115,6 +115,11 @@ On some older versions of Windows you may need to surround parameter values with
|
|||
msiexec /i C:\Users\Administrator\Downloads\windows_exporter.msi ENABLED_COLLECTORS="ad,iis,logon,memory,process,tcp,thermalzone" TEXTFILE_DIR="C:\custom_metrics\"
|
||||
```
|
||||
|
||||
|
||||
## Kubernetes Implementation
|
||||
|
||||
See detailed steps to install on Windows Kubernetes [here](./kubernetes/kubernetes.md).
|
||||
|
||||
## Supported versions
|
||||
|
||||
windows_exporter supports Windows Server versions 2008R2 and later, and desktop Windows version 7 and later.
|
||||
|
|
|
|||
11
exporter.go
11
exporter.go
|
|
@ -9,6 +9,7 @@ import (
|
|||
"net/http"
|
||||
_ "net/http/pprof"
|
||||
"os"
|
||||
"os/user"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
|
@ -345,6 +346,16 @@ func main() {
|
|||
log.Fatalf("Couldn't load collectors: %s", err)
|
||||
}
|
||||
|
||||
u, err := user.Current()
|
||||
if err != nil {
|
||||
log.Fatalf(err.Error())
|
||||
}
|
||||
|
||||
log.Infof("Running as %v", u.Username)
|
||||
if strings.Contains(u.Username, "ContainerAdministrator") || strings.Contains(u.Username, "ContainerUser") {
|
||||
log.Warnf("Running as a preconfigured Windows Container user. This may mean you do not have Windows HostProcess containers configured correctly and some functionality will not work as expected.")
|
||||
}
|
||||
|
||||
log.Infof("Enabled collectors: %v", strings.Join(keys(collectors), ", "))
|
||||
|
||||
h := &metricsHandler{
|
||||
|
|
|
|||
|
|
@ -0,0 +1,91 @@
|
|||
# windows_exporter on Kubernetes
|
||||
|
||||
With Kubernetes supporting HostProcess containers on Windows nodes (as of [v1.22](https://kubernetes.io/blog/2021/08/16/windows-hostprocess-containers/), it is useful to run the `windows_exporter` as a container on Windows to export metrics for your Prometheus implementation. Read the [Kubernetes HostProcess documentation](https://kubernetes.io/docs/tasks/configure-pod-container/create-hostprocess-pod/) for more information.
|
||||
|
||||
Requirements:
|
||||
|
||||
- Kubernetes 1.22+
|
||||
- containerd 1.6 Beta+
|
||||
- WindowsHostProcessContainers feature-gate turned on for `kube-apiserver` and `kubelet`
|
||||
|
||||
> IMPORTANT: This does not work unless you are specifically targeting Host Process Containers with Containerd (Docker doesn't have support). The image will build but will **not** be able to access the host.
|
||||
|
||||
## Container Image
|
||||
|
||||
The image is multi arch image (WS 2019, WS 2022) built on Windows. To build the images:
|
||||
|
||||
```
|
||||
DOCKER_REPO=<your repo> make push-all
|
||||
```
|
||||
|
||||
If you don't have a version of `make` on your Windows machine, You can use WSL to build the image with Windows Containers by creating a symbolic link to the docker cli and then override the docker command in the `Makefile`:
|
||||
|
||||
On Windows:
|
||||
```
|
||||
Item -ItemType SymbolicLink -Path "c:\docker" -Target "C:\Program Files\Docker\Docker\resources\bin\docker.exe"
|
||||
|
||||
In WSL:
|
||||
```
|
||||
DOCKER_REPO=<your repo> DOCKER=/mnt/c/docker make push-all
|
||||
```
|
||||
|
||||
## Kubernetes Quick Start
|
||||
|
||||
Before beginning you need to deploy the [prometheus operator](https://github.com/prometheus-operator/prometheus-operator) to your cluster. As a quick start, you can use a project like https://github.com/prometheus-operator/kube-prometheus. The export itself doesn't have any dependency on prometheus operator and the exporter image can be used in manual configurations.
|
||||
|
||||
### Windows Exporter DaemonSet
|
||||
|
||||
This create a deployment on every node. A config map is created for to handle the configuration of the Windows exporter with [configuration file](../README.md#using-a-configuration-file). Adjust the configuration file for the collectors you are interested in.
|
||||
|
||||
```bash
|
||||
kubectl apply -f kubernetes/windows-exporter-daemonset.yaml
|
||||
```
|
||||
|
||||
> Note: This example manifest deploys the latest bleeding edge image `ghcr.io/prometheus-community/windows-exporter:latest` built from the main branch. You should update this to use a released version which you can find at https://github.com/prometheus-community/windows_exporter/releases
|
||||
|
||||
#### Configuring the firewall
|
||||
The firewall on the node needs to be configured to allow connections on the node: `New-NetFirewallRule -DisplayName 'windows-exporter' -Direction inbound -Profile Any -Action Allow -LocalPort 9182 -Protocol TCP`
|
||||
|
||||
You could do this by adding an init container but if you remove the deployment at a later date you will need to remove the firewall rule manually. The following could be added to the `windows-exporter-daemonset.yaml`:
|
||||
|
||||
```
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
initContainers:
|
||||
- name: configure-firewall
|
||||
image: mcr.microsoft.com/windows/nanoserver:1809
|
||||
command: ["powershell"]
|
||||
args: ["New-NetFirewallRule", "-DisplayName", "'windows-exporter'", "-Direction", "inbound", "-Profile", "Any", "-Action", "Allow", "-LocalPort", "9182", "-Protocol", "TCP"]
|
||||
```
|
||||
|
||||
### Prometheus PodMonitor
|
||||
|
||||
Create the [Pod Monitor](https://prometheus-operator.dev/docs/operator/design/#podmonitor) to configure the scraping:
|
||||
|
||||
```bash
|
||||
kubectl apply -f windows-exporter-podmonitor.yaml
|
||||
```
|
||||
|
||||
### View Metrics
|
||||
|
||||
Open Prometheus with
|
||||
|
||||
```
|
||||
kubectl --namespace monitoring port-forward svc/prometheus-k8s 9091:9090
|
||||
```
|
||||
|
||||
Navigate to prometheus UI and add a query to see node cpu (replacing with your ip address)
|
||||
|
||||
```
|
||||
sum by (mode) (irate(windows_cpu_time_total{instance="10.1.0.5:9182"}[5m]))
|
||||
```
|
||||
|
||||
|
||||
|
||||
|
||||
## Configuring TLS
|
||||
|
||||
It is possible to configure TLS of the solution using `--web.config.file`. Read more at https://github.com/prometheus/exporter-toolkit/blob/master/docs/web-configuration.md
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
labels:
|
||||
app: windows-exporter
|
||||
name: windows-exporter
|
||||
namespace: monitoring
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: windows-exporter
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: windows-exporter
|
||||
spec:
|
||||
securityContext:
|
||||
windowsOptions:
|
||||
hostProcess: true
|
||||
runAsUserName: "NT AUTHORITY\\system"
|
||||
hostNetwork: true
|
||||
initContainers:
|
||||
- name: configure-firewall
|
||||
image: mcr.microsoft.com/windows/nanoserver:1809
|
||||
command: ["powershell"]
|
||||
args: ["New-NetFirewallRule", "-DisplayName", "'windows-exporter'", "-Direction", "inbound", "-Profile", "Any", "-Action", "Allow", "-LocalPort", "9182", "-Protocol", "TCP"]
|
||||
containers:
|
||||
- args:
|
||||
- --config.file=%CONTAINER_SANDBOX_MOUNT_POINT%/config.yml
|
||||
name: windows-exporter
|
||||
image: ghcr.io/prometheus-community/windows-exporter:latest
|
||||
imagePullPolicy: Always
|
||||
ports:
|
||||
- containerPort: 9182
|
||||
hostPort: 9182
|
||||
name: http
|
||||
volumeMounts:
|
||||
- name: windows-exporter-config
|
||||
mountPath: /config.yml
|
||||
subPath: config.yml
|
||||
nodeSelector:
|
||||
kubernetes.io/os: windows
|
||||
volumes:
|
||||
- name: windows-exporter-config
|
||||
configMap:
|
||||
name: windows-exporter-config
|
||||
---
|
||||
kind: ConfigMap
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: windows-exporter-config
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app: windows-exporter
|
||||
data:
|
||||
config.yml: |
|
||||
collectors:
|
||||
enabled: '[defaults],container'
|
||||
collector:
|
||||
service:
|
||||
services-where: "Name='containerd' or Name='kubelet'"
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PodMonitor
|
||||
metadata:
|
||||
labels:
|
||||
app: windows-exporter
|
||||
name: windows-exporter
|
||||
namespace: monitoring
|
||||
spec:
|
||||
jobLabel: windows-exporter
|
||||
selector:
|
||||
matchLabels:
|
||||
app: windows-exporter
|
||||
podMetricsEndpoints:
|
||||
- port: http
|
||||
scheme: http
|
||||
Loading…
Reference in New Issue