Merge branch 'master' into cleanup/gitlab-exporter-usr-idx

.github/workflows/golangci-lint.yml

@@ -1,3 +1,5 @@
+---
+# This action is synced from https://github.com/prometheus/prometheus
 name: golangci-lint
 on:
   push:
@@ -27,4 +29,4 @@ jobs:
       - name: Lint
         uses: golangci/golangci-lint-action@v3.4.0
         with:
-          version: v1.51.2
+          version: v1.53.3

.golangci.yml

@@ -1,6 +1,7 @@
 ---
 linters:
   enable:
+  - misspell
   - revive
 
 issues:
@@ -14,3 +15,9 @@ linters-settings:
     exclude-functions:
       # Never check for logger errors.
       - (github.com/go-kit/log.Logger).Log
+  revive:
+    rules:
+      # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#unused-parameter
+      - name: unused-parameter
+        severity: warning
+        disabled: true

.yamllint

@@ -20,5 +20,4 @@ rules:
       config/testdata/section_key_dup.bad.yml
   line-length: disable
   truthy:
-    ignore: |
-      .github/workflows/*.yml
+    check-keys: false

Makefile.common

@@ -55,7 +55,7 @@ ifneq ($(shell command -v gotestsum > /dev/null),)
 endif
 endif
 
-PROMU_VERSION ?= 0.14.0
+PROMU_VERSION ?= 0.15.0
 PROMU_URL     := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz
 
 SKIP_GOLANGCI_LINT :=

collector/collector_test.go

@@ -50,7 +50,12 @@ func sanitizeQuery(q string) string {
 	q = strings.Join(strings.Fields(q), " ")
 	q = strings.Replace(q, "(", "\\(", -1)
 	q = strings.Replace(q, ")", "\\)", -1)
+	q = strings.Replace(q, "[", "\\[", -1)
+	q = strings.Replace(q, "]", "\\]", -1)
+	q = strings.Replace(q, "{", "\\{", -1)
+	q = strings.Replace(q, "}", "\\}", -1)
 	q = strings.Replace(q, "*", "\\*", -1)
+	q = strings.Replace(q, "^", "\\^", -1)
 	q = strings.Replace(q, "$", "\\$", -1)
 	return q
 }

collector/pg_database.go

@@ -115,10 +115,7 @@ func (c PGDatabaseCollector) Update(ctx context.Context, instance *instance, ch
 			prometheus.GaugeValue, sizeMetric, datname,
 		)
 	}
-	if err := rows.Err(); err != nil {
-		return err
-	}
-	return nil
+	return rows.Err()
 }
 
 func sliceContains(slice []string, s string) bool {

collector/pg_process_idle.go

@@ -18,6 +18,7 @@ import (
 	"database/sql"
 
 	"github.com/go-kit/log"
+	"github.com/lib/pq"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -39,7 +40,7 @@ func NewPGProcessIdleCollector(config collectorConfig) (Collector, error) {
 var pgProcessIdleSeconds = prometheus.NewDesc(
 	prometheus.BuildFQName(namespace, processIdleSubsystem, "seconds"),
 	"Idle time of server processes",
-	[]string{"application_name"},
+	[]string{"state", "application_name"},
 	prometheus.Labels{},
 )
 
@@ -49,15 +50,17 @@ func (PGProcessIdleCollector) Update(ctx context.Context, instance *instance, ch
 		`WITH
 			metrics AS (
 				SELECT
+				state,
 				application_name,
 				SUM(EXTRACT(EPOCH FROM (CURRENT_TIMESTAMP - state_change))::bigint)::float AS process_idle_seconds_sum,
 				COUNT(*) AS process_idle_seconds_count
 				FROM pg_stat_activity
-				WHERE state = 'idle'
-				GROUP BY application_name
+				WHERE state ~ '^idle'
+				GROUP BY state, application_name
 			),
 			buckets AS (
 				SELECT
+				state,
 				application_name,
 				le,
 				SUM(
@@ -69,35 +72,42 @@ func (PGProcessIdleCollector) Update(ctx context.Context, instance *instance, ch
 				FROM
 				pg_stat_activity,
 				UNNEST(ARRAY[1, 2, 5, 15, 30, 60, 90, 120, 300]) AS le
-				GROUP BY application_name, le
-				ORDER BY application_name, le
+				GROUP BY state, application_name, le
+				ORDER BY state, application_name, le
 			)
 			SELECT
+			state,
 			application_name,
 			process_idle_seconds_sum as seconds_sum,
 			process_idle_seconds_count as seconds_count,
 			ARRAY_AGG(le) AS seconds,
 			ARRAY_AGG(bucket) AS seconds_bucket
-			FROM metrics JOIN buckets USING (application_name)
-			GROUP BY 1, 2, 3;`)
+			FROM metrics JOIN buckets USING (state, application_name)
+			GROUP BY 1, 2, 3, 4;`)
 
+	var state sql.NullString
 	var applicationName sql.NullString
-	var secondsSum sql.NullInt64
+	var secondsSum sql.NullFloat64
 	var secondsCount sql.NullInt64
-	var seconds []uint64
-	var secondsBucket []uint64
+	var seconds []float64
+	var secondsBucket []int64
 
-	err := row.Scan(&applicationName, &secondsSum, &secondsCount, &seconds, &secondsBucket)
+	err := row.Scan(&state, &applicationName, &secondsSum, &secondsCount, pq.Array(&seconds), pq.Array(&secondsBucket))
+	if err != nil {
+		return err
+	}
 
 	var buckets = make(map[float64]uint64, len(seconds))
 	for i, second := range seconds {
 		if i >= len(secondsBucket) {
 			break
 		}
-		buckets[float64(second)] = secondsBucket[i]
+		buckets[second] = uint64(secondsBucket[i])
 	}
-	if err != nil {
-		return err
+
+	stateLabel := "unknown"
+	if state.Valid {
+		stateLabel = state.String
 	}
 
 	applicationNameLabel := "unknown"
@@ -111,12 +121,12 @@ func (PGProcessIdleCollector) Update(ctx context.Context, instance *instance, ch
 	}
 	secondsSumMetric := 0.0
 	if secondsSum.Valid {
-		secondsSumMetric = float64(secondsSum.Int64)
+		secondsSumMetric = secondsSum.Float64
 	}
 	ch <- prometheus.MustNewConstHistogram(
 		pgProcessIdleSeconds,
 		secondsCountMetric, secondsSumMetric, buckets,
-		applicationNameLabel,
+		stateLabel, applicationNameLabel,
 	)
 	return nil
 }

collector/pg_replication.go

@@ -29,7 +29,7 @@ type PGReplicationCollector struct {
 }
 
 func NewPGReplicationCollector(collectorConfig) (Collector, error) {
-	return &PGPostmasterCollector{}, nil
+	return &PGReplicationCollector{}, nil
 }
 
 var (

collector/pg_replication_slot.go

@@ -126,8 +126,5 @@ func (PGReplicationSlotCollector) Update(ctx context.Context, instance *instance
 			prometheus.GaugeValue, isActiveValue, slotNameLabel,
 		)
 	}
-	if err := rows.Err(); err != nil {
-		return err
-	}
-	return nil
+	return rows.Err()
 }

collector/pg_stat_bgwriter_test.go

@@ -51,7 +51,7 @@ func TestPGStatBGWriterCollector(t *testing.T) {
 	}
 
 	rows := sqlmock.NewRows(columns).
-		AddRow(354, 4945, 289097744, 1242257, 3275602074, 89320867, 450139, 2034563757, 0, 2725688749, srT)
+		AddRow(354, 4945, 289097744, 1242257, int64(3275602074), 89320867, 450139, 2034563757, 0, int64(2725688749), srT)
 	mock.ExpectQuery(sanitizeQuery(statBGWriterQuery)).WillReturnRows(rows)
 
 	ch := make(chan prometheus.Metric)

collector/pg_stat_database_test.go

@@ -67,12 +67,12 @@ func TestPGStatDatabaseCollector(t *testing.T) {
 			4945,
 			289097744,
 			1242257,
-			3275602074,
+			int64(3275602074),
 			89320867,
 			450139,
 			2034563757,
 			0,
-			2725688749,
+			int64(2725688749),
 			23,
 			52,
 			74,
@@ -263,12 +263,12 @@ func TestPGStatDatabaseCollectorRowLeakTest(t *testing.T) {
 			4945,
 			289097744,
 			1242257,
-			3275602074,
+			int64(3275602074),
 			89320867,
 			450139,
 			2034563757,
 			0,
-			2725688749,
+			int64(2725688749),
 			23,
 			52,
 			74,

collector/pg_statio_user_tables.go

@@ -218,8 +218,5 @@ func (PGStatIOUserTablesCollector) Update(ctx context.Context, instance *instanc
 			datnameLabel, schemanameLabel, relnameLabel,
 		)
 	}
-	if err := rows.Err(); err != nil {
-		return err
-	}
-	return nil
+	return rows.Err()
 }

collector/pg_wal.go

@@ -0,0 +1,84 @@
+// Copyright 2023 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collector
+
+import (
+	"context"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const walSubsystem = "wal"
+
+func init() {
+	registerCollector(walSubsystem, defaultEnabled, NewPGWALCollector)
+}
+
+type PGWALCollector struct {
+}
+
+func NewPGWALCollector(config collectorConfig) (Collector, error) {
+	return &PGWALCollector{}, nil
+}
+
+var (
+	pgWALSegments = prometheus.NewDesc(
+		prometheus.BuildFQName(
+			namespace,
+			walSubsystem,
+			"segments",
+		),
+		"Number of WAL segments",
+		[]string{}, nil,
+	)
+	pgWALSize = prometheus.NewDesc(
+		prometheus.BuildFQName(
+			namespace,
+			walSubsystem,
+			"size_bytes",
+		),
+		"Total size of WAL segments",
+		[]string{}, nil,
+	)
+
+	pgWALQuery = `
+		SELECT
+			COUNT(*) AS segments,
+			SUM(size) AS size
+		FROM pg_ls_waldir()
+		WHERE name ~ '^[0-9A-F]{24}$'`
+)
+
+func (c PGWALCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error {
+	db := instance.getDB()
+	row := db.QueryRowContext(ctx,
+		pgWALQuery,
+	)
+
+	var segments uint64
+	var size uint64
+	err := row.Scan(&segments, &size)
+	if err != nil {
+		return err
+	}
+	ch <- prometheus.MustNewConstMetric(
+		pgWALSegments,
+		prometheus.GaugeValue, float64(segments),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		pgWALSize,
+		prometheus.GaugeValue, float64(size),
+	)
+	return nil
+}

collector/pg_wal_test.go

@@ -0,0 +1,63 @@
+// Copyright 2023 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package collector
+
+import (
+	"context"
+	"testing"
+
+	"github.com/DATA-DOG/go-sqlmock"
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestPgWALCollector(t *testing.T) {
+	db, mock, err := sqlmock.New()
+	if err != nil {
+		t.Fatalf("Error opening a stub db connection: %s", err)
+	}
+	defer db.Close()
+
+	inst := &instance{db: db}
+
+	columns := []string{"segments", "size"}
+	rows := sqlmock.NewRows(columns).
+		AddRow(47, 788529152)
+	mock.ExpectQuery(sanitizeQuery(pgWALQuery)).WillReturnRows(rows)
+
+	ch := make(chan prometheus.Metric)
+	go func() {
+		defer close(ch)
+		c := PGWALCollector{}
+
+		if err := c.Update(context.Background(), inst, ch); err != nil {
+			t.Errorf("Error calling PGWALCollector.Update: %s", err)
+		}
+	}()
+
+	expected := []MetricResult{
+		{labels: labelMap{}, value: 47, metricType: dto.MetricType_GAUGE},
+		{labels: labelMap{}, value: 788529152, metricType: dto.MetricType_GAUGE},
+	}
+
+	convey.Convey("Metrics comparison", t, func() {
+		for _, expect := range expected {
+			m := readMetric(<-ch)
+			convey.So(expect, convey.ShouldResemble, m)
+		}
+	})
+	if err := mock.ExpectationsWereMet(); err != nil {
+		t.Errorf("there were unfulfilled exceptions: %s", err)
+	}
+}

go.mod

@@ -8,11 +8,11 @@ require (
 	github.com/blang/semver/v4 v4.0.0
 	github.com/go-kit/log v0.2.1
 	github.com/lib/pq v1.10.9
-	github.com/prometheus/client_golang v1.15.1
+	github.com/prometheus/client_golang v1.16.0
 	github.com/prometheus/client_model v0.4.0
 	github.com/prometheus/common v0.44.0
 	github.com/prometheus/exporter-toolkit v0.10.0
-	github.com/smartystreets/goconvey v1.8.0
+	github.com/smartystreets/goconvey v1.8.1
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
@@ -32,14 +32,14 @@ require (
 	github.com/kr/text v0.2.0 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
-	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/prometheus/procfs v0.10.1 // indirect
 	github.com/rogpeppe/go-internal v1.10.0 // indirect
-	github.com/smartystreets/assertions v1.13.1 // indirect
+	github.com/smarty/assertions v1.15.0 // indirect
 	github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
 	golang.org/x/crypto v0.8.0 // indirect
 	golang.org/x/net v0.10.0 // indirect
 	golang.org/x/oauth2 v0.8.0 // indirect
-	golang.org/x/sync v0.1.0 // indirect
+	golang.org/x/sync v0.2.0 // indirect
 	golang.org/x/sys v0.8.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect

go.sum

@@ -49,23 +49,23 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI=
-github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk=
+github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
+github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
 github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY=
 github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
 github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
 github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
 github.com/prometheus/exporter-toolkit v0.10.0 h1:yOAzZTi4M22ZzVxD+fhy1URTuNRj/36uQJJ5S8IPza8=
 github.com/prometheus/exporter-toolkit v0.10.0/go.mod h1:+sVFzuvV5JDyw+Ih6p3zFxZNVnKQa3x5qPmDSiPu4ZY=
-github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
-github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
+github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
+github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
-github.com/smartystreets/assertions v1.13.1 h1:Ef7KhSmjZcK6AVf9YbJdvPYG9avaF0ZxudX+ThRdWfU=
-github.com/smartystreets/assertions v1.13.1/go.mod h1:cXr/IwVfSo/RbCSPhoAPv73p3hlSdrBH/b3SdnW/LMY=
-github.com/smartystreets/goconvey v1.8.0 h1:Oi49ha/2MURE0WexF052Z0m+BNSGirfjg5RL+JXWq3w=
-github.com/smartystreets/goconvey v1.8.0/go.mod h1:EdX8jtrTIj26jmjCOVNMVSIYAtgexqXKHOXW2Dx9JLg=
+github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY=
+github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec=
+github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY=
+github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
@@ -80,8 +80,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8=
 golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
-golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
+golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=