Add cpu metrics based on newer and more accurate perflib sources

This change adds 4 new CPU related metrics:

 * process_mperf_total
 * processor_rtc_total
 * processor_utility_total
 * processor_privileged_utility_total

and renames the existing process_performance to
processor_performance_total, since it was previously misunderstood and
was unlikely to be have been useful without the above new metrics

The data sources for these are not particularly well understood, and the
examples show that in some cases, arbitrary scaling factors are required
to actually make them useful, but in my testing on hundreds of systems
with a broad range of CPUs and operating systems from 2012r2 through to
2019 has proved out that we can use them to accurately display actual
CPU frequencies and CPU utilisation as it is represented in taskmgr.

Things I don't particularly like and would like input on:

 * I would have preferred to do the scaling of processor_mperf_total in
the code, but there isn't an elegant way of doing this right now.
 * Maybe processor_mperf_total should be called
processor_mperformance_total.

See #787 for discussion.

Signed-off-by: Steffen Higel <higels@valvesoftware.com>

collector/cpu.go

@@ -37,6 +37,10 @@ type cpuCollectorFull struct {
 	ProcessorFrequencyMHz    *prometheus.Desc
 	ProcessorMaxFrequencyMHz *prometheus.Desc
 	ProcessorPerformance     *prometheus.Desc
+	ProcessorMPerf           *prometheus.Desc
+	ProcessorRTC             *prometheus.Desc
+	ProcessorUtility         *prometheus.Desc
+	ProcessorPrivUtility     *prometheus.Desc
 }
 
 // newCPUCollector constructs a new cpuCollector, appropriate for the running OS
@@ -129,11 +133,35 @@ func newCPUCollector() (Collector, error) {
 			nil,
 		),
 		ProcessorPerformance: prometheus.NewDesc(
-			prometheus.BuildFQName(Namespace, subsystem, "processor_performance"),
+			prometheus.BuildFQName(Namespace, subsystem, "processor_performance_total"),
 			"Processor Performance is the average performance of the processor while it is executing instructions, as a percentage of the nominal performance of the processor. On some processors, Processor Performance may exceed 100%",
 			[]string{"core"},
 			nil,
 		),
+		ProcessorMPerf: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "processor_mperf_total"),
+			"Processor MPerf is the number of TSC ticks incremented while executing instructions",
+			[]string{"core"},
+			nil,
+		),
+		ProcessorRTC: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "processor_rtc_total"),
+			"Processor RTC represents the number of RTC ticks made since the system booted. It should consistently be 64e6, and can be used to properly derive Processor Utility Rate",
+			[]string{"core"},
+			nil,
+		),
+		ProcessorUtility: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "processor_utility_total"),
+			"Processor Utility represents is the amount of time the core spends executing instructions",
+			[]string{"core"},
+			nil,
+		),
+		ProcessorPrivUtility: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "processor_privileged_utility_total"),
+			"Processor Privilieged Utility represents is the amount of time the core has spent executing instructions inside the kernel",
+			[]string{"core"},
+			nil,
+		),
 	}, nil
 }
 
@@ -258,8 +286,10 @@ type perflibProcessorInformation struct {
 	PrivilegedUtilitySeconds float64 `perflib:"% Privileged Utility"`
 	ProcessorFrequencyMHz    float64 `perflib:"Processor Frequency"`
 	ProcessorPerformance     float64 `perflib:"% Processor Performance"`
+	ProcessorMPerf           float64 `perflib:"% Processor Performance,secondvalue"`
 	ProcessorTimeSeconds     float64 `perflib:"% Processor Time"`
 	ProcessorUtilityRate     float64 `perflib:"% Processor Utility"`
+	ProcessorRTC             float64 `perflib:"% Processor Utility,secondvalue"`
 	UserTimeSeconds          float64 `perflib:"% User Time"`
 }
 
@@ -366,10 +396,34 @@ func (c *cpuCollectorFull) Collect(ctx *ScrapeContext, ch chan<- prometheus.Metr
 		)
 		ch <- prometheus.MustNewConstMetric(
 			c.ProcessorPerformance,
-			prometheus.GaugeValue,
+			prometheus.CounterValue,
 			cpu.ProcessorPerformance,
 			core,
 		)
+		ch <- prometheus.MustNewConstMetric(
+			c.ProcessorMPerf,
+			prometheus.CounterValue,
+			cpu.ProcessorMPerf,
+			core,
+		)
+		ch <- prometheus.MustNewConstMetric(
+			c.ProcessorRTC,
+			prometheus.CounterValue,
+			cpu.ProcessorRTC,
+			core,
+		)
+		ch <- prometheus.MustNewConstMetric(
+			c.ProcessorUtility,
+			prometheus.CounterValue,
+			cpu.ProcessorUtilityRate,
+			core,
+		)
+		ch <- prometheus.MustNewConstMetric(
+			c.ProcessorPrivUtility,
+			prometheus.CounterValue,
+			cpu.PrivilegedUtilitySeconds,
+			core,
+		)
 	}
 
 	return nil

collector/perflib.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"reflect"
 	"strconv"
+	"strings"
 
 	perflibCollector "github.com/leoluk/perflib_exporter/collector"
 	"github.com/leoluk/perflib_exporter/perflib"
@@ -67,6 +68,16 @@ func unmarshalObject(obj *perflib.PerfObject, vs interface{}) error {
 			if tag == "" {
 				continue
 			}
+			secondValue := false
+
+			st := strings.Split(tag, ",")
+			tag = st[0]
+
+			for _, t := range st {
+				if t == "secondvalue" {
+					secondValue = true
+				}
+			}
 
 			ctr, found := counters[tag]
 			if !found {
@@ -80,6 +91,14 @@ func unmarshalObject(obj *perflib.PerfObject, vs interface{}) error {
 				return fmt.Errorf("tagged field %v has wrong type %v, must be float64", f.Name, fieldType)
 			}
 
+			if secondValue {
+				if !ctr.Def.HasSecondValue {
+					return fmt.Errorf("tagged field %v expected a SecondValue, which was not present", f.Name)
+				}
+				target.Field(i).SetFloat(float64(ctr.SecondValue))
+				continue
+			}
+
 			switch ctr.Def.CounterType {
 			case perflibCollector.PERF_ELAPSED_TIME:
 				target.Field(i).SetFloat(float64(ctr.Value-windowsEpoch) / float64(obj.Frequency))

collector/perflib_test.go

@@ -11,6 +11,7 @@ import (
 type simple struct {
 	ValA float64 `perflib:"Something"`
 	ValB float64 `perflib:"Something Else"`
+	ValC float64 `perflib:"Something Else,secondvalue"`
 }
 
 func TestUnmarshalPerflib(t *testing.T) {
@@ -62,16 +63,18 @@ func TestUnmarshalPerflib(t *testing.T) {
 							},
 							{
 								Def: &perflib.PerfCounterDef{
-									Name:        "Something Else",
-									CounterType: perflibCollector.PERF_COUNTER_COUNTER,
+									Name:           "Something Else",
+									CounterType:    perflibCollector.PERF_COUNTER_COUNTER,
+									HasSecondValue: true,
 								},
-								Value: 256,
+								Value:       256,
+								SecondValue: 222,
 							},
 						},
 					},
 				},
 			},
-			expectedOutput: []simple{{ValA: 123, ValB: 256}},
+			expectedOutput: []simple{{ValA: 123, ValB: 256, ValC: 222}},
 			expectError:    false,
 		},
 		{

docs/collector.cpu.md

@@ -31,7 +31,11 @@ Name | Description | Type | Labels
 `windows_cpu_idle_break_events_total` | Total number of time processor was woken from idle | counter | `core`
 `windows_cpu_parking_status` | Parking Status represents whether a processor is parked or not | gauge | `core`
 `windows_cpu_core_frequency_mhz` | Core frequency in megahertz | gauge | `core`
-`windows_cpu_processor_performance` | Processor Performance is the average performance of the processor while it is executing instructions, as a percentage of the nominal performance of the processor. On some processors, Processor Performance may exceed 100% | gauge | `core`
+`windows_cpu_processor_performance_total` | Processor Performance is the number of CPU cycles executing instructions by each core; it is believed to be similar to the value that the APERF MSR would show, were it exposed | counter | `core`
+`windows_cpu_processor_mperf_total` | Processor MPerf Total is proportioanl to the number of TSC ticks each core has accumulated while executing instructions. Due to the manner in which it is presented, it should be scaled by 1e2 to properly line up with Processor Performance Total. As above, it is believed to be closely related to the MPERF MSR. | counter | `core`
+`windows_cpu_processor_rtc_total` | RTC total is assumed to represent the 64Hz tick rate in Windows. It is not by itself useful, but can be used with `windows_cpu_processor_utility_total` to more accurately measure CPU utilisation than with `windows_cpu_time_total` | counter | `core`
+`windows_cpu_processor_utility_total` | Processor Utility Total is a newer, more accurate measure of CPU utilization, in particular handling modern CPUs with variant CPU frequencies. The rate of this counter divided by the rate of `windows_cpu_processor_rtc_total` should provide an accurate view of CPU utilisation on modern systems, as observed in Task Manager. | counter | `core`
+`windows_cpu_processor_privileged_utility_total` | Processor Privilged Utility Total, when used in a similar fashion to `windows_cpu_processor_utility_total` will show the portion of CPU utilization which is happening in privileged mode. | counter | `core`
 
 ### Example metric
 Show frequency of host CPU cores
@@ -44,6 +48,19 @@ Show cpu usage by mode.
 ```
 sum by (mode) (irate(windows_cpu_time_total{instance="localhost"}[5m]))
 ```
+Show per-cpu utilisation using the processor utility metrics
+```
+rate(windows_cpu_processor_utility_total{instance="localhost"}[5m]) / rate(windows_cpu_processor_rtc_total{instance="localhost"}[5m])
+```
+Show actual average CPU frequency in Hz
+```
+avg by(instance) (
+    1e4 * windows_cpu_core_frequency_mhz{}
+    * rate(windows_cpu_processor_performance_total{}[5m])
+    / rate(windows_cpu_processor_mperf_total{}[5m])
+)
+```
+
 
 ## Alerting examples
 **prometheus.rules**
@@ -57,4 +74,18 @@ sum by (mode) (irate(windows_cpu_time_total{instance="localhost"}[5m]))
   annotations:
     summary: "CPU Usage (instance {{ $labels.instance }})"
     description: "CPU Usage is more than 80%\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
+# Alert on hosts which are not boosting their CPU frequencies
+- alert: NoCpuTurbo
+  expr: |
+    avg by(instance) (
+        1e4 * windows_cpu_core_frequency_mhz{}
+        * rate(windows_cpu_processor_performance_total{}[5m])
+        / rate(windows_cpu_processor_mperf_total{}[5m])
+    )
+    /
+    (1e6 * avg by (instance) (windows_cpu_core_frequency_mhz))
+    < 1.1
+  for: 1h
+  annotations:
+    summary: "CPU Frequency on {{ $labels.instance }} is less than 110% of base frequency, suggesting it is not able to boost.
 ```