From 187dbfc4ae175e561b1966539b03e1c8b7775574 Mon Sep 17 00:00:00 2001 From: Steffen Higel Date: Fri, 26 Aug 2022 15:40:14 -0700 Subject: [PATCH 1/2] Add cpu metrics based on newer and more accurate perflib sources This change adds 4 new CPU related metrics: * process_mperf_total * processor_rtc_total * processor_utility_total * processor_privileged_utility_total and renames the existing process_performance to processor_performance_total, since it was previously misunderstood and was unlikely to be have been useful without the above new metrics The data sources for these are not particularly well understood, and the examples show that in some cases, arbitrary scaling factors are required to actually make them useful, but in my testing on hundreds of systems with a broad range of CPUs and operating systems from 2012r2 through to 2019 has proved out that we can use them to accurately display actual CPU frequencies and CPU utilisation as it is represented in taskmgr. Things I don't particularly like and would like input on: * I would have preferred to do the scaling of processor_mperf_total in the code, but there isn't an elegant way of doing this right now. * Maybe processor_mperf_total should be called processor_mperformance_total. See #787 for discussion. Signed-off-by: Steffen Higel --- collector/cpu.go | 58 +++++++++++++++++++++++++++++++++++++-- collector/perflib.go | 19 +++++++++++++ collector/perflib_test.go | 11 +++++--- docs/collector.cpu.md | 33 +++++++++++++++++++++- 4 files changed, 114 insertions(+), 7 deletions(-) diff --git a/collector/cpu.go b/collector/cpu.go index 8c5c4c31..e3a43a0d 100644 --- a/collector/cpu.go +++ b/collector/cpu.go @@ -37,6 +37,10 @@ type cpuCollectorFull struct { ProcessorFrequencyMHz *prometheus.Desc ProcessorMaxFrequencyMHz *prometheus.Desc ProcessorPerformance *prometheus.Desc + ProcessorMPerf *prometheus.Desc + ProcessorRTC *prometheus.Desc + ProcessorUtility *prometheus.Desc + ProcessorPrivUtility *prometheus.Desc } // newCPUCollector constructs a new cpuCollector, appropriate for the running OS @@ -129,11 +133,35 @@ func newCPUCollector() (Collector, error) { nil, ), ProcessorPerformance: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "processor_performance"), + prometheus.BuildFQName(Namespace, subsystem, "processor_performance_total"), "Processor Performance is the average performance of the processor while it is executing instructions, as a percentage of the nominal performance of the processor. On some processors, Processor Performance may exceed 100%", []string{"core"}, nil, ), + ProcessorMPerf: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "processor_mperf_total"), + "Processor MPerf is the number of TSC ticks incremented while executing instructions", + []string{"core"}, + nil, + ), + ProcessorRTC: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "processor_rtc_total"), + "Processor RTC represents the number of RTC ticks made since the system booted. It should consistently be 64e6, and can be used to properly derive Processor Utility Rate", + []string{"core"}, + nil, + ), + ProcessorUtility: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "processor_utility_total"), + "Processor Utility represents is the amount of time the core spends executing instructions", + []string{"core"}, + nil, + ), + ProcessorPrivUtility: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "processor_privileged_utility_total"), + "Processor Privilieged Utility represents is the amount of time the core has spent executing instructions inside the kernel", + []string{"core"}, + nil, + ), }, nil } @@ -258,8 +286,10 @@ type perflibProcessorInformation struct { PrivilegedUtilitySeconds float64 `perflib:"% Privileged Utility"` ProcessorFrequencyMHz float64 `perflib:"Processor Frequency"` ProcessorPerformance float64 `perflib:"% Processor Performance"` + ProcessorMPerf float64 `perflib:"% Processor Performance,secondvalue"` ProcessorTimeSeconds float64 `perflib:"% Processor Time"` ProcessorUtilityRate float64 `perflib:"% Processor Utility"` + ProcessorRTC float64 `perflib:"% Processor Utility,secondvalue"` UserTimeSeconds float64 `perflib:"% User Time"` } @@ -366,10 +396,34 @@ func (c *cpuCollectorFull) Collect(ctx *ScrapeContext, ch chan<- prometheus.Metr ) ch <- prometheus.MustNewConstMetric( c.ProcessorPerformance, - prometheus.GaugeValue, + prometheus.CounterValue, cpu.ProcessorPerformance, core, ) + ch <- prometheus.MustNewConstMetric( + c.ProcessorMPerf, + prometheus.CounterValue, + cpu.ProcessorMPerf, + core, + ) + ch <- prometheus.MustNewConstMetric( + c.ProcessorRTC, + prometheus.CounterValue, + cpu.ProcessorRTC, + core, + ) + ch <- prometheus.MustNewConstMetric( + c.ProcessorUtility, + prometheus.CounterValue, + cpu.ProcessorUtilityRate, + core, + ) + ch <- prometheus.MustNewConstMetric( + c.ProcessorPrivUtility, + prometheus.CounterValue, + cpu.PrivilegedUtilitySeconds, + core, + ) } return nil diff --git a/collector/perflib.go b/collector/perflib.go index 153429e6..0d8ca1ad 100644 --- a/collector/perflib.go +++ b/collector/perflib.go @@ -4,6 +4,7 @@ import ( "fmt" "reflect" "strconv" + "strings" perflibCollector "github.com/leoluk/perflib_exporter/collector" "github.com/leoluk/perflib_exporter/perflib" @@ -67,6 +68,16 @@ func unmarshalObject(obj *perflib.PerfObject, vs interface{}) error { if tag == "" { continue } + secondValue := false + + st := strings.Split(tag, ",") + tag = st[0] + + for _, t := range st { + if t == "secondvalue" { + secondValue = true + } + } ctr, found := counters[tag] if !found { @@ -80,6 +91,14 @@ func unmarshalObject(obj *perflib.PerfObject, vs interface{}) error { return fmt.Errorf("tagged field %v has wrong type %v, must be float64", f.Name, fieldType) } + if secondValue { + if !ctr.Def.HasSecondValue { + return fmt.Errorf("tagged field %v expected a SecondValue, which was not present", f.Name) + } + target.Field(i).SetFloat(float64(ctr.SecondValue)) + continue + } + switch ctr.Def.CounterType { case perflibCollector.PERF_ELAPSED_TIME: target.Field(i).SetFloat(float64(ctr.Value-windowsEpoch) / float64(obj.Frequency)) diff --git a/collector/perflib_test.go b/collector/perflib_test.go index 98f0693b..101404e6 100644 --- a/collector/perflib_test.go +++ b/collector/perflib_test.go @@ -11,6 +11,7 @@ import ( type simple struct { ValA float64 `perflib:"Something"` ValB float64 `perflib:"Something Else"` + ValC float64 `perflib:"Something Else,secondvalue"` } func TestUnmarshalPerflib(t *testing.T) { @@ -62,16 +63,18 @@ func TestUnmarshalPerflib(t *testing.T) { }, { Def: &perflib.PerfCounterDef{ - Name: "Something Else", - CounterType: perflibCollector.PERF_COUNTER_COUNTER, + Name: "Something Else", + CounterType: perflibCollector.PERF_COUNTER_COUNTER, + HasSecondValue: true, }, - Value: 256, + Value: 256, + SecondValue: 222, }, }, }, }, }, - expectedOutput: []simple{{ValA: 123, ValB: 256}}, + expectedOutput: []simple{{ValA: 123, ValB: 256, ValC: 222}}, expectError: false, }, { diff --git a/docs/collector.cpu.md b/docs/collector.cpu.md index ca30fb2a..3a18af79 100644 --- a/docs/collector.cpu.md +++ b/docs/collector.cpu.md @@ -31,7 +31,11 @@ Name | Description | Type | Labels `windows_cpu_idle_break_events_total` | Total number of time processor was woken from idle | counter | `core` `windows_cpu_parking_status` | Parking Status represents whether a processor is parked or not | gauge | `core` `windows_cpu_core_frequency_mhz` | Core frequency in megahertz | gauge | `core` -`windows_cpu_processor_performance` | Processor Performance is the average performance of the processor while it is executing instructions, as a percentage of the nominal performance of the processor. On some processors, Processor Performance may exceed 100% | gauge | `core` +`windows_cpu_processor_performance_total` | Processor Performance is the number of CPU cycles executing instructions by each core; it is believed to be similar to the value that the APERF MSR would show, were it exposed | counter | `core` +`windows_cpu_processor_mperf_total` | Processor MPerf Total is proportioanl to the number of TSC ticks each core has accumulated while executing instructions. Due to the manner in which it is presented, it should be scaled by 1e2 to properly line up with Processor Performance Total. As above, it is believed to be closely related to the MPERF MSR. | counter | `core` +`windows_cpu_processor_rtc_total` | RTC total is assumed to represent the 64Hz tick rate in Windows. It is not by itself useful, but can be used with `windows_cpu_processor_utility_total` to more accurately measure CPU utilisation than with `windows_cpu_time_total` | counter | `core` +`windows_cpu_processor_utility_total` | Processor Utility Total is a newer, more accurate measure of CPU utilization, in particular handling modern CPUs with variant CPU frequencies. The rate of this counter divided by the rate of `windows_cpu_processor_rtc_total` should provide an accurate view of CPU utilisation on modern systems, as observed in Task Manager. | counter | `core` +`windows_cpu_processor_privileged_utility_total` | Processor Privilged Utility Total, when used in a similar fashion to `windows_cpu_processor_utility_total` will show the portion of CPU utilization which is happening in privileged mode. | counter | `core` ### Example metric Show frequency of host CPU cores @@ -44,6 +48,19 @@ Show cpu usage by mode. ``` sum by (mode) (irate(windows_cpu_time_total{instance="localhost"}[5m])) ``` +Show per-cpu utilisation using the processor utility metrics +``` +rate(windows_cpu_processor_utility_total{instance="localhost"}[5m]) / rate(windows_cpu_processor_rtc_total{instance="localhost"}[5m]) +``` +Show actual average CPU frequency in Hz +``` +avg by(instance) ( + 1e4 * windows_cpu_core_frequency_mhz{} + * rate(windows_cpu_processor_performance_total{}[5m]) + / rate(windows_cpu_processor_mperf_total{}[5m]) +) +``` + ## Alerting examples **prometheus.rules** @@ -57,4 +74,18 @@ sum by (mode) (irate(windows_cpu_time_total{instance="localhost"}[5m])) annotations: summary: "CPU Usage (instance {{ $labels.instance }})" description: "CPU Usage is more than 80%\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" +# Alert on hosts which are not boosting their CPU frequencies +- alert: NoCpuTurbo + expr: | + avg by(instance) ( + 1e4 * windows_cpu_core_frequency_mhz{} + * rate(windows_cpu_processor_performance_total{}[5m]) + / rate(windows_cpu_processor_mperf_total{}[5m]) + ) + / + (1e6 * avg by (instance) (windows_cpu_core_frequency_mhz)) + < 1.1 + for: 1h + annotations: + summary: "CPU Frequency on {{ $labels.instance }} is less than 110% of base frequency, suggesting it is not able to boost. ``` From f32342400fda4318d0154e242220c9bf0ac5e460 Mon Sep 17 00:00:00 2001 From: Steffen Higel Date: Thu, 8 Dec 2022 14:08:32 -0800 Subject: [PATCH 2/2] Fixing up e2e tests for new CPU perf metrics Signed-off-by: Steffen Higel --- tools/e2e-output.txt | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tools/e2e-output.txt b/tools/e2e-output.txt index 2197ca5b..24ef3d84 100644 --- a/tools/e2e-output.txt +++ b/tools/e2e-output.txt @@ -81,10 +81,18 @@ test_alpha_total 42 # TYPE windows_cpu_interrupts_total counter # HELP windows_cpu_parking_status Parking Status represents whether a processor is parked or not # TYPE windows_cpu_parking_status gauge -# HELP windows_cpu_processor_performance Processor Performance is the average performance of the processor while it is executing instructions, as a percentage of the nominal performance of the processor. On some processors, Processor Performance may exceed 100% -# TYPE windows_cpu_processor_performance gauge +# HELP windows_cpu_processor_performance_total Processor Performance is the average performance of the processor while it is executing instructions, as a percentage of the nominal performance of the processor. On some processors, Processor Performance may exceed 100% +# TYPE windows_cpu_processor_performance_total counter # HELP windows_cpu_time_total Time that processor spent in different modes (dpc, idle, interrupt, privileged, user) # TYPE windows_cpu_time_total counter +# HELP windows_cpu_processor_mperf_total Processor MPerf is the number of TSC ticks incremented while executing instructions +# TYPE windows_cpu_processor_mperf_total counter +# HELP windows_cpu_processor_privileged_utility_total Processor Privilieged Utility represents is the amount of time the core has spent executing instructions inside the kernel +# TYPE windows_cpu_processor_privileged_utility_total counter +# HELP windows_cpu_processor_rtc_total Processor RTC represents the number of RTC ticks made since the system booted. It should consistently be 64e6, and can be used to properly derive Processor Utility Rate +# TYPE windows_cpu_processor_rtc_total counter +# HELP windows_cpu_processor_utility_total Processor Utility represents is the amount of time the core spends executing instructions +# TYPE windows_cpu_processor_utility_total counter # HELP windows_cs_hostname Labeled system hostname information as provided by ComputerSystem.DNSHostName and ComputerSystem.Domain # TYPE windows_cs_hostname gauge # HELP windows_cs_logical_processors ComputerSystem.NumberOfLogicalProcessors