From d90b2d83d7c88a42e1616868e96bfb5d6f611bb5 Mon Sep 17 00:00:00 2001 From: Ben Kochie Date: Thu, 19 Aug 2021 12:19:26 +0200 Subject: [PATCH] Replace load average with PSI metric The load average metric is misleading as a representation of CPU saturation. Normal CPU utilization is a better real representation of saturation. On newer Linux, there is a new Pressure Stall Information[0] metric that better represents CPU over saturation. This is also useful as it can make single-core saturation more visible. [0]: https://www.kernel.org/doc/html/latest/accounting/psi.html Signed-off-by: Ben Kochie --- docs/node-mixin/rules/rules.libsonnet | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/docs/node-mixin/rules/rules.libsonnet b/docs/node-mixin/rules/rules.libsonnet index 5367ac3f..97358da5 100644 --- a/docs/node-mixin/rules/rules.libsonnet +++ b/docs/node-mixin/rules/rules.libsonnet @@ -16,7 +16,7 @@ ||| % $._config, }, { - // CPU utilisation is % CPU is not idle. + // CPU utilisation is % CPU is not idle. This represents CPU saturation. record: 'instance:node_cpu_utilisation:rate%(rateInterval)s' % $._config, expr: ||| 1 - avg without (cpu, mode) ( @@ -25,17 +25,14 @@ ||| % $._config, }, { - // This is CPU saturation: 1min avg run queue length / number of CPUs. - // Can go over 1. - // TODO: There are situation where a run queue >1/core is just normal and fine. - // We need to clarify how to read this metric and if its usage is helpful at all. - record: 'instance:node_load1_per_cpu:ratio', + // CPU pressure represents over-saturation. This is the amount of CPU seconds + // requested, but the kernel was not able to schedule. + // NOTE: This is only availalbe on Linux >= 4.19 and `CONFIG_PSI` is enabled. + // See also: + // - https://www.kernel.org/doc/html/latest/accounting/psi.html + // - https://facebookmicrosites.github.io/psi/docs/overview expr: ||| - ( - node_load1{%(nodeExporterSelector)s} - / - instance:node_num_cpu:sum{%(nodeExporterSelector)s} - ) + rate(node_pressure_cpu_waiting_seconds_total{%(nodeExporterSelector)s}[%(rateInterval)s]) ||| % $._config, }, {