Update diskstats for linux kernel 4.19 (#1109)

The format of /proc/diskstats is changing in linux-4.19 to include some
additional fields.  See: https://www.kernel.org/doc/Documentation/iostats.txt

* collector/diskstats: use constants for some hard coded strings
* collector/diskstats: update diskstats for linux-4.19
* collector/diskstats: remove kernel doc url from individual metrics

Signed-off-by: Paul Gier <pgier@redhat.com>
This commit is contained in:
Paul Gier 2018-10-15 10:24:28 -05:00 committed by Ben Kochie
parent 0880d460d7
commit e8d8199072
5 changed files with 107 additions and 24 deletions

View File

@ -30,8 +30,9 @@ import (
)
const (
diskSubsystem = "disk"
diskSectorSize = 512
diskSubsystem = "disk"
diskSectorSize = 512
diskstatsFilename = "diskstats"
)
var (
@ -61,12 +62,12 @@ func init() {
}
// NewDiskstatsCollector returns a new Collector exposing disk device stats.
// Docs from https://www.kernel.org/doc/Documentation/iostats.txt
func NewDiskstatsCollector() (Collector, error) {
var diskLabelNames = []string{"device"}
return &diskstatsCollector{
ignoredDevicesPattern: regexp.MustCompile(*ignoredDevices),
// Docs from https://www.kernel.org/doc/Documentation/iostats.txt
descs: []typedFactorDesc{
{
desc: prometheus.NewDesc(
@ -79,7 +80,7 @@ func NewDiskstatsCollector() (Collector, error) {
{
desc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, diskSubsystem, "reads_merged_total"),
"The total number of reads merged. See https://www.kernel.org/doc/Documentation/iostats.txt.",
"The total number of reads merged.",
diskLabelNames,
nil,
), valueType: prometheus.CounterValue,
@ -113,7 +114,7 @@ func NewDiskstatsCollector() (Collector, error) {
{
desc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, diskSubsystem, "writes_merged_total"),
"The number of writes merged. See https://www.kernel.org/doc/Documentation/iostats.txt.",
"The number of writes merged.",
diskLabelNames,
nil,
), valueType: prometheus.CounterValue,
@ -156,7 +157,40 @@ func NewDiskstatsCollector() (Collector, error) {
{
desc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, diskSubsystem, "io_time_weighted_seconds_total"),
"The weighted # of seconds spent doing I/Os. See https://www.kernel.org/doc/Documentation/iostats.txt.",
"The weighted # of seconds spent doing I/Os.",
diskLabelNames,
nil,
), valueType: prometheus.CounterValue,
factor: .001,
},
{
desc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, diskSubsystem, "discards_completed_total"),
"The total number of discards completed successfully.",
diskLabelNames,
nil,
), valueType: prometheus.CounterValue,
},
{
desc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, diskSubsystem, "discards_merged_total"),
"The total number of discards merged.",
diskLabelNames,
nil,
), valueType: prometheus.CounterValue,
},
{
desc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, diskSubsystem, "discarded_sectors_total"),
"The total number of sectors discard successfully.",
diskLabelNames,
nil,
), valueType: prometheus.CounterValue,
},
{
desc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, diskSubsystem, "discard_time_seconds_total"),
"This is the total number of seconds spent by all discards.",
diskLabelNames,
nil,
), valueType: prometheus.CounterValue,
@ -167,7 +201,6 @@ func NewDiskstatsCollector() (Collector, error) {
}
func (c *diskstatsCollector) Update(ch chan<- prometheus.Metric) error {
procDiskStats := procFilePath("diskstats")
diskStats, err := getDiskStats()
if err != nil {
return fmt.Errorf("couldn't get diskstats: %s", err)
@ -179,8 +212,8 @@ func (c *diskstatsCollector) Update(ch chan<- prometheus.Metric) error {
continue
}
if len(stats) != len(c.descs) {
return fmt.Errorf("invalid line for %s for %s", procDiskStats, dev)
if len(stats) > len(c.descs) {
return fmt.Errorf("invalid line for %s for %s", procFilePath(diskstatsFilename), dev)
}
for i, value := range stats {
@ -194,8 +227,8 @@ func (c *diskstatsCollector) Update(ch chan<- prometheus.Metric) error {
return nil
}
func getDiskStats() (map[string]map[int]string, error) {
file, err := os.Open(procFilePath("diskstats"))
func getDiskStats() (map[string][]string, error) {
file, err := os.Open(procFilePath(diskstatsFilename))
if err != nil {
return nil, err
}
@ -204,22 +237,19 @@ func getDiskStats() (map[string]map[int]string, error) {
return parseDiskStats(file)
}
func parseDiskStats(r io.Reader) (map[string]map[int]string, error) {
func parseDiskStats(r io.Reader) (map[string][]string, error) {
var (
diskStats = map[string]map[int]string{}
diskStats = map[string][]string{}
scanner = bufio.NewScanner(r)
)
for scanner.Scan() {
parts := strings.Fields(scanner.Text())
if len(parts) < 4 { // we strip major, minor and dev
return nil, fmt.Errorf("invalid line in %s: %s", procFilePath("diskstats"), scanner.Text())
return nil, fmt.Errorf("invalid line in %s: %s", procFilePath(diskstatsFilename), scanner.Text())
}
dev := parts[2]
diskStats[dev] = map[int]string{}
for i, v := range parts[3:] {
diskStats[dev][i] = v
}
diskStats[dev] = parts[3:]
}
return diskStats, scanner.Err()

View File

@ -37,4 +37,8 @@ func TestDiskStats(t *testing.T) {
if want, got := "68", diskStats["mmcblk0p2"][10]; want != got {
t.Errorf("want diskstats mmcblk0p2 %s, got %s", want, got)
}
if want, got := "11130", diskStats["sdb"][14]; want != got {
t.Errorf("want diskstats sdb %s, got %s", want, got)
}
}

View File

@ -290,6 +290,18 @@ node_cpu_seconds_total{cpu="7",mode="softirq"} 0.31
node_cpu_seconds_total{cpu="7",mode="steal"} 0
node_cpu_seconds_total{cpu="7",mode="system"} 101.64
node_cpu_seconds_total{cpu="7",mode="user"} 290.98
# HELP node_disk_discard_time_seconds_total This is the total number of seconds spent by all discards.
# TYPE node_disk_discard_time_seconds_total counter
node_disk_discard_time_seconds_total{device="sdb"} 11.13
# HELP node_disk_discarded_sectors_total The total number of sectors discard successfully.
# TYPE node_disk_discarded_sectors_total counter
node_disk_discarded_sectors_total{device="sdb"} 1.925173784e+09
# HELP node_disk_discards_completed_total The total number of discards completed successfully.
# TYPE node_disk_discards_completed_total counter
node_disk_discards_completed_total{device="sdb"} 68851
# HELP node_disk_discards_merged_total The total number of discards merged.
# TYPE node_disk_discards_merged_total counter
node_disk_discards_merged_total{device="sdb"} 0
# HELP node_disk_io_now The number of I/Os currently in progress.
# TYPE node_disk_io_now gauge
node_disk_io_now{device="dm-0"} 0
@ -303,6 +315,7 @@ node_disk_io_now{device="mmcblk0p1"} 0
node_disk_io_now{device="mmcblk0p2"} 0
node_disk_io_now{device="nvme0n1"} 0
node_disk_io_now{device="sda"} 0
node_disk_io_now{device="sdb"} 0
node_disk_io_now{device="sr0"} 0
node_disk_io_now{device="vda"} 0
# HELP node_disk_io_time_seconds_total Total seconds spent doing I/Os.
@ -318,9 +331,10 @@ node_disk_io_time_seconds_total{device="mmcblk0p1"} 0.024
node_disk_io_time_seconds_total{device="mmcblk0p2"} 0.068
node_disk_io_time_seconds_total{device="nvme0n1"} 222.766
node_disk_io_time_seconds_total{device="sda"} 9653.880000000001
node_disk_io_time_seconds_total{device="sdb"} 60.730000000000004
node_disk_io_time_seconds_total{device="sr0"} 0
node_disk_io_time_seconds_total{device="vda"} 41614.592000000004
# HELP node_disk_io_time_weighted_seconds_total The weighted # of seconds spent doing I/Os. See https://www.kernel.org/doc/Documentation/iostats.txt.
# HELP node_disk_io_time_weighted_seconds_total The weighted # of seconds spent doing I/Os.
# TYPE node_disk_io_time_weighted_seconds_total counter
node_disk_io_time_weighted_seconds_total{device="dm-0"} 1.206301256e+06
node_disk_io_time_weighted_seconds_total{device="dm-1"} 0.084
@ -333,6 +347,7 @@ node_disk_io_time_weighted_seconds_total{device="mmcblk0p1"} 0.024
node_disk_io_time_weighted_seconds_total{device="mmcblk0p2"} 0.068
node_disk_io_time_weighted_seconds_total{device="nvme0n1"} 1032.546
node_disk_io_time_weighted_seconds_total{device="sda"} 82621.804
node_disk_io_time_weighted_seconds_total{device="sdb"} 67.07000000000001
node_disk_io_time_weighted_seconds_total{device="sr0"} 0
node_disk_io_time_weighted_seconds_total{device="vda"} 2.0778722280000001e+06
# HELP node_disk_read_bytes_total The total number of bytes read successfully.
@ -348,6 +363,7 @@ node_disk_read_bytes_total{device="mmcblk0p1"} 81920
node_disk_read_bytes_total{device="mmcblk0p2"} 389120
node_disk_read_bytes_total{device="nvme0n1"} 2.377714176e+09
node_disk_read_bytes_total{device="sda"} 5.13713216512e+11
node_disk_read_bytes_total{device="sdb"} 4.944782848e+09
node_disk_read_bytes_total{device="sr0"} 0
node_disk_read_bytes_total{device="vda"} 1.6727491584e+10
# HELP node_disk_read_time_seconds_total The total number of seconds spent by all reads.
@ -363,6 +379,7 @@ node_disk_read_time_seconds_total{device="mmcblk0p1"} 0.024
node_disk_read_time_seconds_total{device="mmcblk0p2"} 0.068
node_disk_read_time_seconds_total{device="nvme0n1"} 21.650000000000002
node_disk_read_time_seconds_total{device="sda"} 18492.372
node_disk_read_time_seconds_total{device="sdb"} 0.084
node_disk_read_time_seconds_total{device="sr0"} 0
node_disk_read_time_seconds_total{device="vda"} 8655.768
# HELP node_disk_reads_completed_total The total number of reads completed successfully.
@ -378,9 +395,10 @@ node_disk_reads_completed_total{device="mmcblk0p1"} 17
node_disk_reads_completed_total{device="mmcblk0p2"} 95
node_disk_reads_completed_total{device="nvme0n1"} 47114
node_disk_reads_completed_total{device="sda"} 2.5354637e+07
node_disk_reads_completed_total{device="sdb"} 326552
node_disk_reads_completed_total{device="sr0"} 0
node_disk_reads_completed_total{device="vda"} 1.775784e+06
# HELP node_disk_reads_merged_total The total number of reads merged. See https://www.kernel.org/doc/Documentation/iostats.txt.
# HELP node_disk_reads_merged_total The total number of reads merged.
# TYPE node_disk_reads_merged_total counter
node_disk_reads_merged_total{device="dm-0"} 0
node_disk_reads_merged_total{device="dm-1"} 0
@ -393,6 +411,7 @@ node_disk_reads_merged_total{device="mmcblk0p1"} 3
node_disk_reads_merged_total{device="mmcblk0p2"} 0
node_disk_reads_merged_total{device="nvme0n1"} 4
node_disk_reads_merged_total{device="sda"} 3.4367663e+07
node_disk_reads_merged_total{device="sdb"} 841
node_disk_reads_merged_total{device="sr0"} 0
node_disk_reads_merged_total{device="vda"} 15386
# HELP node_disk_write_time_seconds_total This is the total number of seconds spent by all writes.
@ -408,6 +427,7 @@ node_disk_write_time_seconds_total{device="mmcblk0p1"} 0
node_disk_write_time_seconds_total{device="mmcblk0p2"} 0
node_disk_write_time_seconds_total{device="nvme0n1"} 1011.053
node_disk_write_time_seconds_total{device="sda"} 63877.96
node_disk_write_time_seconds_total{device="sdb"} 5.007
node_disk_write_time_seconds_total{device="sr0"} 0
node_disk_write_time_seconds_total{device="vda"} 2.069221364e+06
# HELP node_disk_writes_completed_total The total number of writes completed successfully.
@ -423,9 +443,10 @@ node_disk_writes_completed_total{device="mmcblk0p1"} 0
node_disk_writes_completed_total{device="mmcblk0p2"} 0
node_disk_writes_completed_total{device="nvme0n1"} 1.07832e+06
node_disk_writes_completed_total{device="sda"} 2.8444756e+07
node_disk_writes_completed_total{device="sdb"} 41822
node_disk_writes_completed_total{device="sr0"} 0
node_disk_writes_completed_total{device="vda"} 6.038856e+06
# HELP node_disk_writes_merged_total The number of writes merged. See https://www.kernel.org/doc/Documentation/iostats.txt.
# HELP node_disk_writes_merged_total The number of writes merged.
# TYPE node_disk_writes_merged_total counter
node_disk_writes_merged_total{device="dm-0"} 0
node_disk_writes_merged_total{device="dm-1"} 0
@ -438,6 +459,7 @@ node_disk_writes_merged_total{device="mmcblk0p1"} 0
node_disk_writes_merged_total{device="mmcblk0p2"} 0
node_disk_writes_merged_total{device="nvme0n1"} 43950
node_disk_writes_merged_total{device="sda"} 1.1134226e+07
node_disk_writes_merged_total{device="sdb"} 2895
node_disk_writes_merged_total{device="sr0"} 0
node_disk_writes_merged_total{device="vda"} 2.0711856e+07
# HELP node_disk_written_bytes_total The total number of bytes written successfully.
@ -453,6 +475,7 @@ node_disk_written_bytes_total{device="mmcblk0p1"} 0
node_disk_written_bytes_total{device="mmcblk0p2"} 0
node_disk_written_bytes_total{device="nvme0n1"} 2.0199236096e+10
node_disk_written_bytes_total{device="sda"} 2.58916880384e+11
node_disk_written_bytes_total{device="sdb"} 1.01012736e+09
node_disk_written_bytes_total{device="sr0"} 0
node_disk_written_bytes_total{device="vda"} 1.0938236928e+11
# HELP node_drbd_activitylog_writes_total Number of updates of the activity log area of the meta data.

View File

@ -290,6 +290,18 @@ node_cpu_seconds_total{cpu="7",mode="softirq"} 0.31
node_cpu_seconds_total{cpu="7",mode="steal"} 0
node_cpu_seconds_total{cpu="7",mode="system"} 101.64
node_cpu_seconds_total{cpu="7",mode="user"} 290.98
# HELP node_disk_discard_time_seconds_total This is the total number of seconds spent by all discards.
# TYPE node_disk_discard_time_seconds_total counter
node_disk_discard_time_seconds_total{device="sdb"} 11.13
# HELP node_disk_discarded_sectors_total The total number of sectors discard successfully.
# TYPE node_disk_discarded_sectors_total counter
node_disk_discarded_sectors_total{device="sdb"} 1.925173784e+09
# HELP node_disk_discards_completed_total The total number of discards completed successfully.
# TYPE node_disk_discards_completed_total counter
node_disk_discards_completed_total{device="sdb"} 68851
# HELP node_disk_discards_merged_total The total number of discards merged.
# TYPE node_disk_discards_merged_total counter
node_disk_discards_merged_total{device="sdb"} 0
# HELP node_disk_io_now The number of I/Os currently in progress.
# TYPE node_disk_io_now gauge
node_disk_io_now{device="dm-0"} 0
@ -303,6 +315,7 @@ node_disk_io_now{device="mmcblk0p1"} 0
node_disk_io_now{device="mmcblk0p2"} 0
node_disk_io_now{device="nvme0n1"} 0
node_disk_io_now{device="sda"} 0
node_disk_io_now{device="sdb"} 0
node_disk_io_now{device="sr0"} 0
node_disk_io_now{device="vda"} 0
# HELP node_disk_io_time_seconds_total Total seconds spent doing I/Os.
@ -318,9 +331,10 @@ node_disk_io_time_seconds_total{device="mmcblk0p1"} 0.024
node_disk_io_time_seconds_total{device="mmcblk0p2"} 0.068
node_disk_io_time_seconds_total{device="nvme0n1"} 222.766
node_disk_io_time_seconds_total{device="sda"} 9653.880000000001
node_disk_io_time_seconds_total{device="sdb"} 60.730000000000004
node_disk_io_time_seconds_total{device="sr0"} 0
node_disk_io_time_seconds_total{device="vda"} 41614.592000000004
# HELP node_disk_io_time_weighted_seconds_total The weighted # of seconds spent doing I/Os. See https://www.kernel.org/doc/Documentation/iostats.txt.
# HELP node_disk_io_time_weighted_seconds_total The weighted # of seconds spent doing I/Os.
# TYPE node_disk_io_time_weighted_seconds_total counter
node_disk_io_time_weighted_seconds_total{device="dm-0"} 1.206301256e+06
node_disk_io_time_weighted_seconds_total{device="dm-1"} 0.084
@ -333,6 +347,7 @@ node_disk_io_time_weighted_seconds_total{device="mmcblk0p1"} 0.024
node_disk_io_time_weighted_seconds_total{device="mmcblk0p2"} 0.068
node_disk_io_time_weighted_seconds_total{device="nvme0n1"} 1032.546
node_disk_io_time_weighted_seconds_total{device="sda"} 82621.804
node_disk_io_time_weighted_seconds_total{device="sdb"} 67.07000000000001
node_disk_io_time_weighted_seconds_total{device="sr0"} 0
node_disk_io_time_weighted_seconds_total{device="vda"} 2.0778722280000001e+06
# HELP node_disk_read_bytes_total The total number of bytes read successfully.
@ -348,6 +363,7 @@ node_disk_read_bytes_total{device="mmcblk0p1"} 81920
node_disk_read_bytes_total{device="mmcblk0p2"} 389120
node_disk_read_bytes_total{device="nvme0n1"} 2.377714176e+09
node_disk_read_bytes_total{device="sda"} 5.13713216512e+11
node_disk_read_bytes_total{device="sdb"} 4.944782848e+09
node_disk_read_bytes_total{device="sr0"} 0
node_disk_read_bytes_total{device="vda"} 1.6727491584e+10
# HELP node_disk_read_time_seconds_total The total number of seconds spent by all reads.
@ -363,6 +379,7 @@ node_disk_read_time_seconds_total{device="mmcblk0p1"} 0.024
node_disk_read_time_seconds_total{device="mmcblk0p2"} 0.068
node_disk_read_time_seconds_total{device="nvme0n1"} 21.650000000000002
node_disk_read_time_seconds_total{device="sda"} 18492.372
node_disk_read_time_seconds_total{device="sdb"} 0.084
node_disk_read_time_seconds_total{device="sr0"} 0
node_disk_read_time_seconds_total{device="vda"} 8655.768
# HELP node_disk_reads_completed_total The total number of reads completed successfully.
@ -378,9 +395,10 @@ node_disk_reads_completed_total{device="mmcblk0p1"} 17
node_disk_reads_completed_total{device="mmcblk0p2"} 95
node_disk_reads_completed_total{device="nvme0n1"} 47114
node_disk_reads_completed_total{device="sda"} 2.5354637e+07
node_disk_reads_completed_total{device="sdb"} 326552
node_disk_reads_completed_total{device="sr0"} 0
node_disk_reads_completed_total{device="vda"} 1.775784e+06
# HELP node_disk_reads_merged_total The total number of reads merged. See https://www.kernel.org/doc/Documentation/iostats.txt.
# HELP node_disk_reads_merged_total The total number of reads merged.
# TYPE node_disk_reads_merged_total counter
node_disk_reads_merged_total{device="dm-0"} 0
node_disk_reads_merged_total{device="dm-1"} 0
@ -393,6 +411,7 @@ node_disk_reads_merged_total{device="mmcblk0p1"} 3
node_disk_reads_merged_total{device="mmcblk0p2"} 0
node_disk_reads_merged_total{device="nvme0n1"} 4
node_disk_reads_merged_total{device="sda"} 3.4367663e+07
node_disk_reads_merged_total{device="sdb"} 841
node_disk_reads_merged_total{device="sr0"} 0
node_disk_reads_merged_total{device="vda"} 15386
# HELP node_disk_write_time_seconds_total This is the total number of seconds spent by all writes.
@ -408,6 +427,7 @@ node_disk_write_time_seconds_total{device="mmcblk0p1"} 0
node_disk_write_time_seconds_total{device="mmcblk0p2"} 0
node_disk_write_time_seconds_total{device="nvme0n1"} 1011.053
node_disk_write_time_seconds_total{device="sda"} 63877.96
node_disk_write_time_seconds_total{device="sdb"} 5.007
node_disk_write_time_seconds_total{device="sr0"} 0
node_disk_write_time_seconds_total{device="vda"} 2.069221364e+06
# HELP node_disk_writes_completed_total The total number of writes completed successfully.
@ -423,9 +443,10 @@ node_disk_writes_completed_total{device="mmcblk0p1"} 0
node_disk_writes_completed_total{device="mmcblk0p2"} 0
node_disk_writes_completed_total{device="nvme0n1"} 1.07832e+06
node_disk_writes_completed_total{device="sda"} 2.8444756e+07
node_disk_writes_completed_total{device="sdb"} 41822
node_disk_writes_completed_total{device="sr0"} 0
node_disk_writes_completed_total{device="vda"} 6.038856e+06
# HELP node_disk_writes_merged_total The number of writes merged. See https://www.kernel.org/doc/Documentation/iostats.txt.
# HELP node_disk_writes_merged_total The number of writes merged.
# TYPE node_disk_writes_merged_total counter
node_disk_writes_merged_total{device="dm-0"} 0
node_disk_writes_merged_total{device="dm-1"} 0
@ -438,6 +459,7 @@ node_disk_writes_merged_total{device="mmcblk0p1"} 0
node_disk_writes_merged_total{device="mmcblk0p2"} 0
node_disk_writes_merged_total{device="nvme0n1"} 43950
node_disk_writes_merged_total{device="sda"} 1.1134226e+07
node_disk_writes_merged_total{device="sdb"} 2895
node_disk_writes_merged_total{device="sr0"} 0
node_disk_writes_merged_total{device="vda"} 2.0711856e+07
# HELP node_disk_written_bytes_total The total number of bytes written successfully.
@ -453,6 +475,7 @@ node_disk_written_bytes_total{device="mmcblk0p1"} 0
node_disk_written_bytes_total{device="mmcblk0p2"} 0
node_disk_written_bytes_total{device="nvme0n1"} 2.0199236096e+10
node_disk_written_bytes_total{device="sda"} 2.58916880384e+11
node_disk_written_bytes_total{device="sdb"} 1.01012736e+09
node_disk_written_bytes_total{device="sr0"} 0
node_disk_written_bytes_total{device="vda"} 1.0938236928e+11
# HELP node_drbd_activitylog_writes_total Number of updates of the activity log area of the meta data.

View File

@ -44,3 +44,6 @@
259 0 nvme0n1 47114 4 4643973 21650 1078320 43950 39451633 1011053 0 222766 1032546
259 1 nvme0n1p1 1140 0 9370 16 1 0 1 0 0 16 16
259 2 nvme0n1p2 45914 4 4631243 21626 1036885 43950 39451632 919480 0 131580 940970
8 0 sdb 326552 841 9657779 84 41822 2895 1972905 5007 0 60730 67070 68851 0 1925173784 11130
8 1 sdb1 231 3 34466 4 24 23 106 0 0 64 64 0 0 0 0
8 2 sdb2 326310 838 9622281 67 40726 2872 1972799 4924 0 58250 64567 68851 0 1925173784 11130