From 6807e5319bcc8698e0acbe0a04b7f139e5bac8ee Mon Sep 17 00:00:00 2001 From: Sudhar287 Date: Wed, 13 May 2020 15:06:00 -0400 Subject: [PATCH] read contents of objset file (#1632) * added objread functionality Signed-off-by: Sudharshann D --- collector/fixtures/e2e-64k-page-output.txt | 36 ++++++++++ collector/fixtures/e2e-output.txt | 36 ++++++++++ .../proc/spl/kstat/zfs/pool1/objset-1 | 9 +++ .../proc/spl/kstat/zfs/pool1/objset-2 | 9 +++ .../proc/spl/kstat/zfs/poolz1/objset-1 | 9 +++ .../proc/spl/kstat/zfs/poolz1/objset-2 | 9 +++ collector/zfs.go | 31 +++++++-- collector/zfs_linux.go | 65 +++++++++++++++++++ collector/zfs_linux_test.go | 40 ++++++++++++ 9 files changed, 238 insertions(+), 6 deletions(-) create mode 100644 collector/fixtures/proc/spl/kstat/zfs/pool1/objset-1 create mode 100644 collector/fixtures/proc/spl/kstat/zfs/pool1/objset-2 create mode 100644 collector/fixtures/proc/spl/kstat/zfs/poolz1/objset-1 create mode 100644 collector/fixtures/proc/spl/kstat/zfs/poolz1/objset-2 diff --git a/collector/fixtures/e2e-64k-page-output.txt b/collector/fixtures/e2e-64k-page-output.txt index 0ea506b3..3e9f6cb9 100644 --- a/collector/fixtures/e2e-64k-page-output.txt +++ b/collector/fixtures/e2e-64k-page-output.txt @@ -3584,6 +3584,42 @@ node_zfs_zil_zil_itx_needcopy_bytes 1.8446744073709537e+19 # HELP node_zfs_zil_zil_itx_needcopy_count kstat.zfs.misc.zil.zil_itx_needcopy_count # TYPE node_zfs_zil_zil_itx_needcopy_count untyped node_zfs_zil_zil_itx_needcopy_count 0 +# HELP node_zfs_zpool_dataset_nread kstat.zfs.misc.objset.nread +# TYPE node_zfs_zpool_dataset_nread untyped +node_zfs_zpool_dataset_nread{dataset="pool1",zpool="pool1"} 0 +node_zfs_zpool_dataset_nread{dataset="pool1/dataset1",zpool="pool1"} 28 +node_zfs_zpool_dataset_nread{dataset="poolz1",zpool="poolz1"} 0 +node_zfs_zpool_dataset_nread{dataset="poolz1/dataset1",zpool="poolz1"} 28 +# HELP node_zfs_zpool_dataset_nunlinked kstat.zfs.misc.objset.nunlinked +# TYPE node_zfs_zpool_dataset_nunlinked untyped +node_zfs_zpool_dataset_nunlinked{dataset="pool1",zpool="pool1"} 0 +node_zfs_zpool_dataset_nunlinked{dataset="pool1/dataset1",zpool="pool1"} 3 +node_zfs_zpool_dataset_nunlinked{dataset="poolz1",zpool="poolz1"} 0 +node_zfs_zpool_dataset_nunlinked{dataset="poolz1/dataset1",zpool="poolz1"} 14 +# HELP node_zfs_zpool_dataset_nunlinks kstat.zfs.misc.objset.nunlinks +# TYPE node_zfs_zpool_dataset_nunlinks untyped +node_zfs_zpool_dataset_nunlinks{dataset="pool1",zpool="pool1"} 0 +node_zfs_zpool_dataset_nunlinks{dataset="pool1/dataset1",zpool="pool1"} 3 +node_zfs_zpool_dataset_nunlinks{dataset="poolz1",zpool="poolz1"} 0 +node_zfs_zpool_dataset_nunlinks{dataset="poolz1/dataset1",zpool="poolz1"} 14 +# HELP node_zfs_zpool_dataset_nwritten kstat.zfs.misc.objset.nwritten +# TYPE node_zfs_zpool_dataset_nwritten untyped +node_zfs_zpool_dataset_nwritten{dataset="pool1",zpool="pool1"} 0 +node_zfs_zpool_dataset_nwritten{dataset="pool1/dataset1",zpool="pool1"} 12302 +node_zfs_zpool_dataset_nwritten{dataset="poolz1",zpool="poolz1"} 0 +node_zfs_zpool_dataset_nwritten{dataset="poolz1/dataset1",zpool="poolz1"} 32806 +# HELP node_zfs_zpool_dataset_reads kstat.zfs.misc.objset.reads +# TYPE node_zfs_zpool_dataset_reads untyped +node_zfs_zpool_dataset_reads{dataset="pool1",zpool="pool1"} 0 +node_zfs_zpool_dataset_reads{dataset="pool1/dataset1",zpool="pool1"} 2 +node_zfs_zpool_dataset_reads{dataset="poolz1",zpool="poolz1"} 0 +node_zfs_zpool_dataset_reads{dataset="poolz1/dataset1",zpool="poolz1"} 2 +# HELP node_zfs_zpool_dataset_writes kstat.zfs.misc.objset.writes +# TYPE node_zfs_zpool_dataset_writes untyped +node_zfs_zpool_dataset_writes{dataset="pool1",zpool="pool1"} 0 +node_zfs_zpool_dataset_writes{dataset="pool1/dataset1",zpool="pool1"} 4 +node_zfs_zpool_dataset_writes{dataset="poolz1",zpool="poolz1"} 0 +node_zfs_zpool_dataset_writes{dataset="poolz1/dataset1",zpool="poolz1"} 10 # HELP node_zfs_zpool_nread kstat.zfs.misc.io.nread # TYPE node_zfs_zpool_nread untyped node_zfs_zpool_nread{zpool="pool1"} 1.88416e+06 diff --git a/collector/fixtures/e2e-output.txt b/collector/fixtures/e2e-output.txt index 5a2ba253..df04679d 100644 --- a/collector/fixtures/e2e-output.txt +++ b/collector/fixtures/e2e-output.txt @@ -3677,6 +3677,42 @@ node_zfs_zil_zil_itx_needcopy_bytes 1.8446744073709537e+19 # HELP node_zfs_zil_zil_itx_needcopy_count kstat.zfs.misc.zil.zil_itx_needcopy_count # TYPE node_zfs_zil_zil_itx_needcopy_count untyped node_zfs_zil_zil_itx_needcopy_count 0 +# HELP node_zfs_zpool_dataset_nread kstat.zfs.misc.objset.nread +# TYPE node_zfs_zpool_dataset_nread untyped +node_zfs_zpool_dataset_nread{dataset="pool1",zpool="pool1"} 0 +node_zfs_zpool_dataset_nread{dataset="pool1/dataset1",zpool="pool1"} 28 +node_zfs_zpool_dataset_nread{dataset="poolz1",zpool="poolz1"} 0 +node_zfs_zpool_dataset_nread{dataset="poolz1/dataset1",zpool="poolz1"} 28 +# HELP node_zfs_zpool_dataset_nunlinked kstat.zfs.misc.objset.nunlinked +# TYPE node_zfs_zpool_dataset_nunlinked untyped +node_zfs_zpool_dataset_nunlinked{dataset="pool1",zpool="pool1"} 0 +node_zfs_zpool_dataset_nunlinked{dataset="pool1/dataset1",zpool="pool1"} 3 +node_zfs_zpool_dataset_nunlinked{dataset="poolz1",zpool="poolz1"} 0 +node_zfs_zpool_dataset_nunlinked{dataset="poolz1/dataset1",zpool="poolz1"} 14 +# HELP node_zfs_zpool_dataset_nunlinks kstat.zfs.misc.objset.nunlinks +# TYPE node_zfs_zpool_dataset_nunlinks untyped +node_zfs_zpool_dataset_nunlinks{dataset="pool1",zpool="pool1"} 0 +node_zfs_zpool_dataset_nunlinks{dataset="pool1/dataset1",zpool="pool1"} 3 +node_zfs_zpool_dataset_nunlinks{dataset="poolz1",zpool="poolz1"} 0 +node_zfs_zpool_dataset_nunlinks{dataset="poolz1/dataset1",zpool="poolz1"} 14 +# HELP node_zfs_zpool_dataset_nwritten kstat.zfs.misc.objset.nwritten +# TYPE node_zfs_zpool_dataset_nwritten untyped +node_zfs_zpool_dataset_nwritten{dataset="pool1",zpool="pool1"} 0 +node_zfs_zpool_dataset_nwritten{dataset="pool1/dataset1",zpool="pool1"} 12302 +node_zfs_zpool_dataset_nwritten{dataset="poolz1",zpool="poolz1"} 0 +node_zfs_zpool_dataset_nwritten{dataset="poolz1/dataset1",zpool="poolz1"} 32806 +# HELP node_zfs_zpool_dataset_reads kstat.zfs.misc.objset.reads +# TYPE node_zfs_zpool_dataset_reads untyped +node_zfs_zpool_dataset_reads{dataset="pool1",zpool="pool1"} 0 +node_zfs_zpool_dataset_reads{dataset="pool1/dataset1",zpool="pool1"} 2 +node_zfs_zpool_dataset_reads{dataset="poolz1",zpool="poolz1"} 0 +node_zfs_zpool_dataset_reads{dataset="poolz1/dataset1",zpool="poolz1"} 2 +# HELP node_zfs_zpool_dataset_writes kstat.zfs.misc.objset.writes +# TYPE node_zfs_zpool_dataset_writes untyped +node_zfs_zpool_dataset_writes{dataset="pool1",zpool="pool1"} 0 +node_zfs_zpool_dataset_writes{dataset="pool1/dataset1",zpool="pool1"} 4 +node_zfs_zpool_dataset_writes{dataset="poolz1",zpool="poolz1"} 0 +node_zfs_zpool_dataset_writes{dataset="poolz1/dataset1",zpool="poolz1"} 10 # HELP node_zfs_zpool_nread kstat.zfs.misc.io.nread # TYPE node_zfs_zpool_nread untyped node_zfs_zpool_nread{zpool="pool1"} 1.88416e+06 diff --git a/collector/fixtures/proc/spl/kstat/zfs/pool1/objset-1 b/collector/fixtures/proc/spl/kstat/zfs/pool1/objset-1 new file mode 100644 index 00000000..28a9d2e3 --- /dev/null +++ b/collector/fixtures/proc/spl/kstat/zfs/pool1/objset-1 @@ -0,0 +1,9 @@ +23 1 0x01 7 2160 221578688875 6665999035587 +name type data +dataset_name 7 pool1 +writes 4 0 +nwritten 4 0 +reads 4 0 +nread 4 0 +nunlinks 4 0 +nunlinked 4 0 diff --git a/collector/fixtures/proc/spl/kstat/zfs/pool1/objset-2 b/collector/fixtures/proc/spl/kstat/zfs/pool1/objset-2 new file mode 100644 index 00000000..79ee31d2 --- /dev/null +++ b/collector/fixtures/proc/spl/kstat/zfs/pool1/objset-2 @@ -0,0 +1,9 @@ +24 1 0x01 7 2160 221611904716 7145015038451 +name type data +dataset_name 7 pool1/dataset1 +writes 4 4 +nwritten 4 12302 +reads 4 2 +nread 4 28 +nunlinks 4 3 +nunlinked 4 3 diff --git a/collector/fixtures/proc/spl/kstat/zfs/poolz1/objset-1 b/collector/fixtures/proc/spl/kstat/zfs/poolz1/objset-1 new file mode 100644 index 00000000..189b6555 --- /dev/null +++ b/collector/fixtures/proc/spl/kstat/zfs/poolz1/objset-1 @@ -0,0 +1,9 @@ +30 1 0x01 7 2160 217993779684 2621674546179 +name type data +dataset_name 7 poolz1 +writes 4 0 +nwritten 4 0 +reads 4 0 +nread 4 0 +nunlinks 4 0 +nunlinked 4 0 \ No newline at end of file diff --git a/collector/fixtures/proc/spl/kstat/zfs/poolz1/objset-2 b/collector/fixtures/proc/spl/kstat/zfs/poolz1/objset-2 new file mode 100644 index 00000000..3d446235 --- /dev/null +++ b/collector/fixtures/proc/spl/kstat/zfs/poolz1/objset-2 @@ -0,0 +1,9 @@ +31 1 0x01 7 2160 218133979890 3024169078920 +name type data +dataset_name 7 poolz1/dataset1 +writes 4 10 +nwritten 4 32806 +reads 4 2 +nread 4 28 +nunlinks 4 14 +nunlinked 4 14 \ No newline at end of file diff --git a/collector/zfs.go b/collector/zfs.go index 093ded5c..b530e350 100644 --- a/collector/zfs.go +++ b/collector/zfs.go @@ -34,17 +34,19 @@ func init() { } type zfsCollector struct { - linuxProcpathBase string - linuxZpoolIoPath string - linuxPathMap map[string]string - logger log.Logger + linuxProcpathBase string + linuxZpoolIoPath string + linuxZpoolObjsetPath string + linuxPathMap map[string]string + logger log.Logger } // NewZFSCollector returns a new Collector exposing ZFS statistics. func NewZFSCollector(logger log.Logger) (Collector, error) { return &zfsCollector{ - linuxProcpathBase: "spl/kstat/zfs", - linuxZpoolIoPath: "/*/io", + linuxProcpathBase: "spl/kstat/zfs", + linuxZpoolIoPath: "/*/io", + linuxZpoolObjsetPath: "/*/objset-*", linuxPathMap: map[string]string{ "zfs_abd": "abdstats", "zfs_arc": "arcstats", @@ -113,3 +115,20 @@ func (c *zfsCollector) constPoolMetric(poolName string, sysctl zfsSysctl, value poolName, ) } + +func (c *zfsCollector) constPoolObjsetMetric(poolName string, datasetName string, sysctl zfsSysctl, value uint64) prometheus.Metric { + metricName := sysctl.metricName() + + return prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(namespace, "zfs_zpool_dataset", metricName), + string(sysctl), + []string{"zpool", "dataset"}, + nil, + ), + prometheus.UntypedValue, + float64(value), + poolName, + datasetName, + ) +} diff --git a/collector/zfs_linux.go b/collector/zfs_linux.go index aa27db84..e2c9749a 100644 --- a/collector/zfs_linux.go +++ b/collector/zfs_linux.go @@ -90,6 +90,31 @@ func (c *zfsCollector) updatePoolStats(ch chan<- prometheus.Metric) error { } } + zpoolObjsetPaths, err := filepath.Glob(procFilePath(filepath.Join(c.linuxProcpathBase, c.linuxZpoolObjsetPath))) + if err != nil { + return err + } + + if zpoolObjsetPaths == nil { + return nil + } + + for _, zpoolPath := range zpoolObjsetPaths { + file, err := os.Open(zpoolPath) + if err != nil { + // this file should exist, but there is a race where an exporting pool can remove the files -- ok to ignore + level.Debug(c.logger).Log("msg", "Cannot open file for reading", "path", zpoolPath) + return errZFSNotAvailable + } + + err = c.parsePoolObjsetFile(file, zpoolPath, func(poolName string, datasetName string, s zfsSysctl, v uint64) { + ch <- c.constPoolObjsetMetric(poolName, datasetName, s, v) + }) + file.Close() + if err != nil { + return err + } + } return nil } @@ -168,3 +193,43 @@ func (c *zfsCollector) parsePoolProcfsFile(reader io.Reader, zpoolPath string, h return scanner.Err() } + +func (c *zfsCollector) parsePoolObjsetFile(reader io.Reader, zpoolPath string, handler func(string, string, zfsSysctl, uint64)) error { + scanner := bufio.NewScanner(reader) + + parseLine := false + var zpoolName, datasetName string + for scanner.Scan() { + parts := strings.Fields(scanner.Text()) + + if !parseLine && len(parts) == 3 && parts[0] == "name" && parts[1] == "type" && parts[2] == "data" { + parseLine = true + continue + } + + if !parseLine || len(parts) < 3 { + continue + } + if parts[0] == "dataset_name" { + zpoolPathElements := strings.Split(zpoolPath, "/") + pathLen := len(zpoolPathElements) + zpoolName = zpoolPathElements[pathLen-2] + datasetName = parts[2] + continue + } + + if parts[1] == kstatDataUint64 { + key := fmt.Sprintf("kstat.zfs.misc.objset.%s", parts[0]) + value, err := strconv.ParseUint(parts[2], 10, 64) + if err != nil { + return fmt.Errorf("could not parse expected integer value for %q", key) + } + handler(zpoolName, datasetName, zfsSysctl(key), value) + } + } + if !parseLine { + return fmt.Errorf("did not parse a single %s %s metric", zpoolName, datasetName) + } + + return scanner.Err() +} diff --git a/collector/zfs_linux_test.go b/collector/zfs_linux_test.go index 3574858f..29d28e8f 100644 --- a/collector/zfs_linux_test.go +++ b/collector/zfs_linux_test.go @@ -311,6 +311,46 @@ func TestZpoolParsing(t *testing.T) { } } +func TestZpoolObjsetParsing(t *testing.T) { + zpoolPaths, err := filepath.Glob("fixtures/proc/spl/kstat/zfs/*/objset-*") + if err != nil { + t.Fatal(err) + } + + c := zfsCollector{} + if err != nil { + t.Fatal(err) + } + + handlerCalled := false + for _, zpoolPath := range zpoolPaths { + file, err := os.Open(zpoolPath) + if err != nil { + t.Fatal(err) + } + + err = c.parsePoolObjsetFile(file, zpoolPath, func(poolName string, datasetName string, s zfsSysctl, v uint64) { + if s != zfsSysctl("kstat.zfs.misc.objset.writes") { + return + } + + handlerCalled = true + + if v != uint64(0) && v != uint64(4) && v != uint64(10) { + t.Fatalf("Incorrect value parsed from procfs data %v", v) + } + + }) + file.Close() + if err != nil { + t.Fatal(err) + } + } + if !handlerCalled { + t.Fatal("Zpool parsing handler was not called for some expected sysctls") + } +} + func TestAbdstatsParsing(t *testing.T) { abdstatsFile, err := os.Open("fixtures/proc/spl/kstat/zfs/abdstats") if err != nil {