From eb78b9268f4a79722432f0efe98c79556cd1fbd6 Mon Sep 17 00:00:00 2001 From: Allen Petersen Date: Tue, 8 Dec 2015 01:50:42 -0800 Subject: [PATCH 1/3] Add zfs pool stats collection. --- plugins/zfs/zfs.go | 55 ++++++++++- plugins/zfs/zfs_test.go | 203 ++++++++++++++++++++++++++++++---------- 2 files changed, 205 insertions(+), 53 deletions(-) diff --git a/plugins/zfs/zfs.go b/plugins/zfs/zfs.go index 47684d1b666b5..e1e51217af0fc 100644 --- a/plugins/zfs/zfs.go +++ b/plugins/zfs/zfs.go @@ -1,6 +1,7 @@ package zfs import ( + "fmt" "path/filepath" "strconv" "strings" @@ -12,6 +13,7 @@ import ( type Zfs struct { KstatPath string KstatMetrics []string + PoolMetrics bool } var sampleConfig = ` @@ -22,6 +24,9 @@ var sampleConfig = ` # By default, telegraf gather all zfs stats # If not specified, then default is: # kstatMetrics = ["arcstats", "zfetchstats", "vdev_cache_stats"] + # + # By default, don't gather zpool stats + # poolMetrics = false ` func (z *Zfs) SampleConfig() string { @@ -32,7 +37,7 @@ func (z *Zfs) Description() string { return "Read metrics of ZFS from arcstats, zfetchstats and vdev_cache_stats" } -func getTags(kstatPath string) map[string]string { +func getPoolStats(kstatPath string, poolMetrics bool, acc plugins.Accumulator) (map[string]string, error) { var pools string poolsDirs, _ := filepath.Glob(kstatPath + "/*/io") for _, poolDir := range poolsDirs { @@ -42,8 +47,49 @@ func getTags(kstatPath string) map[string]string { pools += "::" } pools += pool + + if poolMetrics { + err := readPoolStats(poolDir, pool, acc) + if err != nil { + return nil, err + } + } + } + + return map[string]string{"pools": pools}, nil +} + +func readPoolStats(poolIoPath string, poolName string, acc plugins.Accumulator) error { + lines, err := internal.ReadLines(poolIoPath) + if err != nil { + return err + } + + if len(lines) != 3 { + return err + } + + keys := strings.Fields(lines[1]) + values := strings.Fields(lines[2]) + + keyCount := len(keys) + + if keyCount != len(values) { + return fmt.Errorf("Key and value count don't match Keys:%v Values:%v", keys, values) + } + + tag := map[string]string{"pool": poolName} + + for i := 0; i < keyCount; i++ { + value, err := strconv.ParseInt(values[i], 10, 64) + if err != nil { + return err + } + + acc.Add(keys[i], value, tag) } - return map[string]string{"pools": pools} + + return nil } func (z *Zfs) Gather(acc plugins.Accumulator) error { @@ -57,7 +103,10 @@ func (z *Zfs) Gather(acc plugins.Accumulator) error { kstatPath = "/proc/spl/kstat/zfs" } - tags := getTags(kstatPath) + tags, err := getPoolStats(kstatPath, z.PoolMetrics, acc) + if err != nil { + return err + } for _, metric := range kstatMetrics { lines, err := internal.ReadLines(kstatPath + "/" + metric) diff --git a/plugins/zfs/zfs_test.go b/plugins/zfs/zfs_test.go index 71c9252349096..229bf1be2c6d0 100644 --- a/plugins/zfs/zfs_test.go +++ b/plugins/zfs/zfs_test.go @@ -1,7 +1,6 @@ package zfs import ( - "fmt" "io/ioutil" "os" "testing" @@ -121,6 +120,10 @@ delegations 4 0 hits 4 0 misses 4 0 ` +const pool_ioContents = `11 3 0x00 1 80 2225326830828 32953476980628 +nread nwritten reads writes wtime wlentime wupdate rtime rlentime rupdate wcnt rcnt +1884160 6450688 22 978 272187126 2850519036 2263669418655 424226814 2850519036 2263669871823 0 0 +` var testKstatPath = os.TempDir() + "/telegraf/proc/spl/kstat/zfs" @@ -129,6 +132,50 @@ type metrics struct { value int64 } +func TestZfsPoolMetrics(t *testing.T) { + err := os.MkdirAll(testKstatPath, 0755) + require.NoError(t, err) + + err = os.MkdirAll(testKstatPath+"/HOME", 0755) + require.NoError(t, err) + + err = ioutil.WriteFile(testKstatPath+"/HOME/io", []byte(pool_ioContents), 0644) + require.NoError(t, err) + + err = ioutil.WriteFile(testKstatPath+"/arcstats", []byte(arcstatsContents), 0644) + require.NoError(t, err) + + poolMetrics := getPoolMetrics() + + var acc testutil.Accumulator + + //one pool, all metrics + tags := map[string]string{ + "pool": "HOME", + } + + z := &Zfs{KstatPath: testKstatPath, KstatMetrics: []string{"arcstats"}} + err = z.Gather(&acc) + require.NoError(t, err) + + for _, metric := range poolMetrics { + assert.True(t, !acc.HasIntValue(metric.name), metric.name) + assert.True(t, !acc.CheckTaggedValue(metric.name, metric.value, tags)) + } + + z = &Zfs{KstatPath: testKstatPath, KstatMetrics: []string{"arcstats"}, PoolMetrics: true} + err = z.Gather(&acc) + require.NoError(t, err) + + for _, metric := range poolMetrics { + assert.True(t, acc.HasIntValue(metric.name), metric.name) + assert.True(t, acc.CheckTaggedValue(metric.name, metric.value, tags)) + } + + err = os.RemoveAll(os.TempDir() + "/telegraf") + require.NoError(t, err) +} + func TestZfsGeneratesMetrics(t *testing.T) { err := os.MkdirAll(testKstatPath, 0755) require.NoError(t, err) @@ -148,7 +195,60 @@ func TestZfsGeneratesMetrics(t *testing.T) { err = ioutil.WriteFile(testKstatPath+"/vdev_cache_stats", []byte(vdev_cache_statsContents), 0644) require.NoError(t, err) - intMetrics := []*metrics{ + intMetrics := getKstatMetrics() + + var acc testutil.Accumulator + + //one pool, all metrics + tags := map[string]string{ + "pools": "HOME", + } + + z := &Zfs{KstatPath: testKstatPath} + err = z.Gather(&acc) + require.NoError(t, err) + + for _, metric := range intMetrics { + assert.True(t, acc.HasIntValue(metric.name), metric.name) + assert.True(t, acc.CheckTaggedValue(metric.name, metric.value, tags)) + } + + //two pools, all metrics + err = os.MkdirAll(testKstatPath+"/STORAGE", 0755) + require.NoError(t, err) + + err = ioutil.WriteFile(testKstatPath+"/STORAGE/io", []byte(""), 0644) + require.NoError(t, err) + + tags = map[string]string{ + "pools": "HOME::STORAGE", + } + + z = &Zfs{KstatPath: testKstatPath} + err = z.Gather(&acc) + require.NoError(t, err) + + for _, metric := range intMetrics { + assert.True(t, acc.HasIntValue(metric.name), metric.name) + assert.True(t, acc.CheckTaggedValue(metric.name, metric.value, tags)) + } + + //two pools, one metric + z = &Zfs{KstatPath: testKstatPath, KstatMetrics: []string{"arcstats"}} + err = z.Gather(&acc) + require.NoError(t, err) + + for _, metric := range intMetrics { + assert.True(t, acc.HasIntValue(metric.name), metric.name) + assert.True(t, acc.CheckTaggedValue(metric.name, metric.value, tags)) + } + + err = os.RemoveAll(os.TempDir() + "/telegraf") + require.NoError(t, err) +} + +func getKstatMetrics() []*metrics { + return []*metrics{ { name: "arcstats_hits", value: 5968846374, @@ -550,54 +650,57 @@ func TestZfsGeneratesMetrics(t *testing.T) { value: 0, }, } +} - var acc testutil.Accumulator - - //one pool, all metrics - tags := map[string]string{ - "pools": "HOME", - } - - z := &Zfs{KstatPath: testKstatPath} - err = z.Gather(&acc) - require.NoError(t, err) - - for _, metric := range intMetrics { - fmt.Println(metric.name) - assert.True(t, acc.HasIntValue(metric.name), metric.name) - assert.True(t, acc.CheckTaggedValue(metric.name, metric.value, tags)) - } - - //two pools, all metrics - err = os.MkdirAll(testKstatPath+"/STORAGE", 0755) - require.NoError(t, err) - - err = ioutil.WriteFile(testKstatPath+"/STORAGE/io", []byte(""), 0644) - require.NoError(t, err) - - tags = map[string]string{ - "pools": "HOME::STORAGE", - } - - z = &Zfs{KstatPath: testKstatPath} - err = z.Gather(&acc) - require.NoError(t, err) - - for _, metric := range intMetrics { - assert.True(t, acc.HasIntValue(metric.name), metric.name) - assert.True(t, acc.CheckTaggedValue(metric.name, metric.value, tags)) - } - - //two pools, one metric - z = &Zfs{KstatPath: testKstatPath, KstatMetrics: []string{"arcstats"}} - err = z.Gather(&acc) - require.NoError(t, err) - - for _, metric := range intMetrics { - assert.True(t, acc.HasIntValue(metric.name), metric.name) - assert.True(t, acc.CheckTaggedValue(metric.name, metric.value, tags)) +func getPoolMetrics() []*metrics { + return []*metrics{ + { + name: "nread", + value: 1884160, + }, + { + name: "nwritten", + value: 6450688, + }, + { + name: "reads", + value: 22, + }, + { + name: "writes", + value: 978, + }, + { + name: "wtime", + value: 272187126, + }, + { + name: "wlentime", + value: 2850519036, + }, + { + name: "wupdate", + value: 2263669418655, + }, + { + name: "rtime", + value: 424226814, + }, + { + name: "rlentime", + value: 2850519036, + }, + { + name: "rupdate", + value: 2263669871823, + }, + { + name: "wcnt", + value: 0, + }, + { + name: "rcnt", + value: 0, + }, } - - err = os.RemoveAll(os.TempDir() + "/telegraf") - require.NoError(t, err) } From e3c8a1131a7cef4110872bbe6c5b535302a40bdd Mon Sep 17 00:00:00 2001 From: Allen Petersen Date: Tue, 8 Dec 2015 05:53:11 -0800 Subject: [PATCH 2/3] Fix single dataset test. The "two pool, one metic" test was only passing because of previous calls to Gather() had already populated the values. --- plugins/zfs/zfs_test.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/plugins/zfs/zfs_test.go b/plugins/zfs/zfs_test.go index 229bf1be2c6d0..c81e4889afdc7 100644 --- a/plugins/zfs/zfs_test.go +++ b/plugins/zfs/zfs_test.go @@ -195,7 +195,7 @@ func TestZfsGeneratesMetrics(t *testing.T) { err = ioutil.WriteFile(testKstatPath+"/vdev_cache_stats", []byte(vdev_cache_statsContents), 0644) require.NoError(t, err) - intMetrics := getKstatMetrics() + intMetrics := getKstatMetricsAll() var acc testutil.Accumulator @@ -225,6 +225,7 @@ func TestZfsGeneratesMetrics(t *testing.T) { } z = &Zfs{KstatPath: testKstatPath} + acc = testutil.Accumulator{} err = z.Gather(&acc) require.NoError(t, err) @@ -233,8 +234,11 @@ func TestZfsGeneratesMetrics(t *testing.T) { assert.True(t, acc.CheckTaggedValue(metric.name, metric.value, tags)) } + intMetrics = getKstatMetricsArcOnly() + //two pools, one metric z = &Zfs{KstatPath: testKstatPath, KstatMetrics: []string{"arcstats"}} + acc = testutil.Accumulator{} err = z.Gather(&acc) require.NoError(t, err) @@ -247,7 +251,7 @@ func TestZfsGeneratesMetrics(t *testing.T) { require.NoError(t, err) } -func getKstatMetrics() []*metrics { +func getKstatMetricsArcOnly() []*metrics { return []*metrics{ { name: "arcstats_hits", @@ -593,6 +597,11 @@ func getKstatMetrics() []*metrics { name: "arcstats_arc_meta_max", value: 18327165696, }, + } +} + +func getKstatMetricsAll() []*metrics { + otherMetrics := []*metrics{ { name: "zfetchstats_hits", value: 7812959060, @@ -650,6 +659,8 @@ func getKstatMetrics() []*metrics { value: 0, }, } + + return append(getKstatMetricsArcOnly(), otherMetrics...) } func getPoolMetrics() []*metrics { From 33f6c7aa5fa1a5b0b0eff1f0bdc7f8ddb543a46c Mon Sep 17 00:00:00 2001 From: Allen Petersen Date: Thu, 10 Dec 2015 10:38:47 -0800 Subject: [PATCH 3/3] Separate pool tag and stat collection. --- plugins/zfs/zfs.go | 52 ++++++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/plugins/zfs/zfs.go b/plugins/zfs/zfs.go index e1e51217af0fc..f655c4abfae2c 100644 --- a/plugins/zfs/zfs.go +++ b/plugins/zfs/zfs.go @@ -16,6 +16,11 @@ type Zfs struct { PoolMetrics bool } +type poolInfo struct { + name string + ioFilename string +} + var sampleConfig = ` # ZFS kstat path # If not specified, then default is: @@ -37,30 +42,34 @@ func (z *Zfs) Description() string { return "Read metrics of ZFS from arcstats, zfetchstats and vdev_cache_stats" } -func getPoolStats(kstatPath string, poolMetrics bool, acc plugins.Accumulator) (map[string]string, error) { - var pools string +func getPools(kstatPath string) []poolInfo { + pools := make([]poolInfo, 0) poolsDirs, _ := filepath.Glob(kstatPath + "/*/io") + for _, poolDir := range poolsDirs { poolDirSplit := strings.Split(poolDir, "/") pool := poolDirSplit[len(poolDirSplit)-2] - if len(pools) != 0 { - pools += "::" - } - pools += pool + pools = append(pools, poolInfo{name: pool, ioFilename: poolDir}) + } - if poolMetrics { - err := readPoolStats(poolDir, pool, acc) - if err != nil { - return nil, err - } + return pools +} + +func getTags(pools []poolInfo) map[string]string { + var poolNames string + + for _, pool := range pools { + if len(poolNames) != 0 { + poolNames += "::" } + poolNames += pool.name } - return map[string]string{"pools": pools}, nil + return map[string]string{"pools": poolNames} } -func readPoolStats(poolIoPath string, poolName string, acc plugins.Accumulator) error { - lines, err := internal.ReadLines(poolIoPath) +func gatherPoolStats(pool poolInfo, acc plugins.Accumulator) error { + lines, err := internal.ReadLines(pool.ioFilename) if err != nil { return err } @@ -78,7 +87,7 @@ func readPoolStats(poolIoPath string, poolName string, acc plugins.Accumulator) return fmt.Errorf("Key and value count don't match Keys:%v Values:%v", keys, values) } - tag := map[string]string{"pool": poolName} + tag := map[string]string{"pool": pool.name} for i := 0; i < keyCount; i++ { value, err := strconv.ParseInt(values[i], 10, 64) @@ -103,9 +112,16 @@ func (z *Zfs) Gather(acc plugins.Accumulator) error { kstatPath = "/proc/spl/kstat/zfs" } - tags, err := getPoolStats(kstatPath, z.PoolMetrics, acc) - if err != nil { - return err + pools := getPools(kstatPath) + tags := getTags(pools) + + if z.PoolMetrics { + for _, pool := range pools { + err := gatherPoolStats(pool, acc) + if err != nil { + return err + } + } } for _, metric := range kstatMetrics {