Issue #2616: ZFS Zpool properties on Linux

Add freeing and leaked options collection

plugins/inputs/zfs/README.md

@@ -1,7 +1,7 @@
 # ZFS plugin
 
 This ZFS plugin provides metrics from your ZFS filesystems. It supports ZFS on
-Linux and FreeBSD. It gets ZFS stat from `/proc/spl/kstat/zfs` on Linux and
+Linux and FreeBSD. It gets ZFS stat from `/proc/spl/kstat/zfs` and zpool on Linux and
 from `sysctl` and `zpool` on FreeBSD.
 
 ### Configuration:
@@ -180,7 +180,7 @@ each pool.
 
 #### Pool Metrics (optional)
 
-On Linux (reference: kstat accumulated time and queue length statistics):
+On Linux (reference: kstat accumulated time and queue length statistics) and zpool statistics:
 
 - zfs_pool
     - nread (integer, bytes)
@@ -195,6 +195,12 @@ On Linux (reference: kstat accumulated time and queue length statistics):
     - rupdate (integer, timestamp)
     - wcnt (integer, count)
     - rcnt (integer, count)
+    - allocated (integer, bytes)
+    - capacity (integer, bytes)
+    - dedupratio (float, ratio)
+    - free (integer, bytes)
+    - size (integer, bytes)
+    - fragmentation (integer, percent)
 
 On FreeBSD:
 
@@ -327,3 +333,27 @@ ABD is a linear/scatter dual typed buffer for ARC
 note: ZIL measurements are system-wide, neither per-pool nor per-dataset
 
 `zil_commit_count` counts when ZFS transactions are committed to a ZIL
+
+#### Pool Metrics (optional)
+
+`nread` - counts bytes read from the pool
+`nwritten` - counts bytes written to the pool 
+`reads` - counts read requests to the pool 
+`writes` - counts write request to the pool
+`wtime` - counts total time all read/write requests spend in  wait queue (integer, nanoseconds) 
+`wcnt` - its a gauge, not increment counter. It counts number of IO requests in the wait queue of the pool at the moment of measurement. 
+`wlentime` - cumulative value, calculated as number of IO requests in a wait queue multiplied by time(in nanoseconds) this requests spend in a wait queue. Value is updated every time new IO request added/removed to/from wait queue. It can be very helpful to notice short-living spikes in number of IO requests, which can not be reveal by `wcnt` 
+`wupdate` - timestamp of last wait queue update(new IO request added or exist one removed)
+`rtime` - counts total time all read/write requests spend in run queue (integer, nanoseconds)
+`rlentime` - same as `wlentime` but for run queue
+`rupdate` - timestamp of last run queue update
+`rcnt` - same as `wcnt`, but for run queue
+`allocated` - amount of storage available within the pool (integer, bytes)
+`capacity` - percentage of pool space used (integer)
+`dedupratio` (float, ratio)
+`free` - the amount of free space available in the pool (integer, bytes)
+`size` - total size of the storage pool (integer, bytes)
+`fragmentation` - the amount of fragmentation in the pool (integer, percent). Calculated only for free space, not the total pool capacity.
+`freeing` - After a file system or snapshot is destroyed, the space it was using is returned to the pool asynchronously.  freeing is the amount of space remaining to be reclaimed.  Over time freeing will decrease while free increases.(integer, bytes)
+`leaked` - (integer, bytes)
+

plugins/inputs/zfs/zfs.go

@@ -1,5 +1,13 @@
 package zfs
 
+import (
+	"bytes"
+	"fmt"
+	"os/exec"
+	"strconv"
+	"strings"
+)
+
 type Sysctl func(metric string) ([]string, error)
 type Zpool func() ([]string, error)
 
@@ -30,6 +38,109 @@ func (z *Zfs) SampleConfig() string {
 	return sampleConfig
 }
 
+func (z *Zfs) getZpoolStats() (map[string]map[string]interface{}, error) {
+
+	poolFields := map[string]map[string]interface{}{}
+
+	lines, err := z.zpool()
+	if err != nil {
+		return poolFields, err
+	}
+
+	for _, line := range lines {
+		col := strings.Split(line, "\t")
+		if len(col) != 10 {
+			continue
+		}
+
+		health := col[1]
+		name := col[0]
+
+		fields := map[string]interface{}{
+			"name":   name,
+			"health": health,
+		}
+
+		if health == "UNAVAIL" {
+
+			fields["size"] = int64(0)
+
+		} else {
+
+			size, err := strconv.ParseInt(col[2], 10, 64)
+			if err != nil {
+				return poolFields, fmt.Errorf("Error parsing size: %s", err)
+			}
+			fields["size"] = size
+
+			alloc, err := strconv.ParseInt(col[3], 10, 64)
+			if err != nil {
+				return poolFields, fmt.Errorf("Error parsing allocation: %s", err)
+			}
+			fields["allocated"] = alloc
+
+			free, err := strconv.ParseInt(col[4], 10, 64)
+			if err != nil {
+				return poolFields, fmt.Errorf("Error parsing free: %s", err)
+			}
+			fields["free"] = free
+
+			frag, err := strconv.ParseInt(strings.TrimSuffix(col[5], "%"), 10, 0)
+			if err != nil { // This might be - for RO devs
+				frag = 0
+			}
+			fields["fragmentation"] = frag
+
+			capval, err := strconv.ParseInt(col[6], 10, 0)
+			if err != nil {
+				return poolFields, fmt.Errorf("Error parsing capacity: %s", err)
+			}
+			fields["capacity"] = capval
+
+			dedup, err := strconv.ParseFloat(strings.TrimSuffix(col[7], "x"), 32)
+			if err != nil {
+				return poolFields, fmt.Errorf("Error parsing dedupratio: %s", err)
+			}
+			fields["dedupratio"] = dedup
+
+			freeing, err := strconv.ParseInt(col[8], 10, 64)
+			if err != nil {
+				return poolFields, fmt.Errorf("Error parsing freeing: %s", err)
+			}
+			fields["freeing"] = freeing
+
+			leaked, err := strconv.ParseInt(col[9], 10, 64)
+			if err != nil {
+				return poolFields, fmt.Errorf("Error parsing leaked: %s", err)
+			}
+			fields["leaked"] = leaked
+		}
+		poolFields[name] = fields
+	}
+
+	return poolFields, nil
+}
+
 func (z *Zfs) Description() string {
 	return "Read metrics of ZFS from arcstats, zfetchstats, vdev_cache_stats, and pools"
 }
+
+func run(command string, args ...string) ([]string, error) {
+	cmd := exec.Command(command, args...)
+	var outbuf, errbuf bytes.Buffer
+	cmd.Stdout = &outbuf
+	cmd.Stderr = &errbuf
+	err := cmd.Run()
+
+	stdout := strings.TrimSpace(outbuf.String())
+	stderr := strings.TrimSpace(errbuf.String())
+
+	if _, ok := err.(*exec.ExitError); ok {
+		return nil, fmt.Errorf("%s error: %s", command, stderr)
+	}
+	return strings.Split(stdout, "\n"), nil
+}
+
+func zpool() ([]string, error) {
+	return run("zpool", []string{"list", "-Hp", "-o", "name,health,size,alloc,free,fragmentation,capacity,dedupratio,freeing,leaked"}...)
+}

plugins/inputs/zfs/zfs_freebsd.go

@@ -3,9 +3,7 @@
 package zfs
 
 import (
-	"bytes"
 	"fmt"
-	"os/exec"
 	"strconv"
 	"strings"
 
@@ -14,71 +12,25 @@ import (
 )
 
 func (z *Zfs) gatherPoolStats(acc telegraf.Accumulator) (string, error) {
-
-	lines, err := z.zpool()
+	poolFields, err := z.getZpoolStats()
 	if err != nil {
 		return "", err
 	}
 
 	pools := []string{}
-	for _, line := range lines {
-		col := strings.Split(line, "\t")
-
-		pools = append(pools, col[0])
+	for name := range poolFields {
+		pools = append(pools, name)
 	}
 
 	if z.PoolMetrics {
-		for _, line := range lines {
-			col := strings.Split(line, "\t")
-			if len(col) != 8 {
-				continue
+		for name, fields := range poolFields {
+			tags := map[string]string{
+				"pool":   name,
+				"health": fields["health"].(string),
 			}
 
-			tags := map[string]string{"pool": col[0], "health": col[1]}
-			fields := map[string]interface{}{}
-
-			if tags["health"] == "UNAVAIL" {
-
-				fields["size"] = int64(0)
-
-			} else {
-
-				size, err := strconv.ParseInt(col[2], 10, 64)
-				if err != nil {
-					return "", fmt.Errorf("Error parsing size: %s", err)
-				}
-				fields["size"] = size
-
-				alloc, err := strconv.ParseInt(col[3], 10, 64)
-				if err != nil {
-					return "", fmt.Errorf("Error parsing allocation: %s", err)
-				}
-				fields["allocated"] = alloc
-
-				free, err := strconv.ParseInt(col[4], 10, 64)
-				if err != nil {
-					return "", fmt.Errorf("Error parsing free: %s", err)
-				}
-				fields["free"] = free
-
-				frag, err := strconv.ParseInt(strings.TrimSuffix(col[5], "%"), 10, 0)
-				if err != nil { // This might be - for RO devs
-					frag = 0
-				}
-				fields["fragmentation"] = frag
-
-				capval, err := strconv.ParseInt(col[6], 10, 0)
-				if err != nil {
-					return "", fmt.Errorf("Error parsing capacity: %s", err)
-				}
-				fields["capacity"] = capval
-
-				dedup, err := strconv.ParseFloat(strings.TrimSuffix(col[7], "x"), 32)
-				if err != nil {
-					return "", fmt.Errorf("Error parsing dedupratio: %s", err)
-				}
-				fields["dedupratio"] = dedup
-			}
+			delete(fields, "name")
+			delete(fields, "health")
 
 			acc.AddFields("zfs_pool", fields, tags)
 		}
@@ -93,11 +45,11 @@ func (z *Zfs) Gather(acc telegraf.Accumulator) error {
 		kstatMetrics = []string{"arcstats", "zfetchstats", "vdev_cache_stats"}
 	}
 
-	tags := map[string]string{}
 	poolNames, err := z.gatherPoolStats(acc)
 	if err != nil {
 		return err
 	}
+	tags := map[string]string{"pools": poolNames}
 	tags["pools"] = poolNames
 
 	fields := make(map[string]interface{})
@@ -117,26 +69,6 @@ func (z *Zfs) Gather(acc telegraf.Accumulator) error {
 	return nil
 }
 
-func run(command string, args ...string) ([]string, error) {
-	cmd := exec.Command(command, args...)
-	var outbuf, errbuf bytes.Buffer
-	cmd.Stdout = &outbuf
-	cmd.Stderr = &errbuf
-	err := cmd.Run()
-
-	stdout := strings.TrimSpace(outbuf.String())
-	stderr := strings.TrimSpace(errbuf.String())
-
-	if _, ok := err.(*exec.ExitError); ok {
-		return nil, fmt.Errorf("%s error: %s", command, stderr)
-	}
-	return strings.Split(stdout, "\n"), nil
-}
-
-func zpool() ([]string, error) {
-	return run("zpool", []string{"list", "-Hp", "-o", "name,health,size,alloc,free,fragmentation,capacity,dedupratio"}...)
-}
-
 func sysctl(metric string) ([]string, error) {
 	return run("sysctl", []string{"-q", fmt.Sprintf("kstat.zfs.misc.%s", metric)}...)
 }

plugins/inputs/zfs/zfs_freebsd_test.go

@@ -10,12 +10,12 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-// $ zpool list -Hp -o name,health,size,alloc,free,fragmentation,capacity,dedupratio
+// $ zpool list -Hp -o name,health,size,alloc,free,fragmentation,capacity,dedupratio,freeing,leaked
 var zpool_output = []string{
-	"freenas-boot	ONLINE	30601641984	2022177280	28579464704	-	6	1.00x",
-	"red1	ONLINE	8933531975680	1126164848640	7807367127040	8%	12	1.83x",
-	"temp1	ONLINE	2989297238016	1626309320704	1362987917312	38%	54	1.28x",
-	"temp2	ONLINE	2989297238016	626958278656	2362338959360	12%	20	1.00x",
+	"freenas-boot	ONLINE	30601641984	2022177280	28579464704	-	6	1.00x	0	0",
+	"red1	ONLINE	8933531975680	1126164848640	7807367127040	8%	12	1.83x	0	0",
+	"temp1	ONLINE	2989297238016	1626309320704	1362987917312	38%	54	1.28x	0	0",
+	"temp2	ONLINE	2989297238016	626958278656	2362338959360	12%	20	1.00x	0	0",
 }
 
 func mock_zpool() ([]string, error) {
@@ -24,7 +24,7 @@ func mock_zpool() ([]string, error) {
 
 // $ zpool list -Hp -o name,health,size,alloc,free,fragmentation,capacity,dedupratio
 var zpool_output_unavail = []string{
-	"temp2	UNAVAIL	-	-	-	-	-	-",
+	"temp2	UNAVAIL	-	-	-	-	-	-	-	-",
 }
 
 func mock_zpool_unavail() ([]string, error) {
@@ -169,6 +169,8 @@ func getFreeNasBootPoolMetrics() map[string]interface{} {
 		"free":          int64(28579464704),
 		"size":          int64(30601641984),
 		"fragmentation": int64(0),
+		"freeing":       int64(0),
+		"leaked":        int64(0),
 	}
 }