collector: reimplement sockstat collector with procfs (#1552)

* collector: reimplement sockstat collector with procfs
* collector: handle sockstat IPv4 disabled, debug logging

Signed-off-by: Matt Layher <mdlayher@gmail.com>

CHANGELOG.md

@@ -25,6 +25,7 @@
 * [ENHANCEMENT] Report non-fatal collection errors in the exporter metric. #1439
 * [ENHANCEMENT] Expose IPVS firewall mark as a label #1455
 * [ENHANCEMENT] Add check for systemd version before attempting to query certain metrics. #1413
+* [ENHANCEMENT] The sockstat collector now exposes IPv6 statistics in addition to the existing IPv4 support.
 * [BUGFIX] Renamed label `state` to `name` on `node_systemd_service_restart_total`. #1393
 * [BUGFIX] Fix netdev nil reference on Darwin #1414
 * [BUGFIX] Strip path.rootfs from mountpoint labels #1421

collector/fixtures/e2e-output.txt

@@ -2575,15 +2575,27 @@ node_scrape_collector_success{collector="vmstat"} 1
 node_scrape_collector_success{collector="wifi"} 1
 node_scrape_collector_success{collector="xfs"} 1
 node_scrape_collector_success{collector="zfs"} 1
+# HELP node_sockstat_FRAG6_inuse Number of FRAG6 sockets in state inuse.
+# TYPE node_sockstat_FRAG6_inuse gauge
+node_sockstat_FRAG6_inuse 0
+# HELP node_sockstat_FRAG6_memory Number of FRAG6 sockets in state memory.
+# TYPE node_sockstat_FRAG6_memory gauge
+node_sockstat_FRAG6_memory 0
 # HELP node_sockstat_FRAG_inuse Number of FRAG sockets in state inuse.
 # TYPE node_sockstat_FRAG_inuse gauge
 node_sockstat_FRAG_inuse 0
 # HELP node_sockstat_FRAG_memory Number of FRAG sockets in state memory.
 # TYPE node_sockstat_FRAG_memory gauge
 node_sockstat_FRAG_memory 0
+# HELP node_sockstat_RAW6_inuse Number of RAW6 sockets in state inuse.
+# TYPE node_sockstat_RAW6_inuse gauge
+node_sockstat_RAW6_inuse 1
 # HELP node_sockstat_RAW_inuse Number of RAW sockets in state inuse.
 # TYPE node_sockstat_RAW_inuse gauge
 node_sockstat_RAW_inuse 0
+# HELP node_sockstat_TCP6_inuse Number of TCP6 sockets in state inuse.
+# TYPE node_sockstat_TCP6_inuse gauge
+node_sockstat_TCP6_inuse 17
 # HELP node_sockstat_TCP_alloc Number of TCP sockets in state alloc.
 # TYPE node_sockstat_TCP_alloc gauge
 node_sockstat_TCP_alloc 17
@@ -2602,6 +2614,12 @@ node_sockstat_TCP_orphan 0
 # HELP node_sockstat_TCP_tw Number of TCP sockets in state tw.
 # TYPE node_sockstat_TCP_tw gauge
 node_sockstat_TCP_tw 4
+# HELP node_sockstat_UDP6_inuse Number of UDP6 sockets in state inuse.
+# TYPE node_sockstat_UDP6_inuse gauge
+node_sockstat_UDP6_inuse 9
+# HELP node_sockstat_UDPLITE6_inuse Number of UDPLITE6 sockets in state inuse.
+# TYPE node_sockstat_UDPLITE6_inuse gauge
+node_sockstat_UDPLITE6_inuse 0
 # HELP node_sockstat_UDPLITE_inuse Number of UDPLITE sockets in state inuse.
 # TYPE node_sockstat_UDPLITE_inuse gauge
 node_sockstat_UDPLITE_inuse 0
@@ -2614,7 +2632,7 @@ node_sockstat_UDP_mem 0
 # HELP node_sockstat_UDP_mem_bytes Number of UDP sockets in state mem_bytes.
 # TYPE node_sockstat_UDP_mem_bytes gauge
 node_sockstat_UDP_mem_bytes 0
-# HELP node_sockstat_sockets_used Number of sockets sockets in state used.
+# HELP node_sockstat_sockets_used Number of IPv4 sockets in use.
 # TYPE node_sockstat_sockets_used gauge
 node_sockstat_sockets_used 229
 # HELP node_textfile_mtime_seconds Unixtime mtime of textfiles successfully read.

collector/fixtures/proc/net/sockstat6

@@ -0,0 +1,5 @@
+TCP6: inuse 17
+UDP6: inuse 9
+UDPLITE6: inuse 0
+RAW6: inuse 1
+FRAG6: inuse 0 memory 0

collector/sockstat_linux.go

@@ -16,14 +16,12 @@
 package collector
 
 import (
-	"bufio"
 	"fmt"
-	"io"
 	"os"
-	"strconv"
-	"strings"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/log"
+	"github.com/prometheus/procfs"
 )
 
 const (
@@ -45,78 +43,138 @@ func NewSockStatCollector() (Collector, error) {
 }
 
 func (c *sockStatCollector) Update(ch chan<- prometheus.Metric) error {
-	sockStats, err := getSockStats(procFilePath("net/sockstat"))
+	fs, err := procfs.NewFS(*procPath)
 	if err != nil {
-		return fmt.Errorf("couldn't get sockstats: %s", err)
+		return fmt.Errorf("failed to open procfs: %v", err)
 	}
-	for protocol, protocolStats := range sockStats {
-		for name, value := range protocolStats {
-			v, err := strconv.ParseFloat(value, 64)
-			if err != nil {
-				return fmt.Errorf("invalid value %s in sockstats: %s", value, err)
-			}
-			ch <- prometheus.MustNewConstMetric(
-				prometheus.NewDesc(
-					prometheus.BuildFQName(namespace, sockStatSubsystem, protocol+"_"+name),
-					fmt.Sprintf("Number of %s sockets in state %s.", protocol, name),
-					nil, nil,
-				),
-				prometheus.GaugeValue, v,
-			)
-		}
+
+	// If IPv4 and/or IPv6 are disabled on this kernel, handle it gracefully.
+	stat4, err := fs.NetSockstat()
+	switch {
+	case err == nil:
+	case os.IsNotExist(err):
+		log.Debug("IPv4 sockstat statistics not found, skipping")
+	default:
+		return fmt.Errorf("failed to get IPv4 sockstat data: %v", err)
 	}
-	return err
-}
 
-func getSockStats(fileName string) (map[string]map[string]string, error) {
-	file, err := os.Open(fileName)
-	if err != nil {
-		return nil, err
+	stat6, err := fs.NetSockstat6()
+	switch {
+	case err == nil:
+	case os.IsNotExist(err):
+		log.Debug("IPv6 sockstat statistics not found, skipping")
+	default:
+		return fmt.Errorf("failed to get IPv6 sockstat data: %v", err)
+	}
+
+	stats := []struct {
+		isIPv6 bool
+		stat   *procfs.NetSockstat
+	}{
+		{
+			stat: stat4,
+		},
+		{
+			isIPv6: true,
+			stat:   stat6,
+		},
 	}
-	defer file.Close()
 
-	return parseSockStats(file, fileName)
+	for _, s := range stats {
+		c.update(ch, s.isIPv6, s.stat)
+	}
+
+	return nil
 }
 
-func parseSockStats(r io.Reader, fileName string) (map[string]map[string]string, error) {
-	var (
-		sockStat = map[string]map[string]string{}
-		scanner  = bufio.NewScanner(r)
-	)
-
-	for scanner.Scan() {
-		line := strings.Split(scanner.Text(), " ")
-		// Remove trailing ':'.
-		protocol := line[0][:len(line[0])-1]
-		sockStat[protocol] = map[string]string{}
-
-		for i := 1; i < len(line) && i+1 < len(line); i++ {
-			sockStat[protocol][line[i]] = line[i+1]
-			i++
-		}
+func (c *sockStatCollector) update(ch chan<- prometheus.Metric, isIPv6 bool, s *procfs.NetSockstat) {
+	if s == nil {
+		// IPv6 disabled or similar; nothing to do.
+		return
 	}
-	if err := scanner.Err(); err != nil {
-		return nil, err
+
+	// If sockstat contains the number of used sockets, export it.
+	if !isIPv6 && s.Used != nil {
+		// TODO: this must be updated if sockstat6 ever exports this data.
+		ch <- prometheus.MustNewConstMetric(
+			prometheus.NewDesc(
+				prometheus.BuildFQName(namespace, sockStatSubsystem, "sockets_used"),
+				"Number of IPv4 sockets in use.",
+				nil,
+				nil,
+			),
+			prometheus.GaugeValue,
+			float64(*s.Used),
+		)
 	}
 
-	// The mem metrics is the count of pages used. Multiply the mem metrics by
-	// the page size from the kernel to get the number of bytes used.
-	//
-	// Update the TCP mem from page count to bytes.
-	pageCount, err := strconv.Atoi(sockStat["TCP"]["mem"])
-	if err != nil {
-		return nil, fmt.Errorf("invalid value %s in sockstats: %s", sockStat["TCP"]["mem"], err)
+	// A name and optional value for a sockstat metric.
+	type ssPair struct {
+		name string
+		v    *int
 	}
-	sockStat["TCP"]["mem_bytes"] = strconv.Itoa(pageCount * pageSize)
 
-	// Update the UDP mem from page count to bytes.
-	if udpMem := sockStat["UDP"]["mem"]; udpMem != "" {
-		pageCount, err = strconv.Atoi(udpMem)
-		if err != nil {
-			return nil, fmt.Errorf("invalid value %s in sockstats: %s", sockStat["UDP"]["mem"], err)
+	// Previously these metric names were generated directly from the file output.
+	// In order to keep the same level of compatibility, we must map the fields
+	// to their correct names.
+	for _, p := range s.Protocols {
+		pairs := []ssPair{
+			{
+				name: "inuse",
+				v:    &p.InUse,
+			},
+			{
+				name: "orphan",
+				v:    p.Orphan,
+			},
+			{
+				name: "tw",
+				v:    p.TW,
+			},
+			{
+				name: "alloc",
+				v:    p.Alloc,
+			},
+			{
+				name: "mem",
+				v:    p.Mem,
+			},
+			{
+				name: "memory",
+				v:    p.Memory,
+			},
 		}
-		sockStat["UDP"]["mem_bytes"] = strconv.Itoa(pageCount * pageSize)
-	}
 
-	return sockStat, nil
+		// Also export mem_bytes values for sockets which have a mem value
+		// stored in pages.
+		if p.Mem != nil {
+			v := *p.Mem * pageSize
+			pairs = append(pairs, ssPair{
+				name: "mem_bytes",
+				v:    &v,
+			})
+		}
+
+		for _, pair := range pairs {
+			if pair.v == nil {
+				// This value is not set for this protocol; nothing to do.
+				continue
+			}
+
+			ch <- prometheus.MustNewConstMetric(
+				prometheus.NewDesc(
+					prometheus.BuildFQName(
+						namespace,
+						sockStatSubsystem,
+						fmt.Sprintf("%s_%s", p.Protocol, pair.name),
+					),
+					fmt.Sprintf("Number of %s sockets in state %s.", p.Protocol, pair.name),
+					nil,
+					nil,
+				),
+				prometheus.GaugeValue,
+				float64(*pair.v),
+			)
+		}
+	}
 }

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/prometheus/client_golang v1.0.0
 	github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90
 	github.com/prometheus/common v0.7.0
-	github.com/prometheus/procfs v0.0.7
+	github.com/prometheus/procfs v0.0.8
 	github.com/siebenmann/go-kstat v0.0.0-20160321171754-d34789b79745
 	github.com/soundcloud/go-runit v0.0.0-20150630195641-06ad41a06c4a
 	go.uber.org/atomic v1.3.2 // indirect

go.sum

@@ -85,8 +85,8 @@ github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt2
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/procfs v0.0.7 h1:RS5GAlMbnkWkhs4+bPocMTmGjYkuCY5djjqEDdXOhcQ=
-github.com/prometheus/procfs v0.0.7/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
+github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8=
+github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/siebenmann/go-kstat v0.0.0-20160321171754-d34789b79745 h1:IuH7WumZNax0D+rEqmy2TyhKCzrtMGqbZO0b8rO00JA=
 github.com/siebenmann/go-kstat v0.0.0-20160321171754-d34789b79745/go.mod h1:G81aIFAMS9ECrwBYR9YxhlPjWgrItd+Kje78O6+uqm8=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=