Improve PHP-PFM module (#3450)

* Add docker environment for integration and system testing
* Add system test file to check for correct docs.
* Brings docs in line with generated output
* Update data.json
* Remove hostname fields as already part of metricset
* Apply schema instead of manual conversion
* Rename pool.pool to pool.name
* Remove separate http client as not needed anymore

This is a follow up PR for #3415

metricbeat/docker-compose.yml

@@ -6,11 +6,12 @@ services:
       - apache
       - ceph
       - couchbase
-      - mongodb
       - haproxy
       - kafka
+      - mongodb
       - mysql
       - nginx
+      - phpfpm
       - postgresql
       - prometheus
       - redis
@@ -36,6 +37,8 @@ services:
       - MYSQL_DSN=root:test@tcp(mysql:3306)/
       - MYSQL_HOST=mysql
       - MYSQL_PORT=3306
+      - PHPFPM_HOST=phpfpm
+      - PHPFPM_PORT=81
       - POSTGRESQL_DSN=postgres://postgresql:5432?sslmode=disable
       - POSTGRESQL_HOST=postgresql
       - POSTGRESQL_PORT=5432
@@ -90,6 +93,9 @@ services:
   haproxy:
     build: ${PWD}/module/haproxy/_meta
 
+  phpfpm:
+    build: ${PWD}/module/php_fpm/_meta
+
   postgresql:
     image: postgres:9.5.3
 

metricbeat/docker-entrypoint.sh

@@ -27,8 +27,10 @@ waitFor ${CEPH_HOST} ${CEPH_PORT} Ceph
 waitFor ${COUCHBASE_HOST} ${COUCHBASE_PORT} Couchbase
 waitFor ${HAPROXY_HOST} ${HAPROXY_PORT} HAProxy
 waitFor ${KAFKA_HOST} ${KAFKA_PORT} Kafka
+waitFor ${MONGODB_HOST} ${MONGODB_PORT} MongoDB
 waitFor ${MYSQL_HOST} ${MYSQL_PORT} MySQL
 waitFor ${NGINX_HOST} ${NGINX_PORT} Nginx
+waitFor ${PHPFPM_HOST} ${PHPFPM_PORT} PHP_FPM
 waitFor ${POSTGRESQL_HOST} ${POSTGRESQL_PORT} Postgresql
 waitFor ${PROMETHEUS_HOST} ${PROMETHEUS_PORT} Prometheus
 waitFor ${REDIS_HOST} ${REDIS_PORT} Redis

metricbeat/docs/fields.asciidoc

@@ -4046,7 +4046,7 @@ experimental[]
 
 
 [float]
-=== php_fpm.pool.pool
+=== php_fpm.pool.name
 
 type: keyword
 

metricbeat/metricbeat.template-es2x.json

@@ -2324,7 +2324,7 @@
                     }
                   }
                 },
-                "pool": {
+                "name": {
                   "ignore_above": 1024,
                   "index": "not_analyzed",
                   "type": "string"

metricbeat/metricbeat.template.json

@@ -2297,7 +2297,7 @@
                     }
                   }
                 },
-                "pool": {
+                "name": {
                   "ignore_above": 1024,
                   "type": "keyword"
                 },

metricbeat/module/php_fpm/_meta/Dockerfile

@@ -2,3 +2,5 @@ FROM richarvey/nginx-php-fpm
 
 RUN echo "pm.status_path = /status" >> /etc/php7/php-fpm.d/www.conf
 ADD ./php-fpm.conf /etc/nginx/sites-enabled
+
+EXPOSE 81

metricbeat/module/php_fpm/pool/_meta/data.json

@@ -1,25 +1,28 @@
 {
-  "@timestamp": "2017-01-18T23:57:23.960Z",
-  "beat": {
-    "hostname": "host.example.com",
-    "name": "host.example.com"
-  },
-  "metricset": {
-    "host": "localhost:8081",
-    "module": "php_fpm",
-    "name": "pool",
-    "rtt": 1237
-  },
-  "php_fpm": {
-    "pool": {
-      "connections.accepted": 803,
-      "connections.queued": 0,
-      "hostname": "localhost:8081",
-      "pool": "www",
-      "processes.active": 1,
-      "processes.idle": 2,
-      "requests.slow": 0
-    }
-  },
-  "type": "metricsets"
-}
+    "@timestamp": "2016-05-23T08:05:34.853Z",
+    "beat": {
+        "hostname": "host.example.com",
+        "name": "host.example.com"
+    },
+    "metricset": {
+        "host": "127.0.0.1:81",
+        "module": "php_fpm",
+        "name": "pool",
+        "rtt": 115
+    },
+    "php_fpm": {
+        "pool": {
+            "connections": {
+                "accepted": 13,
+                "queued": 0
+            },
+            "pool": "www",
+            "processes": {
+                "active": 1,
+                "idle": 2
+            },
+            "slow_requests": 0
+        }
+    },
+    "type": "metricsets"
+}

metricbeat/module/php_fpm/pool/_meta/fields.yml

@@ -4,7 +4,7 @@
     `pool` contains the metrics that were obtained from the PHP-FPM process
     pool.
   fields:
-    - name: pool
+    - name: name
       type: keyword
       description: >
         The name of the pool.

metricbeat/module/php_fpm/pool/data.go

@@ -1,18 +1,21 @@
 package pool
 
-type poolStats struct {
-	Pool               string `json:"pool"`
-	ProcessManager     string `json:"process manager"`
-	StartTime          int    `json:"start time"`
-	StartSince         int    `json:"start since"`
-	AcceptedConn       int    `json:"accepted conn"`
-	ListenQueue        int    `json:"listen queue"`
-	MaxListenQueue     int    `json:"max listen queue"`
-	ListenQueueLen     int    `json:"listen queue len"`
-	IdleProcesses      int    `json:"idle processes"`
-	ActiveProcesses    int    `json:"active processes"`
-	TotalProcesses     int    `json:"total processes"`
-	MaxActiveProcesses int    `json:"max active processes"`
-	MaxChildrenReached int    `json:"max children reached"`
-	SlowRequests       int    `json:"slow requests"`
-}
+import (
+	s "github.com/elastic/beats/metricbeat/schema"
+	c "github.com/elastic/beats/metricbeat/schema/mapstriface"
+)
+
+var (
+	schema = s.Schema{
+		"name": c.Str("pool"),
+		"connections": s.Object{
+			"accepted": c.Int("accepted conn"),
+			"queued":   c.Int("listen queue"),
+		},
+		"processes": s.Object{
+			"idle":   c.Int("idle processes"),
+			"active": c.Int("active processes"),
+		},
+		"slow_requests": c.Int("slow requests"),
+	}
+)

metricbeat/module/php_fpm/pool/pool.go

@@ -6,69 +6,58 @@ import (
 
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/logp"
+	"github.com/elastic/beats/metricbeat/helper"
 	"github.com/elastic/beats/metricbeat/mb"
-
-	"github.com/elastic/beats/metricbeat/module/php_fpm"
+	"github.com/elastic/beats/metricbeat/mb/parse"
 )
 
 // init registers the MetricSet with the central registry.
-// The New method will be called after the setup of the module and before starting to fetch data
 func init() {
-	if err := mb.Registry.AddMetricSet("php_fpm", "pool", New, php_fpm.HostParser); err != nil {
+	if err := mb.Registry.AddMetricSet("php_fpm", "pool", New, HostParser); err != nil {
 		panic(err)
 	}
 }
 
+const (
+	defaultScheme = "http"
+	defaultPath   = "/status"
+)
+
+// HostParser is used for parsing the configured php-fpm hosts.
+var HostParser = parse.URLHostParserBuilder{
+	DefaultScheme: defaultScheme,
+	DefaultPath:   defaultPath,
+	QueryParams:   "json",
+	PathConfigKey: "status_path",
+}.Build()
+
 // MetricSet type defines all fields of the MetricSet
-// As a minimum it must inherit the mb.BaseMetricSet fields, but can be extended with
-// additional entries. These variables can be used to persist data or configuration between
-// multiple fetch calls.
 type MetricSet struct {
 	mb.BaseMetricSet
-	client *php_fpm.StatsClient // StatsClient that is reused across requests.
+	*helper.HTTP
 }
 
 // New create a new instance of the MetricSet
-// Part of new is also setting up the configuration by processing additional
-// configuration entries if needed.
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 	logp.Warn("EXPERIMENTAL: The php-fpm pool metricset is experimental")
 	return &MetricSet{
-		BaseMetricSet: base,
-		client:        php_fpm.NewStatsClient(base),
+		base,
+		helper.NewHTTP(base),
 	}, nil
 }
 
-// Fetch methods implements the data gathering and data conversion to the right format
-// It returns the event which is then forward to the output. In case of an error, a
-// descriptive error must be returned.
+// Fetch gathers data for the pool metricset
 func (m *MetricSet) Fetch() (common.MapStr, error) {
-	body, err := m.client.Fetch()
-
+	content, err := m.HTTP.FetchContent()
 	if err != nil {
 		return nil, err
 	}
 
-	defer body.Close()
-
-	stats := &poolStats{}
-	err = json.NewDecoder(body).Decode(stats)
+	var stats map[string]interface{}
+	err = json.Unmarshal(content, &stats)
 	if err != nil {
 		return nil, fmt.Errorf("error parsing json: %v", err)
 	}
 
-	return common.MapStr{
-		"hostname": m.Host(),
-
-		"pool": stats.Pool,
-		"connections": common.MapStr{
-			"queue":    stats.ListenQueue,
-			"accepted": stats.AcceptedConn,
-		},
-		"processes": common.MapStr{
-			"idle":   stats.IdleProcesses,
-			"active": stats.ActiveProcesses,
-		},
-		"slow_requests": stats.SlowRequests,
-	}, nil
+	return schema.Apply(stats), nil
 }

metricbeat/module/php_fpm/pool/pool_integration_test.go

@@ -3,6 +3,7 @@
 package pool
 
 import (
+	"os"
 	"testing"
 
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
@@ -20,6 +21,24 @@ func getConfig() map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "php_fpm",
 		"metricsets": []string{"pool"},
-		"hosts":      []string{"127.0.0.1:81"},
+		"hosts":      []string{GetEnvHost() + ":" + GetEnvPort()},
 	}
 }
+
+func GetEnvHost() string {
+	host := os.Getenv("PHPFPM_HOST")
+
+	if len(host) == 0 {
+		host = "127.0.0.1"
+	}
+	return host
+}
+
+func GetEnvPort() string {
+	port := os.Getenv("PHPFPM_PORT")
+
+	if len(port) == 0 {
+		port = "81"
+	}
+	return port
+}