From 539f2290a55ba48cdcfee0550805976c7b8339d1 Mon Sep 17 00:00:00 2001 From: Thiago Souza Date: Wed, 18 Jan 2017 18:39:04 -0200 Subject: [PATCH] Metricbeat php-fpm module Add pool start time and start since stats Add support for proc stats Refactor stats api call Fix listen_queue => listen_queue_len" Revert "Fix listen_queue => listen_queue_len"" This reverts commit 46ff489fa8cce8d73c2aba42758a51bde4d68a77. Add suport for "listen_queue_len" Add documentation Minor const refactor Add generated doc Add basic testing Fix illegal char in asciidoc Fix type Update generated files Fix package name Add integration tests --- metricbeat/_meta/beat.full.yml | 8 + metricbeat/_meta/beat.yml | 8 + metricbeat/docs/fields.asciidoc | 245 ++++++++++++++++++ metricbeat/docs/modules/php_fpm.asciidoc | 65 +++++ metricbeat/docs/modules/php_fpm/pool.asciidoc | 19 ++ metricbeat/docs/modules/php_fpm/proc.asciidoc | 19 ++ metricbeat/docs/modules_list.asciidoc | 2 + metricbeat/include/list.go | 3 + metricbeat/metricbeat.full.yml | 8 + metricbeat/metricbeat.template-es2x.json | 109 ++++++++ metricbeat/metricbeat.template.json | 101 ++++++++ metricbeat/metricbeat.yml | 8 + metricbeat/module/php_fpm/_meta/Dockerfile | 4 + metricbeat/module/php_fpm/_meta/config.yml | 6 + metricbeat/module/php_fpm/_meta/docs.asciidoc | 28 ++ metricbeat/module/php_fpm/_meta/fields.yml | 11 + metricbeat/module/php_fpm/_meta/php-fpm.conf | 16 ++ metricbeat/module/php_fpm/doc.go | 4 + metricbeat/module/php_fpm/php_fpm.go | 103 ++++++++ metricbeat/module/php_fpm/php_fpm_test.go | 30 +++ .../module/php_fpm/pool/_meta/data.json | 33 +++ .../module/php_fpm/pool/_meta/docs.asciidoc | 3 + .../module/php_fpm/pool/_meta/fields.yml | 78 ++++++ metricbeat/module/php_fpm/pool/pool.go | 83 ++++++ .../php_fpm/pool/pool_integration_test.go | 25 ++ .../module/php_fpm/proc/_meta/data.json | 32 +++ .../module/php_fpm/proc/_meta/docs.asciidoc | 3 + .../module/php_fpm/proc/_meta/fields.yml | 74 ++++++ metricbeat/module/php_fpm/proc/proc.go | 88 +++++++ .../php_fpm/proc/proc_integration_test.go | 25 ++ 30 files changed, 1241 insertions(+) create mode 100644 metricbeat/docs/modules/php_fpm.asciidoc create mode 100644 metricbeat/docs/modules/php_fpm/pool.asciidoc create mode 100644 metricbeat/docs/modules/php_fpm/proc.asciidoc create mode 100644 metricbeat/module/php_fpm/_meta/Dockerfile create mode 100644 metricbeat/module/php_fpm/_meta/config.yml create mode 100644 metricbeat/module/php_fpm/_meta/docs.asciidoc create mode 100644 metricbeat/module/php_fpm/_meta/fields.yml create mode 100644 metricbeat/module/php_fpm/_meta/php-fpm.conf create mode 100644 metricbeat/module/php_fpm/doc.go create mode 100644 metricbeat/module/php_fpm/php_fpm.go create mode 100644 metricbeat/module/php_fpm/php_fpm_test.go create mode 100644 metricbeat/module/php_fpm/pool/_meta/data.json create mode 100644 metricbeat/module/php_fpm/pool/_meta/docs.asciidoc create mode 100644 metricbeat/module/php_fpm/pool/_meta/fields.yml create mode 100644 metricbeat/module/php_fpm/pool/pool.go create mode 100644 metricbeat/module/php_fpm/pool/pool_integration_test.go create mode 100644 metricbeat/module/php_fpm/proc/_meta/data.json create mode 100644 metricbeat/module/php_fpm/proc/_meta/docs.asciidoc create mode 100644 metricbeat/module/php_fpm/proc/_meta/fields.yml create mode 100644 metricbeat/module/php_fpm/proc/proc.go create mode 100644 metricbeat/module/php_fpm/proc/proc_integration_test.go diff --git a/metricbeat/_meta/beat.full.yml b/metricbeat/_meta/beat.full.yml index f4065b153c0..15da7fbaac4 100644 --- a/metricbeat/_meta/beat.full.yml +++ b/metricbeat/_meta/beat.full.yml @@ -200,6 +200,14 @@ metricbeat.modules: # Path to server status. Default server-status #server_status_path: "server-status" +#------------------------------- php_fpm Module ------------------------------ +- module: php_fpm + metricsets: ["pool"] + enabled: true + period: 10s + hosts: ["localhost"] + + #----------------------------- PostgreSQL Module ----------------------------- #- module: postgresql #metricsets: diff --git a/metricbeat/_meta/beat.yml b/metricbeat/_meta/beat.yml index 6ab098828ce..e3ff4a38fdf 100644 --- a/metricbeat/_meta/beat.yml +++ b/metricbeat/_meta/beat.yml @@ -46,4 +46,12 @@ metricbeat.modules: period: 10s processes: ['.*'] +#------------------------------- php_fpm Module ------------------------------ +- module: php_fpm + metricsets: ["pool"] + enabled: true + period: 10s + hosts: ["localhost"] + + diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 1afc271c34d..87e2bdeb3eb 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -23,6 +23,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -3802,6 +3803,250 @@ type: long The current number of idle client connections waiting for a request. +[[exported-fields-php_fpm]] +== php_fpm Fields + +PHP-FPM server status metrics collected from PHP-FPM. + + + +[float] +== php_fpm Fields + +`php_fpm` contains the metrics that were obtained from PHP-FPM status page call. + + + +[float] +== pool Fields + +`pool` contains the metrics that were obtained from the PHP-FPM process pool. + + + +[float] +=== php_fpm.pool.pool + +type: keyword + +The name of the pool. + + +[float] +=== php_fpm.pool.process_manager + +type: keyword + +`static`, `dynamic` or `ondemand`. + + +[float] +=== php_fpm.pool.start_time + +type: long + +The date and time FPM has started. + + +[float] +=== php_fpm.pool.start_since + +type: long + +Number of seconds since FPM has started. + + +[float] +=== php_fpm.pool.accepted_conn + +type: long + +The number of request accepted by the pool. + + +[float] +=== php_fpm.pool.listen_queue + +type: long + +The number of request in the queue of pending connections. + + +[float] +=== php_fpm.pool.max_listen_queue + +type: long + +The maximum number of requests in the queue of pending connections since FPM has started. + + +[float] +=== php_fpm.pool.listen_queue_len + +type: long + +The size of the socket queue of pending connections. + + +[float] +=== php_fpm.pool.idle_processes + +type: long + +The number of idle processes. + + +[float] +=== php_fpm.pool.active_processes + +type: long + +The number of active processes. + + +[float] +=== php_fpm.pool.total_processes + +type: long + +The number of idle + active processes. + + +[float] +=== php_fpm.pool.max_active_processes + +type: long + +The maximum number of active processes since FPM has started. + + +[float] +=== php_fpm.pool.max_children_reached + +type: long + +Number of times, the process limit has been reached, when pm tries to start more children (works only for pm `dynamic` and `ondemand`) + + +[float] +=== php_fpm.pool.slow_requests + +type: long + +Number of times a request execution time has exceeded `request_slowlog_timeout`. + + +[float] +== proc Fields + +`proc` contains the metrics that were obtained from a PHP process managed by the PHP-FPM process pool. + + + +[float] +=== php_fpm.proc.pid + +type: keyword + +The PID of the process. + + +[float] +=== php_fpm.proc.state + +type: keyword + +The state of the process (`Idle`, `Running`, ...). + + +[float] +=== php_fpm.proc.start_time + +type: long + +The date and time the process has started. + + +[float] +=== php_fpm.proc.start_since + +type: long + +Number of seconds since the process has started. + + +[float] +=== php_fpm.proc.requests + +type: long + +The number of requests the process has served. + + +[float] +=== php_fpm.proc.request_duration + +type: long + +The duration in microseconds of the requests. + + +[float] +=== php_fpm.proc.request_method + +type: keyword + +The request method (`GET`, `POST`, ...). + + +[float] +=== php_fpm.proc.request_uri + +type: keyword + +The request URI with the query string. + + +[float] +=== php_fpm.proc.content_length + +type: long + +The content length of the request (only with `POST`). + + +[float] +=== php_fpm.proc.user + +type: keyword + +The user (`PHP_AUTH_USER`) (or `-` if not set). + + +[float] +=== php_fpm.proc.script + +type: keyword + +The main script called (or `-` if not set). + + +[float] +=== php_fpm.proc.last_request_cpu + +type: float + +The %cpu the last request consumed it's always 0 if the process is not in `Idle` state because CPU calculation is done when the request processing has terminated. + + +[float] +=== php_fpm.proc.last_request_memory + +type: long + +The max amount of memory the last request consumed it's always 0 if the process is not in `Idle` state because memory calculation is done when the request processing has terminated. + + [[exported-fields-postgresql]] == PostgreSQL Fields diff --git a/metricbeat/docs/modules/php_fpm.asciidoc b/metricbeat/docs/modules/php_fpm.asciidoc new file mode 100644 index 00000000000..5c54610fed0 --- /dev/null +++ b/metricbeat/docs/modules/php_fpm.asciidoc @@ -0,0 +1,65 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-module-php_fpm]] +== PHP-FPM Module + +This module periodically fetches metrics from https://php-fpm.org[PHP-FPM] +servers. + +[float] +=== Module-Specific Configuration Notes + +You need to enable the PHP-FPM status page by properly configuring +`pm.status_path`. + +Here is a sample nginx configuration to forward requests to the PHP-FPM status +page (assuming `pm.status_path` is configured with default value `/status`): +```nginx +location ~ /status { + allow 127.0.0.1; + deny all; + include fastcgi_params; + fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; + fastcgi_pass 127.0.0.1:9000; +} +``` + +[float] +=== Compatibility + +The PHP-FPM metricsets were tested with PHP 5.6.29 and are expected to +work with all versions >= 5. + + +[float] +=== Example Configuration + +The php_fpm module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: php_fpm + metricsets: ["pool"] + enabled: true + period: 10s + hosts: ["localhost"] + +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +include::php_fpm/pool.asciidoc[] + +include::php_fpm/proc.asciidoc[] + diff --git a/metricbeat/docs/modules/php_fpm/pool.asciidoc b/metricbeat/docs/modules/php_fpm/pool.asciidoc new file mode 100644 index 00000000000..2c147e2e7e5 --- /dev/null +++ b/metricbeat/docs/modules/php_fpm/pool.asciidoc @@ -0,0 +1,19 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-php_fpm-pool]] +include::../../../module/php_fpm/pool/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/php_fpm/pool/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/php_fpm/proc.asciidoc b/metricbeat/docs/modules/php_fpm/proc.asciidoc new file mode 100644 index 00000000000..42d3f647294 --- /dev/null +++ b/metricbeat/docs/modules/php_fpm/proc.asciidoc @@ -0,0 +1,19 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-php_fpm-proc]] +include::../../../module/php_fpm/proc/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/php_fpm/proc/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 74b85359d7c..6d2b1b45333 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -10,6 +10,7 @@ This file is generated! See scripts/docs_collector.py * <> * <> * <> + * <> * <> * <> * <> @@ -27,6 +28,7 @@ include::modules/kafka.asciidoc[] include::modules/mongodb.asciidoc[] include::modules/mysql.asciidoc[] include::modules/nginx.asciidoc[] +include::modules/php_fpm.asciidoc[] include::modules/postgresql.asciidoc[] include::modules/prometheus.asciidoc[] include::modules/redis.asciidoc[] diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index 3d88f2937d8..cd789283ba0 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -34,6 +34,9 @@ import ( _ "github.com/elastic/beats/metricbeat/module/mysql/status" _ "github.com/elastic/beats/metricbeat/module/nginx" _ "github.com/elastic/beats/metricbeat/module/nginx/stubstatus" + _ "github.com/elastic/beats/metricbeat/module/php_fpm" + _ "github.com/elastic/beats/metricbeat/module/php_fpm/pool" + _ "github.com/elastic/beats/metricbeat/module/php_fpm/proc" _ "github.com/elastic/beats/metricbeat/module/postgresql" _ "github.com/elastic/beats/metricbeat/module/postgresql/activity" _ "github.com/elastic/beats/metricbeat/module/postgresql/bgwriter" diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml index 683008110ed..1368ce51b9f 100644 --- a/metricbeat/metricbeat.full.yml +++ b/metricbeat/metricbeat.full.yml @@ -200,6 +200,14 @@ metricbeat.modules: # Path to server status. Default server-status #server_status_path: "server-status" +#------------------------------- php_fpm Module ------------------------------ +- module: php_fpm + metricsets: ["pool"] + enabled: true + period: 10s + hosts: ["localhost"] + + #----------------------------- PostgreSQL Module ----------------------------- #- module: postgresql #metricsets: diff --git a/metricbeat/metricbeat.template-es2x.json b/metricbeat/metricbeat.template-es2x.json index f3d85fbb414..116fa2972a8 100644 --- a/metricbeat/metricbeat.template-es2x.json +++ b/metricbeat/metricbeat.template-es2x.json @@ -2168,6 +2168,115 @@ } } }, + "php_fpm": { + "properties": { + "pool": { + "properties": { + "accepted_conn": { + "type": "long" + }, + "active_processes": { + "type": "long" + }, + "idle_processes": { + "type": "long" + }, + "listen_queue": { + "type": "long" + }, + "listen_queue_len": { + "type": "long" + }, + "max_active_processes": { + "type": "long" + }, + "max_children_reached": { + "type": "long" + }, + "max_listen_queue": { + "type": "long" + }, + "pool": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "process_manager": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "slow_requests": { + "type": "long" + }, + "start_since": { + "type": "long" + }, + "start_time": { + "type": "long" + }, + "total_processes": { + "type": "long" + } + } + }, + "proc": { + "properties": { + "content_length": { + "type": "long" + }, + "last_request_cpu": { + "type": "float" + }, + "last_request_memory": { + "type": "long" + }, + "pid": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "request_duration": { + "type": "long" + }, + "request_method": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "request_uri": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "requests": { + "type": "long" + }, + "script": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "start_since": { + "type": "long" + }, + "start_time": { + "type": "long" + }, + "state": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "user": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + } + } + } + } + }, "postgresql": { "properties": { "activity": { diff --git a/metricbeat/metricbeat.template.json b/metricbeat/metricbeat.template.json index d42d1cd726d..63cfd3b5971 100644 --- a/metricbeat/metricbeat.template.json +++ b/metricbeat/metricbeat.template.json @@ -2147,6 +2147,107 @@ } } }, + "php_fpm": { + "properties": { + "pool": { + "properties": { + "accepted_conn": { + "type": "long" + }, + "active_processes": { + "type": "long" + }, + "idle_processes": { + "type": "long" + }, + "listen_queue": { + "type": "long" + }, + "listen_queue_len": { + "type": "long" + }, + "max_active_processes": { + "type": "long" + }, + "max_children_reached": { + "type": "long" + }, + "max_listen_queue": { + "type": "long" + }, + "pool": { + "ignore_above": 1024, + "type": "keyword" + }, + "process_manager": { + "ignore_above": 1024, + "type": "keyword" + }, + "slow_requests": { + "type": "long" + }, + "start_since": { + "type": "long" + }, + "start_time": { + "type": "long" + }, + "total_processes": { + "type": "long" + } + } + }, + "proc": { + "properties": { + "content_length": { + "type": "long" + }, + "last_request_cpu": { + "type": "float" + }, + "last_request_memory": { + "type": "long" + }, + "pid": { + "ignore_above": 1024, + "type": "keyword" + }, + "request_duration": { + "type": "long" + }, + "request_method": { + "ignore_above": 1024, + "type": "keyword" + }, + "request_uri": { + "ignore_above": 1024, + "type": "keyword" + }, + "requests": { + "type": "long" + }, + "script": { + "ignore_above": 1024, + "type": "keyword" + }, + "start_since": { + "type": "long" + }, + "start_time": { + "type": "long" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, "postgresql": { "properties": { "activity": { diff --git a/metricbeat/metricbeat.yml b/metricbeat/metricbeat.yml index 657ea61828d..bc901e9771f 100644 --- a/metricbeat/metricbeat.yml +++ b/metricbeat/metricbeat.yml @@ -46,6 +46,14 @@ metricbeat.modules: period: 10s processes: ['.*'] +#------------------------------- php_fpm Module ------------------------------ +- module: php_fpm + metricsets: ["pool"] + enabled: true + period: 10s + hosts: ["localhost"] + + #================================ General ===================================== diff --git a/metricbeat/module/php_fpm/_meta/Dockerfile b/metricbeat/module/php_fpm/_meta/Dockerfile new file mode 100644 index 00000000000..7e70cdb2b8a --- /dev/null +++ b/metricbeat/module/php_fpm/_meta/Dockerfile @@ -0,0 +1,4 @@ +FROM richarvey/nginx-php-fpm + +RUN echo "pm.status_path = /status" >> /etc/php7/php-fpm.d/www.conf +ADD ./php-fpm.conf /etc/nginx/sites-enabled diff --git a/metricbeat/module/php_fpm/_meta/config.yml b/metricbeat/module/php_fpm/_meta/config.yml new file mode 100644 index 00000000000..a4fc10b4e75 --- /dev/null +++ b/metricbeat/module/php_fpm/_meta/config.yml @@ -0,0 +1,6 @@ +- module: php_fpm + metricsets: ["pool"] + enabled: true + period: 10s + hosts: ["localhost"] + diff --git a/metricbeat/module/php_fpm/_meta/docs.asciidoc b/metricbeat/module/php_fpm/_meta/docs.asciidoc new file mode 100644 index 00000000000..4cd1272731a --- /dev/null +++ b/metricbeat/module/php_fpm/_meta/docs.asciidoc @@ -0,0 +1,28 @@ +== PHP-FPM Module + +This module periodically fetches metrics from https://php-fpm.org[PHP-FPM] +servers. + +[float] +=== Module-Specific Configuration Notes + +You need to enable the PHP-FPM status page by properly configuring +`pm.status_path`. + +Here is a sample nginx configuration to forward requests to the PHP-FPM status +page (assuming `pm.status_path` is configured with default value `/status`): +```nginx +location ~ /status { + allow 127.0.0.1; + deny all; + include fastcgi_params; + fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; + fastcgi_pass 127.0.0.1:9000; +} +``` + +[float] +=== Compatibility + +The PHP-FPM metricsets were tested with PHP 5.6.29 and are expected to +work with all versions >= 5. diff --git a/metricbeat/module/php_fpm/_meta/fields.yml b/metricbeat/module/php_fpm/_meta/fields.yml new file mode 100644 index 00000000000..7d16fa3ceb9 --- /dev/null +++ b/metricbeat/module/php_fpm/_meta/fields.yml @@ -0,0 +1,11 @@ +- key: php_fpm + title: "php_fpm" + description: > + PHP-FPM server status metrics collected from PHP-FPM. + fields: + - name: php_fpm + type: group + description: > + `php_fpm` contains the metrics that were obtained from PHP-FPM status + page call. + fields: diff --git a/metricbeat/module/php_fpm/_meta/php-fpm.conf b/metricbeat/module/php_fpm/_meta/php-fpm.conf new file mode 100644 index 00000000000..470698d5abd --- /dev/null +++ b/metricbeat/module/php_fpm/_meta/php-fpm.conf @@ -0,0 +1,16 @@ +server { + listen 81; ## listen for ipv4; this line is default and implied + listen [::]:81 default ipv6only=on; ## listen for ipv6 + + # Make site accessible from http://localhost/ + server_name _; + + error_log /dev/stdout info; + access_log /dev/stdout; + + location ~ /status { + fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; + fastcgi_pass unix:/var/run/php-fpm.sock; + include fastcgi_params; + } +} diff --git a/metricbeat/module/php_fpm/doc.go b/metricbeat/module/php_fpm/doc.go new file mode 100644 index 00000000000..3fed1f04cf7 --- /dev/null +++ b/metricbeat/module/php_fpm/doc.go @@ -0,0 +1,4 @@ +/* +Package php_fpm is a Metricbeat module that contains MetricSets. +*/ +package php_fpm diff --git a/metricbeat/module/php_fpm/php_fpm.go b/metricbeat/module/php_fpm/php_fpm.go new file mode 100644 index 00000000000..f9c25eb811d --- /dev/null +++ b/metricbeat/module/php_fpm/php_fpm.go @@ -0,0 +1,103 @@ +package php_fpm + +import ( + "fmt" + "io" + "net/http" + + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" +) + +const ( + defaultScheme = "http" + defaultPath = "/status" +) + +// HostParser is used for parsing the configured php-fpm hosts. +var HostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, +}.Build() + +// StatsClient provides access to php-fpm stats api +type StatsClient struct { + address string + user string + password string + http *http.Client +} + +// NewStatsClient creates a new StatsClient +func NewStatsClient(m mb.BaseMetricSet, isFullStats bool) *StatsClient { + var address string + address = m.HostData().SanitizedURI + "?json" + if isFullStats { + address += "&full" + } + return &StatsClient{ + address: address, + user: m.HostData().User, + password: m.HostData().Password, + http: &http.Client{Timeout: m.Module().Config().Timeout}, + } +} + +// Fetch php-fpm stats +func (c *StatsClient) Fetch() (io.ReadCloser, error) { + req, err := http.NewRequest("GET", c.address, nil) + if c.user != "" || c.password != "" { + req.SetBasicAuth(c.user, c.password) + } + resp, err := c.http.Do(req) + if err != nil { + return nil, fmt.Errorf("error making http request: %v", err) + } + + if resp.StatusCode != 200 { + return nil, fmt.Errorf("HTTP error %d: %s", resp.StatusCode, resp.Status) + } + + return resp.Body, nil +} + +// PoolStats defines all stats fields from a php-fpm pool +type PoolStats struct { + Pool string `json:"pool"` + ProcessManager string `json:"process manager"` + StartTime int `json:"start time"` + StartSince int `json:"start since"` + AcceptedConn int `json:"accepted conn"` + ListenQueue int `json:"listen queue"` + MaxListQueue int `json:"max list queue"` + ListenQueueLen int `json:"listen queue len"` + IdleProcesses int `json:"idle processes"` + ActiveProcesses int `json:"active processes"` + TotalProcesses int `json:"total processes"` + MaxActiveProcesses int `json:"max active processes"` + MaxChildrenReached int `json:"max children reached"` + SlowRequests int `json:"slow requests"` +} + +// ProcStats defines all stats fields from a process in php-fpm pool +type ProcStats struct { + Pid int `json:"pid"` + State string `json:"state"` + StartTime int `json:"start time"` + StartSince int `json:"start since"` + Requests int `json:"requests"` + RequestDuration int `json:"request duration"` + RequestMethod string `json:"request method"` + RequestURI string `json:"request uri"` + ContentLength int `json:"content length"` + User string `json:"user"` + Script string `json:"script"` + LastRequestCPU float64 `json:"last request cpu"` + LastRequestMemory int `json:"last request memory"` +} + +// FullStats defines all stats fields of the full stats api call (pool + processes) +type FullStats struct { + PoolStats + Processes []ProcStats `json:"processes"` +} diff --git a/metricbeat/module/php_fpm/php_fpm_test.go b/metricbeat/module/php_fpm/php_fpm_test.go new file mode 100644 index 00000000000..fd3e30dd323 --- /dev/null +++ b/metricbeat/module/php_fpm/php_fpm_test.go @@ -0,0 +1,30 @@ +package php_fpm + +import ( + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + + "github.com/stretchr/testify/assert" +) + +func TestHostParser(t *testing.T) { + tests := []struct { + host, expected string + }{ + {"localhost", "http://localhost/status"}, + {"localhost:123", "http://localhost:123/status"}, + {"http://localhost:123", "http://localhost:123/status"}, + } + + m := mbtest.NewTestModule(t, map[string]interface{}{}) + + for _, test := range tests { + hi, err := HostParser(m, test.host) + if err != nil { + t.Error("failed on", test.host, err) + continue + } + assert.Equal(t, test.expected, hi.URI) + } +} diff --git a/metricbeat/module/php_fpm/pool/_meta/data.json b/metricbeat/module/php_fpm/pool/_meta/data.json new file mode 100644 index 00000000000..b22be80d098 --- /dev/null +++ b/metricbeat/module/php_fpm/pool/_meta/data.json @@ -0,0 +1,33 @@ +{ + "@timestamp": "2017-01-18T23:57:23.960Z", + "beat": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "metricset": { + "host": "localhost:8081", + "module": "php_fpm", + "name": "pool", + "rtt": 1237 + }, + "php_fpm": { + "pool": { + "accepted_conn": 510, + "active_processes": 1, + "hostname": "localhost:8081", + "idle_processes": 2, + "listen_queue": 0, + "listen_queue_len": 0, + "max_active_processes": 2, + "max_children_reached": 0, + "max_list_queue": 0, + "pool": "www", + "process_manager": "dynamic", + "slow_requests": 0, + "start_since": 36785, + "start_time": 1484747058, + "total_processes": 3 + } + }, + "type": "metricsets" +} diff --git a/metricbeat/module/php_fpm/pool/_meta/docs.asciidoc b/metricbeat/module/php_fpm/pool/_meta/docs.asciidoc new file mode 100644 index 00000000000..0e158655e08 --- /dev/null +++ b/metricbeat/module/php_fpm/pool/_meta/docs.asciidoc @@ -0,0 +1,3 @@ +=== php_fpm pool MetricSet + +This is the pool metricset of the module php_fpm. diff --git a/metricbeat/module/php_fpm/pool/_meta/fields.yml b/metricbeat/module/php_fpm/pool/_meta/fields.yml new file mode 100644 index 00000000000..9276c79edb9 --- /dev/null +++ b/metricbeat/module/php_fpm/pool/_meta/fields.yml @@ -0,0 +1,78 @@ +- name: pool + type: group + description: > + `pool` contains the metrics that were obtained from the PHP-FPM process + pool. + fields: + - name: pool + type: keyword + description: > + The name of the pool. + + - name: process_manager + type: keyword + description: > + `static`, `dynamic` or `ondemand`. + + - name: start_time + type: long + description: > + The date and time FPM has started. + + - name: start_since + type: long + description: > + Number of seconds since FPM has started. + + - name: accepted_conn + type: long + description: > + The number of request accepted by the pool. + + - name: listen_queue + type: long + description: > + The number of request in the queue of pending connections. + + - name: max_listen_queue + type: long + description: > + The maximum number of requests in the queue of pending connections + since FPM has started. + + - name: listen_queue_len + type: long + description: > + The size of the socket queue of pending connections. + + - name: idle_processes + type: long + description: > + The number of idle processes. + + - name: active_processes + type: long + description: > + The number of active processes. + + - name: total_processes + type: long + description: > + The number of idle + active processes. + + - name: max_active_processes + type: long + description: > + The maximum number of active processes since FPM has started. + + - name: max_children_reached + type: long + description: > + Number of times, the process limit has been reached, when pm tries to + start more children (works only for pm `dynamic` and `ondemand`) + + - name: slow_requests + type: long + description: > + Number of times a request execution time has exceeded + `request_slowlog_timeout`. diff --git a/metricbeat/module/php_fpm/pool/pool.go b/metricbeat/module/php_fpm/pool/pool.go new file mode 100644 index 00000000000..6bb7e8576e3 --- /dev/null +++ b/metricbeat/module/php_fpm/pool/pool.go @@ -0,0 +1,83 @@ +package pool + +import ( + "encoding/json" + "fmt" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/mb" + + "github.com/elastic/beats/metricbeat/module/php_fpm" +) + +// init registers the MetricSet with the central registry. +// The New method will be called after the setup of the module and before starting to fetch data +func init() { + if err := mb.Registry.AddMetricSet("php_fpm", "pool", New, php_fpm.HostParser); err != nil { + panic(err) + } +} + +// MetricSet type defines all fields of the MetricSet +// As a minimum it must inherit the mb.BaseMetricSet fields, but can be extended with +// additional entries. These variables can be used to persist data or configuration between +// multiple fetch calls. +type MetricSet struct { + mb.BaseMetricSet + client *php_fpm.StatsClient // StatsClient that is reused across requests. +} + +// New create a new instance of the MetricSet +// Part of new is also setting up the configuration by processing additional +// configuration entries if needed. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + + config := struct{}{} + + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + return &MetricSet{ + BaseMetricSet: base, + client: php_fpm.NewStatsClient(base, false), + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right format +// It returns the event which is then forward to the output. In case of an error, a +// descriptive error must be returned. +func (m *MetricSet) Fetch() (common.MapStr, error) { + body, err := m.client.Fetch() + + if err != nil { + return nil, err + } + + defer body.Close() + + stats := &php_fpm.PoolStats{} + err = json.NewDecoder(body).Decode(stats) + if err != nil { + return nil, fmt.Errorf("error parsing json: %v", err) + } + + return common.MapStr{ + "hostname": m.Host(), + + "pool": stats.Pool, + "process_manager": stats.ProcessManager, + "start_time": stats.StartTime, + "start_since": stats.StartSince, + "accepted_conn": stats.AcceptedConn, + "listen_queue": stats.ListenQueue, + "max_list_queue": stats.MaxListQueue, + "listen_queue_len": stats.ListenQueueLen, + "idle_processes": stats.IdleProcesses, + "active_processes": stats.ActiveProcesses, + "total_processes": stats.TotalProcesses, + "max_active_processes": stats.MaxActiveProcesses, + "max_children_reached": stats.MaxChildrenReached, + "slow_requests": stats.SlowRequests, + }, nil +} diff --git a/metricbeat/module/php_fpm/pool/pool_integration_test.go b/metricbeat/module/php_fpm/pool/pool_integration_test.go new file mode 100644 index 00000000000..b5026cfa7d2 --- /dev/null +++ b/metricbeat/module/php_fpm/pool/pool_integration_test.go @@ -0,0 +1,25 @@ +// +build integration + +package pool + +import ( + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + f := mbtest.NewEventFetcher(t, getConfig()) + err := mbtest.WriteEvent(f, t) + if err != nil { + t.Fatal("write", err) + } +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "php_fpm", + "metricsets": []string{"pool"}, + "hosts": []string{"127.0.0.1:81"}, + } +} diff --git a/metricbeat/module/php_fpm/proc/_meta/data.json b/metricbeat/module/php_fpm/proc/_meta/data.json new file mode 100644 index 00000000000..b17127f6bd7 --- /dev/null +++ b/metricbeat/module/php_fpm/proc/_meta/data.json @@ -0,0 +1,32 @@ +{ + "@timestamp": "2017-01-18T23:57:20.965Z", + "beat": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "metricset": { + "host": "localhost:8081", + "module": "php_fpm", + "name": "proc", + "rtt": 1303 + }, + "php_fpm": { + "proc": { + "content_length": 0, + "hostname": "localhost:8081", + "last_request_cpu": 0.000000, + "last_request_memory": 262144, + "pid": "56414", + "request_duration": 573, + "request_method": "GET", + "request_uri": "/status?json\u0026full", + "requests": 170, + "script": "-", + "start_since": 36782, + "start_time": 1484747058, + "state": "Idle", + "user": "-" + } + }, + "type": "metricsets" +} diff --git a/metricbeat/module/php_fpm/proc/_meta/docs.asciidoc b/metricbeat/module/php_fpm/proc/_meta/docs.asciidoc new file mode 100644 index 00000000000..50680ed10bf --- /dev/null +++ b/metricbeat/module/php_fpm/proc/_meta/docs.asciidoc @@ -0,0 +1,3 @@ +=== php_fpm proc MetricSet + +This is the proc metricset of the module php_fpm. diff --git a/metricbeat/module/php_fpm/proc/_meta/fields.yml b/metricbeat/module/php_fpm/proc/_meta/fields.yml new file mode 100644 index 00000000000..47624797edf --- /dev/null +++ b/metricbeat/module/php_fpm/proc/_meta/fields.yml @@ -0,0 +1,74 @@ +- name: proc + type: group + description: > + `proc` contains the metrics that were obtained from a PHP process managed by + the PHP-FPM process pool. + fields: + - name: pid + type: keyword + description: > + The PID of the process. + + - name: state + type: keyword + description: > + The state of the process (`Idle`, `Running`, ...). + + - name: start_time + type: long + description: > + The date and time the process has started. + + - name: start_since + type: long + description: > + Number of seconds since the process has started. + + - name: requests + type: long + description: > + The number of requests the process has served. + + - name: request_duration + type: long + description: > + The duration in microseconds of the requests. + + - name: request_method + type: keyword + description: > + The request method (`GET`, `POST`, ...). + + - name: request_uri + type: keyword + description: > + The request URI with the query string. + + - name: content_length + type: long + description: > + The content length of the request (only with `POST`). + + - name: user + type: keyword + description: > + The user (`PHP_AUTH_USER`) (or `-` if not set). + + - name: script + type: keyword + description: > + The main script called (or `-` if not set). + + - name: last_request_cpu + type: float + description: > + The %cpu the last request consumed it's always 0 if the process is not + in `Idle` state because CPU calculation is done when the request + processing has terminated. + + - name: last_request_memory + type: long + description: > + The max amount of memory the last request consumed it's always 0 if the + process is not in `Idle` state because memory calculation is done when the + request processing has terminated. diff --git a/metricbeat/module/php_fpm/proc/proc.go b/metricbeat/module/php_fpm/proc/proc.go new file mode 100644 index 00000000000..96e345b7091 --- /dev/null +++ b/metricbeat/module/php_fpm/proc/proc.go @@ -0,0 +1,88 @@ +package proc + +import ( + "encoding/json" + "fmt" + "strconv" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/mb" + + "github.com/elastic/beats/metricbeat/module/php_fpm" +) + +// init registers the MetricSet with the central registry. +// The New method will be called after the setup of the module and before starting to fetch data +func init() { + if err := mb.Registry.AddMetricSet("php_fpm", "proc", New, php_fpm.HostParser); err != nil { + panic(err) + } +} + +// MetricSet type defines all fields of the MetricSet +// As a minimum it must inherit the mb.BaseMetricSet fields, but can be extended with +// additional entries. These variables can be used to persist data or configuration between +// multiple fetch calls. +type MetricSet struct { + mb.BaseMetricSet + client *php_fpm.StatsClient // StatsClient that is reused across requests. +} + +// New create a new instance of the MetricSet +// Part of new is also setting up the configuration by processing additional +// configuration entries if needed. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + + config := struct{}{} + + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + return &MetricSet{ + BaseMetricSet: base, + client: php_fpm.NewStatsClient(base, true), + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right format +// It returns the event which is then forward to the output. In case of an error, a +// descriptive error must be returned. +func (m *MetricSet) Fetch() ([]common.MapStr, error) { + body, err := m.client.Fetch() + + if err != nil { + return nil, err + } + + defer body.Close() + + stats := &php_fpm.FullStats{} + err = json.NewDecoder(body).Decode(stats) + if err != nil { + return nil, fmt.Errorf("error parsing json: %v", err) + } + + events := []common.MapStr{} + for _, proc := range stats.Processes { + events = append(events, common.MapStr{ + "hostname": m.Host(), + + "pid": strconv.Itoa(proc.Pid), + "state": proc.State, + "start_time": proc.StartTime, + "start_since": proc.StartSince, + "requests": proc.Requests, + "request_duration": proc.RequestDuration, + "request_method": proc.RequestMethod, + "request_uri": proc.RequestURI, + "content_length": proc.ContentLength, + "user": proc.User, + "script": proc.Script, + "last_request_cpu": proc.LastRequestCPU, + "last_request_memory": proc.LastRequestMemory, + }) + } + + return events, nil +} diff --git a/metricbeat/module/php_fpm/proc/proc_integration_test.go b/metricbeat/module/php_fpm/proc/proc_integration_test.go new file mode 100644 index 00000000000..3ca61b584a4 --- /dev/null +++ b/metricbeat/module/php_fpm/proc/proc_integration_test.go @@ -0,0 +1,25 @@ +// +build integration + +package proc + +import ( + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + f := mbtest.NewEventsFetcher(t, getConfig()) + err := mbtest.WriteEvents(f, t) + if err != nil { + t.Fatal("write", err) + } +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "php_fpm", + "metricsets": []string{"proc"}, + "hosts": []string{"127.0.0.1:81"}, + } +}