From 539f2290a55ba48cdcfee0550805976c7b8339d1 Mon Sep 17 00:00:00 2001 From: Thiago Souza Date: Wed, 18 Jan 2017 18:39:04 -0200 Subject: [PATCH 01/10] Metricbeat php-fpm module Add pool start time and start since stats Add support for proc stats Refactor stats api call Fix listen_queue => listen_queue_len" Revert "Fix listen_queue => listen_queue_len"" This reverts commit 46ff489fa8cce8d73c2aba42758a51bde4d68a77. Add suport for "listen_queue_len" Add documentation Minor const refactor Add generated doc Add basic testing Fix illegal char in asciidoc Fix type Update generated files Fix package name Add integration tests --- metricbeat/_meta/beat.full.yml | 8 + metricbeat/_meta/beat.yml | 8 + metricbeat/docs/fields.asciidoc | 245 ++++++++++++++++++ metricbeat/docs/modules/php_fpm.asciidoc | 65 +++++ metricbeat/docs/modules/php_fpm/pool.asciidoc | 19 ++ metricbeat/docs/modules/php_fpm/proc.asciidoc | 19 ++ metricbeat/docs/modules_list.asciidoc | 2 + metricbeat/include/list.go | 3 + metricbeat/metricbeat.full.yml | 8 + metricbeat/metricbeat.template-es2x.json | 109 ++++++++ metricbeat/metricbeat.template.json | 101 ++++++++ metricbeat/metricbeat.yml | 8 + metricbeat/module/php_fpm/_meta/Dockerfile | 4 + metricbeat/module/php_fpm/_meta/config.yml | 6 + metricbeat/module/php_fpm/_meta/docs.asciidoc | 28 ++ metricbeat/module/php_fpm/_meta/fields.yml | 11 + metricbeat/module/php_fpm/_meta/php-fpm.conf | 16 ++ metricbeat/module/php_fpm/doc.go | 4 + metricbeat/module/php_fpm/php_fpm.go | 103 ++++++++ metricbeat/module/php_fpm/php_fpm_test.go | 30 +++ .../module/php_fpm/pool/_meta/data.json | 33 +++ .../module/php_fpm/pool/_meta/docs.asciidoc | 3 + .../module/php_fpm/pool/_meta/fields.yml | 78 ++++++ metricbeat/module/php_fpm/pool/pool.go | 83 ++++++ .../php_fpm/pool/pool_integration_test.go | 25 ++ .../module/php_fpm/proc/_meta/data.json | 32 +++ .../module/php_fpm/proc/_meta/docs.asciidoc | 3 + .../module/php_fpm/proc/_meta/fields.yml | 74 ++++++ metricbeat/module/php_fpm/proc/proc.go | 88 +++++++ .../php_fpm/proc/proc_integration_test.go | 25 ++ 30 files changed, 1241 insertions(+) create mode 100644 metricbeat/docs/modules/php_fpm.asciidoc create mode 100644 metricbeat/docs/modules/php_fpm/pool.asciidoc create mode 100644 metricbeat/docs/modules/php_fpm/proc.asciidoc create mode 100644 metricbeat/module/php_fpm/_meta/Dockerfile create mode 100644 metricbeat/module/php_fpm/_meta/config.yml create mode 100644 metricbeat/module/php_fpm/_meta/docs.asciidoc create mode 100644 metricbeat/module/php_fpm/_meta/fields.yml create mode 100644 metricbeat/module/php_fpm/_meta/php-fpm.conf create mode 100644 metricbeat/module/php_fpm/doc.go create mode 100644 metricbeat/module/php_fpm/php_fpm.go create mode 100644 metricbeat/module/php_fpm/php_fpm_test.go create mode 100644 metricbeat/module/php_fpm/pool/_meta/data.json create mode 100644 metricbeat/module/php_fpm/pool/_meta/docs.asciidoc create mode 100644 metricbeat/module/php_fpm/pool/_meta/fields.yml create mode 100644 metricbeat/module/php_fpm/pool/pool.go create mode 100644 metricbeat/module/php_fpm/pool/pool_integration_test.go create mode 100644 metricbeat/module/php_fpm/proc/_meta/data.json create mode 100644 metricbeat/module/php_fpm/proc/_meta/docs.asciidoc create mode 100644 metricbeat/module/php_fpm/proc/_meta/fields.yml create mode 100644 metricbeat/module/php_fpm/proc/proc.go create mode 100644 metricbeat/module/php_fpm/proc/proc_integration_test.go diff --git a/metricbeat/_meta/beat.full.yml b/metricbeat/_meta/beat.full.yml index f4065b153c0..15da7fbaac4 100644 --- a/metricbeat/_meta/beat.full.yml +++ b/metricbeat/_meta/beat.full.yml @@ -200,6 +200,14 @@ metricbeat.modules: # Path to server status. Default server-status #server_status_path: "server-status" +#------------------------------- php_fpm Module ------------------------------ +- module: php_fpm + metricsets: ["pool"] + enabled: true + period: 10s + hosts: ["localhost"] + + #----------------------------- PostgreSQL Module ----------------------------- #- module: postgresql #metricsets: diff --git a/metricbeat/_meta/beat.yml b/metricbeat/_meta/beat.yml index 6ab098828ce..e3ff4a38fdf 100644 --- a/metricbeat/_meta/beat.yml +++ b/metricbeat/_meta/beat.yml @@ -46,4 +46,12 @@ metricbeat.modules: period: 10s processes: ['.*'] +#------------------------------- php_fpm Module ------------------------------ +- module: php_fpm + metricsets: ["pool"] + enabled: true + period: 10s + hosts: ["localhost"] + + diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 1afc271c34d..87e2bdeb3eb 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -23,6 +23,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -3802,6 +3803,250 @@ type: long The current number of idle client connections waiting for a request. +[[exported-fields-php_fpm]] +== php_fpm Fields + +PHP-FPM server status metrics collected from PHP-FPM. + + + +[float] +== php_fpm Fields + +`php_fpm` contains the metrics that were obtained from PHP-FPM status page call. + + + +[float] +== pool Fields + +`pool` contains the metrics that were obtained from the PHP-FPM process pool. + + + +[float] +=== php_fpm.pool.pool + +type: keyword + +The name of the pool. + + +[float] +=== php_fpm.pool.process_manager + +type: keyword + +`static`, `dynamic` or `ondemand`. + + +[float] +=== php_fpm.pool.start_time + +type: long + +The date and time FPM has started. + + +[float] +=== php_fpm.pool.start_since + +type: long + +Number of seconds since FPM has started. + + +[float] +=== php_fpm.pool.accepted_conn + +type: long + +The number of request accepted by the pool. + + +[float] +=== php_fpm.pool.listen_queue + +type: long + +The number of request in the queue of pending connections. + + +[float] +=== php_fpm.pool.max_listen_queue + +type: long + +The maximum number of requests in the queue of pending connections since FPM has started. + + +[float] +=== php_fpm.pool.listen_queue_len + +type: long + +The size of the socket queue of pending connections. + + +[float] +=== php_fpm.pool.idle_processes + +type: long + +The number of idle processes. + + +[float] +=== php_fpm.pool.active_processes + +type: long + +The number of active processes. + + +[float] +=== php_fpm.pool.total_processes + +type: long + +The number of idle + active processes. + + +[float] +=== php_fpm.pool.max_active_processes + +type: long + +The maximum number of active processes since FPM has started. + + +[float] +=== php_fpm.pool.max_children_reached + +type: long + +Number of times, the process limit has been reached, when pm tries to start more children (works only for pm `dynamic` and `ondemand`) + + +[float] +=== php_fpm.pool.slow_requests + +type: long + +Number of times a request execution time has exceeded `request_slowlog_timeout`. + + +[float] +== proc Fields + +`proc` contains the metrics that were obtained from a PHP process managed by the PHP-FPM process pool. + + + +[float] +=== php_fpm.proc.pid + +type: keyword + +The PID of the process. + + +[float] +=== php_fpm.proc.state + +type: keyword + +The state of the process (`Idle`, `Running`, ...). + + +[float] +=== php_fpm.proc.start_time + +type: long + +The date and time the process has started. + + +[float] +=== php_fpm.proc.start_since + +type: long + +Number of seconds since the process has started. + + +[float] +=== php_fpm.proc.requests + +type: long + +The number of requests the process has served. + + +[float] +=== php_fpm.proc.request_duration + +type: long + +The duration in microseconds of the requests. + + +[float] +=== php_fpm.proc.request_method + +type: keyword + +The request method (`GET`, `POST`, ...). + + +[float] +=== php_fpm.proc.request_uri + +type: keyword + +The request URI with the query string. + + +[float] +=== php_fpm.proc.content_length + +type: long + +The content length of the request (only with `POST`). + + +[float] +=== php_fpm.proc.user + +type: keyword + +The user (`PHP_AUTH_USER`) (or `-` if not set). + + +[float] +=== php_fpm.proc.script + +type: keyword + +The main script called (or `-` if not set). + + +[float] +=== php_fpm.proc.last_request_cpu + +type: float + +The %cpu the last request consumed it's always 0 if the process is not in `Idle` state because CPU calculation is done when the request processing has terminated. + + +[float] +=== php_fpm.proc.last_request_memory + +type: long + +The max amount of memory the last request consumed it's always 0 if the process is not in `Idle` state because memory calculation is done when the request processing has terminated. + + [[exported-fields-postgresql]] == PostgreSQL Fields diff --git a/metricbeat/docs/modules/php_fpm.asciidoc b/metricbeat/docs/modules/php_fpm.asciidoc new file mode 100644 index 00000000000..5c54610fed0 --- /dev/null +++ b/metricbeat/docs/modules/php_fpm.asciidoc @@ -0,0 +1,65 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-module-php_fpm]] +== PHP-FPM Module + +This module periodically fetches metrics from https://php-fpm.org[PHP-FPM] +servers. + +[float] +=== Module-Specific Configuration Notes + +You need to enable the PHP-FPM status page by properly configuring +`pm.status_path`. + +Here is a sample nginx configuration to forward requests to the PHP-FPM status +page (assuming `pm.status_path` is configured with default value `/status`): +```nginx +location ~ /status { + allow 127.0.0.1; + deny all; + include fastcgi_params; + fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; + fastcgi_pass 127.0.0.1:9000; +} +``` + +[float] +=== Compatibility + +The PHP-FPM metricsets were tested with PHP 5.6.29 and are expected to +work with all versions >= 5. + + +[float] +=== Example Configuration + +The php_fpm module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: php_fpm + metricsets: ["pool"] + enabled: true + period: 10s + hosts: ["localhost"] + +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +include::php_fpm/pool.asciidoc[] + +include::php_fpm/proc.asciidoc[] + diff --git a/metricbeat/docs/modules/php_fpm/pool.asciidoc b/metricbeat/docs/modules/php_fpm/pool.asciidoc new file mode 100644 index 00000000000..2c147e2e7e5 --- /dev/null +++ b/metricbeat/docs/modules/php_fpm/pool.asciidoc @@ -0,0 +1,19 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-php_fpm-pool]] +include::../../../module/php_fpm/pool/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/php_fpm/pool/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/php_fpm/proc.asciidoc b/metricbeat/docs/modules/php_fpm/proc.asciidoc new file mode 100644 index 00000000000..42d3f647294 --- /dev/null +++ b/metricbeat/docs/modules/php_fpm/proc.asciidoc @@ -0,0 +1,19 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-php_fpm-proc]] +include::../../../module/php_fpm/proc/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/php_fpm/proc/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 74b85359d7c..6d2b1b45333 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -10,6 +10,7 @@ This file is generated! See scripts/docs_collector.py * <> * <> * <> + * <> * <> * <> * <> @@ -27,6 +28,7 @@ include::modules/kafka.asciidoc[] include::modules/mongodb.asciidoc[] include::modules/mysql.asciidoc[] include::modules/nginx.asciidoc[] +include::modules/php_fpm.asciidoc[] include::modules/postgresql.asciidoc[] include::modules/prometheus.asciidoc[] include::modules/redis.asciidoc[] diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index 3d88f2937d8..cd789283ba0 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -34,6 +34,9 @@ import ( _ "github.com/elastic/beats/metricbeat/module/mysql/status" _ "github.com/elastic/beats/metricbeat/module/nginx" _ "github.com/elastic/beats/metricbeat/module/nginx/stubstatus" + _ "github.com/elastic/beats/metricbeat/module/php_fpm" + _ "github.com/elastic/beats/metricbeat/module/php_fpm/pool" + _ "github.com/elastic/beats/metricbeat/module/php_fpm/proc" _ "github.com/elastic/beats/metricbeat/module/postgresql" _ "github.com/elastic/beats/metricbeat/module/postgresql/activity" _ "github.com/elastic/beats/metricbeat/module/postgresql/bgwriter" diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml index 683008110ed..1368ce51b9f 100644 --- a/metricbeat/metricbeat.full.yml +++ b/metricbeat/metricbeat.full.yml @@ -200,6 +200,14 @@ metricbeat.modules: # Path to server status. Default server-status #server_status_path: "server-status" +#------------------------------- php_fpm Module ------------------------------ +- module: php_fpm + metricsets: ["pool"] + enabled: true + period: 10s + hosts: ["localhost"] + + #----------------------------- PostgreSQL Module ----------------------------- #- module: postgresql #metricsets: diff --git a/metricbeat/metricbeat.template-es2x.json b/metricbeat/metricbeat.template-es2x.json index f3d85fbb414..116fa2972a8 100644 --- a/metricbeat/metricbeat.template-es2x.json +++ b/metricbeat/metricbeat.template-es2x.json @@ -2168,6 +2168,115 @@ } } }, + "php_fpm": { + "properties": { + "pool": { + "properties": { + "accepted_conn": { + "type": "long" + }, + "active_processes": { + "type": "long" + }, + "idle_processes": { + "type": "long" + }, + "listen_queue": { + "type": "long" + }, + "listen_queue_len": { + "type": "long" + }, + "max_active_processes": { + "type": "long" + }, + "max_children_reached": { + "type": "long" + }, + "max_listen_queue": { + "type": "long" + }, + "pool": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "process_manager": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "slow_requests": { + "type": "long" + }, + "start_since": { + "type": "long" + }, + "start_time": { + "type": "long" + }, + "total_processes": { + "type": "long" + } + } + }, + "proc": { + "properties": { + "content_length": { + "type": "long" + }, + "last_request_cpu": { + "type": "float" + }, + "last_request_memory": { + "type": "long" + }, + "pid": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "request_duration": { + "type": "long" + }, + "request_method": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "request_uri": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "requests": { + "type": "long" + }, + "script": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "start_since": { + "type": "long" + }, + "start_time": { + "type": "long" + }, + "state": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "user": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + } + } + } + } + }, "postgresql": { "properties": { "activity": { diff --git a/metricbeat/metricbeat.template.json b/metricbeat/metricbeat.template.json index d42d1cd726d..63cfd3b5971 100644 --- a/metricbeat/metricbeat.template.json +++ b/metricbeat/metricbeat.template.json @@ -2147,6 +2147,107 @@ } } }, + "php_fpm": { + "properties": { + "pool": { + "properties": { + "accepted_conn": { + "type": "long" + }, + "active_processes": { + "type": "long" + }, + "idle_processes": { + "type": "long" + }, + "listen_queue": { + "type": "long" + }, + "listen_queue_len": { + "type": "long" + }, + "max_active_processes": { + "type": "long" + }, + "max_children_reached": { + "type": "long" + }, + "max_listen_queue": { + "type": "long" + }, + "pool": { + "ignore_above": 1024, + "type": "keyword" + }, + "process_manager": { + "ignore_above": 1024, + "type": "keyword" + }, + "slow_requests": { + "type": "long" + }, + "start_since": { + "type": "long" + }, + "start_time": { + "type": "long" + }, + "total_processes": { + "type": "long" + } + } + }, + "proc": { + "properties": { + "content_length": { + "type": "long" + }, + "last_request_cpu": { + "type": "float" + }, + "last_request_memory": { + "type": "long" + }, + "pid": { + "ignore_above": 1024, + "type": "keyword" + }, + "request_duration": { + "type": "long" + }, + "request_method": { + "ignore_above": 1024, + "type": "keyword" + }, + "request_uri": { + "ignore_above": 1024, + "type": "keyword" + }, + "requests": { + "type": "long" + }, + "script": { + "ignore_above": 1024, + "type": "keyword" + }, + "start_since": { + "type": "long" + }, + "start_time": { + "type": "long" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, "postgresql": { "properties": { "activity": { diff --git a/metricbeat/metricbeat.yml b/metricbeat/metricbeat.yml index 657ea61828d..bc901e9771f 100644 --- a/metricbeat/metricbeat.yml +++ b/metricbeat/metricbeat.yml @@ -46,6 +46,14 @@ metricbeat.modules: period: 10s processes: ['.*'] +#------------------------------- php_fpm Module ------------------------------ +- module: php_fpm + metricsets: ["pool"] + enabled: true + period: 10s + hosts: ["localhost"] + + #================================ General ===================================== diff --git a/metricbeat/module/php_fpm/_meta/Dockerfile b/metricbeat/module/php_fpm/_meta/Dockerfile new file mode 100644 index 00000000000..7e70cdb2b8a --- /dev/null +++ b/metricbeat/module/php_fpm/_meta/Dockerfile @@ -0,0 +1,4 @@ +FROM richarvey/nginx-php-fpm + +RUN echo "pm.status_path = /status" >> /etc/php7/php-fpm.d/www.conf +ADD ./php-fpm.conf /etc/nginx/sites-enabled diff --git a/metricbeat/module/php_fpm/_meta/config.yml b/metricbeat/module/php_fpm/_meta/config.yml new file mode 100644 index 00000000000..a4fc10b4e75 --- /dev/null +++ b/metricbeat/module/php_fpm/_meta/config.yml @@ -0,0 +1,6 @@ +- module: php_fpm + metricsets: ["pool"] + enabled: true + period: 10s + hosts: ["localhost"] + diff --git a/metricbeat/module/php_fpm/_meta/docs.asciidoc b/metricbeat/module/php_fpm/_meta/docs.asciidoc new file mode 100644 index 00000000000..4cd1272731a --- /dev/null +++ b/metricbeat/module/php_fpm/_meta/docs.asciidoc @@ -0,0 +1,28 @@ +== PHP-FPM Module + +This module periodically fetches metrics from https://php-fpm.org[PHP-FPM] +servers. + +[float] +=== Module-Specific Configuration Notes + +You need to enable the PHP-FPM status page by properly configuring +`pm.status_path`. + +Here is a sample nginx configuration to forward requests to the PHP-FPM status +page (assuming `pm.status_path` is configured with default value `/status`): +```nginx +location ~ /status { + allow 127.0.0.1; + deny all; + include fastcgi_params; + fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; + fastcgi_pass 127.0.0.1:9000; +} +``` + +[float] +=== Compatibility + +The PHP-FPM metricsets were tested with PHP 5.6.29 and are expected to +work with all versions >= 5. diff --git a/metricbeat/module/php_fpm/_meta/fields.yml b/metricbeat/module/php_fpm/_meta/fields.yml new file mode 100644 index 00000000000..7d16fa3ceb9 --- /dev/null +++ b/metricbeat/module/php_fpm/_meta/fields.yml @@ -0,0 +1,11 @@ +- key: php_fpm + title: "php_fpm" + description: > + PHP-FPM server status metrics collected from PHP-FPM. + fields: + - name: php_fpm + type: group + description: > + `php_fpm` contains the metrics that were obtained from PHP-FPM status + page call. + fields: diff --git a/metricbeat/module/php_fpm/_meta/php-fpm.conf b/metricbeat/module/php_fpm/_meta/php-fpm.conf new file mode 100644 index 00000000000..470698d5abd --- /dev/null +++ b/metricbeat/module/php_fpm/_meta/php-fpm.conf @@ -0,0 +1,16 @@ +server { + listen 81; ## listen for ipv4; this line is default and implied + listen [::]:81 default ipv6only=on; ## listen for ipv6 + + # Make site accessible from http://localhost/ + server_name _; + + error_log /dev/stdout info; + access_log /dev/stdout; + + location ~ /status { + fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; + fastcgi_pass unix:/var/run/php-fpm.sock; + include fastcgi_params; + } +} diff --git a/metricbeat/module/php_fpm/doc.go b/metricbeat/module/php_fpm/doc.go new file mode 100644 index 00000000000..3fed1f04cf7 --- /dev/null +++ b/metricbeat/module/php_fpm/doc.go @@ -0,0 +1,4 @@ +/* +Package php_fpm is a Metricbeat module that contains MetricSets. +*/ +package php_fpm diff --git a/metricbeat/module/php_fpm/php_fpm.go b/metricbeat/module/php_fpm/php_fpm.go new file mode 100644 index 00000000000..f9c25eb811d --- /dev/null +++ b/metricbeat/module/php_fpm/php_fpm.go @@ -0,0 +1,103 @@ +package php_fpm + +import ( + "fmt" + "io" + "net/http" + + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" +) + +const ( + defaultScheme = "http" + defaultPath = "/status" +) + +// HostParser is used for parsing the configured php-fpm hosts. +var HostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, +}.Build() + +// StatsClient provides access to php-fpm stats api +type StatsClient struct { + address string + user string + password string + http *http.Client +} + +// NewStatsClient creates a new StatsClient +func NewStatsClient(m mb.BaseMetricSet, isFullStats bool) *StatsClient { + var address string + address = m.HostData().SanitizedURI + "?json" + if isFullStats { + address += "&full" + } + return &StatsClient{ + address: address, + user: m.HostData().User, + password: m.HostData().Password, + http: &http.Client{Timeout: m.Module().Config().Timeout}, + } +} + +// Fetch php-fpm stats +func (c *StatsClient) Fetch() (io.ReadCloser, error) { + req, err := http.NewRequest("GET", c.address, nil) + if c.user != "" || c.password != "" { + req.SetBasicAuth(c.user, c.password) + } + resp, err := c.http.Do(req) + if err != nil { + return nil, fmt.Errorf("error making http request: %v", err) + } + + if resp.StatusCode != 200 { + return nil, fmt.Errorf("HTTP error %d: %s", resp.StatusCode, resp.Status) + } + + return resp.Body, nil +} + +// PoolStats defines all stats fields from a php-fpm pool +type PoolStats struct { + Pool string `json:"pool"` + ProcessManager string `json:"process manager"` + StartTime int `json:"start time"` + StartSince int `json:"start since"` + AcceptedConn int `json:"accepted conn"` + ListenQueue int `json:"listen queue"` + MaxListQueue int `json:"max list queue"` + ListenQueueLen int `json:"listen queue len"` + IdleProcesses int `json:"idle processes"` + ActiveProcesses int `json:"active processes"` + TotalProcesses int `json:"total processes"` + MaxActiveProcesses int `json:"max active processes"` + MaxChildrenReached int `json:"max children reached"` + SlowRequests int `json:"slow requests"` +} + +// ProcStats defines all stats fields from a process in php-fpm pool +type ProcStats struct { + Pid int `json:"pid"` + State string `json:"state"` + StartTime int `json:"start time"` + StartSince int `json:"start since"` + Requests int `json:"requests"` + RequestDuration int `json:"request duration"` + RequestMethod string `json:"request method"` + RequestURI string `json:"request uri"` + ContentLength int `json:"content length"` + User string `json:"user"` + Script string `json:"script"` + LastRequestCPU float64 `json:"last request cpu"` + LastRequestMemory int `json:"last request memory"` +} + +// FullStats defines all stats fields of the full stats api call (pool + processes) +type FullStats struct { + PoolStats + Processes []ProcStats `json:"processes"` +} diff --git a/metricbeat/module/php_fpm/php_fpm_test.go b/metricbeat/module/php_fpm/php_fpm_test.go new file mode 100644 index 00000000000..fd3e30dd323 --- /dev/null +++ b/metricbeat/module/php_fpm/php_fpm_test.go @@ -0,0 +1,30 @@ +package php_fpm + +import ( + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + + "github.com/stretchr/testify/assert" +) + +func TestHostParser(t *testing.T) { + tests := []struct { + host, expected string + }{ + {"localhost", "http://localhost/status"}, + {"localhost:123", "http://localhost:123/status"}, + {"http://localhost:123", "http://localhost:123/status"}, + } + + m := mbtest.NewTestModule(t, map[string]interface{}{}) + + for _, test := range tests { + hi, err := HostParser(m, test.host) + if err != nil { + t.Error("failed on", test.host, err) + continue + } + assert.Equal(t, test.expected, hi.URI) + } +} diff --git a/metricbeat/module/php_fpm/pool/_meta/data.json b/metricbeat/module/php_fpm/pool/_meta/data.json new file mode 100644 index 00000000000..b22be80d098 --- /dev/null +++ b/metricbeat/module/php_fpm/pool/_meta/data.json @@ -0,0 +1,33 @@ +{ + "@timestamp": "2017-01-18T23:57:23.960Z", + "beat": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "metricset": { + "host": "localhost:8081", + "module": "php_fpm", + "name": "pool", + "rtt": 1237 + }, + "php_fpm": { + "pool": { + "accepted_conn": 510, + "active_processes": 1, + "hostname": "localhost:8081", + "idle_processes": 2, + "listen_queue": 0, + "listen_queue_len": 0, + "max_active_processes": 2, + "max_children_reached": 0, + "max_list_queue": 0, + "pool": "www", + "process_manager": "dynamic", + "slow_requests": 0, + "start_since": 36785, + "start_time": 1484747058, + "total_processes": 3 + } + }, + "type": "metricsets" +} diff --git a/metricbeat/module/php_fpm/pool/_meta/docs.asciidoc b/metricbeat/module/php_fpm/pool/_meta/docs.asciidoc new file mode 100644 index 00000000000..0e158655e08 --- /dev/null +++ b/metricbeat/module/php_fpm/pool/_meta/docs.asciidoc @@ -0,0 +1,3 @@ +=== php_fpm pool MetricSet + +This is the pool metricset of the module php_fpm. diff --git a/metricbeat/module/php_fpm/pool/_meta/fields.yml b/metricbeat/module/php_fpm/pool/_meta/fields.yml new file mode 100644 index 00000000000..9276c79edb9 --- /dev/null +++ b/metricbeat/module/php_fpm/pool/_meta/fields.yml @@ -0,0 +1,78 @@ +- name: pool + type: group + description: > + `pool` contains the metrics that were obtained from the PHP-FPM process + pool. + fields: + - name: pool + type: keyword + description: > + The name of the pool. + + - name: process_manager + type: keyword + description: > + `static`, `dynamic` or `ondemand`. + + - name: start_time + type: long + description: > + The date and time FPM has started. + + - name: start_since + type: long + description: > + Number of seconds since FPM has started. + + - name: accepted_conn + type: long + description: > + The number of request accepted by the pool. + + - name: listen_queue + type: long + description: > + The number of request in the queue of pending connections. + + - name: max_listen_queue + type: long + description: > + The maximum number of requests in the queue of pending connections + since FPM has started. + + - name: listen_queue_len + type: long + description: > + The size of the socket queue of pending connections. + + - name: idle_processes + type: long + description: > + The number of idle processes. + + - name: active_processes + type: long + description: > + The number of active processes. + + - name: total_processes + type: long + description: > + The number of idle + active processes. + + - name: max_active_processes + type: long + description: > + The maximum number of active processes since FPM has started. + + - name: max_children_reached + type: long + description: > + Number of times, the process limit has been reached, when pm tries to + start more children (works only for pm `dynamic` and `ondemand`) + + - name: slow_requests + type: long + description: > + Number of times a request execution time has exceeded + `request_slowlog_timeout`. diff --git a/metricbeat/module/php_fpm/pool/pool.go b/metricbeat/module/php_fpm/pool/pool.go new file mode 100644 index 00000000000..6bb7e8576e3 --- /dev/null +++ b/metricbeat/module/php_fpm/pool/pool.go @@ -0,0 +1,83 @@ +package pool + +import ( + "encoding/json" + "fmt" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/mb" + + "github.com/elastic/beats/metricbeat/module/php_fpm" +) + +// init registers the MetricSet with the central registry. +// The New method will be called after the setup of the module and before starting to fetch data +func init() { + if err := mb.Registry.AddMetricSet("php_fpm", "pool", New, php_fpm.HostParser); err != nil { + panic(err) + } +} + +// MetricSet type defines all fields of the MetricSet +// As a minimum it must inherit the mb.BaseMetricSet fields, but can be extended with +// additional entries. These variables can be used to persist data or configuration between +// multiple fetch calls. +type MetricSet struct { + mb.BaseMetricSet + client *php_fpm.StatsClient // StatsClient that is reused across requests. +} + +// New create a new instance of the MetricSet +// Part of new is also setting up the configuration by processing additional +// configuration entries if needed. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + + config := struct{}{} + + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + return &MetricSet{ + BaseMetricSet: base, + client: php_fpm.NewStatsClient(base, false), + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right format +// It returns the event which is then forward to the output. In case of an error, a +// descriptive error must be returned. +func (m *MetricSet) Fetch() (common.MapStr, error) { + body, err := m.client.Fetch() + + if err != nil { + return nil, err + } + + defer body.Close() + + stats := &php_fpm.PoolStats{} + err = json.NewDecoder(body).Decode(stats) + if err != nil { + return nil, fmt.Errorf("error parsing json: %v", err) + } + + return common.MapStr{ + "hostname": m.Host(), + + "pool": stats.Pool, + "process_manager": stats.ProcessManager, + "start_time": stats.StartTime, + "start_since": stats.StartSince, + "accepted_conn": stats.AcceptedConn, + "listen_queue": stats.ListenQueue, + "max_list_queue": stats.MaxListQueue, + "listen_queue_len": stats.ListenQueueLen, + "idle_processes": stats.IdleProcesses, + "active_processes": stats.ActiveProcesses, + "total_processes": stats.TotalProcesses, + "max_active_processes": stats.MaxActiveProcesses, + "max_children_reached": stats.MaxChildrenReached, + "slow_requests": stats.SlowRequests, + }, nil +} diff --git a/metricbeat/module/php_fpm/pool/pool_integration_test.go b/metricbeat/module/php_fpm/pool/pool_integration_test.go new file mode 100644 index 00000000000..b5026cfa7d2 --- /dev/null +++ b/metricbeat/module/php_fpm/pool/pool_integration_test.go @@ -0,0 +1,25 @@ +// +build integration + +package pool + +import ( + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + f := mbtest.NewEventFetcher(t, getConfig()) + err := mbtest.WriteEvent(f, t) + if err != nil { + t.Fatal("write", err) + } +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "php_fpm", + "metricsets": []string{"pool"}, + "hosts": []string{"127.0.0.1:81"}, + } +} diff --git a/metricbeat/module/php_fpm/proc/_meta/data.json b/metricbeat/module/php_fpm/proc/_meta/data.json new file mode 100644 index 00000000000..b17127f6bd7 --- /dev/null +++ b/metricbeat/module/php_fpm/proc/_meta/data.json @@ -0,0 +1,32 @@ +{ + "@timestamp": "2017-01-18T23:57:20.965Z", + "beat": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "metricset": { + "host": "localhost:8081", + "module": "php_fpm", + "name": "proc", + "rtt": 1303 + }, + "php_fpm": { + "proc": { + "content_length": 0, + "hostname": "localhost:8081", + "last_request_cpu": 0.000000, + "last_request_memory": 262144, + "pid": "56414", + "request_duration": 573, + "request_method": "GET", + "request_uri": "/status?json\u0026full", + "requests": 170, + "script": "-", + "start_since": 36782, + "start_time": 1484747058, + "state": "Idle", + "user": "-" + } + }, + "type": "metricsets" +} diff --git a/metricbeat/module/php_fpm/proc/_meta/docs.asciidoc b/metricbeat/module/php_fpm/proc/_meta/docs.asciidoc new file mode 100644 index 00000000000..50680ed10bf --- /dev/null +++ b/metricbeat/module/php_fpm/proc/_meta/docs.asciidoc @@ -0,0 +1,3 @@ +=== php_fpm proc MetricSet + +This is the proc metricset of the module php_fpm. diff --git a/metricbeat/module/php_fpm/proc/_meta/fields.yml b/metricbeat/module/php_fpm/proc/_meta/fields.yml new file mode 100644 index 00000000000..47624797edf --- /dev/null +++ b/metricbeat/module/php_fpm/proc/_meta/fields.yml @@ -0,0 +1,74 @@ +- name: proc + type: group + description: > + `proc` contains the metrics that were obtained from a PHP process managed by + the PHP-FPM process pool. + fields: + - name: pid + type: keyword + description: > + The PID of the process. + + - name: state + type: keyword + description: > + The state of the process (`Idle`, `Running`, ...). + + - name: start_time + type: long + description: > + The date and time the process has started. + + - name: start_since + type: long + description: > + Number of seconds since the process has started. + + - name: requests + type: long + description: > + The number of requests the process has served. + + - name: request_duration + type: long + description: > + The duration in microseconds of the requests. + + - name: request_method + type: keyword + description: > + The request method (`GET`, `POST`, ...). + + - name: request_uri + type: keyword + description: > + The request URI with the query string. + + - name: content_length + type: long + description: > + The content length of the request (only with `POST`). + + - name: user + type: keyword + description: > + The user (`PHP_AUTH_USER`) (or `-` if not set). + + - name: script + type: keyword + description: > + The main script called (or `-` if not set). + + - name: last_request_cpu + type: float + description: > + The %cpu the last request consumed it's always 0 if the process is not + in `Idle` state because CPU calculation is done when the request + processing has terminated. + + - name: last_request_memory + type: long + description: > + The max amount of memory the last request consumed it's always 0 if the + process is not in `Idle` state because memory calculation is done when the + request processing has terminated. diff --git a/metricbeat/module/php_fpm/proc/proc.go b/metricbeat/module/php_fpm/proc/proc.go new file mode 100644 index 00000000000..96e345b7091 --- /dev/null +++ b/metricbeat/module/php_fpm/proc/proc.go @@ -0,0 +1,88 @@ +package proc + +import ( + "encoding/json" + "fmt" + "strconv" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/mb" + + "github.com/elastic/beats/metricbeat/module/php_fpm" +) + +// init registers the MetricSet with the central registry. +// The New method will be called after the setup of the module and before starting to fetch data +func init() { + if err := mb.Registry.AddMetricSet("php_fpm", "proc", New, php_fpm.HostParser); err != nil { + panic(err) + } +} + +// MetricSet type defines all fields of the MetricSet +// As a minimum it must inherit the mb.BaseMetricSet fields, but can be extended with +// additional entries. These variables can be used to persist data or configuration between +// multiple fetch calls. +type MetricSet struct { + mb.BaseMetricSet + client *php_fpm.StatsClient // StatsClient that is reused across requests. +} + +// New create a new instance of the MetricSet +// Part of new is also setting up the configuration by processing additional +// configuration entries if needed. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + + config := struct{}{} + + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + return &MetricSet{ + BaseMetricSet: base, + client: php_fpm.NewStatsClient(base, true), + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right format +// It returns the event which is then forward to the output. In case of an error, a +// descriptive error must be returned. +func (m *MetricSet) Fetch() ([]common.MapStr, error) { + body, err := m.client.Fetch() + + if err != nil { + return nil, err + } + + defer body.Close() + + stats := &php_fpm.FullStats{} + err = json.NewDecoder(body).Decode(stats) + if err != nil { + return nil, fmt.Errorf("error parsing json: %v", err) + } + + events := []common.MapStr{} + for _, proc := range stats.Processes { + events = append(events, common.MapStr{ + "hostname": m.Host(), + + "pid": strconv.Itoa(proc.Pid), + "state": proc.State, + "start_time": proc.StartTime, + "start_since": proc.StartSince, + "requests": proc.Requests, + "request_duration": proc.RequestDuration, + "request_method": proc.RequestMethod, + "request_uri": proc.RequestURI, + "content_length": proc.ContentLength, + "user": proc.User, + "script": proc.Script, + "last_request_cpu": proc.LastRequestCPU, + "last_request_memory": proc.LastRequestMemory, + }) + } + + return events, nil +} diff --git a/metricbeat/module/php_fpm/proc/proc_integration_test.go b/metricbeat/module/php_fpm/proc/proc_integration_test.go new file mode 100644 index 00000000000..3ca61b584a4 --- /dev/null +++ b/metricbeat/module/php_fpm/proc/proc_integration_test.go @@ -0,0 +1,25 @@ +// +build integration + +package proc + +import ( + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + f := mbtest.NewEventsFetcher(t, getConfig()) + err := mbtest.WriteEvents(f, t) + if err != nil { + t.Fatal("write", err) + } +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "php_fpm", + "metricsets": []string{"proc"}, + "hosts": []string{"127.0.0.1:81"}, + } +} From 70478068ed15aaef86754f5a6f4069825f8dd15c Mon Sep 17 00:00:00 2001 From: Thiago Souza Date: Sat, 21 Jan 2017 14:08:05 -0200 Subject: [PATCH 02/10] PR review: refactor data structures --- metricbeat/module/php_fpm/php_fpm.go | 42 +------------------------- metricbeat/module/php_fpm/pool/data.go | 18 +++++++++++ metricbeat/module/php_fpm/pool/pool.go | 11 ++----- metricbeat/module/php_fpm/proc/data.go | 21 +++++++++++++ metricbeat/module/php_fpm/proc/proc.go | 11 ++----- 5 files changed, 46 insertions(+), 57 deletions(-) create mode 100644 metricbeat/module/php_fpm/pool/data.go create mode 100644 metricbeat/module/php_fpm/proc/data.go diff --git a/metricbeat/module/php_fpm/php_fpm.go b/metricbeat/module/php_fpm/php_fpm.go index f9c25eb811d..e8beb767205 100644 --- a/metricbeat/module/php_fpm/php_fpm.go +++ b/metricbeat/module/php_fpm/php_fpm.go @@ -18,6 +18,7 @@ const ( var HostParser = parse.URLHostParserBuilder{ DefaultScheme: defaultScheme, DefaultPath: defaultPath, + PathConfigKey: "status_path", }.Build() // StatsClient provides access to php-fpm stats api @@ -60,44 +61,3 @@ func (c *StatsClient) Fetch() (io.ReadCloser, error) { return resp.Body, nil } - -// PoolStats defines all stats fields from a php-fpm pool -type PoolStats struct { - Pool string `json:"pool"` - ProcessManager string `json:"process manager"` - StartTime int `json:"start time"` - StartSince int `json:"start since"` - AcceptedConn int `json:"accepted conn"` - ListenQueue int `json:"listen queue"` - MaxListQueue int `json:"max list queue"` - ListenQueueLen int `json:"listen queue len"` - IdleProcesses int `json:"idle processes"` - ActiveProcesses int `json:"active processes"` - TotalProcesses int `json:"total processes"` - MaxActiveProcesses int `json:"max active processes"` - MaxChildrenReached int `json:"max children reached"` - SlowRequests int `json:"slow requests"` -} - -// ProcStats defines all stats fields from a process in php-fpm pool -type ProcStats struct { - Pid int `json:"pid"` - State string `json:"state"` - StartTime int `json:"start time"` - StartSince int `json:"start since"` - Requests int `json:"requests"` - RequestDuration int `json:"request duration"` - RequestMethod string `json:"request method"` - RequestURI string `json:"request uri"` - ContentLength int `json:"content length"` - User string `json:"user"` - Script string `json:"script"` - LastRequestCPU float64 `json:"last request cpu"` - LastRequestMemory int `json:"last request memory"` -} - -// FullStats defines all stats fields of the full stats api call (pool + processes) -type FullStats struct { - PoolStats - Processes []ProcStats `json:"processes"` -} diff --git a/metricbeat/module/php_fpm/pool/data.go b/metricbeat/module/php_fpm/pool/data.go new file mode 100644 index 00000000000..babc06c36cf --- /dev/null +++ b/metricbeat/module/php_fpm/pool/data.go @@ -0,0 +1,18 @@ +package pool + +type poolStats struct { + Pool string `json:"pool"` + ProcessManager string `json:"process manager"` + StartTime int `json:"start time"` + StartSince int `json:"start since"` + AcceptedConn int `json:"accepted conn"` + ListenQueue int `json:"listen queue"` + MaxListQueue int `json:"max list queue"` + ListenQueueLen int `json:"listen queue len"` + IdleProcesses int `json:"idle processes"` + ActiveProcesses int `json:"active processes"` + TotalProcesses int `json:"total processes"` + MaxActiveProcesses int `json:"max active processes"` + MaxChildrenReached int `json:"max children reached"` + SlowRequests int `json:"slow requests"` +} diff --git a/metricbeat/module/php_fpm/pool/pool.go b/metricbeat/module/php_fpm/pool/pool.go index 6bb7e8576e3..40b3c0f7cb7 100644 --- a/metricbeat/module/php_fpm/pool/pool.go +++ b/metricbeat/module/php_fpm/pool/pool.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/module/php_fpm" @@ -31,13 +32,7 @@ type MetricSet struct { // Part of new is also setting up the configuration by processing additional // configuration entries if needed. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - - config := struct{}{} - - if err := base.Module().UnpackConfig(&config); err != nil { - return nil, err - } - + logp.Warn("EXPERIMENTAL: The php-fpm pool metricset is experimental") return &MetricSet{ BaseMetricSet: base, client: php_fpm.NewStatsClient(base, false), @@ -56,7 +51,7 @@ func (m *MetricSet) Fetch() (common.MapStr, error) { defer body.Close() - stats := &php_fpm.PoolStats{} + stats := &poolStats{} err = json.NewDecoder(body).Decode(stats) if err != nil { return nil, fmt.Errorf("error parsing json: %v", err) diff --git a/metricbeat/module/php_fpm/proc/data.go b/metricbeat/module/php_fpm/proc/data.go new file mode 100644 index 00000000000..7a7e072e916 --- /dev/null +++ b/metricbeat/module/php_fpm/proc/data.go @@ -0,0 +1,21 @@ +package proc + +type procStats struct { + Pid int `json:"pid"` + State string `json:"state"` + StartTime int `json:"start time"` + StartSince int `json:"start since"` + Requests int `json:"requests"` + RequestDuration int `json:"request duration"` + RequestMethod string `json:"request method"` + RequestURI string `json:"request uri"` + ContentLength int `json:"content length"` + User string `json:"user"` + Script string `json:"script"` + LastRequestCPU float64 `json:"last request cpu"` + LastRequestMemory int `json:"last request memory"` +} + +type fullStats struct { + Processes []procStats `json:"processes"` +} diff --git a/metricbeat/module/php_fpm/proc/proc.go b/metricbeat/module/php_fpm/proc/proc.go index 96e345b7091..379118703ab 100644 --- a/metricbeat/module/php_fpm/proc/proc.go +++ b/metricbeat/module/php_fpm/proc/proc.go @@ -6,6 +6,7 @@ import ( "strconv" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/module/php_fpm" @@ -32,13 +33,7 @@ type MetricSet struct { // Part of new is also setting up the configuration by processing additional // configuration entries if needed. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - - config := struct{}{} - - if err := base.Module().UnpackConfig(&config); err != nil { - return nil, err - } - + logp.Warn("EXPERIMENTAL: The php-fpm proc metricset is experimental") return &MetricSet{ BaseMetricSet: base, client: php_fpm.NewStatsClient(base, true), @@ -57,7 +52,7 @@ func (m *MetricSet) Fetch() ([]common.MapStr, error) { defer body.Close() - stats := &php_fpm.FullStats{} + stats := &fullStats{} err = json.NewDecoder(body).Decode(stats) if err != nil { return nil, fmt.Errorf("error parsing json: %v", err) From 4f87ead2448356e04c6820bb5f8869f05a2ef353 Mon Sep 17 00:00:00 2001 From: Thiago Souza Date: Sat, 21 Jan 2017 14:08:31 -0200 Subject: [PATCH 03/10] PR review: minor changes --- metricbeat/_meta/beat.full.yml | 12 ++++++------ metricbeat/_meta/beat.yml | 8 -------- metricbeat/docs/fields.asciidoc | 1 + metricbeat/docs/modules/php_fpm.asciidoc | 12 ++++++------ metricbeat/metricbeat.full.yml | 12 ++++++------ metricbeat/metricbeat.yml | 8 -------- metricbeat/module/php_fpm/_meta/config.yml | 12 ++++++------ metricbeat/module/php_fpm/_meta/fields.yml | 3 +++ 8 files changed, 28 insertions(+), 40 deletions(-) diff --git a/metricbeat/_meta/beat.full.yml b/metricbeat/_meta/beat.full.yml index 15da7fbaac4..e93bebd8bdf 100644 --- a/metricbeat/_meta/beat.full.yml +++ b/metricbeat/_meta/beat.full.yml @@ -201,12 +201,12 @@ metricbeat.modules: #server_status_path: "server-status" #------------------------------- php_fpm Module ------------------------------ -- module: php_fpm - metricsets: ["pool"] - enabled: true - period: 10s - hosts: ["localhost"] - +#- module: php_fpm + #metricsets: ["pool"] + #enabled: true + #period: 10s + #status_path: "/status" + #hosts: ["localhost:8080"] #----------------------------- PostgreSQL Module ----------------------------- #- module: postgresql diff --git a/metricbeat/_meta/beat.yml b/metricbeat/_meta/beat.yml index e3ff4a38fdf..6ab098828ce 100644 --- a/metricbeat/_meta/beat.yml +++ b/metricbeat/_meta/beat.yml @@ -46,12 +46,4 @@ metricbeat.modules: period: 10s processes: ['.*'] -#------------------------------- php_fpm Module ------------------------------ -- module: php_fpm - metricsets: ["pool"] - enabled: true - period: 10s - hosts: ["localhost"] - - diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 87e2bdeb3eb..49e7d9682ec 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -3807,6 +3807,7 @@ The current number of idle client connections waiting for a request. == php_fpm Fields PHP-FPM server status metrics collected from PHP-FPM. +experimental[] diff --git a/metricbeat/docs/modules/php_fpm.asciidoc b/metricbeat/docs/modules/php_fpm.asciidoc index 5c54610fed0..783604941c0 100644 --- a/metricbeat/docs/modules/php_fpm.asciidoc +++ b/metricbeat/docs/modules/php_fpm.asciidoc @@ -42,12 +42,12 @@ in <>. Here is an example configuration: [source,yaml] ---- metricbeat.modules: -- module: php_fpm - metricsets: ["pool"] - enabled: true - period: 10s - hosts: ["localhost"] - +#- module: php_fpm + #metricsets: ["pool"] + #enabled: true + #period: 10s + #status_path: "/status" + #hosts: ["localhost:8080"] ---- [float] diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml index 1368ce51b9f..ede25a1c8ed 100644 --- a/metricbeat/metricbeat.full.yml +++ b/metricbeat/metricbeat.full.yml @@ -201,12 +201,12 @@ metricbeat.modules: #server_status_path: "server-status" #------------------------------- php_fpm Module ------------------------------ -- module: php_fpm - metricsets: ["pool"] - enabled: true - period: 10s - hosts: ["localhost"] - +#- module: php_fpm + #metricsets: ["pool"] + #enabled: true + #period: 10s + #status_path: "/status" + #hosts: ["localhost:8080"] #----------------------------- PostgreSQL Module ----------------------------- #- module: postgresql diff --git a/metricbeat/metricbeat.yml b/metricbeat/metricbeat.yml index bc901e9771f..657ea61828d 100644 --- a/metricbeat/metricbeat.yml +++ b/metricbeat/metricbeat.yml @@ -46,14 +46,6 @@ metricbeat.modules: period: 10s processes: ['.*'] -#------------------------------- php_fpm Module ------------------------------ -- module: php_fpm - metricsets: ["pool"] - enabled: true - period: 10s - hosts: ["localhost"] - - #================================ General ===================================== diff --git a/metricbeat/module/php_fpm/_meta/config.yml b/metricbeat/module/php_fpm/_meta/config.yml index a4fc10b4e75..89527a864c4 100644 --- a/metricbeat/module/php_fpm/_meta/config.yml +++ b/metricbeat/module/php_fpm/_meta/config.yml @@ -1,6 +1,6 @@ -- module: php_fpm - metricsets: ["pool"] - enabled: true - period: 10s - hosts: ["localhost"] - +#- module: php_fpm + #metricsets: ["pool"] + #enabled: true + #period: 10s + #status_path: "/status" + #hosts: ["localhost:8080"] diff --git a/metricbeat/module/php_fpm/_meta/fields.yml b/metricbeat/module/php_fpm/_meta/fields.yml index 7d16fa3ceb9..a8a8ac8ea36 100644 --- a/metricbeat/module/php_fpm/_meta/fields.yml +++ b/metricbeat/module/php_fpm/_meta/fields.yml @@ -2,6 +2,9 @@ title: "php_fpm" description: > PHP-FPM server status metrics collected from PHP-FPM. + + experimental[] + short_config: false fields: - name: php_fpm type: group From 3de7963169a09c077f53e91139c175159d0b1c40 Mon Sep 17 00:00:00 2001 From: Thiago Souza Date: Sat, 21 Jan 2017 16:42:06 -0200 Subject: [PATCH 04/10] PR review: use HostParser.QueryString --- metricbeat/module/php_fpm/php_fpm.go | 3 ++- metricbeat/module/php_fpm/php_fpm_test.go | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/metricbeat/module/php_fpm/php_fpm.go b/metricbeat/module/php_fpm/php_fpm.go index e8beb767205..7df51200f68 100644 --- a/metricbeat/module/php_fpm/php_fpm.go +++ b/metricbeat/module/php_fpm/php_fpm.go @@ -18,6 +18,7 @@ const ( var HostParser = parse.URLHostParserBuilder{ DefaultScheme: defaultScheme, DefaultPath: defaultPath, + QueryParams: "json", PathConfigKey: "status_path", }.Build() @@ -32,7 +33,7 @@ type StatsClient struct { // NewStatsClient creates a new StatsClient func NewStatsClient(m mb.BaseMetricSet, isFullStats bool) *StatsClient { var address string - address = m.HostData().SanitizedURI + "?json" + address = m.HostData().SanitizedURI if isFullStats { address += "&full" } diff --git a/metricbeat/module/php_fpm/php_fpm_test.go b/metricbeat/module/php_fpm/php_fpm_test.go index fd3e30dd323..e5e6fe1342d 100644 --- a/metricbeat/module/php_fpm/php_fpm_test.go +++ b/metricbeat/module/php_fpm/php_fpm_test.go @@ -12,9 +12,9 @@ func TestHostParser(t *testing.T) { tests := []struct { host, expected string }{ - {"localhost", "http://localhost/status"}, - {"localhost:123", "http://localhost:123/status"}, - {"http://localhost:123", "http://localhost:123/status"}, + {"localhost", "http://localhost/status?json="}, + {"localhost:123", "http://localhost:123/status?json="}, + {"http://localhost:123", "http://localhost:123/status?json="}, } m := mbtest.NewTestModule(t, map[string]interface{}{}) From a36196f7c0c7af0014081032a6d5778dd0c28622 Mon Sep 17 00:00:00 2001 From: Thiago Souza Date: Tue, 24 Jan 2017 01:25:04 -0200 Subject: [PATCH 05/10] PR review: no need for "var address" --- metricbeat/module/php_fpm/php_fpm.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/metricbeat/module/php_fpm/php_fpm.go b/metricbeat/module/php_fpm/php_fpm.go index 7df51200f68..d4f5791d980 100644 --- a/metricbeat/module/php_fpm/php_fpm.go +++ b/metricbeat/module/php_fpm/php_fpm.go @@ -32,8 +32,7 @@ type StatsClient struct { // NewStatsClient creates a new StatsClient func NewStatsClient(m mb.BaseMetricSet, isFullStats bool) *StatsClient { - var address string - address = m.HostData().SanitizedURI + address := m.HostData().SanitizedURI if isFullStats { address += "&full" } From 60bba3ada7c84afce7c8fe4ae2c11b80e65002f1 Mon Sep 17 00:00:00 2001 From: Thiago Souza Date: Tue, 24 Jan 2017 01:28:05 -0200 Subject: [PATCH 06/10] PR review: remove proc metricset (metrics does not seems really useful) --- metricbeat/docs/fields.asciidoc | 111 ------------------ metricbeat/docs/modules/php_fpm.asciidoc | 4 - metricbeat/docs/modules/php_fpm/proc.asciidoc | 19 --- metricbeat/include/list.go | 1 - metricbeat/metricbeat.template-es2x.json | 55 --------- metricbeat/metricbeat.template.json | 49 -------- .../module/php_fpm/proc/_meta/data.json | 32 ----- .../module/php_fpm/proc/_meta/docs.asciidoc | 3 - .../module/php_fpm/proc/_meta/fields.yml | 74 ------------ metricbeat/module/php_fpm/proc/data.go | 21 ---- metricbeat/module/php_fpm/proc/proc.go | 83 ------------- .../php_fpm/proc/proc_integration_test.go | 25 ---- 12 files changed, 477 deletions(-) delete mode 100644 metricbeat/docs/modules/php_fpm/proc.asciidoc delete mode 100644 metricbeat/module/php_fpm/proc/_meta/data.json delete mode 100644 metricbeat/module/php_fpm/proc/_meta/docs.asciidoc delete mode 100644 metricbeat/module/php_fpm/proc/_meta/fields.yml delete mode 100644 metricbeat/module/php_fpm/proc/data.go delete mode 100644 metricbeat/module/php_fpm/proc/proc.go delete mode 100644 metricbeat/module/php_fpm/proc/proc_integration_test.go diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 49e7d9682ec..dd4ae7e1e96 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -3937,117 +3937,6 @@ type: long Number of times a request execution time has exceeded `request_slowlog_timeout`. -[float] -== proc Fields - -`proc` contains the metrics that were obtained from a PHP process managed by the PHP-FPM process pool. - - - -[float] -=== php_fpm.proc.pid - -type: keyword - -The PID of the process. - - -[float] -=== php_fpm.proc.state - -type: keyword - -The state of the process (`Idle`, `Running`, ...). - - -[float] -=== php_fpm.proc.start_time - -type: long - -The date and time the process has started. - - -[float] -=== php_fpm.proc.start_since - -type: long - -Number of seconds since the process has started. - - -[float] -=== php_fpm.proc.requests - -type: long - -The number of requests the process has served. - - -[float] -=== php_fpm.proc.request_duration - -type: long - -The duration in microseconds of the requests. - - -[float] -=== php_fpm.proc.request_method - -type: keyword - -The request method (`GET`, `POST`, ...). - - -[float] -=== php_fpm.proc.request_uri - -type: keyword - -The request URI with the query string. - - -[float] -=== php_fpm.proc.content_length - -type: long - -The content length of the request (only with `POST`). - - -[float] -=== php_fpm.proc.user - -type: keyword - -The user (`PHP_AUTH_USER`) (or `-` if not set). - - -[float] -=== php_fpm.proc.script - -type: keyword - -The main script called (or `-` if not set). - - -[float] -=== php_fpm.proc.last_request_cpu - -type: float - -The %cpu the last request consumed it's always 0 if the process is not in `Idle` state because CPU calculation is done when the request processing has terminated. - - -[float] -=== php_fpm.proc.last_request_memory - -type: long - -The max amount of memory the last request consumed it's always 0 if the process is not in `Idle` state because memory calculation is done when the request processing has terminated. - - [[exported-fields-postgresql]] == PostgreSQL Fields diff --git a/metricbeat/docs/modules/php_fpm.asciidoc b/metricbeat/docs/modules/php_fpm.asciidoc index 783604941c0..2440b1935f0 100644 --- a/metricbeat/docs/modules/php_fpm.asciidoc +++ b/metricbeat/docs/modules/php_fpm.asciidoc @@ -57,9 +57,5 @@ The following metricsets are available: * <> -* <> - include::php_fpm/pool.asciidoc[] -include::php_fpm/proc.asciidoc[] - diff --git a/metricbeat/docs/modules/php_fpm/proc.asciidoc b/metricbeat/docs/modules/php_fpm/proc.asciidoc deleted file mode 100644 index 42d3f647294..00000000000 --- a/metricbeat/docs/modules/php_fpm/proc.asciidoc +++ /dev/null @@ -1,19 +0,0 @@ -//// -This file is generated! See scripts/docs_collector.py -//// - -[[metricbeat-metricset-php_fpm-proc]] -include::../../../module/php_fpm/proc/_meta/docs.asciidoc[] - - -==== Fields - -For a description of each field in the metricset, see the -<> section. - -Here is an example document generated by this metricset: - -[source,json] ----- -include::../../../module/php_fpm/proc/_meta/data.json[] ----- diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index cd789283ba0..aec19892987 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -36,7 +36,6 @@ import ( _ "github.com/elastic/beats/metricbeat/module/nginx/stubstatus" _ "github.com/elastic/beats/metricbeat/module/php_fpm" _ "github.com/elastic/beats/metricbeat/module/php_fpm/pool" - _ "github.com/elastic/beats/metricbeat/module/php_fpm/proc" _ "github.com/elastic/beats/metricbeat/module/postgresql" _ "github.com/elastic/beats/metricbeat/module/postgresql/activity" _ "github.com/elastic/beats/metricbeat/module/postgresql/bgwriter" diff --git a/metricbeat/metricbeat.template-es2x.json b/metricbeat/metricbeat.template-es2x.json index 116fa2972a8..9072c9a79e3 100644 --- a/metricbeat/metricbeat.template-es2x.json +++ b/metricbeat/metricbeat.template-es2x.json @@ -2219,61 +2219,6 @@ "type": "long" } } - }, - "proc": { - "properties": { - "content_length": { - "type": "long" - }, - "last_request_cpu": { - "type": "float" - }, - "last_request_memory": { - "type": "long" - }, - "pid": { - "ignore_above": 1024, - "index": "not_analyzed", - "type": "string" - }, - "request_duration": { - "type": "long" - }, - "request_method": { - "ignore_above": 1024, - "index": "not_analyzed", - "type": "string" - }, - "request_uri": { - "ignore_above": 1024, - "index": "not_analyzed", - "type": "string" - }, - "requests": { - "type": "long" - }, - "script": { - "ignore_above": 1024, - "index": "not_analyzed", - "type": "string" - }, - "start_since": { - "type": "long" - }, - "start_time": { - "type": "long" - }, - "state": { - "ignore_above": 1024, - "index": "not_analyzed", - "type": "string" - }, - "user": { - "ignore_above": 1024, - "index": "not_analyzed", - "type": "string" - } - } } } }, diff --git a/metricbeat/metricbeat.template.json b/metricbeat/metricbeat.template.json index 63cfd3b5971..de0c2d25116 100644 --- a/metricbeat/metricbeat.template.json +++ b/metricbeat/metricbeat.template.json @@ -2196,55 +2196,6 @@ "type": "long" } } - }, - "proc": { - "properties": { - "content_length": { - "type": "long" - }, - "last_request_cpu": { - "type": "float" - }, - "last_request_memory": { - "type": "long" - }, - "pid": { - "ignore_above": 1024, - "type": "keyword" - }, - "request_duration": { - "type": "long" - }, - "request_method": { - "ignore_above": 1024, - "type": "keyword" - }, - "request_uri": { - "ignore_above": 1024, - "type": "keyword" - }, - "requests": { - "type": "long" - }, - "script": { - "ignore_above": 1024, - "type": "keyword" - }, - "start_since": { - "type": "long" - }, - "start_time": { - "type": "long" - }, - "state": { - "ignore_above": 1024, - "type": "keyword" - }, - "user": { - "ignore_above": 1024, - "type": "keyword" - } - } } } }, diff --git a/metricbeat/module/php_fpm/proc/_meta/data.json b/metricbeat/module/php_fpm/proc/_meta/data.json deleted file mode 100644 index b17127f6bd7..00000000000 --- a/metricbeat/module/php_fpm/proc/_meta/data.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "@timestamp": "2017-01-18T23:57:20.965Z", - "beat": { - "hostname": "host.example.com", - "name": "host.example.com" - }, - "metricset": { - "host": "localhost:8081", - "module": "php_fpm", - "name": "proc", - "rtt": 1303 - }, - "php_fpm": { - "proc": { - "content_length": 0, - "hostname": "localhost:8081", - "last_request_cpu": 0.000000, - "last_request_memory": 262144, - "pid": "56414", - "request_duration": 573, - "request_method": "GET", - "request_uri": "/status?json\u0026full", - "requests": 170, - "script": "-", - "start_since": 36782, - "start_time": 1484747058, - "state": "Idle", - "user": "-" - } - }, - "type": "metricsets" -} diff --git a/metricbeat/module/php_fpm/proc/_meta/docs.asciidoc b/metricbeat/module/php_fpm/proc/_meta/docs.asciidoc deleted file mode 100644 index 50680ed10bf..00000000000 --- a/metricbeat/module/php_fpm/proc/_meta/docs.asciidoc +++ /dev/null @@ -1,3 +0,0 @@ -=== php_fpm proc MetricSet - -This is the proc metricset of the module php_fpm. diff --git a/metricbeat/module/php_fpm/proc/_meta/fields.yml b/metricbeat/module/php_fpm/proc/_meta/fields.yml deleted file mode 100644 index 47624797edf..00000000000 --- a/metricbeat/module/php_fpm/proc/_meta/fields.yml +++ /dev/null @@ -1,74 +0,0 @@ -- name: proc - type: group - description: > - `proc` contains the metrics that were obtained from a PHP process managed by - the PHP-FPM process pool. - fields: - - name: pid - type: keyword - description: > - The PID of the process. - - - name: state - type: keyword - description: > - The state of the process (`Idle`, `Running`, ...). - - - name: start_time - type: long - description: > - The date and time the process has started. - - - name: start_since - type: long - description: > - Number of seconds since the process has started. - - - name: requests - type: long - description: > - The number of requests the process has served. - - - name: request_duration - type: long - description: > - The duration in microseconds of the requests. - - - name: request_method - type: keyword - description: > - The request method (`GET`, `POST`, ...). - - - name: request_uri - type: keyword - description: > - The request URI with the query string. - - - name: content_length - type: long - description: > - The content length of the request (only with `POST`). - - - name: user - type: keyword - description: > - The user (`PHP_AUTH_USER`) (or `-` if not set). - - - name: script - type: keyword - description: > - The main script called (or `-` if not set). - - - name: last_request_cpu - type: float - description: > - The %cpu the last request consumed it's always 0 if the process is not - in `Idle` state because CPU calculation is done when the request - processing has terminated. - - - name: last_request_memory - type: long - description: > - The max amount of memory the last request consumed it's always 0 if the - process is not in `Idle` state because memory calculation is done when the - request processing has terminated. diff --git a/metricbeat/module/php_fpm/proc/data.go b/metricbeat/module/php_fpm/proc/data.go deleted file mode 100644 index 7a7e072e916..00000000000 --- a/metricbeat/module/php_fpm/proc/data.go +++ /dev/null @@ -1,21 +0,0 @@ -package proc - -type procStats struct { - Pid int `json:"pid"` - State string `json:"state"` - StartTime int `json:"start time"` - StartSince int `json:"start since"` - Requests int `json:"requests"` - RequestDuration int `json:"request duration"` - RequestMethod string `json:"request method"` - RequestURI string `json:"request uri"` - ContentLength int `json:"content length"` - User string `json:"user"` - Script string `json:"script"` - LastRequestCPU float64 `json:"last request cpu"` - LastRequestMemory int `json:"last request memory"` -} - -type fullStats struct { - Processes []procStats `json:"processes"` -} diff --git a/metricbeat/module/php_fpm/proc/proc.go b/metricbeat/module/php_fpm/proc/proc.go deleted file mode 100644 index 379118703ab..00000000000 --- a/metricbeat/module/php_fpm/proc/proc.go +++ /dev/null @@ -1,83 +0,0 @@ -package proc - -import ( - "encoding/json" - "fmt" - "strconv" - - "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/logp" - "github.com/elastic/beats/metricbeat/mb" - - "github.com/elastic/beats/metricbeat/module/php_fpm" -) - -// init registers the MetricSet with the central registry. -// The New method will be called after the setup of the module and before starting to fetch data -func init() { - if err := mb.Registry.AddMetricSet("php_fpm", "proc", New, php_fpm.HostParser); err != nil { - panic(err) - } -} - -// MetricSet type defines all fields of the MetricSet -// As a minimum it must inherit the mb.BaseMetricSet fields, but can be extended with -// additional entries. These variables can be used to persist data or configuration between -// multiple fetch calls. -type MetricSet struct { - mb.BaseMetricSet - client *php_fpm.StatsClient // StatsClient that is reused across requests. -} - -// New create a new instance of the MetricSet -// Part of new is also setting up the configuration by processing additional -// configuration entries if needed. -func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - logp.Warn("EXPERIMENTAL: The php-fpm proc metricset is experimental") - return &MetricSet{ - BaseMetricSet: base, - client: php_fpm.NewStatsClient(base, true), - }, nil -} - -// Fetch methods implements the data gathering and data conversion to the right format -// It returns the event which is then forward to the output. In case of an error, a -// descriptive error must be returned. -func (m *MetricSet) Fetch() ([]common.MapStr, error) { - body, err := m.client.Fetch() - - if err != nil { - return nil, err - } - - defer body.Close() - - stats := &fullStats{} - err = json.NewDecoder(body).Decode(stats) - if err != nil { - return nil, fmt.Errorf("error parsing json: %v", err) - } - - events := []common.MapStr{} - for _, proc := range stats.Processes { - events = append(events, common.MapStr{ - "hostname": m.Host(), - - "pid": strconv.Itoa(proc.Pid), - "state": proc.State, - "start_time": proc.StartTime, - "start_since": proc.StartSince, - "requests": proc.Requests, - "request_duration": proc.RequestDuration, - "request_method": proc.RequestMethod, - "request_uri": proc.RequestURI, - "content_length": proc.ContentLength, - "user": proc.User, - "script": proc.Script, - "last_request_cpu": proc.LastRequestCPU, - "last_request_memory": proc.LastRequestMemory, - }) - } - - return events, nil -} diff --git a/metricbeat/module/php_fpm/proc/proc_integration_test.go b/metricbeat/module/php_fpm/proc/proc_integration_test.go deleted file mode 100644 index 3ca61b584a4..00000000000 --- a/metricbeat/module/php_fpm/proc/proc_integration_test.go +++ /dev/null @@ -1,25 +0,0 @@ -// +build integration - -package proc - -import ( - "testing" - - mbtest "github.com/elastic/beats/metricbeat/mb/testing" -) - -func TestData(t *testing.T) { - f := mbtest.NewEventsFetcher(t, getConfig()) - err := mbtest.WriteEvents(f, t) - if err != nil { - t.Fatal("write", err) - } -} - -func getConfig() map[string]interface{} { - return map[string]interface{}{ - "module": "php_fpm", - "metricsets": []string{"proc"}, - "hosts": []string{"127.0.0.1:81"}, - } -} From 7294c40c9ba35a12d99729d31b85f841d8d99ace Mon Sep 17 00:00:00 2001 From: Thiago Souza Date: Tue, 24 Jan 2017 02:07:48 -0200 Subject: [PATCH 07/10] PR review: reduced metric set to a bare minimum --- metricbeat/docs/fields.asciidoc | 24 ------ metricbeat/metricbeat.template-es2x.json | 11 --- metricbeat/metricbeat.template.json | 10 --- .../module/php_fpm/pool/_meta/data.json | 18 ++--- .../module/php_fpm/pool/_meta/fields.yml | 76 ++++++------------- metricbeat/module/php_fpm/pool/data.go | 2 +- metricbeat/module/php_fpm/pool/pool.go | 18 ++--- 7 files changed, 35 insertions(+), 124 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index dd4ae7e1e96..2663311fadf 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -3833,30 +3833,6 @@ type: keyword The name of the pool. -[float] -=== php_fpm.pool.process_manager - -type: keyword - -`static`, `dynamic` or `ondemand`. - - -[float] -=== php_fpm.pool.start_time - -type: long - -The date and time FPM has started. - - -[float] -=== php_fpm.pool.start_since - -type: long - -Number of seconds since FPM has started. - - [float] === php_fpm.pool.accepted_conn diff --git a/metricbeat/metricbeat.template-es2x.json b/metricbeat/metricbeat.template-es2x.json index 9072c9a79e3..45fe3c7e185 100644 --- a/metricbeat/metricbeat.template-es2x.json +++ b/metricbeat/metricbeat.template-es2x.json @@ -2201,20 +2201,9 @@ "index": "not_analyzed", "type": "string" }, - "process_manager": { - "ignore_above": 1024, - "index": "not_analyzed", - "type": "string" - }, "slow_requests": { "type": "long" }, - "start_since": { - "type": "long" - }, - "start_time": { - "type": "long" - }, "total_processes": { "type": "long" } diff --git a/metricbeat/metricbeat.template.json b/metricbeat/metricbeat.template.json index de0c2d25116..803dfda5326 100644 --- a/metricbeat/metricbeat.template.json +++ b/metricbeat/metricbeat.template.json @@ -2179,19 +2179,9 @@ "ignore_above": 1024, "type": "keyword" }, - "process_manager": { - "ignore_above": 1024, - "type": "keyword" - }, "slow_requests": { "type": "long" }, - "start_since": { - "type": "long" - }, - "start_time": { - "type": "long" - }, "total_processes": { "type": "long" } diff --git a/metricbeat/module/php_fpm/pool/_meta/data.json b/metricbeat/module/php_fpm/pool/_meta/data.json index b22be80d098..06cb2852633 100644 --- a/metricbeat/module/php_fpm/pool/_meta/data.json +++ b/metricbeat/module/php_fpm/pool/_meta/data.json @@ -12,21 +12,13 @@ }, "php_fpm": { "pool": { - "accepted_conn": 510, - "active_processes": 1, + "connections.accepted": 803, + "connections.queued": 0, "hostname": "localhost:8081", - "idle_processes": 2, - "listen_queue": 0, - "listen_queue_len": 0, - "max_active_processes": 2, - "max_children_reached": 0, - "max_list_queue": 0, "pool": "www", - "process_manager": "dynamic", - "slow_requests": 0, - "start_since": 36785, - "start_time": 1484747058, - "total_processes": 3 + "processes.active": 1, + "processes.idle": 2, + "requests.slow": 0 } }, "type": "metricsets" diff --git a/metricbeat/module/php_fpm/pool/_meta/fields.yml b/metricbeat/module/php_fpm/pool/_meta/fields.yml index 9276c79edb9..837c5982d83 100644 --- a/metricbeat/module/php_fpm/pool/_meta/fields.yml +++ b/metricbeat/module/php_fpm/pool/_meta/fields.yml @@ -9,70 +9,42 @@ description: > The name of the pool. - - name: process_manager - type: keyword - description: > - `static`, `dynamic` or `ondemand`. - - - name: start_time - type: long - description: > - The date and time FPM has started. - - - name: start_since - type: long - description: > - Number of seconds since FPM has started. - - - name: accepted_conn - type: long - description: > - The number of request accepted by the pool. - - - name: listen_queue - type: long - description: > - The number of request in the queue of pending connections. - - - name: max_listen_queue - type: long - description: > - The maximum number of requests in the queue of pending connections - since FPM has started. - - - name: listen_queue_len - type: long - description: > - The size of the socket queue of pending connections. - - - name: idle_processes - type: long - description: > - The number of idle processes. - - - name: active_processes + - name: connections.accepted type: long description: > - The number of active processes. + The number of incoming requests that the PHP-FPM server has accepted; + when a connection is accepted it is removed from the listen queue + (displayed in real time). - - name: total_processes + - name: connections.queued type: long description: > - The number of idle + active processes. + The current number of connections that have been initiated, but not yet + accepted. If this value is non-zero it typically means that all the + available server processes are currently busy, and there are no + processes available to serve the next request. Raising `pm.max_children` + (provided the server can handle it) should help keep this number low. + This property follows from the fact that PHP-FPM listens via a socket + (TCP or file based), and thus inherits some of the characteristics of + sockets.. - - name: max_active_processes + - name: processes.idle type: long description: > - The maximum number of active processes since FPM has started. + The number of servers in the `waiting to process` state (i.e. not + currently serving a page). This value should fall between the + `pm.min_spare_servers` and `pm.max_spare_servers` values when the + process manager is `dynamic`. (updated once per second) - - name: max_children_reached + - name: processes.active type: long description: > - Number of times, the process limit has been reached, when pm tries to - start more children (works only for pm `dynamic` and `ondemand`) + The number of servers current processing a page - the minimum is `1` (so + even on a fully idle server, the result will be not read `0`). (updated + once per second) - - name: slow_requests + - name: requests.slow type: long description: > - Number of times a request execution time has exceeded + The number of times a request execution time has exceeded `request_slowlog_timeout`. diff --git a/metricbeat/module/php_fpm/pool/data.go b/metricbeat/module/php_fpm/pool/data.go index babc06c36cf..48a797f36cf 100644 --- a/metricbeat/module/php_fpm/pool/data.go +++ b/metricbeat/module/php_fpm/pool/data.go @@ -7,7 +7,7 @@ type poolStats struct { StartSince int `json:"start since"` AcceptedConn int `json:"accepted conn"` ListenQueue int `json:"listen queue"` - MaxListQueue int `json:"max list queue"` + MaxListenQueue int `json:"max listen queue"` ListenQueueLen int `json:"listen queue len"` IdleProcesses int `json:"idle processes"` ActiveProcesses int `json:"active processes"` diff --git a/metricbeat/module/php_fpm/pool/pool.go b/metricbeat/module/php_fpm/pool/pool.go index 40b3c0f7cb7..bffafa1ae91 100644 --- a/metricbeat/module/php_fpm/pool/pool.go +++ b/metricbeat/module/php_fpm/pool/pool.go @@ -61,18 +61,10 @@ func (m *MetricSet) Fetch() (common.MapStr, error) { "hostname": m.Host(), "pool": stats.Pool, - "process_manager": stats.ProcessManager, - "start_time": stats.StartTime, - "start_since": stats.StartSince, - "accepted_conn": stats.AcceptedConn, - "listen_queue": stats.ListenQueue, - "max_list_queue": stats.MaxListQueue, - "listen_queue_len": stats.ListenQueueLen, - "idle_processes": stats.IdleProcesses, - "active_processes": stats.ActiveProcesses, - "total_processes": stats.TotalProcesses, - "max_active_processes": stats.MaxActiveProcesses, - "max_children_reached": stats.MaxChildrenReached, - "slow_requests": stats.SlowRequests, + "connections.queued": stats.ListenQueue, + "connections.accepted": stats.AcceptedConn, + "processes.idle": stats.IdleProcesses, + "processes.active": stats.ActiveProcesses, + "requests.slow": stats.SlowRequests, }, nil } From 9bc8a1a24c3800886ea006c97c9f3c78ea64c30c Mon Sep 17 00:00:00 2001 From: Thiago Souza Date: Tue, 24 Jan 2017 03:01:41 -0200 Subject: [PATCH 08/10] Generate artifacts --- metricbeat/docs/fields.asciidoc | 60 ++++-------------------- metricbeat/metricbeat.template-es2x.json | 51 ++++++++++---------- metricbeat/metricbeat.template.json | 51 ++++++++++---------- 3 files changed, 58 insertions(+), 104 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 2663311fadf..0857b17e15f 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -3834,83 +3834,43 @@ The name of the pool. [float] -=== php_fpm.pool.accepted_conn +=== php_fpm.pool.connections.accepted type: long -The number of request accepted by the pool. +The number of incoming requests that the PHP-FPM server has accepted; when a connection is accepted it is removed from the listen queue (displayed in real time). [float] -=== php_fpm.pool.listen_queue +=== php_fpm.pool.connections.queued type: long -The number of request in the queue of pending connections. +The current number of connections that have been initiated, but not yet accepted. If this value is non-zero it typically means that all the available server processes are currently busy, and there are no processes available to serve the next request. Raising `pm.max_children` (provided the server can handle it) should help keep this number low. This property follows from the fact that PHP-FPM listens via a socket (TCP or file based), and thus inherits some of the characteristics of sockets.. [float] -=== php_fpm.pool.max_listen_queue +=== php_fpm.pool.processes.idle type: long -The maximum number of requests in the queue of pending connections since FPM has started. +The number of servers in the `waiting to process` state (i.e. not currently serving a page). This value should fall between the `pm.min_spare_servers` and `pm.max_spare_servers` values when the process manager is `dynamic`. (updated once per second) [float] -=== php_fpm.pool.listen_queue_len +=== php_fpm.pool.processes.active type: long -The size of the socket queue of pending connections. +The number of servers current processing a page - the minimum is `1` (so even on a fully idle server, the result will be not read `0`). (updated once per second) [float] -=== php_fpm.pool.idle_processes +=== php_fpm.pool.requests.slow type: long -The number of idle processes. - - -[float] -=== php_fpm.pool.active_processes - -type: long - -The number of active processes. - - -[float] -=== php_fpm.pool.total_processes - -type: long - -The number of idle + active processes. - - -[float] -=== php_fpm.pool.max_active_processes - -type: long - -The maximum number of active processes since FPM has started. - - -[float] -=== php_fpm.pool.max_children_reached - -type: long - -Number of times, the process limit has been reached, when pm tries to start more children (works only for pm `dynamic` and `ondemand`) - - -[float] -=== php_fpm.pool.slow_requests - -type: long - -Number of times a request execution time has exceeded `request_slowlog_timeout`. +The number of times a request execution time has exceeded `request_slowlog_timeout`. [[exported-fields-postgresql]] diff --git a/metricbeat/metricbeat.template-es2x.json b/metricbeat/metricbeat.template-es2x.json index 45fe3c7e185..bff33018efe 100644 --- a/metricbeat/metricbeat.template-es2x.json +++ b/metricbeat/metricbeat.template-es2x.json @@ -2172,40 +2172,37 @@ "properties": { "pool": { "properties": { - "accepted_conn": { - "type": "long" - }, - "active_processes": { - "type": "long" - }, - "idle_processes": { - "type": "long" - }, - "listen_queue": { - "type": "long" - }, - "listen_queue_len": { - "type": "long" - }, - "max_active_processes": { - "type": "long" - }, - "max_children_reached": { - "type": "long" - }, - "max_listen_queue": { - "type": "long" + "connections": { + "properties": { + "accepted": { + "type": "long" + }, + "queued": { + "type": "long" + } + } }, "pool": { "ignore_above": 1024, "index": "not_analyzed", "type": "string" }, - "slow_requests": { - "type": "long" + "processes": { + "properties": { + "active": { + "type": "long" + }, + "idle": { + "type": "long" + } + } }, - "total_processes": { - "type": "long" + "requests": { + "properties": { + "slow": { + "type": "long" + } + } } } } diff --git a/metricbeat/metricbeat.template.json b/metricbeat/metricbeat.template.json index 803dfda5326..e55c16276ef 100644 --- a/metricbeat/metricbeat.template.json +++ b/metricbeat/metricbeat.template.json @@ -2151,39 +2151,36 @@ "properties": { "pool": { "properties": { - "accepted_conn": { - "type": "long" - }, - "active_processes": { - "type": "long" - }, - "idle_processes": { - "type": "long" - }, - "listen_queue": { - "type": "long" - }, - "listen_queue_len": { - "type": "long" - }, - "max_active_processes": { - "type": "long" - }, - "max_children_reached": { - "type": "long" - }, - "max_listen_queue": { - "type": "long" + "connections": { + "properties": { + "accepted": { + "type": "long" + }, + "queued": { + "type": "long" + } + } }, "pool": { "ignore_above": 1024, "type": "keyword" }, - "slow_requests": { - "type": "long" + "processes": { + "properties": { + "active": { + "type": "long" + }, + "idle": { + "type": "long" + } + } }, - "total_processes": { - "type": "long" + "requests": { + "properties": { + "slow": { + "type": "long" + } + } } } } From 28352deb20f9552b3097b8d151e445bae3953a27 Mon Sep 17 00:00:00 2001 From: Thiago Souza Date: Tue, 24 Jan 2017 03:13:11 -0200 Subject: [PATCH 09/10] Remove uneeded bool flag --- metricbeat/module/php_fpm/php_fpm.go | 8 ++------ metricbeat/module/php_fpm/pool/pool.go | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/metricbeat/module/php_fpm/php_fpm.go b/metricbeat/module/php_fpm/php_fpm.go index d4f5791d980..e20d352e3db 100644 --- a/metricbeat/module/php_fpm/php_fpm.go +++ b/metricbeat/module/php_fpm/php_fpm.go @@ -31,13 +31,9 @@ type StatsClient struct { } // NewStatsClient creates a new StatsClient -func NewStatsClient(m mb.BaseMetricSet, isFullStats bool) *StatsClient { - address := m.HostData().SanitizedURI - if isFullStats { - address += "&full" - } +func NewStatsClient(m mb.BaseMetricSet) *StatsClient { return &StatsClient{ - address: address, + address: m.HostData().SanitizedURI, user: m.HostData().User, password: m.HostData().Password, http: &http.Client{Timeout: m.Module().Config().Timeout}, diff --git a/metricbeat/module/php_fpm/pool/pool.go b/metricbeat/module/php_fpm/pool/pool.go index bffafa1ae91..8390b0ed3bd 100644 --- a/metricbeat/module/php_fpm/pool/pool.go +++ b/metricbeat/module/php_fpm/pool/pool.go @@ -35,7 +35,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { logp.Warn("EXPERIMENTAL: The php-fpm pool metricset is experimental") return &MetricSet{ BaseMetricSet: base, - client: php_fpm.NewStatsClient(base, false), + client: php_fpm.NewStatsClient(base), }, nil } From f7ea5e6b3a5c3ea61ca04a2f6256cf8f70e3892b Mon Sep 17 00:00:00 2001 From: Thiago Souza Date: Tue, 24 Jan 2017 08:59:09 -0200 Subject: [PATCH 10/10] PR review: add structures --- metricbeat/docs/fields.asciidoc | 24 +++++-- metricbeat/metricbeat.template-es2x.json | 8 +-- metricbeat/metricbeat.template.json | 8 +-- .../module/php_fpm/pool/_meta/fields.yml | 71 ++++++++++--------- metricbeat/module/php_fpm/pool/pool.go | 16 +++-- 5 files changed, 70 insertions(+), 57 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 0857b17e15f..72222092b27 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -3833,12 +3833,19 @@ type: keyword The name of the pool. +[float] +== connections Fields + +Connection state specific statistics. + + + [float] === php_fpm.pool.connections.accepted type: long -The number of incoming requests that the PHP-FPM server has accepted; when a connection is accepted it is removed from the listen queue (displayed in real time). +The number of incoming requests that the PHP-FPM server has accepted; when a connection is accepted it is removed from the listen queue. [float] @@ -3846,7 +3853,14 @@ The number of incoming requests that the PHP-FPM server has accepted; when a con type: long -The current number of connections that have been initiated, but not yet accepted. If this value is non-zero it typically means that all the available server processes are currently busy, and there are no processes available to serve the next request. Raising `pm.max_children` (provided the server can handle it) should help keep this number low. This property follows from the fact that PHP-FPM listens via a socket (TCP or file based), and thus inherits some of the characteristics of sockets.. +The current number of connections that have been initiated, but not yet accepted. If this value is non-zero it typically means that all the available server processes are currently busy, and there are no processes available to serve the next request. Raising `pm.max_children` (provided the server can handle it) should help keep this number low. This property follows from the fact that PHP-FPM listens via a socket (TCP or file based), and thus inherits some of the characteristics of sockets. + + +[float] +== processes Fields + +Process state specific statistics. + [float] @@ -3854,7 +3868,7 @@ The current number of connections that have been initiated, but not yet accepted type: long -The number of servers in the `waiting to process` state (i.e. not currently serving a page). This value should fall between the `pm.min_spare_servers` and `pm.max_spare_servers` values when the process manager is `dynamic`. (updated once per second) +The number of servers in the `waiting to process` state (i.e. not currently serving a page). This value should fall between the `pm.min_spare_servers` and `pm.max_spare_servers` values when the process manager is `dynamic`. [float] @@ -3862,11 +3876,11 @@ The number of servers in the `waiting to process` state (i.e. not currently serv type: long -The number of servers current processing a page - the minimum is `1` (so even on a fully idle server, the result will be not read `0`). (updated once per second) +The number of servers current processing a page - the minimum is `1` (so even on a fully idle server, the result will be not read `0`). [float] -=== php_fpm.pool.requests.slow +=== php_fpm.pool.slow_requests type: long diff --git a/metricbeat/metricbeat.template-es2x.json b/metricbeat/metricbeat.template-es2x.json index bff33018efe..67d90128a55 100644 --- a/metricbeat/metricbeat.template-es2x.json +++ b/metricbeat/metricbeat.template-es2x.json @@ -2197,12 +2197,8 @@ } } }, - "requests": { - "properties": { - "slow": { - "type": "long" - } - } + "slow_requests": { + "type": "long" } } } diff --git a/metricbeat/metricbeat.template.json b/metricbeat/metricbeat.template.json index e55c16276ef..5d4b27fb515 100644 --- a/metricbeat/metricbeat.template.json +++ b/metricbeat/metricbeat.template.json @@ -2175,12 +2175,8 @@ } } }, - "requests": { - "properties": { - "slow": { - "type": "long" - } - } + "slow_requests": { + "type": "long" } } } diff --git a/metricbeat/module/php_fpm/pool/_meta/fields.yml b/metricbeat/module/php_fpm/pool/_meta/fields.yml index 837c5982d83..ba22858f91e 100644 --- a/metricbeat/module/php_fpm/pool/_meta/fields.yml +++ b/metricbeat/module/php_fpm/pool/_meta/fields.yml @@ -8,42 +8,45 @@ type: keyword description: > The name of the pool. - - - name: connections.accepted - type: long - description: > - The number of incoming requests that the PHP-FPM server has accepted; - when a connection is accepted it is removed from the listen queue - (displayed in real time). - - - name: connections.queued - type: long + - name: connections + type: group description: > - The current number of connections that have been initiated, but not yet - accepted. If this value is non-zero it typically means that all the - available server processes are currently busy, and there are no - processes available to serve the next request. Raising `pm.max_children` - (provided the server can handle it) should help keep this number low. - This property follows from the fact that PHP-FPM listens via a socket - (TCP or file based), and thus inherits some of the characteristics of - sockets.. - - - name: processes.idle - type: long - description: > - The number of servers in the `waiting to process` state (i.e. not - currently serving a page). This value should fall between the - `pm.min_spare_servers` and `pm.max_spare_servers` values when the - process manager is `dynamic`. (updated once per second) - - - name: processes.active - type: long + Connection state specific statistics. + fields: + - name: accepted + type: long + description: > + The number of incoming requests that the PHP-FPM server has accepted; + when a connection is accepted it is removed from the listen queue. + - name: queued + type: long + description: > + The current number of connections that have been initiated, but not + yet accepted. If this value is non-zero it typically means that all + the available server processes are currently busy, and there are no + processes available to serve the next request. Raising + `pm.max_children` (provided the server can handle it) should help + keep this number low. This property follows from the fact that + PHP-FPM listens via a socket (TCP or file based), and thus inherits + some of the characteristics of sockets. + - name: processes + type: group description: > - The number of servers current processing a page - the minimum is `1` (so - even on a fully idle server, the result will be not read `0`). (updated - once per second) - - - name: requests.slow + Process state specific statistics. + fields: + - name: idle + type: long + description: > + The number of servers in the `waiting to process` state (i.e. not + currently serving a page). This value should fall between the + `pm.min_spare_servers` and `pm.max_spare_servers` values when the + process manager is `dynamic`. + - name: active + type: long + description: > + The number of servers current processing a page - the minimum is `1` + (so even on a fully idle server, the result will be not read `0`). + - name: slow_requests type: long description: > The number of times a request execution time has exceeded diff --git a/metricbeat/module/php_fpm/pool/pool.go b/metricbeat/module/php_fpm/pool/pool.go index 8390b0ed3bd..982fe1e3899 100644 --- a/metricbeat/module/php_fpm/pool/pool.go +++ b/metricbeat/module/php_fpm/pool/pool.go @@ -60,11 +60,15 @@ func (m *MetricSet) Fetch() (common.MapStr, error) { return common.MapStr{ "hostname": m.Host(), - "pool": stats.Pool, - "connections.queued": stats.ListenQueue, - "connections.accepted": stats.AcceptedConn, - "processes.idle": stats.IdleProcesses, - "processes.active": stats.ActiveProcesses, - "requests.slow": stats.SlowRequests, + "pool": stats.Pool, + "connections": common.MapStr{ + "queue": stats.ListenQueue, + "accepted": stats.AcceptedConn, + }, + "processes": common.MapStr{ + "idle": stats.IdleProcesses, + "active": stats.ActiveProcesses, + }, + "slow_requests": stats.SlowRequests, }, nil }