Fix: nginx module can't handle multiline entries (#14499) (#14512)

* Fix: nginx module can't handle multiline entries * Update ingest pipeline and test case * Update next CHANGELOG (cherry picked from commit 72f2bf9)
elastic · Nov 14, 2019 · 8116e3f · 8116e3f
1 parent d405956
commit 8116e3f
Show file tree

Hide file tree

Showing 5 changed files with 46 additions and 1 deletion.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -188,6 +188,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Improve error message in s3 input when handleSQSMessage failed. {pull}14113[14113]
 - Fix race condition in S3 input plugin. {pull}14359[14359]
 - Decode hex values in auditd module. {pull}14471[14471]
+- Fix handling multiline log entries in nginx module. {issue}14349[14349] {pull}14499[14499]
 
 *Heartbeat*
 

diff --git a/filebeat/module/nginx/error/config/nginx-error.yml b/filebeat/module/nginx/error/config/nginx-error.yml
@@ -4,6 +4,10 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
+multiline:
+  pattern: '^\d{4}\/\d{2}\/\d{2} '
+  negate: true
+  match: after
 
 processors:
 - add_locale: ~
diff --git a/filebeat/module/nginx/error/ingest/pipeline.json b/filebeat/module/nginx/error/ingest/pipeline.json
@@ -4,8 +4,11 @@
     "grok": {
       "field": "message",
       "patterns": [
-        "%{DATA:nginx.error.time} \\[%{DATA:log.level}\\] %{NUMBER:process.pid:long}#%{NUMBER:process.thread.id:long}: (\\*%{NUMBER:nginx.error.connection_id:long} )?%{GREEDYDATA:message}"
+        "%{DATA:nginx.error.time} \\[%{DATA:log.level}\\] %{NUMBER:process.pid:long}#%{NUMBER:process.thread.id:long}: (\\*%{NUMBER:nginx.error.connection_id:long} )?%{GREEDYMULTILINE:message}"
       ],
+      "pattern_definitions": {
+        "GREEDYMULTILINE":"(.|\n|\t)*"
+      },
       "ignore_missing": true
     }
   }, {

diff --git a/filebeat/module/nginx/error/test/error.log b/filebeat/module/nginx/error/test/error.log
@@ -1,2 +1,6 @@
 2016/10/25 14:49:34 [error] 54053#0: *1 open() "/usr/local/Cellar/nginx/1.10.2_1/html/favicon.ico" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: "GET /favicon.ico HTTP/1.1", host: "localhost:8080", referrer: "http://localhost:8080/"
 2016/10/25 14:50:44 [error] 54053#0: *3 open() "/usr/local/Cellar/nginx/1.10.2_1/html/adsasd" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: "GET /adsasd HTTP/1.1", host: "localhost:8080"
+2019/10/30 23:26:34 [error] 205860#205860: *180289 FastCGI sent in stderr: "PHP message: PHP Warning:  Declaration of FEE_Field_Terms::wrap($content, $taxonomy, $before, $sep, $after) should be compatible with FEE_Field_Post::wrap($content, $post_id = 0) in /var/www/xxx/web/wp-content/plugins/front-end-editor/php/fields/post.php on line 0
+PHP message: PHP Warning:  Declaration of FEE_Field_Tags::wrap($content, $before, $sep, $after) should be compatible with FEE_Field_Terms::wrap($content, $taxonomy, $before, $sep, $after) in /var/www/xxx/web/wp-content/plugins/front-end-editor/php/fields/post.php on line 0
+PHP message: PHP Warning:  Declaration of FEE_Field_Category::wrap($content, $sep, $parents) should be compatible with FEE_Field_Terms::wrap($content, $taxonomy, $before, $sep, $after) in /var/www/xxx/web/wp-content/plugins/front-end-editor/php/fields/post.php on line 0
+2019/11/05 14:50:44 [error] 54053#0: *3 open() "/usr/local/Cellar/nginx/1.10.2_1/html/adsasd" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: "GET /pysio HTTP/1.1", host: "localhost:8080"
diff --git a/filebeat/module/nginx/error/test/error.log-expected.json b/filebeat/module/nginx/error/test/error.log-expected.json
@@ -28,5 +28,38 @@
         "process.pid": 54053,
         "process.thread.id": 0,
         "service.type": "nginx"
+    },
+    {
+        "@timestamp": "2019-10-30T23:26:34.000-02:00",
+        "event.dataset": "nginx.error",
+        "event.module": "nginx",
+        "event.timezone": "-02:00",
+        "fileset.name": "error",
+        "input.type": "log",
+        "log.flags": [
+            "multiline"
+        ],
+        "log.level": "error",
+        "log.offset": 500,
+        "message": "FastCGI sent in stderr: \"PHP message: PHP Warning:  Declaration of FEE_Field_Terms::wrap($content, $taxonomy, $before, $sep, $after) should be compatible with FEE_Field_Post::wrap($content, $post_id = 0) in /var/www/xxx/web/wp-content/plugins/front-end-editor/php/fields/post.php on line 0\nPHP message: PHP Warning:  Declaration of FEE_Field_Tags::wrap($content, $before, $sep, $after) should be compatible with FEE_Field_Terms::wrap($content, $taxonomy, $before, $sep, $after) in /var/www/xxx/web/wp-content/plugins/front-end-editor/php/fields/post.php on line 0\nPHP message: PHP Warning:  Declaration of FEE_Field_Category::wrap($content, $sep, $parents) should be compatible with FEE_Field_Terms::wrap($content, $taxonomy, $before, $sep, $after) in /var/www/xxx/web/wp-content/plugins/front-end-editor/php/fields/post.php on line 0",
+        "nginx.error.connection_id": 180289,
+        "process.pid": 205860,
+        "process.thread.id": 205860,
+        "service.type": "nginx"
+    },
+    {
+        "@timestamp": "2019-11-05T14:50:44.000-02:00",
+        "event.dataset": "nginx.error",
+        "event.module": "nginx",
+        "event.timezone": "-02:00",
+        "fileset.name": "error",
+        "input.type": "log",
+        "log.level": "error",
+        "log.offset": 1386,
+        "message": "open() \"/usr/local/Cellar/nginx/1.10.2_1/html/adsasd\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /pysio HTTP/1.1\", host: \"localhost:8080\"",
+        "nginx.error.connection_id": 3,
+        "process.pid": 54053,
+        "process.thread.id": 0,
+        "service.type": "nginx"
     }
 ]