add support for new timestamp format (#34211)

* add support for new timestamp format
elastic · Jun 1, 2023 · d8fa52a · d8fa52a
1 parent f143eeb
commit d8fa52a
Show file tree

Hide file tree

Showing 6 changed files with 596 additions and 235 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -190,6 +190,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 - Add pagination support for Salesforce module. {issue}34057[34057] {pull}34065[34065]
 - Allow users to redact sensitive data from CEL input debug logs. {pull}34302[34302]
 - Added support for HTTP destination override to Google Cloud Storage input. {pull}34413[34413]
+- Add support for new Rabbitmq timestamp format for logs {pull}34211[34211]
 
 *Auditbeat*
 

diff --git a/filebeat/docs/modules/rabbitmq.asciidoc b/filebeat/docs/modules/rabbitmq.asciidoc
@@ -13,6 +13,7 @@ This file is generated! See scripts/docs_collector.py
 include::{libbeat-dir}/shared/integration-link.asciidoc[]
 
 This is the module for parsing https://www.rabbitmq.com/logging.html[RabbitMQ log files]
+It will only support RabbitMQ default i.e RFC 3339 timestamp format using TIMESTAMP_ISO8601.
 
 include::../include/what-happens.asciidoc[]
 

diff --git a/x-pack/filebeat/module/rabbitmq/_meta/docs.asciidoc b/x-pack/filebeat/module/rabbitmq/_meta/docs.asciidoc
@@ -6,6 +6,7 @@
 include::{libbeat-dir}/shared/integration-link.asciidoc[]
 
 This is the module for parsing https://www.rabbitmq.com/logging.html[RabbitMQ log files]
+It will only support RabbitMQ default i.e RFC 3339 timestamp format using TIMESTAMP_ISO8601.
 
 include::../include/what-happens.asciidoc[]
 

diff --git a/x-pack/filebeat/module/rabbitmq/log/ingest/pipeline.yml b/x-pack/filebeat/module/rabbitmq/log/ingest/pipeline.yml
@@ -10,7 +10,7 @@ processors:
       GREEDYMULTILINE: "(.|\n)*"
       ERL_PID: "\\<%{INT}+\\.%{INT}+\\.%{INT}+\\>"
     patterns:
-    - "%{DATESTAMP:timestamp} \\[%{WORD:log.level}\\] %{ERL_PID:rabbitmq.log.pid}
+    - "%{TIMESTAMP_ISO8601:timestamp} \\[%{WORD:log.level}\\] %{ERL_PID:rabbitmq.log.pid}
       %{GREEDYMULTILINE:message}"
     ignore_missing: true
 - grok:
@@ -66,14 +66,14 @@ processors:
     field: timestamp
     target_field: "@timestamp"
     formats:
-    - yy-MM-dd HH:mm:ss.SSS
+    - yyyy-MM-dd HH:mm:ss.SSSSSSZZZZZ
 - date:
     if: "ctx.event.timezone != null"
     field: "timestamp"
     target_field: "@timestamp"
     timezone: "{{ event.timezone }}"
     formats:
-    - yy-MM-dd HH:mm:ss.SSS
+    - yyyy-MM-dd HH:mm:ss.SSSSSSZZZZZ
 - remove:
     field:
     - timestamp

diff --git a/x-pack/filebeat/module/rabbitmq/log/test/test.log b/x-pack/filebeat/module/rabbitmq/log/test/test.log
@@ -1,79 +1,67 @@
-2019-04-03 11:13:15.076 [info] <0.8.0> Log file opened with Lager
-2019-04-03 11:13:15.510 [info] <0.222.0> 
- Starting RabbitMQ 3.7.14 on Erlang 21.3.2
- Copyright (C) 2007-2019 Pivotal Software, Inc.
- Licensed under the MPL.  See https://www.rabbitmq.com/
-2019-04-03 11:13:15.512 [info] <0.222.0>  
- node           : rabbit@localhost
- home dir       : /Users/jfsiii
- config file(s) : (none)
- cookie hash    : 1FLKC2GJUcbFjO6klcgs8Q==
- log(s)         : /usr/local/var/log/rabbitmq/rabbit@localhost.log
-                : /usr/local/var/log/rabbitmq/rabbit@localhost_upgrade.log
- database dir   : /usr/local/var/lib/rabbitmq/mnesia/rabbit@localhost
-2019-04-12 10:00:53.458 [info] <0.1398.0> RabbitMQ is asked to stop...
-2019-04-12 10:00:53.550 [info] <0.1398.0> Stopping RabbitMQ applications and their dependencies in the following order:
-    rabbitmq_management
-    rabbitmq_stomp
-    rabbitmq_amqp1_0
-    rabbitmq_mqtt
-    amqp_client
-    rabbitmq_web_dispatch
-    cowboy
-    cowlib
-    rabbitmq_management_agent
-    rabbit
-    mnesia
-    rabbit_common
-    sysmon_handler
-    os_mon
-    amqp10_common
-2019-04-12 10:00:53.550 [info] <0.1398.0> Stopping application 'rabbitmq_management'
-2019-04-12 10:00:54.553 [warning] <0.490.0> RabbitMQ HTTP listener registry could not find context rabbitmq_management_tls
-2019-04-12 10:00:54.555 [info] <0.43.0> Application rabbitmq_management exited with reason: stopped
-2019-04-12 10:00:54.567 [info] <0.1398.0> Stopping application 'rabbit'
-2019-04-12 10:00:54.567 [info] <0.286.0> Peer discovery backend rabbit_peer_discovery_classic_config does not support registration, skipping unregistration.
-2019-04-12 10:00:54.568 [info] <0.419.0> stopped TCP listener on 127.0.0.1:5672
-2019-04-12 10:00:54.569 [info] <0.324.0> Closing all connections in vhost '/' on node 'rabbit@localhost' because the vhost is stopping
-2019-04-12 10:00:54.579 [info] <0.374.0> Stopping message store for directory '/usr/local/var/lib/rabbitmq/mnesia/rabbit@localhost/msg_stores/vhosts/628WB79CIFDYO9LJI6DKMI09L/msg_store_persistent'
-2019-04-12 10:00:54.588 [info] <0.374.0> Message store for directory '/usr/local/var/lib/rabbitmq/mnesia/rabbit@localhost/msg_stores/vhosts/628WB79CIFDYO9LJI6DKMI09L/msg_store_persistent' is stopped
-2019-04-12 10:00:54.589 [info] <0.371.0> Stopping message store for directory '/usr/local/var/lib/rabbitmq/mnesia/rabbit@localhost/msg_stores/vhosts/628WB79CIFDYO9LJI6DKMI09L/msg_store_transient'
-2019-04-12 10:00:54.598 [info] <0.371.0> Message store for directory '/usr/local/var/lib/rabbitmq/mnesia/rabbit@localhost/msg_stores/vhosts/628WB79CIFDYO9LJI6DKMI09L/msg_store_transient' is stopped
-2019-04-12 10:00:54.606 [info] <0.43.0> Application rabbit exited with reason: stopped
-2019-04-12 10:00:54.615 [info] <0.1398.0> Successfully stopped RabbitMQ and its dependencies
-2019-04-12 10:00:54.615 [info] <0.1398.0> Halting Erlang VM with the following applications:
-    ranch
-    ssl
-    public_key
-    sasl
-    inets
-    asn1
-    crypto
-    jsx
-    xmerl
-    recon
-    lager
-    goldrush
-    compiler
-    syntax_tools
-    stdlib
-    kernel
-2019-04-12 10:01:01.031 [info] <0.8.0> Server startup complete; 6 plugins started.
- * rabbitmq_stomp
- * rabbitmq_management
- * rabbitmq_web_dispatch
- * rabbitmq_amqp1_0
- * rabbitmq_mqtt
- * rabbitmq_management_agent
-2019-04-12 10:11:15.094 [info] <0.1345.0> accepting AMQP connection <0.1345.0> (127.0.0.1:64875 -> 127.0.0.1:5672)
-2019-04-12 10:11:15.101 [info] <0.1345.0> connection <0.1345.0> (127.0.0.1:64875 -> 127.0.0.1:5672): user 'guest' authenticated and granted access to vhost '/'
-2019-04-12 10:19:14.450 [error] <0.1345.0> Error on AMQP connection <0.1345.0> (127.0.0.1:64875 -> 127.0.0.1:5672, vhost: '/', user: 'guest', state: running), channel 0:
- operation none caused a connection exception connection_forced: [240,159,145,
-                                                                 139,240,159,
-                                                                 143,190,240,
-                                                                 159,144,135,
-                                                                 240,159,164,
-                                                                 163]
-2019-04-12 10:19:14.450 [info] <0.1902.0> Closing connection <0.1345.0> because <<240,159,145,139,240,159,143,190,240,159,144,135,240,159,164,163>>
-2019-04-12 10:19:14.451 [info] <0.1345.0> closing AMQP connection <0.1345.0> (127.0.0.1:64875 -> 127.0.0.1:5672, vhost: '/', user: 'guest')
-2021-11-22 17:48:20.003 [warning] <0.8084.263> HTTP access denied: user 'guest' - Not monitor user
+2023-01-24 10:38:45.236018+00:00 [info] <0.229.0>  
+node           : rabbit@af6809c8510d
+home dir       : /var/lib/rabbitmq
+config file(s) : /etc/rabbitmq/conf.d/10-defaults.conf
+cookie hash    : ibMcme1ZByOOJPIBTHvhzg==
+log(s)         : /var/log/rabbitmq/rabbit@af6809c8510d_upgrade.log
+               : <stdout>
+database dir   : /var/lib/rabbitmq/mnesia/rabbit@af6809c8510d
+2023-01-24 10:38:48.987396+00:00 [info] <0.229.0> Running boot step pre_boot defined by app rabbit
+2023-01-24 10:38:49.001012+00:00 [info] <0.229.0> Running boot step code_server_cache defined by app rabbit
+2023-01-24 10:38:49.001076+00:00 [info] <0.229.0> Running boot step file_handle_cache defined by app rabbit
+2023-01-24 10:38:49.002188+00:00 [info] <0.229.0> Running boot step worker_pool defined by app rabbit
+2023-01-24 10:38:49.002241+00:00 [info] <0.286.0> Will use 5 processes for default worker pool
+2023-01-24 10:38:49.002287+00:00 [info] <0.286.0> Starting worker pool 'worker_pool' with 5 processes in it
+2023-01-24 10:38:49.002531+00:00 [info] <0.229.0> Running boot step database defined by app rabbit
+2023-01-24 10:38:49.003858+00:00 [info] <0.229.0> Node database directory at /var/lib/rabbitmq/mnesia/rabbit@af6809c8510d is empty. Assuming we need to join an existing cluster or initialise from scratch...
+2023-01-24 10:38:49.003915+00:00 [info] <0.229.0> Configured peer discovery backend: rabbit_peer_discovery_classic_config
+2023-01-24 10:38:49.003932+00:00 [info] <0.229.0> Will try to lock with peer discovery backend rabbit_peer_discovery_classic_config
+2023-01-24 10:38:49.003976+00:00 [info] <0.229.0> All discovered existing cluster peers:
+2023-01-24 10:38:49.003989+00:00 [info] <0.229.0> Discovered no peer nodes to cluster with. Some discovery backends can filter nodes out based on a readiness criteria. Enabling debug logging might help troubleshoot.
+2023-01-24 10:38:49.119439+00:00 [info] <0.229.0> Waiting for Mnesia tables for 30000 ms, 9 retries left
+2023-01-24 10:38:49.119725+00:00 [info] <0.229.0> Successfully synced tables from a peer
+2023-01-24 10:38:49.126213+00:00 [info] <0.229.0> Feature flags: `feature_flags_v2`: supported, attempt to enable...
+2023-01-24 10:38:49.220869+00:00 [info] <0.499.0> Waiting for Mnesia tables for 30000 ms, 9 retries left
+2023-01-24 10:38:49.220981+00:00 [info] <0.499.0> Successfully synced tables from a peer
+2023-01-24 10:38:49.429984+00:00 [info] <0.229.0> Waiting for Mnesia tables for 30000 ms, 9 retries left
+2023-01-24 10:38:49.430060+00:00 [info] <0.229.0> Successfully synced tables from a peer
+2023-01-24 10:38:49.437978+00:00 [info] <0.229.0> Waiting for Mnesia tables for 30000 ms, 9 retries left
+2023-01-24 10:38:49.438072+00:00 [info] <0.229.0> Successfully synced tables from a peer
+2023-01-24 10:38:49.438095+00:00 [info] <0.229.0> Peer discovery backend rabbit_peer_discovery_classic_config does not support registration, skipping registration.
+2023-01-24 10:38:49.438118+00:00 [info] <0.229.0> Will try to unlock with peer discovery backend rabbit_peer_discovery_classic_config
+2023-01-24 10:38:49.438156+00:00 [info] <0.229.0> Running boot step tracking_metadata_store defined by app rabbit
+2023-01-24 10:38:49.438205+00:00 [info] <0.621.0> Setting up a table for connection tracking on this node: tracked_connection
+2023-01-24 10:38:49.438234+00:00 [info] <0.621.0> Setting up a table for per-vhost connection counting on this node: tracked_connection_per_vhost
+2023-01-24 10:38:49.438258+00:00 [info] <0.621.0> Setting up a table for per-user connection counting on this node: tracked_connection_per_user
+2023-01-24 10:38:49.438277+00:00 [info] <0.621.0> Setting up a table for channel tracking on this node: tracked_channel
+2023-01-24 10:38:49.438295+00:00 [info] <0.621.0> Setting up a table for channel tracking on this node: tracked_channel_per_user
+2023-01-24 10:38:49.439229+00:00 [info] <0.229.0> Priority queues enabled, real BQ is rabbit_variable_queue
+2023-01-24 10:38:49.457064+00:00 [info] <0.229.0> Management plugin: using rates mode 'basic'
+2023-01-24 10:38:49.459815+00:00 [info] <0.229.0> Will seed default virtual host and user...
+2023-01-24 10:38:49.459877+00:00 [info] <0.229.0> Adding vhost '/' (description: 'Default virtual host', tags: [])
+2023-01-24 10:38:49.462284+00:00 [info] <0.229.0> Applying default limits to vhost '<<"/">>': []
+2023-01-24 10:38:49.475207+00:00 [info] <0.676.0> Making sure data directory '/var/lib/rabbitmq/mnesia/rabbit@af6809c8510d/msg_stores/vhosts/628WB79CIFDYO9LJI6DKMI09L' for vhost '/' exists
+2023-01-24 10:38:49.477540+00:00 [info] <0.676.0> Setting segment_entry_count for vhost '/' with 0 queues to '2048'
+2023-01-24 10:38:49.480811+00:00 [info] <0.676.0> Starting message stores for vhost '/'
+2023-01-24 10:38:49.481068+00:00 [info] <0.681.0> Message store "628WB79CIFDYO9LJI6DKMI09L/msg_store_transient": using rabbit_msg_store_ets_index to provide index
+2023-01-24 10:38:49.482916+00:00 [info] <0.676.0> Started message store of type transient for vhost '/'
+2023-01-24 10:38:49.485324+00:00 [info] <0.676.0> Started message store of type persistent for vhost '/'
+2023-01-24 10:38:49.485389+00:00 [info] <0.676.0> Recovering 0 queues of type rabbit_classic_queue took 7ms
+2023-01-24 10:38:49.485419+00:00 [info] <0.676.0> Recovering 0 queues of type rabbit_quorum_queue took 0ms
+2023-01-24 10:38:49.485436+00:00 [info] <0.676.0> Recovering 0 queues of type rabbit_stream_queue took 0ms
+2023-01-24 10:38:49.487133+00:00 [info] <0.229.0> Created user 'guest'
+2023-01-24 10:38:49.488641+00:00 [info] <0.229.0> Successfully set user tags for user 'guest' to [administrator]
+2023-01-24 10:38:49.490051+00:00 [info] <0.229.0> Successfully set permissions for 'guest' in virtual host '/' to '.*', '.*', '.*'
+2023-01-24 10:38:49.490500+00:00 [info] <0.286.0> Starting worker pool 'definition_import_pool' with 5 processes in it
+2023-01-24 10:38:49.490717+00:00 [info] <0.229.0> Running boot step cluster_name defined by app rabbit
+2023-01-24 10:38:49.490758+00:00 [info] <0.229.0> Initialising internal cluster ID to 'rabbitmq-cluster-id-nZJPoEIR_-4jZYWewYYOZQ'
+2023-01-24 10:38:49.519212+00:00 [info] <0.809.0> Starting worker pool 'management_worker_pool' with 3 processes in it
+2023-01-24 10:38:49.524893+00:00 [info] <0.824.0> Prometheus metrics: HTTP (non-TLS) listener started on port 15692
+2023-01-24 10:38:49.525012+00:00 [info] <0.723.0> Ready to start client connection listeners
+2023-01-24 10:38:49.525875+00:00 [info] <0.868.0> started TCP listener on [::]:5672
+completed with 4 plugins.
+2023-01-24 10:38:49.664998+00:00 [info] <0.723.0> Server startup complete; 4 plugins started.
+* rabbitmq_prometheus
+* rabbitmq_management
+* rabbitmq_web_dispatch
+* rabbitmq_management_agent