From a5e2e73dcfd180085d3b1298c3f67365e918742e Mon Sep 17 00:00:00 2001 From: Corbin Phelps Date: Tue, 26 Apr 2022 12:04:13 -0400 Subject: [PATCH 1/4] Added mongodb plugin Signed-off-by: Corbin Phelps --- plugins/mongodb_logs.yaml | 128 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 plugins/mongodb_logs.yaml diff --git a/plugins/mongodb_logs.yaml b/plugins/mongodb_logs.yaml new file mode 100644 index 000000000..1328d42b2 --- /dev/null +++ b/plugins/mongodb_logs.yaml @@ -0,0 +1,128 @@ +version: 0.0.1 +title: MongoDB +description: Log parser for MongoDB +parameters: + - name: log_paths + type: "[]string" + default: ["/var/log/mongodb/mongodb.log*"] + - name: start_at + type: string + supported: + - beginning + - end + default: end +template: | + receivers: + filelog: + include: + {{ range $fp := .daemon_log_paths }} + - '{{ $fp }}' + {{end}} + exclude: + {{ range $fp := .daemon_log_paths }} + - '{{ $fp }}*.gz' + {{end}} + start_at: {{ .start_at }} + operators: + - type: router + default: legacy_parser + routes: + - expr: 'body matches "^{.*}$|^{.*}\\n$"' + output: 4_4_parser + + - id: legacy_parser + type: regex_parser + regex: '^(?P\S+)\s+(?P\w+)\s+(?P[\w-]+)\s+\[(?P\S+)\]\s+(?P.*)$' + timestamp: + parse_from: attributes:timestamp + #2019-02-06T09:22:43.967-0500 + layout: '%Y-%m-%dT%H:%M:%S.%f%z' + severity: + parse_from: attributes.severity + mapping: + fatal: F + error: E + warn: W + info: I + debug: + - D + - D1 + - D2 + - D3 + - D4 + - D5 + output: add_plugin_id + + - id: 4_4_parser + type: json_parser + parse_from: body + timestamp: + parse_from: attributes.t.date + #2020-11-03T14:24:07.436-05:00 + layout: '%Y-%m-%dT%H:%M:%S.%f%j' + severity: + parse_from: attributes.s + mapping: + fatal: F + error: E + warn: W + info: I + debug: + - D + - D1 + - D2 + - D3 + - D4 + - D5 + output: remove_t + + - id: remove_t + type: remove + field: attributes.t + + - id: move_component + type: move + from: attributes.c + to: attributes.component + + - id: move_context + type: move + from: attributes.ctx + to: attributes.context + + - id: move_message + type: move + from: attributes.msg + to: attributes.message + + # When message is 'WiredTiger message', data.attr.message + # always exists, and should be promoted to message + - id: wiredtiger_router + type: router + default: add_log_type + routes: + - expr: 'attributes.message == "WiredTiger message"' + output: move_wiredtiger_type + + - id: move_wiredtiger_type + type: move + from: attributes.message + to: attributes.message_type + output: move_wiredtiger_msg + + - id: move_wiredtiger_msg + type: move + from: attributes.attr.message + to: attributes.message + output: add_log_type + + - id: add_log_type + type: add + field: 'attributes.log_type' + value: 'mongodb' + + service: + pipelines: + logs: + receivers: + - filelog From 55e0f0d454fcbf919a314b4ee6b85a69fc9a6844 Mon Sep 17 00:00:00 2001 From: Corbin Phelps Date: Tue, 26 Apr 2022 12:28:31 -0400 Subject: [PATCH 2/4] Added multiple logging paths to mongo Signed-off-by: Corbin Phelps --- plugins/mongodb_logs.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/mongodb_logs.yaml b/plugins/mongodb_logs.yaml index 1328d42b2..1bb37df77 100644 --- a/plugins/mongodb_logs.yaml +++ b/plugins/mongodb_logs.yaml @@ -4,7 +4,9 @@ description: Log parser for MongoDB parameters: - name: log_paths type: "[]string" - default: ["/var/log/mongodb/mongodb.log*"] + default: + - "/var/log/mongodb/mongodb.log*" + - "/var/log/mongodb/mongod.log*" - name: start_at type: string supported: From 7069195a869c5001c8c123394bff2826e521a51a Mon Sep 17 00:00:00 2001 From: Corbin Phelps Date: Tue, 26 Apr 2022 14:54:18 -0400 Subject: [PATCH 3/4] Fixedup mondb plugin Signed-off-by: Corbin Phelps --- plugins/mongodb_logs.yaml | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/plugins/mongodb_logs.yaml b/plugins/mongodb_logs.yaml index 1bb37df77..b02280a25 100644 --- a/plugins/mongodb_logs.yaml +++ b/plugins/mongodb_logs.yaml @@ -17,11 +17,11 @@ template: | receivers: filelog: include: - {{ range $fp := .daemon_log_paths }} + {{ range $fp := .log_paths }} - '{{ $fp }}' {{end}} exclude: - {{ range $fp := .daemon_log_paths }} + {{ range $fp := .log_paths }} - '{{ $fp }}*.gz' {{end}} start_at: {{ .start_at }} @@ -36,7 +36,7 @@ template: | type: regex_parser regex: '^(?P\S+)\s+(?P\w+)\s+(?P[\w-]+)\s+\[(?P\S+)\]\s+(?P.*)$' timestamp: - parse_from: attributes:timestamp + parse_from: attributes.timestamp #2019-02-06T09:22:43.967-0500 layout: '%Y-%m-%dT%H:%M:%S.%f%z' severity: @@ -53,13 +53,13 @@ template: | - D3 - D4 - D5 - output: add_plugin_id + output: add_log_type - id: 4_4_parser type: json_parser parse_from: body timestamp: - parse_from: attributes.t.date + parse_from: attributes.t.$date #2020-11-03T14:24:07.436-05:00 layout: '%Y-%m-%dT%H:%M:%S.%f%j' severity: @@ -76,11 +76,15 @@ template: | - D3 - D4 - D5 - output: remove_t + output: move_component + # Commented out remove operatore out and output until remove operator is added in. + # Tracking in PR https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/9545 + #output: remove_t + - - id: remove_t - type: remove - field: attributes.t + # - id: remove_t + # type: remove + # field: attributes.t - id: move_component type: move From 9e4d7556c1607f66abb4f14aadb064496df9df96 Mon Sep 17 00:00:00 2001 From: Corbin Phelps Date: Tue, 26 Apr 2022 14:55:00 -0400 Subject: [PATCH 4/4] Added example log line to mongodb plugin Signed-off-by: Corbin Phelps --- plugins/mongodb_logs.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/mongodb_logs.yaml b/plugins/mongodb_logs.yaml index b02280a25..ec8eb6098 100644 --- a/plugins/mongodb_logs.yaml +++ b/plugins/mongodb_logs.yaml @@ -55,6 +55,8 @@ template: | - D5 output: add_log_type + # Example log line: + # {"t":{"$date":"2022-04-26T16:15:44.876+00:00"},"s":"I", "c":"INDEX", "id":20345, "ctx":"LogicalSessionCacheRefresh","msg":"Index build: done building","attr":{"buildUUID":null,"namespace":"config.system.sessions","index":"lsidTTLIndex","commitTimestamp":null}} - id: 4_4_parser type: json_parser parse_from: body