journald - Add 8.x support (elastic#4037)

Update kibana.constraint to allow 8.1.0+. 8.1.0 is required because elastic/beats#30032 is needed due to a breaking change that was introduced in 8.0.0. Rename message_id to ECS event.code. This field was added in 8.0. From man systemd.journal-fields MESSAGE_ID= A 128-bit message identifier ID for recognizing certain message types, if this is desirable. ...
WildDogOne · Aug 18, 2022 · 4966fd3 · 4966fd3
1 parent 01bc513
commit 4966fd3
Show file tree

Hide file tree

Showing 8 changed files with 234 additions and 8 deletions.
diff --git a/packages/journald/_dev/build/build.yml b/packages/journald/_dev/build/build.yml
@@ -1,3 +1,3 @@
 dependencies:
   ecs:
-    reference: git@1.11
+    reference: git@v8.4.0-rc1
diff --git a/packages/journald/_dev/build/docs/README.md b/packages/journald/_dev/build/docs/README.md
@@ -5,4 +5,6 @@ The journald input reads the log data and the metadata associated with it.
 
 The journald input is available on Linux systems with `systemd` installed.
 
-{{fields "log"}}
+{{event "log"}}
+
+{{fields "log"}}
diff --git a/packages/journald/changelog.yml b/packages/journald/changelog.yml
@@ -1,4 +1,12 @@
 # newer versions go on top
+- version: "0.0.4"
+  changes:
+    - description: Update Kibana requirement to support 8.x.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/4037
+    - description: Map `message_id` to ECS `event.code`.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/4037
 - version: "0.0.3"
   changes:
     - description: Add documentation for multi-fields

diff --git a/packages/journald/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/journald/data_stream/log/elasticsearch/ingest_pipeline/default.yml
@@ -32,6 +32,11 @@ processors:
       target_field: log.syslog.identifier
       ignore_missing: true
 
+  - rename:
+      field: message_id
+      target_field: event.code
+      ignore_missing: true
+
   # Cleanup an empty syslog object.
   - remove:
       if: ctx?.syslog != null && ctx.syslog instanceof Map && ctx.syslog.isEmpty()

diff --git a/packages/journald/data_stream/log/fields/ecs.yml b/packages/journald/data_stream/log/fields/ecs.yml
@@ -1,5 +1,7 @@
 - name: ecs.version
   external: ecs
+- name: event.code
+  external: ecs
 - name: host.hostname
   external: ecs
 - name: host.id

diff --git a/packages/journald/data_stream/log/sample_event.json b/packages/journald/data_stream/log/sample_event.json
@@ -0,0 +1,102 @@
+{
+    "@timestamp": "2020-07-22T13:17:10.012Z",
+    "agent": {
+        "ephemeral_id": "27e2a00a-dab2-4790-8d45-29ad272d0392",
+        "id": "bef8099b-68f6-4621-8089-2229b35a669d",
+        "name": "docker-fleet-agent",
+        "type": "filebeat",
+        "version": "8.3.2"
+    },
+    "data_stream": {
+        "dataset": "journald.log",
+        "namespace": "ep",
+        "type": "logs"
+    },
+    "ecs": {
+        "version": "8.0.0"
+    },
+    "elastic_agent": {
+        "id": "bef8099b-68f6-4621-8089-2229b35a669d",
+        "snapshot": false,
+        "version": "8.3.2"
+    },
+    "event": {
+        "agent_id_status": "verified",
+        "code": "ec387f577b844b8fa948f33cad9a75e6",
+        "created": "2022-08-18T18:14:11.588Z",
+        "dataset": "journald.log",
+        "ingested": "2022-08-18T18:14:15Z",
+        "kind": "event"
+    },
+    "host": {
+        "hostname": "sleipnir",
+        "id": "505afdafda3b4f33a63749ae39284742"
+    },
+    "input": {
+        "type": "journald"
+    },
+    "journald": {
+        "custom": {
+            "available": "0",
+            "available_pretty": "0B",
+            "current_use": "1023455232",
+            "current_use_pretty": "976.0M",
+            "disk_available": "6866636800",
+            "disk_available_pretty": "6.3G",
+            "disk_keep_free": "1466253312",
+            "disk_keep_free_pretty": "1.3G",
+            "journal_name": "System journal",
+            "journal_path": "/var/log/journal/505afdafda3b4f33a63749ae39284742",
+            "limit": "977502208",
+            "limit_pretty": "932.2M",
+            "max_use": "977502208",
+            "max_use_pretty": "932.2M"
+        },
+        "gid": 0,
+        "host": {
+            "boot_id": "fa3c2e3080dc4cd5be5cb5a43e140d51"
+        },
+        "pid": 19317,
+        "process": {
+            "capabilities": "25402800cf",
+            "command_line": "/lib/systemd/systemd-journald",
+            "executable": "/lib/systemd/systemd-journald",
+            "name": "systemd-journal"
+        },
+        "uid": 0
+    },
+    "log": {
+        "syslog": {
+            "facility": {
+                "code": 3
+            },
+            "identifier": "systemd-journald",
+            "priority": 6
+        }
+    },
+    "message": "System journal (/var/log/journal/505afdafda3b4f33a63749ae39284742) is 976.0M, max 932.2M, 0B free.",
+    "process": {
+        "args": [
+            "/lib/systemd/systemd-journald"
+        ],
+        "args_count": 1,
+        "command_line": "/lib/systemd/systemd-journald",
+        "pid": 19317
+    },
+    "systemd": {
+        "cgroup": "/system.slice/systemd-journald.service",
+        "invocation_id": "7c11cda63635437bafe21c92851618a8",
+        "slice": "system.slice",
+        "transport": "driver",
+        "unit": "systemd-journald.service"
+    },
+    "tags": [
+        "forwarded"
+    ],
+    "user": {
+        "group": {
+            "id": "0"
+        },
+        "id": "0"
+    }
+}
diff --git a/packages/journald/docs/README.md b/packages/journald/docs/README.md
@@ -5,6 +5,113 @@ The journald input reads the log data and the metadata associated with it.
 
 The journald input is available on Linux systems with `systemd` installed.
 
+An example event for `log` looks as following:
+
+```json
+{
+    "@timestamp": "2020-07-22T13:17:10.012Z",
+    "agent": {
+        "ephemeral_id": "27e2a00a-dab2-4790-8d45-29ad272d0392",
+        "id": "bef8099b-68f6-4621-8089-2229b35a669d",
+        "name": "docker-fleet-agent",
+        "type": "filebeat",
+        "version": "8.3.2"
+    },
+    "data_stream": {
+        "dataset": "journald.log",
+        "namespace": "ep",
+        "type": "logs"
+    },
+    "ecs": {
+        "version": "8.0.0"
+    },
+    "elastic_agent": {
+        "id": "bef8099b-68f6-4621-8089-2229b35a669d",
+        "snapshot": false,
+        "version": "8.3.2"
+    },
+    "event": {
+        "agent_id_status": "verified",
+        "code": "ec387f577b844b8fa948f33cad9a75e6",
+        "created": "2022-08-18T18:14:11.588Z",
+        "dataset": "journald.log",
+        "ingested": "2022-08-18T18:14:15Z",
+        "kind": "event"
+    },
+    "host": {
+        "hostname": "sleipnir",
+        "id": "505afdafda3b4f33a63749ae39284742"
+    },
+    "input": {
+        "type": "journald"
+    },
+    "journald": {
+        "custom": {
+            "available": "0",
+            "available_pretty": "0B",
+            "current_use": "1023455232",
+            "current_use_pretty": "976.0M",
+            "disk_available": "6866636800",
+            "disk_available_pretty": "6.3G",
+            "disk_keep_free": "1466253312",
+            "disk_keep_free_pretty": "1.3G",
+            "journal_name": "System journal",
+            "journal_path": "/var/log/journal/505afdafda3b4f33a63749ae39284742",
+            "limit": "977502208",
+            "limit_pretty": "932.2M",
+            "max_use": "977502208",
+            "max_use_pretty": "932.2M"
+        },
+        "gid": 0,
+        "host": {
+            "boot_id": "fa3c2e3080dc4cd5be5cb5a43e140d51"
+        },
+        "pid": 19317,
+        "process": {
+            "capabilities": "25402800cf",
+            "command_line": "/lib/systemd/systemd-journald",
+            "executable": "/lib/systemd/systemd-journald",
+            "name": "systemd-journal"
+        },
+        "uid": 0
+    },
+    "log": {
+        "syslog": {
+            "facility": {
+                "code": 3
+            },
+            "identifier": "systemd-journald",
+            "priority": 6
+        }
+    },
+    "message": "System journal (/var/log/journal/505afdafda3b4f33a63749ae39284742) is 976.0M, max 932.2M, 0B free.",
+    "process": {
+        "args": [
+            "/lib/systemd/systemd-journald"
+        ],
+        "args_count": 1,
+        "command_line": "/lib/systemd/systemd-journald",
+        "pid": 19317
+    },
+    "systemd": {
+        "cgroup": "/system.slice/systemd-journald.service",
+        "invocation_id": "7c11cda63635437bafe21c92851618a8",
+        "slice": "system.slice",
+        "transport": "driver",
+        "unit": "systemd-journald.service"
+    },
+    "tags": [
+        "forwarded"
+    ],
+    "user": {
+        "group": {
+            "id": "0"
+        },
+        "id": "0"
+    }
+}
+```
+
 **Exported fields**
 
 | Field | Description | Type |
@@ -16,6 +123,7 @@ The journald input is available on Linux systems with `systemd` installed.
 | data_stream.namespace | Data stream namespace. | constant_keyword |
 | data_stream.type | Data stream type. | constant_keyword |
 | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword |
+| event.code | Identification code for this event, if one exists. Some event sources use event codes to identify messages unambiguously, regardless of message language or wording adjustments over time. An example of this is the Windows Event ID. | keyword |
 | host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword |
 | host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword |
 | input.type |  | keyword |
@@ -56,11 +164,11 @@ The journald input is available on Linux systems with `systemd` installed.
 | log.syslog.identifier | Identifier (usually process) contained in the syslog header. | keyword |
 | log.syslog.pid | PID contained in the syslog header. | long |
 | log.syslog.priority | Syslog numeric priority of the event, if available. According to RFCs 5424 and 3164, the priority is 8 \* facility + severity. This number is therefore expected to contain a value between 0 and 191. | long |
-| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | text |
+| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text |
 | process.args | Array of process arguments, starting with the absolute path to the executable. May be filtered to protect sensitive information. | keyword |
 | process.args_count | Length of the process.args array. This field can be useful for querying or performing bucket analysis on how many arguments were provided to start a process. More arguments may be an indication of suspicious activity. | long |
-| process.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | keyword |
-| process.command_line.text | Multi-field of `process.command_line`. | text |
+| process.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | wildcard |
+| process.command_line.text | Multi-field of `process.command_line`. | match_only_text |
 | process.pid | Process id. | long |
 | systemd.cgroup | The control group path in the systemd hierarchy. | keyword |
 | systemd.invocation_id | The invocation ID for the runtime cycle of the unit the message was generated in, as available to processes of the unit in $INVOCATION_ID. | keyword |
@@ -74,4 +182,3 @@ The journald input is available on Linux systems with `systemd` installed.
 | tags | List of keywords used to tag each event. | keyword |
 | user.group.id | Unique identifier for the group on the system/platform. | keyword |
 | user.id | Unique identifier of the user. | keyword |
-
diff --git a/packages/journald/manifest.yml b/packages/journald/manifest.yml
@@ -1,15 +1,15 @@
 format_version: 1.0.0
 name: journald
 title: "Custom Journald logs"
-version: 0.0.3
+version: 0.0.4
 license: basic
 description: Collect logs from journald with Elastic Agent.
 type: integration
 categories:
   - custom
 release: experimental
 conditions:
-  kibana.version: "^7.16.0"
+  kibana.version: "^7.17.0 || ^8.1.0"
 icons:
   - src: /img/systemd-logo.svg
     title: systemd logo