[elasticsearch] add static and pipeline tests (#4122)

* audit system test * message is ecs field * gc system test * sample event * deprecation logs test * server logs tests * slowlog test * readme update * format * index metrics test * index_summary test * node metrics test * shard metrics test * readme * index_recovery test * node_stats metrics test * remove pipeline test * update readme * add pipeline tests for logs * add ccr sample_event for static test * deprecation pipeline test * remove system tests * cleanup * remove node_stats test * comment container * Revert "comment container" This reverts commit 075372a. * fix gc logs timestamp * fix slowlog timestamp processor * fix server logs timestamp processor * fix deprecation timestamp processor * fix audit logs timestamp processor
elastic · Sep 12, 2022 · 027b248 · 027b248
1 parent 57a42bf
commit 027b248
Show file tree

Hide file tree

Showing 53 changed files with 2,968 additions and 769 deletions.
diff --git a/packages/elasticsearch/_dev/deploy/docker/scripts/generate-logs.sh b/packages/elasticsearch/_dev/deploy/docker/scripts/generate-logs.sh
@@ -125,5 +125,5 @@ do
       }
   }'
 
-  sleep 5
+  sleep 10
 done
diff --git a/packages/elasticsearch/data_stream/audit/_dev/test/pipeline/test-audit-logs.log b/packages/elasticsearch/data_stream/audit/_dev/test/pipeline/test-audit-logs.log
@@ -0,0 +1,3 @@
+{"type":"audit", "timestamp":"2022-09-04T22:54:53,028+0000", "cluster.uuid":"sh0FdC0tRUGgzD6U7OsO3g", "node.id":"rsRsMdvhREeQqLkk3twtqA", "event.type":"transport", "event.action":"access_granted", "authentication.type":"REALM", "user.name":"elastic", "user.realm":"reserved", "user.roles":["superuser"], "origin.type":"rest", "origin.address":"172.19.0.3:48524", "request.id":"sdzMxhL5Rga_wTaN7_pfsw", "action":"indices:admin/create", "request.name":"CreateIndexRequest", "indices":["test_2"]}
+{"type":"audit", "timestamp":"2022-09-04T22:54:53,034+0000", "cluster.uuid":"sh0FdC0tRUGgzD6U7OsO3g", "node.id":"rsRsMdvhREeQqLkk3twtqA", "event.type":"rest", "event.action":"anonymous_access_denied", "origin.type":"rest", "origin.address":"172.19.0.3:48526", "url.path":"/test_3", "request.method":"PUT", "request.id":"kDWih8w0SC6mY7Q5ExEI2w", "opaque_id":"myApp1", "trace.id":"0af7651916cd43dd8448eb211c80319c"}
+{"type":"audit", "timestamp":"2022-09-04T22:54:53,040+0000", "cluster.uuid":"sh0FdC0tRUGgzD6U7OsO3g", "node.id":"rsRsMdvhREeQqLkk3twtqA", "event.type":"transport", "event.action":"access_granted", "authentication.type":"REALM", "user.name":"elastic", "user.realm":"reserved", "user.roles":["superuser"], "origin.type":"rest", "origin.address":"172.19.0.3:48528", "request.id":"fTP-0rxJQyyZUNGIs4Hpdg", "action":"indices:admin/create", "request.name":"CreateIndexRequest", "indices":["testindex2"], "opaque_id":"myAppId", "trace.id":"0af7651916cd43dd8448eb211c80319c"}
diff --git a/...ages/elasticsearch/data_stream/audit/_dev/test/pipeline/test-audit-logs.log-expected.json b/...ages/elasticsearch/data_stream/audit/_dev/test/pipeline/test-audit-logs.log-expected.json
@@ -0,0 +1,201 @@
+{
+    "expected": [
+        {
+            "@timestamp": "2022-09-04T22:54:53.028Z",
+            "elasticsearch": {
+                "audit": {
+                    "action": "indices:admin/create",
+                    "authentication.type": "REALM",
+                    "cluster": {},
+                    "event": {},
+                    "indices": [
+                        "test_2"
+                    ],
+                    "layer": "transport",
+                    "origin": {},
+                    "origin.type": "rest",
+                    "request": {
+                        "id": "sdzMxhL5Rga_wTaN7_pfsw"
+                    },
+                    "request.name": "CreateIndexRequest",
+                    "user": {},
+                    "user.realm": "reserved",
+                    "user.roles": [
+                        "superuser"
+                    ]
+                },
+                "cluster": {
+                    "uuid": "sh0FdC0tRUGgzD6U7OsO3g"
+                },
+                "node": {
+                    "id": "rsRsMdvhREeQqLkk3twtqA"
+                }
+            },
+            "event": {
+                "action": "access_granted",
+                "category": "database",
+                "ingested": "2022-09-04T23:00:22.485831147Z",
+                "kind": "event",
+                "outcome": "success"
+            },
+            "host": {
+                "id": "rsRsMdvhREeQqLkk3twtqA"
+            },
+            "http": {
+                "request": {
+                    "id": "sdzMxhL5Rga_wTaN7_pfsw"
+                }
+            },
+            "log": {
+                "level": "info"
+            },
+            "message": "{\"type\":\"audit\", \"timestamp\":\"2022-09-04T22:54:53,028+0000\", \"cluster.uuid\":\"sh0FdC0tRUGgzD6U7OsO3g\", \"node.id\":\"rsRsMdvhREeQqLkk3twtqA\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"authentication.type\":\"REALM\", \"user.name\":\"elastic\", \"user.realm\":\"reserved\", \"user.roles\":[\"superuser\"], \"origin.type\":\"rest\", \"origin.address\":\"172.19.0.3:48524\", \"request.id\":\"sdzMxhL5Rga_wTaN7_pfsw\", \"action\":\"indices:admin/create\", \"request.name\":\"CreateIndexRequest\", \"indices\":[\"test_2\"]}",
+            "related": {
+                "user": [
+                    "elastic"
+                ]
+            },
+            "service": {
+                "type": "elasticsearch"
+            },
+            "source": {
+                "address": "172.19.0.3:48524",
+                "ip": "172.19.0.3",
+                "port": 48524
+            },
+            "user": {
+                "name": "elastic"
+            }
+        },
+        {
+            "@timestamp": "2022-09-04T22:54:53.034Z",
+            "elasticsearch": {
+                "audit": {
+                    "cluster": {},
+                    "event": {},
+                    "layer": "rest",
+                    "opaque_id": "myApp1",
+                    "origin": {},
+                    "origin.type": "rest",
+                    "request": {
+                        "id": "kDWih8w0SC6mY7Q5ExEI2w"
+                    },
+                    "trace": {},
+                    "url": {}
+                },
+                "cluster": {
+                    "uuid": "sh0FdC0tRUGgzD6U7OsO3g"
+                },
+                "node": {
+                    "id": "rsRsMdvhREeQqLkk3twtqA"
+                }
+            },
+            "event": {
+                "action": "anonymous_access_denied",
+                "category": "database",
+                "ingested": "2022-09-04T23:00:22.485849152Z",
+                "kind": "event",
+                "outcome": "failure"
+            },
+            "host": {
+                "id": "rsRsMdvhREeQqLkk3twtqA"
+            },
+            "http": {
+                "request": {
+                    "id": "kDWih8w0SC6mY7Q5ExEI2w",
+                    "method": "PUT"
+                }
+            },
+            "log": {
+                "level": "info"
+            },
+            "message": "{\"type\":\"audit\", \"timestamp\":\"2022-09-04T22:54:53,034+0000\", \"cluster.uuid\":\"sh0FdC0tRUGgzD6U7OsO3g\", \"node.id\":\"rsRsMdvhREeQqLkk3twtqA\", \"event.type\":\"rest\", \"event.action\":\"anonymous_access_denied\", \"origin.type\":\"rest\", \"origin.address\":\"172.19.0.3:48526\", \"url.path\":\"/test_3\", \"request.method\":\"PUT\", \"request.id\":\"kDWih8w0SC6mY7Q5ExEI2w\", \"opaque_id\":\"myApp1\", \"trace.id\":\"0af7651916cd43dd8448eb211c80319c\"}",
+            "service": {
+                "type": "elasticsearch"
+            },
+            "source": {
+                "address": "172.19.0.3:48526",
+                "ip": "172.19.0.3",
+                "port": 48526
+            },
+            "trace": {
+                "id": "0af7651916cd43dd8448eb211c80319c"
+            },
+            "url": {
+                "original": "/test_3"
+            }
+        },
+        {
+            "@timestamp": "2022-09-04T22:54:53.040Z",
+            "elasticsearch": {
+                "audit": {
+                    "action": "indices:admin/create",
+                    "authentication.type": "REALM",
+                    "cluster": {},
+                    "event": {},
+                    "indices": [
+                        "testindex2"
+                    ],
+                    "layer": "transport",
+                    "opaque_id": "myAppId",
+                    "origin": {},
+                    "origin.type": "rest",
+                    "request": {
+                        "id": "fTP-0rxJQyyZUNGIs4Hpdg"
+                    },
+                    "request.name": "CreateIndexRequest",
+                    "trace": {},
+                    "user": {},
+                    "user.realm": "reserved",
+                    "user.roles": [
+                        "superuser"
+                    ]
+                },
+                "cluster": {
+                    "uuid": "sh0FdC0tRUGgzD6U7OsO3g"
+                },
+                "node": {
+                    "id": "rsRsMdvhREeQqLkk3twtqA"
+                }
+            },
+            "event": {
+                "action": "access_granted",
+                "category": "database",
+                "ingested": "2022-09-04T23:00:22.485851956Z",
+                "kind": "event",
+                "outcome": "success"
+            },
+            "host": {
+                "id": "rsRsMdvhREeQqLkk3twtqA"
+            },
+            "http": {
+                "request": {
+                    "id": "fTP-0rxJQyyZUNGIs4Hpdg"
+                }
+            },
+            "log": {
+                "level": "info"
+            },
+            "message": "{\"type\":\"audit\", \"timestamp\":\"2022-09-04T22:54:53,040+0000\", \"cluster.uuid\":\"sh0FdC0tRUGgzD6U7OsO3g\", \"node.id\":\"rsRsMdvhREeQqLkk3twtqA\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"authentication.type\":\"REALM\", \"user.name\":\"elastic\", \"user.realm\":\"reserved\", \"user.roles\":[\"superuser\"], \"origin.type\":\"rest\", \"origin.address\":\"172.19.0.3:48528\", \"request.id\":\"fTP-0rxJQyyZUNGIs4Hpdg\", \"action\":\"indices:admin/create\", \"request.name\":\"CreateIndexRequest\", \"indices\":[\"testindex2\"], \"opaque_id\":\"myAppId\", \"trace.id\":\"0af7651916cd43dd8448eb211c80319c\"}",
+            "related": {
+                "user": [
+                    "elastic"
+                ]
+            },
+            "service": {
+                "type": "elasticsearch"
+            },
+            "source": {
+                "address": "172.19.0.3:48528",
+                "ip": "172.19.0.3",
+                "port": 48528
+            },
+            "trace": {
+                "id": "0af7651916cd43dd8448eb211c80319c"
+            },
+            "user": {
+                "name": "elastic"
+            }
+        }
+    ]
+}
diff --git a/packages/elasticsearch/data_stream/audit/_dev/test/pipeline/test-common-config.yml b/packages/elasticsearch/data_stream/audit/_dev/test/pipeline/test-common-config.yml
@@ -0,0 +1,3 @@
+dynamic_fields:
+  event.ingested: ".*"
+  event.created: ".*"
diff --git a/packages/elasticsearch/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/elasticsearch/data_stream/audit/elasticsearch/ingest_pipeline/default.yml
@@ -4,9 +4,6 @@ processors:
   - set:
       field: event.ingested
       value: "{{_ingest.timestamp}}"
-  - set:
-      copy_from: "@timestamp"
-      field: event.created
   - grok:
       field: message
       patterns:
@@ -17,7 +14,10 @@ processors:
       if: ctx.first_char != '{'
   - pipeline:
       if: ctx.first_char == '{'
-      name: '{< IngestPipeline "pipeline-json" >}'
+      name: '{{ IngestPipeline "pipeline-json" }}'
+  - set:
+      copy_from: "@timestamp"
+      field: event.created
   - set:
       field: event.kind
       value: event

diff --git a/packages/elasticsearch/data_stream/audit/fields/ecs.yml b/packages/elasticsearch/data_stream/audit/fields/ecs.yml
@@ -1,3 +1,5 @@
+- external: ecs
+  name: ecs.version
 - external: ecs
   name: http
 - external: ecs
@@ -14,3 +16,21 @@
   name: user
 - external: ecs
   name: user.name
+- external: ecs
+  name: http.request.id
+- external: ecs
+  name: http.request.method
+- external: ecs
+  name: log.file.path
+- external: ecs
+  name: log.level
+- external: ecs
+  name: service.type
+- external: ecs
+  name: source.address
+- external: ecs
+  name: source.port
+- external: ecs
+  name: trace.id
+- external: ecs
+  name: message
diff --git a/packages/elasticsearch/data_stream/audit/fields/fields.yml b/packages/elasticsearch/data_stream/audit/fields/fields.yml
@@ -1,6 +1,10 @@
 - name: elasticsearch.audit
   type: group
   fields:
+    - name: authentication.type
+      type: keyword
+    - name: opaque_id
+      type: keyword
     - name: layer
       type: keyword
       description: 'The layer from which this event originated: rest, transport or ip_filter'

diff --git a/packages/elasticsearch/data_stream/audit/fields/package-fields.yml b/packages/elasticsearch/data_stream/audit/fields/package-fields.yml
@@ -1,3 +1,9 @@
+- name: input.type
+  type: keyword
+- name: log.offset
+  type: long
+- name: related.user
+  type: keyword
 - name: elasticsearch
   type: group
   fields: