From 175b3db24633a9f5a3664ccdda9d5fc002e8e2e5 Mon Sep 17 00:00:00 2001 From: Lei Qiu Date: Sat, 23 Nov 2019 19:25:14 +0000 Subject: [PATCH 1/6] Add ssl config to input.yml for Filebeat httpjson input --- x-pack/filebeat/module/misp/threat/config/input.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/filebeat/module/misp/threat/config/input.yml b/x-pack/filebeat/module/misp/threat/config/input.yml index 302d535ce5e..ca06ddcd002 100644 --- a/x-pack/filebeat/module/misp/threat/config/input.yml +++ b/x-pack/filebeat/module/misp/threat/config/input.yml @@ -10,6 +10,7 @@ interval: {{ .interval }} json_objects_array: {{ .json_objects_array }} pagination: {{ .pagination }} url: {{ .url }} +ssl: {{ .ssl }} {{ else if eq .input "file" }} From fb51e96d734cf2c99419f7d3b760442e4aaa06db Mon Sep 17 00:00:00 2001 From: Ray Qiu Date: Mon, 25 Nov 2019 09:08:47 -0800 Subject: [PATCH 2/6] Fix CHANGELOG.next.asciidoc --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 39fbb2b1d2c..8ec0738f5cb 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -198,6 +198,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix parsing of Elasticsearch node name by `elasticsearch/slowlog` fileset. {pull}14547[14547] - cisco/asa fileset: Fix parsing of 302021 message code. {pull}14519[14519] - Fix filebeat azure dashboards, event category should be `Alert`. {pull}14668[14668] +- Fix a problem in Filebeat input httpjson where interval is not used as time.Duration. {issue}14752[14752] *Heartbeat* From 3148c649e30cd42ab523cd93eefb3bafe9813ddb Mon Sep 17 00:00:00 2001 From: Ray Qiu Date: Wed, 11 Dec 2019 09:56:35 -0800 Subject: [PATCH 3/6] Fix merge problems --- CHANGELOG.next.asciidoc | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index adce621b65f..c849550ba4c 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -212,7 +212,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Update Logstash module's Grok patterns to support Logstash 7.4 logs. {pull}14743[14743] - Fix a problem in Filebeat input httpjson where interval is not used as time.Duration. {issue}14752[14752] {pull}14753[14753] - Fix SSL config in input.yml for Filebeat httpjson input in the MISP module. {pull}14767[14767] -- Fix a problem in Filebeat input httpjson where interval is not used as time.Duration. {issue}14752[14752] *Heartbeat* From 6de44ce9febc567340673ac46e6f40ffec39d4b5 Mon Sep 17 00:00:00 2001 From: Lei Qiu Date: Wed, 22 Jan 2020 05:59:12 +0000 Subject: [PATCH 4/6] Set event.outcome based on googlecloud audit log output --- .../googlecloud/audit/config/pipeline.js | 23 ++++++++ .../audit/test/audit-log-entries.json.log | 3 +- .../audit-log-entries.json.log-expected.json | 53 ++++++++++++++++++- 3 files changed, 76 insertions(+), 3 deletions(-) diff --git a/x-pack/filebeat/module/googlecloud/audit/config/pipeline.js b/x-pack/filebeat/module/googlecloud/audit/config/pipeline.js index 93d8b8648f1..1a928be1049 100644 --- a/x-pack/filebeat/module/googlecloud/audit/config/pipeline.js +++ b/x-pack/filebeat/module/googlecloud/audit/config/pipeline.js @@ -123,6 +123,28 @@ function Audit(keep_original_message) { } }; + // Set event.outcome based on authenticatio_info and status + var setEventOutcome = function(evt) { + if (evt.Get("googlecloud.audit.status.code") == null) { + var authorization_info = evt.Get("googlecloud.audit.authorization_info"); + if (authorization_info.length == 1) { + if (authorization_info[0].granted == null) { + evt.Put("event.outcome", "unknown"); + } else if (authorization_info[0].granted == true) { + evt.Put("event.outcome", "success"); + } else { + evt.Put("event.outcome", "failure"); + } + } else { + evt.Put("event.outcome", "unknown"); + } + } else if (evt.Get("googlecloud.audit.status.code") == 0) { + evt.Put("event.outcome", "success"); + } else { + evt.Put("event.outcome", "failure"); + } + }; + var pipeline = new processor.Chain() .Add(decodeJson) .Add(parseTimestamp) @@ -135,6 +157,7 @@ function Audit(keep_original_message) { .Add(copyFields) .Add(dropExtraFields) .Add(RenameNestedFields) + .Add(setEventOutcome) .Build(); return { diff --git a/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log b/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log index e42e9106287..2120a297a5f 100644 --- a/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log +++ b/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log @@ -1,3 +1,4 @@ {"insertId":"-uihnmjctwo","logName":"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"xxx@xxx.xxx"},"authorizationInfo":[{"granted":true,"permission":"resourcemanager.projects.get","resource":"projects/elastic-beats","resourceAttributes":{}}],"methodName":"GetResourceBillingInfo","request":{"@type":"type.googleapis.com/google.internal.cloudbilling.billingaccount.v1.GetResourceBillingInfoRequest","resourceName":"projects/189716325846"},"requestMetadata":{"callerIp":"192.168.1.1","destinationAttributes":{},"requestAttributes":{}},"resourceName":"projects/elastic-beats","serviceName":"cloudbilling.googleapis.com","status":{}},"receiveTimestamp":"2019-12-19T00:49:36.313482371Z","resource":{"labels":{"project_id":"elastic-beats"},"type":"project"},"severity":"INFO","timestamp":"2019-12-19T00:49:36.086Z"} -{"insertId":"-h6onuze1h7dg","logName":"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"xxx@xxx.xxx"},"authorizationInfo":[{"granted":true,"permission":"compute.machineTypes.list","resourceAttributes":{"name":"projects/elastic-beats","service":"resourcemanager","type":"resourcemanager.projects"}}],"methodName":"beta.compute.machineTypes.aggregatedList","numResponseItems":"71","request":{"@type":"type.googleapis.com/compute.machineTypes.aggregatedList"},"requestMetadata":{"callerIp":"192.168.1.1","callerSuppliedUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2019-12-19T00:45:51.711Z"}},"resourceLocation":{"currentLocations":["global"]},"resourceName":"projects/elastic-beats/global/machineTypes","serviceName":"compute.googleapis.com"},"receiveTimestamp":"2019-12-19T00:45:52.367887078Z","resource":{"labels":{"location":"global","method":"compute.machineTypes.aggregatedList","project_id":"elastic-beats","service":"compute.googleapis.com","version":"beta"},"type":"api"},"severity":"INFO","timestamp":"2019-12-19T00:45:51.228Z"} +{"insertId":"-h6onuze1h7dg","logName":"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"xxx@xxx.xxx"},"authorizationInfo":[{"granted":false,"permission":"compute.machineTypes.list","resourceAttributes":{"name":"projects/elastic-beats","service":"resourcemanager","type":"resourcemanager.projects"}}],"methodName":"beta.compute.machineTypes.aggregatedList","numResponseItems":"71","request":{"@type":"type.googleapis.com/compute.machineTypes.aggregatedList"},"requestMetadata":{"callerIp":"192.168.1.1","callerSuppliedUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2019-12-19T00:45:51.711Z"}},"resourceLocation":{"currentLocations":["global"]},"resourceName":"projects/elastic-beats/global/machineTypes","serviceName":"compute.googleapis.com"},"receiveTimestamp":"2019-12-19T00:45:52.367887078Z","resource":{"labels":{"location":"global","method":"compute.machineTypes.aggregatedList","project_id":"elastic-beats","service":"compute.googleapis.com","version":"beta"},"type":"api"},"severity":"INFO","timestamp":"2019-12-19T00:45:51.228Z"} {"insertId":"yonau2dg2zi","logName":"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"xxx@xxx.xxx"},"authorizationInfo":[{"granted":true,"permission":"compute.instances.list","resourceAttributes":{"name":"projects/elastic-beats","service":"resourcemanager","type":"resourcemanager.projects"}}],"methodName":"beta.compute.instances.aggregatedList","numResponseItems":"61","request":{"@type":"type.googleapis.com/compute.instances.aggregatedList"},"requestMetadata":{"callerIp":"192.168.1.1","callerSuppliedUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2019-12-19T00:44:25.198Z"}},"resourceLocation":{"currentLocations":["global"]},"resourceName":"projects/elastic-beats/global/instances","serviceName":"compute.googleapis.com"},"receiveTimestamp":"2019-12-19T00:44:25.262379373Z","resource":{"labels":{"location":"global","method":"compute.instances.aggregatedList","project_id":"elastic-beats","service":"compute.googleapis.com","version":"beta"},"type":"api"},"severity":"INFO","timestamp":"2019-12-19T00:44:25.051Z"} +{"insertId":"yonau3dc2zi","logName":"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"xxx@xxx.xxx"},"authorizationInfo":[{"permission":"compute.instances.list","resourceAttributes":{"name":"projects/elastic-beats","service":"resourcemanager","type":"resourcemanager.projects"}}],"methodName":"beta.compute.instances.aggregatedList","numResponseItems":"61","request":{"@type":"type.googleapis.com/compute.instances.aggregatedList"},"requestMetadata":{"callerIp":"192.168.1.1","callerSuppliedUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2019-12-19T00:44:25.198Z"}},"resourceLocation":{"currentLocations":["global"]},"resourceName":"projects/elastic-beats/global/instances","serviceName":"compute.googleapis.com","status":{"code":7,"message":"PERMISSION_DENIED"}},"receiveTimestamp":"2019-12-19T00:44:25.262379373Z","resource":{"labels":{"location":"global","method":"compute.instances.aggregatedList","project_id":"elastic-beats","service":"compute.googleapis.com","version":"beta"},"type":"api"},"severity":"INFO","timestamp":"2019-12-19T00:44:25.051Z"} diff --git a/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log-expected.json b/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log-expected.json index d303ad1076c..37ef7275861 100644 --- a/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log-expected.json +++ b/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log-expected.json @@ -4,6 +4,7 @@ "cloud.project.id": "elastic-beats", "event.dataset": "googlecloud.audit", "event.module": "googlecloud", + "event.outcome": "success", "fileset.name": "audit", "googlecloud.audit.authentication_info.principal_email": "xxx@xxx.xxx", "googlecloud.audit.authorization_info": [ @@ -34,11 +35,12 @@ "cloud.project.id": "elastic-beats", "event.dataset": "googlecloud.audit", "event.module": "googlecloud", + "event.outcome": "failure", "fileset.name": "audit", "googlecloud.audit.authentication_info.principal_email": "xxx@xxx.xxx", "googlecloud.audit.authorization_info": [ { - "granted": true, + "granted": false, "permission": "compute.machineTypes.list", "resource_attributes": { "name": "projects/elastic-beats", @@ -78,6 +80,7 @@ "cloud.project.id": "elastic-beats", "event.dataset": "googlecloud.audit", "event.module": "googlecloud", + "event.outcome": "success", "fileset.name": "audit", "googlecloud.audit.authentication_info.principal_email": "xxx@xxx.xxx", "googlecloud.audit.authorization_info": [ @@ -104,7 +107,53 @@ "googlecloud.audit.type": "type.googleapis.com/google.cloud.audit.AuditLog", "input.type": "log", "log.logger": "projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access", - "log.offset": 2251, + "log.offset": 2252, + "service.name": "compute.googleapis.com", + "service.type": "googlecloud", + "source.ip": "192.168.1.1", + "user.email": "xxx@xxx.xxx", + "user_agent.device.name": "Other", + "user_agent.name": "Firefox", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)", + "user_agent.os.full": "Mac OS X 10.15", + "user_agent.os.name": "Mac OS X", + "user_agent.os.version": "10.15", + "user_agent.version": "71.0." + }, + { + "@timestamp": "2019-12-19T00:44:25.051Z", + "cloud.project.id": "elastic-beats", + "event.dataset": "googlecloud.audit", + "event.module": "googlecloud", + "event.outcome": "failure", + "fileset.name": "audit", + "googlecloud.audit.authentication_info.principal_email": "xxx@xxx.xxx", + "googlecloud.audit.authorization_info": [ + { + "permission": "compute.instances.list", + "resource_attributes": { + "name": "projects/elastic-beats", + "service": "resourcemanager", + "type": "resourcemanager.projects" + } + } + ], + "googlecloud.audit.method_name": "beta.compute.instances.aggregatedList", + "googlecloud.audit.num_response_items": 61, + "googlecloud.audit.request.proto_name": "type.googleapis.com/compute.instances.aggregatedList", + "googlecloud.audit.request_metadata.caller_ip": "192.168.1.1", + "googlecloud.audit.request_metadata.caller_supplied_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)", + "googlecloud.audit.resource_location.current_locations": [ + "global" + ], + "googlecloud.audit.resource_name": "projects/elastic-beats/global/instances", + "googlecloud.audit.service_name": "compute.googleapis.com", + "googlecloud.audit.status.code": 7, + "googlecloud.audit.status.message": "PERMISSION_DENIED", + "googlecloud.audit.type": "type.googleapis.com/google.cloud.audit.AuditLog", + "input.type": "log", + "log.logger": "projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access", + "log.offset": 3541, "service.name": "compute.googleapis.com", "service.type": "googlecloud", "source.ip": "192.168.1.1", From a363763d8ad81ed976cc196d514f36909116aaa9 Mon Sep 17 00:00:00 2001 From: Lei Qiu Date: Wed, 22 Jan 2020 06:06:15 +0000 Subject: [PATCH 5/6] Update CHANGELOG.next.asciidoc --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 252e214e044..9319ab2212e 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -74,7 +74,7 @@ TLS or Beats that accept connections over TLS and validate client certificates. *Filebeat* - +- Set event.outcome field based on googlecloud audit log output. {pull}15731[15731] *Heartbeat* From a84cfc9eb478b8e721ed7b26312da30dbe30a3ff Mon Sep 17 00:00:00 2001 From: Lei Qiu Date: Thu, 23 Jan 2020 01:15:14 +0000 Subject: [PATCH 6/6] Fix some problems according to review comments --- .../module/googlecloud/audit/config/pipeline.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/x-pack/filebeat/module/googlecloud/audit/config/pipeline.js b/x-pack/filebeat/module/googlecloud/audit/config/pipeline.js index 1a928be1049..65819ff6a92 100644 --- a/x-pack/filebeat/module/googlecloud/audit/config/pipeline.js +++ b/x-pack/filebeat/module/googlecloud/audit/config/pipeline.js @@ -118,19 +118,19 @@ function Audit(keep_original_message) { var RenameNestedFields = function(evt) { var arr = evt.Get("googlecloud.audit.authorization_info"); for (var i = 0; i < arr.length; i++) { - arr[i].resource_attributes = arr[i].resourceAttributes; - delete arr[i].resourceAttributes; + arr[i].resource_attributes = arr[i].resourceAttributes; + delete arr[i].resourceAttributes; } }; - // Set event.outcome based on authenticatio_info and status + // Set event.outcome based on authentication_info and status. var setEventOutcome = function(evt) { if (evt.Get("googlecloud.audit.status.code") == null) { var authorization_info = evt.Get("googlecloud.audit.authorization_info"); - if (authorization_info.length == 1) { + if (authorization_info.length === 1) { if (authorization_info[0].granted == null) { evt.Put("event.outcome", "unknown"); - } else if (authorization_info[0].granted == true) { + } else if (authorization_info[0].granted === true) { evt.Put("event.outcome", "success"); } else { evt.Put("event.outcome", "failure"); @@ -138,10 +138,10 @@ function Audit(keep_original_message) { } else { evt.Put("event.outcome", "unknown"); } - } else if (evt.Get("googlecloud.audit.status.code") == 0) { - evt.Put("event.outcome", "success"); + } else if (evt.Get("googlecloud.audit.status.code") === 0) { + evt.Put("event.outcome", "success"); } else { - evt.Put("event.outcome", "failure"); + evt.Put("event.outcome", "failure"); } };