From b9ffdef2ce223bd931059a0d1a48e95c1d3020a6 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Mon, 7 Jun 2021 09:30:19 -0500 Subject: [PATCH 1/2] Improve cloud region and AZ field descriptions (#1452) * improve region and az descriptions * artifacts * changelog --- CHANGELOG.next.md | 1 + code/go/ecs/cloud.go | 4 ++-- docs/field-details.asciidoc | 4 ++-- experimental/generated/beats/fields.ecs.yml | 4 ++-- experimental/generated/csv/fields.csv | 4 ++-- experimental/generated/ecs/ecs_flat.yml | 8 ++++---- experimental/generated/ecs/ecs_nested.yml | 8 ++++---- generated/beats/fields.ecs.yml | 4 ++-- generated/csv/fields.csv | 4 ++-- generated/ecs/ecs_flat.yml | 8 ++++---- generated/ecs/ecs_nested.yml | 8 ++++---- schemas/cloud.yml | 4 ++-- 12 files changed, 31 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index 96b1beaa47..30ea326764 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -30,6 +30,7 @@ Thanks, you're awesome :-) --> * Fix ecs GitHub repo link source branch #1393 * Add --exclude flag to Generator to support field removal testing #1411 * Explicitly include user identifiers in `relater.user` description. #1420 +* Improve descriptions for `cloud.region` and `cloud.availability` fields. #1452 #### Deprecated diff --git a/code/go/ecs/cloud.go b/code/go/ecs/cloud.go index 13b7ff551a..66a208b47f 100644 --- a/code/go/ecs/cloud.go +++ b/code/go/ecs/cloud.go @@ -25,10 +25,10 @@ type Cloud struct { // digitalocean. Provider string `ecs:"provider"` - // Availability zone in which this host is running. + // Availability zone in which this host, resource, or service is located. AvailabilityZone string `ecs:"availability_zone"` - // Region in which this host is running. + // Region in which this host, resource, or service is located. Region string `ecs:"region"` // Instance ID of the host machine. diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index 62dceb76ba..5e1d02175c 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -607,7 +607,7 @@ example: `elastic-dev` [[field-cloud-availability-zone]] <> -| Availability zone in which this host is running. +| Availability zone in which this host, resource, or service is located. type: keyword @@ -723,7 +723,7 @@ example: `aws` [[field-cloud-region]] <> -| Region in which this host is running. +| Region in which this host, resource, or service is located. type: keyword diff --git a/experimental/generated/beats/fields.ecs.yml b/experimental/generated/beats/fields.ecs.yml index 4b550cd2a7..4585e042d7 100644 --- a/experimental/generated/beats/fields.ecs.yml +++ b/experimental/generated/beats/fields.ecs.yml @@ -456,7 +456,7 @@ level: extended type: keyword ignore_above: 1024 - description: Availability zone in which this host is running. + description: Availability zone in which this host, resource, or service is located. example: us-east-1c - name: instance.id level: extended @@ -504,7 +504,7 @@ level: extended type: keyword ignore_above: 1024 - description: Region in which this host is running. + description: Region in which this host, resource, or service is located. example: us-east-1 - name: service.name level: extended diff --git a/experimental/generated/csv/fields.csv b/experimental/generated/csv/fields.csv index c5f6939dc9..2c08b3da77 100644 --- a/experimental/generated/csv/fields.csv +++ b/experimental/generated/csv/fields.csv @@ -49,14 +49,14 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 2.0.0-dev+exp,true,client,client.user.roles,keyword,extended,array,"[""kibana_admin"", ""reporting_user""]",Array of user roles at the time of the event. 2.0.0-dev+exp,true,cloud,cloud.account.id,keyword,extended,,666777888999,The cloud account or organization id. 2.0.0-dev+exp,true,cloud,cloud.account.name,keyword,extended,,elastic-dev,The cloud account name. -2.0.0-dev+exp,true,cloud,cloud.availability_zone,keyword,extended,,us-east-1c,Availability zone in which this host is running. +2.0.0-dev+exp,true,cloud,cloud.availability_zone,keyword,extended,,us-east-1c,"Availability zone in which this host, resource, or service is located." 2.0.0-dev+exp,true,cloud,cloud.instance.id,keyword,extended,,i-1234567890abcdef0,Instance ID of the host machine. 2.0.0-dev+exp,true,cloud,cloud.instance.name,keyword,extended,,,Instance name of the host machine. 2.0.0-dev+exp,true,cloud,cloud.machine.type,keyword,extended,,t2.medium,Machine type of the host machine. 2.0.0-dev+exp,true,cloud,cloud.project.id,keyword,extended,,my-project,The cloud project id. 2.0.0-dev+exp,true,cloud,cloud.project.name,keyword,extended,,my project,The cloud project name. 2.0.0-dev+exp,true,cloud,cloud.provider,keyword,extended,,aws,Name of the cloud provider. -2.0.0-dev+exp,true,cloud,cloud.region,keyword,extended,,us-east-1,Region in which this host is running. +2.0.0-dev+exp,true,cloud,cloud.region,keyword,extended,,us-east-1,"Region in which this host, resource, or service is located." 2.0.0-dev+exp,true,cloud,cloud.service.name,keyword,extended,,lambda,The cloud service name. 2.0.0-dev+exp,true,container,container.id,keyword,core,,,Unique container id. 2.0.0-dev+exp,true,container,container.image.name,keyword,extended,,,Name of the image the container was built on. diff --git a/experimental/generated/ecs/ecs_flat.yml b/experimental/generated/ecs/ecs_flat.yml index 9862f6f9a6..be633be0bd 100644 --- a/experimental/generated/ecs/ecs_flat.yml +++ b/experimental/generated/ecs/ecs_flat.yml @@ -581,14 +581,14 @@ cloud.account.name: type: keyword cloud.availability_zone: dashed_name: cloud-availability-zone - description: Availability zone in which this host is running. + description: Availability zone in which this host, resource, or service is located. example: us-east-1c flat_name: cloud.availability_zone ignore_above: 1024 level: extended name: availability_zone normalize: [] - short: Availability zone in which this host is running. + short: Availability zone in which this host, resource, or service is located. type: keyword cloud.instance.id: dashed_name: cloud-instance-id @@ -662,14 +662,14 @@ cloud.provider: type: keyword cloud.region: dashed_name: cloud-region - description: Region in which this host is running. + description: Region in which this host, resource, or service is located. example: us-east-1 flat_name: cloud.region ignore_above: 1024 level: extended name: region normalize: [] - short: Region in which this host is running. + short: Region in which this host, resource, or service is located. type: keyword cloud.service.name: dashed_name: cloud-service-name diff --git a/experimental/generated/ecs/ecs_nested.yml b/experimental/generated/ecs/ecs_nested.yml index cf674b67f9..29a8f5f666 100644 --- a/experimental/generated/ecs/ecs_nested.yml +++ b/experimental/generated/ecs/ecs_nested.yml @@ -754,14 +754,14 @@ cloud: type: keyword cloud.availability_zone: dashed_name: cloud-availability-zone - description: Availability zone in which this host is running. + description: Availability zone in which this host, resource, or service is located. example: us-east-1c flat_name: cloud.availability_zone ignore_above: 1024 level: extended name: availability_zone normalize: [] - short: Availability zone in which this host is running. + short: Availability zone in which this host, resource, or service is located. type: keyword cloud.instance.id: dashed_name: cloud-instance-id @@ -835,14 +835,14 @@ cloud: type: keyword cloud.region: dashed_name: cloud-region - description: Region in which this host is running. + description: Region in which this host, resource, or service is located. example: us-east-1 flat_name: cloud.region ignore_above: 1024 level: extended name: region normalize: [] - short: Region in which this host is running. + short: Region in which this host, resource, or service is located. type: keyword cloud.service.name: dashed_name: cloud-service-name diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index 31bef9cb32..70254fbe2d 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -465,7 +465,7 @@ level: extended type: keyword ignore_above: 1024 - description: Availability zone in which this host is running. + description: Availability zone in which this host, resource, or service is located. example: us-east-1c - name: instance.id level: extended @@ -513,7 +513,7 @@ level: extended type: keyword ignore_above: 1024 - description: Region in which this host is running. + description: Region in which this host, resource, or service is located. example: us-east-1 - name: service.name level: extended diff --git a/generated/csv/fields.csv b/generated/csv/fields.csv index 7e4afe9103..a08d5174b8 100644 --- a/generated/csv/fields.csv +++ b/generated/csv/fields.csv @@ -49,14 +49,14 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 2.0.0-dev,true,client,client.user.roles,keyword,extended,array,"[""kibana_admin"", ""reporting_user""]",Array of user roles at the time of the event. 2.0.0-dev,true,cloud,cloud.account.id,keyword,extended,,666777888999,The cloud account or organization id. 2.0.0-dev,true,cloud,cloud.account.name,keyword,extended,,elastic-dev,The cloud account name. -2.0.0-dev,true,cloud,cloud.availability_zone,keyword,extended,,us-east-1c,Availability zone in which this host is running. +2.0.0-dev,true,cloud,cloud.availability_zone,keyword,extended,,us-east-1c,"Availability zone in which this host, resource, or service is located." 2.0.0-dev,true,cloud,cloud.instance.id,keyword,extended,,i-1234567890abcdef0,Instance ID of the host machine. 2.0.0-dev,true,cloud,cloud.instance.name,keyword,extended,,,Instance name of the host machine. 2.0.0-dev,true,cloud,cloud.machine.type,keyword,extended,,t2.medium,Machine type of the host machine. 2.0.0-dev,true,cloud,cloud.project.id,keyword,extended,,my-project,The cloud project id. 2.0.0-dev,true,cloud,cloud.project.name,keyword,extended,,my project,The cloud project name. 2.0.0-dev,true,cloud,cloud.provider,keyword,extended,,aws,Name of the cloud provider. -2.0.0-dev,true,cloud,cloud.region,keyword,extended,,us-east-1,Region in which this host is running. +2.0.0-dev,true,cloud,cloud.region,keyword,extended,,us-east-1,"Region in which this host, resource, or service is located." 2.0.0-dev,true,cloud,cloud.service.name,keyword,extended,,lambda,The cloud service name. 2.0.0-dev,true,container,container.id,keyword,core,,,Unique container id. 2.0.0-dev,true,container,container.image.name,keyword,extended,,,Name of the image the container was built on. diff --git a/generated/ecs/ecs_flat.yml b/generated/ecs/ecs_flat.yml index eafdc69658..c4addd7373 100644 --- a/generated/ecs/ecs_flat.yml +++ b/generated/ecs/ecs_flat.yml @@ -589,14 +589,14 @@ cloud.account.name: type: keyword cloud.availability_zone: dashed_name: cloud-availability-zone - description: Availability zone in which this host is running. + description: Availability zone in which this host, resource, or service is located. example: us-east-1c flat_name: cloud.availability_zone ignore_above: 1024 level: extended name: availability_zone normalize: [] - short: Availability zone in which this host is running. + short: Availability zone in which this host, resource, or service is located. type: keyword cloud.instance.id: dashed_name: cloud-instance-id @@ -670,14 +670,14 @@ cloud.provider: type: keyword cloud.region: dashed_name: cloud-region - description: Region in which this host is running. + description: Region in which this host, resource, or service is located. example: us-east-1 flat_name: cloud.region ignore_above: 1024 level: extended name: region normalize: [] - short: Region in which this host is running. + short: Region in which this host, resource, or service is located. type: keyword cloud.service.name: dashed_name: cloud-service-name diff --git a/generated/ecs/ecs_nested.yml b/generated/ecs/ecs_nested.yml index a17651cff0..d02e82464f 100644 --- a/generated/ecs/ecs_nested.yml +++ b/generated/ecs/ecs_nested.yml @@ -760,14 +760,14 @@ cloud: type: keyword cloud.availability_zone: dashed_name: cloud-availability-zone - description: Availability zone in which this host is running. + description: Availability zone in which this host, resource, or service is located. example: us-east-1c flat_name: cloud.availability_zone ignore_above: 1024 level: extended name: availability_zone normalize: [] - short: Availability zone in which this host is running. + short: Availability zone in which this host, resource, or service is located. type: keyword cloud.instance.id: dashed_name: cloud-instance-id @@ -841,14 +841,14 @@ cloud: type: keyword cloud.region: dashed_name: cloud-region - description: Region in which this host is running. + description: Region in which this host, resource, or service is located. example: us-east-1 flat_name: cloud.region ignore_above: 1024 level: extended name: region normalize: [] - short: Region in which this host is running. + short: Region in which this host, resource, or service is located. type: keyword cloud.service.name: dashed_name: cloud-service-name diff --git a/schemas/cloud.yml b/schemas/cloud.yml index 789b2d7485..ef0651ba63 100644 --- a/schemas/cloud.yml +++ b/schemas/cloud.yml @@ -28,14 +28,14 @@ example: us-east-1c type: keyword description: > - Availability zone in which this host is running. + Availability zone in which this host, resource, or service is located. - name: region level: extended type: keyword example: us-east-1 description: > - Region in which this host is running. + Region in which this host, resource, or service is located. - name: instance.id level: extended From 0416af591fbc2c0f539497e4cf1de997000301e3 Mon Sep 17 00:00:00 2001 From: Ross Wolf <31489089+rw-access@users.noreply.github.com> Date: Wed, 9 Jun 2021 10:11:44 -0600 Subject: [PATCH 2/2] [RFC] Continue target process RFC to stage 1 (#1297) Co-authored-by: Eric Beahan --- rfcs/text/0016-target-process.md | 83 +++++++++++++++++++++++++++++--- 1 file changed, 76 insertions(+), 7 deletions(-) diff --git a/rfcs/text/0016-target-process.md b/rfcs/text/0016-target-process.md index fd8d41f575..3aa9bb74ec 100644 --- a/rfcs/text/0016-target-process.md +++ b/rfcs/text/0016-target-process.md @@ -1,8 +1,8 @@ # 0016: Target process fields -- Stage: **0 (strawperson)** -- Date: **2021-03-09** +- Stage: **1 (draft)** +- Date: **2021-06-09** @@ -29,6 +29,27 @@ The most common use cases for Windows: Stage 1: Describe at a high level how this change affects fields. Include new or updated yml field definitions for all of the essential fields in this draft. While not exhaustive, the fields documented here should be comprehensive enough to deeply evaluate the technical considerations of this change. The goal here is to validate the technical details for all essential fields and to provide a basis for adding experimental field definitions to the schema. Use GitHub code blocks with yml syntax formatting. --> +**Stage 1** +This causes reuse of the `process.*` field set at two locations: +* `process.target.*` +* `process.parent.target*` + + +The `process.parent.target` reused fieldset could be descoped if it's too complex or increases the field count too significantly. It does have value, because information of the parent process of the target remains valuable. More on that utility in the next section. + +```yml + reusable: + top_level: true + expected: + - at: process + as: parent + - at: process + as: target + # collect the parent of the target process at process.target.parent + - at: process.target + as: parent +``` + @@ -39,12 +60,60 @@ Stage 2: Add or update all remaining field definitions. The list should now be e Stage 1: Describe at a high-level how these field changes will be used in practice. Real world examples are encouraged. The goal here is to understand how people would leverage these fields to gain insights or solve problems. ~1-3 paragraphs. --> +Target process information is valuable to detect several kinds of attacker behavior, but also good to profile or audit a system. +The most commonly known attacker behaviors where one process directly accesses another: + +* Process injection [T1055](https://attack.mitre.org/techniques/T1055/) +* Access token manipulation [T1134](https://attack.mitre.org/techniques/T1134/) +* Credential theft from lsass [T1003.001](https://attack.mitre.org/techniques/T1003/001/) + + +Here are some example detections that could be written in KQL: + +| Example rule name | KQL query | +| ------------------------------ | ------------------------------------------------------------------------------------------------------------ | +| Injection to a browser | event.action : "process_injection" and process.target.name : ("GoogleChrome.exe", "iexplore.exe", "firefox.exe") | +| Token theft from explorer | event.action : "token_theft" and process.name : (not "explorer.exe") and process.target.name : "explorer.exe" | +| Injection to a service process | event.action : "process_injection" and process.target.parent.name : "services.exe" | +| Password dumping from lsass | event.action : "process_memory_read" and process.target.name : "lsass.exe" | +| Generic process access | event.action : "process_access" an process.target.name : * | + + ## Source data +Example sources of data include EDR-like products that collect operating system telemetry. Although cross-process events are more commonly known with Windows (injection, memory reads), they are also possible with Linux and macOS. The most universal use case across operating systems is attaching remote debuggers, which could be used for benign or malicious purposes. + + +Example event from Microsoft Sysmon [source](https://www.ultimatewindowssecurity.com/securitylog/encyclopedia/event.aspx?eventid=90010), which is used by Winlogbeat: + + Process accessed: + UtcTime: 2017-05-15 00:02:01.463 + SourceProcessGUID: {d49b2de5-efa6-5918-0000-00104d553c00} + SourceProcessId: 4704 + SourceThreadId: 4124 + SourceImage: C:\mimikatz\x64\mimikatz.exe + TargetProcessGUID: {d49b2de5-e852-5918-0000-00100b0f0700} + TargetProcessId: 1576 + TargetImage: C:\Windows\system32\winlogon.exe + GrantedAccess: 0x40 + CallTrace: C:\Windows\SYSTEM32\ntdll.dll+a5594|C:\Windows\system32\KERNELBASE.dll+1e865|C:\mimikatz\x64\mimikatz.exe+77ad|C:\mimikatz\x64\mimikatz.exe+7759|C:\mimikatz\x64\mimikatz.exe+f095|C:\mimikatz\x64\mimikatz.exe+6610a|C:\mimikatz\x64\mimikatz.exe+65dc4|C:\mimikatz\x64\mimikatz.exe+4ac00|C:\mimikatz\x64\mimikatz.exe+4aa36|C:\mimikatz\x64\mimikatz.exe+4a81d|C:\mimikatz\x64\mimikatz.exe+6ebe5|C:\Windows\system32\KERNEL32.DLL+18102|C:\Windows\SYSTEM32\ntdll.dll+5c5b4 + +The `Target*` fields of the Sysmon event would map: +* `TargetProcessGUID` -> `process.target.entity_id` +* `TargetProcessID` -> `process.target.pid` +* `TargetProcessImage` -> `process.target.executable` and `process.target.name` + +The `Source*` fields of the Sysmon would map: +* `SourceProcessGUID` -> `process.entity_id` +* `SourceProcessId` -> `process.pid` +* `SourceThreadId` -> `process.thread.tid` (side question: does it make sense to move `thread.*` from `process`?) +* `SourceImage` -> `process.executable` and `process.name` + + @@ -65,10 +134,13 @@ The goal here is to research and understand the impact of these changes on users ## Concerns + +The biggest concern is the duplication of fields and the double-nested `process` group at `process.target.parent`. This could require some updates to our reuse mechanism, but that's an issue internal to this repository. We should make sure that we don't accidentally populate `process.parent.target`, which would have different meaning. Because of this, we will need to make sure that we articulate what each reuse means, similar to https://www.elastic.co/guide/en/ecs/current/ecs-user.html#ecs-user-nestings. + @@ -83,6 +155,7 @@ The following are the people that consulted on the contents of this RFC. * @rw-access | author * @andrewstucki | co-author +* @devonakerr | sponsor @@ -95,8 +168,4 @@ The following are the people that consulted on the contents of this RFC. * Stage 0: https://github.com/elastic/ecs/pull/1286 - - +* Stage 1: https://github.com/elastic/ecs/pull/1297