Merge remote-tracking branch 'elastic/master' into feature/event-agen…

…t-id-status

CHANGELOG.next.md

@@ -31,6 +31,7 @@ Thanks, you're awesome :-) -->
 * Fix ecs GitHub repo link source branch #1393
 * Add --exclude flag to Generator to support field removal testing #1411
 * Explicitly include user identifiers in `relater.user` description. #1420
+* Improve descriptions for `cloud.region` and `cloud.availability` fields. #1452
 
 #### Deprecated
 

docs/field-details.asciidoc

@@ -607,7 +607,7 @@ example: `elastic-dev`
 [[field-cloud-availability-zone]]
 <<field-cloud-availability-zone, cloud.availability_zone>>
 
-| Availability zone in which this host is running.
+| Availability zone in which this host, resource, or service is located.
 
 type: keyword
 
@@ -723,7 +723,7 @@ example: `aws`
 [[field-cloud-region]]
 <<field-cloud-region, cloud.region>>
 
-| Region in which this host is running.
+| Region in which this host, resource, or service is located.
 
 type: keyword
 

experimental/generated/beats/fields.ecs.yml

@@ -456,7 +456,7 @@
       level: extended
       type: keyword
       ignore_above: 1024
-      description: Availability zone in which this host is running.
+      description: Availability zone in which this host, resource, or service is located.
       example: us-east-1c
     - name: instance.id
       level: extended
@@ -504,7 +504,7 @@
       level: extended
       type: keyword
       ignore_above: 1024
-      description: Region in which this host is running.
+      description: Region in which this host, resource, or service is located.
       example: us-east-1
     - name: service.name
       level: extended

experimental/generated/csv/fields.csv

@@ -49,14 +49,14 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description
 2.0.0-dev+exp,true,client,client.user.roles,keyword,extended,array,"[""kibana_admin"", ""reporting_user""]",Array of user roles at the time of the event.
 2.0.0-dev+exp,true,cloud,cloud.account.id,keyword,extended,,666777888999,The cloud account or organization id.
 2.0.0-dev+exp,true,cloud,cloud.account.name,keyword,extended,,elastic-dev,The cloud account name.
-2.0.0-dev+exp,true,cloud,cloud.availability_zone,keyword,extended,,us-east-1c,Availability zone in which this host is running.
+2.0.0-dev+exp,true,cloud,cloud.availability_zone,keyword,extended,,us-east-1c,"Availability zone in which this host, resource, or service is located."
 2.0.0-dev+exp,true,cloud,cloud.instance.id,keyword,extended,,i-1234567890abcdef0,Instance ID of the host machine.
 2.0.0-dev+exp,true,cloud,cloud.instance.name,keyword,extended,,,Instance name of the host machine.
 2.0.0-dev+exp,true,cloud,cloud.machine.type,keyword,extended,,t2.medium,Machine type of the host machine.
 2.0.0-dev+exp,true,cloud,cloud.project.id,keyword,extended,,my-project,The cloud project id.
 2.0.0-dev+exp,true,cloud,cloud.project.name,keyword,extended,,my project,The cloud project name.
 2.0.0-dev+exp,true,cloud,cloud.provider,keyword,extended,,aws,Name of the cloud provider.
-2.0.0-dev+exp,true,cloud,cloud.region,keyword,extended,,us-east-1,Region in which this host is running.
+2.0.0-dev+exp,true,cloud,cloud.region,keyword,extended,,us-east-1,"Region in which this host, resource, or service is located."
 2.0.0-dev+exp,true,cloud,cloud.service.name,keyword,extended,,lambda,The cloud service name.
 2.0.0-dev+exp,true,container,container.id,keyword,core,,,Unique container id.
 2.0.0-dev+exp,true,container,container.image.name,keyword,extended,,,Name of the image the container was built on.

experimental/generated/ecs/ecs_flat.yml

@@ -581,14 +581,14 @@ cloud.account.name:
   type: keyword
 cloud.availability_zone:
   dashed_name: cloud-availability-zone
-  description: Availability zone in which this host is running.
+  description: Availability zone in which this host, resource, or service is located.
   example: us-east-1c
   flat_name: cloud.availability_zone
   ignore_above: 1024
   level: extended
   name: availability_zone
   normalize: []
-  short: Availability zone in which this host is running.
+  short: Availability zone in which this host, resource, or service is located.
   type: keyword
 cloud.instance.id:
   dashed_name: cloud-instance-id
@@ -662,14 +662,14 @@ cloud.provider:
   type: keyword
 cloud.region:
   dashed_name: cloud-region
-  description: Region in which this host is running.
+  description: Region in which this host, resource, or service is located.
   example: us-east-1
   flat_name: cloud.region
   ignore_above: 1024
   level: extended
   name: region
   normalize: []
-  short: Region in which this host is running.
+  short: Region in which this host, resource, or service is located.
   type: keyword
 cloud.service.name:
   dashed_name: cloud-service-name

experimental/generated/ecs/ecs_nested.yml

@@ -754,14 +754,14 @@ cloud:
       type: keyword
     cloud.availability_zone:
       dashed_name: cloud-availability-zone
-      description: Availability zone in which this host is running.
+      description: Availability zone in which this host, resource, or service is located.
       example: us-east-1c
       flat_name: cloud.availability_zone
       ignore_above: 1024
       level: extended
       name: availability_zone
       normalize: []
-      short: Availability zone in which this host is running.
+      short: Availability zone in which this host, resource, or service is located.
       type: keyword
     cloud.instance.id:
       dashed_name: cloud-instance-id
@@ -835,14 +835,14 @@ cloud:
       type: keyword
     cloud.region:
       dashed_name: cloud-region
-      description: Region in which this host is running.
+      description: Region in which this host, resource, or service is located.
       example: us-east-1
       flat_name: cloud.region
       ignore_above: 1024
       level: extended
       name: region
       normalize: []
-      short: Region in which this host is running.
+      short: Region in which this host, resource, or service is located.
       type: keyword
     cloud.service.name:
       dashed_name: cloud-service-name

generated/beats/fields.ecs.yml

@@ -465,7 +465,7 @@
       level: extended
       type: keyword
       ignore_above: 1024
-      description: Availability zone in which this host is running.
+      description: Availability zone in which this host, resource, or service is located.
       example: us-east-1c
     - name: instance.id
       level: extended
@@ -513,7 +513,7 @@
       level: extended
       type: keyword
       ignore_above: 1024
-      description: Region in which this host is running.
+      description: Region in which this host, resource, or service is located.
       example: us-east-1
     - name: service.name
       level: extended

generated/csv/fields.csv

@@ -49,14 +49,14 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description
 2.0.0-dev,true,client,client.user.roles,keyword,extended,array,"[""kibana_admin"", ""reporting_user""]",Array of user roles at the time of the event.
 2.0.0-dev,true,cloud,cloud.account.id,keyword,extended,,666777888999,The cloud account or organization id.
 2.0.0-dev,true,cloud,cloud.account.name,keyword,extended,,elastic-dev,The cloud account name.
-2.0.0-dev,true,cloud,cloud.availability_zone,keyword,extended,,us-east-1c,Availability zone in which this host is running.
+2.0.0-dev,true,cloud,cloud.availability_zone,keyword,extended,,us-east-1c,"Availability zone in which this host, resource, or service is located."
 2.0.0-dev,true,cloud,cloud.instance.id,keyword,extended,,i-1234567890abcdef0,Instance ID of the host machine.
 2.0.0-dev,true,cloud,cloud.instance.name,keyword,extended,,,Instance name of the host machine.
 2.0.0-dev,true,cloud,cloud.machine.type,keyword,extended,,t2.medium,Machine type of the host machine.
 2.0.0-dev,true,cloud,cloud.project.id,keyword,extended,,my-project,The cloud project id.
 2.0.0-dev,true,cloud,cloud.project.name,keyword,extended,,my project,The cloud project name.
 2.0.0-dev,true,cloud,cloud.provider,keyword,extended,,aws,Name of the cloud provider.
-2.0.0-dev,true,cloud,cloud.region,keyword,extended,,us-east-1,Region in which this host is running.
+2.0.0-dev,true,cloud,cloud.region,keyword,extended,,us-east-1,"Region in which this host, resource, or service is located."
 2.0.0-dev,true,cloud,cloud.service.name,keyword,extended,,lambda,The cloud service name.
 2.0.0-dev,true,container,container.id,keyword,core,,,Unique container id.
 2.0.0-dev,true,container,container.image.name,keyword,extended,,,Name of the image the container was built on.

generated/ecs/ecs_flat.yml

@@ -589,14 +589,14 @@ cloud.account.name:
   type: keyword
 cloud.availability_zone:
   dashed_name: cloud-availability-zone
-  description: Availability zone in which this host is running.
+  description: Availability zone in which this host, resource, or service is located.
   example: us-east-1c
   flat_name: cloud.availability_zone
   ignore_above: 1024
   level: extended
   name: availability_zone
   normalize: []
-  short: Availability zone in which this host is running.
+  short: Availability zone in which this host, resource, or service is located.
   type: keyword
 cloud.instance.id:
   dashed_name: cloud-instance-id
@@ -670,14 +670,14 @@ cloud.provider:
   type: keyword
 cloud.region:
   dashed_name: cloud-region
-  description: Region in which this host is running.
+  description: Region in which this host, resource, or service is located.
   example: us-east-1
   flat_name: cloud.region
   ignore_above: 1024
   level: extended
   name: region
   normalize: []
-  short: Region in which this host is running.
+  short: Region in which this host, resource, or service is located.
   type: keyword
 cloud.service.name:
   dashed_name: cloud-service-name

generated/ecs/ecs_nested.yml

@@ -760,14 +760,14 @@ cloud:
       type: keyword
     cloud.availability_zone:
       dashed_name: cloud-availability-zone
-      description: Availability zone in which this host is running.
+      description: Availability zone in which this host, resource, or service is located.
       example: us-east-1c
       flat_name: cloud.availability_zone
       ignore_above: 1024
       level: extended
       name: availability_zone
       normalize: []
-      short: Availability zone in which this host is running.
+      short: Availability zone in which this host, resource, or service is located.
       type: keyword
     cloud.instance.id:
       dashed_name: cloud-instance-id
@@ -841,14 +841,14 @@ cloud:
       type: keyword
     cloud.region:
       dashed_name: cloud-region
-      description: Region in which this host is running.
+      description: Region in which this host, resource, or service is located.
       example: us-east-1
       flat_name: cloud.region
       ignore_above: 1024
       level: extended
       name: region
       normalize: []
-      short: Region in which this host is running.
+      short: Region in which this host, resource, or service is located.
       type: keyword
     cloud.service.name:
       dashed_name: cloud-service-name

rfcs/text/0016-target-process.md

@@ -1,8 +1,8 @@
 # 0016: Target process fields
 <!-- Leave this ID at 0000. The ECS team will assign a unique, contiguous RFC number upon merging the initial stage of this RFC. -->
 
-- Stage: **0 (strawperson)** <!-- Update to reflect target stage. See https://elastic.github.io/ecs/stages.html -->
-- Date: **2021-03-09** <!-- The ECS team sets this date at merge time. This is the date of the latest stage advancement. -->
+- Stage: **1 (draft)** <!-- Update to reflect target stage. See https://elastic.github.io/ecs/stages.html -->
+- Date: **2021-06-09** <!-- The ECS team sets this date at merge time. This is the date of the latest stage advancement. -->
 
 
 
@@ -29,6 +29,27 @@ The most common use cases for Windows:
 Stage 1: Describe at a high level how this change affects fields. Include new or updated yml field definitions for all of the essential fields in this draft. While not exhaustive, the fields documented here should be comprehensive enough to deeply evaluate the technical considerations of this change. The goal here is to validate the technical details for all essential fields and to provide a basis for adding experimental field definitions to the schema. Use GitHub code blocks with yml syntax formatting.
 -->
 
+**Stage 1**
+This causes reuse of the `process.*` field set at two locations:
+* `process.target.*`
+* `process.parent.target*`
+
+
+The `process.parent.target` reused fieldset could be descoped if it's too complex or increases the field count too significantly. It does have value, because information of the parent process of the target remains valuable. More on that utility in the next section.
+
+```yml
+  reusable:
+    top_level: true
+    expected:
+      - at: process
+        as: parent
+      - at: process
+        as: target
+      # collect the parent of the target process at process.target.parent
+      - at: process.target
+        as: parent
+```
+
 <!--
 Stage 2: Add or update all remaining field definitions. The list should now be exhaustive. The goal here is to validate the technical details of all remaining fields and to provide a basis for releasing these field definitions as beta in the schema. Use GitHub code blocks with yml syntax formatting.
 -->
@@ -39,12 +60,60 @@ Stage 2: Add or update all remaining field definitions. The list should now be e
 Stage 1: Describe at a high-level how these field changes will be used in practice. Real world examples are encouraged. The goal here is to understand how people would leverage these fields to gain insights or solve problems. ~1-3 paragraphs.
 -->
 
+Target process information is valuable to detect several kinds of attacker behavior, but also good to profile or audit a system.
+The most commonly known attacker behaviors where one process directly accesses another:
+
+* Process injection [T1055](https://attack.mitre.org/techniques/T1055/)
+* Access token manipulation [T1134](https://attack.mitre.org/techniques/T1134/)
+* Credential theft from lsass [T1003.001](https://attack.mitre.org/techniques/T1003/001/)
+
+
+Here are some example detections that could be written in KQL:
+
+| Example rule name              | KQL query                                                                                                    |
+| ------------------------------ | ------------------------------------------------------------------------------------------------------------ |
+| Injection to a browser         | event.action : "process_injection" and process.target.name : ("GoogleChrome.exe", "iexplore.exe", "firefox.exe") |
+| Token theft from explorer      | event.action : "token_theft" and process.name : (not "explorer.exe") and process.target.name : "explorer.exe" |
+| Injection to a service process | event.action : "process_injection" and process.target.parent.name : "services.exe" |
+| Password dumping from lsass    | event.action : "process_memory_read" and process.target.name : "lsass.exe" |
+| Generic process access         | event.action : "process_access" an process.target.name : * |
+
+
 ## Source data
 
 <!--
 Stage 1: Provide a high-level description of example sources of data. This does not yet need to be a concrete example of a source document, but instead can simply describe a potential source (e.g. nginx access log). This will ultimately be fleshed out to include literal source examples in a future stage. The goal here is to identify practical sources for these fields in the real world. ~1-3 sentences or unordered list.
 -->
 
+Example sources of data include EDR-like products that collect operating system telemetry. Although cross-process events are more commonly known with Windows (injection, memory reads), they are also possible with Linux and macOS. The most universal use case across operating systems is attaching remote debuggers, which could be used for benign or malicious purposes.
+
+
+Example event from Microsoft Sysmon [source](https://www.ultimatewindowssecurity.com/securitylog/encyclopedia/event.aspx?eventid=90010), which is used by Winlogbeat:
+
+    Process accessed:
+    UtcTime: 2017-05-15 00:02:01.463
+    SourceProcessGUID: {d49b2de5-efa6-5918-0000-00104d553c00}
+    SourceProcessId: 4704
+    SourceThreadId: 4124
+    SourceImage: C:\mimikatz\x64\mimikatz.exe
+    TargetProcessGUID: {d49b2de5-e852-5918-0000-00100b0f0700}
+    TargetProcessId: 1576
+    TargetImage: C:\Windows\system32\winlogon.exe
+    GrantedAccess: 0x40
+    CallTrace: C:\Windows\SYSTEM32\ntdll.dll+a5594|C:\Windows\system32\KERNELBASE.dll+1e865|C:\mimikatz\x64\mimikatz.exe+77ad|C:\mimikatz\x64\mimikatz.exe+7759|C:\mimikatz\x64\mimikatz.exe+f095|C:\mimikatz\x64\mimikatz.exe+6610a|C:\mimikatz\x64\mimikatz.exe+65dc4|C:\mimikatz\x64\mimikatz.exe+4ac00|C:\mimikatz\x64\mimikatz.exe+4aa36|C:\mimikatz\x64\mimikatz.exe+4a81d|C:\mimikatz\x64\mimikatz.exe+6ebe5|C:\Windows\system32\KERNEL32.DLL+18102|C:\Windows\SYSTEM32\ntdll.dll+5c5b4
+
+The `Target*` fields of the Sysmon event would map:
+* `TargetProcessGUID` -> `process.target.entity_id`
+* `TargetProcessID` -> `process.target.pid`
+* `TargetProcessImage` -> `process.target.executable` and `process.target.name`
+
+The `Source*` fields of the Sysmon would map:
+* `SourceProcessGUID` -> `process.entity_id`
+* `SourceProcessId` -> `process.pid`
+* `SourceThreadId` -> `process.thread.tid` (side question: does it make sense to move `thread.*` from `process`?)
+* `SourceImage` -> `process.executable` and `process.name`
+
+
 <!--
 Stage 2: Included a real world example source document. Ideally this example comes from the source(s) identified in stage 1. If not, it should replace them. The goal here is to validate the utility of these field changes in the context of a real world example. Format with the source name as a ### header and the example document in a GitHub code block with json formatting.
 -->
@@ -65,10 +134,13 @@ The goal here is to research and understand the impact of these changes on users
 
 ## Concerns
 
+
 <!--
 Stage 1: Identify potential concerns, implementation challenges, or complexity. Spend some time on this. Play devil's advocate. Try to identify the sort of non-obvious challenges that tend to surface later. The goal here is to surface risks early, allow everyone the time to work through them, and ultimately document resolution for posterity's sake.
 -->
 
+The biggest concern is the duplication of fields and the double-nested `process` group at `process.target.parent`. This could require some updates to our reuse mechanism, but that's an issue internal to this repository. We should make sure that we don't accidentally populate `process.parent.target`, which would have different meaning. Because of this, we will need to make sure that we articulate what each reuse means, similar to https://www.elastic.co/guide/en/ecs/current/ecs-user.html#ecs-user-nestings.
+
 <!--
 Stage 2: Document new concerns or resolutions to previously listed concerns. It's not critical that all concerns have resolutions at this point, but it would be helpful if resolutions were taking shape for the most significant concerns.
 -->
@@ -83,6 +155,7 @@ The following are the people that consulted on the contents of this RFC.
 
 * @rw-access    | author
 * @andrewstucki | co-author
+* @devonakerr   | sponsor
 
 
 
@@ -95,8 +168,4 @@ The following are the people that consulted on the contents of this RFC.
 <!-- An RFC should link to the PRs for each of it stage advancements. -->
 
 * Stage 0: https://github.com/elastic/ecs/pull/1286
-
-<!--
-* Stage 1: https://github.com/elastic/ecs/pull/NNN
-...
--->
+* Stage 1: https://github.com/elastic/ecs/pull/1297

schemas/cloud.yml

@@ -28,14 +28,14 @@
       example: us-east-1c
       type: keyword
       description: >
-        Availability zone in which this host is running.
+        Availability zone in which this host, resource, or service is located.
 
     - name: region
       level: extended
       type: keyword
       example: us-east-1
       description: >
-        Region in which this host is running.
+        Region in which this host, resource, or service is located.
 
     - name: instance.id
       level: extended