[1.x] Add ssdeep hash (elastic#1169) (elastic#1227)

Co-authored-by: Andrew Stucki <andrew.stucki@elastic.co>

CHANGELOG.next.md

@@ -19,6 +19,7 @@ Thanks, you're awesome :-) -->
 
 * Added `http.request.id`. #1208
 * Added `cloud.service.name`. #1204
+* Added `hash.ssdeep`. #1169
 
 #### Improvements
 

docs/field-details.asciidoc

@@ -3023,10 +3023,12 @@ Note also that the `group` fields may be used directly at the root of the events
 [[ecs-hash]]
 === Hash Fields
 
-The hash fields represent different hash algorithms and their values.
+The hash fields represent different bitwise hash algorithms and their values.
 
 Field names for common hashes (e.g. MD5, SHA1) are predefined. Add fields for other hashes by lowercasing the hash algorithm name and using underscore separators as appropriate (snake case, e.g. sha3_512).
 
+Note that this fieldset is used for common hashes that may be computed over a range of generic bytes. Entity-specific hashes such as ja3 or imphash are placed in the fieldsets to which they relate (tls and pe, respectively).
+
 [discrete]
 ==== Hash Field Details
 
@@ -3096,6 +3098,22 @@ type: keyword
 
 
 
+| extended
+
+// ===============================================================
+
+|
+[[field-hash-ssdeep]]
+<<field-hash-ssdeep, hash.ssdeep>>
+
+| SSDEEP hash.
+
+type: keyword
+
+
+
+
+
 | extended
 
 // ===============================================================

experimental/generated/beats/fields.ecs.yml

@@ -951,6 +951,12 @@
       ignore_above: 1024
       description: SHA512 hash.
       default_field: false
+    - name: hash.ssdeep
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: SSDEEP hash.
+      default_field: false
     - name: name
       level: core
       type: keyword
@@ -1682,6 +1688,12 @@
       type: keyword
       ignore_above: 1024
       description: SHA512 hash.
+    - name: hash.ssdeep
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: SSDEEP hash.
+      default_field: false
     - name: inode
       level: extended
       type: keyword
@@ -2068,11 +2080,16 @@
   - name: hash
     title: Hash
     group: 2
-    description: 'The hash fields represent different hash algorithms and their values.
+    description: 'The hash fields represent different bitwise hash algorithms and
+      their values.
 
       Field names for common hashes (e.g. MD5, SHA1) are predefined. Add fields for
       other hashes by lowercasing the hash algorithm name and using underscore separators
-      as appropriate (snake case, e.g. sha3_512).'
+      as appropriate (snake case, e.g. sha3_512).
+
+      Note that this fieldset is used for common hashes that may be computed over
+      a range of generic bytes. Entity-specific hashes such as ja3 or imphash are
+      placed in the fieldsets to which they relate (tls and pe, respectively).'
     type: group
     fields:
     - name: md5
@@ -2095,6 +2112,12 @@
       type: keyword
       ignore_above: 1024
       description: SHA512 hash.
+    - name: ssdeep
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: SSDEEP hash.
+      default_field: false
   - name: host
     title: Host
     group: 2
@@ -3500,6 +3523,12 @@
       type: keyword
       ignore_above: 1024
       description: SHA512 hash.
+    - name: hash.ssdeep
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: SSDEEP hash.
+      default_field: false
     - name: name
       level: extended
       type: wildcard
@@ -3645,6 +3674,12 @@
       ignore_above: 1024
       description: SHA512 hash.
       default_field: false
+    - name: parent.hash.ssdeep
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: SSDEEP hash.
+      default_field: false
     - name: parent.name
       level: extended
       type: wildcard

experimental/generated/csv/fields.csv

@@ -108,6 +108,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description
 1.9.0-dev+exp,true,dll,dll.hash.sha1,keyword,extended,,,SHA1 hash.
 1.9.0-dev+exp,true,dll,dll.hash.sha256,keyword,extended,,,SHA256 hash.
 1.9.0-dev+exp,true,dll,dll.hash.sha512,keyword,extended,,,SHA512 hash.
+1.9.0-dev+exp,true,dll,dll.hash.ssdeep,keyword,extended,,,SSDEEP hash.
 1.9.0-dev+exp,true,dll,dll.name,keyword,core,,kernel32.dll,Name of the library.
 1.9.0-dev+exp,true,dll,dll.path,keyword,extended,,C:\Windows\System32\kernel32.dll,Full file path of the library.
 1.9.0-dev+exp,true,dll,dll.pe.architecture,keyword,extended,,x64,CPU architecture target for the file.
@@ -186,6 +187,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description
 1.9.0-dev+exp,true,file,file.hash.sha1,keyword,extended,,,SHA1 hash.
 1.9.0-dev+exp,true,file,file.hash.sha256,keyword,extended,,,SHA256 hash.
 1.9.0-dev+exp,true,file,file.hash.sha512,keyword,extended,,,SHA512 hash.
+1.9.0-dev+exp,true,file,file.hash.ssdeep,keyword,extended,,,SSDEEP hash.
 1.9.0-dev+exp,true,file,file.inode,keyword,extended,,256383,Inode representing the file in the filesystem.
 1.9.0-dev+exp,true,file,file.mime_type,keyword,extended,,,"Media type of file, document, or arrangement of bytes."
 1.9.0-dev+exp,true,file,file.mode,keyword,extended,,0640,Mode of the file in octal representation.
@@ -395,6 +397,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description
 1.9.0-dev+exp,true,process,process.hash.sha1,keyword,extended,,,SHA1 hash.
 1.9.0-dev+exp,true,process,process.hash.sha256,keyword,extended,,,SHA256 hash.
 1.9.0-dev+exp,true,process,process.hash.sha512,keyword,extended,,,SHA512 hash.
+1.9.0-dev+exp,true,process,process.hash.ssdeep,keyword,extended,,,SSDEEP hash.
 1.9.0-dev+exp,true,process,process.name,wildcard,extended,,ssh,Process name.
 1.9.0-dev+exp,true,process,process.name.text,text,extended,,ssh,Process name.
 1.9.0-dev+exp,true,process,process.parent.args,keyword,extended,array,"[""/usr/bin/ssh"", ""-l"", ""user"", ""10.0.0.16""]",Array of process arguments.
@@ -414,6 +417,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description
 1.9.0-dev+exp,true,process,process.parent.hash.sha1,keyword,extended,,,SHA1 hash.
 1.9.0-dev+exp,true,process,process.parent.hash.sha256,keyword,extended,,,SHA256 hash.
 1.9.0-dev+exp,true,process,process.parent.hash.sha512,keyword,extended,,,SHA512 hash.
+1.9.0-dev+exp,true,process,process.parent.hash.ssdeep,keyword,extended,,,SSDEEP hash.
 1.9.0-dev+exp,true,process,process.parent.name,wildcard,extended,,ssh,Process name.
 1.9.0-dev+exp,true,process,process.parent.name.text,text,extended,,ssh,Process name.
 1.9.0-dev+exp,true,process,process.parent.pe.architecture,keyword,extended,,x64,CPU architecture target for the file.

experimental/generated/ecs/ecs_flat.yml

@@ -1298,6 +1298,17 @@ dll.hash.sha512:
   original_fieldset: hash
   short: SHA512 hash.
   type: keyword
+dll.hash.ssdeep:
+  dashed_name: dll-hash-ssdeep
+  description: SSDEEP hash.
+  flat_name: dll.hash.ssdeep
+  ignore_above: 1024
+  level: extended
+  name: ssdeep
+  normalize: []
+  original_fieldset: hash
+  short: SSDEEP hash.
+  type: keyword
 dll.name:
   dashed_name: dll-name
   description: 'Name of the library.
@@ -2722,6 +2733,17 @@ file.hash.sha512:
   original_fieldset: hash
   short: SHA512 hash.
   type: keyword
+file.hash.ssdeep:
+  dashed_name: file-hash-ssdeep
+  description: SSDEEP hash.
+  flat_name: file.hash.ssdeep
+  ignore_above: 1024
+  level: extended
+  name: ssdeep
+  normalize: []
+  original_fieldset: hash
+  short: SSDEEP hash.
+  type: keyword
 file.inode:
   dashed_name: file-inode
   description: Inode representing the file in the filesystem.
@@ -5283,6 +5305,17 @@ process.hash.sha512:
   original_fieldset: hash
   short: SHA512 hash.
   type: keyword
+process.hash.ssdeep:
+  dashed_name: process-hash-ssdeep
+  description: SSDEEP hash.
+  flat_name: process.hash.ssdeep
+  ignore_above: 1024
+  level: extended
+  name: ssdeep
+  normalize: []
+  original_fieldset: hash
+  short: SSDEEP hash.
+  type: keyword
 process.name:
   beta: Note the usage of `wildcard` type is considered beta. This field used to be
     type `keyword`.
@@ -5518,6 +5551,17 @@ process.parent.hash.sha512:
   original_fieldset: hash
   short: SHA512 hash.
   type: keyword
+process.parent.hash.ssdeep:
+  dashed_name: process-parent-hash-ssdeep
+  description: SSDEEP hash.
+  flat_name: process.parent.hash.ssdeep
+  ignore_above: 1024
+  level: extended
+  name: ssdeep
+  normalize: []
+  original_fieldset: hash
+  short: SSDEEP hash.
+  type: keyword
 process.parent.name:
   beta: Note the usage of `wildcard` type is considered beta. This field used to be
     type `keyword`.

experimental/generated/ecs/ecs_nested.yml

@@ -1644,6 +1644,17 @@ dll:
       original_fieldset: hash
       short: SHA512 hash.
       type: keyword
+    dll.hash.ssdeep:
+      dashed_name: dll-hash-ssdeep
+      description: SSDEEP hash.
+      flat_name: dll.hash.ssdeep
+      ignore_above: 1024
+      level: extended
+      name: ssdeep
+      normalize: []
+      original_fieldset: hash
+      short: SSDEEP hash.
+      type: keyword
     dll.name:
       dashed_name: dll-name
       description: 'Name of the library.
@@ -3170,6 +3181,17 @@ file:
       original_fieldset: hash
       short: SHA512 hash.
       type: keyword
+    file.hash.ssdeep:
+      dashed_name: file-hash-ssdeep
+      description: SSDEEP hash.
+      flat_name: file.hash.ssdeep
+      ignore_above: 1024
+      level: extended
+      name: ssdeep
+      normalize: []
+      original_fieldset: hash
+      short: SSDEEP hash.
+      type: keyword
     file.inode:
       dashed_name: file-inode
       description: Inode representing the file in the filesystem.
@@ -3902,11 +3924,16 @@ group:
   title: Group
   type: group
 hash:
-  description: 'The hash fields represent different hash algorithms and their values.
+  description: 'The hash fields represent different bitwise hash algorithms and their
+    values.
 
     Field names for common hashes (e.g. MD5, SHA1) are predefined. Add fields for
     other hashes by lowercasing the hash algorithm name and using underscore separators
-    as appropriate (snake case, e.g. sha3_512).'
+    as appropriate (snake case, e.g. sha3_512).
+
+    Note that this fieldset is used for common hashes that may be computed over a
+    range of generic bytes. Entity-specific hashes such as ja3 or imphash are placed
+    in the fieldsets to which they relate (tls and pe, respectively).'
   fields:
     hash.md5:
       dashed_name: hash-md5
@@ -3948,6 +3975,16 @@ hash:
       normalize: []
       short: SHA512 hash.
       type: keyword
+    hash.ssdeep:
+      dashed_name: hash-ssdeep
+      description: SSDEEP hash.
+      flat_name: hash.ssdeep
+      ignore_above: 1024
+      level: extended
+      name: ssdeep
+      normalize: []
+      short: SSDEEP hash.
+      type: keyword
   group: 2
   name: hash
   prefix: hash.
@@ -6379,6 +6416,17 @@ process:
       original_fieldset: hash
       short: SHA512 hash.
       type: keyword
+    process.hash.ssdeep:
+      dashed_name: process-hash-ssdeep
+      description: SSDEEP hash.
+      flat_name: process.hash.ssdeep
+      ignore_above: 1024
+      level: extended
+      name: ssdeep
+      normalize: []
+      original_fieldset: hash
+      short: SSDEEP hash.
+      type: keyword
     process.name:
       beta: Note the usage of `wildcard` type is considered beta. This field used
         to be type `keyword`.
@@ -6614,6 +6662,17 @@ process:
       original_fieldset: hash
       short: SHA512 hash.
       type: keyword
+    process.parent.hash.ssdeep:
+      dashed_name: process-parent-hash-ssdeep
+      description: SSDEEP hash.
+      flat_name: process.parent.hash.ssdeep
+      ignore_above: 1024
+      level: extended
+      name: ssdeep
+      normalize: []
+      original_fieldset: hash
+      short: SSDEEP hash.
+      type: keyword
     process.parent.name:
       beta: Note the usage of `wildcard` type is considered beta. This field used
         to be type `keyword`.