[ML] Add allow_lazy_open and max_empty_searches to SIEM jobs (#48238)

This change augments the SIEM jobs and datafeeds that were added in #47848 with the allow_lazy_open and max_empty_searches options that were added in elastic/elasticsearch#47726 and elastic/elasticsearch#47922 respectively.
elastic · Oct 15, 2019 · cd7dfee · cd7dfee
1 parent 21841f6
commit cd7dfee
Show file tree

Hide file tree

Showing 48 changed files with 53 additions and 5 deletions.
diff --git a/...a_recognizer/modules/siem_auditbeat/ml/datafeed_linux_anomalous_network_activity_ecs.json b/...a_recognizer/modules/siem_auditbeat/ml/datafeed_linux_anomalous_network_activity_ecs.json
@@ -3,6 +3,7 @@
     "indices": [
       "INDEX_PATTERN_NAME"
     ],
+    "max_empty_searches": 10,
     "query": {
             "bool": {
               "filter": [

diff --git a/...ognizer/modules/siem_auditbeat/ml/datafeed_linux_anomalous_network_port_activity_ecs.json b/...ognizer/modules/siem_auditbeat/ml/datafeed_linux_anomalous_network_port_activity_ecs.json
@@ -3,6 +3,7 @@
     "indices": [
       "INDEX_PATTERN_NAME"
     ],
+    "max_empty_searches": 10,
     "query": {
       "bool": {
         "filter": [
@@ -24,4 +25,4 @@
         ]
       }
     }
-  }
+  }
diff --git a/...s/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_anomalous_network_service.json b/...s/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_anomalous_network_service.json
@@ -3,6 +3,7 @@
     "indices": [
       "INDEX_PATTERN_NAME"
     ],
+    "max_empty_searches": 10,
     "query": {
         "bool": {
           "filter": [
@@ -23,4 +24,4 @@
         ]
         }
       }
-  }
+  }
diff --git a/...cognizer/modules/siem_auditbeat/ml/datafeed_linux_anomalous_network_url_activity_ecs.json b/...cognizer/modules/siem_auditbeat/ml/datafeed_linux_anomalous_network_url_activity_ecs.json
@@ -3,6 +3,7 @@
     "indices": [
       "INDEX_PATTERN_NAME"
     ],
+    "max_empty_searches": 10,
     "query": {
         "bool":{
           "filter": [

diff --git a/..._recognizer/modules/siem_auditbeat/ml/datafeed_linux_anomalous_process_all_hosts_ecs.json b/..._recognizer/modules/siem_auditbeat/ml/datafeed_linux_anomalous_process_all_hosts_ecs.json
@@ -3,6 +3,7 @@
     "indices": [
       "INDEX_PATTERN_NAME"
     ],
+    "max_empty_searches": 10,
     "query": {
         "bool": {
           "filter": [

diff --git a/...els/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_anomalous_user_name_ecs.json b/...els/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_anomalous_user_name_ecs.json
@@ -3,6 +3,7 @@
     "indices": [
       "INDEX_PATTERN_NAME"
     ],
+    "max_empty_searches": 10,
     "query": {
         "bool": {
             "filter": [

diff --git a/...ls/data_recognizer/modules/siem_auditbeat/ml/datafeed_rare_process_by_host_linux_ecs.json b/...ls/data_recognizer/modules/siem_auditbeat/ml/datafeed_rare_process_by_host_linux_ecs.json
@@ -3,6 +3,7 @@
     "indexes": [
       "INDEX_PATTERN_NAME"
     ],
+    "max_empty_searches": 10,
     "query": {
       "bool": {
         "filter": [

diff --git a/...els/data_recognizer/modules/siem_auditbeat/ml/datafeed_suspicious_login_activity_ecs.json b/...els/data_recognizer/modules/siem_auditbeat/ml/datafeed_suspicious_login_activity_ecs.json
@@ -3,6 +3,7 @@
   "indexes": [
     "INDEX_PATTERN_NAME"
   ],
+  "max_empty_searches": 10,
   "query": {
     "bool": {
       "filter": {

diff --git a/...odels/data_recognizer/modules/siem_auditbeat/ml/linux_anomalous_network_activity_ecs.json b/...odels/data_recognizer/modules/siem_auditbeat/ml/linux_anomalous_network_activity_ecs.json
@@ -22,6 +22,7 @@
         "destination.ip"
       ]
     },
+    "allow_lazy_open": true,
     "analysis_limits": {
       "model_memory_limit": "64mb"
     },

diff --git a/.../data_recognizer/modules/siem_auditbeat/ml/linux_anomalous_network_port_activity_ecs.json b/.../data_recognizer/modules/siem_auditbeat/ml/linux_anomalous_network_port_activity_ecs.json
@@ -22,6 +22,7 @@
       "destination.ip"
     ]
   },
+  "allow_lazy_open": true,
   "analysis_limits": {
     "model_memory_limit": "32mb"
   },

diff --git a/...ver/models/data_recognizer/modules/siem_auditbeat/ml/linux_anomalous_network_service.json b/...ver/models/data_recognizer/modules/siem_auditbeat/ml/linux_anomalous_network_service.json
@@ -21,6 +21,7 @@
       "user.name"
     ]
   },
+  "allow_lazy_open": true,
   "analysis_limits": {
     "model_memory_limit": "128mb"
   },

diff --git a/...s/data_recognizer/modules/siem_auditbeat/ml/linux_anomalous_network_url_activity_ecs.json b/...s/data_recognizer/modules/siem_auditbeat/ml/linux_anomalous_network_url_activity_ecs.json
@@ -21,6 +21,7 @@
         "destination.port"
       ]
     },
+    "allow_lazy_open": true,
     "analysis_limits": {
       "model_memory_limit": "32mb"
     },

diff --git a/...dels/data_recognizer/modules/siem_auditbeat/ml/linux_anomalous_process_all_hosts_ecs.json b/...dels/data_recognizer/modules/siem_auditbeat/ml/linux_anomalous_process_all_hosts_ecs.json
@@ -21,6 +21,7 @@
       "user.name"
     ]
   },
+  "allow_lazy_open": true,
   "analysis_limits": {
     "model_memory_limit": "512mb"
   },

diff --git a/...erver/models/data_recognizer/modules/siem_auditbeat/ml/linux_anomalous_user_name_ecs.json b/...erver/models/data_recognizer/modules/siem_auditbeat/ml/linux_anomalous_user_name_ecs.json
@@ -21,6 +21,7 @@
         "user.name"
       ]
     },
+    "allow_lazy_open": true,
     "analysis_limits": {
       "model_memory_limit": "32mb"
     },

diff --git a/...rver/models/data_recognizer/modules/siem_auditbeat/ml/rare_process_by_host_linux_ecs.json b/...rver/models/data_recognizer/modules/siem_auditbeat/ml/rare_process_by_host_linux_ecs.json
@@ -22,6 +22,7 @@
       "user.name"
     ]
   },
+  "allow_lazy_open": true,
   "analysis_limits": {
     "model_memory_limit": "256mb"
   },

diff --git a/...erver/models/data_recognizer/modules/siem_auditbeat/ml/suspicious_login_activity_ecs.json b/...erver/models/data_recognizer/modules/siem_auditbeat/ml/suspicious_login_activity_ecs.json
@@ -17,6 +17,7 @@
       "source.ip"
     ]
   },
+  "allow_lazy_open": true,
   "analysis_limits": {
    "model_memory_limit": "256mb"
   },

diff --git a/...ata_recognizer/modules/siem_auditbeat_auth/ml/datafeed_suspicious_login_activity_ecs.json b/...ata_recognizer/modules/siem_auditbeat_auth/ml/datafeed_suspicious_login_activity_ecs.json
@@ -3,6 +3,7 @@
   "indexes": [
     "INDEX_PATTERN_NAME"
   ],
+  "max_empty_searches": 10,
   "query": {
     "bool": {
       "filter": [

diff --git a/.../models/data_recognizer/modules/siem_auditbeat_auth/ml/suspicious_login_activity_ecs.json b/.../models/data_recognizer/modules/siem_auditbeat_auth/ml/suspicious_login_activity_ecs.json
@@ -21,6 +21,7 @@
       "source.ip"
     ]
   },
+  "allow_lazy_open": true,
   "analysis_limits": {
    "model_memory_limit": "256mb"
   },

diff --git a/.../models/data_recognizer/modules/siem_packetbeat/ml/datafeed_packetbeat_dns_tunneling.json b/.../models/data_recognizer/modules/siem_packetbeat/ml/datafeed_packetbeat_dns_tunneling.json
@@ -3,6 +3,7 @@
   "indices": [
     "INDEX_PATTERN_NAME"
   ],
+  "max_empty_searches": 10,
   "query": {
     "bool": {
       "filter": [

diff --git a/...els/data_recognizer/modules/siem_packetbeat/ml/datafeed_packetbeat_rare_dns_question.json b/...els/data_recognizer/modules/siem_packetbeat/ml/datafeed_packetbeat_rare_dns_question.json
@@ -3,6 +3,7 @@
   "indices": [
     "INDEX_PATTERN_NAME"
   ],
+  "max_empty_searches": 10,
   "query": {
     "bool": {
       "filter": [

diff --git a/...ls/data_recognizer/modules/siem_packetbeat/ml/datafeed_packetbeat_rare_server_domain.json b/...ls/data_recognizer/modules/siem_packetbeat/ml/datafeed_packetbeat_rare_server_domain.json
@@ -3,6 +3,7 @@
   "indices": [
     "INDEX_PATTERN_NAME"
   ],
+  "max_empty_searches": 10,
   "query": {
     "bool": {
       "filter": [

diff --git a/...rver/models/data_recognizer/modules/siem_packetbeat/ml/datafeed_packetbeat_rare_urls.json b/...rver/models/data_recognizer/modules/siem_packetbeat/ml/datafeed_packetbeat_rare_urls.json
@@ -3,6 +3,7 @@
   "indices": [
     "INDEX_PATTERN_NAME"
   ],
+  "max_empty_searches": 10,
   "query": {
     "bool": {
       "filter": [
@@ -14,4 +15,4 @@
       ]
     }
   }
-}
+}
diff --git a/...odels/data_recognizer/modules/siem_packetbeat/ml/datafeed_packetbeat_rare_user_agent.json b/...odels/data_recognizer/modules/siem_packetbeat/ml/datafeed_packetbeat_rare_user_agent.json
@@ -3,6 +3,7 @@
   "indices": [
     "INDEX_PATTERN_NAME"
   ],
+  "max_empty_searches": 10,
   "query": {
     "bool": {
       "filter": [
@@ -14,4 +15,4 @@
         ]
     }
   }
-}
+}
diff --git a/...ml/server/models/data_recognizer/modules/siem_packetbeat/ml/packetbeat_dns_tunneling.json b/...ml/server/models/data_recognizer/modules/siem_packetbeat/ml/packetbeat_dns_tunneling.json
@@ -36,6 +36,7 @@
       "dns.question.etld_plus_one"
     ]
   },
+  "allow_lazy_open": true,
   "analysis_limits": {
     "model_memory_limit": "256mb"
   },

diff --git a/...erver/models/data_recognizer/modules/siem_packetbeat/ml/packetbeat_rare_dns_question.json b/...erver/models/data_recognizer/modules/siem_packetbeat/ml/packetbeat_rare_dns_question.json
@@ -19,6 +19,7 @@
       "host.name"
     ]
   },
+  "allow_lazy_open": true,
   "analysis_limits": {
     "model_memory_limit": "256mb"
   },

diff --git a/...rver/models/data_recognizer/modules/siem_packetbeat/ml/packetbeat_rare_server_domain.json b/...rver/models/data_recognizer/modules/siem_packetbeat/ml/packetbeat_rare_server_domain.json
@@ -21,6 +21,7 @@
       "source.ip"
     ]
   },
+  "allow_lazy_open": true,
   "analysis_limits": {
     "model_memory_limit": "256mb"
   },

diff --git a/...ins/ml/server/models/data_recognizer/modules/siem_packetbeat/ml/packetbeat_rare_urls.json b/...ins/ml/server/models/data_recognizer/modules/siem_packetbeat/ml/packetbeat_rare_urls.json
@@ -20,6 +20,7 @@
       "destination.ip"
     ]
   },
+  "allow_lazy_open": true,
   "analysis_limits": {
     "model_memory_limit": "256mb"
   },

diff --git a/.../server/models/data_recognizer/modules/siem_packetbeat/ml/packetbeat_rare_user_agent.json b/.../server/models/data_recognizer/modules/siem_packetbeat/ml/packetbeat_rare_user_agent.json
@@ -20,6 +20,7 @@
       "destination.ip"
     ]
   },
+  "allow_lazy_open": true,
   "analysis_limits": {
     "model_memory_limit": "256mb"
   },

diff --git a/...data_recognizer/modules/siem_winlogbeat/ml/datafeed_rare_process_by_host_windows_ecs.json b/...data_recognizer/modules/siem_winlogbeat/ml/datafeed_rare_process_by_host_windows_ecs.json
@@ -3,6 +3,7 @@
     "indexes": [
       "INDEX_PATTERN_NAME"
     ],
+    "max_empty_searches": 10,
     "query": {
       "bool": {
         "filter": [

diff --git a/...ecognizer/modules/siem_winlogbeat/ml/datafeed_windows_anomalous_network_activity_ecs.json b/...ecognizer/modules/siem_winlogbeat/ml/datafeed_windows_anomalous_network_activity_ecs.json
@@ -3,6 +3,7 @@
     "indices": [
       "INDEX_PATTERN_NAME"
     ],
+    "max_empty_searches": 10,
     "query": {
         "bool": {
           "filter": [

diff --git a/...a_recognizer/modules/siem_winlogbeat/ml/datafeed_windows_anomalous_path_activity_ecs.json b/...a_recognizer/modules/siem_winlogbeat/ml/datafeed_windows_anomalous_path_activity_ecs.json
@@ -3,6 +3,7 @@
     "indices": [
       "INDEX_PATTERN_NAME"
     ],
+    "max_empty_searches": 10,
     "query": {
         "bool": {
           "filter": [

diff --git a/...cognizer/modules/siem_winlogbeat/ml/datafeed_windows_anomalous_process_all_hosts_ecs.json b/...cognizer/modules/siem_winlogbeat/ml/datafeed_windows_anomalous_process_all_hosts_ecs.json
@@ -3,6 +3,7 @@
     "indices": [
       "INDEX_PATTERN_NAME"
     ],
+    "max_empty_searches": 10,
     "query": {
         "bool": {
           "filter": [

diff --git a/...ta_recognizer/modules/siem_winlogbeat/ml/datafeed_windows_anomalous_process_creation.json b/...ta_recognizer/modules/siem_winlogbeat/ml/datafeed_windows_anomalous_process_creation.json
@@ -3,6 +3,7 @@
     "indices": [
       "INDEX_PATTERN_NAME"
     ],
+    "max_empty_searches": 10,
     "query": {
         "bool": {
           "filter": [

diff --git a/.../models/data_recognizer/modules/siem_winlogbeat/ml/datafeed_windows_anomalous_script.json b/.../models/data_recognizer/modules/siem_winlogbeat/ml/datafeed_windows_anomalous_script.json
@@ -3,6 +3,7 @@
     "indices": [
       "INDEX_PATTERN_NAME"
     ],
+    "max_empty_searches": 10,
     "query": {
         "bool": {
           "filter": [

diff --git a/...models/data_recognizer/modules/siem_winlogbeat/ml/datafeed_windows_anomalous_service.json b/...models/data_recognizer/modules/siem_winlogbeat/ml/datafeed_windows_anomalous_service.json
@@ -3,6 +3,7 @@
     "indices": [
       "INDEX_PATTERN_NAME"
     ],
+    "max_empty_searches": 10,
     "query": {
         "bool": {
           "filter": [

diff --git a/.../data_recognizer/modules/siem_winlogbeat/ml/datafeed_windows_anomalous_user_name_ecs.json b/.../data_recognizer/modules/siem_winlogbeat/ml/datafeed_windows_anomalous_user_name_ecs.json
@@ -3,6 +3,7 @@
     "indices": [
       "INDEX_PATTERN_NAME"
     ],
+    "max_empty_searches": 10,
     "query": {
         "bool": {
           "filter": [

diff --git a/...ls/data_recognizer/modules/siem_winlogbeat/ml/datafeed_windows_rare_user_runas_event.json b/...ls/data_recognizer/modules/siem_winlogbeat/ml/datafeed_windows_rare_user_runas_event.json
@@ -3,6 +3,7 @@
     "indices": [
       "INDEX_PATTERN_NAME"
     ],
+    "max_empty_searches": 10,
     "query": {
         "bool": {
           "filter": [
@@ -11,4 +12,4 @@
           ]
         }
       }
-    }
+    }
diff --git a/...r/models/data_recognizer/modules/siem_winlogbeat/ml/rare_process_by_host_windows_ecs.json b/...r/models/data_recognizer/modules/siem_winlogbeat/ml/rare_process_by_host_windows_ecs.json
@@ -21,6 +21,7 @@
       "user.name"
     ]
   },
+  "allow_lazy_open": true,
   "analysis_limits": {
     "model_memory_limit": "256mb"
   },

diff --git a/...ls/data_recognizer/modules/siem_winlogbeat/ml/windows_anomalous_network_activity_ecs.json b/...ls/data_recognizer/modules/siem_winlogbeat/ml/windows_anomalous_network_activity_ecs.json
@@ -22,6 +22,7 @@
         "destination.ip"
       ]
     },
+    "allow_lazy_open": true,
     "analysis_limits": {
       "model_memory_limit": "64mb"
     },

diff --git a/...odels/data_recognizer/modules/siem_winlogbeat/ml/windows_anomalous_path_activity_ecs.json b/...odels/data_recognizer/modules/siem_winlogbeat/ml/windows_anomalous_path_activity_ecs.json
@@ -21,6 +21,7 @@
       "user.name"
     ]
   },
+  "allow_lazy_open": true,
   "analysis_limits": {
     "model_memory_limit": "256mb"
   },

diff --git a/...s/data_recognizer/modules/siem_winlogbeat/ml/windows_anomalous_process_all_hosts_ecs.json b/...s/data_recognizer/modules/siem_winlogbeat/ml/windows_anomalous_process_all_hosts_ecs.json
@@ -21,6 +21,7 @@
         "user.name"
       ]
     },
+    "allow_lazy_open": true,
     "analysis_limits": {
       "model_memory_limit": "256mb"
     },

diff --git a/...models/data_recognizer/modules/siem_winlogbeat/ml/windows_anomalous_process_creation.json b/...models/data_recognizer/modules/siem_winlogbeat/ml/windows_anomalous_process_creation.json
@@ -22,6 +22,7 @@
       "user.name"
     ]
   },
+  "allow_lazy_open": true,
   "analysis_limits": {
     "model_memory_limit": "256mb"
   },

diff --git a/...ml/server/models/data_recognizer/modules/siem_winlogbeat/ml/windows_anomalous_script.json b/...ml/server/models/data_recognizer/modules/siem_winlogbeat/ml/windows_anomalous_script.json
@@ -21,6 +21,7 @@
       "winlog.event_data.Path"
     ]
   },
+  "allow_lazy_open": true,
   "analysis_limits": {
     "model_memory_limit": "256mb"
   },

diff --git a/...l/server/models/data_recognizer/modules/siem_winlogbeat/ml/windows_anomalous_service.json b/...l/server/models/data_recognizer/modules/siem_winlogbeat/ml/windows_anomalous_service.json
@@ -20,6 +20,7 @@
       "winlog.event_data.ServiceName"
     ]
   },
+  "allow_lazy_open": true,
   "analysis_limits": {
     "model_memory_limit": "256mb"
   },

diff --git a/...er/models/data_recognizer/modules/siem_winlogbeat/ml/windows_anomalous_user_name_ecs.json b/...er/models/data_recognizer/modules/siem_winlogbeat/ml/windows_anomalous_user_name_ecs.json
@@ -21,6 +21,7 @@
       "user.name"
     ]
   },
+  "allow_lazy_open": true,
   "analysis_limits": {
     "model_memory_limit": "256mb"
   },