[SIEM][Detections] - Update KQL/Lucene search to include lists except…

…ions (elastic#63264) (elastic#63856)

[SIEM][Detections] - Update KQL/Lucene search to include lists exceptions (elastic#63264)

### Summary 

This PR adds logic to include the lists values in search so we can now filter off of lists.

- does work with KQL search
- does work with Lucene search
- does NOT yet filter off of large lists (will be separate PR)
- does NOT yet work on ML search (will be separate PR)

x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/create_rules_schema.test.ts

@@ -556,7 +556,9 @@ describe('create rules schema', () => {
 
   test('language does not validate with something made up', () => {
     expect(
-      createRulesSchema.validate<Partial<RuleAlertParamsRest>>({
+      createRulesSchema.validate<
+        Partial<Omit<RuleAlertParamsRest, 'language'> & { language: string }>
+      >({
         rule_id: 'rule-1',
         output_index: '.siem-signals',
         risk_score: 50,

x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/patch_rules_schema.test.ts

@@ -572,7 +572,9 @@ describe('patch rules schema', () => {
 
   test('language does not validate with something made up', () => {
     expect(
-      patchRulesSchema.validate<Partial<PatchRuleAlertParamsRest>>({
+      patchRulesSchema.validate<
+        Partial<Omit<PatchRuleAlertParamsRest, 'language'> & { language: string }>
+      >({
         id: 'rule-1',
         description: 'some description',
         from: 'now-5m',

x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/types/lists_default_array.ts

@@ -7,10 +7,16 @@
 import * as t from 'io-ts';
 import { Either } from 'fp-ts/lib/Either';
 
-import { list_and as listAnd } from '../response/schemas';
+import {
+  list_and as listAnd,
+  list_values as listValues,
+  list_values_operator as listOperator,
+} from '../response/schemas';
 
 export type ListsDefaultArrayC = t.Type<List[], List[], unknown>;
-type List = t.TypeOf<typeof listAnd>;
+export type List = t.TypeOf<typeof listAnd>;
+export type ListValues = t.TypeOf<typeof listValues>;
+export type ListOperator = t.TypeOf<typeof listOperator>;
 
 /**
  * Types the ListsDefaultArray as:

x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/update_rules_schema.test.ts

@@ -575,7 +575,9 @@ describe('create rules schema', () => {
 
   test('language does not validate with something made up', () => {
     expect(
-      updateRulesSchema.validate<Partial<RuleAlertParamsRest>>({
+      updateRulesSchema.validate<
+        Partial<Omit<RuleAlertParamsRest, 'language'> & { language: string }>
+      >({
         rule_id: 'rule-1',
         output_index: '.siem-signals',
         risk_score: 50,

x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/queries/lists/query_with_and.json

@@ -0,0 +1,35 @@
+{
+  "name": "List - and",
+  "description": "Query with a list that includes and. This rule should only produce signals when host.name exists and when both event.module is endgame and event.category is anything other than file",
+  "rule_id": "query-with-list-and",
+  "risk_score": 1,
+  "severity": "high",
+  "type": "query",
+  "query": "host.name: *",
+  "interval": "30s",
+  "language": "kuery",
+  "lists": [
+    {
+      "field": "event.module",
+      "values_operator": "excluded",
+      "values_type": "match",
+      "values": [
+        {
+          "name": "endgame"
+        }
+      ],
+      "and": [
+        {
+          "field": "event.category",
+          "values_operator": "included",
+          "values_type": "match",
+          "values": [
+            {
+              "name": "file"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/queries/lists/query_with_excluded.json

@@ -0,0 +1,23 @@
+{
+  "name": "List - excluded",
+  "description": "Query with a list of values_operator excluded. This rule should only produce signals when host.name exists and event.module is suricata",
+  "rule_id": "query-with-list-excluded",
+  "risk_score": 1,
+  "severity": "high",
+  "type": "query",
+  "query": "host.name: *",
+  "interval": "30s",
+  "language": "kuery",
+  "lists": [
+    {
+      "field": "event.module",
+      "values_operator": "excluded",
+      "values_type": "match",
+      "values": [
+        {
+          "name": "suricata"
+        }
+      ]
+    }
+  ]
+}

x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/queries/lists/query_with_exists.json

@@ -0,0 +1,18 @@
+{
+  "name": "List - exists",
+  "description": "Query with a list that includes exists. This rule should only produce signals when host.name exists and event.action does not exist",
+  "rule_id": "query-with-list-exists",
+  "risk_score": 1,
+  "severity": "high",
+  "type": "query",
+  "query": "host.name: *",
+  "interval": "30s",
+  "language": "kuery",
+  "lists": [
+    {
+      "field": "event.action",
+      "values_operator": "included",
+      "values_type": "exists"
+    }
+  ]
+}

x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/queries/lists/query_with_list.json

@@ -0,0 +1,54 @@
+{
+  "name": "Query with a list",
+  "description": "Query with a list. This rule should only produce signals when either host.name exists and event.module is system and user.name is zeek or gdm OR when host.name exists and event.module is not endgame or zeek or system.",
+  "rule_id": "query-with-list",
+  "risk_score": 2,
+  "severity": "high",
+  "type": "query",
+  "query": "host.name: *",
+  "interval": "30s",
+  "language": "kuery",
+  "lists": [
+    {
+      "field": "event.module",
+      "values_operator": "excluded",
+      "values_type": "match",
+      "values": [
+        {
+          "name": "system"
+        }
+      ],
+      "and": [
+        {
+          "field": "user.name",
+          "values_operator": "excluded",
+          "values_type": "match_all",
+          "values": [
+            {
+              "name": "zeek"
+            },
+            {
+              "name": "gdm"
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "field": "event.module",
+      "values_operator": "included",
+      "values_type": "match_all",
+      "values": [
+        {
+          "name": "endgame"
+        },
+        {
+          "name": "zeek"
+        },
+        {
+          "name": "system"
+        }
+      ]
+    }
+  ]
+}

x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/queries/lists/query_with_match.json

@@ -0,0 +1,23 @@
+{
+  "name": "List - match",
+  "description": "Query with a list that includes match. This rule should only produce signals when host.name exists and event.module is not suricata",
+  "rule_id": "query-with-list-match",
+  "risk_score": 1,
+  "severity": "high",
+  "type": "query",
+  "query": "host.name: *",
+  "interval": "30s",
+  "language": "kuery",
+  "lists": [
+    {
+      "field": "event.module",
+      "values_operator": "included",
+      "values_type": "match",
+      "values": [
+        {
+          "name": "suricata"
+        }
+      ]
+    }
+  ]
+}

x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/queries/lists/query_with_match_all.json

@@ -0,0 +1,26 @@
+{
+  "name": "List - match_all",
+  "description": "Query with a list that includes match_all. This rule should only produce signals when host.name exists and event.module is not suricata or auditd",
+  "rule_id": "query-with-list-match-all",
+  "risk_score": 1,
+  "severity": "high",
+  "type": "query",
+  "query": "host.name: *",
+  "interval": "30s",
+  "language": "kuery",
+  "lists": [
+    {
+      "field": "event.module",
+      "values_operator": "included",
+      "values_type": "match_all",
+      "values": [
+        {
+          "name": "suricata"
+        },
+        {
+          "name": "auditd"
+        }
+      ]
+    }
+  ]
+}

x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/queries/lists/query_with_or.json

@@ -0,0 +1,32 @@
+{
+  "name": "List - or",
+  "description": "Query with a list that includes or. This rule should only produce signals when host.name exists and event.module is suricata OR when host.name exists and event.category is file",
+  "rule_id": "query-with-list-or",
+  "risk_score": 1,
+  "severity": "high",
+  "type": "query",
+  "query": "host.name: *",
+  "interval": "30s",
+  "lists": [
+    {
+      "field": "event.module",
+      "values_operator": "excluded",
+      "values_type": "match",
+      "values": [
+        {
+          "name": "suricata"
+        }
+      ]
+    },
+    {
+      "field": "event.category",
+      "values_operator": "excluded",
+      "values_type": "match",
+      "values": [
+        {
+          "name": "file"
+        }
+      ]
+    }
+  ]
+}