[SIEM][Detection Engine][Lists] Adds the ability for exception lists to be multi-list queried.

x-pack/plugins/lists/README.md

@@ -57,7 +57,7 @@ which will:
 - Delete any existing exception list items you have
 - Delete any existing mapping, policies, and templates, you might have previously had.
 - Add the latest list and list item index and its mappings using your settings from `kibana.dev.yml` environment variable of `xpack.lists.listIndex` and `xpack.lists.listItemIndex`.
-- Posts the sample list from `./lists/new/list_ip.json`
+- Posts the sample list from `./lists/new/ip_list.json`
 
 Now you can run
 
@@ -69,7 +69,7 @@ You should see the new list created like so:
 
 ```sh
 {
-  "id": "list_ip",
+  "id": "ip_list",
   "created_at": "2020-05-28T19:15:22.344Z",
   "created_by": "yo",
   "description": "This list describes bad internet ip",
@@ -96,7 +96,7 @@ You should see the new list item created and attached to the above list like so:
   "value": "127.0.0.1",
   "created_at": "2020-05-28T19:15:49.790Z",
   "created_by": "yo",
-  "list_id": "list_ip",
+  "list_id": "ip_list",
   "tie_breaker_id": "a881bf2e-1e17-4592-bba8-d567cb07d234",
   "updated_at": "2020-05-28T19:15:49.790Z",
   "updated_by": "yo"
@@ -195,7 +195,7 @@ You can then do find for each one like so:
   "cursor": "WzIwLFsiYzU3ZWZiYzQtNDk3Ny00YTMyLTk5NWYtY2ZkMjk2YmVkNTIxIl1d",
   "data": [
     {
-      "id": "list_ip",
+      "id": "ip_list",
       "created_at": "2020-05-28T19:15:22.344Z",
       "created_by": "yo",
       "description": "This list describes bad internet ip",

x-pack/plugins/lists/common/schemas/common/schemas.ts

@@ -273,7 +273,6 @@ export const cursorOrUndefined = t.union([cursor, t.undefined]);
 export type CursorOrUndefined = t.TypeOf<typeof cursorOrUndefined>;
 
 export const namespace_type = DefaultNamespace;
-export type NamespaceType = t.TypeOf<typeof namespace_type>;
 
 export const operator = t.keyof({ excluded: null, included: null });
 export type Operator = t.TypeOf<typeof operator>;

x-pack/plugins/lists/common/schemas/request/create_exception_list_item_schema.ts

@@ -10,7 +10,6 @@ import * as t from 'io-ts';
 
 import {
   ItemId,
-  NamespaceType,
   Tags,
   _Tags,
   _tags,
@@ -23,7 +22,12 @@ import {
   tags,
 } from '../common/schemas';
 import { Identity, RequiredKeepUndefined } from '../../types';
-import { CreateCommentsArray, DefaultCreateCommentsArray, DefaultEntryArray } from '../types';
+import {
+  CreateCommentsArray,
+  DefaultCreateCommentsArray,
+  DefaultEntryArray,
+  NamespaceType,
+} from '../types';
 import { EntriesArray } from '../types/entries';
 import { DefaultUuid } from '../../siem_common_deps';
 

x-pack/plugins/lists/common/schemas/request/create_exception_list_schema.ts

@@ -10,7 +10,6 @@ import * as t from 'io-ts';
 
 import {
   ListId,
-  NamespaceType,
   Tags,
   _Tags,
   _tags,
@@ -23,6 +22,7 @@ import {
 } from '../common/schemas';
 import { Identity, RequiredKeepUndefined } from '../../types';
 import { DefaultUuid } from '../../siem_common_deps';
+import { NamespaceType } from '../types';
 
 export const createExceptionListSchema = t.intersection([
   t.exact(

x-pack/plugins/lists/common/schemas/request/delete_exception_list_item_schema.ts

@@ -8,7 +8,8 @@
 
 import * as t from 'io-ts';
 
-import { NamespaceType, id, item_id, namespace_type } from '../common/schemas';
+import { id, item_id, namespace_type } from '../common/schemas';
+import { NamespaceType } from '../types';
 
 export const deleteExceptionListItemSchema = t.exact(
   t.partial({

x-pack/plugins/lists/common/schemas/request/delete_exception_list_schema.ts

@@ -8,7 +8,8 @@
 
 import * as t from 'io-ts';
 
-import { NamespaceType, id, list_id, namespace_type } from '../common/schemas';
+import { id, list_id, namespace_type } from '../common/schemas';
+import { NamespaceType } from '../types';
 
 export const deleteExceptionListSchema = t.exact(
   t.partial({

x-pack/plugins/lists/common/schemas/request/find_exception_list_item_schema.ts

@@ -8,27 +8,26 @@
 
 import * as t from 'io-ts';
 
-import {
-  NamespaceType,
-  filter,
-  list_id,
-  namespace_type,
-  sort_field,
-  sort_order,
-} from '../common/schemas';
+import { sort_field, sort_order } from '../common/schemas';
 import { RequiredKeepUndefined } from '../../types';
 import { StringToPositiveNumber } from '../types/string_to_positive_number';
+import {
+  DefaultNamespaceArray,
+  DefaultNamespaceArrayTypeDecoded,
+} from '../types/default_namespace_array';
+import { NonEmptyStringArray } from '../types/non_empty_string_array';
+import { EmptyStringArray, EmptyStringArrayDecoded } from '../types/empty_string_array';
 
 export const findExceptionListItemSchema = t.intersection([
   t.exact(
     t.type({
-      list_id,
+      list_id: NonEmptyStringArray,
     })
   ),
   t.exact(
     t.partial({
-      filter, // defaults to undefined if not set during decode
-      namespace_type, // defaults to 'single' if not set during decode
+      filter: EmptyStringArray, // defaults to undefined if not set during decode
+      namespace_type: DefaultNamespaceArray, // defaults to ['single'] if not set during decode
       page: StringToPositiveNumber, // defaults to undefined if not set during decode
       per_page: StringToPositiveNumber, // defaults to undefined if not set during decode
       sort_field, // defaults to undefined if not set during decode
@@ -37,14 +36,15 @@ export const findExceptionListItemSchema = t.intersection([
   ),
 ]);
 
-export type FindExceptionListItemSchemaPartial = t.TypeOf<typeof findExceptionListItemSchema>;
+export type FindExceptionListItemSchemaPartial = t.OutputOf<typeof findExceptionListItemSchema>;
 
 // This type is used after a decode since some things are defaults after a decode.
 export type FindExceptionListItemSchemaPartialDecoded = Omit<
-  FindExceptionListItemSchemaPartial,
-  'namespace_type'
+  t.TypeOf<typeof findExceptionListItemSchema>,
+  'namespace_type' | 'filter'
 > & {
-  namespace_type: NamespaceType;
+  filter: EmptyStringArrayDecoded;
+  namespace_type: DefaultNamespaceArrayTypeDecoded;
 };
 
 // This type is used after a decode since some things are defaults after a decode.

x-pack/plugins/lists/common/schemas/request/find_exception_list_schema.ts

@@ -8,9 +8,10 @@
 
 import * as t from 'io-ts';
 
-import { NamespaceType, filter, namespace_type, sort_field, sort_order } from '../common/schemas';
+import { filter, namespace_type, sort_field, sort_order } from '../common/schemas';
 import { RequiredKeepUndefined } from '../../types';
 import { StringToPositiveNumber } from '../types/string_to_positive_number';
+import { NamespaceType } from '../types';
 
 export const findExceptionListSchema = t.exact(
   t.partial({

x-pack/plugins/lists/common/schemas/request/read_exception_list_item_schema.ts

@@ -8,8 +8,9 @@
 
 import * as t from 'io-ts';
 
-import { NamespaceType, id, item_id, namespace_type } from '../common/schemas';
+import { id, item_id, namespace_type } from '../common/schemas';
 import { RequiredKeepUndefined } from '../../types';
+import { NamespaceType } from '../types';
 
 export const readExceptionListItemSchema = t.exact(
   t.partial({

x-pack/plugins/lists/common/schemas/request/read_exception_list_schema.ts

@@ -8,8 +8,9 @@
 
 import * as t from 'io-ts';
 
-import { NamespaceType, id, list_id, namespace_type } from '../common/schemas';
+import { id, list_id, namespace_type } from '../common/schemas';
 import { RequiredKeepUndefined } from '../../types';
+import { NamespaceType } from '../types';
 
 export const readExceptionListSchema = t.exact(
   t.partial({

x-pack/plugins/lists/common/schemas/request/update_exception_list_item_schema.ts

@@ -9,7 +9,6 @@
 import * as t from 'io-ts';
 
 import {
-  NamespaceType,
   Tags,
   _Tags,
   _tags,
@@ -26,6 +25,7 @@ import {
   DefaultEntryArray,
   DefaultUpdateCommentsArray,
   EntriesArray,
+  NamespaceType,
   UpdateCommentsArray,
 } from '../types';
 

x-pack/plugins/lists/common/schemas/request/update_exception_list_schema.ts

@@ -9,7 +9,6 @@
 import * as t from 'io-ts';
 
 import {
-  NamespaceType,
   Tags,
   _Tags,
   _tags,
@@ -21,6 +20,7 @@ import {
   tags,
 } from '../common/schemas';
 import { Identity, RequiredKeepUndefined } from '../../types';
+import { NamespaceType } from '../types';
 
 export const updateExceptionListSchema = t.intersection([
   t.exact(

x-pack/plugins/lists/common/schemas/types/default_namespace.ts

@@ -8,23 +8,18 @@ import * as t from 'io-ts';
 import { Either } from 'fp-ts/lib/Either';
 
 export const namespaceType = t.keyof({ agnostic: null, single: null });
-
-type NamespaceType = t.TypeOf<typeof namespaceType>;
-
-export type DefaultNamespaceC = t.Type<NamespaceType, NamespaceType, unknown>;
+export type NamespaceType = t.TypeOf<typeof namespaceType>;
 
 /**
  * Types the DefaultNamespace as:
  *   - If null or undefined, then a default string/enumeration of "single" will be used.
  */
-export const DefaultNamespace: DefaultNamespaceC = new t.Type<
-  NamespaceType,
-  NamespaceType,
-  unknown
->(
+export const DefaultNamespace = new t.Type<NamespaceType, NamespaceType, unknown>(
   'DefaultNamespace',
   namespaceType.is,
   (input, context): Either<t.Errors, NamespaceType> =>
     input == null ? t.success('single') : namespaceType.validate(input, context),
   t.identity
 );
+
+export type DefaultNamespaceC = typeof DefaultNamespace;

x-pack/plugins/lists/common/schemas/types/default_namespace_array.test.ts

@@ -0,0 +1,99 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { pipe } from 'fp-ts/lib/pipeable';
+import { left } from 'fp-ts/lib/Either';
+
+import { foldLeftRight, getPaths } from '../../siem_common_deps';
+
+import { DefaultNamespaceArray, DefaultNamespaceArrayTypeEncoded } from './default_namespace_array';
+
+describe('default_namespace_array', () => {
+  test('it should validate "null" single item as an array with a "single" value', () => {
+    const payload: DefaultNamespaceArrayTypeEncoded = null;
+    const decoded = DefaultNamespaceArray.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual(['single']);
+  });
+
+  test('it should NOT validate a numeric value', () => {
+    const payload = 5;
+    const decoded = DefaultNamespaceArray.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([
+      'Invalid value "5" supplied to "DefaultNamespaceArray"',
+    ]);
+    expect(message.schema).toEqual({});
+  });
+
+  test('it should validate "undefined" item as an array with a "single" value', () => {
+    const payload: DefaultNamespaceArrayTypeEncoded = undefined;
+    const decoded = DefaultNamespaceArray.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual(['single']);
+  });
+
+  test('it should validate "single" as an array of a "single" value', () => {
+    const payload: DefaultNamespaceArrayTypeEncoded = 'single';
+    const decoded = DefaultNamespaceArray.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual([payload]);
+  });
+
+  test('it should validate "agnostic" as an array of a "agnostic" value', () => {
+    const payload: DefaultNamespaceArrayTypeEncoded = 'agnostic';
+    const decoded = DefaultNamespaceArray.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual([payload]);
+  });
+
+  test('it should validate "single,agnostic" as an array of 2 values of ["single", "agnostic"] values', () => {
+    const payload: DefaultNamespaceArrayTypeEncoded = 'agnostic,single';
+    const decoded = DefaultNamespaceArray.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual(['agnostic', 'single']);
+  });
+
+  test('it should validate 3 elements of "single,agnostic,single" as an array of 3 values of ["single", "agnostic", "single"] values', () => {
+    const payload: DefaultNamespaceArrayTypeEncoded = 'single,agnostic,single';
+    const decoded = DefaultNamespaceArray.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual(['single', 'agnostic', 'single']);
+  });
+
+  test('it should validate 3 elements of "single,agnostic, single" as an array of 3 values of ["single", "agnostic", "single"] values when there are spaces', () => {
+    const payload: DefaultNamespaceArrayTypeEncoded = '  single,  agnostic, single  ';
+    const decoded = DefaultNamespaceArray.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual(['single', 'agnostic', 'single']);
+  });
+
+  test('it should not validate 3 elements of "single,agnostic,junk" since the 3rd value is junk', () => {
+    const payload: DefaultNamespaceArrayTypeEncoded = 'single,agnostic,junk';
+    const decoded = DefaultNamespaceArray.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([
+      'Invalid value "junk" supplied to "DefaultNamespaceArray"',
+    ]);
+    expect(message.schema).toEqual({});
+  });
+});

x-pack/plugins/lists/common/schemas/types/default_namespace_array.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as t from 'io-ts';
+import { Either } from 'fp-ts/lib/Either';
+
+import { namespaceType } from './default_namespace';
+
+export const namespaceTypeArray = t.array(namespaceType);
+export type NamespaceTypeArray = t.TypeOf<typeof namespaceTypeArray>;
+
+/**
+ * Types the DefaultNamespaceArray as:
+ *   - If null or undefined, then a default string array of "single" will be used.
+ *   - If it contains a string, then it is split along the commas and puts them into an array and validates it
+ */
+export const DefaultNamespaceArray = new t.Type<
+  NamespaceTypeArray,
+  string | undefined | null,
+  unknown
+>(
+  'DefaultNamespaceArray',
+  namespaceTypeArray.is,
+  (input, context): Either<t.Errors, NamespaceTypeArray> => {
+    if (input == null) {
+      return t.success(['single']);
+    } else if (typeof input === 'string') {
+      const commaSeparatedValues = input
+        .trim()
+        .split(',')
+        .map((value) => value.trim());
+      return namespaceTypeArray.validate(commaSeparatedValues, context);
+    }
+    return t.failure(input, context);
+  },
+  String
+);
+
+export type DefaultNamespaceC = typeof DefaultNamespaceArray;
+
+export type DefaultNamespaceArrayTypeEncoded = t.OutputOf<typeof DefaultNamespaceArray>;
+export type DefaultNamespaceArrayTypeDecoded = t.TypeOf<typeof DefaultNamespaceArray>;