[SIEM] move away from Joi for importing/exporting timeline (#62125) (#…

…63673) * move away from joi * update schema for filterQuery * fix types * update schemas * remove boom * remove redundant params * reuse utils from case * update schemas for query params and body * fix types * update validation schema * fix unit test * update description for test cases * remove import from case * lifting common libs * fix dependency * lifting validation builder function * add unit test * fix for code review * reve comments * rename common utils * fix types
elastic · Apr 16, 2020 · e8b1317 · e8b1317
1 parent 773c25d
commit e8b1317
Show file tree

Hide file tree

Showing 21 changed files with 378 additions and 398 deletions.
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.ts
@@ -26,13 +26,13 @@ import {
   buildSiemResponse,
   validateLicenseForRuleType,
 } from '../utils';
-import { createRulesStreamFromNdJson } from '../../rules/create_rules_stream_from_ndjson';
 import { ImportRuleAlertRest } from '../../types';
 import { patchRules } from '../../rules/patch_rules';
 import { importRulesQuerySchema, importRulesPayloadSchema } from '../schemas/import_rules_schema';
 import { ImportRulesSchema, importRulesSchema } from '../schemas/response/import_rules_schema';
 import { getTupleDuplicateErrorsAndUniqueRules } from './utils';
 import { validate } from './validate';
+import { createRulesStreamFromNdJson } from '../../rules/create_rules_stream_from_ndjson';
 
 type PromiseFromStreams = ImportRuleAlertRest | Error;
 

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.test.ts
@@ -12,7 +12,6 @@ import {
   transformTags,
   getIdBulkError,
   transformOrBulkError,
-  transformDataToNdjson,
   transformAlertsToRules,
   transformOrImportError,
   getDuplicates,
@@ -22,14 +21,13 @@ import { getResult } from '../__mocks__/request_responses';
 import { INTERNAL_IDENTIFIER } from '../../../../../common/constants';
 import { ImportRuleAlertRest, RuleAlertParamsRest, RuleTypeParams } from '../../types';
 import { BulkError, ImportSuccessError } from '../utils';
-import { sampleRule } from '../../signals/__mocks__/es_results';
 import { getSimpleRule, getOutputRuleAlertForRest } from '../__mocks__/utils';
-import { createRulesStreamFromNdJson } from '../../rules/create_rules_stream_from_ndjson';
 import { createPromiseFromStreams } from '../../../../../../../../../src/legacy/utils/streams';
 import { PartialAlert } from '../../../../../../../../plugins/alerting/server';
 import { SanitizedAlert } from '../../../../../../../../plugins/alerting/server/types';
 import { RuleAlertType } from '../../rules/types';
 import { setFeatureFlagsForTestsOnly, unSetFeatureFlagsForTestsOnly } from '../../feature_flags';
+import { createRulesStreamFromNdJson } from '../../rules/create_rules_stream_from_ndjson';
 
 type PromiseFromStreams = ImportRuleAlertRest | Error;
 
@@ -396,47 +394,6 @@ describe('utils', () => {
     });
   });
 
-  describe('transformDataToNdjson', () => {
-    test('if rules are empty it returns an empty string', () => {
-      const ruleNdjson = transformDataToNdjson([]);
-      expect(ruleNdjson).toEqual('');
-    });
-
-    test('single rule will transform with new line ending character for ndjson', () => {
-      const rule = sampleRule();
-      const ruleNdjson = transformDataToNdjson([rule]);
-      expect(ruleNdjson.endsWith('\n')).toBe(true);
-    });
-
-    test('multiple rules will transform with two new line ending characters for ndjson', () => {
-      const result1 = sampleRule();
-      const result2 = sampleRule();
-      result2.id = 'some other id';
-      result2.rule_id = 'some other id';
-      result2.name = 'Some other rule';
-
-      const ruleNdjson = transformDataToNdjson([result1, result2]);
-      // this is how we count characters in JavaScript :-)
-      const count = ruleNdjson.split('\n').length - 1;
-      expect(count).toBe(2);
-    });
-
-    test('you can parse two rules back out without errors', () => {
-      const result1 = sampleRule();
-      const result2 = sampleRule();
-      result2.id = 'some other id';
-      result2.rule_id = 'some other id';
-      result2.name = 'Some other rule';
-
-      const ruleNdjson = transformDataToNdjson([result1, result2]);
-      const ruleStrings = ruleNdjson.split('\n');
-      const reParsed1 = JSON.parse(ruleStrings[0]);
-      const reParsed2 = JSON.parse(ruleStrings[1]);
-      expect(reParsed1).toEqual(result1);
-      expect(reParsed2).toEqual(result2);
-    });
-  });
-
   describe('transformAlertsToRules', () => {
     test('given an empty array returns an empty array', () => {
       expect(transformAlertsToRules([])).toEqual([]);

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts
@@ -152,15 +152,6 @@ export const transformAlertToRule = (
   });
 };
 
-export const transformDataToNdjson = (data: unknown[]): string => {
-  if (data.length !== 0) {
-    const dataString = data.map(rule => JSON.stringify(rule)).join('\n');
-    return `${dataString}\n`;
-  } else {
-    return '';
-  }
-};
-
 export const transformAlertsToRules = (
   alerts: RuleAlertType[]
 ): Array<Partial<OutputRuleAlertRest>> => {

diff --git a/.../legacy/plugins/siem/server/lib/detection_engine/rules/create_rules_stream_from_ndjson.ts b/.../legacy/plugins/siem/server/lib/detection_engine/rules/create_rules_stream_from_ndjson.ts
@@ -4,39 +4,19 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 import { Transform } from 'stream';
-import { has, isString } from 'lodash/fp';
 import { ImportRuleAlertRest } from '../types';
 import {
   createSplitStream,
   createMapStream,
-  createFilterStream,
   createConcatStream,
 } from '../../../../../../../../src/legacy/utils/streams';
 import { importRulesSchema } from '../routes/schemas/import_rules_schema';
 import { BadRequestError } from '../errors/bad_request_error';
-
-export interface RulesObjectsExportResultDetails {
-  /** number of successfully exported objects */
-  exportedCount: number;
-}
-
-export const parseNdjsonStrings = (): Transform => {
-  return createMapStream((ndJsonStr: string) => {
-    if (isString(ndJsonStr) && ndJsonStr.trim() !== '') {
-      try {
-        return JSON.parse(ndJsonStr);
-      } catch (err) {
-        return err;
-      }
-    }
-  });
-};
-
-export const filterExportedCounts = (): Transform => {
-  return createFilterStream<ImportRuleAlertRest | RulesObjectsExportResultDetails>(
-    obj => obj != null && !has('exported_count', obj)
-  );
-};
+import {
+  parseNdjsonStrings,
+  filterExportedCounts,
+  createLimitStream,
+} from '../../../utils/read_stream/create_stream_from_ndjson';
 
 export const validateRules = (): Transform => {
   return createMapStream((obj: ImportRuleAlertRest) => {
@@ -53,21 +33,6 @@ export const validateRules = (): Transform => {
   });
 };
 
-// Adaptation from: saved_objects/import/create_limit_stream.ts
-export const createLimitStream = (limit: number): Transform => {
-  let counter = 0;
-  return new Transform({
-    objectMode: true,
-    async transform(obj, _, done) {
-      if (counter >= limit) {
-        return done(new Error(`Can't import more than ${limit} rules`));
-      }
-      counter++;
-      done(undefined, obj);
-    },
-  });
-};
-
 // TODO: Capture both the line number and the rule_id if you have that information for the error message
 // eventually and then pass it down so we can give error messages on the line number
 

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_export_all.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_export_all.ts
@@ -7,7 +7,8 @@
 import { AlertsClient } from '../../../../../../../plugins/alerting/server';
 import { getNonPackagedRules } from './get_existing_prepackaged_rules';
 import { getExportDetailsNdjson } from './get_export_details_ndjson';
-import { transformAlertsToRules, transformDataToNdjson } from '../routes/rules/utils';
+import { transformAlertsToRules } from '../routes/rules/utils';
+import { transformDataToNdjson } from '../../../utils/read_stream/create_stream_from_ndjson';
 
 export const getExportAll = async (
   alertsClient: AlertsClient

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_export_by_object_ids.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_export_by_object_ids.ts
@@ -8,8 +8,9 @@ import { AlertsClient } from '../../../../../../../plugins/alerting/server';
 import { getExportDetailsNdjson } from './get_export_details_ndjson';
 import { isAlertType } from '../rules/types';
 import { readRules } from './read_rules';
-import { transformDataToNdjson, transformAlertToRule } from '../routes/rules/utils';
+import { transformAlertToRule } from '../routes/rules/utils';
 import { OutputRuleAlertRest } from '../types';
+import { transformDataToNdjson } from '../../../utils/read_stream/create_stream_from_ndjson';
 
 interface ExportSuccesRule {
   statusCode: 200;

diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts
@@ -3,8 +3,12 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-
+import * as rt from 'io-ts';
 import { Transform } from 'stream';
+import { pipe } from 'fp-ts/lib/pipeable';
+import { fold } from 'fp-ts/lib/Either';
+import { failure } from 'io-ts/lib/PathReporter';
+import { identity } from 'fp-ts/lib/function';
 import {
   createConcatStream,
   createSplitStream,
@@ -14,26 +18,28 @@ import {
   parseNdjsonStrings,
   filterExportedCounts,
   createLimitStream,
-} from '../detection_engine/rules/create_rules_stream_from_ndjson';
-import { importTimelinesSchema } from './routes/schemas/import_timelines_schema';
-import { BadRequestError } from '../detection_engine/errors/bad_request_error';
+} from '../../utils/read_stream/create_stream_from_ndjson';
+
 import { ImportTimelineResponse } from './routes/utils/import_timelines';
+import { ImportTimelinesSchemaRt } from './routes/schemas/import_timelines_schema';
+
+type ErrorFactory = (message: string) => Error;
 
-export const validateTimelines = (): Transform => {
-  return createMapStream((obj: ImportTimelineResponse) => {
-    if (!(obj instanceof Error)) {
-      const validated = importTimelinesSchema.validate(obj);
-      if (validated.error != null) {
-        return new BadRequestError(validated.error.message);
-      } else {
-        return validated.value;
-      }
-    } else {
-      return obj;
-    }
-  });
+export const createPlainError = (message: string) => new Error(message);
+
+export const throwErrors = (createError: ErrorFactory) => (errors: rt.Errors) => {
+  throw createError(failure(errors).join('\n'));
 };
 
+export const decodeOrThrow = <A, O, I>(
+  runtimeType: rt.Type<A, O, I>,
+  createError: ErrorFactory = createPlainError
+) => (inputValue: I) =>
+  pipe(runtimeType.decode(inputValue), fold(throwErrors(createError), identity));
+
+export const validateTimelines = (): Transform =>
+  createMapStream((obj: ImportTimelineResponse) => decodeOrThrow(ImportTimelinesSchemaRt)(obj));
+
 export const createTimelinesStreamFromNdJson = (ruleLimit: number) => {
   return [
     createSplitStream('\n'),

diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/__mocks__/request_responses.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/__mocks__/request_responses.ts
@@ -6,11 +6,16 @@
 
 import { TIMELINE_EXPORT_URL, TIMELINE_IMPORT_URL } from '../../../../../common/constants';
 import { requestMock } from '../../../detection_engine/routes/__mocks__';
-
+import stream from 'stream';
+const readable = new stream.Readable();
 export const getExportTimelinesRequest = () =>
   requestMock.create({
     method: 'get',
     path: TIMELINE_EXPORT_URL,
+    query: {
+      file_name: 'mock_export_timeline.ndjson',
+      exclude_export_details: 'false',
+    },
     body: {
       ids: ['f0e58720-57b6-11ea-b88d-3f1a31716be8', '890b8ae0-57df-11ea-a7c9-3976b7f1cb37'],
     },
@@ -22,7 +27,7 @@ export const getImportTimelinesRequest = (filename?: string) =>
     path: TIMELINE_IMPORT_URL,
     query: { overwrite: false },
     body: {
-      file: { hapi: { filename: filename ?? 'filename.ndjson' } },
+      file: { ...readable, hapi: { filename: filename ?? 'filename.ndjson' } },
     },
   });
 

diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.test.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.test.ts
@@ -83,15 +83,34 @@ describe('export timelines', () => {
   });
 
   describe('request validation', () => {
-    test('disallows singular id query param', async () => {
+    test('return validation error for request body', async () => {
       const request = requestMock.create({
         method: 'get',
         path: TIMELINE_EXPORT_URL,
         body: { id: 'someId' },
       });
       const result = server.validate(request);
 
-      expect(result.badRequest).toHaveBeenCalledWith('"id" is not allowed');
+      expect(result.badRequest.mock.calls[0][0]).toEqual(
+        'Invalid value undefined supplied to : { ids: Array<string> }/ids: Array<string>'
+      );
+    });
+
+    test('return validation error for request params', async () => {
+      const request = requestMock.create({
+        method: 'get',
+        path: TIMELINE_EXPORT_URL,
+        body: { id: 'someId' },
+      });
+      const result = server.validate(request);
+
+      expect(result.badRequest.mock.calls[1][0]).toEqual(
+        [
+          'Invalid value undefined supplied to : { file_name: string, exclude_export_details: ("true" | "false") }/file_name: string',
+          'Invalid value undefined supplied to : { file_name: string, exclude_export_details: ("true" | "false") }/exclude_export_details: ("true" | "false")/0: "true"',
+          'Invalid value undefined supplied to : { file_name: string, exclude_export_details: ("true" | "false") }/exclude_export_details: ("true" | "false")/1: "false"',
+        ].join('\n')
+      );
     });
   });
 });
diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.ts
@@ -7,31 +7,24 @@
 import { set as _set } from 'lodash/fp';
 import { IRouter } from '../../../../../../../../src/core/server';
 import { LegacyServices } from '../../../types';
-import { ExportTimelineRequestParams } from '../types';
 
-import {
-  transformError,
-  buildRouteValidation,
-  buildSiemResponse,
-} from '../../detection_engine/routes/utils';
+import { transformError, buildSiemResponse } from '../../detection_engine/routes/utils';
 import { TIMELINE_EXPORT_URL } from '../../../../common/constants';
 
+import { getExportTimelineByObjectIds } from './utils/export_timelines';
 import {
-  exportTimelinesSchema,
   exportTimelinesQuerySchema,
+  exportTimelinesRequestBodySchema,
 } from './schemas/export_timelines_schema';
-
-import { getExportTimelineByObjectIds } from './utils/export_timelines';
+import { buildRouteValidation } from '../../../utils/build_validation/route_validation';
 
 export const exportTimelinesRoute = (router: IRouter, config: LegacyServices['config']) => {
   router.post(
     {
       path: TIMELINE_EXPORT_URL,
       validate: {
-        query: buildRouteValidation<ExportTimelineRequestParams['query']>(
-          exportTimelinesQuerySchema
-        ),
-        body: buildRouteValidation<ExportTimelineRequestParams['body']>(exportTimelinesSchema),
+        query: buildRouteValidation(exportTimelinesQuerySchema),
+        body: buildRouteValidation(exportTimelinesRequestBodySchema),
       },
       options: {
         tags: ['access:siem'],
@@ -42,6 +35,7 @@ export const exportTimelinesRoute = (router: IRouter, config: LegacyServices['co
         const siemResponse = buildSiemResponse(response);
         const savedObjectsClient = context.core.savedObjects.client;
         const exportSizeLimit = config().get<number>('savedObjects.maxImportExportSize');
+
         if (request.body?.ids != null && request.body.ids.length > exportSizeLimit) {
           return siemResponse.error({
             statusCode: 400,
@@ -51,7 +45,7 @@ export const exportTimelinesRoute = (router: IRouter, config: LegacyServices['co
 
         const responseBody = await getExportTimelineByObjectIds({
           client: savedObjectsClient,
-          request,
+          ids: request.body.ids,
         });
 
         return response.ok({

diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.test.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.test.ts
@@ -334,7 +334,10 @@ describe('import timelines', () => {
       const result = server.validate(request);
 
       expect(result.badRequest).toHaveBeenCalledWith(
-        'child "file" fails because ["file" is required]'
+        [
+          'Invalid value undefined supplied to : { file: (ReadableRt & { hapi: { filename: string } }) }/file: (ReadableRt & { hapi: { filename: string } })/0: ReadableRt',
+          'Invalid value undefined supplied to : { file: (ReadableRt & { hapi: { filename: string } }) }/file: (ReadableRt & { hapi: { filename: string } })/1: { hapi: { filename: string } }',
+        ].join('\n')
       );
     });
   });