Skip to content

Commit

Permalink
[SIEM] move away from Joi for importing/exporting timeline (#62125) (#…
Browse files Browse the repository at this point in the history
…63673)

* move away from joi

* update schema for filterQuery

* fix types

* update schemas

* remove boom

* remove redundant params

* reuse utils from case

* update schemas for query params and body

* fix types

* update validation schema

* fix unit test

* update description for test cases

* remove import from case

* lifting common libs

* fix dependency

* lifting validation builder function

* add unit test

* fix for code review

* reve comments

* rename common utils

* fix types
  • Loading branch information
angorayc authored Apr 16, 2020
1 parent 773c25d commit e8b1317
Show file tree
Hide file tree
Showing 21 changed files with 378 additions and 398 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,13 @@ import {
buildSiemResponse,
validateLicenseForRuleType,
} from '../utils';
import { createRulesStreamFromNdJson } from '../../rules/create_rules_stream_from_ndjson';
import { ImportRuleAlertRest } from '../../types';
import { patchRules } from '../../rules/patch_rules';
import { importRulesQuerySchema, importRulesPayloadSchema } from '../schemas/import_rules_schema';
import { ImportRulesSchema, importRulesSchema } from '../schemas/response/import_rules_schema';
import { getTupleDuplicateErrorsAndUniqueRules } from './utils';
import { validate } from './validate';
import { createRulesStreamFromNdJson } from '../../rules/create_rules_stream_from_ndjson';

type PromiseFromStreams = ImportRuleAlertRest | Error;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import {
transformTags,
getIdBulkError,
transformOrBulkError,
transformDataToNdjson,
transformAlertsToRules,
transformOrImportError,
getDuplicates,
Expand All @@ -22,14 +21,13 @@ import { getResult } from '../__mocks__/request_responses';
import { INTERNAL_IDENTIFIER } from '../../../../../common/constants';
import { ImportRuleAlertRest, RuleAlertParamsRest, RuleTypeParams } from '../../types';
import { BulkError, ImportSuccessError } from '../utils';
import { sampleRule } from '../../signals/__mocks__/es_results';
import { getSimpleRule, getOutputRuleAlertForRest } from '../__mocks__/utils';
import { createRulesStreamFromNdJson } from '../../rules/create_rules_stream_from_ndjson';
import { createPromiseFromStreams } from '../../../../../../../../../src/legacy/utils/streams';
import { PartialAlert } from '../../../../../../../../plugins/alerting/server';
import { SanitizedAlert } from '../../../../../../../../plugins/alerting/server/types';
import { RuleAlertType } from '../../rules/types';
import { setFeatureFlagsForTestsOnly, unSetFeatureFlagsForTestsOnly } from '../../feature_flags';
import { createRulesStreamFromNdJson } from '../../rules/create_rules_stream_from_ndjson';

type PromiseFromStreams = ImportRuleAlertRest | Error;

Expand Down Expand Up @@ -396,47 +394,6 @@ describe('utils', () => {
});
});

describe('transformDataToNdjson', () => {
test('if rules are empty it returns an empty string', () => {
const ruleNdjson = transformDataToNdjson([]);
expect(ruleNdjson).toEqual('');
});

test('single rule will transform with new line ending character for ndjson', () => {
const rule = sampleRule();
const ruleNdjson = transformDataToNdjson([rule]);
expect(ruleNdjson.endsWith('\n')).toBe(true);
});

test('multiple rules will transform with two new line ending characters for ndjson', () => {
const result1 = sampleRule();
const result2 = sampleRule();
result2.id = 'some other id';
result2.rule_id = 'some other id';
result2.name = 'Some other rule';

const ruleNdjson = transformDataToNdjson([result1, result2]);
// this is how we count characters in JavaScript :-)
const count = ruleNdjson.split('\n').length - 1;
expect(count).toBe(2);
});

test('you can parse two rules back out without errors', () => {
const result1 = sampleRule();
const result2 = sampleRule();
result2.id = 'some other id';
result2.rule_id = 'some other id';
result2.name = 'Some other rule';

const ruleNdjson = transformDataToNdjson([result1, result2]);
const ruleStrings = ruleNdjson.split('\n');
const reParsed1 = JSON.parse(ruleStrings[0]);
const reParsed2 = JSON.parse(ruleStrings[1]);
expect(reParsed1).toEqual(result1);
expect(reParsed2).toEqual(result2);
});
});

describe('transformAlertsToRules', () => {
test('given an empty array returns an empty array', () => {
expect(transformAlertsToRules([])).toEqual([]);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -152,15 +152,6 @@ export const transformAlertToRule = (
});
};

export const transformDataToNdjson = (data: unknown[]): string => {
if (data.length !== 0) {
const dataString = data.map(rule => JSON.stringify(rule)).join('\n');
return `${dataString}\n`;
} else {
return '';
}
};

export const transformAlertsToRules = (
alerts: RuleAlertType[]
): Array<Partial<OutputRuleAlertRest>> => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,39 +4,19 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { Transform } from 'stream';
import { has, isString } from 'lodash/fp';
import { ImportRuleAlertRest } from '../types';
import {
createSplitStream,
createMapStream,
createFilterStream,
createConcatStream,
} from '../../../../../../../../src/legacy/utils/streams';
import { importRulesSchema } from '../routes/schemas/import_rules_schema';
import { BadRequestError } from '../errors/bad_request_error';

export interface RulesObjectsExportResultDetails {
/** number of successfully exported objects */
exportedCount: number;
}

export const parseNdjsonStrings = (): Transform => {
return createMapStream((ndJsonStr: string) => {
if (isString(ndJsonStr) && ndJsonStr.trim() !== '') {
try {
return JSON.parse(ndJsonStr);
} catch (err) {
return err;
}
}
});
};

export const filterExportedCounts = (): Transform => {
return createFilterStream<ImportRuleAlertRest | RulesObjectsExportResultDetails>(
obj => obj != null && !has('exported_count', obj)
);
};
import {
parseNdjsonStrings,
filterExportedCounts,
createLimitStream,
} from '../../../utils/read_stream/create_stream_from_ndjson';

export const validateRules = (): Transform => {
return createMapStream((obj: ImportRuleAlertRest) => {
Expand All @@ -53,21 +33,6 @@ export const validateRules = (): Transform => {
});
};

// Adaptation from: saved_objects/import/create_limit_stream.ts
export const createLimitStream = (limit: number): Transform => {
let counter = 0;
return new Transform({
objectMode: true,
async transform(obj, _, done) {
if (counter >= limit) {
return done(new Error(`Can't import more than ${limit} rules`));
}
counter++;
done(undefined, obj);
},
});
};

// TODO: Capture both the line number and the rule_id if you have that information for the error message
// eventually and then pass it down so we can give error messages on the line number

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@
import { AlertsClient } from '../../../../../../../plugins/alerting/server';
import { getNonPackagedRules } from './get_existing_prepackaged_rules';
import { getExportDetailsNdjson } from './get_export_details_ndjson';
import { transformAlertsToRules, transformDataToNdjson } from '../routes/rules/utils';
import { transformAlertsToRules } from '../routes/rules/utils';
import { transformDataToNdjson } from '../../../utils/read_stream/create_stream_from_ndjson';

export const getExportAll = async (
alertsClient: AlertsClient
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@ import { AlertsClient } from '../../../../../../../plugins/alerting/server';
import { getExportDetailsNdjson } from './get_export_details_ndjson';
import { isAlertType } from '../rules/types';
import { readRules } from './read_rules';
import { transformDataToNdjson, transformAlertToRule } from '../routes/rules/utils';
import { transformAlertToRule } from '../routes/rules/utils';
import { OutputRuleAlertRest } from '../types';
import { transformDataToNdjson } from '../../../utils/read_stream/create_stream_from_ndjson';

interface ExportSuccesRule {
statusCode: 200;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,12 @@
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import * as rt from 'io-ts';
import { Transform } from 'stream';
import { pipe } from 'fp-ts/lib/pipeable';
import { fold } from 'fp-ts/lib/Either';
import { failure } from 'io-ts/lib/PathReporter';
import { identity } from 'fp-ts/lib/function';
import {
createConcatStream,
createSplitStream,
Expand All @@ -14,26 +18,28 @@ import {
parseNdjsonStrings,
filterExportedCounts,
createLimitStream,
} from '../detection_engine/rules/create_rules_stream_from_ndjson';
import { importTimelinesSchema } from './routes/schemas/import_timelines_schema';
import { BadRequestError } from '../detection_engine/errors/bad_request_error';
} from '../../utils/read_stream/create_stream_from_ndjson';

import { ImportTimelineResponse } from './routes/utils/import_timelines';
import { ImportTimelinesSchemaRt } from './routes/schemas/import_timelines_schema';

type ErrorFactory = (message: string) => Error;

export const validateTimelines = (): Transform => {
return createMapStream((obj: ImportTimelineResponse) => {
if (!(obj instanceof Error)) {
const validated = importTimelinesSchema.validate(obj);
if (validated.error != null) {
return new BadRequestError(validated.error.message);
} else {
return validated.value;
}
} else {
return obj;
}
});
export const createPlainError = (message: string) => new Error(message);

export const throwErrors = (createError: ErrorFactory) => (errors: rt.Errors) => {
throw createError(failure(errors).join('\n'));
};

export const decodeOrThrow = <A, O, I>(
runtimeType: rt.Type<A, O, I>,
createError: ErrorFactory = createPlainError
) => (inputValue: I) =>
pipe(runtimeType.decode(inputValue), fold(throwErrors(createError), identity));

export const validateTimelines = (): Transform =>
createMapStream((obj: ImportTimelineResponse) => decodeOrThrow(ImportTimelinesSchemaRt)(obj));

export const createTimelinesStreamFromNdJson = (ruleLimit: number) => {
return [
createSplitStream('\n'),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,16 @@

import { TIMELINE_EXPORT_URL, TIMELINE_IMPORT_URL } from '../../../../../common/constants';
import { requestMock } from '../../../detection_engine/routes/__mocks__';

import stream from 'stream';
const readable = new stream.Readable();
export const getExportTimelinesRequest = () =>
requestMock.create({
method: 'get',
path: TIMELINE_EXPORT_URL,
query: {
file_name: 'mock_export_timeline.ndjson',
exclude_export_details: 'false',
},
body: {
ids: ['f0e58720-57b6-11ea-b88d-3f1a31716be8', '890b8ae0-57df-11ea-a7c9-3976b7f1cb37'],
},
Expand All @@ -22,7 +27,7 @@ export const getImportTimelinesRequest = (filename?: string) =>
path: TIMELINE_IMPORT_URL,
query: { overwrite: false },
body: {
file: { hapi: { filename: filename ?? 'filename.ndjson' } },
file: { ...readable, hapi: { filename: filename ?? 'filename.ndjson' } },
},
});

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,15 +83,34 @@ describe('export timelines', () => {
});

describe('request validation', () => {
test('disallows singular id query param', async () => {
test('return validation error for request body', async () => {
const request = requestMock.create({
method: 'get',
path: TIMELINE_EXPORT_URL,
body: { id: 'someId' },
});
const result = server.validate(request);

expect(result.badRequest).toHaveBeenCalledWith('"id" is not allowed');
expect(result.badRequest.mock.calls[0][0]).toEqual(
'Invalid value undefined supplied to : { ids: Array<string> }/ids: Array<string>'
);
});

test('return validation error for request params', async () => {
const request = requestMock.create({
method: 'get',
path: TIMELINE_EXPORT_URL,
body: { id: 'someId' },
});
const result = server.validate(request);

expect(result.badRequest.mock.calls[1][0]).toEqual(
[
'Invalid value undefined supplied to : { file_name: string, exclude_export_details: ("true" | "false") }/file_name: string',
'Invalid value undefined supplied to : { file_name: string, exclude_export_details: ("true" | "false") }/exclude_export_details: ("true" | "false")/0: "true"',
'Invalid value undefined supplied to : { file_name: string, exclude_export_details: ("true" | "false") }/exclude_export_details: ("true" | "false")/1: "false"',
].join('\n')
);
});
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -7,31 +7,24 @@
import { set as _set } from 'lodash/fp';
import { IRouter } from '../../../../../../../../src/core/server';
import { LegacyServices } from '../../../types';
import { ExportTimelineRequestParams } from '../types';

import {
transformError,
buildRouteValidation,
buildSiemResponse,
} from '../../detection_engine/routes/utils';
import { transformError, buildSiemResponse } from '../../detection_engine/routes/utils';
import { TIMELINE_EXPORT_URL } from '../../../../common/constants';

import { getExportTimelineByObjectIds } from './utils/export_timelines';
import {
exportTimelinesSchema,
exportTimelinesQuerySchema,
exportTimelinesRequestBodySchema,
} from './schemas/export_timelines_schema';

import { getExportTimelineByObjectIds } from './utils/export_timelines';
import { buildRouteValidation } from '../../../utils/build_validation/route_validation';

export const exportTimelinesRoute = (router: IRouter, config: LegacyServices['config']) => {
router.post(
{
path: TIMELINE_EXPORT_URL,
validate: {
query: buildRouteValidation<ExportTimelineRequestParams['query']>(
exportTimelinesQuerySchema
),
body: buildRouteValidation<ExportTimelineRequestParams['body']>(exportTimelinesSchema),
query: buildRouteValidation(exportTimelinesQuerySchema),
body: buildRouteValidation(exportTimelinesRequestBodySchema),
},
options: {
tags: ['access:siem'],
Expand All @@ -42,6 +35,7 @@ export const exportTimelinesRoute = (router: IRouter, config: LegacyServices['co
const siemResponse = buildSiemResponse(response);
const savedObjectsClient = context.core.savedObjects.client;
const exportSizeLimit = config().get<number>('savedObjects.maxImportExportSize');

if (request.body?.ids != null && request.body.ids.length > exportSizeLimit) {
return siemResponse.error({
statusCode: 400,
Expand All @@ -51,7 +45,7 @@ export const exportTimelinesRoute = (router: IRouter, config: LegacyServices['co

const responseBody = await getExportTimelineByObjectIds({
client: savedObjectsClient,
request,
ids: request.body.ids,
});

return response.ok({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -334,7 +334,10 @@ describe('import timelines', () => {
const result = server.validate(request);

expect(result.badRequest).toHaveBeenCalledWith(
'child "file" fails because ["file" is required]'
[
'Invalid value undefined supplied to : { file: (ReadableRt & { hapi: { filename: string } }) }/file: (ReadableRt & { hapi: { filename: string } })/0: ReadableRt',
'Invalid value undefined supplied to : { file: (ReadableRt & { hapi: { filename: string } }) }/file: (ReadableRt & { hapi: { filename: string } })/1: { hapi: { filename: string } }',
].join('\n')
);
});
});
Expand Down
Loading

0 comments on commit e8b1317

Please sign in to comment.