Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[SIEM][Detection Engine] Converts from joi to use io-ts and moves the types to common #68127

Merged
merged 34 commits into from
Jun 9, 2020
Merged
Show file tree
Hide file tree
Changes from 31 commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
52c53d6
WIP with create_rules_schema
FrankHassanabad Jun 1, 2020
c8d607f
Added createSchema and put it into place
FrankHassanabad Jun 3, 2020
87740c9
Merge branch 'master' into add-create-schema
FrankHassanabad Jun 3, 2020
1c31834
Updated test to pass
FrankHassanabad Jun 3, 2020
427019b
Added workaround for the UI sending down strings to get the server to…
FrankHassanabad Jun 3, 2020
e7b6a5d
Fixed code for the unit test for the workaround
FrankHassanabad Jun 3, 2020
c96a6e3
Merge branch 'master' into add-create-schema
FrankHassanabad Jun 3, 2020
1a8e004
Fixed the code and tests for risk score on the front end to use a num…
FrankHassanabad Jun 4, 2020
2bcfaf3
Merge branch 'master' into add-create-schema
FrankHassanabad Jun 4, 2020
c822956
Removed duplicate files in favor of types existing within SIEM
FrankHassanabad Jun 4, 2020
115a803
Added the create rules bulk and fixed type errors as well as deleted …
FrankHassanabad Jun 4, 2020
f9849d7
Added forgotten query and language sections for the create rules bulk
FrankHassanabad Jun 4, 2020
2f78f59
Merge branch 'master' into add-create-schema
FrankHassanabad Jun 4, 2020
bb2c276
Updated to the latest folder change
FrankHassanabad Jun 4, 2020
7105a2d
Deleted the old joi schemas
FrankHassanabad Jun 4, 2020
c785a0d
Added update rules and update rules bulk and fixed bugs found within …
FrankHassanabad Jun 5, 2020
503c616
Merge branch 'master' into add-create-schema
FrankHassanabad Jun 5, 2020
0a809be
Fixed end to end tests where the schema is now stricter to work with …
FrankHassanabad Jun 5, 2020
1919db8
Deleted files I forgot to delete and removed big blocks of TODO code …
FrankHassanabad Jun 5, 2020
b9fc944
Fixed bad schema type
FrankHassanabad Jun 5, 2020
6ad9cff
One more small schema fix
FrankHassanabad Jun 5, 2020
9378017
Added the patch conversion and updated tests
FrankHassanabad Jun 6, 2020
d6cc52c
Merge branch 'master' into add-create-schema
FrankHassanabad Jun 6, 2020
26b37dd
prepackaged rules converted
FrankHassanabad Jun 6, 2020
09e50a4
Converted the query schema which includes the read and delete endpoin…
FrankHassanabad Jun 6, 2020
ae07f4b
Converted import from joi to io-ts, updated tests and had to remove o…
FrankHassanabad Jun 6, 2020
347c5a7
Converted export from joi -> iots
FrankHassanabad Jun 6, 2020
a6ebe61
Converted find from joi -> io-ts
FrankHassanabad Jun 7, 2020
df139c2
Converted the statuses and left overs from joi -> io-ts
FrankHassanabad Jun 7, 2020
fa51402
Merge branch 'master' into add-create-schema
FrankHassanabad Jun 8, 2020
8be7378
Merge branch 'master' into add-create-schema
FrankHassanabad Jun 8, 2020
3d41e5a
Fixed issue with bad conflict
FrankHassanabad Jun 8, 2020
d9993dd
Merge branch 'master' into add-create-schema
FrankHassanabad Jun 8, 2020
b06403c
Fixes from code review
FrankHassanabad Jun 9, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion x-pack/plugins/lists/common/schemas/common/schemas.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@

import * as t from 'io-ts';

import { DefaultStringArray, NonEmptyString } from '../types';
import { DefaultNamespace } from '../types/default_namespace';
import { DefaultStringArray, NonEmptyString } from '../../siem_common_deps';

export const name = t.string;
export type Name = t.TypeOf<typeof name>;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,9 @@ import {
tags,
} from '../common/schemas';
import { Identity, RequiredKeepUndefined } from '../../types';
import { DefaultEntryArray, DefaultUuid } from '../types';
import { DefaultEntryArray } from '../types';
import { EntriesArray } from '../types/entries';
import { DefaultUuid } from '../../siem_common_deps';

export const createExceptionListItemSchema = t.intersection([
t.exact(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import {
tags,
} from '../common/schemas';
import { Identity, RequiredKeepUndefined } from '../../types';
import { DefaultUuid } from '../types/default_uuid';
import { DefaultUuid } from '../../siem_common_deps';

export const createExceptionListSchema = t.intersection([
t.exact(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

/* eslint-disable @typescript-eslint/camelcase */

// TODO: You cannot import a stream from common into the front end code! CHANGE THIS
import { Readable } from 'stream';

import * as t from 'io-ts';
Expand All @@ -20,6 +21,7 @@ export const importListItemSchema = t.exact(

export type ImportListItemSchema = t.TypeOf<typeof importListItemSchema>;

// TODO: You cannot import a stream from common into the front end code! CHANGE THIS
export interface HapiReadableStream extends Readable {
hapi: {
filename: string;
Expand Down
3 changes: 0 additions & 3 deletions x-pack/plugins/lists/common/schemas/types/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,4 @@
* you may not use this file except in compliance with the Elastic License.
*/
export * from './default_entries_array';
export * from './default_string_array';
export * from './default_uuid';
export * from './entries';
export * from './non_empty_string';
3 changes: 3 additions & 0 deletions x-pack/plugins/lists/common/siem_common_deps.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,8 @@
* you may not use this file except in compliance with the Elastic License.
*/

export { NonEmptyString } from '../../security_solution/common/detection_engine/schemas/types/non_empty_string';
export { DefaultUuid } from '../../security_solution/common/detection_engine/schemas/types/default_uuid';
export { DefaultStringArray } from '../../security_solution/common/detection_engine/schemas/types/default_string_array';
export { exactCheck } from '../../security_solution/common/exact_check';
export { getPaths, foldLeftRight } from '../../security_solution/common/test_utils';
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,35 @@ import { PositiveIntegerGreaterThanZero } from '../types/positive_integer_greate
import { PositiveInteger } from '../types/positive_integer';

export const description = t.string;
export type Description = t.TypeOf<typeof description>;

export const descriptionOrUndefined = t.union([description, t.undefined]);
export type DescriptionOrUndefined = t.TypeOf<typeof descriptionOrUndefined>;

export const enabled = t.boolean;
export const exclude_export_details = t.boolean;
export type Enabled = t.TypeOf<typeof enabled>;

export const enabledOrUndefined = t.union([enabled, t.undefined]);
export type EnabledOrUndefined = t.TypeOf<typeof enabledOrUndefined>;

export const false_positives = t.array(t.string);
export type FalsePositives = t.TypeOf<typeof false_positives>;

export const falsePositivesOrUndefined = t.union([false_positives, t.undefined]);
export type FalsePositivesOrUndefined = t.TypeOf<typeof falsePositivesOrUndefined>;

export const file_name = t.string;
export type FileName = t.TypeOf<typeof file_name>;

export const exclude_export_details = t.boolean;
export type ExcludeExportDetails = t.TypeOf<typeof exclude_export_details>;

/**
* TODO: Right now the filters is an "unknown", when it could more than likely
* become the actual ESFilter as a type.
*/
export const filters = t.array(t.unknown); // Filters are not easily type-able yet
export type Filters = t.TypeOf<typeof filters>; // Filters are not easily type-able yet

/**
* Params is an "object", since it is a type of AlertActionParams which is action templates.
Expand All @@ -43,30 +62,98 @@ export const action = t.exact(
);

export const actions = t.array(action);
export type Actions = t.TypeOf<typeof actions>;

// TODO: Create a regular expression type or custom date math part type here
export const from = t.string;
export type From = t.TypeOf<typeof from>;

export const fromOrUndefined = t.union([from, t.undefined]);
export type FromOrUndefined = t.TypeOf<typeof fromOrUndefined>;

export const immutable = t.boolean;
export type Immutable = t.TypeOf<typeof immutable>;

// Note: Never make this a strict uuid, we allow the rule_id to be any string at the moment
// in case we encounter 3rd party rule systems which might be using auto incrementing numbers
// or other different things.
export const rule_id = t.string;
export type RuleId = t.TypeOf<typeof rule_id>;

export const ruleIdOrUndefined = t.union([rule_id, t.undefined]);
export type RuleIdOrUndefined = t.TypeOf<typeof ruleIdOrUndefined>;

export const id = UUID;
export const idOrUndefined = t.union([id, t.undefined]);
export type IdOrUndefined = t.TypeOf<typeof idOrUndefined>;

export const index = t.array(t.string);
export type Index = t.TypeOf<typeof index>;

export const indexOrUndefined = t.union([index, t.undefined]);
export type IndexOrUndefined = t.TypeOf<typeof indexOrUndefined>;

export const interval = t.string;
export type Interval = t.TypeOf<typeof interval>;

export const intervalOrUndefined = t.union([interval, t.undefined]);
export type IntervalOrUndefined = t.TypeOf<typeof intervalOrUndefined>;

export const query = t.string;
export type Query = t.TypeOf<typeof query>;

export const queryOrUndefined = t.union([query, t.undefined]);
export type QueryOrUndefined = t.TypeOf<typeof queryOrUndefined>;

export const language = t.keyof({ kuery: null, lucene: null });
export type Language = t.TypeOf<typeof language>;

export const languageOrUndefined = t.union([language, t.undefined]);
export type LanguageOrUndefined = t.TypeOf<typeof languageOrUndefined>;

export const objects = t.array(t.type({ rule_id }));

export const output_index = t.string;
export type OutputIndex = t.TypeOf<typeof output_index>;

export const outputIndexOrUndefined = t.union([output_index, t.undefined]);
export type OutputIndexOrUndefined = t.TypeOf<typeof outputIndexOrUndefined>;

export const saved_id = t.string;
export type SavedId = t.TypeOf<typeof saved_id>;

export const savedIdOrUndefined = t.union([saved_id, t.undefined]);
export type SavedIdOrUndefined = t.TypeOf<typeof savedIdOrUndefined>;

export const timeline_id = t.string;
export type TimelineId = t.TypeOf<typeof timeline_id>;

export const timelineIdOrUndefined = t.union([timeline_id, t.undefined]);
export type TimelineIdOrUndefined = t.TypeOf<typeof timelineIdOrUndefined>;

export const timeline_title = t.string;
export type TimelineTitle = t.TypeOf<typeof t.string>;

export const timelineTitleOrUndefined = t.union([timeline_title, t.undefined]);
export type TimelineTitleOrUndefined = t.TypeOf<typeof timelineTitleOrUndefined>;

export const throttle = t.string;
export type Throttle = t.TypeOf<typeof throttle>;

export const throttleOrNull = t.union([throttle, t.null]);
export type ThrottleOrNull = t.TypeOf<typeof throttleOrNull>;

export const anomaly_threshold = PositiveInteger;
export type AnomalyThreshold = t.TypeOf<typeof PositiveInteger>;

export const anomalyThresholdOrUndefined = t.union([anomaly_threshold, t.undefined]);
export type AnomalyThresholdOrUndefined = t.TypeOf<typeof anomalyThresholdOrUndefined>;

export const machine_learning_job_id = t.string;
export type MachineLearningJobId = t.TypeOf<typeof machine_learning_job_id>;

export const machineLearningJobIdOrUndefined = t.union([machine_learning_job_id, t.undefined]);
export type MachineLearningJobIdOrUndefined = t.TypeOf<typeof machineLearningJobIdOrUndefined>;

/**
* Note that this is a plain unknown object because we allow the UI
Expand All @@ -76,30 +163,103 @@ export const machine_learning_job_id = t.string;
* so we have tighter control over 3rd party data structures.
*/
export const meta = t.object;
export type Meta = t.TypeOf<typeof meta>;
export const metaOrUndefined = t.union([meta, t.undefined]);
export type MetaOrUndefined = t.TypeOf<typeof metaOrUndefined>;

export const max_signals = PositiveIntegerGreaterThanZero;
export type MaxSignals = t.TypeOf<typeof max_signals>;

export const maxSignalsOrUndefined = t.union([max_signals, t.undefined]);
export type MaxSignalsOrUndefined = t.TypeOf<typeof maxSignalsOrUndefined>;

export const name = t.string;
export type Name = t.TypeOf<typeof name>;

export const nameOrUndefined = t.union([name, t.undefined]);
export type NameOrUndefined = t.TypeOf<typeof nameOrUndefined>;

export const risk_score = RiskScore;
export type RiskScore = t.TypeOf<typeof risk_score>;

export const riskScoreOrUndefined = t.union([risk_score, t.undefined]);
export type RiskScoreOrUndefined = t.TypeOf<typeof riskScoreOrUndefined>;

export const severity = t.keyof({ low: null, medium: null, high: null, critical: null });
export type Severity = t.TypeOf<typeof severity>;

export const severityOrUndefined = t.union([severity, t.undefined]);
export type SeverityOrUndefined = t.TypeOf<typeof severityOrUndefined>;

export const status = t.keyof({ open: null, closed: null });

export const job_status = t.keyof({ succeeded: null, failed: null, 'going to run': null });

// TODO: Create a regular expression type or custom date math part type here
export const to = t.string;
export type To = t.TypeOf<typeof to>;

export const toOrUndefined = t.union([to, t.undefined]);
export type ToOrUndefined = t.TypeOf<typeof toOrUndefined>;

export const type = t.keyof({ machine_learning: null, query: null, saved_query: null });
export type Type = t.TypeOf<typeof type>;

export const typeOrUndefined = t.union([type, t.undefined]);
export type TypeOrUndefined = t.TypeOf<typeof typeOrUndefined>;

export const queryFilter = t.string;
export type QueryFilter = t.TypeOf<typeof queryFilter>;

export const queryFilterOrUndefined = t.union([queryFilter, t.undefined]);
export type QueryFilterOrUndefined = t.TypeOf<typeof queryFilterOrUndefined>;

export const references = t.array(t.string);
export type References = t.TypeOf<typeof references>;

export const referencesOrUndefined = t.union([references, t.undefined]);
export type ReferencesOrUndefined = t.TypeOf<typeof referencesOrUndefined>;

export const per_page = PositiveInteger;
export type PerPage = t.TypeOf<typeof per_page>;

export const perPageOrUndefined = t.union([per_page, t.undefined]);
export type PerPageOrUndefined = t.TypeOf<typeof perPageOrUndefined>;

export const page = PositiveIntegerGreaterThanZero;
export type Page = t.TypeOf<typeof page>;

export const pageOrUndefined = t.union([page, t.undefined]);
export type PageOrUndefined = t.TypeOf<typeof pageOrUndefined>;

export const signal_ids = t.array(t.string);

// TODO: Can this be more strict or is this is the set of all Elastic Queries?
export const signal_status_query = t.object;

export const sort_field = t.string;
export type SortField = t.TypeOf<typeof sort_field>;

export const sortFieldOrUndefined = t.union([sort_field, t.undefined]);
export type SortFieldOrUndefined = t.TypeOf<typeof sortFieldOrUndefined>;

export const sort_order = t.keyof({ asc: null, desc: null });
export type sortOrder = t.TypeOf<typeof sort_order>;

export const sortOrderOrUndefined = t.union([sort_order, t.undefined]);
export type SortOrderOrUndefined = t.TypeOf<typeof sortOrderOrUndefined>;

export const tags = t.array(t.string);
export type Tags = t.TypeOf<typeof tags>;

export const tagsOrUndefined = t.union([tags, t.undefined]);
export type TagsOrUndefined = t.TypeOf<typeof tagsOrUndefined>;

export const fields = t.array(t.string);
export type Fields = t.TypeOf<typeof fields>;
export const fieldsOrUndefined = t.union([fields, t.undefined]);
export type FieldsOrUndefined = t.TypeOf<typeof fieldsOrUndefined>;

export const threat_framework = t.string;
export const threat_tactic_id = t.string;
export const threat_tactic_name = t.string;
Expand Down Expand Up @@ -129,11 +289,23 @@ export const threat = t.array(
})
)
);

export type Threat = t.TypeOf<typeof threat>;

export const threatOrUndefined = t.union([threat, t.undefined]);
export type ThreatOrUndefined = t.TypeOf<typeof threatOrUndefined>;

export const created_at = IsoDateString;
export const updated_at = IsoDateString;
export const updated_by = t.string;
export const created_by = t.string;

export const version = PositiveIntegerGreaterThanZero;
export type Version = t.TypeOf<typeof version>;

export const versionOrUndefined = t.union([version, t.undefined]);
export type VersionOrUndefined = t.TypeOf<typeof versionOrUndefined>;

export const last_success_at = IsoDateString;
export const last_success_message = t.string;
export const last_failure_at = IsoDateString;
Expand All @@ -150,7 +322,12 @@ export const success_count = PositiveInteger;
export const rules_custom_installed = PositiveInteger;
export const rules_not_installed = PositiveInteger;
export const rules_not_updated = PositiveInteger;

export const note = t.string;
export type Note = t.TypeOf<typeof note>;

export const noteOrUndefined = t.union([note, t.undefined]);
export type NoteOrUndefined = t.TypeOf<typeof noteOrUndefined>;

// NOTE: Experimental list support not being shipped currently and behind a feature flag
// TODO: Remove this comment once we lists have passed testing and is ready for the release
Expand Down Expand Up @@ -185,3 +362,6 @@ export const list_and = t.intersection([
and: t.array(list),
}),
]);

export const listAndOrUndefined = t.union([t.array(list_and), t.undefined]);
export type ListAndOrUndefined = t.TypeOf<typeof listAndOrUndefined>;
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import {
AddPrepackagedRulesSchema,
AddPrepackagedRulesSchemaDecoded,
} from './add_prepackaged_rules_schema';
import { DEFAULT_MAX_SIGNALS } from '../../../constants';

export const getAddPrepackagedRulesSchemaMock = (): AddPrepackagedRulesSchema => ({
description: 'some description',
name: 'Query with a rule id',
query: 'user.name: root or user.name: admin',
severity: 'high',
type: 'query',
risk_score: 55,
language: 'kuery',
rule_id: 'rule-1',
version: 1,
});

export const getAddPrepackagedRulesSchemaDecodedMock = (): AddPrepackagedRulesSchemaDecoded => ({
description: 'some description',
name: 'Query with a rule id',
query: 'user.name: root or user.name: admin',
severity: 'high',
type: 'query',
risk_score: 55,
language: 'kuery',
references: [],
actions: [],
enabled: false,
false_positives: [],
from: 'now-6m',
interval: '5m',
max_signals: DEFAULT_MAX_SIGNALS,
tags: [],
to: 'now',
threat: [],
throttle: null,
version: 1,
exceptions_list: [],
rule_id: 'rule-1',
});
Loading