Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Security Solution][Detections] Set default indicator path to reduce friction with new filebeat modules #92081

Merged
merged 6 commits into from
Feb 25, 2021
Merged

[Security Solution][Detections] Set default indicator path to reduce friction with new filebeat modules #92081

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
7c90377
Distinguish source and destination config for indicator matches
rylnd Feb 19, 2021
d35a8fc
Update our integration tests following change of default
rylnd Feb 19, 2021
c9ec203
Merge branch 'master' into default_indicator_path
kibanamachine Feb 22, 2021
40c280b
DRY up unit tests a bit
rylnd Feb 24, 2021
4790415
Add a note for future devs
rylnd Feb 24, 2021
d5c24d4
Merge remote-tracking branch 'upstream/master' into default_indicator…
rylnd Feb 24, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions x-pack/plugins/security_solution/common/constants.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,10 @@ export const DEFAULT_RULE_REFRESH_INTERVAL_VALUE = 60000; // ms
export const DEFAULT_RULE_REFRESH_IDLE_VALUE = 2700000; // ms
export const DEFAULT_RULE_NOTIFICATION_QUERY_SIZE = 100;

// Document path where threat indicator fields are expected. Used as
// both the source of enrichment fields and the destination for enrichment in
// the generated detection alert
export const DEFAULT_INDICATOR_PATH = 'threat.indicator';
// Document path where threat indicator fields are expected. Fields are used
// to enrich signals, and are copied to threat.indicator.
export const DEFAULT_INDICATOR_SOURCE_PATH = 'threatintel.indicator';
export const INDICATOR_DESTINATION_PATH = 'threat.indicator';

export enum SecurityPageName {
detections = 'detections',
Expand Down
4 changes: 2 additions & 2 deletions ...ck/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.mock.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
* 2.0.
*/

import { DEFAULT_INDICATOR_PATH } from '../../../constants';
import { DEFAULT_INDICATOR_SOURCE_PATH } from '../../../constants';
import {
MachineLearningCreateSchema,
MachineLearningUpdateSchema,
Expand Down Expand Up @@ -57,7 +57,7 @@ export const getCreateThreatMatchRulesSchemaMock = (
rule_id: ruleId,
threat_query: '*:*',
threat_index: ['list-index'],
threat_indicator_path: DEFAULT_INDICATOR_PATH,
threat_indicator_path: DEFAULT_INDICATOR_SOURCE_PATH,
threat_mapping: [
{
entries: [
Expand Down
4 changes: 2 additions & 2 deletions .../plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
* 2.0.
*/

import { DEFAULT_INDICATOR_PATH } from '../../../constants';
import { DEFAULT_INDICATOR_SOURCE_PATH } from '../../../constants';
import { getListArrayMock } from '../types/lists.mock';

import { RulesSchema } from './rules_schema';
Expand Down Expand Up @@ -151,7 +151,7 @@ export const getThreatMatchingSchemaPartialMock = (enabled = false): Partial<Rul
language: 'kuery',
threat_query: '*:*',
threat_index: ['list-index'],
threat_indicator_path: DEFAULT_INDICATOR_PATH,
threat_indicator_path: DEFAULT_INDICATOR_SOURCE_PATH,
threat_mapping: [
{
entries: [
Expand Down
4 changes: 2 additions & 2 deletions ...ck/plugins/security_solution/public/detections/components/rules/step_about_rule/index.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ import { RiskScoreField } from '../risk_score_mapping';
import { AutocompleteField } from '../autocomplete_field';
import { useFetchIndex } from '../../../../common/containers/source';
import { isThreatMatchRule } from '../../../../../common/detection_engine/utils';
import { DEFAULT_INDICATOR_PATH } from '../../../../../common/constants';
import { DEFAULT_INDICATOR_SOURCE_PATH } from '../../../../../common/constants';

const CommonUseField = getUseField({ component: Field });

Expand Down Expand Up @@ -310,7 +310,7 @@ const StepAboutRuleComponent: FC<StepAboutRuleProps> = ({
euiFieldProps: {
fullWidth: true,
disabled: isLoading,
placeholder: DEFAULT_INDICATOR_PATH,
placeholder: DEFAULT_INDICATOR_SOURCE_PATH,
},
}}
/>
Expand Down
2 changes: 1 addition & 1 deletion ...k/plugins/security_solution/public/detections/components/rules/step_about_rule/schema.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ export const schema: FormSchema<AboutStepRule> = {
'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThreatIndicatorPathHelpText',
{
defaultMessage:
'Specify the document path containing your threat indicator fields. Used for enrichment of indicator match alerts. Defaults to threat.indicator unless otherwise specified.',
'Specify the document path containing your threat indicator fields. Used for enrichment of indicator match alerts.',
}
),
labelAppend: OptionalFieldLabel,
Expand Down
6 changes: 4 additions & 2 deletions ...ty_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_enrichment.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
* 2.0.
*/

import { DEFAULT_INDICATOR_PATH } from '../../../../../common/constants';
import { DEFAULT_INDICATOR_SOURCE_PATH } from '../../../../../common/constants';
import { SignalSearchResponse, SignalsEnrichment } from '../types';
import { enrichSignalThreatMatches } from './enrich_signal_threat_matches';
import { getThreatList } from './get_threat_list';
Expand Down Expand Up @@ -52,7 +52,9 @@ export const buildThreatEnrichment = ({
return threatResponse.hits.hits;
};

const defaultedIndicatorPath = threatIndicatorPath ? threatIndicatorPath : DEFAULT_INDICATOR_PATH;
const defaultedIndicatorPath = threatIndicatorPath
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this could be simplified as threatIndicatorPath || DEFAULT_INDICATOR_SOURCE_PATH

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch, thanks!

? threatIndicatorPath
: DEFAULT_INDICATOR_SOURCE_PATH;
return (signals: SignalSearchResponse): Promise<SignalSearchResponse> =>
enrichSignalThreatMatches(signals, getMatchedThreats, defaultedIndicatorPath);
};
50 changes: 27 additions & 23 deletions ...n/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
*/

import { get } from 'lodash';
import { DEFAULT_INDICATOR_PATH } from '../../../../../common/constants';
import { INDICATOR_DESTINATION_PATH } from '../../../../../common/constants';

import { getThreatListItemMock } from './build_threat_mapping_filter.mock';
import {
Expand Down Expand Up @@ -75,8 +75,10 @@ describe('groupAndMergeSignalMatches', () => {
describe('buildMatchedIndicator', () => {
let threats: ThreatListItem[];
let queries: ThreatMatchNamedQuery[];
let indicatorPath: string;

beforeEach(() => {
indicatorPath = 'threat.indicator';
threats = [
getThreatListItemMock({
_id: '123',
Expand All @@ -94,7 +96,7 @@ describe('buildMatchedIndicator', () => {
const indicators = buildMatchedIndicator({
queries: [],
threats,
indicatorPath: DEFAULT_INDICATOR_PATH,
indicatorPath,
});

expect(indicators).toEqual([]);
Expand All @@ -104,7 +106,7 @@ describe('buildMatchedIndicator', () => {
const [indicator] = buildMatchedIndicator({
queries,
threats,
indicatorPath: DEFAULT_INDICATOR_PATH,
indicatorPath,
});

expect(get(indicator, 'matched.atomic')).toEqual('domain_1');
Expand All @@ -114,7 +116,7 @@ describe('buildMatchedIndicator', () => {
const [indicator] = buildMatchedIndicator({
queries,
threats,
indicatorPath: DEFAULT_INDICATOR_PATH,
indicatorPath,
});

expect(get(indicator, 'matched.field')).toEqual('event.field');
Expand All @@ -124,7 +126,7 @@ describe('buildMatchedIndicator', () => {
const [indicator] = buildMatchedIndicator({
queries,
threats,
indicatorPath: DEFAULT_INDICATOR_PATH,
indicatorPath,
});

expect(get(indicator, 'matched.type')).toEqual('type_1');
Expand Down Expand Up @@ -153,7 +155,7 @@ describe('buildMatchedIndicator', () => {
const indicators = buildMatchedIndicator({
queries,
threats,
indicatorPath: DEFAULT_INDICATOR_PATH,
indicatorPath,
});

expect(indicators).toHaveLength(queries.length);
Expand All @@ -163,7 +165,7 @@ describe('buildMatchedIndicator', () => {
const indicators = buildMatchedIndicator({
queries,
threats,
indicatorPath: DEFAULT_INDICATOR_PATH,
indicatorPath,
});

expect(indicators).toEqual([
Expand Down Expand Up @@ -228,7 +230,7 @@ describe('buildMatchedIndicator', () => {
const indicators = buildMatchedIndicator({
queries,
threats,
indicatorPath: DEFAULT_INDICATOR_PATH,
indicatorPath,
});

expect(indicators).toEqual([
Expand All @@ -253,7 +255,7 @@ describe('buildMatchedIndicator', () => {
const indicators = buildMatchedIndicator({
queries,
threats,
indicatorPath: DEFAULT_INDICATOR_PATH,
indicatorPath,
});

expect(indicators).toEqual([
Expand Down Expand Up @@ -285,7 +287,7 @@ describe('buildMatchedIndicator', () => {
const indicators = buildMatchedIndicator({
queries,
threats,
indicatorPath: DEFAULT_INDICATOR_PATH,
indicatorPath,
});

expect(indicators).toEqual([
Expand Down Expand Up @@ -317,7 +319,7 @@ describe('buildMatchedIndicator', () => {
buildMatchedIndicator({
queries,
threats,
indicatorPath: DEFAULT_INDICATOR_PATH,
indicatorPath,
})
).toThrowError('Expected indicator field to be an object, but found: not an object');
});
Expand All @@ -338,7 +340,7 @@ describe('buildMatchedIndicator', () => {
buildMatchedIndicator({
queries,
threats,
indicatorPath: DEFAULT_INDICATOR_PATH,
indicatorPath,
})
).toThrowError('Expected indicator field to be an object, but found: not an object');
});
Expand All @@ -347,8 +349,10 @@ describe('buildMatchedIndicator', () => {
describe('enrichSignalThreatMatches', () => {
let getMatchedThreats: GetMatchedThreats;
let matchedQuery: string;
let indicatorPath: string;

beforeEach(() => {
indicatorPath = 'threat.indicator';
getMatchedThreats = async () => [
getThreatListItemMock({
_id: '123',
Expand All @@ -367,7 +371,7 @@ describe('enrichSignalThreatMatches', () => {
const enrichedSignals = await enrichSignalThreatMatches(
signals,
getMatchedThreats,
DEFAULT_INDICATOR_PATH
indicatorPath
);

expect(enrichedSignals.hits.hits).toEqual([]);
Expand All @@ -382,10 +386,10 @@ describe('enrichSignalThreatMatches', () => {
const enrichedSignals = await enrichSignalThreatMatches(
signals,
getMatchedThreats,
DEFAULT_INDICATOR_PATH
indicatorPath
);
const [enrichedHit] = enrichedSignals.hits.hits;
const indicators = get(enrichedHit._source, DEFAULT_INDICATOR_PATH);
const indicators = get(enrichedHit._source, INDICATOR_DESTINATION_PATH);

expect(indicators).toEqual([
{ existing: 'indicator' },
Expand All @@ -407,10 +411,10 @@ describe('enrichSignalThreatMatches', () => {
const enrichedSignals = await enrichSignalThreatMatches(
signals,
getMatchedThreats,
DEFAULT_INDICATOR_PATH
indicatorPath
);
const [enrichedHit] = enrichedSignals.hits.hits;
const indicators = get(enrichedHit._source, DEFAULT_INDICATOR_PATH);
const indicators = get(enrichedHit._source, INDICATOR_DESTINATION_PATH);

expect(indicators).toEqual([
{
Expand All @@ -428,10 +432,10 @@ describe('enrichSignalThreatMatches', () => {
const enrichedSignals = await enrichSignalThreatMatches(
signals,
getMatchedThreats,
DEFAULT_INDICATOR_PATH
indicatorPath
);
const [enrichedHit] = enrichedSignals.hits.hits;
const indicators = get(enrichedHit._source, DEFAULT_INDICATOR_PATH);
const indicators = get(enrichedHit._source, INDICATOR_DESTINATION_PATH);

expect(indicators).toEqual([
{ existing: 'indicator' },
Expand All @@ -451,7 +455,7 @@ describe('enrichSignalThreatMatches', () => {
});
const signals = getSignalsResponseMock([signalHit]);
await expect(() =>
enrichSignalThreatMatches(signals, getMatchedThreats, DEFAULT_INDICATOR_PATH)
enrichSignalThreatMatches(signals, getMatchedThreats, indicatorPath)
).rejects.toThrowError('Expected threat field to be an object, but found: whoops');
});

Expand Down Expand Up @@ -487,7 +491,7 @@ describe('enrichSignalThreatMatches', () => {
'custom_threat.custom_indicator'
);
const [enrichedHit] = enrichedSignals.hits.hits;
const indicators = get(enrichedHit._source, DEFAULT_INDICATOR_PATH);
const indicators = get(enrichedHit._source, INDICATOR_DESTINATION_PATH);

expect(indicators).toEqual([
{
Expand Down Expand Up @@ -530,13 +534,13 @@ describe('enrichSignalThreatMatches', () => {
const enrichedSignals = await enrichSignalThreatMatches(
signals,
getMatchedThreats,
DEFAULT_INDICATOR_PATH
indicatorPath
);
expect(enrichedSignals.hits.total).toEqual(expect.objectContaining({ value: 1 }));
expect(enrichedSignals.hits.hits).toHaveLength(1);

const [enrichedHit] = enrichedSignals.hits.hits;
const indicators = get(enrichedHit._source, DEFAULT_INDICATOR_PATH);
const indicators = get(enrichedHit._source, INDICATOR_DESTINATION_PATH);

expect(indicators).toEqual([
{
Expand Down
7 changes: 5 additions & 2 deletions ...lution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
*/

import { get, isObject } from 'lodash';
import { DEFAULT_INDICATOR_PATH } from '../../../../../common/constants';

import type { SignalSearchResponse, SignalSourceHit } from '../types';
import type {
Expand Down Expand Up @@ -92,7 +91,11 @@ export const enrichSignalThreatMatches = async (
if (!isObject(threat)) {
throw new Error(`Expected threat field to be an object, but found: ${threat}`);
}
const existingIndicatorValue = get(signalHit._source, DEFAULT_INDICATOR_PATH) ?? [];
// We are not using INDICATOR_DESTINATION_PATH here because the code above
// and below make assumptions about its current value, 'threat.indicator',
// and making this code dynamic on an arbitrary path would introduce several
// new issues.
const existingIndicatorValue = get(signalHit._source, 'threat.indicator') ?? [];
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we could use the previously declared INDICATOR_DESTINATION_PATH here instead of the hardcoded value, as this is not a test file

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm inclined to leave this hardcoded string for now, as the check above and the object generation below this line both make assumptions about this string in different ways.

const existingIndicators = [existingIndicatorValue].flat(); // ensure indicators is an array

return {
Expand Down
4 changes: 4 additions & 0 deletions ...test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,7 @@ export default ({ getService }: FtrProviderContext) => {
rule_id: 'rule-1',
from: '1900-01-01T00:00:00.000Z',
query: '*:*',
threat_indicator_path: 'threat.indicator',
threat_query: 'threat.indicator.domain: *', // narrow things down to indicators with a domain
threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module
threat_mapping: [
Expand Down Expand Up @@ -353,6 +354,7 @@ export default ({ getService }: FtrProviderContext) => {
rule_id: 'rule-1',
from: '1900-01-01T00:00:00.000Z',
query: 'source.port: 57324', // narrow our query to a single record that matches two indicators
threat_indicator_path: 'threat.indicator',
threat_query: 'threat.indicator.ip: *',
threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module
threat_mapping: [
Expand Down Expand Up @@ -422,6 +424,7 @@ export default ({ getService }: FtrProviderContext) => {
rule_id: 'rule-1',
from: '1900-01-01T00:00:00.000Z',
query: 'source.port: 57324', // narrow our query to a single record that matches two indicators
threat_indicator_path: 'threat.indicator',
threat_query: 'threat.indicator.ip: *',
threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module
threat_mapping: [
Expand Down Expand Up @@ -519,6 +522,7 @@ export default ({ getService }: FtrProviderContext) => {
rule_id: 'rule-1',
from: '1900-01-01T00:00:00.000Z',
query: '*:*', // narrow our query to a single record that matches two indicators
threat_indicator_path: 'threat.indicator',
threat_query: '',
threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module
threat_mapping: [
Expand Down