Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Security Solution][Detections] Set default indicator path to reduce friction with new filebeat modules #92081

Merged
merged 6 commits into from
Feb 25, 2021
Merged

[Security Solution][Detections] Set default indicator path to reduce friction with new filebeat modules #92081

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
7c90377
Distinguish source and destination config for indicator matches
rylnd Feb 19, 2021
d35a8fc
Update our integration tests following change of default
rylnd Feb 19, 2021
c9ec203
Merge branch 'master' into default_indicator_path
kibanamachine Feb 22, 2021
40c280b
DRY up unit tests a bit
rylnd Feb 24, 2021
4790415
Add a note for future devs
rylnd Feb 24, 2021
d5c24d4
Merge remote-tracking branch 'upstream/master' into default_indicator…
rylnd Feb 24, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions x-pack/plugins/security_solution/common/constants.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,10 @@ export const DEFAULT_RULE_REFRESH_INTERVAL_VALUE = 60000; // ms
export const DEFAULT_RULE_REFRESH_IDLE_VALUE = 2700000; // ms
export const DEFAULT_RULE_NOTIFICATION_QUERY_SIZE = 100;

// Document path where threat indicator fields are expected. Used as
// both the source of enrichment fields and the destination for enrichment in
// the generated detection alert
export const DEFAULT_INDICATOR_PATH = 'threat.indicator';
// Document path where threat indicator fields are expected. Fields are used
// to enrich signals, and are copied to threat.indicator.
export const DEFAULT_INDICATOR_SOURCE_PATH = 'threatintel.indicator';
export const INDICATOR_DESTINATION_PATH = 'threat.indicator';

export enum SecurityPageName {
detections = 'detections',
Expand Down
4 changes: 2 additions & 2 deletions ...ck/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.mock.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
* 2.0.
*/

import { DEFAULT_INDICATOR_PATH } from '../../../constants';
import { DEFAULT_INDICATOR_SOURCE_PATH } from '../../../constants';
import {
MachineLearningCreateSchema,
MachineLearningUpdateSchema,
Expand Down Expand Up @@ -57,7 +57,7 @@ export const getCreateThreatMatchRulesSchemaMock = (
rule_id: ruleId,
threat_query: '*:*',
threat_index: ['list-index'],
threat_indicator_path: DEFAULT_INDICATOR_PATH,
threat_indicator_path: DEFAULT_INDICATOR_SOURCE_PATH,
threat_mapping: [
{
entries: [
Expand Down
4 changes: 2 additions & 2 deletions .../plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
* 2.0.
*/

import { DEFAULT_INDICATOR_PATH } from '../../../constants';
import { DEFAULT_INDICATOR_SOURCE_PATH } from '../../../constants';
import { getListArrayMock } from '../types/lists.mock';

import { RulesSchema } from './rules_schema';
Expand Down Expand Up @@ -151,7 +151,7 @@ export const getThreatMatchingSchemaPartialMock = (enabled = false): Partial<Rul
language: 'kuery',
threat_query: '*:*',
threat_index: ['list-index'],
threat_indicator_path: DEFAULT_INDICATOR_PATH,
threat_indicator_path: DEFAULT_INDICATOR_SOURCE_PATH,
threat_mapping: [
{
entries: [
Expand Down
4 changes: 2 additions & 2 deletions ...ck/plugins/security_solution/public/detections/components/rules/step_about_rule/index.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ import { RiskScoreField } from '../risk_score_mapping';
import { AutocompleteField } from '../autocomplete_field';
import { useFetchIndex } from '../../../../common/containers/source';
import { isThreatMatchRule } from '../../../../../common/detection_engine/utils';
import { DEFAULT_INDICATOR_PATH } from '../../../../../common/constants';
import { DEFAULT_INDICATOR_SOURCE_PATH } from '../../../../../common/constants';

const CommonUseField = getUseField({ component: Field });

Expand Down Expand Up @@ -310,7 +310,7 @@ const StepAboutRuleComponent: FC<StepAboutRuleProps> = ({
euiFieldProps: {
fullWidth: true,
disabled: isLoading,
placeholder: DEFAULT_INDICATOR_PATH,
placeholder: DEFAULT_INDICATOR_SOURCE_PATH,
},
}}
/>
Expand Down
2 changes: 1 addition & 1 deletion ...k/plugins/security_solution/public/detections/components/rules/step_about_rule/schema.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ export const schema: FormSchema<AboutStepRule> = {
'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThreatIndicatorPathHelpText',
{
defaultMessage:
'Specify the document path containing your threat indicator fields. Used for enrichment of indicator match alerts. Defaults to threat.indicator unless otherwise specified.',
'Specify the document path containing your threat indicator fields. Used for enrichment of indicator match alerts.',
}
),
labelAppend: OptionalFieldLabel,
Expand Down
6 changes: 4 additions & 2 deletions ...ty_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_enrichment.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
* 2.0.
*/

import { DEFAULT_INDICATOR_PATH } from '../../../../../common/constants';
import { DEFAULT_INDICATOR_SOURCE_PATH } from '../../../../../common/constants';
import { SignalSearchResponse, SignalsEnrichment } from '../types';
import { enrichSignalThreatMatches } from './enrich_signal_threat_matches';
import { getThreatList } from './get_threat_list';
Expand Down Expand Up @@ -52,7 +52,9 @@ export const buildThreatEnrichment = ({
return threatResponse.hits.hits;
};

const defaultedIndicatorPath = threatIndicatorPath ? threatIndicatorPath : DEFAULT_INDICATOR_PATH;
const defaultedIndicatorPath = threatIndicatorPath
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this could be simplified as threatIndicatorPath || DEFAULT_INDICATOR_SOURCE_PATH

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch, thanks!

? threatIndicatorPath
: DEFAULT_INDICATOR_SOURCE_PATH;
return (signals: SignalSearchResponse): Promise<SignalSearchResponse> =>
enrichSignalThreatMatches(signals, getMatchedThreats, defaultedIndicatorPath);
};
46 changes: 23 additions & 23 deletions ...n/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
*/

import { get } from 'lodash';
import { DEFAULT_INDICATOR_PATH } from '../../../../../common/constants';
import { INDICATOR_DESTINATION_PATH } from '../../../../../common/constants';

import { getThreatListItemMock } from './build_threat_mapping_filter.mock';
import {
Expand Down Expand Up @@ -94,7 +94,7 @@ describe('buildMatchedIndicator', () => {
const indicators = buildMatchedIndicator({
queries: [],
threats,
indicatorPath: DEFAULT_INDICATOR_PATH,
indicatorPath: 'threat.indicator',
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if the intention is to keep this explicit, no need for modification - although it looks like we could declare an indicatorPath variable and assign it in thebeforeEach block with the value threat.indicator or INDICATOR_DESTINATION_PATH so that we can use shorthand. In the same vein, it looks like there are a couple of tests that build the matched indicators in the same manner, so maybe it could make sense to declare a new var called defaultIndicators in the same beforeEach block where the defaultIndicators would be buildMatchedIndicator({queries, threats, indicatorPath})

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed on DRYing this up, I think I'll do that here 👍. The intention with hardcoding the path here here was to make the tests robust to default values changing; since defaulting happens outside of these functions I only want those integration/UI tests to change if those defaults do.

});

expect(indicators).toEqual([]);
Expand All @@ -104,7 +104,7 @@ describe('buildMatchedIndicator', () => {
const [indicator] = buildMatchedIndicator({
queries,
threats,
indicatorPath: DEFAULT_INDICATOR_PATH,
indicatorPath: 'threat.indicator',
});

expect(get(indicator, 'matched.atomic')).toEqual('domain_1');
Expand All @@ -114,7 +114,7 @@ describe('buildMatchedIndicator', () => {
const [indicator] = buildMatchedIndicator({
queries,
threats,
indicatorPath: DEFAULT_INDICATOR_PATH,
indicatorPath: 'threat.indicator',
});

expect(get(indicator, 'matched.field')).toEqual('event.field');
Expand All @@ -124,7 +124,7 @@ describe('buildMatchedIndicator', () => {
const [indicator] = buildMatchedIndicator({
queries,
threats,
indicatorPath: DEFAULT_INDICATOR_PATH,
indicatorPath: 'threat.indicator',
});

expect(get(indicator, 'matched.type')).toEqual('type_1');
Expand Down Expand Up @@ -153,7 +153,7 @@ describe('buildMatchedIndicator', () => {
const indicators = buildMatchedIndicator({
queries,
threats,
indicatorPath: DEFAULT_INDICATOR_PATH,
indicatorPath: 'threat.indicator',
});

expect(indicators).toHaveLength(queries.length);
Expand All @@ -163,7 +163,7 @@ describe('buildMatchedIndicator', () => {
const indicators = buildMatchedIndicator({
queries,
threats,
indicatorPath: DEFAULT_INDICATOR_PATH,
indicatorPath: 'threat.indicator',
});

expect(indicators).toEqual([
Expand Down Expand Up @@ -228,7 +228,7 @@ describe('buildMatchedIndicator', () => {
const indicators = buildMatchedIndicator({
queries,
threats,
indicatorPath: DEFAULT_INDICATOR_PATH,
indicatorPath: 'threat.indicator',
});

expect(indicators).toEqual([
Expand All @@ -253,7 +253,7 @@ describe('buildMatchedIndicator', () => {
const indicators = buildMatchedIndicator({
queries,
threats,
indicatorPath: DEFAULT_INDICATOR_PATH,
indicatorPath: 'threat.indicator',
});

expect(indicators).toEqual([
Expand Down Expand Up @@ -285,7 +285,7 @@ describe('buildMatchedIndicator', () => {
const indicators = buildMatchedIndicator({
queries,
threats,
indicatorPath: DEFAULT_INDICATOR_PATH,
indicatorPath: 'threat.indicator',
});

expect(indicators).toEqual([
Expand Down Expand Up @@ -317,7 +317,7 @@ describe('buildMatchedIndicator', () => {
buildMatchedIndicator({
queries,
threats,
indicatorPath: DEFAULT_INDICATOR_PATH,
indicatorPath: 'threat.indicator',
})
).toThrowError('Expected indicator field to be an object, but found: not an object');
});
Expand All @@ -338,7 +338,7 @@ describe('buildMatchedIndicator', () => {
buildMatchedIndicator({
queries,
threats,
indicatorPath: DEFAULT_INDICATOR_PATH,
indicatorPath: 'threat.indicator',
})
).toThrowError('Expected indicator field to be an object, but found: not an object');
});
Expand Down Expand Up @@ -367,7 +367,7 @@ describe('enrichSignalThreatMatches', () => {
const enrichedSignals = await enrichSignalThreatMatches(
signals,
getMatchedThreats,
DEFAULT_INDICATOR_PATH
'threat.indicator'
);

expect(enrichedSignals.hits.hits).toEqual([]);
Expand All @@ -382,10 +382,10 @@ describe('enrichSignalThreatMatches', () => {
const enrichedSignals = await enrichSignalThreatMatches(
signals,
getMatchedThreats,
DEFAULT_INDICATOR_PATH
'threat.indicator'
);
const [enrichedHit] = enrichedSignals.hits.hits;
const indicators = get(enrichedHit._source, DEFAULT_INDICATOR_PATH);
const indicators = get(enrichedHit._source, INDICATOR_DESTINATION_PATH);

expect(indicators).toEqual([
{ existing: 'indicator' },
Expand All @@ -407,10 +407,10 @@ describe('enrichSignalThreatMatches', () => {
const enrichedSignals = await enrichSignalThreatMatches(
signals,
getMatchedThreats,
DEFAULT_INDICATOR_PATH
'threat.indicator'
);
const [enrichedHit] = enrichedSignals.hits.hits;
const indicators = get(enrichedHit._source, DEFAULT_INDICATOR_PATH);
const indicators = get(enrichedHit._source, INDICATOR_DESTINATION_PATH);

expect(indicators).toEqual([
{
Expand All @@ -428,10 +428,10 @@ describe('enrichSignalThreatMatches', () => {
const enrichedSignals = await enrichSignalThreatMatches(
signals,
getMatchedThreats,
DEFAULT_INDICATOR_PATH
'threat.indicator'
);
const [enrichedHit] = enrichedSignals.hits.hits;
const indicators = get(enrichedHit._source, DEFAULT_INDICATOR_PATH);
const indicators = get(enrichedHit._source, INDICATOR_DESTINATION_PATH);

expect(indicators).toEqual([
{ existing: 'indicator' },
Expand All @@ -451,7 +451,7 @@ describe('enrichSignalThreatMatches', () => {
});
const signals = getSignalsResponseMock([signalHit]);
await expect(() =>
enrichSignalThreatMatches(signals, getMatchedThreats, DEFAULT_INDICATOR_PATH)
enrichSignalThreatMatches(signals, getMatchedThreats, 'threat.indicator')
).rejects.toThrowError('Expected threat field to be an object, but found: whoops');
});

Expand Down Expand Up @@ -487,7 +487,7 @@ describe('enrichSignalThreatMatches', () => {
'custom_threat.custom_indicator'
);
const [enrichedHit] = enrichedSignals.hits.hits;
const indicators = get(enrichedHit._source, DEFAULT_INDICATOR_PATH);
const indicators = get(enrichedHit._source, INDICATOR_DESTINATION_PATH);

expect(indicators).toEqual([
{
Expand Down Expand Up @@ -530,13 +530,13 @@ describe('enrichSignalThreatMatches', () => {
const enrichedSignals = await enrichSignalThreatMatches(
signals,
getMatchedThreats,
DEFAULT_INDICATOR_PATH
'threat.indicator'
);
expect(enrichedSignals.hits.total).toEqual(expect.objectContaining({ value: 1 }));
expect(enrichedSignals.hits.hits).toHaveLength(1);

const [enrichedHit] = enrichedSignals.hits.hits;
const indicators = get(enrichedHit._source, DEFAULT_INDICATOR_PATH);
const indicators = get(enrichedHit._source, INDICATOR_DESTINATION_PATH);

expect(indicators).toEqual([
{
Expand Down
3 changes: 1 addition & 2 deletions ...lution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
*/

import { get, isObject } from 'lodash';
import { DEFAULT_INDICATOR_PATH } from '../../../../../common/constants';

import type { SignalSearchResponse, SignalSourceHit } from '../types';
import type {
Expand Down Expand Up @@ -92,7 +91,7 @@ export const enrichSignalThreatMatches = async (
if (!isObject(threat)) {
throw new Error(`Expected threat field to be an object, but found: ${threat}`);
}
const existingIndicatorValue = get(signalHit._source, DEFAULT_INDICATOR_PATH) ?? [];
const existingIndicatorValue = get(signalHit._source, 'threat.indicator') ?? [];
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we could use the previously declared INDICATOR_DESTINATION_PATH here instead of the hardcoded value, as this is not a test file

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm inclined to leave this hardcoded string for now, as the check above and the object generation below this line both make assumptions about this string in different ways.

const existingIndicators = [existingIndicatorValue].flat(); // ensure indicators is an array

return {
Expand Down
4 changes: 4 additions & 0 deletions ...test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,7 @@ export default ({ getService }: FtrProviderContext) => {
rule_id: 'rule-1',
from: '1900-01-01T00:00:00.000Z',
query: '*:*',
threat_indicator_path: 'threat.indicator',
threat_query: 'threat.indicator.domain: *', // narrow things down to indicators with a domain
threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module
threat_mapping: [
Expand Down Expand Up @@ -353,6 +354,7 @@ export default ({ getService }: FtrProviderContext) => {
rule_id: 'rule-1',
from: '1900-01-01T00:00:00.000Z',
query: 'source.port: 57324', // narrow our query to a single record that matches two indicators
threat_indicator_path: 'threat.indicator',
threat_query: 'threat.indicator.ip: *',
threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module
threat_mapping: [
Expand Down Expand Up @@ -422,6 +424,7 @@ export default ({ getService }: FtrProviderContext) => {
rule_id: 'rule-1',
from: '1900-01-01T00:00:00.000Z',
query: 'source.port: 57324', // narrow our query to a single record that matches two indicators
threat_indicator_path: 'threat.indicator',
threat_query: 'threat.indicator.ip: *',
threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module
threat_mapping: [
Expand Down Expand Up @@ -519,6 +522,7 @@ export default ({ getService }: FtrProviderContext) => {
rule_id: 'rule-1',
from: '1900-01-01T00:00:00.000Z',
query: '*:*', // narrow our query to a single record that matches two indicators
threat_indicator_path: 'threat.indicator',
threat_query: '',
threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module
threat_mapping: [
Expand Down