From 4d3ca34627101edcf5302983c19a332214851024 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 13 Jul 2020 13:18:47 -0500 Subject: [PATCH] [Security Solution][Detections] Adoption telemetry (#71102) * style: sort plugin interface * WIP: UsageCollector for Security Adoption This uses ML and raw ES calls to query our ML Jobs and Rules, and parse them into a format to be consumed by telemetry. Still to come: * initialization * tests * Initialize usage collectors during plugin setup * Rename usage key The service seems to convert colons to underscores, so let's just use an underscure. * Collector is ready if we have a kibana index * Refactor collector to generate options in a function This allows us to test our adherence to the collector API, focusing particularly on the fetch function. * Refactor usage collector in anticipation of endpoint data We're going to have our usage data under one key corresponding to the app, so this nests the existing data under a 'detections' key while allowing another fetching function to be plugged into the main collector under a separate key. * Update our collector to satisfy telemetry tooling * inlines collector options * inlines schema object * makes DetectionsUsage an interface instead of a type alias * Extracts telemetry mappings via scripts/telemetry_extract * Refactor detections usage logic to perform one loop instead of two We were previously performing two loops over each set of data: one to format it down to just the data we need, and another to convert that into usage data. We now perform both steps within a single loop. * Refactor detections telemetry to be nested * Extract new nested detections telemetry mappings Co-authored-by: Elastic Machine --- .../security_solution/server/plugin.ts | 13 +- .../server/usage/collector.ts | 54 +++++ .../server/usage/detections.mocks.ts | 162 +++++++++++++++ .../server/usage/detections.test.ts | 107 ++++++++++ .../server/usage/detections.ts | 39 ++++ .../server/usage/detections_helpers.ts | 188 ++++++++++++++++++ .../security_solution/server/usage/index.ts | 14 ++ .../security_solution/server/usage/types.ts | 12 ++ .../schema/xpack_plugins.json | 56 ++++++ 9 files changed, 643 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/usage/collector.ts create mode 100644 x-pack/plugins/security_solution/server/usage/detections.mocks.ts create mode 100644 x-pack/plugins/security_solution/server/usage/detections.test.ts create mode 100644 x-pack/plugins/security_solution/server/usage/detections.ts create mode 100644 x-pack/plugins/security_solution/server/usage/detections_helpers.ts create mode 100644 x-pack/plugins/security_solution/server/usage/index.ts create mode 100644 x-pack/plugins/security_solution/server/usage/types.ts diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index d4935f1aabc1c..ebd95fe79ebf5 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -16,6 +16,7 @@ import { PluginInitializerContext, SavedObjectsClient, } from '../../../../src/core/server'; +import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/server'; import { PluginSetupContract as AlertingSetup } from '../../alerts/server'; import { SecurityPluginSetup as SecuritySetup } from '../../security/server'; import { PluginSetupContract as FeaturesSetup } from '../../features/server'; @@ -46,17 +47,19 @@ import { ArtifactClient, ManifestManager } from './endpoint/services'; import { EndpointAppContextService } from './endpoint/endpoint_app_context_services'; import { EndpointAppContext } from './endpoint/types'; import { registerDownloadExceptionListRoute } from './endpoint/routes/artifacts'; +import { initUsageCollectors } from './usage'; export interface SetupPlugins { alerts: AlertingSetup; encryptedSavedObjects?: EncryptedSavedObjectsSetup; features: FeaturesSetup; licensing: LicensingPluginSetup; + lists?: ListPluginSetup; + ml?: MlSetup; security?: SecuritySetup; spaces?: SpacesSetup; taskManager?: TaskManagerSetupContract; - ml?: MlSetup; - lists?: ListPluginSetup; + usageCollection?: UsageCollectionSetup; } export interface StartPlugins { @@ -106,9 +109,15 @@ export class Plugin implements IPlugin void; +export interface UsageData { + detections: DetectionsUsage; +} + +export const registerCollector: RegisterCollector = ({ kibanaIndex, ml, usageCollection }) => { + if (!usageCollection) { + return; + } + + const collector = usageCollection.makeUsageCollector({ + type: 'security_solution', + schema: { + detections: { + detection_rules: { + custom: { + enabled: { type: 'long' }, + disabled: { type: 'long' }, + }, + elastic: { + enabled: { type: 'long' }, + disabled: { type: 'long' }, + }, + }, + ml_jobs: { + custom: { + enabled: { type: 'long' }, + disabled: { type: 'long' }, + }, + elastic: { + enabled: { type: 'long' }, + disabled: { type: 'long' }, + }, + }, + }, + }, + isReady: () => kibanaIndex.length > 0, + fetch: async (callCluster: LegacyAPICaller): Promise => ({ + detections: await fetchDetectionsUsage(kibanaIndex, callCluster, ml), + }), + }); + + usageCollection.registerCollector(collector); +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections.mocks.ts b/x-pack/plugins/security_solution/server/usage/detections.mocks.ts new file mode 100644 index 0000000000000..c80dc6936ec7b --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections.mocks.ts @@ -0,0 +1,162 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { INTERNAL_IMMUTABLE_KEY } from '../../common/constants'; + +export const getMockJobSummaryResponse = () => [ + { + id: 'linux_anomalous_network_activity_ecs', + description: + 'SIEM Auditbeat: Looks for unusual processes using the network which could indicate command-and-control, lateral movement, persistence, or data exfiltration activity (beta)', + groups: ['auditbeat', 'process', 'siem'], + processed_record_count: 141889, + memory_status: 'ok', + jobState: 'opened', + hasDatafeed: true, + datafeedId: 'datafeed-linux_anomalous_network_activity_ecs', + datafeedIndices: ['auditbeat-*'], + datafeedState: 'started', + latestTimestampMs: 1594085401911, + earliestTimestampMs: 1593054845656, + latestResultsTimestampMs: 1594085401911, + isSingleMetricViewerJob: true, + nodeName: 'node', + }, + { + id: 'linux_anomalous_network_port_activity_ecs', + description: + 'SIEM Auditbeat: Looks for unusual destination port activity that could indicate command-and-control, persistence mechanism, or data exfiltration activity (beta)', + groups: ['auditbeat', 'process', 'siem'], + processed_record_count: 0, + memory_status: 'ok', + jobState: 'closed', + hasDatafeed: true, + datafeedId: 'datafeed-linux_anomalous_network_port_activity_ecs', + datafeedIndices: ['auditbeat-*'], + datafeedState: 'stopped', + isSingleMetricViewerJob: true, + }, + { + id: 'other_job', + description: 'a job that is custom', + groups: ['auditbeat', 'process'], + processed_record_count: 0, + memory_status: 'ok', + jobState: 'closed', + hasDatafeed: true, + datafeedId: 'datafeed-other', + datafeedIndices: ['auditbeat-*'], + datafeedState: 'stopped', + isSingleMetricViewerJob: true, + }, + { + id: 'another_job', + description: 'another job that is custom', + groups: ['auditbeat', 'process'], + processed_record_count: 0, + memory_status: 'ok', + jobState: 'opened', + hasDatafeed: true, + datafeedId: 'datafeed-another', + datafeedIndices: ['auditbeat-*'], + datafeedState: 'started', + isSingleMetricViewerJob: true, + }, +]; + +export const getMockListModulesResponse = () => [ + { + id: 'siem_auditbeat', + title: 'SIEM Auditbeat', + description: + 'Detect suspicious network activity and unusual processes in Auditbeat data (beta).', + type: 'Auditbeat data', + logoFile: 'logo.json', + defaultIndexPattern: 'auditbeat-*', + query: { + bool: { + filter: [ + { + term: { + 'agent.type': 'auditbeat', + }, + }, + ], + }, + }, + jobs: [ + { + id: 'linux_anomalous_network_activity_ecs', + config: { + job_type: 'anomaly_detector', + description: + 'SIEM Auditbeat: Looks for unusual processes using the network which could indicate command-and-control, lateral movement, persistence, or data exfiltration activity (beta)', + groups: ['siem', 'auditbeat', 'process'], + analysis_config: { + bucket_span: '15m', + detectors: [ + { + detector_description: 'rare by "process.name"', + function: 'rare', + by_field_name: 'process.name', + }, + ], + influencers: ['host.name', 'process.name', 'user.name', 'destination.ip'], + }, + allow_lazy_open: true, + analysis_limits: { + model_memory_limit: '64mb', + }, + data_description: { + time_field: '@timestamp', + }, + }, + }, + { + id: 'linux_anomalous_network_port_activity_ecs', + config: { + job_type: 'anomaly_detector', + description: + 'SIEM Auditbeat: Looks for unusual destination port activity that could indicate command-and-control, persistence mechanism, or data exfiltration activity (beta)', + groups: ['siem', 'auditbeat', 'network'], + analysis_config: { + bucket_span: '15m', + detectors: [ + { + detector_description: 'rare by "destination.port"', + function: 'rare', + by_field_name: 'destination.port', + }, + ], + influencers: ['host.name', 'process.name', 'user.name', 'destination.ip'], + }, + allow_lazy_open: true, + analysis_limits: { + model_memory_limit: '32mb', + }, + data_description: { + time_field: '@timestamp', + }, + }, + }, + ], + datafeeds: [], + kibana: {}, + }, +]; + +export const getMockRulesResponse = () => ({ + hits: { + hits: [ + { _source: { alert: { enabled: true, tags: [`${INTERNAL_IMMUTABLE_KEY}:true`] } } }, + { _source: { alert: { enabled: true, tags: [`${INTERNAL_IMMUTABLE_KEY}:false`] } } }, + { _source: { alert: { enabled: false, tags: [`${INTERNAL_IMMUTABLE_KEY}:true`] } } }, + { _source: { alert: { enabled: true, tags: [`${INTERNAL_IMMUTABLE_KEY}:true`] } } }, + { _source: { alert: { enabled: false, tags: [`${INTERNAL_IMMUTABLE_KEY}:false`] } } }, + { _source: { alert: { enabled: false, tags: [`${INTERNAL_IMMUTABLE_KEY}:true`] } } }, + { _source: { alert: { enabled: false, tags: [`${INTERNAL_IMMUTABLE_KEY}:true`] } } }, + ], + }, +}); diff --git a/x-pack/plugins/security_solution/server/usage/detections.test.ts b/x-pack/plugins/security_solution/server/usage/detections.test.ts new file mode 100644 index 0000000000000..7fd2d3eb9ff27 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections.test.ts @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { LegacyAPICaller } from '../../../../../src/core/server'; +import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; +import { jobServiceProvider } from '../../../ml/server/models/job_service'; +import { DataRecognizer } from '../../../ml/server/models/data_recognizer'; +import { mlServicesMock } from '../lib/machine_learning/mocks'; +import { + getMockJobSummaryResponse, + getMockListModulesResponse, + getMockRulesResponse, +} from './detections.mocks'; +import { fetchDetectionsUsage } from './detections'; + +jest.mock('../../../ml/server/models/job_service'); +jest.mock('../../../ml/server/models/data_recognizer'); + +describe('Detections Usage', () => { + describe('fetchDetectionsUsage()', () => { + let callClusterMock: jest.Mocked; + let mlMock: ReturnType; + + beforeEach(() => { + callClusterMock = elasticsearchServiceMock.createLegacyClusterClient().callAsInternalUser; + mlMock = mlServicesMock.create(); + }); + + it('returns zeroed counts if both calls are empty', async () => { + const result = await fetchDetectionsUsage('', callClusterMock, mlMock); + + expect(result).toEqual({ + detection_rules: { + custom: { + enabled: 0, + disabled: 0, + }, + elastic: { + enabled: 0, + disabled: 0, + }, + }, + ml_jobs: { + custom: { + enabled: 0, + disabled: 0, + }, + elastic: { + enabled: 0, + disabled: 0, + }, + }, + }); + }); + + it('tallies rules data given rules results', async () => { + (callClusterMock as jest.Mock).mockResolvedValue(getMockRulesResponse()); + const result = await fetchDetectionsUsage('', callClusterMock, mlMock); + + expect(result).toEqual( + expect.objectContaining({ + detection_rules: { + custom: { + enabled: 1, + disabled: 1, + }, + elastic: { + enabled: 2, + disabled: 3, + }, + }, + }) + ); + }); + + it('tallies jobs data given jobs results', async () => { + const mockJobSummary = jest.fn().mockResolvedValue(getMockJobSummaryResponse()); + const mockListModules = jest.fn().mockResolvedValue(getMockListModulesResponse()); + (jobServiceProvider as jest.Mock).mockImplementation(() => ({ + jobsSummary: mockJobSummary, + })); + (DataRecognizer as jest.Mock).mockImplementation(() => ({ + listModules: mockListModules, + })); + + const result = await fetchDetectionsUsage('', callClusterMock, mlMock); + + expect(result).toEqual( + expect.objectContaining({ + ml_jobs: { + custom: { + enabled: 1, + disabled: 1, + }, + elastic: { + enabled: 1, + disabled: 1, + }, + }, + }) + ); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/usage/detections.ts b/x-pack/plugins/security_solution/server/usage/detections.ts new file mode 100644 index 0000000000000..1475a8ae34625 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { LegacyAPICaller } from '../../../../../src/core/server'; +import { getMlJobsUsage, getRulesUsage } from './detections_helpers'; +import { MlPluginSetup } from '../../../ml/server'; + +interface FeatureUsage { + enabled: number; + disabled: number; +} + +export interface DetectionRulesUsage { + custom: FeatureUsage; + elastic: FeatureUsage; +} + +export interface MlJobsUsage { + custom: FeatureUsage; + elastic: FeatureUsage; +} + +export interface DetectionsUsage { + detection_rules: DetectionRulesUsage; + ml_jobs: MlJobsUsage; +} + +export const fetchDetectionsUsage = async ( + kibanaIndex: string, + callCluster: LegacyAPICaller, + ml: MlPluginSetup | undefined +): Promise => { + const rulesUsage = await getRulesUsage(kibanaIndex, callCluster); + const mlJobsUsage = await getMlJobsUsage(ml); + return { detection_rules: rulesUsage, ml_jobs: mlJobsUsage }; +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections_helpers.ts b/x-pack/plugins/security_solution/server/usage/detections_helpers.ts new file mode 100644 index 0000000000000..18a90b12991b2 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections_helpers.ts @@ -0,0 +1,188 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchParams } from 'elasticsearch'; + +import { LegacyAPICaller, SavedObjectsClient } from '../../../../../src/core/server'; +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { jobServiceProvider } from '../../../ml/server/models/job_service'; +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { DataRecognizer } from '../../../ml/server/models/data_recognizer'; +import { MlPluginSetup } from '../../../ml/server'; +import { SIGNALS_ID, INTERNAL_IMMUTABLE_KEY } from '../../common/constants'; +import { DetectionRulesUsage, MlJobsUsage } from './detections'; +import { isJobStarted } from '../../common/machine_learning/helpers'; + +interface DetectionsMetric { + isElastic: boolean; + isEnabled: boolean; +} + +const isElasticRule = (tags: string[]) => tags.includes(`${INTERNAL_IMMUTABLE_KEY}:true`); + +const initialRulesUsage: DetectionRulesUsage = { + custom: { + enabled: 0, + disabled: 0, + }, + elastic: { + enabled: 0, + disabled: 0, + }, +}; + +const initialMlJobsUsage: MlJobsUsage = { + custom: { + enabled: 0, + disabled: 0, + }, + elastic: { + enabled: 0, + disabled: 0, + }, +}; + +const updateRulesUsage = ( + ruleMetric: DetectionsMetric, + usage: DetectionRulesUsage +): DetectionRulesUsage => { + const { isEnabled, isElastic } = ruleMetric; + if (isEnabled && isElastic) { + return { + ...usage, + elastic: { + ...usage.elastic, + enabled: usage.elastic.enabled + 1, + }, + }; + } else if (!isEnabled && isElastic) { + return { + ...usage, + elastic: { + ...usage.elastic, + disabled: usage.elastic.disabled + 1, + }, + }; + } else if (isEnabled && !isElastic) { + return { + ...usage, + custom: { + ...usage.custom, + enabled: usage.custom.enabled + 1, + }, + }; + } else if (!isEnabled && !isElastic) { + return { + ...usage, + custom: { + ...usage.custom, + disabled: usage.custom.disabled + 1, + }, + }; + } else { + return usage; + } +}; + +const updateMlJobsUsage = (jobMetric: DetectionsMetric, usage: MlJobsUsage): MlJobsUsage => { + const { isEnabled, isElastic } = jobMetric; + if (isEnabled && isElastic) { + return { + ...usage, + elastic: { + ...usage.elastic, + enabled: usage.elastic.enabled + 1, + }, + }; + } else if (!isEnabled && isElastic) { + return { + ...usage, + elastic: { + ...usage.elastic, + disabled: usage.elastic.disabled + 1, + }, + }; + } else if (isEnabled && !isElastic) { + return { + ...usage, + custom: { + ...usage.custom, + enabled: usage.custom.enabled + 1, + }, + }; + } else if (!isEnabled && !isElastic) { + return { + ...usage, + custom: { + ...usage.custom, + disabled: usage.custom.disabled + 1, + }, + }; + } else { + return usage; + } +}; + +export const getRulesUsage = async ( + index: string, + callCluster: LegacyAPICaller +): Promise => { + let rulesUsage: DetectionRulesUsage = initialRulesUsage; + const ruleSearchOptions: SearchParams = { + body: { query: { bool: { filter: { term: { 'alert.alertTypeId': SIGNALS_ID } } } } }, + filterPath: ['hits.hits._source.alert.enabled', 'hits.hits._source.alert.tags'], + ignoreUnavailable: true, + index, + size: 10000, // elasticsearch index.max_result_window default value + }; + + try { + const ruleResults = await callCluster<{ alert: { enabled: boolean; tags: string[] } }>( + 'search', + ruleSearchOptions + ); + + if (ruleResults.hits?.hits?.length > 0) { + rulesUsage = ruleResults.hits.hits.reduce((usage, hit) => { + const isElastic = isElasticRule(hit._source.alert.tags); + const isEnabled = hit._source.alert.enabled; + + return updateRulesUsage({ isElastic, isEnabled }, usage); + }, initialRulesUsage); + } + } catch (e) { + // ignore failure, usage will be zeroed + } + + return rulesUsage; +}; + +export const getMlJobsUsage = async (ml: MlPluginSetup | undefined): Promise => { + let jobsUsage: MlJobsUsage = initialMlJobsUsage; + + if (ml) { + try { + const mlCaller = ml.mlClient.callAsInternalUser; + const modules = await new DataRecognizer( + mlCaller, + ({} as unknown) as SavedObjectsClient + ).listModules(); + const moduleJobs = modules.flatMap((module) => module.jobs); + const jobs = await jobServiceProvider(mlCaller).jobsSummary(['siem']); + + jobsUsage = jobs.reduce((usage, job) => { + const isElastic = moduleJobs.some((moduleJob) => moduleJob.id === job.id); + const isEnabled = isJobStarted(job.jobState, job.datafeedState); + + return updateMlJobsUsage({ isElastic, isEnabled }, usage); + }, initialMlJobsUsage); + } catch (e) { + // ignore failure, usage will be zeroed + } + } + + return jobsUsage; +}; diff --git a/x-pack/plugins/security_solution/server/usage/index.ts b/x-pack/plugins/security_solution/server/usage/index.ts new file mode 100644 index 0000000000000..4d8749a83be80 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/index.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { CollectorDependencies } from './types'; +import { registerCollector } from './collector'; + +export type InitUsageCollectors = (deps: CollectorDependencies) => void; + +export const initUsageCollectors: InitUsageCollectors = (dependencies) => { + registerCollector(dependencies); +}; diff --git a/x-pack/plugins/security_solution/server/usage/types.ts b/x-pack/plugins/security_solution/server/usage/types.ts new file mode 100644 index 0000000000000..955a4eaf4be5a --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/types.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SetupPlugins } from '../plugin'; + +export type CollectorDependencies = { kibanaIndex: string } & Pick< + SetupPlugins, + 'ml' | 'usageCollection' +>; diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index fbef75b9aa9cc..44aa5d814bde2 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -127,6 +127,62 @@ } } }, + "security_solution": { + "properties": { + "detections": { + "properties": { + "detection_rules": { + "properties": { + "custom": { + "properties": { + "enabled": { + "type": "long" + }, + "disabled": { + "type": "long" + } + } + }, + "elastic": { + "properties": { + "enabled": { + "type": "long" + }, + "disabled": { + "type": "long" + } + } + } + } + }, + "ml_jobs": { + "properties": { + "custom": { + "properties": { + "enabled": { + "type": "long" + }, + "disabled": { + "type": "long" + } + } + }, + "elastic": { + "properties": { + "enabled": { + "type": "long" + }, + "disabled": { + "type": "long" + } + } + } + } + } + } + } + } + }, "spaces": { "properties": { "usesFeatureControls": {