From 7b8bf0af8e84fe434b2e20f094f23e5b09d8a4ff Mon Sep 17 00:00:00 2001 From: Michael Olorunnisola Date: Tue, 16 Jun 2020 16:15:22 -0400 Subject: [PATCH 1/3] endpoint adoption telemetry --- .../server/lib/telemetry/endpoint.mocks.ts | 131 ++++++++++++++ .../server/lib/telemetry/endpoint.test.ts | 116 +++++++++++++ .../server/lib/telemetry/endpoint.ts | 161 ++++++++++++++++++ .../lib/telemetry/fleet_saved_objects.ts | 37 ++++ .../server/lib/telemetry/index.ts | 52 ++++++ .../security_solution/server/plugin.ts | 11 ++ .../schema/xpack_plugins.json | 43 +++++ 7 files changed, 551 insertions(+) create mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/endpoint.mocks.ts create mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/endpoint.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/endpoint.ts create mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/fleet_saved_objects.ts create mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/index.ts diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/endpoint.mocks.ts b/x-pack/plugins/security_solution/server/lib/telemetry/endpoint.mocks.ts new file mode 100644 index 0000000000000..65b7540fb5582 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/endpoint.mocks.ts @@ -0,0 +1,131 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { SavedObjectsFindResponse } from 'src/core/server'; +import { AgentEventSOAttributes } from './../../../../ingest_manager/common/types/models/agent'; +import { + AGENT_SAVED_OBJECT_TYPE, + AGENT_EVENT_SAVED_OBJECT_TYPE, +} from './../../../../ingest_manager/common/constants/agent'; +import { Agent } from '../../../../ingest_manager/common'; +import { FLEET_ENDPOINT_PACKAGE_CONSTANT } from './fleet_saved_objects'; + +const testAgentId = 'testAgentId'; +const testConfigId = 'testConfigId'; + +/** Mock OS Platform for endpoint telemetry */ +export const MockOSPlatform = 'somePlatform'; +/** Mock OS Name for endpoint telemetry */ +export const MockOSName = 'somePlatformName'; +/** Mock OS Version for endpoint telemetry */ +export const MockOSVersion = '1'; +/** Mock OS Full Name for endpoint telemetry */ +export const MockOSFullName = 'somePlatformFullName'; + +/** + * + * @param lastCheckIn - the last time the agent checked in. Defaults to current ISO time. + * @description We request the install and OS related telemetry information from the 'fleet-agents' saved objects in ingest_manager. This mocks that response + */ +export const mockFleetObjectsResponse = ( + lastCheckIn = new Date().toISOString() +): SavedObjectsFindResponse => ({ + page: 1, + per_page: 20, + total: 1, + saved_objects: [ + { + type: AGENT_SAVED_OBJECT_TYPE, + id: testAgentId, + attributes: { + active: true, + id: testAgentId, + config_id: 'randoConfigId', + type: 'PERMANENT', + user_provided_metadata: {}, + enrolled_at: lastCheckIn, + current_error_events: [], + local_metadata: { + elastic: { + agent: { + id: testAgentId, + }, + }, + host: { + hostname: 'testDesktop', + name: 'testDesktop', + id: 'randoHostId', + }, + os: { + platform: MockOSPlatform, + version: MockOSVersion, + name: MockOSName, + full: MockOSFullName, + }, + }, + packages: [FLEET_ENDPOINT_PACKAGE_CONSTANT, 'system'], + last_checkin: lastCheckIn, + }, + references: [], + updated_at: lastCheckIn, + version: 'WzI4MSwxXQ==', + score: 0, + }, + ], +}); + +/** + * + * @param running - allows us to set whether the mocked endpoint is in an active or disabled/failed state + * @param updatedDate - the last time the endpoint was updated. Defaults to current ISO time. + * @description We request the events triggered by the agent and get the most recent endpoint event to confirm it is still running. This allows us to mock both scenarios + */ +export const mockFleetEventsObjectsResponse = ( + running?: boolean, + updatedDate = new Date().toISOString() +): SavedObjectsFindResponse => { + return { + page: 1, + per_page: 20, + total: 2, + saved_objects: [ + { + type: AGENT_EVENT_SAVED_OBJECT_TYPE, + id: 'id1', + attributes: { + agent_id: testAgentId, + type: running ? 'STATE' : 'ERROR', + timestamp: updatedDate, + subtype: running ? 'RUNNING' : 'FAILED', + message: `Application: endpoint-security--8.0.0[d8f7f6e8-9375-483c-b456-b479f1d7a4f2]: State changed to ${ + running ? 'RUNNING' : 'FAILED' + }: `, + config_id: testConfigId, + }, + references: [], + updated_at: updatedDate, + version: 'WzExOCwxXQ==', + score: 0, + }, + { + type: AGENT_EVENT_SAVED_OBJECT_TYPE, + id: 'id2', + attributes: { + agent_id: testAgentId, + type: 'STATE', + timestamp: updatedDate, + subtype: 'STARTING', + message: + 'Application: endpoint-security--8.0.0[d8f7f6e8-9375-483c-b456-b479f1d7a4f2]: State changed to STARTING: Starting', + config_id: testConfigId, + }, + references: [], + updated_at: updatedDate, + version: 'WzExNywxXQ==', + score: 0, + }, + ], + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/endpoint.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/endpoint.test.ts new file mode 100644 index 0000000000000..02bc189919059 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/endpoint.test.ts @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { savedObjectsRepositoryMock } from 'src/core/server/mocks'; +import { + mockFleetObjectsResponse, + mockFleetEventsObjectsResponse, + MockOSFullName, + MockOSPlatform, + MockOSVersion, +} from './endpoint.mocks'; +import { ISavedObjectsRepository, SavedObjectsFindResponse } from 'src/core/server'; +import { AgentEventSOAttributes } from './../../../../ingest_manager/common/types/models/agent'; +import { Agent } from '../../../../ingest_manager/common'; +import * as endpointTelemetry from './endpoint'; +import * as fleetSavedObjects from './fleet_saved_objects'; + +describe('test security solution endpoint telemetry', () => { + let mockSavedObjectsRepository: jest.Mocked; + let getFleetSavedObjectsMetadataSpy: jest.SpyInstance>>; + let getFleetEventsSavedObjectsSpy: jest.SpyInstance + >>; + + beforeAll(() => { + getFleetEventsSavedObjectsSpy = jest.spyOn(fleetSavedObjects, 'getFleetEventsSavedObjects'); + getFleetSavedObjectsMetadataSpy = jest.spyOn(fleetSavedObjects, 'getFleetSavedObjectsMetadata'); + mockSavedObjectsRepository = savedObjectsRepositoryMock.create(); + }); + + afterAll(() => { + jest.resetAllMocks(); + }); + + it('should have a default shape', () => { + expect(endpointTelemetry.getDefaultEndpointTelemetry()).toMatchInlineSnapshot(` + Object { + "active_within_last_24_hours": 0, + "os": Array [], + "total_installed": 0, + } + `); + }); + + describe('when an agent has not been installed', () => { + it('should return the default shape if no agents are found', async () => { + getFleetSavedObjectsMetadataSpy.mockImplementation(() => + Promise.resolve({ saved_objects: [], total: 0, per_page: 0, page: 0 }) + ); + + const emptyEndpointTelemetryData = await endpointTelemetry.getEndpointTelemetryFromFleet( + mockSavedObjectsRepository + ); + expect(getFleetSavedObjectsMetadataSpy).toHaveBeenCalled(); + expect(emptyEndpointTelemetryData).toEqual({ + total_installed: 0, + active_within_last_24_hours: 0, + os: [], + }); + }); + }); + + describe('when an agent has been installed', () => { + it('should show one enpoint installed but it is inactive', async () => { + getFleetSavedObjectsMetadataSpy.mockImplementation(() => + Promise.resolve(mockFleetObjectsResponse()) + ); + getFleetEventsSavedObjectsSpy.mockImplementation(() => + Promise.resolve(mockFleetEventsObjectsResponse()) + ); + + const emptyEndpointTelemetryData = await endpointTelemetry.getEndpointTelemetryFromFleet( + mockSavedObjectsRepository + ); + expect(emptyEndpointTelemetryData).toEqual({ + total_installed: 1, + active_within_last_24_hours: 0, + os: [ + { + full_name: MockOSFullName, + platform: MockOSPlatform, + version: MockOSVersion, + count: 1, + }, + ], + }); + }); + + it('should show one endpoint installed and it is active', async () => { + getFleetSavedObjectsMetadataSpy.mockImplementation(() => + Promise.resolve(mockFleetObjectsResponse()) + ); + getFleetEventsSavedObjectsSpy.mockImplementation(() => + Promise.resolve(mockFleetEventsObjectsResponse(true)) + ); + + const emptyEndpointTelemetryData = await endpointTelemetry.getEndpointTelemetryFromFleet( + mockSavedObjectsRepository + ); + expect(emptyEndpointTelemetryData).toEqual({ + total_installed: 1, + active_within_last_24_hours: 1, + os: [ + { + full_name: MockOSFullName, + platform: MockOSPlatform, + version: MockOSVersion, + count: 1, + }, + ], + }); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/endpoint.ts b/x-pack/plugins/security_solution/server/lib/telemetry/endpoint.ts new file mode 100644 index 0000000000000..bf4082fddf59f --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/endpoint.ts @@ -0,0 +1,161 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ISavedObjectsRepository } from 'src/core/server'; +import { AgentMetadata } from '../../../../ingest_manager/common/types/models/agent'; +import { + getFleetSavedObjectsMetadata, + getFleetEventsSavedObjects, + FLEET_ENDPOINT_PACKAGE_CONSTANT, +} from './fleet_saved_objects'; + +export interface AgentOSMetadataTelemetry { + full_name: string; + platform: string; + version: string; + count: number; +} + +export interface PoliciesTelemetry { + malware: { + success: number; + warning: number; + failure: number; + }; +} + +export interface EndpointUsage { + total_installed: number; + active_within_last_24_hours: number; + os: AgentOSMetadataTelemetry[]; + policies?: PoliciesTelemetry; // TODO: make required when able to enable policy information +} + +export interface AgentLocalMetadata extends AgentMetadata { + elastic: { + agent: { + id: string; + }; + }; + host: { + id: string; + }; + os: { + name: string; + platform: string; + version: string; + full: string; + }; +} + +export type OSTracker = Record; +/** + * @description returns an empty telemetry object to be incrmented and updated within the `getEndpointTelemetryFromFleet` fn + */ +export const getDefaultEndpointTelemetry = (): EndpointUsage => ({ + total_installed: 0, + active_within_last_24_hours: 0, + os: [], +}); + +export const trackEndpointOSTelemetry = ( + os: AgentLocalMetadata['os'], + osTracker: OSTracker +): OSTracker => { + const updatedOSTracker = { ...osTracker }; + const { version: osVersion, platform: osPlatform, full: osFullName } = os; + if (osFullName && osVersion) { + if (updatedOSTracker[osFullName]) updatedOSTracker[osFullName].count += 1; + else { + updatedOSTracker[osFullName] = { + full_name: osFullName, + platform: osPlatform, + version: osVersion, + count: 1, + }; + } + } + + return updatedOSTracker; +}; + +/** + * @description This aggregates the telemetry details from the two fleet savedObject sources, `fleet-agents` and `fleet-agent-events` to populate + * the telemetry details for endpoint. Since we cannot access our own indices due to `kibana_system` not having access, this is the best alternative. + * Once the data is requested, we iterate over all agents with endpoints registered, and then request the events for each active agent (within last 24 hours) + * to confirm whether or not the endpoint is still active + */ +export const getEndpointTelemetryFromFleet = async ( + savedObjectsClient: ISavedObjectsRepository +) => { + // Retrieve every agent that references the endpoint as an installed package. It will not be listed if it was never installed + const { saved_objects: endpointAgents } = await getFleetSavedObjectsMetadata(savedObjectsClient); + const endpointTelemetry = getDefaultEndpointTelemetry(); + + // If there are no installed endpoints return the default telemetry object + if (!endpointAgents || endpointAgents.length < 1) return endpointTelemetry; + + // Use unique hosts to prevent any potential duplicates + const uniqueHostIds: Set = new Set(); + + // Need unique agents to get events data for those that have run in last 24 hours + const uniqueAgentIds: Set = new Set(); + + const aDayAgo = new Date(); + aDayAgo.setDate(aDayAgo.getDate() - 1); + + let osTracker: OSTracker = {}; + + const endpointMetadataTelemetry = endpointAgents.reduce( + (metadataTelemetry, { attributes: metadataAttributes }) => { + const { last_checkin: lastCheckin, local_metadata: localMetadata } = metadataAttributes; + // The extended AgentMetadata is just an empty blob, so cast to account for our specific use case + const { host, os, elastic } = localMetadata as AgentLocalMetadata; + + if (lastCheckin && new Date(lastCheckin) > aDayAgo) { + // Get agents that have checked in within the last 24 hours to later see if their endpoints are running + uniqueAgentIds.add(elastic.agent.id); + } + if (host && uniqueHostIds.has(host.id)) { + return metadataTelemetry; + } else { + uniqueHostIds.add(host.id); + osTracker = trackEndpointOSTelemetry(os, osTracker); + return metadataTelemetry; + } + }, + endpointTelemetry + ); + + // All agents in the unique host. + endpointTelemetry.total_installed = uniqueHostIds.size; + + // Get the objects to populate our OS Telemetry + endpointMetadataTelemetry.os = Object.values(osTracker); + + // Check for agents running in the last 24 hours whose endpoints are still active + for (const agentId of uniqueAgentIds) { + const { saved_objects: agentEvents } = await getFleetEventsSavedObjects( + savedObjectsClient, + agentId + ); + const lastEndpointStatus = agentEvents.find((agentEvent) => + agentEvent.attributes.message.includes(FLEET_ENDPOINT_PACKAGE_CONSTANT) + ); + + /* + We can assume that if the last status of the endpoint is RUNNING and the agent has checked in within the last 24 hours + then the endpoint has still been running within the last 24 hours. If / when we get the policy response, then we can use that + instead + */ + const endpointIsActive = lastEndpointStatus?.attributes.subtype === 'RUNNING'; + if (endpointIsActive) { + endpointMetadataTelemetry.active_within_last_24_hours += 1; + } + } + + return endpointMetadataTelemetry; +}; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/fleet_saved_objects.ts b/x-pack/plugins/security_solution/server/lib/telemetry/fleet_saved_objects.ts new file mode 100644 index 0000000000000..70657ed9f08f7 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/fleet_saved_objects.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ISavedObjectsRepository } from 'src/core/server'; +import { AgentEventSOAttributes } from './../../../../ingest_manager/common/types/models/agent'; +import { + AGENT_SAVED_OBJECT_TYPE, + AGENT_EVENT_SAVED_OBJECT_TYPE, +} from './../../../../ingest_manager/common/constants/agent'; +import { Agent, DefaultPackages as FleetDefaultPackages } from '../../../../ingest_manager/common'; + +export const FLEET_ENDPOINT_PACKAGE_CONSTANT = FleetDefaultPackages.endpoint; + +export const getFleetSavedObjectsMetadata = async (savedObjectsClient: ISavedObjectsRepository) => + savedObjectsClient.find({ + type: AGENT_SAVED_OBJECT_TYPE, + fields: ['packages', 'last_checkin', 'local_metadata'], + filter: `${AGENT_SAVED_OBJECT_TYPE}.attributes.packages: ${FLEET_ENDPOINT_PACKAGE_CONSTANT}`, + sortField: 'enrolled_at', + sortOrder: 'desc', + }); + +export const getFleetEventsSavedObjects = async ( + savedObjectsClient: ISavedObjectsRepository, + agentId: string +) => + savedObjectsClient.find({ + type: AGENT_EVENT_SAVED_OBJECT_TYPE, + filter: `${AGENT_EVENT_SAVED_OBJECT_TYPE}.attributes.agent_id: ${agentId} and ${AGENT_EVENT_SAVED_OBJECT_TYPE}.attributes.message: "${FLEET_ENDPOINT_PACKAGE_CONSTANT}"`, + sortField: 'timestamp', + sortOrder: 'desc', + search: agentId, + searchFields: ['agent_id'], + }); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/index.ts b/x-pack/plugins/security_solution/server/lib/telemetry/index.ts new file mode 100644 index 0000000000000..58b51b32bd916 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/index.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; +import { ISavedObjectsRepository } from '../../../../../../src/core/server'; +import { getEndpointTelemetryFromFleet, EndpointUsage } from './endpoint'; + +export interface UsageData { + endpoint?: EndpointUsage; +} + +export async function createSiemTelemetry( + usageCollector: UsageCollectionSetup, + savedObjectsClient: ISavedObjectsRepository +) { + const collector = usageCollector.makeUsageCollector({ + type: 'security_solution', + isReady: () => true, + fetch: async () => { + try { + return { + endpoint: await getEndpointTelemetryFromFleet(savedObjectsClient), + }; + } catch (err) { + return {}; + } + }, + schema: { + endpoint: { + total_installed: { type: 'long' }, + active_within_last_24_hours: { type: 'long' }, + os: { + full_name: { type: 'keyword' }, + platform: { type: 'keyword' }, + version: { type: 'keyword' }, + count: { type: 'long' }, + }, + policies: { + malware: { + success: { type: 'long' }, + warning: { type: 'long' }, + failure: { type: 'long' }, + }, + }, + }, + }, + }); + + usageCollector.registerCollector(collector); +} diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index ebd95fe79ebf5..07e5b283823eb 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -48,6 +48,7 @@ import { EndpointAppContextService } from './endpoint/endpoint_app_context_servi import { EndpointAppContext } from './endpoint/types'; import { registerDownloadExceptionListRoute } from './endpoint/routes/artifacts'; import { initUsageCollectors } from './usage'; +import { createSiemTelemetry } from './lib/telemetry'; export interface SetupPlugins { alerts: AlertingSetup; @@ -59,6 +60,8 @@ export interface SetupPlugins { security?: SecuritySetup; spaces?: SpacesSetup; taskManager?: TaskManagerSetupContract; + ml?: MlSetup; + lists?: ListPluginSetup; usageCollection?: UsageCollectionSetup; } @@ -248,6 +251,14 @@ export class Plugin implements IPlugin { + const savedObjectsClient = savedObjects.createInternalRepository(); + createSiemTelemetry(usageCollection, savedObjectsClient); + }); + } + const libs = compose(core, plugins, this.context.env.mode.prod); initServer(libs); diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index c5d528cbcce23..25a887799f80a 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -217,6 +217,49 @@ } } } + }, + "endpoint": { + "properties": { + "total_installed": { + "type": "long" + }, + "active_within_last_24_hours": { + "type": "long" + }, + "os": { + "properties": { + "full_name": { + "type": "keyword" + }, + "platform": { + "type": "keyword" + }, + "version": { + "type": "keyword" + }, + "count": { + "type": "long" + } + } + }, + "policies": { + "properties": { + "malware": { + "properties": { + "success": { + "type": "long" + }, + "warning": { + "type": "long" + }, + "failure": { + "type": "long" + } + } + } + } + } + } } } }, From 65ae3f99e0d764e4057be0d85e0b1a1946fab7cd Mon Sep 17 00:00:00 2001 From: Michael Olorunnisola Date: Mon, 13 Jul 2020 17:09:48 -0400 Subject: [PATCH 2/3] unify endpoints and detections telemetry --- .../server/lib/telemetry/index.ts | 52 ------------------- .../security_solution/server/plugin.ts | 12 +---- .../server/usage/collector.ts | 45 +++++++++++++--- .../{ => detections}/detections.mocks.ts | 2 +- .../usage/{ => detections}/detections.test.ts | 16 +++--- .../{ => detections}/detections_helpers.ts | 14 ++--- .../{detections.ts => detections/index.ts} | 4 +- .../endpoints}/endpoint.mocks.ts | 2 +- .../endpoints}/endpoint.test.ts | 4 +- .../endpoints}/fleet_saved_objects.ts | 0 .../endpoint.ts => usage/endpoints/index.ts} | 6 +-- .../security_solution/server/usage/types.ts | 3 +- .../schema/xpack_plugins.json | 2 +- 13 files changed, 66 insertions(+), 96 deletions(-) delete mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/index.ts rename x-pack/plugins/security_solution/server/usage/{ => detections}/detections.mocks.ts (98%) rename x-pack/plugins/security_solution/server/usage/{ => detections}/detections.test.ts (83%) rename x-pack/plugins/security_solution/server/usage/{ => detections}/detections_helpers.ts (91%) rename x-pack/plugins/security_solution/server/usage/{detections.ts => detections/index.ts} (89%) rename x-pack/plugins/security_solution/server/{lib/telemetry => usage/endpoints}/endpoint.mocks.ts (98%) rename x-pack/plugins/security_solution/server/{lib/telemetry => usage/endpoints}/endpoint.test.ts (96%) rename x-pack/plugins/security_solution/server/{lib/telemetry => usage/endpoints}/fleet_saved_objects.ts (100%) rename x-pack/plugins/security_solution/server/{lib/telemetry/endpoint.ts => usage/endpoints/index.ts} (96%) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/index.ts b/x-pack/plugins/security_solution/server/lib/telemetry/index.ts deleted file mode 100644 index 58b51b32bd916..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/telemetry/index.ts +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; -import { ISavedObjectsRepository } from '../../../../../../src/core/server'; -import { getEndpointTelemetryFromFleet, EndpointUsage } from './endpoint'; - -export interface UsageData { - endpoint?: EndpointUsage; -} - -export async function createSiemTelemetry( - usageCollector: UsageCollectionSetup, - savedObjectsClient: ISavedObjectsRepository -) { - const collector = usageCollector.makeUsageCollector({ - type: 'security_solution', - isReady: () => true, - fetch: async () => { - try { - return { - endpoint: await getEndpointTelemetryFromFleet(savedObjectsClient), - }; - } catch (err) { - return {}; - } - }, - schema: { - endpoint: { - total_installed: { type: 'long' }, - active_within_last_24_hours: { type: 'long' }, - os: { - full_name: { type: 'keyword' }, - platform: { type: 'keyword' }, - version: { type: 'keyword' }, - count: { type: 'long' }, - }, - policies: { - malware: { - success: { type: 'long' }, - warning: { type: 'long' }, - failure: { type: 'long' }, - }, - }, - }, - }, - }); - - usageCollector.registerCollector(collector); -} diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 07e5b283823eb..137c57f04367d 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -48,7 +48,6 @@ import { EndpointAppContextService } from './endpoint/endpoint_app_context_servi import { EndpointAppContext } from './endpoint/types'; import { registerDownloadExceptionListRoute } from './endpoint/routes/artifacts'; import { initUsageCollectors } from './usage'; -import { createSiemTelemetry } from './lib/telemetry'; export interface SetupPlugins { alerts: AlertingSetup; @@ -60,8 +59,6 @@ export interface SetupPlugins { security?: SecuritySetup; spaces?: SpacesSetup; taskManager?: TaskManagerSetupContract; - ml?: MlSetup; - lists?: ListPluginSetup; usageCollection?: UsageCollectionSetup; } @@ -117,6 +114,7 @@ export class Plugin implements IPlugin { - const savedObjectsClient = savedObjects.createInternalRepository(); - createSiemTelemetry(usageCollection, savedObjectsClient); - }); - } - const libs = compose(core, plugins, this.context.env.mode.prod); initServer(libs); diff --git a/x-pack/plugins/security_solution/server/usage/collector.ts b/x-pack/plugins/security_solution/server/usage/collector.ts index 71127f819ca79..ceefbc5e3acbd 100644 --- a/x-pack/plugins/security_solution/server/usage/collector.ts +++ b/x-pack/plugins/security_solution/server/usage/collector.ts @@ -4,20 +4,32 @@ * you may not use this file except in compliance with the Elastic License. */ -import { LegacyAPICaller } from '../../../../../src/core/server'; +import { LegacyAPICaller, CoreSetup } from '../../../../../src/core/server'; import { CollectorDependencies } from './types'; import { DetectionsUsage, fetchDetectionsUsage } from './detections'; +import { EndpointUsage, getEndpointTelemetryFromFleet } from './endpoints'; export type RegisterCollector = (deps: CollectorDependencies) => void; export interface UsageData { detections: DetectionsUsage; + endpoints: EndpointUsage; } -export const registerCollector: RegisterCollector = ({ kibanaIndex, ml, usageCollection }) => { +export async function getInternalSavedObjectsClient(core: CoreSetup) { + return core.getStartServices().then(async ([coreStart]) => { + return coreStart.savedObjects.createInternalRepository(); + }); +} + +export const registerCollector: RegisterCollector = ({ + core, + kibanaIndex, + ml, + usageCollection, +}) => { if (!usageCollection) { return; } - const collector = usageCollection.makeUsageCollector({ type: 'security_solution', schema: { @@ -43,11 +55,32 @@ export const registerCollector: RegisterCollector = ({ kibanaIndex, ml, usageCol }, }, }, + endpoints: { + total_installed: { type: 'long' }, + active_within_last_24_hours: { type: 'long' }, + os: { + full_name: { type: 'keyword' }, + platform: { type: 'keyword' }, + version: { type: 'keyword' }, + count: { type: 'long' }, + }, + policies: { + malware: { + success: { type: 'long' }, + warning: { type: 'long' }, + failure: { type: 'long' }, + }, + }, + }, }, isReady: () => kibanaIndex.length > 0, - fetch: async (callCluster: LegacyAPICaller): Promise => ({ - detections: await fetchDetectionsUsage(kibanaIndex, callCluster, ml), - }), + fetch: async (callCluster: LegacyAPICaller): Promise => { + const savedObjectsClient = await getInternalSavedObjectsClient(core); + return { + detections: await fetchDetectionsUsage(kibanaIndex, callCluster, ml), + endpoint: await getEndpointTelemetryFromFleet(savedObjectsClient), + }; + }, }); usageCollection.registerCollector(collector); diff --git a/x-pack/plugins/security_solution/server/usage/detections.mocks.ts b/x-pack/plugins/security_solution/server/usage/detections/detections.mocks.ts similarity index 98% rename from x-pack/plugins/security_solution/server/usage/detections.mocks.ts rename to x-pack/plugins/security_solution/server/usage/detections/detections.mocks.ts index c80dc6936ec7b..e59b1092978da 100644 --- a/x-pack/plugins/security_solution/server/usage/detections.mocks.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/detections.mocks.ts @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { INTERNAL_IMMUTABLE_KEY } from '../../common/constants'; +import { INTERNAL_IMMUTABLE_KEY } from '../../../common/constants'; export const getMockJobSummaryResponse = () => [ { diff --git a/x-pack/plugins/security_solution/server/usage/detections.test.ts b/x-pack/plugins/security_solution/server/usage/detections/detections.test.ts similarity index 83% rename from x-pack/plugins/security_solution/server/usage/detections.test.ts rename to x-pack/plugins/security_solution/server/usage/detections/detections.test.ts index 7fd2d3eb9ff27..0fc23f90a0ebf 100644 --- a/x-pack/plugins/security_solution/server/usage/detections.test.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/detections.test.ts @@ -4,20 +4,20 @@ * you may not use this file except in compliance with the Elastic License. */ -import { LegacyAPICaller } from '../../../../../src/core/server'; -import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; -import { jobServiceProvider } from '../../../ml/server/models/job_service'; -import { DataRecognizer } from '../../../ml/server/models/data_recognizer'; -import { mlServicesMock } from '../lib/machine_learning/mocks'; +import { LegacyAPICaller } from '../../../../../../src/core/server'; +import { elasticsearchServiceMock } from '../../../../../../src/core/server/mocks'; +import { jobServiceProvider } from '../../../../ml/server/models/job_service'; +import { DataRecognizer } from '../../../../ml/server/models/data_recognizer'; +import { mlServicesMock } from '../../lib/machine_learning/mocks'; import { getMockJobSummaryResponse, getMockListModulesResponse, getMockRulesResponse, } from './detections.mocks'; -import { fetchDetectionsUsage } from './detections'; +import { fetchDetectionsUsage } from './index'; -jest.mock('../../../ml/server/models/job_service'); -jest.mock('../../../ml/server/models/data_recognizer'); +jest.mock('../../../../ml/server/models/job_service'); +jest.mock('../../../../ml/server/models/data_recognizer'); describe('Detections Usage', () => { describe('fetchDetectionsUsage()', () => { diff --git a/x-pack/plugins/security_solution/server/usage/detections_helpers.ts b/x-pack/plugins/security_solution/server/usage/detections/detections_helpers.ts similarity index 91% rename from x-pack/plugins/security_solution/server/usage/detections_helpers.ts rename to x-pack/plugins/security_solution/server/usage/detections/detections_helpers.ts index 18a90b12991b2..3d04c24bab55a 100644 --- a/x-pack/plugins/security_solution/server/usage/detections_helpers.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/detections_helpers.ts @@ -6,15 +6,15 @@ import { SearchParams } from 'elasticsearch'; -import { LegacyAPICaller, SavedObjectsClient } from '../../../../../src/core/server'; +import { LegacyAPICaller, SavedObjectsClient } from '../../../../../../src/core/server'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths -import { jobServiceProvider } from '../../../ml/server/models/job_service'; +import { jobServiceProvider } from '../../../../ml/server/models/job_service'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths -import { DataRecognizer } from '../../../ml/server/models/data_recognizer'; -import { MlPluginSetup } from '../../../ml/server'; -import { SIGNALS_ID, INTERNAL_IMMUTABLE_KEY } from '../../common/constants'; -import { DetectionRulesUsage, MlJobsUsage } from './detections'; -import { isJobStarted } from '../../common/machine_learning/helpers'; +import { DataRecognizer } from '../../../../ml/server/models/data_recognizer'; +import { MlPluginSetup } from '../../../../ml/server'; +import { SIGNALS_ID, INTERNAL_IMMUTABLE_KEY } from '../../../common/constants'; +import { DetectionRulesUsage, MlJobsUsage } from './index'; +import { isJobStarted } from '../../../common/machine_learning/helpers'; interface DetectionsMetric { isElastic: boolean; diff --git a/x-pack/plugins/security_solution/server/usage/detections.ts b/x-pack/plugins/security_solution/server/usage/detections/index.ts similarity index 89% rename from x-pack/plugins/security_solution/server/usage/detections.ts rename to x-pack/plugins/security_solution/server/usage/detections/index.ts index 1475a8ae34625..dd50e79e22cc9 100644 --- a/x-pack/plugins/security_solution/server/usage/detections.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/index.ts @@ -4,9 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -import { LegacyAPICaller } from '../../../../../src/core/server'; +import { LegacyAPICaller } from '../../../../../../src/core/server'; import { getMlJobsUsage, getRulesUsage } from './detections_helpers'; -import { MlPluginSetup } from '../../../ml/server'; +import { MlPluginSetup } from '../../../../ml/server'; interface FeatureUsage { enabled: number; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/endpoint.mocks.ts b/x-pack/plugins/security_solution/server/usage/endpoints/endpoint.mocks.ts similarity index 98% rename from x-pack/plugins/security_solution/server/lib/telemetry/endpoint.mocks.ts rename to x-pack/plugins/security_solution/server/usage/endpoints/endpoint.mocks.ts index 65b7540fb5582..f41cfb773736d 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/endpoint.mocks.ts +++ b/x-pack/plugins/security_solution/server/usage/endpoints/endpoint.mocks.ts @@ -8,7 +8,7 @@ import { AgentEventSOAttributes } from './../../../../ingest_manager/common/type import { AGENT_SAVED_OBJECT_TYPE, AGENT_EVENT_SAVED_OBJECT_TYPE, -} from './../../../../ingest_manager/common/constants/agent'; +} from '../../../../ingest_manager/common/constants/agent'; import { Agent } from '../../../../ingest_manager/common'; import { FLEET_ENDPOINT_PACKAGE_CONSTANT } from './fleet_saved_objects'; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/endpoint.test.ts b/x-pack/plugins/security_solution/server/usage/endpoints/endpoint.test.ts similarity index 96% rename from x-pack/plugins/security_solution/server/lib/telemetry/endpoint.test.ts rename to x-pack/plugins/security_solution/server/usage/endpoints/endpoint.test.ts index 02bc189919059..0b2f4e4ed9dbe 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/endpoint.test.ts +++ b/x-pack/plugins/security_solution/server/usage/endpoints/endpoint.test.ts @@ -12,9 +12,9 @@ import { MockOSVersion, } from './endpoint.mocks'; import { ISavedObjectsRepository, SavedObjectsFindResponse } from 'src/core/server'; -import { AgentEventSOAttributes } from './../../../../ingest_manager/common/types/models/agent'; +import { AgentEventSOAttributes } from '../../../../ingest_manager/common/types/models/agent'; import { Agent } from '../../../../ingest_manager/common'; -import * as endpointTelemetry from './endpoint'; +import * as endpointTelemetry from './index'; import * as fleetSavedObjects from './fleet_saved_objects'; describe('test security solution endpoint telemetry', () => { diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/fleet_saved_objects.ts b/x-pack/plugins/security_solution/server/usage/endpoints/fleet_saved_objects.ts similarity index 100% rename from x-pack/plugins/security_solution/server/lib/telemetry/fleet_saved_objects.ts rename to x-pack/plugins/security_solution/server/usage/endpoints/fleet_saved_objects.ts diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/endpoint.ts b/x-pack/plugins/security_solution/server/usage/endpoints/index.ts similarity index 96% rename from x-pack/plugins/security_solution/server/lib/telemetry/endpoint.ts rename to x-pack/plugins/security_solution/server/usage/endpoints/index.ts index bf4082fddf59f..576d248613d1e 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/endpoint.ts +++ b/x-pack/plugins/security_solution/server/usage/endpoints/index.ts @@ -90,7 +90,7 @@ export const trackEndpointOSTelemetry = ( */ export const getEndpointTelemetryFromFleet = async ( savedObjectsClient: ISavedObjectsRepository -) => { +): Promise => { // Retrieve every agent that references the endpoint as an installed package. It will not be listed if it was never installed const { saved_objects: endpointAgents } = await getFleetSavedObjectsMetadata(savedObjectsClient); const endpointTelemetry = getDefaultEndpointTelemetry(); @@ -100,13 +100,11 @@ export const getEndpointTelemetryFromFleet = async ( // Use unique hosts to prevent any potential duplicates const uniqueHostIds: Set = new Set(); - // Need unique agents to get events data for those that have run in last 24 hours const uniqueAgentIds: Set = new Set(); const aDayAgo = new Date(); aDayAgo.setDate(aDayAgo.getDate() - 1); - let osTracker: OSTracker = {}; const endpointMetadataTelemetry = endpointAgents.reduce( @@ -130,7 +128,7 @@ export const getEndpointTelemetryFromFleet = async ( endpointTelemetry ); - // All agents in the unique host. + // All unique agents with an endpoint installed. You can technically install a new agent on a host, so relying on most recently installed. endpointTelemetry.total_installed = uniqueHostIds.size; // Get the objects to populate our OS Telemetry diff --git a/x-pack/plugins/security_solution/server/usage/types.ts b/x-pack/plugins/security_solution/server/usage/types.ts index 955a4eaf4be5a..9f8ebf80b65b5 100644 --- a/x-pack/plugins/security_solution/server/usage/types.ts +++ b/x-pack/plugins/security_solution/server/usage/types.ts @@ -4,9 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ +import { CoreSetup } from 'src/core/server'; import { SetupPlugins } from '../plugin'; -export type CollectorDependencies = { kibanaIndex: string } & Pick< +export type CollectorDependencies = { kibanaIndex: string; core: CoreSetup } & Pick< SetupPlugins, 'ml' | 'usageCollection' >; diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index 25a887799f80a..a7bc29f9efae2 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -218,7 +218,7 @@ } } }, - "endpoint": { + "endpoints": { "properties": { "total_installed": { "type": "long" From 8f47a7117d9ad97656d8e75940570ee487906b0c Mon Sep 17 00:00:00 2001 From: Michael Olorunnisola Date: Mon, 13 Jul 2020 18:33:38 -0400 Subject: [PATCH 3/3] fix type error --- x-pack/plugins/security_solution/server/usage/collector.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/usage/collector.ts b/x-pack/plugins/security_solution/server/usage/collector.ts index ceefbc5e3acbd..bb3583d50f8e5 100644 --- a/x-pack/plugins/security_solution/server/usage/collector.ts +++ b/x-pack/plugins/security_solution/server/usage/collector.ts @@ -78,7 +78,7 @@ export const registerCollector: RegisterCollector = ({ const savedObjectsClient = await getInternalSavedObjectsClient(core); return { detections: await fetchDetectionsUsage(kibanaIndex, callCluster, ml), - endpoint: await getEndpointTelemetryFromFleet(savedObjectsClient), + endpoints: await getEndpointTelemetryFromFleet(savedObjectsClient), }; }, });