diff --git a/x-pack/plugins/security_solution/server/usage/collector.ts b/x-pack/plugins/security_solution/server/usage/collector.ts index bb3583d50f8e5..9740f57450e80 100644 --- a/x-pack/plugins/security_solution/server/usage/collector.ts +++ b/x-pack/plugins/security_solution/server/usage/collector.ts @@ -66,8 +66,8 @@ export const registerCollector: RegisterCollector = ({ }, policies: { malware: { - success: { type: 'long' }, - warning: { type: 'long' }, + active: { type: 'long' }, + inactive: { type: 'long' }, failure: { type: 'long' }, }, }, diff --git a/x-pack/plugins/security_solution/server/usage/endpoints/endpoint.mocks.ts b/x-pack/plugins/security_solution/server/usage/endpoints/endpoint.mocks.ts index f41cfb773736d..1369a3d398265 100644 --- a/x-pack/plugins/security_solution/server/usage/endpoints/endpoint.mocks.ts +++ b/x-pack/plugins/security_solution/server/usage/endpoints/endpoint.mocks.ts @@ -76,6 +76,108 @@ export const mockFleetObjectsResponse = ( ], }); +const mockPolicyPayload = (malwareStatus: 'success' | 'warning' | 'failure') => + JSON.stringify({ + 'endpoint-security': { + Endpoint: { + configuration: { + inputs: [ + { + id: '0d466df0-c60f-11ea-a5c5-151665e785c4', + policy: { + linux: { + events: { + file: true, + network: true, + process: true, + }, + logging: { + file: 'info', + }, + }, + mac: { + events: { + file: true, + network: true, + process: true, + }, + logging: { + file: 'info', + }, + malware: { + mode: 'prevent', + }, + }, + windows: { + events: { + dll_and_driver_load: true, + dns: true, + file: true, + network: true, + process: true, + registry: true, + security: true, + }, + logging: { + file: 'info', + }, + malware: { + mode: 'prevent', + }, + }, + }, + }, + ], + }, + policy: { + applied: { + id: '0d466df0-c60f-11ea-a5c5-151665e785c4', + response: { + configurations: { + malware: { + concerned_actions: [ + 'load_config', + 'workflow', + 'download_global_artifacts', + 'download_user_artifacts', + 'configure_malware', + 'read_malware_config', + 'load_malware_model', + 'read_kernel_config', + 'configure_kernel', + 'detect_process_events', + 'detect_file_write_events', + 'connect_kernel', + 'detect_file_open_events', + 'detect_sync_image_load_events', + ], + status: `${malwareStatus}`, + }, + }, + }, + status: `${malwareStatus}`, + }, + }, + }, + agent: { + id: 'testAgentId', + version: '8.0.0-SNAPSHOT', + }, + host: { + architecture: 'x86_64', + id: 'a4148b63-1758-ab1f-a6d3-f95075cb1a9c', + os: { + Ext: { + variant: 'Windows 10 Pro', + }, + full: 'Windows 10 Pro 2004 (10.0.19041.329)', + name: 'Windows', + version: '2004 (10.0.19041.329)', + }, + }, + }, + }); + /** * * @param running - allows us to set whether the mocked endpoint is in an active or disabled/failed state @@ -102,6 +204,7 @@ export const mockFleetEventsObjectsResponse = ( message: `Application: endpoint-security--8.0.0[d8f7f6e8-9375-483c-b456-b479f1d7a4f2]: State changed to ${ running ? 'RUNNING' : 'FAILED' }: `, + payload: mockPolicyPayload(running ? 'success' : 'failure'), config_id: testConfigId, }, references: [], diff --git a/x-pack/plugins/security_solution/server/usage/endpoints/endpoint.test.ts b/x-pack/plugins/security_solution/server/usage/endpoints/endpoint.test.ts index 0b2f4e4ed9dbe..06755192bd818 100644 --- a/x-pack/plugins/security_solution/server/usage/endpoints/endpoint.test.ts +++ b/x-pack/plugins/security_solution/server/usage/endpoints/endpoint.test.ts @@ -20,12 +20,12 @@ import * as fleetSavedObjects from './fleet_saved_objects'; describe('test security solution endpoint telemetry', () => { let mockSavedObjectsRepository: jest.Mocked; let getFleetSavedObjectsMetadataSpy: jest.SpyInstance>>; - let getFleetEventsSavedObjectsSpy: jest.SpyInstance >>; beforeAll(() => { - getFleetEventsSavedObjectsSpy = jest.spyOn(fleetSavedObjects, 'getFleetEventsSavedObjects'); + getLatestFleetEndpointEventSpy = jest.spyOn(fleetSavedObjects, 'getLatestFleetEndpointEvent'); getFleetSavedObjectsMetadataSpy = jest.spyOn(fleetSavedObjects, 'getFleetSavedObjectsMetadata'); mockSavedObjectsRepository = savedObjectsRepositoryMock.create(); }); @@ -39,6 +39,13 @@ describe('test security solution endpoint telemetry', () => { Object { "active_within_last_24_hours": 0, "os": Array [], + "policies": Object { + "malware": Object { + "active": 0, + "failure": 0, + "inactive": 0, + }, + }, "total_installed": 0, } `); @@ -58,6 +65,13 @@ describe('test security solution endpoint telemetry', () => { total_installed: 0, active_within_last_24_hours: 0, os: [], + policies: { + malware: { + failure: 0, + active: 0, + inactive: 0, + }, + }, }); }); }); @@ -67,7 +81,7 @@ describe('test security solution endpoint telemetry', () => { getFleetSavedObjectsMetadataSpy.mockImplementation(() => Promise.resolve(mockFleetObjectsResponse()) ); - getFleetEventsSavedObjectsSpy.mockImplementation(() => + getLatestFleetEndpointEventSpy.mockImplementation(() => Promise.resolve(mockFleetEventsObjectsResponse()) ); @@ -85,6 +99,13 @@ describe('test security solution endpoint telemetry', () => { count: 1, }, ], + policies: { + malware: { + failure: 1, + active: 0, + inactive: 0, + }, + }, }); }); @@ -92,7 +113,7 @@ describe('test security solution endpoint telemetry', () => { getFleetSavedObjectsMetadataSpy.mockImplementation(() => Promise.resolve(mockFleetObjectsResponse()) ); - getFleetEventsSavedObjectsSpy.mockImplementation(() => + getLatestFleetEndpointEventSpy.mockImplementation(() => Promise.resolve(mockFleetEventsObjectsResponse(true)) ); @@ -110,6 +131,13 @@ describe('test security solution endpoint telemetry', () => { count: 1, }, ], + policies: { + malware: { + failure: 0, + active: 1, + inactive: 0, + }, + }, }); }); }); diff --git a/x-pack/plugins/security_solution/server/usage/endpoints/fleet_saved_objects.ts b/x-pack/plugins/security_solution/server/usage/endpoints/fleet_saved_objects.ts index 70657ed9f08f7..7e05fdec36169 100644 --- a/x-pack/plugins/security_solution/server/usage/endpoints/fleet_saved_objects.ts +++ b/x-pack/plugins/security_solution/server/usage/endpoints/fleet_saved_objects.ts @@ -19,17 +19,19 @@ export const getFleetSavedObjectsMetadata = async (savedObjectsClient: ISavedObj type: AGENT_SAVED_OBJECT_TYPE, fields: ['packages', 'last_checkin', 'local_metadata'], filter: `${AGENT_SAVED_OBJECT_TYPE}.attributes.packages: ${FLEET_ENDPOINT_PACKAGE_CONSTANT}`, + perPage: 10000, sortField: 'enrolled_at', sortOrder: 'desc', }); -export const getFleetEventsSavedObjects = async ( +export const getLatestFleetEndpointEvent = async ( savedObjectsClient: ISavedObjectsRepository, agentId: string ) => savedObjectsClient.find({ type: AGENT_EVENT_SAVED_OBJECT_TYPE, filter: `${AGENT_EVENT_SAVED_OBJECT_TYPE}.attributes.agent_id: ${agentId} and ${AGENT_EVENT_SAVED_OBJECT_TYPE}.attributes.message: "${FLEET_ENDPOINT_PACKAGE_CONSTANT}"`, + perPage: 1, // Get the most recent endpoint event. sortField: 'timestamp', sortOrder: 'desc', search: agentId, diff --git a/x-pack/plugins/security_solution/server/usage/endpoints/index.ts b/x-pack/plugins/security_solution/server/usage/endpoints/index.ts index 576d248613d1e..ab5669d503275 100644 --- a/x-pack/plugins/security_solution/server/usage/endpoints/index.ts +++ b/x-pack/plugins/security_solution/server/usage/endpoints/index.ts @@ -6,11 +6,7 @@ import { ISavedObjectsRepository } from 'src/core/server'; import { AgentMetadata } from '../../../../ingest_manager/common/types/models/agent'; -import { - getFleetSavedObjectsMetadata, - getFleetEventsSavedObjects, - FLEET_ENDPOINT_PACKAGE_CONSTANT, -} from './fleet_saved_objects'; +import { getFleetSavedObjectsMetadata, getLatestFleetEndpointEvent } from './fleet_saved_objects'; export interface AgentOSMetadataTelemetry { full_name: string; @@ -18,22 +14,25 @@ export interface AgentOSMetadataTelemetry { version: string; count: number; } +export interface PolicyTelemetry { + active: number; + inactive: number; + failure: number; +} export interface PoliciesTelemetry { - malware: { - success: number; - warning: number; - failure: number; - }; + malware: PolicyTelemetry; } export interface EndpointUsage { total_installed: number; active_within_last_24_hours: number; os: AgentOSMetadataTelemetry[]; - policies?: PoliciesTelemetry; // TODO: make required when able to enable policy information + policies: PoliciesTelemetry; } +type EndpointOSNames = 'Linux' | 'Windows' | 'macOs'; + export interface AgentLocalMetadata extends AgentMetadata { elastic: { agent: { @@ -51,7 +50,8 @@ export interface AgentLocalMetadata extends AgentMetadata { }; } -export type OSTracker = Record; +type OSTracker = Record; +type AgentDailyActiveTracker = Map; /** * @description returns an empty telemetry object to be incrmented and updated within the `getEndpointTelemetryFromFleet` fn */ @@ -59,8 +59,18 @@ export const getDefaultEndpointTelemetry = (): EndpointUsage => ({ total_installed: 0, active_within_last_24_hours: 0, os: [], + policies: { + malware: { + active: 0, + inactive: 0, + failure: 0, + }, + }, }); +/** + * @description this fun + */ export const trackEndpointOSTelemetry = ( os: AgentLocalMetadata['os'], osTracker: OSTracker @@ -82,6 +92,80 @@ export const trackEndpointOSTelemetry = ( return updatedOSTracker; }; +/** + * @description This iterates over all unique agents that currently track an endpoint package. It takes a list of agents who have checked in in the last 24 hours + * and then checks whether those agents have endpoints whose latest status is 'RUNNING' to determine an active_within_last_24_hours. Since the policy information is also tracked in these events + * we pull out the status of the current protection (malware) type. This must be done in a compound manner as the desired status is reflected in the config, and the successful application of that policy + * is tracked in the policy.applied.response.configurations[protectionsType].status. Using these two we can determine whether the policy is toggled on, off, or failed to turn on. + */ +export const addEndpointDailyActivityAndPolicyDetailsToTelemetry = async ( + agentDailyActiveTracker: AgentDailyActiveTracker, + savedObjectsClient: ISavedObjectsRepository, + endpointTelemetry: EndpointUsage +): Promise => { + const updatedEndpointTelemetry = { ...endpointTelemetry }; + + const policyHostTypeToPolicyType = { + Linux: 'linux', + macOs: 'mac', + Windows: 'windows', + }; + const enabledMalwarePolicyTypes = ['prevent', 'detect']; + + for (const agentId of agentDailyActiveTracker.keys()) { + const { saved_objects: agentEvents } = await getLatestFleetEndpointEvent( + savedObjectsClient, + agentId + ); + + const latestEndpointEvent = agentEvents[0]; + if (latestEndpointEvent) { + /* + We can assume that if the last status of the endpoint is RUNNING and the agent has checked in within the last 24 hours + then the endpoint has still been running within the last 24 hours. + */ + const { subtype, payload } = latestEndpointEvent.attributes; + const endpointIsActive = + subtype === 'RUNNING' && agentDailyActiveTracker.get(agentId) === true; + + if (endpointIsActive) { + updatedEndpointTelemetry.active_within_last_24_hours += 1; + } + + // The policy details are sent as a string on the 'payload' attribute of the agent event + const endpointPolicyDetails = payload ? JSON.parse(payload) : null; + if (endpointPolicyDetails) { + // We get the setting the user desired to enable (treating prevent and detect as 'active' states) and then see if it succeded or failed. + const hostType = + policyHostTypeToPolicyType[ + endpointPolicyDetails['endpoint-security']?.host?.os?.name as EndpointOSNames + ]; + const userDesiredMalwareState = + endpointPolicyDetails['endpoint-security'].Endpoint?.configuration?.inputs[0]?.policy[ + hostType + ]?.malware?.mode; + + const isAnActiveMalwareState = enabledMalwarePolicyTypes.includes(userDesiredMalwareState); + const malwareStatus = + endpointPolicyDetails['endpoint-security'].Endpoint?.policy?.applied?.response + ?.configurations?.malware?.status; + + if (isAnActiveMalwareState && malwareStatus !== 'failure') { + updatedEndpointTelemetry.policies.malware.active += 1; + } + if (!isAnActiveMalwareState) { + updatedEndpointTelemetry.policies.malware.inactive += 1; + } + if (isAnActiveMalwareState && malwareStatus === 'failure') { + updatedEndpointTelemetry.policies.malware.failure += 1; + } + } + } + } + + return updatedEndpointTelemetry; +}; + /** * @description This aggregates the telemetry details from the two fleet savedObject sources, `fleet-agents` and `fleet-agent-events` to populate * the telemetry details for endpoint. Since we cannot access our own indices due to `kibana_system` not having access, this is the best alternative. @@ -100,8 +184,8 @@ export const getEndpointTelemetryFromFleet = async ( // Use unique hosts to prevent any potential duplicates const uniqueHostIds: Set = new Set(); - // Need unique agents to get events data for those that have run in last 24 hours - const uniqueAgentIds: Set = new Set(); + // Need agents to get events data for those that have run in last 24 hours as well as policy details + const agentDailyActiveTracker: AgentDailyActiveTracker = new Map(); const aDayAgo = new Date(); aDayAgo.setDate(aDayAgo.getDate() - 1); @@ -110,17 +194,15 @@ export const getEndpointTelemetryFromFleet = async ( const endpointMetadataTelemetry = endpointAgents.reduce( (metadataTelemetry, { attributes: metadataAttributes }) => { const { last_checkin: lastCheckin, local_metadata: localMetadata } = metadataAttributes; - // The extended AgentMetadata is just an empty blob, so cast to account for our specific use case - const { host, os, elastic } = localMetadata as AgentLocalMetadata; + const { host, os, elastic } = localMetadata as AgentLocalMetadata; // AgentMetadata is just an empty blob, casting for our use case - if (lastCheckin && new Date(lastCheckin) > aDayAgo) { - // Get agents that have checked in within the last 24 hours to later see if their endpoints are running - uniqueAgentIds.add(elastic.agent.id); - } if (host && uniqueHostIds.has(host.id)) { + // use hosts since new agents could potentially be re-installed on existing hosts return metadataTelemetry; } else { uniqueHostIds.add(host.id); + const isActiveWithinLastDay = !!lastCheckin && new Date(lastCheckin) > aDayAgo; + agentDailyActiveTracker.set(elastic.agent.id, isActiveWithinLastDay); osTracker = trackEndpointOSTelemetry(os, osTracker); return metadataTelemetry; } @@ -128,32 +210,16 @@ export const getEndpointTelemetryFromFleet = async ( endpointTelemetry ); - // All unique agents with an endpoint installed. You can technically install a new agent on a host, so relying on most recently installed. + // All unique hosts with an endpoint installed. endpointTelemetry.total_installed = uniqueHostIds.size; - // Get the objects to populate our OS Telemetry endpointMetadataTelemetry.os = Object.values(osTracker); + // Populate endpoint telemetry with the finalized 24 hour count and policy details + const finalizedEndpointTelemetryData = await addEndpointDailyActivityAndPolicyDetailsToTelemetry( + agentDailyActiveTracker, + savedObjectsClient, + endpointMetadataTelemetry + ); - // Check for agents running in the last 24 hours whose endpoints are still active - for (const agentId of uniqueAgentIds) { - const { saved_objects: agentEvents } = await getFleetEventsSavedObjects( - savedObjectsClient, - agentId - ); - const lastEndpointStatus = agentEvents.find((agentEvent) => - agentEvent.attributes.message.includes(FLEET_ENDPOINT_PACKAGE_CONSTANT) - ); - - /* - We can assume that if the last status of the endpoint is RUNNING and the agent has checked in within the last 24 hours - then the endpoint has still been running within the last 24 hours. If / when we get the policy response, then we can use that - instead - */ - const endpointIsActive = lastEndpointStatus?.attributes.subtype === 'RUNNING'; - if (endpointIsActive) { - endpointMetadataTelemetry.active_within_last_24_hours += 1; - } - } - - return endpointMetadataTelemetry; + return finalizedEndpointTelemetryData; }; diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index a7bc29f9efae2..fd21b70660bb6 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -246,10 +246,10 @@ "properties": { "malware": { "properties": { - "success": { + "active": { "type": "long" }, - "warning": { + "inactive": { "type": "long" }, "failure": {