Skip to content

Commit

Permalink
add policy details and update SO limit requests
Browse files Browse the repository at this point in the history
  • Loading branch information
@michaelolo24
michaelolo24 committed Jul 14, 2020
1 parent 54e09cd commit a9c2095
Show file tree
Hide file tree
Showing 4 changed files with 131 additions and 40 deletions.
42 changes: 42 additions & 0 deletions x-pack/plugins/security_solution/server/usage/endpoints/endpoint.mocks.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,47 @@ export const mockFleetObjectsResponse = (
],
});

const mockPolicyPayload = (malwareStatus: 'success' | 'warning' | 'failure') =>
JSON.stringify({
'endpoint-security': {
Endpoint: {
policy: {
applied: {
id: '0d466df0-c60f-11ea-a5c5-151665e785c4',
response: {
configurations: {
malware: {
concerned_actions: [
'load_config',
'workflow',
'download_global_artifacts',
'download_user_artifacts',
'configure_malware',
'read_malware_config',
'load_malware_model',
'read_kernel_config',
'configure_kernel',
'detect_process_events',
'detect_file_write_events',
'connect_kernel',
'detect_file_open_events',
'detect_sync_image_load_events',
],
status: `${malwareStatus}`,
},
},
},
status: `${malwareStatus}`,
},
},
},
agent: {
id: 'testAgentId',
version: '8.0.0-SNAPSHOT',
},
},
});

/**
*
* @param running - allows us to set whether the mocked endpoint is in an active or disabled/failed state
Expand All @@ -102,6 +143,7 @@ export const mockFleetEventsObjectsResponse = (
message: `Application: endpoint-security--8.0.0[d8f7f6e8-9375-483c-b456-b479f1d7a4f2]: State changed to ${
running ? 'RUNNING' : 'FAILED'
}: `,
payload: mockPolicyPayload(running ? 'success' : 'failure'),
config_id: testConfigId,
},
references: [],
Expand Down
36 changes: 32 additions & 4 deletions x-pack/plugins/security_solution/server/usage/endpoints/endpoint.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,12 @@ import * as fleetSavedObjects from './fleet_saved_objects';
describe('test security solution endpoint telemetry', () => {
let mockSavedObjectsRepository: jest.Mocked<ISavedObjectsRepository>;
let getFleetSavedObjectsMetadataSpy: jest.SpyInstance<Promise<SavedObjectsFindResponse<Agent>>>;
let getFleetEventsSavedObjectsSpy: jest.SpyInstance<Promise<
let getLatestFleetEndpointEventSpy: jest.SpyInstance<Promise<
SavedObjectsFindResponse<AgentEventSOAttributes>
>>;

beforeAll(() => {
getFleetEventsSavedObjectsSpy = jest.spyOn(fleetSavedObjects, 'getFleetEventsSavedObjects');
getLatestFleetEndpointEventSpy = jest.spyOn(fleetSavedObjects, 'getLatestFleetEndpointEvent');
getFleetSavedObjectsMetadataSpy = jest.spyOn(fleetSavedObjects, 'getFleetSavedObjectsMetadata');
mockSavedObjectsRepository = savedObjectsRepositoryMock.create();
});
Expand All @@ -39,6 +39,13 @@ describe('test security solution endpoint telemetry', () => {
Object {
"active_within_last_24_hours": 0,
"os": Array [],
"policies": Object {
"malware": Object {
"failure": 0,
"success": 0,
"warning": 0,
},
},
"total_installed": 0,
}
`);
Expand All @@ -58,6 +65,13 @@ describe('test security solution endpoint telemetry', () => {
total_installed: 0,
active_within_last_24_hours: 0,
os: [],
policies: {
malware: {
failure: 0,
success: 0,
warning: 0,
},
},
});
});
});
Expand All @@ -67,7 +81,7 @@ describe('test security solution endpoint telemetry', () => {
getFleetSavedObjectsMetadataSpy.mockImplementation(() =>
Promise.resolve(mockFleetObjectsResponse())
);
getFleetEventsSavedObjectsSpy.mockImplementation(() =>
getLatestFleetEndpointEventSpy.mockImplementation(() =>
Promise.resolve(mockFleetEventsObjectsResponse())
);

Expand All @@ -85,14 +99,21 @@ describe('test security solution endpoint telemetry', () => {
count: 1,
},
],
policies: {
malware: {
failure: 1,
success: 0,
warning: 0,
},
},
});
});

it('should show one endpoint installed and it is active', async () => {
getFleetSavedObjectsMetadataSpy.mockImplementation(() =>
Promise.resolve(mockFleetObjectsResponse())
);
getFleetEventsSavedObjectsSpy.mockImplementation(() =>
getLatestFleetEndpointEventSpy.mockImplementation(() =>
Promise.resolve(mockFleetEventsObjectsResponse(true))
);

Expand All @@ -110,6 +131,13 @@ describe('test security solution endpoint telemetry', () => {
count: 1,
},
],
policies: {
malware: {
failure: 0,
success: 1,
warning: 0,
},
},
});
});
});
Expand Down
4 changes: 3 additions & 1 deletion x-pack/plugins/security_solution/server/usage/endpoints/fleet_saved_objects.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,19 @@ export const getFleetSavedObjectsMetadata = async (savedObjectsClient: ISavedObj
type: AGENT_SAVED_OBJECT_TYPE,
fields: ['packages', 'last_checkin', 'local_metadata'],
filter: `${AGENT_SAVED_OBJECT_TYPE}.attributes.packages: ${FLEET_ENDPOINT_PACKAGE_CONSTANT}`,
perPage: 10000,
sortField: 'enrolled_at',
sortOrder: 'desc',
});

export const getFleetEventsSavedObjects = async (
export const getLatestFleetEndpointEvent = async (
savedObjectsClient: ISavedObjectsRepository,
agentId: string
) =>
savedObjectsClient.find<AgentEventSOAttributes>({
type: AGENT_EVENT_SAVED_OBJECT_TYPE,
filter: `${AGENT_EVENT_SAVED_OBJECT_TYPE}.attributes.agent_id: ${agentId} and ${AGENT_EVENT_SAVED_OBJECT_TYPE}.attributes.message: "${FLEET_ENDPOINT_PACKAGE_CONSTANT}"`,
perPage: 1, // Get the most recent endpoint event.
sortField: 'timestamp',
sortOrder: 'desc',
search: agentId,
Expand Down
89 changes: 54 additions & 35 deletions x-pack/plugins/security_solution/server/usage/endpoints/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,7 @@

import { ISavedObjectsRepository } from 'src/core/server';
import { AgentMetadata } from '../../../../ingest_manager/common/types/models/agent';
import {
getFleetSavedObjectsMetadata,
getFleetEventsSavedObjects,
FLEET_ENDPOINT_PACKAGE_CONSTANT,
} from './fleet_saved_objects';
import { getFleetSavedObjectsMetadata, getLatestFleetEndpointEvent } from './fleet_saved_objects';

export interface AgentOSMetadataTelemetry {
full_name: string;
Expand All @@ -19,19 +15,20 @@ export interface AgentOSMetadataTelemetry {
count: number;
}

export interface PoliciesTelemetry {
malware: {
success: number;
warning: number;
failure: number;
};
export type PolicyTypes = 'malware';
export interface PolicyTelemetry {
success: number;
warning: number;
failure: number;
}

export type PoliciesTelemetry = Record<PolicyTypes, PolicyTelemetry>;

export interface EndpointUsage {
total_installed: number;
active_within_last_24_hours: number;
os: AgentOSMetadataTelemetry[];
policies?: PoliciesTelemetry; // TODO: make required when able to enable policy information
policies: PoliciesTelemetry;
}

export interface AgentLocalMetadata extends AgentMetadata {
Expand All @@ -52,13 +49,21 @@ export interface AgentLocalMetadata extends AgentMetadata {
}

export type OSTracker = Record<string, AgentOSMetadataTelemetry>;

/**
* @description returns an empty telemetry object to be incrmented and updated within the `getEndpointTelemetryFromFleet` fn
*/
export const getDefaultEndpointTelemetry = (): EndpointUsage => ({
total_installed: 0,
active_within_last_24_hours: 0,
os: [],
policies: {
malware: {
success: 0,
warning: 0,
failure: 0,
},
},
});

export const trackEndpointOSTelemetry = (
Expand Down Expand Up @@ -100,8 +105,8 @@ export const getEndpointTelemetryFromFleet = async (

// Use unique hosts to prevent any potential duplicates
const uniqueHostIds: Set<string> = new Set();
// Need unique agents to get events data for those that have run in last 24 hours
const uniqueAgentIds: Set<string> = new Set();
// Need agents to get events data for those that have run in last 24 hours as well as policy details
const agentDailyActiveTracker: Map<string, boolean> = new Map();

const aDayAgo = new Date();
aDayAgo.setDate(aDayAgo.getDate() - 1);
Expand All @@ -110,17 +115,15 @@ export const getEndpointTelemetryFromFleet = async (
const endpointMetadataTelemetry = endpointAgents.reduce(
(metadataTelemetry, { attributes: metadataAttributes }) => {
const { last_checkin: lastCheckin, local_metadata: localMetadata } = metadataAttributes;
// The extended AgentMetadata is just an empty blob, so cast to account for our specific use case
const { host, os, elastic } = localMetadata as AgentLocalMetadata;
const { host, os, elastic } = localMetadata as AgentLocalMetadata; // AgentMetadata is just an empty blob, casting for our use case

if (lastCheckin && new Date(lastCheckin) > aDayAgo) {
// Get agents that have checked in within the last 24 hours to later see if their endpoints are running
uniqueAgentIds.add(elastic.agent.id);
}
if (host && uniqueHostIds.has(host.id)) {
// use hosts since new agents could potentially be re-installed on existing hosts
return metadataTelemetry;
} else {
uniqueHostIds.add(host.id);
const isActiveWithinLastDay = !!lastCheckin && new Date(lastCheckin) > aDayAgo;
agentDailyActiveTracker.set(elastic.agent.id, isActiveWithinLastDay);
osTracker = trackEndpointOSTelemetry(os, osTracker);
return metadataTelemetry;
}
Expand All @@ -130,28 +133,44 @@ export const getEndpointTelemetryFromFleet = async (

// All unique agents with an endpoint installed. You can technically install a new agent on a host, so relying on most recently installed.
endpointTelemetry.total_installed = uniqueHostIds.size;

// Get the objects to populate our OS Telemetry
endpointMetadataTelemetry.os = Object.values(osTracker);

// Check for agents running in the last 24 hours whose endpoints are still active
for (const agentId of uniqueAgentIds) {
const { saved_objects: agentEvents } = await getFleetEventsSavedObjects(
for (const agentId of agentDailyActiveTracker.keys()) {
const { saved_objects: agentEvents } = await getLatestFleetEndpointEvent(
savedObjectsClient,
agentId
);
const lastEndpointStatus = agentEvents.find((agentEvent) =>
agentEvent.attributes.message.includes(FLEET_ENDPOINT_PACKAGE_CONSTANT)
);

/*
We can assume that if the last status of the endpoint is RUNNING and the agent has checked in within the last 24 hours
then the endpoint has still been running within the last 24 hours. If / when we get the policy response, then we can use that
instead
*/
const endpointIsActive = lastEndpointStatus?.attributes.subtype === 'RUNNING';
if (endpointIsActive) {
endpointMetadataTelemetry.active_within_last_24_hours += 1;
const latestEndpointEvent = agentEvents[0];
if (latestEndpointEvent) {
/*
We can assume that if the last status of the endpoint is RUNNING and the agent has checked in within the last 24 hours
then the endpoint has still been running within the last 24 hours.
*/
const { subtype, payload } = latestEndpointEvent.attributes;
const endpointIsActive =
subtype === 'RUNNING' && agentDailyActiveTracker.get(agentId) === true;

if (endpointIsActive) {
endpointMetadataTelemetry.active_within_last_24_hours += 1;
}
const endpointPolicyDetails = payload ? JSON.parse(payload) : null;
if (endpointPolicyDetails) {
const malwareStatus =
endpointPolicyDetails['endpoint-security'].Endpoint?.policy?.applied?.response
?.configurations?.malware?.status;

if (malwareStatus === 'success') {
endpointMetadataTelemetry.policies.malware.success += 1;
}
if (malwareStatus === 'warning') {
endpointMetadataTelemetry.policies.malware.warning += 1;
}
if (malwareStatus === 'failure') {
endpointMetadataTelemetry.policies.malware.failure += 1;
}
}
}
}

Expand Down

0 comments on commit a9c2095

Please sign in to comment.