improve deduping strategy

elastic · Jul 30, 2020 · 537aa12 · 537aa12
1 parent 4ac318b
commit 537aa12
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 21 deletions.
diff --git a/x-pack/plugins/security_solution/server/usage/endpoints/endpoint.mocks.ts b/x-pack/plugins/security_solution/server/usage/endpoints/endpoint.mocks.ts
@@ -15,6 +15,7 @@ import { FLEET_ENDPOINT_PACKAGE_CONSTANT } from './fleet_saved_objects';
 const testAgentId = 'testAgentId';
 const testConfigId = 'testConfigId';
 const testHostId = 'randoHostId';
+const testHostName = 'testDesktop';
 
 /** Mock OS Platform for endpoint telemetry */
 export const MockOSPlatform = 'somePlatform';
@@ -24,6 +25,8 @@ export const MockOSName = 'somePlatformName';
 export const MockOSVersion = '1';
 /** Mock OS Full Name for endpoint telemetry */
 export const MockOSFullName = 'somePlatformFullName';
+/** Mock OS Full Name for endpoint telemetry */
+export const MockOSKernel = 'popcorn';
 
 /**
  *
@@ -56,15 +59,16 @@ export const mockFleetObjectsResponse = (
             },
           },
           host: {
-            hostname: 'testDesktop',
-            name: 'testDesktop',
+            hostname: testHostName,
+            name: testHostName,
             id: testHostId,
           },
           os: {
             platform: MockOSPlatform,
             version: MockOSVersion,
             name: MockOSName,
             full: MockOSFullName,
+            kernel: MockOSKernel,
           },
         },
         packages: [FLEET_ENDPOINT_PACKAGE_CONSTANT, 'system'],
@@ -93,15 +97,16 @@ export const mockFleetObjectsResponse = (
             },
           },
           host: {
-            hostname: 'testDesktop',
-            name: 'testDesktop',
+            hostname: hasDuplicates ? testHostName : 'oldRandoHostName',
+            name: hasDuplicates ? testHostName : 'oldRandoHostName',
             id: hasDuplicates ? testHostId : 'oldRandoHostId',
           },
           os: {
             platform: MockOSPlatform,
             version: MockOSVersion,
             name: MockOSName,
             full: MockOSFullName,
+            kernel: hasDuplicates ? MockOSKernel : 'unpopped-popocorn',
           },
         },
         packages: [FLEET_ENDPOINT_PACKAGE_CONSTANT, 'system'],

diff --git a/x-pack/plugins/security_solution/server/usage/endpoints/fleet_saved_objects.ts b/x-pack/plugins/security_solution/server/usage/endpoints/fleet_saved_objects.ts
@@ -23,6 +23,8 @@ export const getFleetSavedObjectsMetadata = async (savedObjectsClient: ISavedObj
       'last_checkin',
       'local_metadata.agent.id',
       'local_metadata.host.id',
+      'local_metadata.host.name',
+      'local_metadata.host.hostname',
       'local_metadata.elastic.agent.id',
       'local_metadata.os',
     ],

diff --git a/x-pack/plugins/security_solution/server/usage/endpoints/index.ts b/x-pack/plugins/security_solution/server/usage/endpoints/index.ts
@@ -42,11 +42,14 @@ export interface AgentLocalMetadata extends AgentMetadata {
     };
   };
   host: {
+    hostname: string;
     id: string;
+    name: string;
   };
   os: {
     name: string;
     platform: string;
+    kernel: string;
     version: string;
     full: string;
   };
@@ -78,17 +81,20 @@ export const updateEndpointOSTelemetry = (
   os: AgentLocalMetadata['os'],
   osTracker: OSTracker
 ): OSTracker => {
-  const updatedOSTracker = cloneDeep(osTracker);
-  const { version: osVersion, platform: osPlatform, full: osFullName } = os;
-  if (osFullName && osVersion) {
-    if (updatedOSTracker[osFullName]) updatedOSTracker[osFullName].count += 1;
-    else {
-      updatedOSTracker[osFullName] = {
-        full_name: osFullName,
-        platform: osPlatform,
-        version: osVersion,
-        count: 1,
-      };
+  let updatedOSTracker = osTracker;
+  if (os && typeof os === 'object') {
+    updatedOSTracker = cloneDeep(osTracker);
+    const { version: osVersion, platform: osPlatform, full: osFullName } = os;
+    if (osFullName && osVersion) {
+      if (updatedOSTracker[osFullName]) updatedOSTracker[osFullName].count += 1;
+      else {
+        updatedOSTracker[osFullName] = {
+          full_name: osFullName,
+          platform: osPlatform,
+          version: osVersion,
+          count: 1,
+        };
+      }
     }
   }
 
@@ -211,7 +217,7 @@ export const getEndpointTelemetryFromFleet = async (
   if (!endpointAgents || endpointAgentsCount < 1) return endpointTelemetry;
 
   // Use unique hosts to prevent any potential duplicates
-  const uniqueHostIds: Set<string> = new Set();
+  const uniqueHosts: Set<string> = new Set();
   let osTracker: OSTracker = {};
   let dailyActiveCount = 0;
   let policyTracker: PoliciesTelemetry = { malware: { active: 0, inactive: 0, failure: 0 } };
@@ -222,8 +228,12 @@ export const getEndpointTelemetryFromFleet = async (
       const { last_checkin: lastCheckin, local_metadata: localMetadata } = metadataAttributes;
       const { host, os, elastic } = localMetadata as AgentLocalMetadata;
 
-      if (!uniqueHostIds.has(host.id)) {
-        uniqueHostIds.add(host.id);
+      // Although not perfect, the goal is to dedupe hosts to get the most recent data for a host
+      // An agent re-installed on the same host will have all the same id, name, and kernel details
+      // A cloned VM will have the same id, but "may" have the same name and kernel, but it's really up to the user.
+      const compoundUniqueId = `${host?.id}-${host?.hostname}-${os?.kernel}`;
+      if (!uniqueHosts.has(compoundUniqueId)) {
+        uniqueHosts.add(compoundUniqueId);
         const agentId = elastic?.agent?.id;
         osTracker = updateEndpointOSTelemetry(os, osTracker);
 
@@ -244,18 +254,16 @@ export const getEndpointTelemetryFromFleet = async (
             policyTracker = updateEndpointPolicyTelemetry(latestEndpointEvent, policyTracker);
           }
         }
-        throw new Error('I broke!');
       }
     } catch (error) {
-      console.log("ERROR: ", error); // eslint-disable-line
       // All errors thrown in the loop would be handled here
       // Not logging any errors to avoid leaking any potential PII
       // Depending on when the error is thrown in the loop some specifics may be missing, but it allows the loop to continue
     }
   }
 
   // All unique hosts with an endpoint installed, thus all unique endpoint installs
-  endpointTelemetry.total_installed = uniqueHostIds.size;
+  endpointTelemetry.total_installed = uniqueHosts.size;
   // Set the daily active count for the endpoints
   endpointTelemetry.active_within_last_24_hours = dailyActiveCount;
   // Get the objects to populate our OS Telemetry