Move to modified version of original status count for speed (#41210) (#…

…41325)

This attempts to speed up the now slow snapshot count @justinkambic noticed. It uses a different approach form #41203, it's actually a modified version of the old formula. It should be faster. It achieves this executing a query that's very friendly to composite aggs, even if it returns more data than strictly necessary.

This also fixes a bug in the original implementation, where the total would be equivalent to the filtered value, rather than the pre-filtered value. This made little sense since you'd just see the total twice.

x-pack/legacy/plugins/uptime/server/graphql/monitors/resolvers.ts

@@ -109,7 +109,7 @@ export const createMonitorsResolvers: CreateUMGraphQLResolvers = (
       { dateRangeStart, dateRangeEnd, filters },
       { req }
     ): Promise<Snapshot> {
-      const counts = await libs.monitorStates.getSummaryCount(
+      const counts = await libs.monitors.getSnapshotCount(
         req,
         dateRangeStart,
         dateRangeEnd,

x-pack/legacy/plugins/uptime/server/lib/adapters/monitor_states/adapter_types.ts

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { MonitorSummary, SnapshotCount, StatesIndexStatus } from '../../../../common/graphql/types';
+import { MonitorSummary, StatesIndexStatus } from '../../../../common/graphql/types';
 
 export interface UMMonitorStatesAdapter {
   getMonitorStates(
@@ -20,11 +20,5 @@ export interface UMMonitorStatesAdapter {
     dateRangeEnd: string,
     filters?: string | null
   ): Promise<MonitorSummary[]>;
-  getSummaryCount(
-    request: any,
-    dateRangeStart: string,
-    dateRangeEnd: string,
-    filters?: string | null
-  ): Promise<SnapshotCount>;
   statesIndexExists(request: any): Promise<StatesIndexStatus>;
 }

x-pack/legacy/plugins/uptime/server/lib/adapters/monitor_states/elasticsearch_monitor_states_adapter.ts

@@ -11,7 +11,6 @@ import {
   MonitorSummary,
   SummaryHistogram,
   Check,
-  SnapshotCount,
   StatesIndexStatus,
 } from '../../../../common/graphql/types';
 import { INDEX_NAMES, LEGACY_STATES_QUERY_SIZE } from '../../../../common/constants';
@@ -581,49 +580,6 @@ export class ElasticsearchMonitorStatesAdapter implements UMMonitorStatesAdapter
     }, {});
   }
 
-  public async getSummaryCount(
-    request: any,
-    dateRangeStart: string,
-    dateRangeEnd: string,
-    filters?: string | null
-  ): Promise<SnapshotCount> {
-    // TODO: adapt this to the states index in future release
-    // const { count } = await this.database.count(request, { index: 'heartbeat-states-8.0.0' });
-    // return { count };
-
-    const count: SnapshotCount = {
-      up: 0,
-      down: 0,
-      mixed: 0,
-      total: 0,
-    };
-
-    let searchAfter: any | null = null;
-    do {
-      const { afterKey, result, statusFilter } = await this.runLegacyMonitorStatesQuery(
-        request,
-        dateRangeStart,
-        dateRangeEnd,
-        filters,
-        searchAfter
-      );
-      searchAfter = afterKey;
-      this.getMonitorBuckets(result, statusFilter).reduce((acc: SnapshotCount, monitor: any) => {
-        const status = get<string | undefined>(monitor, 'state.value.monitor.status', undefined);
-        if (status === 'up') {
-          acc.up++;
-        } else if (status === 'down') {
-          acc.down++;
-        } else if (status === 'mixed') {
-          acc.mixed++;
-        }
-        acc.total++;
-        return acc;
-      }, count);
-    } while (searchAfter !== null);
-    return count;
-  }
-
   public async statesIndexExists(request: any): Promise<StatesIndexStatus> {
     // TODO: adapt this to the states index in future release
     const {

x-pack/legacy/plugins/uptime/server/lib/adapters/monitor_states/memory_monitor_states_adapter.ts

@@ -5,7 +5,7 @@
  */
 
 import { UMMonitorStatesAdapter } from './adapter_types';
-import { MonitorSummary, SnapshotCount, StatesIndexStatus } from '../../../../common/graphql/types';
+import { MonitorSummary, StatesIndexStatus } from '../../../../common/graphql/types';
 
 /**
  * This class will be implemented for server-side tests.
@@ -28,14 +28,6 @@ export class UMMemoryMonitorStatesAdapter implements UMMonitorStatesAdapter {
   ): Promise<MonitorSummary[]> {
     throw new Error('Method not implemented.');
   }
-  public async getSummaryCount(
-    request: any,
-    dateRangeStart: string,
-    dateRangeEnd: string,
-    filters?: string | null | undefined
-  ): Promise<SnapshotCount> {
-    throw new Error('Method not implemented.');
-  }
   public async statesIndexExists(request: any): Promise<StatesIndexStatus> {
     throw new Error('Method not implemented.');
   }

x-pack/legacy/plugins/uptime/server/lib/adapters/monitors/elasticsearch_monitors_adapter.ts

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { get, set } from 'lodash';
+import { get, set, reduce } from 'lodash';
 import { INDEX_NAMES } from '../../../../common/constants';
 import {
   ErrorListItem,
@@ -197,11 +197,10 @@ export class ElasticsearchMonitorsAdapter implements UMMonitorsAdapter {
             aggs: {
               latest: {
                 top_hits: {
-                  sort: [
-                    {
-                      '@timestamp': { order: 'desc' },
-                    },
-                  ],
+                  sort: [{ '@timestamp': { order: 'desc' } }],
+                  _source: {
+                    includes: ['summary.*', 'monitor.id', '@timestamp', 'observer.geo.name'],
+                  },
                   size: 1,
                 },
               },
@@ -211,10 +210,16 @@ export class ElasticsearchMonitorsAdapter implements UMMonitorsAdapter {
       },
     };
 
-    let up: number = 0;
-    let down: number = 0;
     let searchAfter: any = null;
 
+    const summaryByIdLocation: {
+      // ID
+      [key: string]: {
+        // Location
+        [key: string]: { up: number; down: number; timestamp: number };
+      };
+    } = {};
+
     do {
       if (searchAfter) {
         set(params, 'body.aggs.ids.composite.after', searchAfter);
@@ -225,20 +230,66 @@ export class ElasticsearchMonitorsAdapter implements UMMonitorsAdapter {
 
       idBuckets.forEach(bucket => {
         // We only get the latest doc
-        const status = get(bucket, 'latest.hits.hits[0]._source.monitor.status', null);
-        if (!statusFilter || (statusFilter && statusFilter === status)) {
-          if (status === 'up') {
-            up++;
-          } else {
-            down++;
-          }
+        const source: any = get(bucket, 'latest.hits.hits[0]._source');
+        const {
+          summary: { up, down },
+          monitor: { id },
+        } = source;
+        const timestamp = get(source, '@timestamp', 0);
+        const location = get(source, 'observer.geo.name', '');
+
+        let idSummary = summaryByIdLocation[id];
+        if (!idSummary) {
+          idSummary = {};
+          summaryByIdLocation[id] = idSummary;
+        }
+        const locationSummary = idSummary[location];
+        if (!locationSummary || locationSummary.timestamp < timestamp) {
+          idSummary[location] = { timestamp, up, down };
         }
       });
 
       searchAfter = get(queryResult, 'aggregations.ids.after_key');
     } while (searchAfter);
 
-    return { up, down, total: up + down };
+    let up: number = 0;
+    let mixed: number = 0;
+    let down: number = 0;
+
+    for (const id in summaryByIdLocation) {
+      if (!summaryByIdLocation.hasOwnProperty(id)) {
+        continue;
+      }
+      const locationInfo = summaryByIdLocation[id];
+      const { up: locationUp, down: locationDown } = reduce(
+        locationInfo,
+        (acc, value, key) => {
+          acc.up += value.up;
+          acc.down += value.down;
+          return acc;
+        },
+        { up: 0, down: 0 }
+      );
+
+      if (locationDown === 0) {
+        up++;
+      } else if (locationUp > 0) {
+        mixed++;
+      } else {
+        down++;
+      }
+    }
+
+    const result: any = { up, down, mixed, total: up + down + mixed };
+    if (statusFilter) {
+      for (const status in result) {
+        if (status !== 'total' && status !== statusFilter) {
+          result[status] = 0;
+        }
+      }
+    }
+
+    return result;
   }
 
   /**

x-pack/legacy/plugins/uptime/server/lib/domains/monitor_states.ts

@@ -5,7 +5,7 @@
  */
 
 import { UMMonitorStatesAdapter } from '../adapters/monitor_states';
-import { MonitorSummary, SnapshotCount, StatesIndexStatus } from '../../../common/graphql/types';
+import { MonitorSummary, StatesIndexStatus } from '../../../common/graphql/types';
 
 export class UMMonitorStatesDomain {
   constructor(private readonly adapter: UMMonitorStatesAdapter, libs: {}) {
@@ -22,15 +22,6 @@ export class UMMonitorStatesDomain {
     return this.adapter.getMonitorStates(request, pageIndex, pageSize, sortField, sortDirection);
   }
 
-  public async getSummaryCount(
-    request: any,
-    dateRangeStart: string,
-    dateRangeEnd: string,
-    filters?: string | null
-  ): Promise<SnapshotCount> {
-    return this.adapter.getSummaryCount(request, dateRangeStart, dateRangeEnd, filters);
-  }
-
   public async statesIndexExists(request: any): Promise<StatesIndexStatus> {
     return this.adapter.statesIndexExists(request);
   }

x-pack/test/api_integration/apis/uptime/graphql/fixtures/snapshot_filtered_by_down.json

@@ -1,5 +1,5 @@
 {
   "snapshot": {
-    "counts": { "down": 2, "mixed": 0, "up": 0, "total": 2 }
+    "counts": { "down": 2, "mixed": 0, "up": 0, "total": 10 }
   }
 }

x-pack/test/api_integration/apis/uptime/graphql/fixtures/snapshot_filtered_by_up.json

@@ -1,5 +1,5 @@
 {
   "snapshot": {
-    "counts": { "down": 0, "mixed": 0, "up": 8, "total": 8 }
+    "counts": { "down": 0, "mixed": 0, "up": 8, "total": 10 }
   }
 }