From d813e2eb874d57276ac928af3a6e6ba8e7aa9e3b Mon Sep 17 00:00:00 2001 From: Dominique Clarke Date: Tue, 6 Jul 2021 16:07:56 -0400 Subject: [PATCH 1/3] check monitor status by monitor.timespan --- .../synthetics_alerts/mappings.json | 212 ++++++++++++++++++ .../server/lib/alerts/status_check.test.ts | 114 +++++++++- .../uptime/server/lib/alerts/status_check.ts | 51 ++++- .../lib/requests/get_monitor_status.test.ts | 76 ++++++- .../server/lib/requests/get_monitor_status.ts | 17 +- 5 files changed, 449 insertions(+), 21 deletions(-) create mode 100644 test/functional/fixtures/es_archiver/synthetics_alerts/mappings.json diff --git a/test/functional/fixtures/es_archiver/synthetics_alerts/mappings.json b/test/functional/fixtures/es_archiver/synthetics_alerts/mappings.json new file mode 100644 index 00000000000000..19dfbfa319c0fd --- /dev/null +++ b/test/functional/fixtures/es_archiver/synthetics_alerts/mappings.json @@ -0,0 +1,212 @@ +{ + "type": "index", + "value": { + "aliases": { + ".kibana-alerts-observability-synthetics": { + "is_write_index": true + } + }, + "index": ".kibana_1", + "mappings": { + "dynamic": "strict", + "properties": { + "@timestamp": { + "type": "date" + }, + "agent": { + "properties": { + "name": { + "type": "keyword" + } + } + }, + "anomaly": { + "properties": { + "expected_response": { + "type": "double" + }, + "observer_location": { + "type": "keyword" + }, + "severity": { + "type": "keyword" + }, + "severity_score": { + "type": "double" + }, + "slowest_response": { + "type": "double" + }, + "start": { + "type": "date" + } + } + }, + "cert": { + "properties": { + "count": { + "type": "integer" + } + } + }, + "cert_status": { + "properties": { + "aging_common_name_and_date": { + "type": "text" + }, + "aging_count": { + "type": "integer" + }, + "expiring_common_name_and_date": { + "type": "text" + }, + "expiring_count": { + "type": "integer" + }, + "has_aging": { + "type": "boolean" + }, + "has_expired": { + "type": "boolean" + } + } + }, + "error": { + "properties": { + "message": { + "type": "text" + } + } + }, + "event": { + "properties": { + "action": { + "type": "keyword" + }, + "kind": { + "type": "keyword" + } + } + }, + "kibana": { + "properties": { + "rac": { + "properties": { + "alert": { + "properties": { + "duration": { + "properties": { + "us": { + "type": "long" + } + } + }, + "end": { + "type": "date" + }, + "evaluation": { + "properties": { + "threshold": { + "scaling_factor": 100, + "type": "scaled_float" + }, + "value": { + "scaling_factor": 100, + "type": "scaled_float" + } + } + }, + "id": { + "type": "keyword" + }, + "producer": { + "type": "keyword" + }, + "severity": { + "properties": { + "level": { + "type": "keyword" + }, + "value": { + "type": "long" + } + } + }, + "start": { + "type": "date" + }, + "status": { + "type": "keyword" + }, + "uuid": { + "type": "keyword" + } + } + } + } + } + } + }, + "monitor": { + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } + }, + "observer": { + "properties": { + "geo": { + "properties": { + "name": { + "type": "keyword" + } + } + } + } + }, + "reason": { + "type": "text" + }, + "rule": { + "properties": { + "category": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "uuid": { + "type": "keyword" + } + } + }, + "tags": { + "type": "keyword" + }, + "url": { + "properties": { + "full": { + "type": "keyword" + } + } + } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +} \ No newline at end of file diff --git a/x-pack/plugins/uptime/server/lib/alerts/status_check.test.ts b/x-pack/plugins/uptime/server/lib/alerts/status_check.test.ts index 29f2f0cca82bcd..743e9f6bc75ac9 100644 --- a/x-pack/plugins/uptime/server/lib/alerts/status_check.test.ts +++ b/x-pack/plugins/uptime/server/lib/alerts/status_check.test.ts @@ -4,7 +4,6 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - import { generateFilterDSL, hasFilters, @@ -62,7 +61,12 @@ const mockOptions = ( shouldCheckStatus: true, }, services = alertsMock.createAlertServices(), - state = {} + state = {}, + rule = { + schedule: { + interval: '5m', + }, + } ): any => { services.scopedClusterClient = elasticsearchServiceMock.createScopedClusterClient(); services.scopedClusterClient.asCurrentUser = (jest.fn() as unknown) as any; @@ -77,13 +81,16 @@ const mockOptions = ( params, services, state, + rule, }; }; describe('status check alert', () => { let toISOStringSpy: jest.SpyInstance; + const mockDate = new Date('2021-05-13T12:33:37.000Z'); beforeEach(() => { toISOStringSpy = jest.spyOn(Date.prototype, 'toISOString'); + Date.now = jest.fn().mockReturnValue(mockDate); }); afterEach(() => { @@ -108,10 +115,14 @@ describe('status check alert', () => { "filters": undefined, "locations": Array [], "numTimes": 5, - "timerange": Object { + "timespanRange": Object { "from": "now-15m", "to": "now", }, + "timestampRange": Object { + "from": 1620821917000, + "to": "now", + }, "uptimeEsClient": Object { "baseESClient": [MockFunction], "count": [Function], @@ -163,10 +174,14 @@ describe('status check alert', () => { "filters": undefined, "locations": Array [], "numTimes": 5, - "timerange": Object { + "timespanRange": Object { "from": "now-15m", "to": "now", }, + "timestampRange": Object { + "from": 1620821917000, + "to": "now", + }, "uptimeEsClient": Object { "baseESClient": [MockFunction], "count": [Function], @@ -476,10 +491,14 @@ describe('status check alert', () => { }, "locations": Array [], "numTimes": 3, - "timerange": Object { + "timespanRange": Object { "from": "now-15m", "to": "now", }, + "timestampRange": Object { + "from": 1620821917000, + "to": "now", + }, "uptimeEsClient": Object { "baseESClient": [MockFunction], "count": [Function], @@ -583,10 +602,14 @@ describe('status check alert', () => { }, "locations": Array [], "numTimes": 20, - "timerange": Object { + "timespanRange": Object { "from": "now-30h", "to": "now", }, + "timestampRange": Object { + "from": 1620714817000, + "to": "now", + }, "uptimeEsClient": Object { "baseESClient": [MockFunction], "count": [Function], @@ -900,6 +923,85 @@ describe('status check alert', () => { }); }); + it('generates timespan and @timestamp ranges appropriately', async () => { + const mockGetter = jest.fn(); + mockGetter.mockReturnValue([]); + const { server, libs, plugins } = bootstrapDependencies({ + getIndexPattern: jest.fn(), + getMonitorStatus: mockGetter, + }); + const alert = statusCheckAlertFactory(server, libs, plugins); + const options = mockOptions({ + numTimes: 20, + timerangeCount: 30, + timerangeUnit: 'h', + filters: { + 'monitor.type': ['http'], + 'observer.geo.name': [], + tags: [], + 'url.port': [], + }, + search: 'url.full: *', + }); + await alert.executor(options); + + expect(mockGetter).toHaveBeenCalledTimes(1); + expect(mockGetter.mock.calls[0][0]).toEqual( + expect.objectContaining({ + timespanRange: { + from: 'now-30h', + to: 'now', + }, + timestampRange: { + from: mockDate.setHours(mockDate.getHours() - 54).valueOf(), // now minus the timerange (30h), plus an additional 24 hour buffer + to: 'now', + }, + }) + ); + }); + + it('uses the larger of alert interval and timerange when defining timestampRange', async () => { + const mockGetter = jest.fn(); + mockGetter.mockReturnValue([]); + const { server, libs, plugins } = bootstrapDependencies({ + getIndexPattern: jest.fn(), + getMonitorStatus: mockGetter, + }); + const alert = statusCheckAlertFactory(server, libs, plugins); + const options = mockOptions( + { + numTimes: 20, + timerangeCount: 30, + timerangeUnit: 'h', + filters: { + 'monitor.type': ['http'], + 'observer.geo.name': [], + tags: [], + 'url.port': [], + }, + search: 'url.full: *', + }, + undefined, + undefined, + { schedule: { interval: '60h' } } + ); + await alert.executor(options); + + expect(mockGetter).toHaveBeenCalledTimes(1); + expect(mockGetter.mock.calls[0][0]).toEqual( + expect.objectContaining({ + timespanRange: { + from: 'now-30h', + to: 'now', + }, + timestampRange: { + from: mockDate.setHours(mockDate.getHours() - 60).valueOf(), // 60h rule schedule interval is larger than 30h timerange, so use now - 60h to define timestamp range + to: 'now', + }, + }) + ); + }); + describe('hasFilters', () => { it('returns false for undefined filters', () => { expect(hasFilters()).toBe(false); diff --git a/x-pack/plugins/uptime/server/lib/alerts/status_check.ts b/x-pack/plugins/uptime/server/lib/alerts/status_check.ts index 6f3e3303f6bdcf..75a642b5d0a30c 100644 --- a/x-pack/plugins/uptime/server/lib/alerts/status_check.ts +++ b/x-pack/plugins/uptime/server/lib/alerts/status_check.ts @@ -4,7 +4,8 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - +import datemath from '@elastic/datemath'; +import { min } from 'lodash'; import { schema } from '@kbn/config-schema'; import { i18n } from '@kbn/i18n'; import Mustache from 'mustache'; @@ -31,6 +32,34 @@ import { UMServerLibs, UptimeESClient } from '../lib'; export type ActionGroupIds = ActionGroupIdsOf; +/** + * Returns the appropriate range for filtering the documents by `@timestamp`. + * + * We check monitor status by `monitor.timespan`, but need to first cut down on the number of documents + * searched by filtering by `@timestamp`. To ensure that we catch as many documents as possible which could + * likely contain a down monitor with a `monitor.timespan` in the given timerange, we create a filter + * range for `@timestamp` that is the greater of either: from now to now - timerange interval - 24 hours + * OR from now to now - rule interval + * @param ruleScheduleLookback - string representing now minus the interval at which the rule is ran + * @param timerangeLookback - string representing now minus the timerange configured by the user for checking down monitors + */ +export function getTimestampRange({ + ruleScheduleLookback, + timerangeLookback, +}: Record<'ruleScheduleLookback' | 'timerangeLookback', string>) { + const scheduleIntervalAbsoluteTime = datemath.parse(ruleScheduleLookback)?.valueOf(); + const defaultIntervalAbsoluteTime = datemath + .parse(timerangeLookback) + ?.subtract('24', 'hours') + .valueOf(); + const absoluteFrom = min([scheduleIntervalAbsoluteTime, defaultIntervalAbsoluteTime]); + + return { + to: 'now', + from: absoluteFrom ? absoluteFrom : 'now-24h', + }; +} + const getMonIdByLoc = (monitorId: string, location: string) => { return monitorId + '-' + location; }; @@ -264,6 +293,9 @@ export const statusCheckAlertFactory: UptimeAlertTypeFactory = ( params: rawParams, state, services: { alertInstanceFactory }, + rule: { + schedule: { interval }, + }, }, uptimeEsClient, }) { @@ -279,14 +311,22 @@ export const statusCheckAlertFactory: UptimeAlertTypeFactory = ( isAutoGenerated, timerange: oldVersionTimeRange, } = rawParams; - const filterString = await formatFilterString(uptimeEsClient, filters, search, libs); - const timerange = oldVersionTimeRange || { - from: `now-${String(timerangeCount) + timerangeUnit}`, + const timespanInterval = `${String(timerangeCount)}${timerangeUnit}`; + + // Range filter for `monitor.timespan`, the range of time the ping is valid + const timespanRange = oldVersionTimeRange || { + from: `now-${timespanInterval}`, to: 'now', }; + // Range filter for `@timestamp`, the time the document was indexed + const timestampRange = getTimestampRange({ + ruleScheduleLookback: `now-${interval}`, + timerangeLookback: timespanRange.from, + }); + let downMonitorsByLocation: GetMonitorStatusResult[] = []; // if oldVersionTimeRange present means it's 7.7 format and @@ -294,7 +334,8 @@ export const statusCheckAlertFactory: UptimeAlertTypeFactory = ( if (!(!oldVersionTimeRange && shouldCheckStatus === false)) { downMonitorsByLocation = await libs.requests.getMonitorStatus({ uptimeEsClient, - timerange, + timespanRange, + timestampRange, numTimes, locations: [], filters: filterString, diff --git a/x-pack/plugins/uptime/server/lib/requests/get_monitor_status.test.ts b/x-pack/plugins/uptime/server/lib/requests/get_monitor_status.test.ts index 6d88ccb9a9efff..08b675576a5d20 100644 --- a/x-pack/plugins/uptime/server/lib/requests/get_monitor_status.test.ts +++ b/x-pack/plugins/uptime/server/lib/requests/get_monitor_status.test.ts @@ -85,10 +85,14 @@ describe('getMonitorStatus', () => { filters: exampleFilter, locations: [], numTimes: 5, - timerange: { + timespanRange: { from: 'now-10m', to: 'now-1m', }, + timestampRange: { + from: 'now-24h', + to: 'now', + }, }); expect(esMock.search).toHaveBeenCalledTimes(1); const [params] = esMock.search.mock.calls[0]; @@ -144,6 +148,14 @@ describe('getMonitorStatus', () => { Object { "range": Object { "@timestamp": Object { + "gte": "now-24h", + "lte": "now", + }, + }, + }, + Object { + "range": Object { + "monitor.timespan": Object { "gte": "now-10m", "lte": "now-1m", }, @@ -202,10 +214,14 @@ describe('getMonitorStatus', () => { uptimeEsClient, locations: ['fairbanks', 'harrisburg'], numTimes: 1, - timerange: { + timespanRange: { from: 'now-2m', to: 'now', }, + timestampRange: { + from: 'now-24h', + to: 'now', + }, }); expect(esMock.search).toHaveBeenCalledTimes(1); const [params] = esMock.search.mock.calls[0]; @@ -261,6 +277,14 @@ describe('getMonitorStatus', () => { Object { "range": Object { "@timestamp": Object { + "gte": "now-24h", + "lte": "now", + }, + }, + }, + Object { + "range": Object { + "monitor.timespan": Object { "gte": "now-2m", "lte": "now", }, @@ -298,10 +322,14 @@ describe('getMonitorStatus', () => { genBucketItem ); const clientParameters = { - timerange: { + timespanRange: { from: 'now-15m', to: 'now', }, + timestampRange: { + from: 'now-24h', + to: 'now', + }, numTimes: 5, locations: [], filters: { @@ -415,6 +443,14 @@ describe('getMonitorStatus', () => { Object { "range": Object { "@timestamp": Object { + "gte": "now-24h", + "lte": "now", + }, + }, + }, + Object { + "range": Object { + "monitor.timespan": Object { "gte": "now-15m", "lte": "now", }, @@ -485,10 +521,14 @@ describe('getMonitorStatus', () => { genBucketItem ); const clientParameters = { - timerange: { + timespanRange: { from: 'now-15m', to: 'now', }, + timestampRange: { + from: 'now-24h', + to: 'now', + }, numTimes: 5, locations: [], filters: { @@ -562,6 +602,14 @@ describe('getMonitorStatus', () => { Object { "range": Object { "@timestamp": Object { + "gte": "now-24h", + "lte": "now", + }, + }, + }, + Object { + "range": Object { + "monitor.timespan": Object { "gte": "now-15m", "lte": "now", }, @@ -618,10 +666,14 @@ describe('getMonitorStatus', () => { filters: undefined, locations: [], numTimes: 5, - timerange: { + timespanRange: { from: 'now-12m', to: 'now-2m', }, + timestampRange: { + from: 'now-24h', + to: 'now', + }, }; const { uptimeEsClient } = getUptimeESMockClient(esMock); @@ -684,6 +736,14 @@ describe('getMonitorStatus', () => { Object { "range": Object { "@timestamp": Object { + "gte": "now-24h", + "lte": "now", + }, + }, + }, + Object { + "range": Object { + "monitor.timespan": Object { "gte": "now-12m", "lte": "now-2m", }, @@ -810,10 +870,14 @@ describe('getMonitorStatus', () => { uptimeEsClient, locations: [], numTimes: 5, - timerange: { + timespanRange: { from: 'now-10m', to: 'now-1m', }, + timestampRange: { + from: 'now-24h', + to: 'now', + }, }); expect(result.length).toBe(8); expect(result).toMatchInlineSnapshot(` diff --git a/x-pack/plugins/uptime/server/lib/requests/get_monitor_status.ts b/x-pack/plugins/uptime/server/lib/requests/get_monitor_status.ts index 07047bd0be7bc5..15e6fe30db1867 100644 --- a/x-pack/plugins/uptime/server/lib/requests/get_monitor_status.ts +++ b/x-pack/plugins/uptime/server/lib/requests/get_monitor_status.ts @@ -15,7 +15,8 @@ export interface GetMonitorStatusParams { filters?: JsonObject; locations: string[]; numTimes: number; - timerange: { from: string; to: string }; + timespanRange: { from: string; to: string }; + timestampRange: { from: string | number; to: string }; } export interface GetMonitorStatusResult { @@ -43,7 +44,7 @@ export type AfterKey = Record | undefined; export const getMonitorStatus: UMElasticsearchQueryFn< GetMonitorStatusParams, GetMonitorStatusResult[] -> = async ({ uptimeEsClient, filters, locations, numTimes, timerange: { from, to } }) => { +> = async ({ uptimeEsClient, filters, locations, numTimes, timespanRange, timestampRange }) => { let afterKey: AfterKey; const STATUS = 'down'; @@ -63,8 +64,16 @@ export const getMonitorStatus: UMElasticsearchQueryFn< { range: { '@timestamp': { - gte: from, - lte: to, + gte: timestampRange.from, + lte: timestampRange.to, + }, + }, + }, + { + range: { + 'monitor.timespan': { + gte: timespanRange.from, + lte: timespanRange.to, }, }, }, From 22fb1cdc3d2443a8ace1ce95d8702d8b09d5918e Mon Sep 17 00:00:00 2001 From: Dominique Clarke Date: Wed, 7 Jul 2021 11:00:05 -0400 Subject: [PATCH 2/3] Delete mappings.json --- .../synthetics_alerts/mappings.json | 212 ------------------ 1 file changed, 212 deletions(-) delete mode 100644 test/functional/fixtures/es_archiver/synthetics_alerts/mappings.json diff --git a/test/functional/fixtures/es_archiver/synthetics_alerts/mappings.json b/test/functional/fixtures/es_archiver/synthetics_alerts/mappings.json deleted file mode 100644 index 19dfbfa319c0fd..00000000000000 --- a/test/functional/fixtures/es_archiver/synthetics_alerts/mappings.json +++ /dev/null @@ -1,212 +0,0 @@ -{ - "type": "index", - "value": { - "aliases": { - ".kibana-alerts-observability-synthetics": { - "is_write_index": true - } - }, - "index": ".kibana_1", - "mappings": { - "dynamic": "strict", - "properties": { - "@timestamp": { - "type": "date" - }, - "agent": { - "properties": { - "name": { - "type": "keyword" - } - } - }, - "anomaly": { - "properties": { - "expected_response": { - "type": "double" - }, - "observer_location": { - "type": "keyword" - }, - "severity": { - "type": "keyword" - }, - "severity_score": { - "type": "double" - }, - "slowest_response": { - "type": "double" - }, - "start": { - "type": "date" - } - } - }, - "cert": { - "properties": { - "count": { - "type": "integer" - } - } - }, - "cert_status": { - "properties": { - "aging_common_name_and_date": { - "type": "text" - }, - "aging_count": { - "type": "integer" - }, - "expiring_common_name_and_date": { - "type": "text" - }, - "expiring_count": { - "type": "integer" - }, - "has_aging": { - "type": "boolean" - }, - "has_expired": { - "type": "boolean" - } - } - }, - "error": { - "properties": { - "message": { - "type": "text" - } - } - }, - "event": { - "properties": { - "action": { - "type": "keyword" - }, - "kind": { - "type": "keyword" - } - } - }, - "kibana": { - "properties": { - "rac": { - "properties": { - "alert": { - "properties": { - "duration": { - "properties": { - "us": { - "type": "long" - } - } - }, - "end": { - "type": "date" - }, - "evaluation": { - "properties": { - "threshold": { - "scaling_factor": 100, - "type": "scaled_float" - }, - "value": { - "scaling_factor": 100, - "type": "scaled_float" - } - } - }, - "id": { - "type": "keyword" - }, - "producer": { - "type": "keyword" - }, - "severity": { - "properties": { - "level": { - "type": "keyword" - }, - "value": { - "type": "long" - } - } - }, - "start": { - "type": "date" - }, - "status": { - "type": "keyword" - }, - "uuid": { - "type": "keyword" - } - } - } - } - } - } - }, - "monitor": { - "properties": { - "id": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "type": { - "type": "keyword" - } - } - }, - "observer": { - "properties": { - "geo": { - "properties": { - "name": { - "type": "keyword" - } - } - } - } - }, - "reason": { - "type": "text" - }, - "rule": { - "properties": { - "category": { - "type": "keyword" - }, - "id": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "uuid": { - "type": "keyword" - } - } - }, - "tags": { - "type": "keyword" - }, - "url": { - "properties": { - "full": { - "type": "keyword" - } - } - } - } - }, - "settings": { - "index": { - "number_of_replicas": "1", - "number_of_shards": "1" - } - } - } -} \ No newline at end of file From 8623c9b867ace6d2f10b13ec656da68ce0fbc305 Mon Sep 17 00:00:00 2001 From: Dominique Clarke Date: Fri, 9 Jul 2021 10:24:03 -0400 Subject: [PATCH 3/3] adjust logic --- x-pack/plugins/uptime/server/lib/alerts/status_check.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/uptime/server/lib/alerts/status_check.ts b/x-pack/plugins/uptime/server/lib/alerts/status_check.ts index 75a642b5d0a30c..364518bba720ad 100644 --- a/x-pack/plugins/uptime/server/lib/alerts/status_check.ts +++ b/x-pack/plugins/uptime/server/lib/alerts/status_check.ts @@ -52,11 +52,11 @@ export function getTimestampRange({ .parse(timerangeLookback) ?.subtract('24', 'hours') .valueOf(); - const absoluteFrom = min([scheduleIntervalAbsoluteTime, defaultIntervalAbsoluteTime]); + const from = min([scheduleIntervalAbsoluteTime, defaultIntervalAbsoluteTime]) ?? 'now-24h'; return { to: 'now', - from: absoluteFrom ? absoluteFrom : 'now-24h', + from, }; }