From 237b2f66ed1b4ad44849aa6a8439771ba16109f8 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Thu, 9 Jul 2020 13:59:00 +0100 Subject: [PATCH 01/13] skip flaky suite (#71216) --- test/functional/apps/discover/_doc_navigation.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/functional/apps/discover/_doc_navigation.js b/test/functional/apps/discover/_doc_navigation.js index 669540c5f4a42..9bcf7fd2d73b5 100644 --- a/test/functional/apps/discover/_doc_navigation.js +++ b/test/functional/apps/discover/_doc_navigation.js @@ -28,7 +28,8 @@ export default function ({ getService, getPageObjects }) { const esArchiver = getService('esArchiver'); const retry = getService('retry'); - describe('doc link in discover', function contextSize() { + // Flaky: https://github.com/elastic/kibana/issues/71216 + describe.skip('doc link in discover', function contextSize() { beforeEach(async function () { log.debug('load kibana index with default index pattern'); await esArchiver.loadIfNeeded('discover'); From c8e675492bad0f1324e9254618b1054fef0862cd Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Thu, 9 Jul 2020 09:05:25 -0400 Subject: [PATCH 02/13] [Ingest Manager] Fix and improve agent status (#71009) --- .../common/services/agent_status.ts | 57 ++++++++----------- .../common/types/models/agent.ts | 12 +++- .../common/types/rest_spec/agent.ts | 1 + .../sections/fleet/agent_list_page/index.tsx | 12 ++-- .../fleet/components/agent_health.tsx | 20 +++++++ .../server/routes/agent/handlers.ts | 7 ++- .../server/saved_objects/index.ts | 2 + .../server/services/agents/checkin/index.ts | 23 ++++---- .../agents/checkin/state_connected_agents.ts | 2 +- .../server/services/agents/status.test.ts | 33 +++++++++++ .../server/types/rest_spec/agent.ts | 3 + 11 files changed, 119 insertions(+), 53 deletions(-) diff --git a/x-pack/plugins/ingest_manager/common/services/agent_status.ts b/x-pack/plugins/ingest_manager/common/services/agent_status.ts index b1d92d3a78e65..6489c30308771 100644 --- a/x-pack/plugins/ingest_manager/common/services/agent_status.ts +++ b/x-pack/plugins/ingest_manager/common/services/agent_status.ts @@ -5,63 +5,52 @@ */ import { - AGENT_TYPE_TEMPORARY, AGENT_POLLING_THRESHOLD_MS, AGENT_TYPE_PERMANENT, - AGENT_TYPE_EPHEMERAL, AGENT_SAVED_OBJECT_TYPE, } from '../constants'; import { Agent, AgentStatus } from '../types'; export function getAgentStatus(agent: Agent, now: number = Date.now()): AgentStatus { - const { type, last_checkin: lastCheckIn } = agent; - const msLastCheckIn = new Date(lastCheckIn || 0).getTime(); - const msSinceLastCheckIn = new Date().getTime() - msLastCheckIn; - const intervalsSinceLastCheckIn = Math.floor(msSinceLastCheckIn / AGENT_POLLING_THRESHOLD_MS); + const { last_checkin: lastCheckIn } = agent; + if (!agent.active) { return 'inactive'; } + if (!agent.last_checkin) { + return 'enrolling'; + } if (agent.unenrollment_started_at && !agent.unenrolled_at) { return 'unenrolling'; } - if (agent.current_error_events.length > 0) { + + const msLastCheckIn = new Date(lastCheckIn || 0).getTime(); + const msSinceLastCheckIn = new Date().getTime() - msLastCheckIn; + const intervalsSinceLastCheckIn = Math.floor(msSinceLastCheckIn / AGENT_POLLING_THRESHOLD_MS); + + if (agent.last_checkin_status === 'error') { return 'error'; } - switch (type) { - case AGENT_TYPE_PERMANENT: - if (intervalsSinceLastCheckIn >= 4) { - return 'error'; - } - case AGENT_TYPE_TEMPORARY: - if (intervalsSinceLastCheckIn >= 3) { - return 'offline'; - } - case AGENT_TYPE_EPHEMERAL: - if (intervalsSinceLastCheckIn >= 3) { - return 'inactive'; - } + if (agent.last_checkin_status === 'degraded') { + return 'degraded'; + } + if (intervalsSinceLastCheckIn >= 4) { + return 'offline'; } + return 'online'; } export function buildKueryForOnlineAgents() { - return `(${AGENT_SAVED_OBJECT_TYPE}.type:${AGENT_TYPE_PERMANENT} and ${AGENT_SAVED_OBJECT_TYPE}.last_checkin >= now-${ - (4 * AGENT_POLLING_THRESHOLD_MS) / 1000 - }s) or (${AGENT_SAVED_OBJECT_TYPE}.type:${AGENT_TYPE_TEMPORARY} and ${AGENT_SAVED_OBJECT_TYPE}.last_checkin >= now-${ - (3 * AGENT_POLLING_THRESHOLD_MS) / 1000 - }s) or (${AGENT_SAVED_OBJECT_TYPE}.type:${AGENT_TYPE_EPHEMERAL} and ${AGENT_SAVED_OBJECT_TYPE}.last_checkin >= now-${ - (3 * AGENT_POLLING_THRESHOLD_MS) / 1000 - }s)`; + return `not (${buildKueryForOfflineAgents()}) AND not (${buildKueryForErrorAgents()})`; } -export function buildKueryForOfflineAgents() { - return `${AGENT_SAVED_OBJECT_TYPE}.type:${AGENT_TYPE_TEMPORARY} AND ${AGENT_SAVED_OBJECT_TYPE}.last_checkin < now-${ - (3 * AGENT_POLLING_THRESHOLD_MS) / 1000 - }s`; +export function buildKueryForErrorAgents() { + return `( ${AGENT_SAVED_OBJECT_TYPE}.last_checkin_status:error or ${AGENT_SAVED_OBJECT_TYPE}.last_checkin_status:degraded )`; } -export function buildKueryForErrorAgents() { - return `${AGENT_SAVED_OBJECT_TYPE}.type:${AGENT_TYPE_PERMANENT} AND ${AGENT_SAVED_OBJECT_TYPE}.last_checkin < now-${ +export function buildKueryForOfflineAgents() { + return `((${AGENT_SAVED_OBJECT_TYPE}.type:${AGENT_TYPE_PERMANENT} AND ${AGENT_SAVED_OBJECT_TYPE}.last_checkin < now-${ (4 * AGENT_POLLING_THRESHOLD_MS) / 1000 - }s`; + }s) AND not ( ${buildKueryForErrorAgents()} ))`; } diff --git a/x-pack/plugins/ingest_manager/common/types/models/agent.ts b/x-pack/plugins/ingest_manager/common/types/models/agent.ts index 1f4718acc2c1f..d3789c58a2c22 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/agent.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/agent.ts @@ -11,7 +11,16 @@ export type AgentType = | typeof AGENT_TYPE_PERMANENT | typeof AGENT_TYPE_TEMPORARY; -export type AgentStatus = 'offline' | 'error' | 'online' | 'inactive' | 'warning' | 'unenrolling'; +export type AgentStatus = + | 'offline' + | 'error' + | 'online' + | 'inactive' + | 'warning' + | 'enrolling' + | 'unenrolling' + | 'degraded'; + export type AgentActionType = 'CONFIG_CHANGE' | 'DATA_DUMP' | 'RESUME' | 'PAUSE' | 'UNENROLL'; export interface NewAgentAction { type: AgentActionType; @@ -82,6 +91,7 @@ interface AgentBase { config_id?: string; config_revision?: number | null; last_checkin?: string; + last_checkin_status?: 'error' | 'online' | 'degraded'; user_provided_metadata: AgentMetadata; local_metadata: AgentMetadata; } diff --git a/x-pack/plugins/ingest_manager/common/types/rest_spec/agent.ts b/x-pack/plugins/ingest_manager/common/types/rest_spec/agent.ts index 1105c8ee7ca82..ed7d73ab0b719 100644 --- a/x-pack/plugins/ingest_manager/common/types/rest_spec/agent.ts +++ b/x-pack/plugins/ingest_manager/common/types/rest_spec/agent.ts @@ -47,6 +47,7 @@ export interface PostAgentCheckinRequest { agentId: string; }; body: { + status?: 'online' | 'error' | 'degraded'; local_metadata?: Record; events?: NewAgentEvent[]; }; diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_list_page/index.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_list_page/index.tsx index 30204603e764c..36a8bf908ddd7 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_list_page/index.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_list_page/index.tsx @@ -178,11 +178,7 @@ export const AgentListPage: React.FunctionComponent<{}> = () => { } if (selectedStatus.length) { - if (kuery) { - kuery = `(${kuery}) and`; - } - - kuery = selectedStatus + const kueryStatus = selectedStatus .map((status) => { switch (status) { case 'online': @@ -196,6 +192,12 @@ export const AgentListPage: React.FunctionComponent<{}> = () => { return ''; }) .join(' or '); + + if (kuery) { + kuery = `(${kuery}) and ${kueryStatus}`; + } else { + kuery = kueryStatus; + } } const agentsRequest = useGetAgents( diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/components/agent_health.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/components/agent_health.tsx index e4dfa520259eb..7c6c95cab420f 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/components/agent_health.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/components/agent_health.tsx @@ -53,6 +53,22 @@ const Status = { /> ), + Degraded: ( + + + + ), + Enrolling: ( + + + + ), Unenrolling: ( = {}; + const { updatedErrorEvents } = await processEventsForCheckin(soClient, agent, data.events); if (updatedErrorEvents) { updateData.current_error_events = JSON.stringify(updatedErrorEvents); } - if (localMetadata) { - updateData.local_metadata = localMetadata; + if (data.localMetadata) { + updateData.local_metadata = data.localMetadata; + } + + if (data.status !== agent.last_checkin_status) { + updateData.last_checkin_status = data.status; } if (Object.keys(updateData).length > 0) { await soClient.update(AGENT_SAVED_OBJECT_TYPE, agent.id, updateData); diff --git a/x-pack/plugins/ingest_manager/server/services/agents/checkin/state_connected_agents.ts b/x-pack/plugins/ingest_manager/server/services/agents/checkin/state_connected_agents.ts index 96e006b78f00f..994ecc64c82a7 100644 --- a/x-pack/plugins/ingest_manager/server/services/agents/checkin/state_connected_agents.ts +++ b/x-pack/plugins/ingest_manager/server/services/agents/checkin/state_connected_agents.ts @@ -59,7 +59,7 @@ export function agentCheckinStateConnectedAgentsFactory() { const internalSOClient = getInternalUserSOClient(); const now = new Date().toISOString(); const updates: Array> = [ - ...connectedAgentsIds.values(), + ...agentToUpdate.values(), ].map((agentId) => ({ type: AGENT_SAVED_OBJECT_TYPE, id: agentId, diff --git a/x-pack/plugins/ingest_manager/server/services/agents/status.test.ts b/x-pack/plugins/ingest_manager/server/services/agents/status.test.ts index 8140b1e6de470..f216cd541eb21 100644 --- a/x-pack/plugins/ingest_manager/server/services/agents/status.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/agents/status.test.ts @@ -33,6 +33,7 @@ describe('Agent status service', () => { type: AGENT_TYPE_PERMANENT, attributes: { active: true, + last_checkin: new Date().toISOString(), local_metadata: {}, user_provided_metadata: {}, }, @@ -40,4 +41,36 @@ describe('Agent status service', () => { const status = await getAgentStatusById(mockSavedObjectsClient, 'id'); expect(status).toEqual('online'); }); + + it('should return enrolling when agent is active but never checkin', async () => { + const mockSavedObjectsClient = savedObjectsClientMock.create(); + mockSavedObjectsClient.get = jest.fn().mockReturnValue({ + id: 'id', + type: AGENT_TYPE_PERMANENT, + attributes: { + active: true, + local_metadata: {}, + user_provided_metadata: {}, + }, + } as SavedObject); + const status = await getAgentStatusById(mockSavedObjectsClient, 'id'); + expect(status).toEqual('enrolling'); + }); + + it('should return unenrolling when agent is unenrolling', async () => { + const mockSavedObjectsClient = savedObjectsClientMock.create(); + mockSavedObjectsClient.get = jest.fn().mockReturnValue({ + id: 'id', + type: AGENT_TYPE_PERMANENT, + attributes: { + active: true, + last_checkin: new Date().toISOString(), + unenrollment_started_at: new Date().toISOString(), + local_metadata: {}, + user_provided_metadata: {}, + }, + } as SavedObject); + const status = await getAgentStatusById(mockSavedObjectsClient, 'id'); + expect(status).toEqual('unenrolling'); + }); }); diff --git a/x-pack/plugins/ingest_manager/server/types/rest_spec/agent.ts b/x-pack/plugins/ingest_manager/server/types/rest_spec/agent.ts index a508c33e0347b..3e9209efcac04 100644 --- a/x-pack/plugins/ingest_manager/server/types/rest_spec/agent.ts +++ b/x-pack/plugins/ingest_manager/server/types/rest_spec/agent.ts @@ -32,6 +32,9 @@ export const PostAgentCheckinRequestSchema = { agentId: schema.string(), }), body: schema.object({ + status: schema.maybe( + schema.oneOf([schema.literal('online'), schema.literal('error'), schema.literal('degraded')]) + ), local_metadata: schema.maybe(schema.recordOf(schema.string(), schema.any())), events: schema.maybe(schema.arrayOf(NewAgentEventSchema)), }), From cfbdf1f55e8d34ef9ea5603657cb70ae7d3fb528 Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Thu, 9 Jul 2020 15:08:51 +0200 Subject: [PATCH 03/13] Unskip data table non timebased test (#71049) * Unskip data table non timebased test * Fix test to use new label Co-authored-by: Elastic Machine --- test/functional/apps/visualize/_data_table_nontimeindex.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/functional/apps/visualize/_data_table_nontimeindex.js b/test/functional/apps/visualize/_data_table_nontimeindex.js index d64629a65c2c3..fd06257a91ff4 100644 --- a/test/functional/apps/visualize/_data_table_nontimeindex.js +++ b/test/functional/apps/visualize/_data_table_nontimeindex.js @@ -112,8 +112,7 @@ export default function ({ getService, getPageObjects }) { expect(data.trim().split('\n')).to.be.eql(['14,004 1,412.6']); }); - // bug https://github.com/elastic/kibana/issues/68977 - describe.skip('data table with date histogram', async () => { + describe('data table with date histogram', async () => { before(async () => { await PageObjects.visualize.navigateToNewVisualization(); await PageObjects.visualize.clickDataTable(); @@ -123,7 +122,7 @@ export default function ({ getService, getPageObjects }) { await PageObjects.visEditor.clickBucket('Split rows'); await PageObjects.visEditor.selectAggregation('Date Histogram'); await PageObjects.visEditor.selectField('@timestamp'); - await PageObjects.visEditor.setInterval('Daily'); + await PageObjects.visEditor.setInterval('Day'); await PageObjects.visEditor.clickGo(); }); From d37b053f28d2e3ae1545438c3a7666ff3108cd09 Mon Sep 17 00:00:00 2001 From: Alison Goryachev Date: Thu, 9 Jul 2020 09:41:11 -0400 Subject: [PATCH 04/13] [Index Management] Fix broken app links (#71007) --- .../components/policy_table/policy_table.js | 2 +- .../helpers/setup_environment.tsx | 6 ++++- .../public/application/sections/home/home.tsx | 1 - .../index_list/index_table/index_table.js | 2 +- .../template_details/tabs/tab_summary.tsx | 15 +++++++++++-- .../public/application/services/navigation.ts | 21 ------------------ .../public/application/services/routing.ts | 22 +++++++++++++++++++ .../plugins/index_management/public/index.ts | 2 +- 8 files changed, 43 insertions(+), 28 deletions(-) delete mode 100644 x-pack/plugins/index_management/public/application/services/navigation.ts diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/policy_table/components/policy_table/policy_table.js b/x-pack/plugins/index_lifecycle_management/public/application/sections/policy_table/components/policy_table/policy_table.js index dad259681eb7a..500ab44d96694 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/policy_table/components/policy_table/policy_table.js +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/policy_table/components/policy_table/policy_table.js @@ -254,7 +254,7 @@ export class PolicyTable extends Component { icon: 'list', onClick: () => { this.props.navigateToApp('management', { - path: `/data/index_management${getIndexListUri(`ilm.policy:${policy.name}`)}`, + path: `/data/index_management${getIndexListUri(`ilm.policy:${policy.name}`, true)}`, }); }, }); diff --git a/x-pack/plugins/index_management/__jest__/client_integration/helpers/setup_environment.tsx b/x-pack/plugins/index_management/__jest__/client_integration/helpers/setup_environment.tsx index d85db94d4a970..ad445f75f047c 100644 --- a/x-pack/plugins/index_management/__jest__/client_integration/helpers/setup_environment.tsx +++ b/x-pack/plugins/index_management/__jest__/client_integration/helpers/setup_environment.tsx @@ -34,7 +34,11 @@ export const services = { services.uiMetricService.setup({ reportUiStats() {} } as any); setExtensionsService(services.extensionsService); setUiMetricService(services.uiMetricService); -const appDependencies = { services, core: { getUrlForApp: () => {} }, plugins: {} } as any; +const appDependencies = { + services, + core: { getUrlForApp: () => {} }, + plugins: {}, +} as any; export const setupEnvironment = () => { // Mock initialization of services diff --git a/x-pack/plugins/index_management/public/application/sections/home/home.tsx b/x-pack/plugins/index_management/public/application/sections/home/home.tsx index 7bd04cdbf0c91..ee8970a3c4509 100644 --- a/x-pack/plugins/index_management/public/application/sections/home/home.tsx +++ b/x-pack/plugins/index_management/public/application/sections/home/home.tsx @@ -144,7 +144,6 @@ export const IndexManagementHome: React.FunctionComponent - index.name); const selectedIndexNames = Object.keys(selectedIndicesMap); diff --git a/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/tabs/tab_summary.tsx b/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/tabs/tab_summary.tsx index fe6c9ad3d8e07..de2fc29ec8543 100644 --- a/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/tabs/tab_summary.tsx +++ b/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/tabs/tab_summary.tsx @@ -18,8 +18,9 @@ import { EuiFlexItem, EuiCodeBlock, } from '@elastic/eui'; +import { useAppContext } from '../../../../../app_context'; import { TemplateDeserialized } from '../../../../../../../common'; -import { getILMPolicyPath } from '../../../../../services/navigation'; +import { getILMPolicyPath } from '../../../../../services/routing'; interface Props { templateDetails: TemplateDeserialized; @@ -51,6 +52,10 @@ export const TabSummary: React.FunctionComponent = ({ templateDetails }) const numIndexPatterns = indexPatterns.length; + const { + core: { getUrlForApp }, + } = useAppContext(); + return ( @@ -153,7 +158,13 @@ export const TabSummary: React.FunctionComponent = ({ templateDetails }) {ilmPolicy && ilmPolicy.name ? ( - {ilmPolicy.name} + + {ilmPolicy.name} + ) : ( i18nTexts.none )} diff --git a/x-pack/plugins/index_management/public/application/services/navigation.ts b/x-pack/plugins/index_management/public/application/services/navigation.ts deleted file mode 100644 index 3b4977bb63751..0000000000000 --- a/x-pack/plugins/index_management/public/application/services/navigation.ts +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -export const getIndexListUri = (filter: any) => { - if (filter) { - // React router tries to decode url params but it can't because the browser partially - // decodes them. So we have to encode both the URL and the filter to get it all to - // work correctly for filters with URL unsafe characters in them. - return encodeURI(`/indices/filter/${encodeURIComponent(filter)}`); - } - - // If no filter, URI is already safe so no need to encode. - return '/indices'; -}; - -export const getILMPolicyPath = (policyName: string) => { - return encodeURI(`/policies/edit/${encodeURIComponent(policyName)}`); -}; diff --git a/x-pack/plugins/index_management/public/application/services/routing.ts b/x-pack/plugins/index_management/public/application/services/routing.ts index 8831fa2368f47..68bf06409e6ab 100644 --- a/x-pack/plugins/index_management/public/application/services/routing.ts +++ b/x-pack/plugins/index_management/public/application/services/routing.ts @@ -31,6 +31,28 @@ export const getTemplateCloneLink = (name: string, isLegacy?: boolean) => { return encodeURI(url); }; +export const getILMPolicyPath = (policyName: string) => { + return encodeURI( + `/data/index_lifecycle_management/policies/edit/${encodeURIComponent(policyName)}` + ); +}; + +export const getIndexListUri = (filter?: string, includeHiddenIndices?: boolean) => { + const hiddenIndicesParam = + typeof includeHiddenIndices !== 'undefined' ? includeHiddenIndices : false; + if (filter) { + // React router tries to decode url params but it can't because the browser partially + // decodes them. So we have to encode both the URL and the filter to get it all to + // work correctly for filters with URL unsafe characters in them. + return encodeURI( + `/indices?includeHiddenIndices=${hiddenIndicesParam}&filter=${encodeURIComponent(filter)}` + ); + } + + // If no filter, URI is already safe so no need to encode. + return '/indices'; +}; + export const decodePathFromReactRouter = (pathname: string): string => { let decodedPath; try { diff --git a/x-pack/plugins/index_management/public/index.ts b/x-pack/plugins/index_management/public/index.ts index 7a76fff7f3ec6..a2e9a41feb165 100644 --- a/x-pack/plugins/index_management/public/index.ts +++ b/x-pack/plugins/index_management/public/index.ts @@ -13,4 +13,4 @@ export const plugin = () => { export { IndexManagementPluginSetup }; -export { getIndexListUri } from './application/services/navigation'; +export { getIndexListUri } from './application/services/routing'; From 5085b62c9c2139526d4f4d6f7d30a75d0a26cea5 Mon Sep 17 00:00:00 2001 From: Andrew Cholakian Date: Thu, 9 Jul 2020 08:51:55 -0500 Subject: [PATCH 05/13] [Uptime] Remove Scripted Metric Query (#69229) Resolves #68096 , removing unnecessary scripted metric query from overview page and unifying the Check and Ping types. This simplifies the types in a number of ways, and reduces the total quantity of code to execute the queries for the overview page. It also fixes the Tls and related types which were inconsistent and presented a problem here since they are used by this JS. There are now three stages where before there were four: Find potential matches: where we determine which monitor IDs are eligible for the overview page Refine potential matches: where we determine which ones actually match and return the summary documents for each location to build the MonitorSummary object Get monitor histograms: where we calculate the histograms for each monitor. In the future we might make this a separate API call. This improves the overall code structure, and leaves the test coverage about the same depending on how you look at it. I think we can do more to improve the quality of code / tests here, but this seemed like a good initial place to draw the line for now. In perfunctory testing on our internal observability clusters I saw perf improve from 2.5s to 1.1s on the Uptime homepage with no filters. So, it looks like there are potentially perf improvements (no real benchmarking was done). Previously, this returned all pings from the latest check group. This was not actually used anywhere, only the summary pings are required for the current UI, so we now only return those from the list API as this saves a query. --- .../common/runtime_types/monitor/state.ts | 91 +- .../uptime/common/runtime_types/ping/ping.ts | 71 +- .../ssl_certificate.test.tsx.snap | 3 +- .../__test__/ssl_certificate.test.tsx | 19 +- .../status_bar/ssl_certificate.tsx | 22 +- .../__snapshots__/monitor_list.test.tsx.snap | 922 ++++++++++++------ .../__tests__/monitor_list.test.tsx | 216 ++-- .../monitor_list_status_column.test.tsx | 84 +- .../monitor_list/cert_status_column.tsx | 13 +- .../overview/monitor_list/monitor_list.tsx | 16 +- .../monitor_list_drawer.test.tsx.snap | 71 +- .../monitor_list_drawer/__tests__/data.json | 2 +- .../__tests__/integration_group.test.tsx | 33 +- .../__tests__/monitor_list_drawer.test.tsx | 61 +- .../__tests__/monitor_status_list.test.tsx | 58 +- .../actions_popover/integration_group.tsx | 6 +- .../monitor_list_drawer.tsx | 4 +- .../monitor_status_list.tsx | 18 +- .../monitor_list_status_column.tsx | 32 +- .../__tests__/get_apm_href.test.ts | 29 +- .../__tests__/get_infra_href.test.ts | 202 ++-- .../__tests__/get_logging_href.test.ts | 50 +- .../observability_integration/build_href.ts | 8 +- .../get_infra_href.ts | 6 +- .../get_logging_href.ts | 6 +- .../public/state/actions/monitor_list.ts | 6 +- .../uptime/public/state/api/monitor_list.ts | 8 +- .../plugins/uptime/public/state/api/utils.ts | 30 +- .../public/state/reducers/monitor_list.ts | 8 +- .../__tests__/get_latest_monitor.test.ts | 14 +- .../server/lib/requests/get_latest_monitor.ts | 14 +- .../server/lib/requests/get_monitor_states.ts | 123 ++- .../__tests__/enrich_monitor_groups.test.ts | 98 -- .../search/__tests__/fetch_page.test.ts | 111 --- .../__tests__/monitor_group_iterator.test.ts | 100 -- .../monitor_summary_iterator.test.ts | 131 +++ .../refine_potential_matches.test.ts | 112 --- .../requests/search/enrich_monitor_groups.ts | 409 -------- .../server/lib/requests/search/fetch_chunk.ts | 7 +- .../server/lib/requests/search/fetch_page.ts | 138 --- .../requests/search/find_potential_matches.ts | 9 +- .../server/lib/requests/search/index.ts | 3 +- ...terator.ts => monitor_summary_iterator.ts} | 105 +- .../lib/requests/search/query_context.ts | 18 + .../search/refine_potential_matches.ts | 118 ++- .../server/lib/requests/uptime_requests.ts | 4 +- .../rest/fixtures/monitor_latest_status.json | 5 +- .../uptime/rest/monitor_states_generated.ts | 12 - .../uptime/rest/monitor_states_real_data.ts | 4 +- 49 files changed, 1683 insertions(+), 1947 deletions(-) delete mode 100644 x-pack/plugins/uptime/server/lib/requests/search/__tests__/enrich_monitor_groups.test.ts delete mode 100644 x-pack/plugins/uptime/server/lib/requests/search/__tests__/fetch_page.test.ts delete mode 100644 x-pack/plugins/uptime/server/lib/requests/search/__tests__/monitor_group_iterator.test.ts create mode 100644 x-pack/plugins/uptime/server/lib/requests/search/__tests__/monitor_summary_iterator.test.ts delete mode 100644 x-pack/plugins/uptime/server/lib/requests/search/__tests__/refine_potential_matches.test.ts delete mode 100644 x-pack/plugins/uptime/server/lib/requests/search/enrich_monitor_groups.ts delete mode 100644 x-pack/plugins/uptime/server/lib/requests/search/fetch_page.ts rename x-pack/plugins/uptime/server/lib/requests/search/{monitor_group_iterator.ts => monitor_summary_iterator.ts} (61%) diff --git a/x-pack/plugins/uptime/common/runtime_types/monitor/state.ts b/x-pack/plugins/uptime/common/runtime_types/monitor/state.ts index 7fad5051919c4..edbaacd725045 100644 --- a/x-pack/plugins/uptime/common/runtime_types/monitor/state.ts +++ b/x-pack/plugins/uptime/common/runtime_types/monitor/state.ts @@ -5,68 +5,9 @@ */ import * as t from 'io-ts'; - -export const CheckMonitorType = t.intersection([ - t.partial({ - name: t.string, - ip: t.union([t.array(t.union([t.string, t.null])), t.string, t.null]), - }), - t.type({ - status: t.string, - }), -]); - -export const CheckType = t.intersection([ - t.partial({ - agent: t.partial({ - id: t.string, - }), - container: t.type({ - id: t.string, - }), - kubernetes: t.type({ - pod: t.type({ - uid: t.string, - }), - }), - observer: t.type({ - geo: t.partial({ - name: t.string, - location: t.partial({ - lat: t.number, - lon: t.number, - }), - }), - }), - }), - t.type({ - monitor: CheckMonitorType, - timestamp: t.number, - }), -]); - -export type Check = t.TypeOf; +import { PingType } from '../ping/ping'; export const StateType = t.intersection([ - t.partial({ - checks: t.array(CheckType), - observer: t.partial({ - geo: t.partial({ - name: t.array(t.string), - }), - }), - summary: t.partial({ - up: t.number, - down: t.number, - geo: t.partial({ - name: t.string, - location: t.partial({ - lat: t.number, - lon: t.number, - }), - }), - }), - }), t.type({ timestamp: t.string, url: t.partial({ @@ -76,9 +17,31 @@ export const StateType = t.intersection([ port: t.number, scheme: t.string, }), + summaryPings: t.array(PingType), + summary: t.partial({ + status: t.string, + up: t.number, + down: t.number, + }), + monitor: t.partial({ + name: t.string, + }), + }), + t.partial({ + tls: t.partial({ + not_after: t.union([t.string, t.null]), + not_before: t.union([t.string, t.null]), + }), + observer: t.type({ + geo: t.type({ + name: t.array(t.string), + }), + }), }), ]); +export type MonitorSummaryState = t.TypeOf; + export const HistogramPointType = t.type({ timestamp: t.number, up: t.union([t.number, t.undefined]), @@ -105,18 +68,18 @@ export const MonitorSummaryType = t.intersection([ export type MonitorSummary = t.TypeOf; -export const MonitorSummaryResultType = t.intersection([ +export const MonitorSummariesResultType = t.intersection([ t.partial({ - summaries: t.array(MonitorSummaryType), + totalSummaryCount: t.number, }), t.type({ + summaries: t.array(MonitorSummaryType), prevPagePagination: t.union([t.string, t.null]), nextPagePagination: t.union([t.string, t.null]), - totalSummaryCount: t.number, }), ]); -export type MonitorSummaryResult = t.TypeOf; +export type MonitorSummariesResult = t.TypeOf; export const FetchMonitorStatesQueryArgsType = t.intersection([ t.partial({ diff --git a/x-pack/plugins/uptime/common/runtime_types/ping/ping.ts b/x-pack/plugins/uptime/common/runtime_types/ping/ping.ts index ab539b38c3e41..d037b4da882a1 100644 --- a/x-pack/plugins/uptime/common/runtime_types/ping/ping.ts +++ b/x-pack/plugins/uptime/common/runtime_types/ping/ping.ts @@ -16,11 +16,51 @@ export const HttpResponseBodyType = t.partial({ export type HttpResponseBody = t.TypeOf; -export const TlsType = t.partial({ +const ECSDistinguishedName = t.type({ + common_name: t.string, + distinguished_name: t.string, +}); + +export const X509ExpiryType = t.type({ not_after: t.string, not_before: t.string, }); +export type X509Expiry = t.TypeOf; + +export const X509Type = t.intersection([ + t.type({ + issuer: ECSDistinguishedName, + subject: ECSDistinguishedName, + serial_number: t.string, + public_key_algorithm: t.string, + signature_algorithm: t.string, + }), + X509ExpiryType, + t.partial({ + public_key_curve: t.string, + public_key_exponent: t.number, + public_key_size: t.number, + }), +]); + +export type X509 = t.TypeOf; + +export const TlsType = t.partial({ + // deprecated in favor of server.x509.not_after/not_before + certificate_not_valid_after: t.string, + certificate_not_valid_before: t.string, + cipher: t.string, + established: t.boolean, + server: t.partial({ + hash: t.type({ + sha256: t.string, + sha1: t.string, + }), + x509: X509Type, + }), +}); + export type Tls = t.TypeOf; export const MonitorType = t.intersection([ @@ -123,6 +163,11 @@ export const PingType = t.intersection([ observer: t.partial({ geo: t.partial({ name: t.string, + location: t.union([ + t.string, + t.partial({ lat: t.number, lon: t.number }), + t.partial({ lat: t.string, lon: t.string }), + ]), }), }), resolve: t.partial({ @@ -156,6 +201,30 @@ export const PingType = t.intersection([ export type Ping = t.TypeOf; +// Convenience function for tests etc that makes an empty ping +// object with the minimum of fields. +export const makePing = (f: { + docId?: string; + type?: string; + id?: string; + timestamp?: string; + ip?: string; + status?: string; + duration?: number; +}): Ping => { + return { + docId: f.docId || 'myDocId', + timestamp: f.timestamp || '2020-07-07T01:14:08Z', + monitor: { + id: f.id || 'myId', + type: f.type || 'myType', + ip: f.ip || '127.0.0.1', + status: f.status || 'up', + duration: { us: f.duration || 100000 }, + }, + }; +}; + export const PingsResponseType = t.type({ total: t.number, locations: t.array(t.string), diff --git a/x-pack/plugins/uptime/public/components/monitor/status_details/__test__/__snapshots__/ssl_certificate.test.tsx.snap b/x-pack/plugins/uptime/public/components/monitor/status_details/__test__/__snapshots__/ssl_certificate.test.tsx.snap index 591b3f222d033..64802ad383b79 100644 --- a/x-pack/plugins/uptime/public/components/monitor/status_details/__test__/__snapshots__/ssl_certificate.test.tsx.snap +++ b/x-pack/plugins/uptime/public/components/monitor/status_details/__test__/__snapshots__/ssl_certificate.test.tsx.snap @@ -176,7 +176,8 @@ exports[`SSL Certificate component shallow renders 1`] = ` diff --git a/x-pack/plugins/uptime/public/components/monitor/status_details/__test__/ssl_certificate.test.tsx b/x-pack/plugins/uptime/public/components/monitor/status_details/__test__/ssl_certificate.test.tsx index a4b360ea690d0..01652044e7a5b 100644 --- a/x-pack/plugins/uptime/public/components/monitor/status_details/__test__/ssl_certificate.test.tsx +++ b/x-pack/plugins/uptime/public/components/monitor/status_details/__test__/ssl_certificate.test.tsx @@ -15,12 +15,13 @@ import { DYNAMIC_SETTINGS_DEFAULTS } from '../../../../../common/constants'; describe('SSL Certificate component', () => { let monitorTls: Tls; + const dateInTwoMonths = moment().add(2, 'month').toString(); + const yesterday = moment().subtract(1, 'day').toString(); beforeEach(() => { - const dateInTwoMonths = moment().add(2, 'month').toString(); - monitorTls = { - not_after: dateInTwoMonths, + certificate_not_valid_after: dateInTwoMonths, + certificate_not_valid_before: yesterday, }; const useDispatchSpy = jest.spyOn(redux, 'useDispatch'); @@ -32,7 +33,8 @@ describe('SSL Certificate component', () => { it('shallow renders', () => { const monitorTls1 = { - not_after: '2020-04-24T11:41:38.200Z', + certificate_not_valid_after: '2020-04-24T11:41:38.200Z', + certificate_not_valid_before: '2019-04-24T11:41:38.200Z', }; const component = shallowWithRouter(); expect(component).toMatchSnapshot(); @@ -45,7 +47,8 @@ describe('SSL Certificate component', () => { it('renders null if invalid date', () => { monitorTls = { - not_after: 'i am so invalid date', + certificate_not_valid_after: 'i am so invalid date', + certificate_not_valid_before: 'i am so invalid date', }; const component = renderWithRouter(); expect(component).toMatchSnapshot(); @@ -54,7 +57,8 @@ describe('SSL Certificate component', () => { it('renders expiration date with a warning state if ssl expiry date is less than 5 days', () => { const dateIn5Days = moment().add(5, 'day').toString(); monitorTls = { - not_after: dateIn5Days, + certificate_not_valid_after: dateIn5Days, + certificate_not_valid_before: yesterday, }; const component = mountWithRouter(); @@ -69,7 +73,8 @@ describe('SSL Certificate component', () => { it('does not render the expiration date with a warning state if expiry date is greater than a month', () => { const dateIn40Days = moment().add(40, 'day').toString(); monitorTls = { - not_after: dateIn40Days, + certificate_not_valid_after: dateIn40Days, + certificate_not_valid_before: yesterday, }; const component = mountWithRouter(); diff --git a/x-pack/plugins/uptime/public/components/monitor/status_details/status_bar/ssl_certificate.tsx b/x-pack/plugins/uptime/public/components/monitor/status_details/status_bar/ssl_certificate.tsx index 93720ab313ee3..ffe4f5d759e03 100644 --- a/x-pack/plugins/uptime/public/components/monitor/status_details/status_bar/ssl_certificate.tsx +++ b/x-pack/plugins/uptime/public/components/monitor/status_details/status_bar/ssl_certificate.tsx @@ -8,7 +8,7 @@ import React from 'react'; import { Link } from 'react-router-dom'; import { EuiSpacer } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; -import { Tls } from '../../../../../common/runtime_types'; +import { Tls, X509Expiry } from '../../../../../common/runtime_types'; import { CERTIFICATES_ROUTE } from '../../../../../common/constants'; import { MonListDescription, MonListTitle } from './status_bar'; import { CertStatusColumn } from '../../../overview/monitor_list/cert_status_column'; @@ -21,7 +21,21 @@ interface Props { } export const MonitorSSLCertificate = ({ tls }: Props) => { - return tls?.not_after ? ( + let expiry: X509Expiry | null = null; + if (tls?.server?.x509) { + expiry = tls.server.x509; + } else if (tls?.certificate_not_valid_after && tls?.certificate_not_valid_before) { + expiry = { + not_after: tls.certificate_not_valid_after, + not_before: tls.certificate_not_valid_before, + }; + } + + if (!expiry) { + return null; + } + + return ( <> { - + - ) : null; + ); }; diff --git a/x-pack/plugins/uptime/public/components/overview/monitor_list/__tests__/__snapshots__/monitor_list.test.tsx.snap b/x-pack/plugins/uptime/public/components/overview/monitor_list/__tests__/__snapshots__/monitor_list.test.tsx.snap index d593dcc21b590..126b2eb9dfdf6 100644 --- a/x-pack/plugins/uptime/public/components/overview/monitor_list/__tests__/__snapshots__/monitor_list.test.tsx.snap +++ b/x-pack/plugins/uptime/public/components/overview/monitor_list/__tests__/__snapshots__/monitor_list.test.tsx.snap @@ -51,21 +51,56 @@ exports[`MonitorList component MonitorListPagination component renders a no item } } > - + > + + `; @@ -120,83 +155,150 @@ exports[`MonitorList component MonitorListPagination component renders the pagin } } > - + + pageSize={10} + setPageSize={[MockFunction]} + /> + `; @@ -251,21 +353,56 @@ exports[`MonitorList component renders a no items message when no data is provid } } > - + > + + `; @@ -320,84 +457,151 @@ exports[`MonitorList component renders error list 1`] = ` } } > - + + pageSize={10} + setPageSize={[MockFunction]} + /> + `; @@ -452,83 +656,150 @@ exports[`MonitorList component renders loading state 1`] = ` } } > - + + pageSize={10} + setPageSize={[MockFunction]} + /> + `; @@ -712,7 +983,7 @@ exports[`MonitorList component renders the monitor list 1`] = ` @@ -760,7 +1031,7 @@ exports[`MonitorList component renders the monitor list 1`] = ` @@ -861,7 +1132,7 @@ exports[`MonitorList component renders the monitor list 1`] = `
- 1897 Yr ago + 1898 Yr ago
@@ -1038,7 +1309,7 @@ exports[`MonitorList component renders the monitor list 1`] = `
- 1895 Yr ago + 1896 Yr ago
@@ -1050,7 +1321,7 @@ exports[`MonitorList component renders the monitor list 1`] = `
- in 1/1 Location + in 0/1 Location
@@ -1301,82 +1572,149 @@ exports[`MonitorList component shallow renders the monitor list 1`] = ` } } > - + + pageSize={10} + setPageSize={[MockFunction]} + /> + `; diff --git a/x-pack/plugins/uptime/public/components/overview/monitor_list/__tests__/monitor_list.test.tsx b/x-pack/plugins/uptime/public/components/overview/monitor_list/__tests__/monitor_list.test.tsx index 7d09f4161fcac..3bba1e9e894c6 100644 --- a/x-pack/plugins/uptime/public/components/overview/monitor_list/__tests__/monitor_list.test.tsx +++ b/x-pack/plugins/uptime/public/components/overview/monitor_list/__tests__/monitor_list.test.tsx @@ -6,17 +6,99 @@ import React from 'react'; import { - MonitorSummaryResult, + MonitorSummariesResult, CursorDirection, SortOrder, + makePing, + Ping, + MonitorSummary, } from '../../../../../common/runtime_types'; import { MonitorListComponent, noItemsMessage } from '../monitor_list'; import { renderWithRouter, shallowWithRouter } from '../../../../lib'; import * as redux from 'react-redux'; +const testFooPings: Ping[] = [ + makePing({ + docId: 'foo1', + id: 'foo', + type: 'icmp', + status: 'up', + duration: 123, + timestamp: '124', + ip: '127.0.0.1', + }), + makePing({ + docId: 'foo2', + id: 'foo', + type: 'icmp', + status: 'up', + duration: 123, + timestamp: '125', + ip: '127.0.0.2', + }), + makePing({ + docId: 'foo3', + id: 'foo', + type: 'icmp', + status: 'down', + duration: 123, + timestamp: '126', + ip: '127.0.0.3', + }), +]; + +const testFooSummary: MonitorSummary = { + monitor_id: 'foo', + state: { + monitor: {}, + summaryPings: testFooPings, + summary: { + up: 1, + down: 2, + }, + timestamp: '123', + url: {}, + }, +}; + +const testBarPings: Ping[] = [ + makePing({ + docId: 'bar1', + id: 'bar', + type: 'icmp', + status: 'down', + duration: 123, + timestamp: '125', + ip: '127.0.0.1', + }), + makePing({ + docId: 'bar2', + id: 'bar', + type: 'icmp', + status: 'down', + duration: 123, + timestamp: '126', + ip: '127.0.0.1', + }), +]; + +const testBarSummary: MonitorSummary = { + monitor_id: 'bar', + state: { + monitor: {}, + summaryPings: testBarPings, + summary: { + up: 2, + down: 0, + }, + timestamp: '125', + url: {}, + }, +}; + // Failing: See https://github.com/elastic/kibana/issues/70386 describe.skip('MonitorList component', () => { - let result: MonitorSummaryResult; + let result: MonitorSummariesResult; let localStorageMock: any; beforeEach(() => { @@ -36,69 +118,7 @@ describe.skip('MonitorList component', () => { result = { nextPagePagination: null, prevPagePagination: null, - summaries: [ - { - monitor_id: 'foo', - state: { - checks: [ - { - monitor: { - ip: '127.0.0.1', - status: 'up', - }, - timestamp: 124, - }, - { - monitor: { - ip: '127.0.0.2', - status: 'down', - }, - timestamp: 125, - }, - { - monitor: { - ip: '127.0.0.3', - status: 'down', - }, - timestamp: 126, - }, - ], - summary: { - up: 1, - down: 2, - }, - timestamp: '123', - url: {}, - }, - }, - { - monitor_id: 'bar', - state: { - checks: [ - { - monitor: { - ip: '127.0.0.1', - status: 'up', - }, - timestamp: 125, - }, - { - monitor: { - ip: '127.0.0.2', - status: 'up', - }, - timestamp: 126, - }, - ], - summary: { - up: 2, - down: 0, - }, - timestamp: '125', - url: {}, - }, - }, - ], + summaries: [testFooSummary, testBarSummary], totalSummaryCount: 2, }; }); @@ -171,7 +191,7 @@ describe.skip('MonitorList component', () => { }); describe('MonitorListPagination component', () => { - let paginationResult: MonitorSummaryResult; + let paginationResult: MonitorSummariesResult; beforeEach(() => { paginationResult = { @@ -185,69 +205,7 @@ describe.skip('MonitorList component', () => { cursorDirection: CursorDirection.AFTER, sortOrder: SortOrder.ASC, }), - summaries: [ - { - monitor_id: 'foo', - state: { - checks: [ - { - monitor: { - ip: '127.0.0.1', - status: 'up', - }, - timestamp: 124, - }, - { - monitor: { - ip: '127.0.0.2', - status: 'down', - }, - timestamp: 125, - }, - { - monitor: { - ip: '127.0.0.3', - status: 'down', - }, - timestamp: 126, - }, - ], - summary: { - up: 1, - down: 2, - }, - timestamp: '123', - url: {}, - }, - }, - { - monitor_id: 'bar', - state: { - checks: [ - { - monitor: { - ip: '127.0.0.1', - status: 'up', - }, - timestamp: 125, - }, - { - monitor: { - ip: '127.0.0.2', - status: 'up', - }, - timestamp: 126, - }, - ], - summary: { - up: 2, - down: 0, - }, - timestamp: '125', - url: {}, - }, - }, - ], + summaries: [testFooSummary, testBarSummary], totalSummaryCount: 2, }; }); diff --git a/x-pack/plugins/uptime/public/components/overview/monitor_list/__tests__/monitor_list_status_column.test.tsx b/x-pack/plugins/uptime/public/components/overview/monitor_list/__tests__/monitor_list_status_column.test.tsx index d765c0b33ea4b..9696ecc2c237b 100644 --- a/x-pack/plugins/uptime/public/components/overview/monitor_list/__tests__/monitor_list_status_column.test.tsx +++ b/x-pack/plugins/uptime/public/components/overview/monitor_list/__tests__/monitor_list_status_column.test.tsx @@ -8,7 +8,7 @@ import React from 'react'; import moment from 'moment'; import { renderWithIntl, shallowWithIntl } from 'test_utils/enzyme_helpers'; import { getLocationStatus, MonitorListStatusColumn } from '../monitor_list_status_column'; -import { Check } from '../../../../../common/runtime_types'; +import { Ping } from '../../../../../common/runtime_types'; import { STATUS } from '../../../../../common/constants'; describe('MonitorListStatusColumn', () => { @@ -20,19 +20,23 @@ describe('MonitorListStatusColumn', () => { Date.prototype.toString = jest.fn(() => 'Tue, 01 Jan 2019 00:00:00 GMT'); }); - let upChecks: Check[]; + let upChecks: Ping[]; - let downChecks: Check[]; + let downChecks: Ping[]; - let checks: Check[]; + let summaryPings: Ping[]; beforeEach(() => { upChecks = [ { + docId: '1', monitor: { ip: '104.86.46.103', name: '', status: 'up', + id: 'myMonitor', + type: 'icmp', + duration: { us: 123 }, }, observer: { geo: { @@ -43,13 +47,17 @@ describe('MonitorListStatusColumn', () => { }, }, }, - timestamp: 1579794631464, + timestamp: '1579794631464', }, { + docId: '2', monitor: { ip: '104.86.46.103', name: '', status: 'up', + id: 'myMonitor', + type: 'icmp', + duration: { us: 123 }, }, observer: { geo: { @@ -60,13 +68,17 @@ describe('MonitorListStatusColumn', () => { }, }, }, - timestamp: 1579794634220, + timestamp: '1579794634220', }, { + docId: '3', monitor: { ip: '104.86.46.103', name: '', status: 'up', + id: 'myMonitor', + type: 'icmp', + duration: { us: 123 }, }, observer: { geo: { @@ -77,16 +89,20 @@ describe('MonitorListStatusColumn', () => { }, }, }, - timestamp: 1579794628368, + timestamp: '1579794628368', }, ]; downChecks = [ { + docId: '4', monitor: { ip: '104.86.46.103', name: '', status: 'down', + id: 'myMonitor', + type: 'icmp', + duration: { us: 123 }, }, observer: { geo: { @@ -97,13 +113,17 @@ describe('MonitorListStatusColumn', () => { }, }, }, - timestamp: 1579794631464, + timestamp: '1579794631464', }, { + docId: '5', monitor: { ip: '104.86.46.103', name: '', status: 'down', + id: 'myMonitor', + type: 'icmp', + duration: { us: 123 }, }, observer: { geo: { @@ -114,13 +134,17 @@ describe('MonitorListStatusColumn', () => { }, }, }, - timestamp: 1579794634220, + timestamp: '1579794634220', }, { + docId: '6', monitor: { ip: '104.86.46.103', name: '', status: 'down', + id: 'myMonitor', + type: 'icmp', + duration: { us: 123 }, }, observer: { geo: { @@ -131,16 +155,20 @@ describe('MonitorListStatusColumn', () => { }, }, }, - timestamp: 1579794628368, + timestamp: '1579794628368', }, ]; - checks = [ + summaryPings = [ { + docId: '7', monitor: { ip: '104.86.46.103', name: '', status: 'up', + id: 'myMonitor', + type: 'icmp', + duration: { us: 123 }, }, observer: { geo: { @@ -151,13 +179,17 @@ describe('MonitorListStatusColumn', () => { }, }, }, - timestamp: 1579794631464, + timestamp: '1579794631464', }, { + docId: '8', monitor: { ip: '104.86.46.103', name: '', status: 'down', + id: 'myMonitor', + type: 'icmp', + duration: { us: 123 }, }, observer: { geo: { @@ -168,13 +200,17 @@ describe('MonitorListStatusColumn', () => { }, }, }, - timestamp: 1579794634220, + timestamp: '1579794634220', }, { + docId: '9', monitor: { ip: '104.86.46.103', name: '', status: 'down', + id: 'myMonitor', + type: 'icmp', + duration: { us: 123 }, }, observer: { geo: { @@ -185,45 +221,53 @@ describe('MonitorListStatusColumn', () => { }, }, }, - timestamp: 1579794628368, + timestamp: '1579794628368', }, ]; }); it('provides expected tooltip and display times', () => { const component = shallowWithIntl( - + ); expect(component).toMatchSnapshot(); }); it('can handle a non-numeric timestamp value', () => { const component = shallowWithIntl( - + ); expect(component).toMatchSnapshot(); }); it('will display location status', () => { const component = shallowWithIntl( - + ); expect(component).toMatchSnapshot(); }); it('will render display location status', () => { const component = renderWithIntl( - + ); expect(component).toMatchSnapshot(); }); it(' will test getLocationStatus location', () => { - let statusMessage = getLocationStatus(checks, STATUS.UP); + let statusMessage = getLocationStatus(summaryPings, STATUS.UP); expect(statusMessage).toBe('in 1/3 Locations'); - statusMessage = getLocationStatus(checks, STATUS.DOWN); + statusMessage = getLocationStatus(summaryPings, STATUS.DOWN); expect(statusMessage).toBe('in 2/3 Locations'); diff --git a/x-pack/plugins/uptime/public/components/overview/monitor_list/cert_status_column.tsx b/x-pack/plugins/uptime/public/components/overview/monitor_list/cert_status_column.tsx index b7c70198912fc..318e18385ba1f 100644 --- a/x-pack/plugins/uptime/public/components/overview/monitor_list/cert_status_column.tsx +++ b/x-pack/plugins/uptime/public/components/overview/monitor_list/cert_status_column.tsx @@ -8,13 +8,13 @@ import React from 'react'; import moment from 'moment'; import styled from 'styled-components'; import { EuiIcon, EuiText, EuiToolTip } from '@elastic/eui'; -import { Cert, Tls } from '../../../../common/runtime_types'; +import { X509Expiry } from '../../../../common/runtime_types'; import { useCertStatus } from '../../../hooks'; import { EXPIRED, EXPIRES, EXPIRES_SOON } from '../../certificates/translations'; import { CERT_STATUS } from '../../../../common/constants'; interface Props { - cert: Cert | Tls; + expiry: X509Expiry; boldStyle?: boolean; } @@ -31,14 +31,15 @@ const H4Text = styled.h4` } `; -export const CertStatusColumn: React.FC = ({ cert, boldStyle = false }) => { - const certStatus = useCertStatus(cert?.not_after); +export const CertStatusColumn: React.FC = ({ expiry, boldStyle = false }) => { + const notAfter = expiry?.not_after; + const certStatus = useCertStatus(notAfter); - const relativeDate = moment(cert?.not_after).fromNow(); + const relativeDate = moment(notAfter).fromNow(); const CertStatus = ({ color, text }: { color: string; text: string }) => { return ( - + {boldStyle ? ( diff --git a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list.tsx b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list.tsx index 75d587579f66f..ce4c518d82255 100644 --- a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list.tsx +++ b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list.tsx @@ -16,7 +16,7 @@ import { } from '@elastic/eui'; import React, { useState } from 'react'; import styled from 'styled-components'; -import { HistogramPoint } from '../../../../common/runtime_types'; +import { HistogramPoint, X509Expiry } from '../../../../common/runtime_types'; import { MonitorSummary } from '../../../../common/runtime_types'; import { MonitorListStatusColumn } from './monitor_list_status_column'; import { ExpandedRowMap } from './types'; @@ -79,14 +79,18 @@ export const MonitorListComponent: React.FC = ({ const columns = [ { align: 'left' as const, - field: 'state.monitor.status', + field: 'state.summary.status', name: labels.STATUS_COLUMN_LABEL, mobileOptions: { fullWidth: true, }, - render: (status: string, { state: { timestamp, checks } }: MonitorSummary) => { + render: (status: string, { state: { timestamp, summaryPings } }: MonitorSummary) => { return ( - + ); }, }, @@ -116,9 +120,9 @@ export const MonitorListComponent: React.FC = ({ }, { align: 'left' as const, - field: 'state.tls', + field: 'state.tls.server.x509', name: labels.TLS_COLUMN_LABEL, - render: (tls: any) => , + render: (x509: X509Expiry) => , }, { align: 'center' as const, diff --git a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/__tests__/__snapshots__/monitor_list_drawer.test.tsx.snap b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/__tests__/__snapshots__/monitor_list_drawer.test.tsx.snap index 877ba889ec1ff..329fb8bade106 100644 --- a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/__tests__/__snapshots__/monitor_list_drawer.test.tsx.snap +++ b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/__tests__/__snapshots__/monitor_list_drawer.test.tsx.snap @@ -99,33 +99,65 @@ exports[`MonitorListDrawer component renders a MonitorListDrawer when there are Object { "monitor_id": "foo", "state": Object { - "checks": Array [ + "monitor": Object {}, + "summary": Object { + "down": 0, + "up": 1, + }, + "summaryPings": Array [ Object { + "docId": "foo", "monitor": Object { + "duration": Object { + "us": 121, + }, + "id": "foo", "ip": "127.0.0.1", "status": "up", + "type": "icmp", }, - "timestamp": 121, + "timestamp": "121", }, Object { + "docId": "foo-0", "monitor": Object { - "ip": "127.0.0.2", + "duration": Object { + "us": 100000, + }, + "id": "foo", + "ip": "127.0.0.1", "status": "down", + "type": "icmp", }, - "timestamp": 123, + "timestamp": "0", }, Object { + "docId": "foo-1", "monitor": Object { - "ip": "127.0.0.3", + "duration": Object { + "us": 1, + }, + "id": "foo", + "ip": "127.0.0.2", "status": "up", + "type": "icmp", + }, + "timestamp": "1", + }, + Object { + "docId": "foo-2", + "monitor": Object { + "duration": Object { + "us": 2, + }, + "id": "foo", + "ip": "127.0.0.3", + "status": "down", + "type": "icmp", }, - "timestamp": 125, + "timestamp": "2", }, ], - "summary": Object { - "down": 0, - "up": 1, - }, "timestamp": "123", "url": Object { "domain": "expired.badssl.com", @@ -238,19 +270,26 @@ exports[`MonitorListDrawer component renders a MonitorListDrawer when there is o Object { "monitor_id": "foo", "state": Object { - "checks": Array [ + "monitor": Object {}, + "summary": Object { + "down": 0, + "up": 1, + }, + "summaryPings": Array [ Object { + "docId": "foo", "monitor": Object { + "duration": Object { + "us": 121, + }, + "id": "foo", "ip": "127.0.0.1", "status": "up", + "type": "icmp", }, - "timestamp": 121, + "timestamp": "121", }, ], - "summary": Object { - "down": 0, - "up": 1, - }, "timestamp": "123", "url": Object { "domain": "expired.badssl.com", diff --git a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/__tests__/data.json b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/__tests__/data.json index e8142f0480c4a..012280c8147dd 100644 --- a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/__tests__/data.json +++ b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/__tests__/data.json @@ -615,7 +615,7 @@ "__typename": "MonitorSummary" } ], - "__typename": "MonitorSummaryResult" + "__typename": "MonitorSummariesResult" } } } diff --git a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/__tests__/integration_group.test.tsx b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/__tests__/integration_group.test.tsx index 1c587568fe61d..df9f1febd40c3 100644 --- a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/__tests__/integration_group.test.tsx +++ b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/__tests__/integration_group.test.tsx @@ -5,7 +5,7 @@ */ import React from 'react'; -import { MonitorSummary } from '../../../../../../common/runtime_types'; +import { MonitorSummary, makePing } from '../../../../../../common/runtime_types'; import { shallowWithIntl } from 'test_utils/enzyme_helpers'; import { IntegrationGroup, extractSummaryValues } from '../actions_popover/integration_group'; @@ -17,7 +17,8 @@ describe('IntegrationGroup', () => { monitor_id: '12345', state: { summary: {}, - checks: [], + monitor: {}, + summaryPings: [], timestamp: '123', url: {}, }, @@ -46,6 +47,12 @@ describe('IntegrationGroup', () => { mockSummary = { state: { timestamp: 'foo', + summaryPings: [], + monitor: {}, + summary: { + up: 0, + down: 0, + }, url: {}, }, }; @@ -76,22 +83,25 @@ describe('IntegrationGroup', () => { }); it('finds pod uid', () => { - mockSummary.state.checks = [ - { kubernetes: { pod: { uid: 'myuid' } }, monitor: { status: 'up' }, timestamp: 123 }, + mockSummary.state.summaryPings = [ + { + ...makePing({}), + kubernetes: { pod: { uid: 'myuid' } }, + }, ]; expect(extractSummaryValues(mockSummary)).toMatchInlineSnapshot(` Object { "containerId": undefined, "domain": "", - "ip": undefined, + "ip": "127.0.0.1", "podUid": "myuid", } `); }); it('does not throw for missing kubernetes fields', () => { - mockSummary.state.checks = []; + mockSummary.state.summaryPings = []; expect(extractSummaryValues(mockSummary)).toMatchInlineSnapshot(` Object { @@ -104,22 +114,25 @@ describe('IntegrationGroup', () => { }); it('finds container id', () => { - mockSummary.state.checks = [ - { container: { id: 'mycontainer' }, monitor: { status: 'up' }, timestamp: 123 }, + mockSummary.state.summaryPings = [ + { + ...makePing({}), + container: { id: 'mycontainer' }, + }, ]; expect(extractSummaryValues(mockSummary)).toMatchInlineSnapshot(` Object { "containerId": "mycontainer", "domain": "", - "ip": undefined, + "ip": "127.0.0.1", "podUid": undefined, } `); }); it('finds ip field', () => { - mockSummary.state.checks = [{ monitor: { ip: '127.0.0.1', status: 'up' }, timestamp: 123 }]; + mockSummary.state.summaryPings = [makePing({ ip: '127.0.0.1', status: 'up' })]; expect(extractSummaryValues(mockSummary)).toMatchInlineSnapshot(` Object { diff --git a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/__tests__/monitor_list_drawer.test.tsx b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/__tests__/monitor_list_drawer.test.tsx index 4bc0c3f0a40ba..502ccd53ef80c 100644 --- a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/__tests__/monitor_list_drawer.test.tsx +++ b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/__tests__/monitor_list_drawer.test.tsx @@ -6,7 +6,7 @@ import 'jest'; import React from 'react'; import { MonitorListDrawerComponent } from '../monitor_list_drawer'; -import { Check, MonitorDetails, MonitorSummary } from '../../../../../../common/runtime_types'; +import { MonitorDetails, MonitorSummary, makePing } from '../../../../../../common/runtime_types'; import { shallowWithRouter } from '../../../../../lib'; describe('MonitorListDrawer component', () => { @@ -17,14 +17,17 @@ describe('MonitorListDrawer component', () => { summary = { monitor_id: 'foo', state: { - checks: [ - { - monitor: { - ip: '127.0.0.1', - status: 'up', - }, - timestamp: 121, - }, + monitor: {}, + summaryPings: [ + makePing({ + docId: 'foo', + id: 'foo', + ip: '127.0.0.1', + type: 'icmp', + status: 'up', + timestamp: '121', + duration: 121, + }), ], summary: { up: 1, @@ -55,7 +58,7 @@ describe('MonitorListDrawer component', () => { }); it('renders nothing when no check data is present', () => { - delete summary.state.checks; + delete summary.state.summaryPings; const component = shallowWithRouter( ); @@ -70,30 +73,20 @@ describe('MonitorListDrawer component', () => { }); it('renders a MonitorListDrawer when there are many checks', () => { - const checks: Check[] = [ - { - monitor: { - ip: '127.0.0.1', - status: 'up', - }, - timestamp: 121, - }, - { - monitor: { - ip: '127.0.0.2', - status: 'down', - }, - timestamp: 123, - }, - { - monitor: { - ip: '127.0.0.3', - status: 'up', - }, - timestamp: 125, - }, - ]; - summary.state.checks = checks; + for (let i = 0; i < 3; i++) { + summary.state.summaryPings.push( + makePing({ + docId: `foo-${i}`, + id: 'foo', + ip: `127.0.0.${1 + i}`, + type: 'icmp', + timestamp: `${i}`, + duration: i, + status: i % 2 !== 0 ? 'up' : 'down', + }) + ); + } + const component = shallowWithRouter( ); diff --git a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/__tests__/monitor_status_list.test.tsx b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/__tests__/monitor_status_list.test.tsx index c7f3aef4075e1..c0143828dc69d 100644 --- a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/__tests__/monitor_status_list.test.tsx +++ b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/__tests__/monitor_status_list.test.tsx @@ -8,10 +8,10 @@ import { shallowWithIntl } from 'test_utils/enzyme_helpers'; import React from 'react'; import moment from 'moment'; import { MonitorStatusList } from '../monitor_status_list'; -import { Check } from '../../../../../../common/runtime_types'; +import { Ping } from '../../../../../../common/runtime_types'; describe('MonitorStatusList component', () => { - let checks: Check[]; + let pings: Ping[]; beforeAll(() => { moment.prototype.toLocaleString = jest.fn(() => '2019-06-21 15:29:26'); @@ -19,105 +19,137 @@ describe('MonitorStatusList component', () => { }); beforeEach(() => { - checks = [ + pings = [ { + docId: '1', monitor: { ip: '151.101.130.217', name: 'elastic', status: 'up', + id: 'myMonitor', + type: 'icmp', + duration: { us: 123 }, }, observer: { geo: {}, }, - timestamp: 1570538236414, + timestamp: '1570538236414', }, { + docId: '2', monitor: { ip: '151.101.194.217', name: 'elastic', status: 'up', + id: 'myMonitor', + type: 'icmp', + duration: { us: 123 }, }, observer: { geo: {}, }, - timestamp: 1570538236414, + timestamp: '1570538236414', }, { + docId: '3', monitor: { ip: '151.101.2.217', name: 'elastic', status: 'up', + id: 'myMonitor', + type: 'icmp', + duration: { us: 123 }, }, observer: { geo: {}, }, - timestamp: 1570538236414, + timestamp: '1570538236414', }, { + docId: '4', monitor: { ip: '151.101.66.217', name: 'elastic', status: 'up', + id: 'myMonitor', + type: 'icmp', + duration: { us: 123 }, }, observer: { geo: {}, }, - timestamp: 1570538236414, + timestamp: '1570538236414', }, { + docId: '4', monitor: { ip: '2a04:4e42:200::729', name: 'elastic', status: 'down', + id: 'myMonitor', + type: 'icmp', + duration: { us: 123 }, }, observer: { geo: {}, }, - timestamp: 1570538236414, + timestamp: '1570538236414', }, { + docId: '5', monitor: { ip: '2a04:4e42:400::729', name: 'elastic', status: 'down', + id: 'myMonitor', + type: 'icmp', + duration: { us: 123 }, }, observer: { geo: {}, }, - timestamp: 1570538236414, + timestamp: '1570538236414', }, { + docId: '6', monitor: { ip: '2a04:4e42:600::729', name: 'elastic', status: 'down', + id: 'myMonitor', + type: 'icmp', + duration: { us: 123 }, }, observer: { geo: {}, }, - timestamp: 1570538236414, + timestamp: '1570538236414', }, { + docId: '5', monitor: { ip: '2a04:4e42::729', name: 'elastic', status: 'down', + id: 'myMonitor', + type: 'icmp', + duration: { us: 123 }, }, observer: { geo: {}, }, - timestamp: 1570538236414, + timestamp: '1570538236414', }, ]; }); it('renders checks', () => { - const component = shallowWithIntl(); + const component = shallowWithIntl(); expect(component).toMatchSnapshot(); }); it('renders null in place of child status with missing ip', () => { - const component = shallowWithIntl(); + const component = shallowWithIntl(); expect(component).toMatchSnapshot(); }); }); diff --git a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/actions_popover/integration_group.tsx b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/actions_popover/integration_group.tsx index 55a99ab8541f8..38aa9287b0c47 100644 --- a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/actions_popover/integration_group.tsx +++ b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/actions_popover/integration_group.tsx @@ -27,9 +27,9 @@ interface IntegrationGroupProps { export const extractSummaryValues = (summary: Pick) => { const domain = summary.state.url?.domain ?? ''; - const podUid = summary.state.checks?.[0]?.kubernetes?.pod.uid ?? undefined; - const containerId = summary.state.checks?.[0]?.container?.id ?? undefined; - const ip = summary.state.checks?.[0]?.monitor.ip ?? undefined; + const podUid = summary.state.summaryPings?.[0]?.kubernetes?.pod?.uid ?? undefined; + const containerId = summary.state.summaryPings?.[0]?.container?.id ?? undefined; + const ip = summary.state.summaryPings?.[0]?.monitor.ip ?? undefined; return { domain, diff --git a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/monitor_list_drawer.tsx b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/monitor_list_drawer.tsx index 8e97ce4d692d7..305455c8ba573 100644 --- a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/monitor_list_drawer.tsx +++ b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/monitor_list_drawer.tsx @@ -36,7 +36,7 @@ interface MonitorListDrawerProps { export function MonitorListDrawerComponent({ summary, monitorDetails }: MonitorListDrawerProps) { const monitorUrl = summary?.state?.url?.full || ''; - return summary && summary.state.checks ? ( + return summary && summary.state.summaryPings ? ( @@ -52,7 +52,7 @@ export function MonitorListDrawerComponent({ summary, monitorDetails }: MonitorL - + {monitorDetails && monitorDetails.error && ( { +export const MonitorStatusList = ({ summaryPings }: MonitorStatusListProps) => { const upChecks: Set = new Set(); const downChecks: Set = new Set(); - checks.forEach((check: Check) => { + summaryPings.forEach((ping: Ping) => { // Doing this way because name is either string or null, get() default value only works on undefined value - const location = get(check, 'observer.geo.name', null) || UNNAMED_LOCATION; + const location = ping.observer?.geo?.name ?? UNNAMED_LOCATION; - if (check.monitor.status === STATUS.UP) { + if (ping.monitor.status === STATUS.UP) { upChecks.add(upperFirst(location)); - } else if (check.monitor.status === STATUS.DOWN) { + } else if (ping.monitor.status === STATUS.DOWN) { downChecks.add(upperFirst(location)); } }); diff --git a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_status_column.tsx b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_status_column.tsx index f80c73dcf5bb0..68ddf512e4d3c 100644 --- a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_status_column.tsx +++ b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_status_column.tsx @@ -11,7 +11,7 @@ import { upperFirst } from 'lodash'; import styled from 'styled-components'; import { EuiHealth, EuiFlexGroup, EuiFlexItem, EuiText, EuiToolTip } from '@elastic/eui'; import { parseTimestamp } from './parse_timestamp'; -import { Check } from '../../../../common/runtime_types'; +import { Ping } from '../../../../common/runtime_types'; import { STATUS, SHORT_TIMESPAN_LOCALE, @@ -24,7 +24,7 @@ import * as labels from './translations'; interface MonitorListStatusColumnProps { status: string; timestamp: string; - checks: Check[]; + summaryPings: Ping[]; } const PaddedSpan = styled.span` @@ -75,27 +75,27 @@ const getRelativeShortTimeStamp = (timeStamp: any) => { return shortTimestamp; }; -export const getLocationStatus = (checks: Check[], status: string) => { - const upChecks: Set = new Set(); - const downChecks: Set = new Set(); +export const getLocationStatus = (summaryPings: Ping[], status: string) => { + const upPings: Set = new Set(); + const downPings: Set = new Set(); - checks.forEach((check: Check) => { - const location = check?.observer?.geo?.name ?? UNNAMED_LOCATION; + summaryPings.forEach((summaryPing: Ping) => { + const location = summaryPing?.observer?.geo?.name ?? UNNAMED_LOCATION; - if (check.monitor.status === STATUS.UP) { - upChecks.add(upperFirst(location)); - } else if (check.monitor.status === STATUS.DOWN) { - downChecks.add(upperFirst(location)); + if (summaryPing.monitor.status === STATUS.UP) { + upPings.add(upperFirst(location)); + } else if (summaryPing.monitor.status === STATUS.DOWN) { + downPings.add(upperFirst(location)); } }); // if monitor is down in one dns, it will be considered down so removing it from up list - const absUpChecks: Set = new Set([...upChecks].filter((item) => !downChecks.has(item))); + const absUpChecks: Set = new Set([...upPings].filter((item) => !downPings.has(item))); - const totalLocations = absUpChecks.size + downChecks.size; + const totalLocations = absUpChecks.size + downPings.size; let statusMessage = ''; if (status === STATUS.DOWN) { - statusMessage = `${downChecks.size}/${totalLocations}`; + statusMessage = `${downPings.size}/${totalLocations}`; } else { statusMessage = `${absUpChecks.size}/${totalLocations}`; } @@ -115,7 +115,7 @@ export const getLocationStatus = (checks: Check[], status: string) => { export const MonitorListStatusColumn = ({ status, - checks = [], + summaryPings = [], timestamp: tsString, }: MonitorListStatusColumnProps) => { const timestamp = parseTimestamp(tsString); @@ -140,7 +140,7 @@ export const MonitorListStatusColumn = ({ - {getLocationStatus(checks, status)} + {getLocationStatus(summaryPings, status)} ); diff --git a/x-pack/plugins/uptime/public/lib/helper/observability_integration/__tests__/get_apm_href.test.ts b/x-pack/plugins/uptime/public/lib/helper/observability_integration/__tests__/get_apm_href.test.ts index f27ed78d593ac..8e320a8d9533a 100644 --- a/x-pack/plugins/uptime/public/lib/helper/observability_integration/__tests__/get_apm_href.test.ts +++ b/x-pack/plugins/uptime/public/lib/helper/observability_integration/__tests__/get_apm_href.test.ts @@ -5,7 +5,7 @@ */ import { getApmHref } from '../get_apm_href'; -import { MonitorSummary } from '../../../../../common/runtime_types'; +import { MonitorSummary, makePing } from '../../../../../common/runtime_types'; describe('getApmHref', () => { let summary: MonitorSummary; @@ -15,22 +15,17 @@ describe('getApmHref', () => { monitor_id: 'foo', state: { summary: {}, - checks: [ - { - monitor: { - ip: '151.101.202.217', - status: 'up', - }, - container: { - id: 'test-container-id', - }, - kubernetes: { - pod: { - uid: 'test-pod-id', - }, - }, - timestamp: 123, - }, + monitor: {}, + summaryPings: [ + makePing({ + docId: 'foo', + id: 'foo', + type: 'test', + ip: '151.101.202.217', + status: 'up', + timestamp: '123', + duration: 123, + }), ], timestamp: '123', url: { diff --git a/x-pack/plugins/uptime/public/lib/helper/observability_integration/__tests__/get_infra_href.test.ts b/x-pack/plugins/uptime/public/lib/helper/observability_integration/__tests__/get_infra_href.test.ts index ee5db74af22c2..befa11d74df4a 100644 --- a/x-pack/plugins/uptime/public/lib/helper/observability_integration/__tests__/get_infra_href.test.ts +++ b/x-pack/plugins/uptime/public/lib/helper/observability_integration/__tests__/get_infra_href.test.ts @@ -5,33 +5,32 @@ */ import { getInfraContainerHref, getInfraKubernetesHref, getInfraIpHref } from '../get_infra_href'; -import { MonitorSummary } from '../../../../../common/runtime_types'; +import { MonitorSummary, makePing, Ping } from '../../../../../common/runtime_types'; describe('getInfraHref', () => { let summary: MonitorSummary; beforeEach(() => { + const ping: Ping = { + ...makePing({ + docId: 'myDocId', + type: 'test', + id: 'myId', + ip: '151.101.202.217', + status: 'up', + duration: 123, + timestamp: '123', + }), + container: { id: 'test-container-id' }, + kubernetes: { pod: { uid: 'test-pod-uid' } }, + }; + summary = { monitor_id: 'foo', state: { - checks: [ - { - monitor: { - ip: '151.101.202.217', - status: 'up', - }, - container: { - id: 'test-container-id', - }, - kubernetes: { - pod: { - uid: 'test-pod-uid', - }, - }, - timestamp: 123, - }, - ], + summaryPings: [ping], summary: {}, url: {}, + monitor: {}, timestamp: '123', }, }; @@ -51,50 +50,36 @@ describe('getInfraHref', () => { }); it('getInfraContainerHref returns undefined when no container id is present', () => { - summary.state.checks = []; + summary.state.summaryPings = []; expect(getInfraContainerHref(summary, 'foo')).toBeUndefined(); }); it('getInfraContainerHref returns the first item when multiple container ids are supplied', () => { - summary.state.checks = [ - { - monitor: { - ip: '151.101.202.217', - status: 'up', - }, - container: { - id: 'test-container-id', - }, - kubernetes: { - pod: { - uid: 'test-pod-uid', - }, - }, - timestamp: 123, - }, - { - monitor: { - ip: '151.101.202.27', - status: 'up', - }, - container: { - id: 'test-container-id-foo', - }, - kubernetes: { - pod: { - uid: 'test-pod-uid-bar', - }, - }, - timestamp: 123, - }, - ]; + const pingBase = makePing({ + docId: 'myDocId', + type: 'test', + id: 'myId', + ip: '151.101.202.217', + status: 'up', + duration: 123, + timestamp: '123', + }); + const pingTestContainerId: Ping = { + ...pingBase, + container: { id: 'test-container-id' }, + }; + const pingTestFooContainerId: Ping = { + ...pingBase, + container: { id: 'test-container-id-foo' }, + }; + summary.state.summaryPings = [pingTestContainerId, pingTestFooContainerId]; expect(getInfraContainerHref(summary, 'bar')).toMatchInlineSnapshot( `"bar/app/metrics/link-to/container-detail/test-container-id"` ); }); - it('getInfraContainerHref returns undefined when checks are undefined', () => { - delete summary.state.checks; + it('getInfraContainerHref returns undefined when summaryPings are undefined', () => { + delete summary.state.summaryPings; expect(getInfraContainerHref(summary, '')).toBeUndefined(); }); @@ -111,55 +96,41 @@ describe('getInfraHref', () => { }); it('getInfraKubernetesHref returns undefined when no pod data is present', () => { - summary.state.checks = []; + summary.state.summaryPings = []; expect(getInfraKubernetesHref(summary, 'foo')).toBeUndefined(); }); it('getInfraKubernetesHref selects the first pod uid when there are multiple', () => { - summary.state.checks = [ - { - monitor: { - ip: '151.101.202.217', - status: 'up', - }, - container: { - id: 'test-container-id', - }, - kubernetes: { - pod: { - uid: 'test-pod-uid', - }, - }, - timestamp: 123, - }, - { - monitor: { - ip: '151.101.202.27', - status: 'up', - }, - container: { - id: 'test-container-id-foo', - }, - kubernetes: { - pod: { - uid: 'test-pod-uid-bar', - }, - }, - timestamp: 123, - }, - ]; + const pingBase = makePing({ + docId: 'myDocId', + type: 'test', + id: 'myId', + ip: '151.101.202.217', + status: 'up', + duration: 123, + timestamp: '123', + }); + const pingTestPodId: Ping = { + ...pingBase, + kubernetes: { pod: { uid: 'test-pod-uid' } }, + }; + const pingTestBarPodId: Ping = { + ...pingBase, + kubernetes: { pod: { uid: 'test-pod-uid-bar' } }, + }; + summary.state.summaryPings = [pingTestPodId, pingTestBarPodId]; expect(getInfraKubernetesHref(summary, '')).toMatchInlineSnapshot( `"/app/metrics/link-to/pod-detail/test-pod-uid"` ); }); - it('getInfraKubernetesHref returns undefined when checks are undefined', () => { - delete summary.state.checks; + it('getInfraKubernetesHref returns undefined when summaryPings are undefined', () => { + delete summary.state.summaryPings; expect(getInfraKubernetesHref(summary, '')).toBeUndefined(); }); - it('getInfraKubernetesHref returns undefined when checks are null', () => { - delete summary.state.checks![0]!.kubernetes!.pod!.uid; + it('getInfraKubernetesHref returns undefined when summaryPings are null', () => { + delete summary.state.summaryPings![0]!.kubernetes!.pod!.uid; expect(getInfraKubernetesHref(summary, '')).toBeUndefined(); }); @@ -177,47 +148,42 @@ describe('getInfraHref', () => { }); it('getInfraIpHref returns undefined when ip is undefined', () => { - summary.state.checks = []; + summary.state.summaryPings = []; expect(getInfraIpHref(summary, 'foo')).toBeUndefined(); }); it('getInfraIpHref returns undefined when ip is null', () => { - delete summary.state.checks![0].monitor.ip; + delete summary.state.summaryPings![0].monitor.ip; expect(getInfraIpHref(summary, 'foo')).toBeUndefined(); }); it('getInfraIpHref returns a url for ors between multiple ips', () => { - summary.state.checks = [ - { - timestamp: 123, - monitor: { - ip: '152.151.23.192', - status: 'up', - }, - }, - { - monitor: { - ip: '151.101.202.217', - status: 'up', - }, - container: { - id: 'test-container-id', - }, - kubernetes: { - pod: { - uid: 'test-pod-uid', - }, - }, - timestamp: 123, - }, - ]; + const pingOne = makePing({ + docId: 'myDocId', + type: 'test', + id: 'myId', + ip: '152.151.23.192', + status: 'up', + duration: 123, + timestamp: '123', + }); + const pingTwo = makePing({ + docId: 'myDocId2', + type: 'test', + id: 'myId', + ip: '151.101.202.217', + status: 'up', + duration: 123, + timestamp: '123', + }); + summary.state.summaryPings = [pingOne, pingTwo]; expect(getInfraIpHref(summary, 'foo')).toMatchInlineSnapshot( `"foo/app/metrics/inventory?waffleFilter=(expression:'host.ip%20%3A%20152.151.23.192%20or%20host.ip%20%3A%20151.101.202.217',kind:kuery)"` ); }); - it('getInfraIpHref returns undefined if checks are undefined', () => { - delete summary.state.checks; + it('getInfraIpHref returns undefined if summaryPings are undefined', () => { + delete summary.state.summaryPings; expect(getInfraIpHref(summary, 'foo')).toBeUndefined(); }); }); diff --git a/x-pack/plugins/uptime/public/lib/helper/observability_integration/__tests__/get_logging_href.test.ts b/x-pack/plugins/uptime/public/lib/helper/observability_integration/__tests__/get_logging_href.test.ts index b188a8d1b8ef6..203d5b84c433a 100644 --- a/x-pack/plugins/uptime/public/lib/helper/observability_integration/__tests__/get_logging_href.test.ts +++ b/x-pack/plugins/uptime/public/lib/helper/observability_integration/__tests__/get_logging_href.test.ts @@ -9,34 +9,36 @@ import { getLoggingKubernetesHref, getLoggingIpHref, } from '../get_logging_href'; -import { MonitorSummary } from '../../../../../common/runtime_types'; +import { MonitorSummary, makePing } from '../../../../../common/runtime_types'; describe('getLoggingHref', () => { let summary: MonitorSummary; beforeEach(() => { + const ping = makePing({ + docId: 'myDocId', + type: 'test', + id: 'myId', + ip: '151.101.202.217', + status: 'up', + duration: 123, + timestamp: '123', + }); + ping.container = { + id: 'test-container-id', + }; + ping.kubernetes = { + pod: { + uid: 'test-pod-id', + }, + }; summary = { monitor_id: 'foo', state: { summary: {}, - checks: [ - { - monitor: { - ip: '151.101.202.217', - status: 'up', - }, - container: { - id: 'test-container-id', - }, - kubernetes: { - pod: { - uid: 'test-pod-id', - }, - }, - timestamp: 123, - }, - ], + summaryPings: [ping], timestamp: '123', + monitor: {}, url: {}, }, }; @@ -91,32 +93,32 @@ describe('getLoggingHref', () => { }); it('returns undefined if necessary container is not present', () => { - delete summary.state.checks; + delete summary.state.summaryPings; expect(getLoggingContainerHref(summary, '')).toBeUndefined(); }); it('returns undefined if necessary container is null', () => { - delete summary.state.checks![0].container!.id; + delete summary.state.summaryPings![0].container!.id; expect(getLoggingContainerHref(summary, '')).toBeUndefined(); }); it('returns undefined if necessary pod is not present', () => { - delete summary.state.checks; + delete summary.state.summaryPings; expect(getLoggingKubernetesHref(summary, '')).toBeUndefined(); }); it('returns undefined if necessary pod is null', () => { - delete summary.state.checks![0].kubernetes!.pod!.uid; + delete summary.state.summaryPings![0].kubernetes!.pod!.uid; expect(getLoggingKubernetesHref(summary, '')).toBeUndefined(); }); it('returns undefined ip href if ip is not present', () => { - delete summary.state.checks; + delete summary.state.summaryPings; expect(getLoggingIpHref(summary, '')).toBeUndefined(); }); it('returns undefined ip href if ip is null', () => { - delete summary.state.checks![0].monitor.ip; + delete summary.state.summaryPings![0].monitor.ip; expect(getLoggingIpHref(summary, '')).toBeUndefined(); }); }); diff --git a/x-pack/plugins/uptime/public/lib/helper/observability_integration/build_href.ts b/x-pack/plugins/uptime/public/lib/helper/observability_integration/build_href.ts index 397d23a18332c..94383262b0acd 100644 --- a/x-pack/plugins/uptime/public/lib/helper/observability_integration/build_href.ts +++ b/x-pack/plugins/uptime/public/lib/helper/observability_integration/build_href.ts @@ -5,7 +5,7 @@ */ import { get } from 'lodash'; -import { Check } from '../../../../common/runtime_types'; +import { Ping } from '../../../../common/runtime_types'; /** * Builds URLs to the designated features by extracting values from the provided @@ -16,12 +16,12 @@ import { Check } from '../../../../common/runtime_types'; * @param getHref a function that returns the full URL */ export const buildHref = ( - checks: Check[], + summaryPings: Ping[], path: string, getHref: (value: string | string[] | undefined) => string | undefined ): string | undefined => { - const queryValue = checks - .map((check) => get(check, path, undefined)) + const queryValue = summaryPings + .map((ping) => get(ping, path, undefined)) .filter((value: string | undefined) => value !== undefined); if (queryValue.length === 0) { return getHref(undefined); diff --git a/x-pack/plugins/uptime/public/lib/helper/observability_integration/get_infra_href.ts b/x-pack/plugins/uptime/public/lib/helper/observability_integration/get_infra_href.ts index 384067e4b033b..33d24a0f081b4 100644 --- a/x-pack/plugins/uptime/public/lib/helper/observability_integration/get_infra_href.ts +++ b/x-pack/plugins/uptime/public/lib/helper/observability_integration/get_infra_href.ts @@ -22,7 +22,7 @@ export const getInfraContainerHref = ( `/app/metrics/link-to/container-detail/${encodeURIComponent(ret)}` ); }; - return buildHref(summary.state.checks || [], 'container.id', getHref); + return buildHref(summary.state.summaryPings || [], 'container.id', getHref); }; export const getInfraKubernetesHref = ( @@ -37,7 +37,7 @@ export const getInfraKubernetesHref = ( return addBasePath(basePath, `/app/metrics/link-to/pod-detail/${encodeURIComponent(ret)}`); }; - return buildHref(summary.state.checks || [], 'kubernetes.pod.uid', getHref); + return buildHref(summary.state.summaryPings || [], 'kubernetes.pod.uid', getHref); }; export const getInfraIpHref = (summary: MonitorSummary, basePath: string) => { @@ -63,5 +63,5 @@ export const getInfraIpHref = (summary: MonitorSummary, basePath: string) => { `/app/metrics/inventory?waffleFilter=(expression:'${encodeURIComponent(ips)}',kind:kuery)` ); }; - return buildHref(summary.state.checks || [], 'monitor.ip', getHref); + return buildHref(summary.state.summaryPings || [], 'monitor.ip', getHref); }; diff --git a/x-pack/plugins/uptime/public/lib/helper/observability_integration/get_logging_href.ts b/x-pack/plugins/uptime/public/lib/helper/observability_integration/get_logging_href.ts index 222c7b57c9272..c4fee330e9763 100644 --- a/x-pack/plugins/uptime/public/lib/helper/observability_integration/get_logging_href.ts +++ b/x-pack/plugins/uptime/public/lib/helper/observability_integration/get_logging_href.ts @@ -22,7 +22,7 @@ export const getLoggingContainerHref = ( `/app/logs?logFilter=${encodeURI(`(expression:'container.id : ${ret}',kind:kuery)`)}` ); }; - return buildHref(summary.state.checks || [], 'container.id', getHref); + return buildHref(summary.state.summaryPings || [], 'container.id', getHref); }; export const getLoggingKubernetesHref = (summary: MonitorSummary, basePath: string) => { @@ -36,7 +36,7 @@ export const getLoggingKubernetesHref = (summary: MonitorSummary, basePath: stri `/app/logs?logFilter=${encodeURI(`(expression:'pod.uid : ${ret}',kind:kuery)`)}` ); }; - return buildHref(summary.state.checks || [], 'kubernetes.pod.uid', getHref); + return buildHref(summary.state.summaryPings || [], 'kubernetes.pod.uid', getHref); }; export const getLoggingIpHref = (summary: MonitorSummary, basePath: string) => { @@ -50,5 +50,5 @@ export const getLoggingIpHref = (summary: MonitorSummary, basePath: string) => { `/app/logs?logFilter=(expression:'${encodeURIComponent(`host.ip : ${ret}`)}',kind:kuery)` ); }; - return buildHref(summary.state.checks || [], 'monitor.ip', getHref); + return buildHref(summary.state.summaryPings || [], 'monitor.ip', getHref); }; diff --git a/x-pack/plugins/uptime/public/state/actions/monitor_list.ts b/x-pack/plugins/uptime/public/state/actions/monitor_list.ts index ee2267a9058af..da1c80f061c3e 100644 --- a/x-pack/plugins/uptime/public/state/actions/monitor_list.ts +++ b/x-pack/plugins/uptime/public/state/actions/monitor_list.ts @@ -5,8 +5,10 @@ */ import { createAction } from 'redux-actions'; -import { FetchMonitorStatesQueryArgs, MonitorSummaryResult } from '../../../common/runtime_types'; +import { FetchMonitorStatesQueryArgs, MonitorSummariesResult } from '../../../common/runtime_types'; export const getMonitorList = createAction('GET_MONITOR_LIST'); -export const getMonitorListSuccess = createAction('GET_MONITOR_LIST_SUCCESS'); +export const getMonitorListSuccess = createAction( + 'GET_MONITOR_LIST_SUCCESS' +); export const getMonitorListFailure = createAction('GET_MONITOR_LIST_FAIL'); diff --git a/x-pack/plugins/uptime/public/state/api/monitor_list.ts b/x-pack/plugins/uptime/public/state/api/monitor_list.ts index 084bcb4bd2a91..7c595dc255c84 100644 --- a/x-pack/plugins/uptime/public/state/api/monitor_list.ts +++ b/x-pack/plugins/uptime/public/state/api/monitor_list.ts @@ -8,12 +8,12 @@ import { API_URLS } from '../../../common/constants'; import { apiService } from './utils'; import { FetchMonitorStatesQueryArgs, - MonitorSummaryResult, - MonitorSummaryResultType, + MonitorSummariesResult, + MonitorSummariesResultType, } from '../../../common/runtime_types'; export const fetchMonitorList = async ( params: FetchMonitorStatesQueryArgs -): Promise => { - return await apiService.get(API_URLS.MONITOR_LIST, params, MonitorSummaryResultType); +): Promise => { + return await apiService.get(API_URLS.MONITOR_LIST, params, MonitorSummariesResultType); }; diff --git a/x-pack/plugins/uptime/public/state/api/utils.ts b/x-pack/plugins/uptime/public/state/api/utils.ts index f2efd2ecb875c..4f3765275c49a 100644 --- a/x-pack/plugins/uptime/public/state/api/utils.ts +++ b/x-pack/plugins/uptime/public/state/api/utils.ts @@ -7,6 +7,31 @@ import { PathReporter } from 'io-ts/lib/PathReporter'; import { isRight } from 'fp-ts/lib/Either'; import { HttpFetchQuery, HttpSetup } from 'src/core/public'; +import * as t from 'io-ts'; +import { isObject } from 'lodash'; + +// TODO: Copied from https://github.com/elastic/kibana/blob/master/x-pack/plugins/security_solution/common/format_errors.ts +// We should figure out a better way to share this +export const formatErrors = (errors: t.Errors): string[] => { + return errors.map((error) => { + if (error.message != null) { + return error.message; + } else { + const keyContext = error.context + .filter( + (entry) => entry.key != null && !Number.isInteger(+entry.key) && entry.key.trim() !== '' + ) + .map((entry) => entry.key) + .join('.'); + + const nameContext = error.context.find((entry) => entry.type?.name?.length > 0); + const suppliedValue = + keyContext !== '' ? keyContext : nameContext != null ? nameContext.type.name : ''; + const value = isObject(error.value) ? JSON.stringify(error.value) : error.value; + return `Invalid value "${value}" supplied to "${suppliedValue}"`; + } + }); +}; class ApiService { private static instance: ApiService; @@ -40,7 +65,10 @@ class ApiService { } else { // eslint-disable-next-line no-console console.error( - `API ${apiUrl} is not returning expected response, ${PathReporter.report(decoded)}` + `API ${apiUrl} is not returning expected response, ${formatErrors( + decoded.left + )} for response`, + response ); } } diff --git a/x-pack/plugins/uptime/public/state/reducers/monitor_list.ts b/x-pack/plugins/uptime/public/state/reducers/monitor_list.ts index 59a794a549d57..c4ba445eb123f 100644 --- a/x-pack/plugins/uptime/public/state/reducers/monitor_list.ts +++ b/x-pack/plugins/uptime/public/state/reducers/monitor_list.ts @@ -6,12 +6,12 @@ import { handleActions, Action } from 'redux-actions'; import { getMonitorList, getMonitorListSuccess, getMonitorListFailure } from '../actions'; -import { MonitorSummaryResult } from '../../../common/runtime_types'; +import { MonitorSummariesResult } from '../../../common/runtime_types'; export interface MonitorList { error?: Error; loading: boolean; - list: MonitorSummaryResult; + list: MonitorSummariesResult; } export const initialState: MonitorList = { @@ -24,7 +24,7 @@ export const initialState: MonitorList = { loading: false, }; -type Payload = MonitorSummaryResult & Error; +type Payload = MonitorSummariesResult & Error; export const monitorListReducer = handleActions( { @@ -34,7 +34,7 @@ export const monitorListReducer = handleActions( }), [String(getMonitorListSuccess)]: ( state: MonitorList, - action: Action + action: Action ) => ({ ...state, loading: false, diff --git a/x-pack/plugins/uptime/server/lib/requests/__tests__/get_latest_monitor.test.ts b/x-pack/plugins/uptime/server/lib/requests/__tests__/get_latest_monitor.test.ts index aa3574874af8e..01384ec145236 100644 --- a/x-pack/plugins/uptime/server/lib/requests/__tests__/get_latest_monitor.test.ts +++ b/x-pack/plugins/uptime/server/lib/requests/__tests__/get_latest_monitor.test.ts @@ -32,14 +32,7 @@ describe('getLatestMonitor', () => { }, }, size: 1, - _source: [ - 'url', - 'monitor', - 'observer', - '@timestamp', - 'tls.certificate_not_valid_after', - 'tls.certificate_not_valid_before', - ], + _source: ['url', 'monitor', 'observer', '@timestamp', 'tls.*'], sort: { '@timestamp': { order: 'desc' }, }, @@ -90,10 +83,7 @@ describe('getLatestMonitor', () => { "type": "http", }, "timestamp": "123456", - "tls": Object { - "not_after": undefined, - "not_before": undefined, - }, + "tls": undefined, } `); expect(result.timestamp).toBe('123456'); diff --git a/x-pack/plugins/uptime/server/lib/requests/get_latest_monitor.ts b/x-pack/plugins/uptime/server/lib/requests/get_latest_monitor.ts index a47e6173d9f00..a58208fc2bb96 100644 --- a/x-pack/plugins/uptime/server/lib/requests/get_latest_monitor.ts +++ b/x-pack/plugins/uptime/server/lib/requests/get_latest_monitor.ts @@ -45,14 +45,7 @@ export const getLatestMonitor: UMElasticsearchQueryFn { if (!p) { @@ -43,7 +38,7 @@ const jsonifyPagination = (p: any): string | null => { // Gets a page of monitor states. export const getMonitorStates: UMElasticsearchQueryFn< GetMonitorStatesParams, - GetMonitorStatesResult + MonitorSummariesResult > = async ({ callES, dynamicSettings, @@ -68,11 +63,113 @@ export const getMonitorStates: UMElasticsearchQueryFn< statusFilter ); - const page = await fetchPage(queryContext); + const size = Math.min(queryContext.size, QUERY.DEFAULT_AGGS_CAP); + + const iterator = new MonitorSummaryIterator(queryContext); + const page = await iterator.nextPage(size); + + const histograms = await getHistogramForMonitors( + queryContext, + page.monitorSummaries.map((s) => s.monitor_id) + ); + + page.monitorSummaries.forEach((s) => { + s.histogram = histograms[s.monitor_id]; + }); return { - summaries: page.items, + summaries: page.monitorSummaries, nextPagePagination: jsonifyPagination(page.nextPagePagination), prevPagePagination: jsonifyPagination(page.prevPagePagination), }; }; + +export const getHistogramForMonitors = async ( + queryContext: QueryContext, + monitorIds: string[] +): Promise<{ [key: string]: Histogram }> => { + const params = { + index: queryContext.heartbeatIndices, + body: { + size: 0, + query: { + bool: { + filter: [ + { + range: { + 'summary.down': { gt: 0 }, + }, + }, + { + terms: { + 'monitor.id': monitorIds, + }, + }, + { + range: { + '@timestamp': { + gte: queryContext.dateRangeStart, + lte: queryContext.dateRangeEnd, + }, + }, + }, + ], + }, + }, + aggs: { + histogram: { + date_histogram: { + field: '@timestamp', + // 12 seems to be a good size for performance given + // long monitor lists of up to 100 on the overview page + fixed_interval: + getHistogramInterval(queryContext.dateRangeStart, queryContext.dateRangeEnd, 12) + + 'ms', + missing: 0, + }, + aggs: { + by_id: { + terms: { + field: 'monitor.id', + size: Math.max(monitorIds.length, 1), + }, + aggs: { + totalDown: { + sum: { field: 'summary.down' }, + }, + }, + }, + }, + }, + }, + }, + }; + const result = await queryContext.search(params); + + const histoBuckets: any[] = result.aggregations.histogram.buckets; + const simplified = histoBuckets.map((histoBucket: any): { timestamp: number; byId: any } => { + const byId: { [key: string]: number } = {}; + histoBucket.by_id.buckets.forEach((idBucket: any) => { + byId[idBucket.key] = idBucket.totalDown.value; + }); + return { + timestamp: parseInt(histoBucket.key, 10), + byId, + }; + }); + + const histosById: { [key: string]: Histogram } = {}; + monitorIds.forEach((id: string) => { + const points: HistogramPoint[] = []; + simplified.forEach((simpleHisto) => { + points.push({ + timestamp: simpleHisto.timestamp, + up: undefined, + down: simpleHisto.byId[id], + }); + }); + histosById[id] = { points }; + }); + + return histosById; +}; diff --git a/x-pack/plugins/uptime/server/lib/requests/search/__tests__/enrich_monitor_groups.test.ts b/x-pack/plugins/uptime/server/lib/requests/search/__tests__/enrich_monitor_groups.test.ts deleted file mode 100644 index dd7996b68c41f..0000000000000 --- a/x-pack/plugins/uptime/server/lib/requests/search/__tests__/enrich_monitor_groups.test.ts +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { sortChecksBy } from '../enrich_monitor_groups'; - -describe('enrich monitor groups', () => { - describe('sortChecksBy', () => { - it('identifies lesser geo name', () => { - expect( - sortChecksBy( - { observer: { geo: { name: 'less' } }, monitor: { status: 'up' } }, - { observer: { geo: { name: 'more' } }, monitor: { status: 'up' } } - ) - ).toBe(-1); - }); - - it('identifies greater geo name', () => { - expect( - sortChecksBy( - { observer: { geo: { name: 'more' } }, monitor: { status: 'up' } }, - { observer: { geo: { name: 'less' } }, monitor: { status: 'up' } } - ) - ).toBe(1); - }); - - it('identifies equivalent geo name and sorts by lesser ip', () => { - expect( - sortChecksBy( - { observer: { geo: { name: 'same' } }, monitor: { ip: '127.0.0.1', status: 'up' } }, - { observer: { geo: { name: 'same' } }, monitor: { ip: '127.0.0.2', status: 'up' } } - ) - ).toBe(-1); - }); - - it('identifies equivalent geo name and sorts by greater ip', () => { - expect( - sortChecksBy( - { observer: { geo: { name: 'same' } }, monitor: { ip: '127.0.0.2', status: 'up' } }, - { observer: { geo: { name: 'same' } }, monitor: { ip: '127.0.0.1', status: 'up' } } - ) - ).toBe(1); - }); - - it('identifies equivalent geo name and sorts by equivalent ip', () => { - expect( - sortChecksBy( - { observer: { geo: { name: 'same' } }, monitor: { ip: '127.0.0.1', status: 'up' } }, - { observer: { geo: { name: 'same' } }, monitor: { ip: '127.0.0.1', status: 'up' } } - ) - ).toBe(0); - }); - - it('handles equivalent ip arrays', () => { - expect( - sortChecksBy( - { observer: { geo: { name: 'same' } }, monitor: { ip: ['127.0.0.1'], status: 'up' } }, - { observer: { geo: { name: 'same' } }, monitor: { ip: ['127.0.0.1'], status: 'up' } } - ) - ).toBe(0); - }); - - it('handles non-equal ip arrays', () => { - expect( - sortChecksBy( - { - observer: { geo: { name: 'same' } }, - monitor: { ip: ['127.0.0.2', '127.0.0.9'], status: 'up' }, - }, - { - observer: { geo: { name: 'same' } }, - monitor: { ip: ['127.0.0.3', '127.0.0.1'], status: 'up' }, - } - ) - ).toBe(1); - }); - - it('handles undefined observer fields', () => { - expect( - sortChecksBy( - { observer: undefined, monitor: { ip: ['127.0.0.1'], status: 'up' } }, - { observer: { geo: { name: 'same' } }, monitor: { ip: ['127.0.0.1'], status: 'up' } } - ) - ).toBe(-1); - }); - - it('handles undefined ip fields', () => { - expect( - sortChecksBy( - { observer: { geo: { name: 'same' } }, monitor: { ip: undefined, status: 'up' } }, - { observer: { geo: { name: 'same' } }, monitor: { ip: ['127.0.0.1'], status: 'up' } } - ) - ).toBe(-1); - }); - }); -}); diff --git a/x-pack/plugins/uptime/server/lib/requests/search/__tests__/fetch_page.test.ts b/x-pack/plugins/uptime/server/lib/requests/search/__tests__/fetch_page.test.ts deleted file mode 100644 index b98a5afcca334..0000000000000 --- a/x-pack/plugins/uptime/server/lib/requests/search/__tests__/fetch_page.test.ts +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { - fetchPage, - MonitorEnricher, - MonitorGroups, - MonitorGroupsFetcher, - MonitorGroupsPage, -} from '../fetch_page'; -import { QueryContext } from '../query_context'; -import { MonitorSummary } from '../../../../../common/runtime_types'; -import { nextPagination, prevPagination, simpleQueryContext } from './test_helpers'; - -const simpleFixture: MonitorGroups[] = [ - { - id: 'foo', - groups: [ - { - monitorId: 'foo', - location: 'foo-loc', - checkGroup: 'foo-check', - status: 'up', - summaryTimestamp: new Date(), - }, - ], - }, - { - id: 'bar', - groups: [ - { - monitorId: 'bar', - location: 'bar-loc', - checkGroup: 'bar-check', - status: 'down', - summaryTimestamp: new Date(), - }, - ], - }, -]; - -const simpleFetcher = (monitorGroups: MonitorGroups[]): MonitorGroupsFetcher => { - return async (queryContext: QueryContext, size: number): Promise => { - return { - monitorGroups, - prevPagePagination: prevPagination(monitorGroups[0].id), - nextPagePagination: nextPagination(monitorGroups[monitorGroups.length - 1].id), - }; - }; -}; - -const simpleEnricher = (monitorGroups: MonitorGroups[]): MonitorEnricher => { - return async (_queryContext: QueryContext, checkGroups: string[]): Promise => { - return checkGroups.map((cg) => { - const monitorGroup = monitorGroups.find((mg) => mg.groups.some((g) => g.checkGroup === cg))!; - return { - monitor_id: monitorGroup.id, - state: { - summary: {}, - timestamp: new Date(Date.parse('1999-12-31')).valueOf().toString(), - url: {}, - }, - }; - }); - }; -}; - -describe('fetching a page', () => { - it('returns the enriched groups', async () => { - const res = await fetchPage( - simpleQueryContext(), - simpleFetcher(simpleFixture), - simpleEnricher(simpleFixture) - ); - expect(res).toMatchInlineSnapshot(` - Object { - "items": Array [ - Object { - "monitor_id": "foo", - "state": Object { - "summary": Object {}, - "timestamp": "946598400000", - "url": Object {}, - }, - }, - Object { - "monitor_id": "bar", - "state": Object { - "summary": Object {}, - "timestamp": "946598400000", - "url": Object {}, - }, - }, - ], - "nextPagePagination": Object { - "cursorDirection": "AFTER", - "cursorKey": "bar", - "sortOrder": "ASC", - }, - "prevPagePagination": Object { - "cursorDirection": "BEFORE", - "cursorKey": "foo", - "sortOrder": "ASC", - }, - } - `); - }); -}); diff --git a/x-pack/plugins/uptime/server/lib/requests/search/__tests__/monitor_group_iterator.test.ts b/x-pack/plugins/uptime/server/lib/requests/search/__tests__/monitor_group_iterator.test.ts deleted file mode 100644 index 0ce5e75195475..0000000000000 --- a/x-pack/plugins/uptime/server/lib/requests/search/__tests__/monitor_group_iterator.test.ts +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { - CHUNK_SIZE, - ChunkFetcher, - ChunkResult, - MonitorGroupIterator, -} from '../monitor_group_iterator'; -import { simpleQueryContext } from './test_helpers'; -import { MonitorGroups } from '../fetch_page'; -import { QueryContext } from '../query_context'; - -describe('iteration', () => { - let iterator: MonitorGroupIterator | null = null; - let fetched: MonitorGroups[]; - - const setup = async (numGroups: number) => { - fetched = []; - const expectedMonitorGroups = makeMonitorGroups(numGroups); - const chunkFetcher = mockChunkFetcher(expectedMonitorGroups); - iterator = new MonitorGroupIterator(simpleQueryContext(), [], -1, chunkFetcher); - - while (true) { - const got = await iterator.next(); - if (got) { - fetched.push(got); - } else { - break; - } - } - }; - - describe('matching', () => { - [ - { name: 'zero results', numGroups: 0 }, - { name: 'one result', numGroups: 1 }, - { name: 'less than chunk', numGroups: CHUNK_SIZE - 1 }, - { name: 'multiple full chunks', numGroups: CHUNK_SIZE * 3 }, - { name: 'multiple full chunks + partial', numGroups: CHUNK_SIZE * 3 + 3 }, - ].forEach(({ name, numGroups }) => { - describe(`scenario given ${name}`, () => { - beforeEach(async () => { - await setup(numGroups); - }); - - it('should receive the expected number of results', () => { - expect(fetched.length).toEqual(numGroups); - }); - - it('should have no remaining pages', async () => { - expect(await iterator!.paginationAfterCurrent()).toBeNull(); - }); - }); - }); - }); -}); - -const makeMonitorGroups = (count: number): MonitorGroups[] => { - const groups: MonitorGroups[] = []; - for (let i = 0; i < count; i++) { - const id = `monitor-${i}`; - - groups.push({ - id, - groups: [ - { - monitorId: id, - location: 'a-location', - status: 'up', - checkGroup: `check-group-${i}`, - summaryTimestamp: new Date(), - }, - ], - }); - } - return groups; -}; - -const mockChunkFetcher = (groups: MonitorGroups[]): ChunkFetcher => { - const buffer = groups.slice(0); // Clone it since we'll modify it - return async ( - queryContext: QueryContext, - searchAfter: any, - size: number - ): Promise => { - const resultMonitorGroups = buffer.splice(0, size); - const resultSearchAfter = - buffer.length === 0 - ? null - : { monitor_id: resultMonitorGroups[resultMonitorGroups.length - 1].id }; - return { - monitorGroups: resultMonitorGroups, - searchAfter: resultSearchAfter, - }; - }; -}; diff --git a/x-pack/plugins/uptime/server/lib/requests/search/__tests__/monitor_summary_iterator.test.ts b/x-pack/plugins/uptime/server/lib/requests/search/__tests__/monitor_summary_iterator.test.ts new file mode 100644 index 0000000000000..8ba5be943304c --- /dev/null +++ b/x-pack/plugins/uptime/server/lib/requests/search/__tests__/monitor_summary_iterator.test.ts @@ -0,0 +1,131 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + CHUNK_SIZE, + ChunkFetcher, + ChunkResult, + MonitorSummaryIterator, +} from '../monitor_summary_iterator'; +import { simpleQueryContext } from './test_helpers'; +import { QueryContext } from '../query_context'; +import { MonitorSummary } from '../../../../../common/runtime_types'; + +describe('iteration', () => { + let iterator: MonitorSummaryIterator | null = null; + let fetched: MonitorSummary[]; + let fullSummaryDataset: MonitorSummary[]; + + const setup = async (numSummaries: number) => { + fetched = []; + fullSummaryDataset = makeMonitorSummaries(numSummaries); + const chunkFetcher = mockChunkFetcher(fullSummaryDataset); + iterator = new MonitorSummaryIterator(simpleQueryContext(), [], -1, chunkFetcher); + }; + + const fetchAllViaNext = async () => { + while (true) { + const got = await iterator!.next(); + if (got) { + fetched.push(got); + } else { + break; + } + } + }; + + describe('matching', () => { + [ + { name: 'zero results', numSummaries: 0 }, + { name: 'one result', numSummaries: 1 }, + { name: 'less than chunk', numSummaries: CHUNK_SIZE - 1 }, + { name: 'multiple full chunks', numSummaries: CHUNK_SIZE * 3 }, + { name: 'multiple full chunks + partial', numSummaries: CHUNK_SIZE * 3 + 3 }, + ].forEach(({ name, numSummaries }) => { + describe(`scenario given ${name}`, () => { + beforeEach(async () => { + await setup(numSummaries); + }); + + describe('fetching via next', () => { + beforeEach(async () => { + await fetchAllViaNext(); + }); + + it('should receive the expected number of results', async () => { + expect(fetched.length).toEqual(numSummaries); + }); + + it('should have no remaining pages', async () => { + expect(await iterator!.paginationAfterCurrent()).toBeNull(); + }); + }); + + describe('nextPage()', () => { + const pageSize = 900; + + it('should fetch no more than the page size results', async () => { + const page = await iterator!.nextPage(pageSize); + const expectedLength = numSummaries < pageSize ? numSummaries : pageSize; + expect(page.monitorSummaries).toHaveLength(expectedLength); + }); + + it('should return all the results if called until none remain', async () => { + const receivedResults: MonitorSummary[] = []; + while (true) { + const page = await iterator!.nextPage(pageSize); + if (page.monitorSummaries.length === 0) { + break; + } + page.monitorSummaries.forEach((s) => receivedResults.push(s)); + } + expect(receivedResults.length).toEqual(fullSummaryDataset.length); + }); + }); + }); + }); + }); +}); + +const makeMonitorSummaries = (count: number): MonitorSummary[] => { + const summaries: MonitorSummary[] = []; + for (let i = 0; i < count; i++) { + const id = `monitor-${i}`; + + summaries.push({ + monitor_id: id, + state: { + monitor: {}, + timestamp: (123 + i).toString(), + url: {}, + summaryPings: [], + summary: { up: 1, down: 0 }, + }, + }); + } + return summaries; +}; + +const mockChunkFetcher = (summaries: MonitorSummary[]): ChunkFetcher => { + const buffer = summaries.slice(0); // Clone it since we'll modify it + return async ( + queryContext: QueryContext, + searchAfter: any, + size: number + ): Promise => { + const offset = searchAfter?.monitor_id ? parseInt(searchAfter.monitor_id.split('-')[1], 10) : 0; + const resultMonitorSummaries = buffer.slice(offset, offset + size); + const resultSearchAfter = + offset > buffer.length - 1 + ? null + : { monitor_id: `monitor-${offset + resultMonitorSummaries.length}` }; + + return { + monitorSummaries: resultMonitorSummaries, + searchAfter: resultSearchAfter, + }; + }; +}; diff --git a/x-pack/plugins/uptime/server/lib/requests/search/__tests__/refine_potential_matches.test.ts b/x-pack/plugins/uptime/server/lib/requests/search/__tests__/refine_potential_matches.test.ts deleted file mode 100644 index 283f5fb8909f6..0000000000000 --- a/x-pack/plugins/uptime/server/lib/requests/search/__tests__/refine_potential_matches.test.ts +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { fullyMatchingIds } from '../refine_potential_matches'; -import { MonitorLocCheckGroup } from '..'; - -const mockQueryResult = (opts: { latestSummary: any; latestMatching: any }) => { - return { - aggregations: { - monitor: { - buckets: [ - { - key: 'my-monitor', - location: { - buckets: [ - { - key: 'my-location', - summaries: { - latest: { - hits: { - hits: [ - { - _source: opts.latestSummary, - }, - ], - }, - }, - }, - latest_matching: { - top: { - hits: { - hits: [ - { - _source: opts.latestMatching, - }, - ], - }, - }, - }, - }, - ], - }, - }, - ], - }, - }, - }; -}; - -describe('fully matching IDs', () => { - it('should exclude items whose latest result does not match', () => { - const queryRes = mockQueryResult({ - latestSummary: { - '@timestamp': '2020-06-04T12:39:54.698-0500', - monitor: { - check_group: 'latest-summary-check-group', - }, - summary: { - up: 1, - down: 0, - }, - }, - latestMatching: { - '@timestamp': '2019-06-04T12:39:54.698-0500', - summary: { - up: 1, - down: 0, - }, - }, - }); - const res = fullyMatchingIds(queryRes, undefined); - const expected = new Map(); - expect(res).toEqual(expected); - }); - - it('should include items whose latest result does match', () => { - const queryRes = mockQueryResult({ - latestSummary: { - '@timestamp': '2020-06-04T12:39:54.698-0500', - monitor: { - check_group: 'latest-summary-check-group', - }, - summary: { - up: 1, - down: 0, - }, - }, - latestMatching: { - '@timestamp': '2020-06-04T12:39:54.698-0500', - summary: { - up: 1, - down: 0, - }, - }, - }); - const res = fullyMatchingIds(queryRes, undefined); - const expected = new Map(); - expected.set('my-monitor', [ - { - checkGroup: 'latest-summary-check-group', - location: 'my-location', - monitorId: 'my-monitor', - status: 'up', - summaryTimestamp: new Date('2020-06-04T12:39:54.698-0500'), - }, - ]); - expect(res).toEqual(expected); - }); -}); diff --git a/x-pack/plugins/uptime/server/lib/requests/search/enrich_monitor_groups.ts b/x-pack/plugins/uptime/server/lib/requests/search/enrich_monitor_groups.ts deleted file mode 100644 index f5c4c55a4e300..0000000000000 --- a/x-pack/plugins/uptime/server/lib/requests/search/enrich_monitor_groups.ts +++ /dev/null @@ -1,409 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { QueryContext } from './query_context'; -import { - Check, - Histogram, - HistogramPoint, - MonitorSummary, - CursorDirection, - SortOrder, -} from '../../../../common/runtime_types'; -import { MonitorEnricher } from './fetch_page'; -import { getHistogramInterval } from '../../helper/get_histogram_interval'; - -export const enrichMonitorGroups: MonitorEnricher = async ( - queryContext: QueryContext, - checkGroups: string[] -): Promise => { - // TODO the scripted metric query here is totally unnecessary and largely - // redundant with the way the code works now. This could be simplified - // to a much simpler query + some JS processing. - const params = { - index: queryContext.heartbeatIndices, - body: { - query: { - bool: { - filter: [{ terms: { 'monitor.check_group': checkGroups } }], - }, - }, - size: 0, - aggs: { - monitors: { - composite: { - /** - * TODO: extract this to a constant; we can't be looking for _only_ - * ten monitors, because it's possible our check groups selection will represent more than ten. - * - * We were previously passing the after key from the check groups query regardless of the number of monitors we had, - * it's important that the latest check group from the final monitor we use is what we return, or we will be way ahead in terms - * of check groups and end up skipping monitors on subsequent calls. - */ - size: 500, - sources: [ - { - monitor_id: { - terms: { - field: 'monitor.id', - order: cursorDirectionToOrder(queryContext.pagination.cursorDirection), - }, - }, - }, - ], - }, - aggregations: { - state: { - scripted_metric: { - init_script: ` - // Globals are values that should be identical across all docs - // We can cheat a bit by always overwriting these and make the - // assumption that there is no variation in these across checks - state.globals = new HashMap(); - // Here we store stuff broken out by agent.id and monitor.id - // This should correspond to a unique check. - state.checksByAgentIdIP = new HashMap(); - `, - map_script: ` - Map curCheck = new HashMap(); - String agentId = doc["agent.id"][0]; - String ip = null; - if (doc["monitor.ip"].length > 0) { - ip = doc["monitor.ip"][0]; - } - String agentIdIP = agentId + "-" + (ip == null ? "" : ip.toString()); - def ts = doc["@timestamp"][0].toInstant().toEpochMilli(); - - def lastCheck = state.checksByAgentIdIP[agentId]; - Instant lastTs = lastCheck != null ? lastCheck["@timestamp"] : null; - if (lastTs != null && lastTs > ts) { - return; - } - - curCheck.put("@timestamp", ts); - - Map agent = new HashMap(); - agent.id = agentId; - curCheck.put("agent", agent); - - if (state.globals.url == null) { - Map url = new HashMap(); - Collection fields = ["full", "original", "scheme", "username", "password", "domain", "port", "path", "query", "fragment"]; - for (field in fields) { - String docPath = "url." + field; - def val = doc[docPath]; - if (!val.isEmpty()) { - url[field] = val[0]; - } - } - state.globals.url = url; - } - - Map monitor = new HashMap(); - monitor.status = doc["monitor.status"][0]; - monitor.ip = ip; - if (!doc["monitor.name"].isEmpty()) { - String monitorName = doc["monitor.name"][0]; - if (monitor.name != "") { - monitor.name = monitorName; - } - } - curCheck.monitor = monitor; - - if (curCheck.observer == null) { - curCheck.observer = new HashMap(); - } - if (curCheck.observer.geo == null) { - curCheck.observer.geo = new HashMap(); - } - if (!doc["observer.geo.name"].isEmpty()) { - curCheck.observer.geo.name = doc["observer.geo.name"][0]; - } - if (!doc["observer.geo.location"].isEmpty()) { - curCheck.observer.geo.location = doc["observer.geo.location"][0]; - } - if (!doc["kubernetes.pod.uid"].isEmpty() && curCheck.kubernetes == null) { - curCheck.kubernetes = new HashMap(); - curCheck.kubernetes.pod = new HashMap(); - curCheck.kubernetes.pod.uid = doc["kubernetes.pod.uid"][0]; - } - if (!doc["container.id"].isEmpty() && curCheck.container == null) { - curCheck.container = new HashMap(); - curCheck.container.id = doc["container.id"][0]; - } - if (curCheck.tls == null) { - curCheck.tls = new HashMap(); - } - if (!doc["tls.certificate_not_valid_after"].isEmpty()) { - curCheck.tls.not_after = doc["tls.certificate_not_valid_after"][0]; - } - if (!doc["tls.certificate_not_valid_before"].isEmpty()) { - curCheck.tls.not_before = doc["tls.certificate_not_valid_before"][0]; - } - - state.checksByAgentIdIP[agentIdIP] = curCheck; - `, - combine_script: 'return state;', - reduce_script: ` - // The final document - Map result = new HashMap(); - - Map checks = new HashMap(); - Instant maxTs = Instant.ofEpochMilli(0); - Collection ips = new HashSet(); - Collection geoNames = new HashSet(); - Collection podUids = new HashSet(); - Collection containerIds = new HashSet(); - Collection tls = new HashSet(); - String name = null; - for (state in states) { - result.putAll(state.globals); - for (entry in state.checksByAgentIdIP.entrySet()) { - def agentIdIP = entry.getKey(); - def check = entry.getValue(); - def lastBestCheck = checks.get(agentIdIP); - def checkTs = Instant.ofEpochMilli(check.get("@timestamp")); - - if (maxTs.isBefore(checkTs)) { maxTs = checkTs} - - if (lastBestCheck == null || lastBestCheck.get("@timestamp") < checkTs) { - check["@timestamp"] = check["@timestamp"]; - checks[agentIdIP] = check - } - - if (check.monitor.name != null && check.monitor.name != "") { - name = check.monitor.name; - } - - ips.add(check.monitor.ip); - if (check.observer != null && check.observer.geo != null && check.observer.geo.name != null) { - geoNames.add(check.observer.geo.name); - } - if (check.kubernetes != null && check.kubernetes.pod != null) { - podUids.add(check.kubernetes.pod.uid); - } - if (check.container != null) { - containerIds.add(check.container.id); - } - if (check.tls != null) { - tls.add(check.tls); - } - } - } - - // We just use the values so we can store these as nested docs - result.checks = checks.values(); - result.put("@timestamp", maxTs); - - - Map summary = new HashMap(); - summary.up = checks.entrySet().stream().filter(c -> c.getValue().monitor.status == "up").count(); - summary.down = checks.size() - summary.up; - result.summary = summary; - - Map monitor = new HashMap(); - monitor.ip = ips; - monitor.name = name; - monitor.status = summary.down > 0 ? "down" : "up"; - result.monitor = monitor; - - Map observer = new HashMap(); - Map geo = new HashMap(); - observer.geo = geo; - geo.name = geoNames; - result.observer = observer; - - if (!podUids.isEmpty()) { - result.kubernetes = new HashMap(); - result.kubernetes.pod = new HashMap(); - result.kubernetes.pod.uid = podUids; - } - - if (!containerIds.isEmpty()) { - result.container = new HashMap(); - result.container.id = containerIds; - } - - if (!tls.isEmpty()) { - result.tls = new HashMap(); - result.tls = tls; - } - - return result; - `, - }, - }, - }, - }, - }, - }, - }; - - const items = await queryContext.search(params); - - const monitorBuckets = items?.aggregations?.monitors?.buckets ?? []; - - const monitorIds: string[] = []; - const summaries: MonitorSummary[] = monitorBuckets.map((monitor: any) => { - const monitorId = monitor.key.monitor_id; - monitorIds.push(monitorId); - const state: any = monitor.state?.value; - state.timestamp = state['@timestamp']; - const { checks } = state; - if (Array.isArray(checks)) { - checks.sort(sortChecksBy); - state.checks = state.checks.map((check: any) => ({ - ...check, - timestamp: check['@timestamp'], - })); - } else { - state.checks = []; - } - return { - monitor_id: monitorId, - state, - }; - }); - - const histogramMap = await getHistogramForMonitors(queryContext, monitorIds); - - const resItems = summaries.map((summary) => ({ - ...summary, - histogram: histogramMap[summary.monitor_id], - })); - - const sortedResItems: any = resItems.sort((a, b) => { - if (a.monitor_id === b.monitor_id) return 0; - return a.monitor_id > b.monitor_id ? 1 : -1; - }); - - if (queryContext.pagination.sortOrder === SortOrder.DESC) { - sortedResItems.reverse(); - } - - return sortedResItems; -}; - -const getHistogramForMonitors = async ( - queryContext: QueryContext, - monitorIds: string[] -): Promise<{ [key: string]: Histogram }> => { - const params = { - index: queryContext.heartbeatIndices, - body: { - size: 0, - query: { - bool: { - filter: [ - { - range: { - 'summary.down': { gt: 0 }, - }, - }, - { - terms: { - 'monitor.id': monitorIds, - }, - }, - { - range: { - '@timestamp': { - gte: queryContext.dateRangeStart, - lte: queryContext.dateRangeEnd, - }, - }, - }, - ], - }, - }, - aggs: { - histogram: { - date_histogram: { - field: '@timestamp', - // 12 seems to be a good size for performance given - // long monitor lists of up to 100 on the overview page - fixed_interval: - getHistogramInterval(queryContext.dateRangeStart, queryContext.dateRangeEnd, 12) + - 'ms', - missing: 0, - }, - aggs: { - by_id: { - terms: { - field: 'monitor.id', - size: Math.max(monitorIds.length, 1), - }, - aggs: { - totalDown: { - sum: { field: 'summary.down' }, - }, - }, - }, - }, - }, - }, - }, - }; - const result = await queryContext.search(params); - - const histoBuckets: any[] = result.aggregations.histogram.buckets; - const simplified = histoBuckets.map((histoBucket: any): { timestamp: number; byId: any } => { - const byId: { [key: string]: number } = {}; - histoBucket.by_id.buckets.forEach((idBucket: any) => { - byId[idBucket.key] = idBucket.totalDown.value; - }); - return { - timestamp: parseInt(histoBucket.key, 10), - byId, - }; - }); - - const histosById: { [key: string]: Histogram } = {}; - monitorIds.forEach((id: string) => { - const points: HistogramPoint[] = []; - simplified.forEach((simpleHisto) => { - points.push({ - timestamp: simpleHisto.timestamp, - up: undefined, - down: simpleHisto.byId[id], - }); - }); - histosById[id] = { points }; - }); - - return histosById; -}; - -const cursorDirectionToOrder = (cd: CursorDirection): 'asc' | 'desc' => { - return CursorDirection[cd] === CursorDirection.AFTER ? 'asc' : 'desc'; -}; - -const getStringValue = (value: string | Array | null | undefined): string => { - if (Array.isArray(value)) { - value.sort(); - return value[0] ?? ''; - } - return value ?? ''; -}; - -export const sortChecksBy = ( - a: Pick, - b: Pick -) => { - const nameA: string = a.observer?.geo?.name ?? ''; - const nameB: string = b.observer?.geo?.name ?? ''; - - if (nameA === nameB) { - const ipA = getStringValue(a.monitor.ip); - const ipB = getStringValue(b.monitor.ip); - - if (ipA === ipB) { - return 0; - } - return ipA > ipB ? 1 : -1; - } - return nameA > nameB ? 1 : -1; -}; diff --git a/x-pack/plugins/uptime/server/lib/requests/search/fetch_chunk.ts b/x-pack/plugins/uptime/server/lib/requests/search/fetch_chunk.ts index 2a5f1f1261cb3..82bc5e0123866 100644 --- a/x-pack/plugins/uptime/server/lib/requests/search/fetch_chunk.ts +++ b/x-pack/plugins/uptime/server/lib/requests/search/fetch_chunk.ts @@ -6,7 +6,7 @@ import { refinePotentialMatches } from './refine_potential_matches'; import { findPotentialMatches } from './find_potential_matches'; -import { ChunkFetcher, ChunkResult } from './monitor_group_iterator'; +import { ChunkFetcher, ChunkResult } from './monitor_summary_iterator'; import { QueryContext } from './query_context'; /** @@ -18,9 +18,6 @@ import { QueryContext } from './query_context'; * @param searchAfter indicates where Elasticsearch should continue querying on subsequent requests, if at all * @param size the minimum size of the matches to chunk */ -// Note that all returned data may be erroneous. If `searchAfter` is returned the caller should invoke this function -// repeatedly with the new searchAfter value as there may be more matching data in a future chunk. If `searchAfter` -// is falsey there is no more data to fetch. export const fetchChunk: ChunkFetcher = async ( queryContext: QueryContext, searchAfter: any, @@ -34,7 +31,7 @@ export const fetchChunk: ChunkFetcher = async ( const matching = await refinePotentialMatches(queryContext, monitorIds); return { - monitorGroups: matching, + monitorSummaries: matching, searchAfter: foundSearchAfter, }; }; diff --git a/x-pack/plugins/uptime/server/lib/requests/search/fetch_page.ts b/x-pack/plugins/uptime/server/lib/requests/search/fetch_page.ts deleted file mode 100644 index 831236d8ccb51..0000000000000 --- a/x-pack/plugins/uptime/server/lib/requests/search/fetch_page.ts +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { flatten } from 'lodash'; -import { CursorPagination } from './types'; -import { QueryContext } from './query_context'; -import { QUERY } from '../../../../common/constants'; -import { CursorDirection, MonitorSummary, SortOrder } from '../../../../common/runtime_types'; -import { enrichMonitorGroups } from './enrich_monitor_groups'; -import { MonitorGroupIterator } from './monitor_group_iterator'; - -/** - * - * Gets a single page of results per the settings in the provided queryContext. These results are very minimal, - * just monitor IDs and check groups. This takes an optional `MonitorGroupEnricher` that post-processes the minimal - * data, decorating it appropriately. The function also takes a fetcher, which does all the actual fetching. - * @param queryContext defines the criteria for the data on the current page - * @param monitorGroupFetcher performs paginated monitor fetching - * @param monitorEnricher decorates check group results with additional data - */ -// just monitor IDs and check groups. This takes an optional `MonitorGroupEnricher` that post-processes the minimal -// data, decorating it appropriately. The function also takes a fetcher, which does all the actual fetching. -export const fetchPage = async ( - queryContext: QueryContext, - monitorGroupFetcher: MonitorGroupsFetcher = fetchPageMonitorGroups, - monitorEnricher: MonitorEnricher = enrichMonitorGroups -): Promise => { - const size = Math.min(queryContext.size, QUERY.DEFAULT_AGGS_CAP); - const monitorPage = await monitorGroupFetcher(queryContext, size); - - const checkGroups: string[] = flatten( - monitorPage.monitorGroups.map((monitorGroups) => monitorGroups.groups.map((g) => g.checkGroup)) - ); - - const enrichedMonitors = await monitorEnricher(queryContext, checkGroups); - - return { - items: enrichedMonitors, - nextPagePagination: monitorPage.nextPagePagination, - prevPagePagination: monitorPage.prevPagePagination, - }; -}; - -// Fetches the most recent monitor groups for the given page, -// in the manner demanded by the `queryContext` and return at most `size` results. -const fetchPageMonitorGroups: MonitorGroupsFetcher = async ( - queryContext: QueryContext, - size: number -): Promise => { - const monitorGroups: MonitorGroups[] = []; - - const iterator = new MonitorGroupIterator(queryContext); - - let paginationBefore: CursorPagination | null = null; - while (monitorGroups.length < size) { - const monitor = await iterator.next(); - if (!monitor) { - break; // No more items to fetch - } - monitorGroups.push(monitor); - - // We want the before pagination to be before the first item we encounter - if (monitorGroups.length === 1) { - paginationBefore = await iterator.paginationBeforeCurrent(); - } - } - - // We have to create these objects before checking if we can navigate backward - const paginationAfter = await iterator.paginationAfterCurrent(); - - const ssAligned = searchSortAligned(queryContext.pagination); - - if (!ssAligned) { - monitorGroups.reverse(); - } - - return { - monitorGroups, - nextPagePagination: ssAligned ? paginationAfter : paginationBefore, - prevPagePagination: ssAligned ? paginationBefore : paginationAfter, - }; -}; - -// Returns true if the order returned by the ES query matches the requested sort order. -// This useful to determine if the results need to be reversed from their ES results order. -// I.E. when navigating backwards using prevPagePagination (CursorDirection.Before) yet using a SortOrder.ASC. -const searchSortAligned = (pagination: CursorPagination): boolean => { - if (pagination.cursorDirection === CursorDirection.AFTER) { - return pagination.sortOrder === SortOrder.ASC; - } else { - return pagination.sortOrder === SortOrder.DESC; - } -}; - -// Minimal interface representing the most recent set of groups accompanying a MonitorId in a given context. -export interface MonitorGroups { - id: string; - groups: MonitorLocCheckGroup[]; -} - -// Representation of the data returned when aggregating summary check groups. -export interface MonitorLocCheckGroup { - monitorId: string; - location: string | null; - checkGroup: string; - status: 'up' | 'down'; - summaryTimestamp: Date; -} - -// Represents a page that has not yet been enriched. -export interface MonitorGroupsPage { - monitorGroups: MonitorGroups[]; - nextPagePagination: CursorPagination | null; - prevPagePagination: CursorPagination | null; -} - -// Representation of a full page of results with pagination data for constructing next/prev links. -export interface EnrichedPage { - items: MonitorSummary[]; - nextPagePagination: CursorPagination | null; - prevPagePagination: CursorPagination | null; -} - -// A function that does the work of matching the minimal set of data for this query, returning just matching fields -// that are efficient to access while performing the query. -export type MonitorGroupsFetcher = ( - queryContext: QueryContext, - size: number -) => Promise; - -// A function that takes a set of check groups and returns richer MonitorSummary objects. -export type MonitorEnricher = ( - queryContext: QueryContext, - checkGroups: string[] -) => Promise; diff --git a/x-pack/plugins/uptime/server/lib/requests/search/find_potential_matches.ts b/x-pack/plugins/uptime/server/lib/requests/search/find_potential_matches.ts index ac4ff91230b95..8bdf7faf380e8 100644 --- a/x-pack/plugins/uptime/server/lib/requests/search/find_potential_matches.ts +++ b/x-pack/plugins/uptime/server/lib/requests/search/find_potential_matches.ts @@ -5,7 +5,6 @@ */ import { get, set } from 'lodash'; -import { CursorDirection } from '../../../../common/runtime_types'; import { QueryContext } from './query_context'; /** @@ -44,8 +43,6 @@ const query = async (queryContext: QueryContext, searchAfter: any, size: number) }; const queryBody = async (queryContext: QueryContext, searchAfter: any, size: number) => { - const compositeOrder = cursorDirectionToOrder(queryContext.pagination.cursorDirection); - const filters = await queryContext.dateAndCustomFilters(); if (queryContext.statusFilter) { @@ -66,7 +63,7 @@ const queryBody = async (queryContext: QueryContext, searchAfter: any, size: num size, sources: [ { - monitor_id: { terms: { field: 'monitor.id', order: compositeOrder } }, + monitor_id: { terms: { field: 'monitor.id', order: queryContext.cursorOrder() } }, }, ], }, @@ -80,7 +77,3 @@ const queryBody = async (queryContext: QueryContext, searchAfter: any, size: num return body; }; - -const cursorDirectionToOrder = (cd: CursorDirection): 'asc' | 'desc' => { - return CursorDirection[cd] === CursorDirection.AFTER ? 'asc' : 'desc'; -}; diff --git a/x-pack/plugins/uptime/server/lib/requests/search/index.ts b/x-pack/plugins/uptime/server/lib/requests/search/index.ts index 8de5687808d61..dd8ca56f1fd7d 100644 --- a/x-pack/plugins/uptime/server/lib/requests/search/index.ts +++ b/x-pack/plugins/uptime/server/lib/requests/search/index.ts @@ -4,6 +4,5 @@ * you may not use this file except in compliance with the Elastic License. */ -export { fetchPage, MonitorGroups, MonitorLocCheckGroup, MonitorGroupsPage } from './fetch_page'; -export { MonitorGroupIterator } from './monitor_group_iterator'; +export { MonitorSummaryIterator } from './monitor_summary_iterator'; export { QueryContext } from './query_context'; diff --git a/x-pack/plugins/uptime/server/lib/requests/search/monitor_group_iterator.ts b/x-pack/plugins/uptime/server/lib/requests/search/monitor_summary_iterator.ts similarity index 61% rename from x-pack/plugins/uptime/server/lib/requests/search/monitor_group_iterator.ts rename to x-pack/plugins/uptime/server/lib/requests/search/monitor_summary_iterator.ts index 1ead81fa7102c..0a726eb383949 100644 --- a/x-pack/plugins/uptime/server/lib/requests/search/monitor_group_iterator.ts +++ b/x-pack/plugins/uptime/server/lib/requests/search/monitor_summary_iterator.ts @@ -6,8 +6,7 @@ import { QueryContext } from './query_context'; import { fetchChunk } from './fetch_chunk'; -import { CursorDirection } from '../../../../common/runtime_types'; -import { MonitorGroups } from './fetch_page'; +import { CursorDirection, MonitorSummary } from '../../../../common/runtime_types'; import { CursorPagination } from './types'; // Hardcoded chunk size for how many monitors to fetch at a time when querying @@ -22,7 +21,7 @@ export type ChunkFetcher = ( // Result of fetching more results from the search. export interface ChunkResult { - monitorGroups: MonitorGroups[]; + monitorSummaries: MonitorSummary[]; searchAfter: any; } @@ -32,21 +31,19 @@ export interface ChunkResult { * querying, this class provides a `next` function that is cleaner to call. `next` provides the next matching result, * which may require many subsequent fetches, while keeping the external API clean. */ -// matches, or may simple be empty results that tell us a to keep looking for more, this class exists to simplify things. -// The idea is that you can call next() on it and receive the next matching result, even if internally we need to fetch -// multiple chunks to find that result. -export class MonitorGroupIterator { +export class MonitorSummaryIterator { queryContext: QueryContext; // Cache representing pre-fetched query results. - // The first item is the CheckGroup this represents. - buffer: MonitorGroups[]; + // The first item is the MonitorSummary this represents. + buffer: MonitorSummary[]; bufferPos: number; searchAfter: any; chunkFetcher: ChunkFetcher; + endOfResults: boolean; // true if we've hit the end of results from ES constructor( queryContext: QueryContext, - initialBuffer: MonitorGroups[] = [], + initialBuffer: MonitorSummary[] = [], initialBufferPos: number = -1, chunkFetcher: ChunkFetcher = fetchChunk ) { @@ -55,11 +52,12 @@ export class MonitorGroupIterator { this.bufferPos = initialBufferPos; this.searchAfter = queryContext.pagination.cursorKey; this.chunkFetcher = chunkFetcher; + this.endOfResults = false; } // Fetch the next matching result. - async next(): Promise { - await this.bufferNext(CHUNK_SIZE); + async next(): Promise { + await this.bufferNext(); const found = this.buffer[this.bufferPos + 1]; if (found) { @@ -69,29 +67,62 @@ export class MonitorGroupIterator { return null; } + async nextPage(size: number): Promise { + const monitorSummaries: MonitorSummary[] = []; + let paginationBefore: CursorPagination | null = null; + while (monitorSummaries.length < size) { + const monitor = await this.next(); + if (!monitor) { + break; // No more items to fetch + } + monitorSummaries.push(monitor); + + // We want the before pagination to be before the first item we encounter + if (monitorSummaries.length === 1) { + paginationBefore = await this.paginationBeforeCurrent(); + } + } + + // We have to create these objects before checking if we can navigate backward + const paginationAfter = await this.paginationAfterCurrent(); + + const ssAligned = this.queryContext.searchSortAligned(); + + if (!ssAligned) { + monitorSummaries.reverse(); + } + + return { + monitorSummaries, + nextPagePagination: ssAligned ? paginationAfter : paginationBefore, + prevPagePagination: ssAligned ? paginationBefore : paginationAfter, + }; + } + // Look ahead to see if there are additional results. - async peek(): Promise { - await this.bufferNext(CHUNK_SIZE); + async peek(): Promise { + await this.bufferNext(); return this.buffer[this.bufferPos + 1] || null; } // Returns the last item fetched with next(). null if no items fetched with // next or if next has not yet been invoked. - getCurrent(): MonitorGroups | null { + getCurrent(): MonitorSummary | null { return this.buffer[this.bufferPos] || null; } // Attempts to buffer at most `size` number of additional results, stopping when at least one additional // result is buffered or there are no more matching items to be found. - async bufferNext(size: number = CHUNK_SIZE): Promise { - // The next element is already buffered. + async bufferNext(): Promise { + // Nothing to do if there are no more results or + // the next element is already buffered. if (this.buffer[this.bufferPos + 1]) { return; } - while (true) { - const result = await this.attemptBufferMore(size); - if (result.gotHit || !result.hasMore) { + while (!this.endOfResults) { + const result = await this.attemptBufferMore(); + if (result.gotHit) { return; } } @@ -103,9 +134,7 @@ export class MonitorGroupIterator { * to free up space. * @param size the number of items to chunk */ - async attemptBufferMore( - size: number = CHUNK_SIZE - ): Promise<{ hasMore: boolean; gotHit: boolean }> { + async attemptBufferMore(): Promise<{ gotHit: boolean }> { // Trim the buffer to just the current element since we'll be fetching more const current = this.getCurrent(); @@ -117,17 +146,21 @@ export class MonitorGroupIterator { this.bufferPos = 0; } - const results = await this.chunkFetcher(this.queryContext, this.searchAfter, size); + const results = await this.chunkFetcher(this.queryContext, this.searchAfter, CHUNK_SIZE); // If we've hit the end of the stream searchAfter will be empty - - results.monitorGroups.forEach((mig: MonitorGroups) => this.buffer.push(mig)); + results.monitorSummaries.forEach((ms: MonitorSummary) => this.buffer.push(ms)); if (results.searchAfter) { this.searchAfter = results.searchAfter; } + // Remember, the chunk fetcher might return no results in one chunk, but still have more matching + // results, so we use the searchAfter field to determine whether we keep going. + if (!results.searchAfter) { + this.endOfResults = true; + } + return { - gotHit: results.monitorGroups.length > 0, - hasMore: !!results.searchAfter, + gotHit: results.monitorSummaries.length > 0, }; } @@ -142,7 +175,7 @@ export class MonitorGroupIterator { if (!current) { return null; } - const cursorKey = { monitor_id: current.id }; + const cursorKey = { monitor_id: current.monitor_id }; return Object.assign({}, this.queryContext.pagination, { cursorKey }); } @@ -154,12 +187,12 @@ export class MonitorGroupIterator { } // Returns a copy of this fetcher that goes backwards from the current position - reverse(): MonitorGroupIterator | null { + reverse(): MonitorSummaryIterator | null { const reverseContext = this.queryContext.clone(); const current = this.getCurrent(); reverseContext.pagination = { - cursorKey: current ? { monitor_id: current.id } : null, + cursorKey: current ? { monitor_id: current.monitor_id } : null, sortOrder: this.queryContext.pagination.sortOrder, cursorDirection: this.queryContext.pagination.cursorDirection === CursorDirection.AFTER @@ -168,12 +201,18 @@ export class MonitorGroupIterator { }; return current - ? new MonitorGroupIterator(reverseContext, [current], 0, this.chunkFetcher) + ? new MonitorSummaryIterator(reverseContext, [current], 0, this.chunkFetcher) : null; } // Returns a copy of this with a shallow copied buffer. Note that the queryContext is still shared! clone() { - return new MonitorGroupIterator(this.queryContext, this.buffer.slice(0), this.bufferPos); + return new MonitorSummaryIterator(this.queryContext, this.buffer.slice(0), this.bufferPos); } } + +export interface MonitorSummariesPage { + monitorSummaries: MonitorSummary[]; + nextPagePagination: CursorPagination | null; + prevPagePagination: CursorPagination | null; +} diff --git a/x-pack/plugins/uptime/server/lib/requests/search/query_context.ts b/x-pack/plugins/uptime/server/lib/requests/search/query_context.ts index b2fb9ce68c132..5d97e635f3e7d 100644 --- a/x-pack/plugins/uptime/server/lib/requests/search/query_context.ts +++ b/x-pack/plugins/uptime/server/lib/requests/search/query_context.ts @@ -8,6 +8,7 @@ import moment from 'moment'; import { LegacyAPICaller } from 'src/core/server'; import { CursorPagination } from './types'; import { parseRelativeDate } from '../../helper'; +import { CursorDirection, SortOrder } from '../../../../common/runtime_types'; export class QueryContext { callES: LegacyAPICaller; @@ -146,4 +147,21 @@ export class QueryContext { this.statusFilter ); } + + // Returns true if the order returned by the ES query matches the requested sort order. + // This useful to determine if the results need to be reversed from their ES results order. + // I.E. when navigating backwards using prevPagePagination (CursorDirection.Before) yet using a SortOrder.ASC. + searchSortAligned(): boolean { + if (this.pagination.cursorDirection === CursorDirection.AFTER) { + return this.pagination.sortOrder === SortOrder.ASC; + } else { + return this.pagination.sortOrder === SortOrder.DESC; + } + } + + cursorOrder(): 'asc' | 'desc' { + return CursorDirection[this.pagination.cursorDirection] === CursorDirection.AFTER + ? 'asc' + : 'desc'; + } } diff --git a/x-pack/plugins/uptime/server/lib/requests/search/refine_potential_matches.ts b/x-pack/plugins/uptime/server/lib/requests/search/refine_potential_matches.ts index 2f54f3f6dd689..f631d5c963ca5 100644 --- a/x-pack/plugins/uptime/server/lib/requests/search/refine_potential_matches.ts +++ b/x-pack/plugins/uptime/server/lib/requests/search/refine_potential_matches.ts @@ -5,8 +5,7 @@ */ import { QueryContext } from './query_context'; -import { CursorDirection } from '../../../../common/runtime_types'; -import { MonitorGroups, MonitorLocCheckGroup } from './fetch_page'; +import { MonitorSummary, Ping } from '../../../../common/runtime_types'; /** * Determines whether the provided check groups are the latest complete check groups for their associated monitor ID's. @@ -19,80 +18,83 @@ import { MonitorGroups, MonitorLocCheckGroup } from './fetch_page'; export const refinePotentialMatches = async ( queryContext: QueryContext, potentialMatchMonitorIDs: string[] -): Promise => { +): Promise => { if (potentialMatchMonitorIDs.length === 0) { return []; } const queryResult = await query(queryContext, potentialMatchMonitorIDs); - const recentGroupsMatchingStatus = await fullyMatchingIds(queryResult, queryContext.statusFilter); - - // Return the monitor groups filtering out potential matches that weren't current - const matches: MonitorGroups[] = potentialMatchMonitorIDs - .map((id: string) => { - return { id, groups: recentGroupsMatchingStatus.get(id) || [] }; - }) - .filter((mrg) => mrg.groups.length > 0); - - // Sort matches by ID - matches.sort((a: MonitorGroups, b: MonitorGroups) => { - return a.id === b.id ? 0 : a.id > b.id ? 1 : -1; - }); - - if (queryContext.pagination.cursorDirection === CursorDirection.BEFORE) { - matches.reverse(); - } - return matches; + return await fullyMatchingIds(queryResult, queryContext.statusFilter); }; -export const fullyMatchingIds = (queryResult: any, statusFilter?: string) => { - const matching = new Map(); - MonitorLoop: for (const monBucket of queryResult.aggregations.monitor.buckets) { - const monitorId: string = monBucket.key; - const groups: MonitorLocCheckGroup[] = []; +export const fullyMatchingIds = (queryResult: any, statusFilter?: string): MonitorSummary[] => { + const summaries: MonitorSummary[] = []; + for (const monBucket of queryResult.aggregations.monitor.buckets) { // Did at least one location match? let matched = false; + + const summaryPings: Ping[] = []; + for (const locBucket of monBucket.location.buckets) { - const location = locBucket.key; - const latestSource = locBucket.summaries.latest.hits.hits[0]._source; - const latestStillMatchingSource = locBucket.latest_matching.top.hits.hits[0]?._source; + const latest = locBucket.summaries.latest.hits.hits[0]; + const latestStillMatching = locBucket.latest_matching.top.hits.hits[0]; // If the most recent document still matches the most recent document matching the current filters // we can include this in the result // // We just check if the timestamp is greater. Note this may match an incomplete check group // that has not yet sent a summary doc if ( - latestStillMatchingSource && - latestStillMatchingSource['@timestamp'] >= latestSource['@timestamp'] + latestStillMatching && + latestStillMatching._source['@timestamp'] >= latest._source['@timestamp'] ) { matched = true; } - const checkGroup = latestSource.monitor.check_group; - const status = latestSource.summary.down > 0 ? 'down' : 'up'; - - // This monitor doesn't match, so just skip ahead and don't add it to the output - // Only skip in case of up statusFilter, for a monitor to be up, all checks should be up - if (statusFilter === 'up' && statusFilter !== status) { - continue MonitorLoop; - } - groups.push({ - monitorId, - location, - checkGroup, - status, - summaryTimestamp: new Date(latestSource['@timestamp']), + summaryPings.push({ + docId: latest._id, + timestamp: latest._source['@timestamp'], + ...latest._source, }); } - // If one location matched, include data from all locations in the result set - if (matched) { - matching.set(monitorId, groups); + const someDown = summaryPings.some((p) => (p.summary?.down ?? 0) > 0); + const statusFilterOk = !statusFilter ? true : statusFilter === 'up' ? !someDown : someDown; + + if (matched && statusFilterOk) { + summaries.push(summaryPingsToSummary(summaryPings)); } } - return matching; + return summaries; +}; + +export const summaryPingsToSummary = (summaryPings: Ping[]): MonitorSummary => { + summaryPings.sort((a, b) => + a.timestamp > b.timestamp ? 1 : a.timestamp === b.timestamp ? 0 : -1 + ); + const latest = summaryPings[summaryPings.length - 1]; + return { + monitor_id: latest.monitor.id, + state: { + timestamp: latest.timestamp, + monitor: { + name: latest.monitor?.name, + }, + url: latest.url ?? {}, + summary: { + up: summaryPings.reduce((acc, p) => (p.summary?.up ?? 0) + acc, 0), + down: summaryPings.reduce((acc, p) => (p.summary?.down ?? 0) + acc, 0), + status: summaryPings.some((p) => (p.summary?.down ?? 0) > 0) ? 'down' : 'up', + }, + summaryPings, + tls: latest.tls, + // easier to ensure to use '' for an empty geo name in terms of types + observer: { + geo: { name: summaryPings.map((p) => p.observer?.geo?.name ?? '').filter((n) => n !== '') }, + }, + }, + }; }; export const query = async ( @@ -113,7 +115,11 @@ export const query = async ( }, aggs: { monitor: { - terms: { field: 'monitor.id', size: potentialMatchMonitorIDs.length }, + terms: { + field: 'monitor.id', + size: potentialMatchMonitorIDs.length, + order: { _key: queryContext.cursorOrder() }, + }, aggs: { location: { terms: { field: 'observer.geo.name', missing: 'N/A', size: 100 }, @@ -125,14 +131,6 @@ export const query = async ( latest: { top_hits: { sort: [{ '@timestamp': 'desc' }], - _source: { - includes: [ - 'monitor.check_group', - '@timestamp', - 'summary.up', - 'summary.down', - ], - }, size: 1, }, }, @@ -144,10 +142,8 @@ export const query = async ( aggs: { top: { top_hits: { + _source: ['@timestamp'], sort: [{ '@timestamp': 'desc' }], - _source: { - includes: ['monitor.check_group', '@timestamp'], - }, size: 1, }, }, diff --git a/x-pack/plugins/uptime/server/lib/requests/uptime_requests.ts b/x-pack/plugins/uptime/server/lib/requests/uptime_requests.ts index 85fc2c3ef9771..ae3a729e41c70 100644 --- a/x-pack/plugins/uptime/server/lib/requests/uptime_requests.ts +++ b/x-pack/plugins/uptime/server/lib/requests/uptime_requests.ts @@ -17,6 +17,7 @@ import { GetCertsParams, GetPingsParams, CertResult, + MonitorSummariesResult, } from '../../../common/runtime_types'; import { MonitorDurationResult } from '../../../common/types'; @@ -31,7 +32,6 @@ import { GetMonitorStatusParams, GetMonitorStatusResult, } from '.'; -import { GetMonitorStatesResult } from './get_monitor_states'; import { GetSnapshotCountParams } from './get_snapshot_counts'; import { IIndexPattern } from '../../../../../../src/plugins/data/server'; @@ -45,7 +45,7 @@ export interface UptimeRequests { getMonitorDurationChart: ESQ; getMonitorDetails: ESQ; getMonitorLocations: ESQ; - getMonitorStates: ESQ; + getMonitorStates: ESQ; getMonitorStatus: ESQ; getPings: ESQ; getPingHistogram: ESQ; diff --git a/x-pack/test/api_integration/apis/uptime/rest/fixtures/monitor_latest_status.json b/x-pack/test/api_integration/apis/uptime/rest/fixtures/monitor_latest_status.json index 1baff443bd97f..9a33be807670e 100644 --- a/x-pack/test/api_integration/apis/uptime/rest/fixtures/monitor_latest_status.json +++ b/x-pack/test/api_integration/apis/uptime/rest/fixtures/monitor_latest_status.json @@ -27,6 +27,5 @@ "full": "http://localhost:5678/pattern?r=200x1" }, "docId": "h5toHm0B0I9WX_CznN_V", - "timestamp": "2019-09-11T03:40:34.371Z", - "tls": {} -} + "timestamp": "2019-09-11T03:40:34.371Z" +} \ No newline at end of file diff --git a/x-pack/test/api_integration/apis/uptime/rest/monitor_states_generated.ts b/x-pack/test/api_integration/apis/uptime/rest/monitor_states_generated.ts index 34cf15e061569..3e06373042d59 100644 --- a/x-pack/test/api_integration/apis/uptime/rest/monitor_states_generated.ts +++ b/x-pack/test/api_integration/apis/uptime/rest/monitor_states_generated.ts @@ -53,18 +53,6 @@ export default function ({ getService }: FtrProviderContext) { nonSummaryIp = checks[0][0].monitor.ip; }); - it('should return all IPs', async () => { - const filters = makeApiParams(testMonitorId); - const url = getBaseUrl(dateRangeStart, dateRangeEnd) + `&filters=${filters}`; - const apiResponse = await supertest.get(url); - const res = apiResponse.body; - - const uniqueIps = new Set(); - res.summaries[0].state.checks.forEach((c: any) => uniqueIps.add(c.monitor.ip)); - - expect(uniqueIps.size).to.eql(4); - }); - it('should match non summary documents without a status filter', async () => { const filters = makeApiParams(testMonitorId, [{ match: { 'monitor.ip': nonSummaryIp } }]); diff --git a/x-pack/test/api_integration/apis/uptime/rest/monitor_states_real_data.ts b/x-pack/test/api_integration/apis/uptime/rest/monitor_states_real_data.ts index bcf4e63a9f6eb..d3c49bb49ff52 100644 --- a/x-pack/test/api_integration/apis/uptime/rest/monitor_states_real_data.ts +++ b/x-pack/test/api_integration/apis/uptime/rest/monitor_states_real_data.ts @@ -7,7 +7,7 @@ import expect from '@kbn/expect'; import { isRight } from 'fp-ts/lib/Either'; import { FtrProviderContext } from '../../../ftr_provider_context'; -import { MonitorSummaryResultType } from '../../../../../plugins/uptime/common/runtime_types'; +import { MonitorSummariesResultType } from '../../../../../plugins/uptime/common/runtime_types'; import { API_URLS } from '../../../../../plugins/uptime/common/constants'; interface ExpectedMonitorStatesPage { @@ -38,7 +38,7 @@ const checkMonitorStatesResponse = ({ prevPagination, nextPagination, }: ExpectedMonitorStatesPage) => { - const decoded = MonitorSummaryResultType.decode(response); + const decoded = MonitorSummariesResultType.decode(response); expect(isRight(decoded)).to.be.ok(); if (isRight(decoded)) { const { summaries, prevPagePagination, nextPagePagination, totalSummaryCount } = decoded.right; From 59d3096e55eb30f735e121160e7d3dba3706cfbf Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Thu, 9 Jul 2020 15:56:20 +0200 Subject: [PATCH 06/13] Test that savedObjectsTable exists AND isn't loading (#71207) * Wait until savedObjectsTable exists AND isn't loading * Use :not css selector Co-authored-by: Elastic Machine --- .../page_objects/management/saved_objects_page.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/functional/page_objects/management/saved_objects_page.ts b/test/functional/page_objects/management/saved_objects_page.ts index d058695ea6819..03d21aa4aa52f 100644 --- a/test/functional/page_objects/management/saved_objects_page.ts +++ b/test/functional/page_objects/management/saved_objects_page.ts @@ -87,13 +87,15 @@ export function SavedObjectsPageProvider({ getService, getPageObjects }: FtrProv async waitTableIsLoaded() { return retry.try(async () => { - const exists = await find.existsByDisplayedByCssSelector( - '*[data-test-subj="savedObjectsTable"] .euiBasicTable-loading' + const isLoaded = await find.existsByDisplayedByCssSelector( + '*[data-test-subj="savedObjectsTable"] :not(.euiBasicTable-loading)' ); - if (exists) { + + if (isLoaded) { + return true; + } else { throw new Error('Waiting'); } - return true; }); } From cf3133a5df60a51d9592b66697d74bc526165c15 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen <43350163+qn895@users.noreply.github.com> Date: Thu, 9 Jul 2020 09:15:57 -0500 Subject: [PATCH 07/13] [ML] Add switch to enable model plot annotations independently (#70678) --- .../types/anomaly_detection_jobs/job.ts | 2 +- .../new_job/common/job_creator/job_creator.ts | 30 +++++++-- .../advanced_section/advanced_section.tsx | 3 + .../annotations/annotations_switch.tsx | 65 +++++++++++++++++++ .../components/annotations/description.tsx | 34 ++++++++++ .../components/annotations/index.ts | 7 ++ .../jobs/new_job/pages/new_job/page.tsx | 1 + .../ml/server/routes/anomaly_detectors.ts | 4 +- 8 files changed, 137 insertions(+), 9 deletions(-) create mode 100644 x-pack/plugins/ml/public/application/jobs/new_job/pages/components/job_details_step/components/advanced_section/components/annotations/annotations_switch.tsx create mode 100644 x-pack/plugins/ml/public/application/jobs/new_job/pages/components/job_details_step/components/advanced_section/components/annotations/description.tsx create mode 100644 x-pack/plugins/ml/public/application/jobs/new_job/pages/components/job_details_step/components/advanced_section/components/annotations/index.ts diff --git a/x-pack/plugins/ml/common/types/anomaly_detection_jobs/job.ts b/x-pack/plugins/ml/common/types/anomaly_detection_jobs/job.ts index e2c4f1bae1a10..7c08daa9f05c5 100644 --- a/x-pack/plugins/ml/common/types/anomaly_detection_jobs/job.ts +++ b/x-pack/plugins/ml/common/types/anomaly_detection_jobs/job.ts @@ -80,7 +80,7 @@ export interface DataDescription { } export interface ModelPlotConfig { - enabled: boolean; + enabled?: boolean; annotations_enabled?: boolean; terms?: string; } diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator.ts index d8c4dab150fb5..29e8aafffef7e 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator.ts @@ -226,16 +226,23 @@ export class JobCreator { this._calendars = calendars; } - public set modelPlot(enable: boolean) { - if (enable) { - this._job_config.model_plot_config = { - enabled: true, - }; - } else { - delete this._job_config.model_plot_config; + private _initModelPlotConfig() { + // initialize configs to false if they are missing + if (this._job_config.model_plot_config === undefined) { + this._job_config.model_plot_config = {}; + } + if (this._job_config.model_plot_config.enabled === undefined) { + this._job_config.model_plot_config.enabled = false; + } + if (this._job_config.model_plot_config.annotations_enabled === undefined) { + this._job_config.model_plot_config.annotations_enabled = false; } } + public set modelPlot(enable: boolean) { + this._initModelPlotConfig(); + this._job_config.model_plot_config!.enabled = enable; + } public get modelPlot() { return ( this._job_config.model_plot_config !== undefined && @@ -243,6 +250,15 @@ export class JobCreator { ); } + public set modelChangeAnnotations(enable: boolean) { + this._initModelPlotConfig(); + this._job_config.model_plot_config!.annotations_enabled = enable; + } + + public get modelChangeAnnotations() { + return this._job_config.model_plot_config?.annotations_enabled === true; + } + public set useDedicatedIndex(enable: boolean) { this._useDedicatedIndex = enable; if (enable) { diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/job_details_step/components/advanced_section/advanced_section.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/job_details_step/components/advanced_section/advanced_section.tsx index 9a158f78c39be..18bd6f7fc6e23 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/job_details_step/components/advanced_section/advanced_section.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/job_details_step/components/advanced_section/advanced_section.tsx @@ -14,6 +14,7 @@ import { EuiHorizontalRule, } from '@elastic/eui'; import { ModelPlotSwitch } from './components/model_plot'; +import { AnnotationsSwitch } from './components/annotations'; import { DedicatedIndexSwitch } from './components/dedicated_index'; import { ModelMemoryLimitInput } from '../../../common/model_memory_limit'; import { JobCreatorContext } from '../../../job_creator_context'; @@ -41,6 +42,7 @@ export const AdvancedSection: FC = ({ advancedExpanded, setAdvancedExpand + @@ -68,6 +70,7 @@ export const AdvancedSection: FC = ({ advancedExpanded, setAdvancedExpand > + diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/job_details_step/components/advanced_section/components/annotations/annotations_switch.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/job_details_step/components/advanced_section/components/annotations/annotations_switch.tsx new file mode 100644 index 0000000000000..9defbb12207e2 --- /dev/null +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/job_details_step/components/advanced_section/components/annotations/annotations_switch.tsx @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC, useState, useContext, useEffect } from 'react'; +import { i18n } from '@kbn/i18n'; +import { EuiCallOut, EuiSpacer, EuiSwitch } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { JobCreatorContext } from '../../../../../job_creator_context'; +import { Description } from './description'; + +export const AnnotationsSwitch: FC = () => { + const { jobCreator, jobCreatorUpdate, jobCreatorUpdated } = useContext(JobCreatorContext); + const [annotationsEnabled, setAnnotationsEnabled] = useState(jobCreator.modelChangeAnnotations); + const [showCallOut, setShowCallout] = useState( + jobCreator.modelPlot && !jobCreator.modelChangeAnnotations + ); + + useEffect(() => { + jobCreator.modelChangeAnnotations = annotationsEnabled; + jobCreatorUpdate(); + }, [annotationsEnabled]); + + useEffect(() => { + setShowCallout(jobCreator.modelPlot && !annotationsEnabled); + }, [jobCreatorUpdated, annotationsEnabled]); + + function toggleAnnotations() { + setAnnotationsEnabled(!annotationsEnabled); + } + + return ( + <> + + + + {showCallOut && ( + + } + color="primary" + iconType="help" + /> + )} + + + ); +}; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/job_details_step/components/advanced_section/components/annotations/description.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/job_details_step/components/advanced_section/components/annotations/description.tsx new file mode 100644 index 0000000000000..92b07ff8d0910 --- /dev/null +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/job_details_step/components/advanced_section/components/annotations/description.tsx @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { memo, FC } from 'react'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { EuiDescribedFormGroup, EuiFormRow } from '@elastic/eui'; + +export const Description: FC = memo(({ children }) => { + const title = i18n.translate( + 'xpack.ml.newJob.wizard.jobDetailsStep.advancedSection.enableModelPlotAnnotations.title', + { + defaultMessage: 'Enable model change annotations', + } + ); + return ( + {title}} + description={ + + } + > + + <>{children} + + + ); +}); diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/job_details_step/components/advanced_section/components/annotations/index.ts b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/job_details_step/components/advanced_section/components/annotations/index.ts new file mode 100644 index 0000000000000..04bd97e140055 --- /dev/null +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/job_details_step/components/advanced_section/components/annotations/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { AnnotationsSwitch } from './annotations_switch'; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx index 171c7bbdd550c..48b044e5371de 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx @@ -125,6 +125,7 @@ export const Page: FC = ({ existingJobsAndGroups, jobType }) => { if (jobCreator.type === JOB_TYPE.SINGLE_METRIC) { jobCreator.modelPlot = true; + jobCreator.modelChangeAnnotations = true; } if (mlContext.currentSavedSearch !== null) { diff --git a/x-pack/plugins/ml/server/routes/anomaly_detectors.ts b/x-pack/plugins/ml/server/routes/anomaly_detectors.ts index 0b544d4eca0ed..78e05c9a6d07b 100644 --- a/x-pack/plugins/ml/server/routes/anomaly_detectors.ts +++ b/x-pack/plugins/ml/server/routes/anomaly_detectors.ts @@ -175,9 +175,11 @@ export function jobRoutes({ router, mlLicense }: RouteInitialization) { mlLicense.fullLicenseAPIGuard(async (context, request, response) => { try { const { jobId } = request.params; + const body = request.body; + const results = await context.ml!.mlClient.callAsCurrentUser('ml.addJob', { jobId, - body: request.body, + body, }); return response.ok({ body: results, From fc62f4fee0731d3b6323a0874f34d3bf26266869 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Thu, 9 Jul 2020 16:26:45 +0200 Subject: [PATCH 08/13] [ML] string_utils cleanup. (#71092) - Converts string_utils to TypeScript. - Removes sortByKey() from string_utils, we no longer make use of it. - Fixes #69499, stringMatch() was defined twice, now moved to string_utils. - Fixes #69498, OMIT_FIELDS was defined twice, now moved to common/constants/field_types.ts. --- .../ml/common/constants/field_types.ts | 3 + .../types/anomaly_detection_jobs/job.ts | 2 +- .../form_options_validation.ts | 3 - .../supported_fields_message.tsx | 3 +- .../analytics_list/analytics_list.tsx | 9 +-- .../index_based/data_loader/data_loader.ts | 3 +- .../jobs/jobs_list/components/utils.js | 9 +-- .../public/application/util/string_utils.d.ts | 21 ------ .../application/util/string_utils.test.ts | 60 +++------------- .../util/{string_utils.js => string_utils.ts} | 72 +++++++++---------- 10 files changed, 53 insertions(+), 132 deletions(-) delete mode 100644 x-pack/plugins/ml/public/application/util/string_utils.d.ts rename x-pack/plugins/ml/public/application/util/{string_utils.js => string_utils.ts} (75%) diff --git a/x-pack/plugins/ml/common/constants/field_types.ts b/x-pack/plugins/ml/common/constants/field_types.ts index 9402e4c20e46f..93641fd45c499 100644 --- a/x-pack/plugins/ml/common/constants/field_types.ts +++ b/x-pack/plugins/ml/common/constants/field_types.ts @@ -17,3 +17,6 @@ export enum ML_JOB_FIELD_TYPES { export const MLCATEGORY = 'mlcategory'; export const DOC_COUNT = 'doc_count'; + +// List of system fields we don't want to display. +export const OMIT_FIELDS: string[] = ['_source', '_type', '_index', '_id', '_version', '_score']; diff --git a/x-pack/plugins/ml/common/types/anomaly_detection_jobs/job.ts b/x-pack/plugins/ml/common/types/anomaly_detection_jobs/job.ts index 7c08daa9f05c5..744f9c4d759dd 100644 --- a/x-pack/plugins/ml/common/types/anomaly_detection_jobs/job.ts +++ b/x-pack/plugins/ml/common/types/anomaly_detection_jobs/job.ts @@ -65,7 +65,7 @@ export interface Detector { function: string; over_field_name?: string; partition_field_name?: string; - use_null?: string; + use_null?: boolean; custom_rules?: CustomRule[]; } export interface AnalysisLimits { diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/form_options_validation.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/form_options_validation.ts index bf3ab01549139..0935ed15a1a4a 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/form_options_validation.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/form_options_validation.ts @@ -12,9 +12,6 @@ import { BASIC_NUMERICAL_TYPES, EXTENDED_NUMERICAL_TYPES } from '../../../../com export const CATEGORICAL_TYPES = new Set(['ip', 'keyword']); -// List of system fields we want to ignore for the numeric field check. -export const OMIT_FIELDS: string[] = ['_source', '_type', '_index', '_id', '_version', '_score']; - // Regression supports numeric fields. Classification supports categorical, numeric, and boolean. export const shouldAddAsDepVarOption = (field: Field, jobType: AnalyticsJobType) => { if (field.id === EVENT_RATE_FIELD_ID) return false; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/supported_fields_message.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/supported_fields_message.tsx index 0a4ba67831818..88c89df86b29a 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/supported_fields_message.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/supported_fields_message.tsx @@ -11,8 +11,9 @@ import { FormattedMessage } from '@kbn/i18n/react'; import { AnalyticsJobType } from '../../../analytics_management/hooks/use_create_analytics_form/state'; import { ANALYSIS_CONFIG_TYPE } from '../../../../common/analytics'; import { Field, EVENT_RATE_FIELD_ID } from '../../../../../../../common/types/fields'; +import { OMIT_FIELDS } from '../../../../../../../common/constants/field_types'; import { BASIC_NUMERICAL_TYPES, EXTENDED_NUMERICAL_TYPES } from '../../../../common/fields'; -import { OMIT_FIELDS, CATEGORICAL_TYPES } from './form_options_validation'; +import { CATEGORICAL_TYPES } from './form_options_validation'; import { ES_FIELD_TYPES } from '../../../../../../../../../../src/plugins/data/public'; import { newJobCapsService } from '../../../../../services/new_job_capabilities_service'; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_list/analytics_list.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_list/analytics_list.tsx index 405231aef5774..4080f6cd7a77e 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_list/analytics_list.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_list/analytics_list.tsx @@ -39,6 +39,7 @@ import { import { getAnalyticsFactory } from '../../services/analytics_service'; import { getTaskStateBadge, getJobTypeBadge, useColumns } from './use_columns'; import { ExpandedRow } from './expanded_row'; +import { stringMatch } from '../../../../../util/string_utils'; import { ProgressBar, mlInMemoryTableFactory, @@ -65,14 +66,6 @@ function getItemIdToExpandedRowMap( }, {} as ItemIdToExpandedRowMap); } -function stringMatch(str: string | undefined, substr: any) { - return ( - typeof str === 'string' && - typeof substr === 'string' && - (str.toLowerCase().match(substr.toLowerCase()) === null) === false - ); -} - const MlInMemoryTable = mlInMemoryTableFactory(); interface Props { diff --git a/x-pack/plugins/ml/public/application/datavisualizer/index_based/data_loader/data_loader.ts b/x-pack/plugins/ml/public/application/datavisualizer/index_based/data_loader/data_loader.ts index 7d1f456d2334f..a08821c65bfe7 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/index_based/data_loader/data_loader.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/index_based/data_loader/data_loader.ts @@ -10,13 +10,12 @@ import { getToastNotifications } from '../../../util/dependency_cache'; import { IndexPattern } from '../../../../../../../../src/plugins/data/public'; import { SavedSearchQuery } from '../../../contexts/ml'; +import { OMIT_FIELDS } from '../../../../../common/constants/field_types'; import { IndexPatternTitle } from '../../../../../common/types/kibana'; import { ml } from '../../../services/ml_api_service'; import { FieldRequestConfig } from '../common'; -// List of system fields we don't want to display. -const OMIT_FIELDS: string[] = ['_source', '_type', '_index', '_id', '_version', '_score']; // Maximum number of examples to obtain for text type fields. const MAX_EXAMPLES_DEFAULT: number = 10; diff --git a/x-pack/plugins/ml/public/application/jobs/jobs_list/components/utils.js b/x-pack/plugins/ml/public/application/jobs/jobs_list/components/utils.js index 15c54fc5b3a46..569eca4aba949 100644 --- a/x-pack/plugins/ml/public/application/jobs/jobs_list/components/utils.js +++ b/x-pack/plugins/ml/public/application/jobs/jobs_list/components/utils.js @@ -11,6 +11,7 @@ import rison from 'rison-node'; import { mlJobService } from '../../../services/job_service'; import { ml } from '../../../services/ml_api_service'; import { getToastNotifications } from '../../../util/dependency_cache'; +import { stringMatch } from '../../../util/string_utils'; import { JOB_STATE, DATAFEED_STATE } from '../../../../../common/constants/states'; import { parseInterval } from '../../../../../common/util/parse_interval'; import { i18n } from '@kbn/i18n'; @@ -350,14 +351,6 @@ export function checkForAutoStartDatafeed() { } } -function stringMatch(str, substr) { - return ( - typeof str === 'string' && - typeof substr === 'string' && - (str.toLowerCase().match(substr.toLowerCase()) === null) === false - ); -} - function jobProperty(job, prop) { const propMap = { job_state: 'jobState', diff --git a/x-pack/plugins/ml/public/application/util/string_utils.d.ts b/x-pack/plugins/ml/public/application/util/string_utils.d.ts deleted file mode 100644 index 531e44e3e78c1..0000000000000 --- a/x-pack/plugins/ml/public/application/util/string_utils.d.ts +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -export function escapeForElasticsearchQuery(str: string): string; - -export function replaceStringTokens( - str: string, - valuesByTokenName: {}, - encodeForURI: boolean -): string; - -export function detectorToString(dtr: any): string; - -export function sortByKey(list: any, reverse: boolean, comparator?: any): any; - -export function toLocaleString(x: number): string; - -export function mlEscape(str: string): string; diff --git a/x-pack/plugins/ml/public/application/util/string_utils.test.ts b/x-pack/plugins/ml/public/application/util/string_utils.test.ts index 25f1cbd3abac3..034c406afb4b2 100644 --- a/x-pack/plugins/ml/public/application/util/string_utils.test.ts +++ b/x-pack/plugins/ml/public/application/util/string_utils.test.ts @@ -4,10 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ +import { CustomUrlAnomalyRecordDoc } from '../../../common/types/custom_urls'; +import { Detector } from '../../../common/types/anomaly_detection_jobs'; + import { replaceStringTokens, detectorToString, - sortByKey, toLocaleString, mlEscape, escapeForElasticsearchQuery, @@ -15,7 +17,7 @@ import { describe('ML - string utils', () => { describe('replaceStringTokens', () => { - const testRecord = { + const testRecord: CustomUrlAnomalyRecordDoc = { job_id: 'test_job', result_type: 'record', probability: 0.0191711, @@ -30,6 +32,10 @@ describe('ML - string utils', () => { testfield1: 'test$tring=[+-?]', testfield2: '{<()>}', testfield3: 'host=\\\\test@uk.dev', + earliest: '0', + latest: '0', + is_interim: false, + initial_record_score: 0, }; test('returns correct values without URI encoding', () => { @@ -68,17 +74,17 @@ describe('ML - string utils', () => { describe('detectorToString', () => { test('returns the correct descriptions for detectors', () => { - const detector1 = { + const detector1: Detector = { function: 'count', }; - const detector2 = { + const detector2: Detector = { function: 'count', by_field_name: 'airline', use_null: false, }; - const detector3 = { + const detector3: Detector = { function: 'mean', field_name: 'CPUUtilization', partition_field_name: 'region', @@ -95,50 +101,6 @@ describe('ML - string utils', () => { }); }); - describe('sortByKey', () => { - const obj = { - zebra: 'stripes', - giraffe: 'neck', - elephant: 'trunk', - }; - - const valueComparator = function (value: string) { - return value; - }; - - test('returns correct ordering with default comparator', () => { - const result = sortByKey(obj, false); - const keys = Object.keys(result); - expect(keys[0]).toBe('elephant'); - expect(keys[1]).toBe('giraffe'); - expect(keys[2]).toBe('zebra'); - }); - - test('returns correct ordering with default comparator and order reversed', () => { - const result = sortByKey(obj, true); - const keys = Object.keys(result); - expect(keys[0]).toBe('zebra'); - expect(keys[1]).toBe('giraffe'); - expect(keys[2]).toBe('elephant'); - }); - - test('returns correct ordering with comparator', () => { - const result = sortByKey(obj, false, valueComparator); - const keys = Object.keys(result); - expect(keys[0]).toBe('giraffe'); - expect(keys[1]).toBe('zebra'); - expect(keys[2]).toBe('elephant'); - }); - - test('returns correct ordering with comparator and order reversed', () => { - const result = sortByKey(obj, true, valueComparator); - const keys = Object.keys(result); - expect(keys[0]).toBe('elephant'); - expect(keys[1]).toBe('zebra'); - expect(keys[2]).toBe('giraffe'); - }); - }); - describe('toLocaleString', () => { test('returns correct comma placement for large numbers', () => { expect(toLocaleString(1)).toBe('1'); diff --git a/x-pack/plugins/ml/public/application/util/string_utils.js b/x-pack/plugins/ml/public/application/util/string_utils.ts similarity index 75% rename from x-pack/plugins/ml/public/application/util/string_utils.js rename to x-pack/plugins/ml/public/application/util/string_utils.ts index 7411820ba3239..aa283fd71bf79 100644 --- a/x-pack/plugins/ml/public/application/util/string_utils.js +++ b/x-pack/plugins/ml/public/application/util/string_utils.ts @@ -10,6 +10,9 @@ import _ from 'lodash'; import d3 from 'd3'; +import { CustomUrlAnomalyRecordDoc } from '../../../common/types/custom_urls'; +import { Detector } from '../../../common/types/anomaly_detection_jobs'; + // Replaces all instances of dollar delimited tokens in the specified String // with corresponding values from the supplied object, optionally // encoding the replacement for a URI component. @@ -17,7 +20,11 @@ import d3 from 'd3'; // and valuesByTokenName of {"airline":"AAL"}, will return // 'http://www.google.co.uk/#q=airline+code+AAL'. // If a corresponding key is not found in valuesByTokenName, then the String is not replaced. -export function replaceStringTokens(str, valuesByTokenName, encodeForURI) { +export function replaceStringTokens( + str: string, + valuesByTokenName: CustomUrlAnomalyRecordDoc, + encodeForURI: boolean +) { return String(str).replace(/\$([^?&$\'"]+)\$/g, (match, name) => { // Use lodash get to allow nested JSON fields to be retrieved. let tokenValue = _.get(valuesByTokenName, name, null); @@ -31,7 +38,7 @@ export function replaceStringTokens(str, valuesByTokenName, encodeForURI) { } // creates the default description for a given detector -export function detectorToString(dtr) { +export function detectorToString(dtr: Detector): string { const BY_TOKEN = ' by '; const OVER_TOKEN = ' over '; const USE_NULL_OPTION = ' use_null='; @@ -73,7 +80,7 @@ export function detectorToString(dtr) { } // wrap a the inputed string in quotes if it contains non-word characters -function quoteField(field) { +function quoteField(field: string): string { if (field.match(/\W/g)) { return '"' + field + '"'; } else { @@ -81,28 +88,10 @@ function quoteField(field) { } } -// re-order an object based on the value of the keys -export function sortByKey(list, reverse, comparator) { - let keys = _.sortBy(_.keys(list), (key) => { - return comparator ? comparator(list[key], key) : key; - }); - - if (reverse) { - keys = keys.reverse(); - } - - return _.zipObject( - keys, - _.map(keys, (key) => { - return list[key]; - }) - ); -} - // add commas to large numbers // Number.toLocaleString is not supported on safari -export function toLocaleString(x) { - let result = x; +export function toLocaleString(x: number): string { + let result = x.toString(); if (x && typeof x === 'number') { const parts = x.toString().split('.'); parts[0] = parts[0].replace(/\B(?=(\d{3})+(?!\d))/g, ','); @@ -112,8 +101,8 @@ export function toLocaleString(x) { } // escape html characters -export function mlEscape(str) { - const entityMap = { +export function mlEscape(str: string): string { + const entityMap: { [escapeChar: string]: string } = { '&': '&', '<': '<', '>': '>', @@ -125,7 +114,7 @@ export function mlEscape(str) { } // Escapes reserved characters for use in Elasticsearch query terms. -export function escapeForElasticsearchQuery(str) { +export function escapeForElasticsearchQuery(str: string): string { // Escape with a leading backslash any of the characters that // Elastic document may cause a syntax error when used in queries: // + - = && || > < ! ( ) { } [ ] ^ " ~ * ? : \ / @@ -133,27 +122,24 @@ export function escapeForElasticsearchQuery(str) { return String(str).replace(/[-[\]{}()+!<>=?:\/\\^"~*&|\s]/g, '\\$&'); } -export function calculateTextWidth(txt, isNumber, elementSelection) { - txt = isNumber ? d3.format(',')(txt) : txt; - let svg = elementSelection; - let $el; - if (elementSelection === undefined) { - // Create a temporary selection to append the label to. - // Note styling of font will be inherited from CSS of page. - const $body = d3.select('body'); - $el = $body.append('div'); - svg = $el.append('svg'); - } +export function calculateTextWidth(txt: string | number, isNumber: boolean) { + txt = isNumber && typeof txt === 'number' ? d3.format(',')(txt) : txt; + + // Create a temporary selection to append the label to. + // Note styling of font will be inherited from CSS of page. + const $body = d3.select('body'); + const $el = $body.append('div'); + const svg = $el.append('svg'); const tempLabelText = svg .append('g') .attr('class', 'temp-axis-label tick') .selectAll('text.temp.axis') - .data('a') + .data(['a']) .enter() .append('text') .text(txt); - const width = tempLabelText[0][0].getBBox().width; + const width = (tempLabelText[0][0] as SVGSVGElement).getBBox().width; d3.select('.temp-axis-label').remove(); if ($el !== undefined) { @@ -161,3 +147,11 @@ export function calculateTextWidth(txt, isNumber, elementSelection) { } return Math.ceil(width); } + +export function stringMatch(str: string | undefined, substr: any) { + return ( + typeof str === 'string' && + typeof substr === 'string' && + (str.toLowerCase().match(substr.toLowerCase()) === null) === false + ); +} From 526de12e16ae77814105cc3ef35479226dc80832 Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Thu, 9 Jul 2020 07:44:21 -0700 Subject: [PATCH 09/13] Updates translations Signed-off-by: Tyler Smalley --- x-pack/plugins/translations/translations/ja-JP.json | 8 -------- x-pack/plugins/translations/translations/zh-CN.json | 8 -------- 2 files changed, 16 deletions(-) diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index ae37cc9dabc61..edc5cd04bef1e 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -13531,7 +13531,6 @@ "xpack.securitySolution.case.configureCases.mappingFieldComments": "コメント", "xpack.securitySolution.case.configureCases.mappingFieldDescription": "説明", "xpack.securitySolution.case.configureCases.mappingFieldNotMapped": "マップされません", - "xpack.securitySolution.case.configureCases.mappingFieldShortDescription": "短い説明", "xpack.securitySolution.case.configureCases.mappingFieldSummary": "まとめ", "xpack.securitySolution.case.configureCases.noConnector": "コネクターを選択していません", "xpack.securitySolution.case.configureCases.updateConnector": "コネクターを更新", @@ -13560,8 +13559,6 @@ "xpack.securitySolution.case.connectors.jira.projectKey": "プロジェクトキー", "xpack.securitySolution.case.connectors.jira.requiredProjectKeyTextField": "プロジェクトキーが必要です", "xpack.securitySolution.case.connectors.jira.selectMessageText": "JiraでSIEMケースデータを更新するか、新しいインシデントにプッシュ", - "xpack.securitySolution.case.connectors.servicenow.actionTypeTitle": "ServiceNow", - "xpack.securitySolution.case.connectors.servicenow.selectMessageText": "ServiceNow で Security ケースデータをb\\更新するか、または新しいインシデントにプッシュする", "xpack.securitySolution.case.createCase.descriptionFieldRequiredError": "説明が必要です。", "xpack.securitySolution.case.createCase.fieldTagsHelpText": "このケースの 1 つ以上のカスタム識別タグを入力します。新しいタグを開始するには、各タグの後でEnterを押します。", "xpack.securitySolution.case.createCase.titleFieldRequiredError": "タイトルが必要です。", @@ -13722,8 +13719,6 @@ "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldMitreThreatLabel": "MITRE ATT&CK\\u2122", "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldNameLabel": "名前", "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldReferenceUrlsLabel": "参照URL", - "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldRiskScoreLabel": "リスクスコア", - "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldSeverityLabel": "深刻度", "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldTagsHelpText": "このルールの1つ以上のカスタム識別タグを入力します。新しいタグを開始するには、各タグの後でEnterを押します。", "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldTagsLabel": "タグ", "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldTimelineTemplateHelpText": "生成されたシグナルを調査するときにテンプレートとして使用する既存のタイムラインを選択します。", @@ -14152,7 +14147,6 @@ "xpack.securitySolution.detectionEngine.rules.allRules.tabs.rules": "ルール", "xpack.securitySolution.detectionEngine.rules.backOptionsHeader": "検出に戻る", "xpack.securitySolution.detectionEngine.rules.components.ruleActionsOverflow.allActionsTitle": "すべてのアクション", - "xpack.securitySolution.detectionEngine.rules.components.ruleDownloader.exportFailureTitle": "ルールをエクスポートできませんでした...", "xpack.securitySolution.detectionEngine.rules.continueButtonTitle": "続行", "xpack.securitySolution.detectionEngine.rules.create.successfullyCreatedRuleTitle": "{ruleName}が作成されました", "xpack.securitySolution.detectionEngine.rules.defineRuleTitle": "ルールの定義", @@ -14506,7 +14500,6 @@ "xpack.securitySolution.open.timeline.untitledTimelineLabel": "無題のタイムライン", "xpack.securitySolution.open.timeline.withLabel": "With", "xpack.securitySolution.open.timeline.zeroTimelinesMatchLabel": "0 件のタイムラインが検索条件に一致", - "xpack.securitySolution.overview.alertsGraphTitle": "外部アラート数", "xpack.securitySolution.overview.auditBeatAuditTitle": "監査", "xpack.securitySolution.overview.auditBeatFimTitle": "File Integrityモジュール", "xpack.securitySolution.overview.auditBeatLoginTitle": "ログイン", @@ -14567,7 +14560,6 @@ "xpack.securitySolution.overview.winlogbeatSecurityTitle": "セキュリティ", "xpack.securitySolution.pages.common.emptyActionPrimary": "Beatsでデータを表示", "xpack.securitySolution.pages.common.emptyActionSecondary": "入門ガイドを表示", - "xpack.securitySolution.pages.common.emptyMessage": "セキュリティ情報とイベント管理（SIEM）を使用して開始するには、Elastic StackにElastic Common Schema（ECS）フォーマットでSIEM関連データを追加する必要があります。簡単に開始するには、Beatsと呼ばれるデータシッパーをインストールして設定するという方法があります。今すぐ始めましょう。", "xpack.securitySolution.pages.common.emptyTitle": "SIEMへようこそ。始めましょう。", "xpack.securitySolution.pages.fourohfour.noContentFoundDescription": "コンテンツがありません", "xpack.securitySolution.paginatedTable.rowsButtonLabel": "ページごとの行数", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 2bf5f48384403..c24cac9dae9d2 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -13537,7 +13537,6 @@ "xpack.securitySolution.case.configureCases.mappingFieldComments": "注释", "xpack.securitySolution.case.configureCases.mappingFieldDescription": "描述", "xpack.securitySolution.case.configureCases.mappingFieldNotMapped": "未映射", - "xpack.securitySolution.case.configureCases.mappingFieldShortDescription": "简短描述", "xpack.securitySolution.case.configureCases.mappingFieldSummary": "摘要", "xpack.securitySolution.case.configureCases.noConnector": "未选择连接器", "xpack.securitySolution.case.configureCases.updateConnector": "更新连接器", @@ -13566,8 +13565,6 @@ "xpack.securitySolution.case.connectors.jira.projectKey": "项目键", "xpack.securitySolution.case.connectors.jira.requiredProjectKeyTextField": "项目键必填。", "xpack.securitySolution.case.connectors.jira.selectMessageText": "将 Security 案例数据推送或更新到 Jira 中的新问题", - "xpack.securitySolution.case.connectors.servicenow.actionTypeTitle": "ServiceNow", - "xpack.securitySolution.case.connectors.servicenow.selectMessageText": "将 Security 案例数据推送或更新到 ServiceNow 中的新事件", "xpack.securitySolution.case.createCase.descriptionFieldRequiredError": "描述必填。", "xpack.securitySolution.case.createCase.fieldTagsHelpText": "为此案例键入一个或多个定制识别标记。在每个标记后按 Enter 键可开始新的标记。", "xpack.securitySolution.case.createCase.titleFieldRequiredError": "标题必填。", @@ -13728,8 +13725,6 @@ "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldMitreThreatLabel": "MITRE ATT&CK\\u2122", "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldNameLabel": "名称", "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldReferenceUrlsLabel": "引用 URL", - "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldRiskScoreLabel": "风险分数", - "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldSeverityLabel": "严重性", "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldTagsHelpText": "为此规则键入一个或多个定制识别标记。在每个标记后按 Enter 键可开始新的标记。", "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldTagsLabel": "标记", "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldTimelineTemplateHelpText": "选择现有时间线以将其用作调查生成的信号时的模板。", @@ -14158,7 +14153,6 @@ "xpack.securitySolution.detectionEngine.rules.allRules.tabs.rules": "规则", "xpack.securitySolution.detectionEngine.rules.backOptionsHeader": "返回到检测", "xpack.securitySolution.detectionEngine.rules.components.ruleActionsOverflow.allActionsTitle": "所有操作", - "xpack.securitySolution.detectionEngine.rules.components.ruleDownloader.exportFailureTitle": "无法导出规则……", "xpack.securitySolution.detectionEngine.rules.continueButtonTitle": "继续", "xpack.securitySolution.detectionEngine.rules.create.successfullyCreatedRuleTitle": "{ruleName} 已创建", "xpack.securitySolution.detectionEngine.rules.defineRuleTitle": "定义规则", @@ -14512,7 +14506,6 @@ "xpack.securitySolution.open.timeline.untitledTimelineLabel": "未命名时间线", "xpack.securitySolution.open.timeline.withLabel": "具有", "xpack.securitySolution.open.timeline.zeroTimelinesMatchLabel": "0 个时间线匹配搜索条件", - "xpack.securitySolution.overview.alertsGraphTitle": "外部告警计数", "xpack.securitySolution.overview.auditBeatAuditTitle": "审计", "xpack.securitySolution.overview.auditBeatFimTitle": "文件完整性模块", "xpack.securitySolution.overview.auditBeatLoginTitle": "登录", @@ -14573,7 +14566,6 @@ "xpack.securitySolution.overview.winlogbeatSecurityTitle": "安全", "xpack.securitySolution.pages.common.emptyActionPrimary": "使用 Beats 添加数据", "xpack.securitySolution.pages.common.emptyActionSecondary": "查看入门指南", - "xpack.securitySolution.pages.common.emptyMessage": "要开始使用安全信息和事件管理 (Security)，您将需要将 Security 相关数据以 Elastic Common Schema (ECS) 格式添加到 Elastic Stack。较为轻松的入门方式是安装并配置我们称作 Beats 的数据采集器。让我们现在就动手！", "xpack.securitySolution.pages.common.emptyTitle": "欢迎使用 SIEM。让我们教您如何入门。", "xpack.securitySolution.pages.fourohfour.noContentFoundDescription": "未找到任何内容", "xpack.securitySolution.paginatedTable.rowsButtonLabel": "每页行数", From 83c7d33120773ef4afbc8964cd2a68b2fc116aa4 Mon Sep 17 00:00:00 2001 From: Yuliia Naumenko Date: Thu, 9 Jul 2020 07:58:05 -0700 Subject: [PATCH 10/13] Changed "Elastic" to "Elastic Alerts" in PagerDuty action doc. (#71184) --- docs/user/alerting/action-types/pagerduty.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/user/alerting/action-types/pagerduty.asciidoc b/docs/user/alerting/action-types/pagerduty.asciidoc index 0468ab042e57e..5fd85a1045265 100644 --- a/docs/user/alerting/action-types/pagerduty.asciidoc +++ b/docs/user/alerting/action-types/pagerduty.asciidoc @@ -68,11 +68,11 @@ Then, select the *Integrations* tab and click the *New Integration* button. * If you are creating a new service for your integration, go to https://support.pagerduty.com/docs/services-and-integrations#section-configuring-services-and-integrations[Configuring Services and Integrations] -and follow the steps outlined in the *Create a New Service* section, selecting *Elastic* as the *Integration Type* in step 4. +and follow the steps outlined in the *Create a New Service* section, selecting *Elastic Alerts* as the *Integration Type* in step 4. Continue with the <> section once you have finished these steps. . Enter an *Integration Name* in the format Elastic-service-name (for example, Elastic-Alerting or Kibana-APM-Alerting) -and select Elastic from the *Integration Type* menu. +and select *Elastic Alerts* from the *Integration Type* menu. . Click *Add Integration* to save your new integration. + You will be redirected to the *Integrations* tab for your service. An Integration Key is generated on this screen. From f9da7afc4338880b70f89f93f521f6b19af41902 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Thu, 9 Jul 2020 16:01:44 +0100 Subject: [PATCH 11/13] skip flaky suite (#65949) --- x-pack/test/functional/apps/dashboard/_async_dashboard.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/functional/apps/dashboard/_async_dashboard.ts b/x-pack/test/functional/apps/dashboard/_async_dashboard.ts index b4dfffcdeff57..cc30a7a7e640f 100644 --- a/x-pack/test/functional/apps/dashboard/_async_dashboard.ts +++ b/x-pack/test/functional/apps/dashboard/_async_dashboard.ts @@ -27,7 +27,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { 'timePicker', ]); - describe('sample data dashboard', function describeIndexTests() { + // Flakky: https://github.com/elastic/kibana/issues/65949 + describe.skip('sample data dashboard', function describeIndexTests() { before(async () => { await PageObjects.common.sleep(5000); await PageObjects.common.navigateToUrl('home', '/tutorial_directory/sampleData', { From 7145216f7a1b6fd44f1a9d37b940d169027f6eec Mon Sep 17 00:00:00 2001 From: nnamdifrankie <56440728+nnamdifrankie@users.noreply.github.com> Date: Thu, 9 Jul 2020 11:04:49 -0400 Subject: [PATCH 12/13] [SecuritySolution - Endpoint]: update payload for metadata and policy response tests (#71084) [SecuritySolution - Endpoint]: update payload for metadata and policy response tests --- .../endpoint/metadata/api_feature/data.json | 153 ++++++++++++++++-- .../es_archives/endpoint/policy/data.json.gz | Bin 1323 -> 1545 bytes 2 files changed, 144 insertions(+), 9 deletions(-) diff --git a/x-pack/test/functional/es_archives/endpoint/metadata/api_feature/data.json b/x-pack/test/functional/es_archives/endpoint/metadata/api_feature/data.json index ac35ef3dcab99..60679f9072c74 100644 --- a/x-pack/test/functional/es_archives/endpoint/metadata/api_feature/data.json +++ b/x-pack/test/functional/es_archives/endpoint/metadata/api_feature/data.json @@ -16,6 +16,7 @@ } }, "Endpoint": { + "status": "enrolled", "policy": { "applied": { "name": "Default", @@ -25,11 +26,23 @@ } }, "event": { - "created": 1579881969541 + "created": 1579881969541, + "id": "32f5fda2-48e4-4fae-b89e-a18038294d14", + "kind": "metric", + "category": [ + "host" + ], + "type": [ + "info" + ], + "module": "endpoint", + "action": "endpoint_metadata", + "dataset": "endpoint.metadata" }, "host": { "architecture": "x86", "hostname": "cadmann-4.example.com", + "name": "cadmann-4.example.com", "id": "1fb3e58f-6ab0-4406-9d2a-91911207a712", "ip": [ "10.192.213.130", @@ -43,6 +56,8 @@ "os": { "full": "Windows 10", "name": "windows 10.0", + "platform": "Windows", + "family": "Windows", "version": "10.0", "Ext": { "variant" : "Windows Pro" @@ -71,6 +86,7 @@ } }, "Endpoint": { + "status": "enrolled", "policy": { "applied": { "name": "Default", @@ -80,11 +96,23 @@ } }, "event": { - "created": 1579881969541 + "created": 1579881969541, + "id": "32f5fda2-48e4-4fae-b89e-a18038294d15", + "kind": "metric", + "category": [ + "host" + ], + "type": [ + "info" + ], + "module": "endpoint", + "action": "endpoint_metadata", + "dataset": "endpoint.metadata" }, "host": { "architecture": "x86_64", "hostname": "thurlow-9.example.com", + "name": "thurlow-9.example.com", "id": "2f735e3d-be14-483b-9822-bad06e9045ca", "ip": [ "10.46.229.234" @@ -97,6 +125,8 @@ "os": { "full": "Windows Server 2016", "name": "windows 10.0", + "platform": "Windows", + "family": "Windows", "version": "10.0", "Ext": { "variant" : "Windows Server" @@ -125,6 +155,7 @@ } }, "Endpoint": { + "status": "enrolled", "policy": { "applied": { "name": "Default", @@ -134,10 +165,22 @@ } }, "event": { - "created": 1579881969541 + "created": 1579881969541, + "id": "32f5fda2-48e4-4fae-b89e-a18038294d16", + "kind": "metric", + "category": [ + "host" + ], + "type": [ + "info" + ], + "module": "endpoint", + "action": "endpoint_metadata", + "dataset": "endpoint.metadata" }, "host": { "hostname": "rezzani-7.example.com", + "name": "rezzani-7.example.com", "id": "fc0ff548-feba-41b6-8367-65e8790d0eaf", "ip": [ "10.101.149.26", @@ -149,6 +192,8 @@ "os": { "full": "Windows 10", "name": "windows 10.0", + "platform": "Windows", + "family": "Windows", "version": "10.0", "Ext": { "variant" : "Windows Pro" @@ -177,6 +222,7 @@ } }, "Endpoint": { + "status": "enrolled", "policy": { "applied": { "name": "Default", @@ -186,11 +232,23 @@ } }, "event": { - "created": 1579878369541 + "created": 1579878369541, + "id": "32f5fda2-48e4-4fae-b89e-a18038294d18", + "kind": "metric", + "category": [ + "host" + ], + "type": [ + "info" + ], + "module": "endpoint", + "action": "endpoint_metadata", + "dataset": "endpoint.metadata" }, "host": { "architecture": "x86", "hostname": "cadmann-4.example.com", + "name": "cadmann-4.example.com", "id": "1fb3e58f-6ab0-4406-9d2a-91911207a712", "ip": [ "10.192.213.130", @@ -204,6 +262,8 @@ "os": { "full": "Windows Server 2016", "name": "windows 10.0", + "platform": "Windows", + "family": "Windows", "version": "10.0", "Ext": { "variant" : "Windows Server 2016" @@ -232,6 +292,7 @@ } }, "Endpoint": { + "status": "enrolled", "policy": { "applied": { "name": "Default", @@ -241,10 +302,22 @@ } }, "event": { - "created": 1579878369541 + "created": 1579878369541, + "id": "32f5fda2-48e4-4fae-b89e-a18038294d19", + "kind": "metric", + "category": [ + "host" + ], + "type": [ + "info" + ], + "module": "endpoint", + "action": "endpoint_metadata", + "dataset": "endpoint.metadata" }, "host": { "hostname": "thurlow-9.example.com", + "name": "thurlow-9.example.com", "id": "2f735e3d-be14-483b-9822-bad06e9045ca", "ip": [ "10.46.229.234" @@ -257,6 +330,8 @@ "os": { "full": "Windows Server 2012", "name": "windows 6.2", + "platform": "Windows", + "family": "Windows", "version": "6.2", "Ext": { "variant" : "Windows Server 2012" @@ -285,6 +360,7 @@ } }, "Endpoint": { + "status": "enrolled", "policy": { "applied": { "name": "With Eventing", @@ -294,11 +370,23 @@ } }, "event": { - "created": 1579878369541 + "created": 1579878369541, + "id": "32f5fda2-48e4-4fae-b89e-a18038294d39", + "kind": "metric", + "category": [ + "host" + ], + "type": [ + "info" + ], + "module": "endpoint", + "action": "endpoint_metadata", + "dataset": "endpoint.metadata" }, "host": { "architecture": "x86", "hostname": "rezzani-7.example.com", + "name": "rezzani-7.example.com", "id": "fc0ff548-feba-41b6-8367-65e8790d0eaf", "ip": [ "10.101.149.26", @@ -310,6 +398,8 @@ "os": { "full": "Windows Server 2012", "name": "windows 6.2", + "platform": "Windows", + "family": "Windows", "version": "6.2", "Ext": { "variant" : "Windows Server 2012" @@ -338,6 +428,7 @@ } }, "Endpoint": { + "status": "enrolled", "policy": { "applied": { "name": "With Eventing", @@ -347,10 +438,22 @@ } }, "event": { - "created": 1579874769541 + "created": 1579874769541, + "id": "32f5fda2-48e4-4fae-b89e-a18038294d31", + "kind": "metric", + "category": [ + "host" + ], + "type": [ + "info" + ], + "module": "endpoint", + "action": "endpoint_metadata", + "dataset": "endpoint.metadata" }, "host": { "hostname": "cadmann-4.example.com", + "name": "cadmann-4.example.com", "id": "1fb3e58f-6ab0-4406-9d2a-91911207a712", "ip": [ "10.192.213.130", @@ -364,6 +467,8 @@ "os": { "full": "Windows Server 2012R2", "name": "windows 6.3", + "platform": "Windows", + "family": "Windows", "version": "6.3", "Ext": { "variant" : "Windows Server 2012 R2" @@ -392,6 +497,7 @@ } }, "Endpoint": { + "status": "enrolled", "policy": { "applied": { "name": "Default", @@ -401,10 +507,22 @@ } }, "event": { - "created": 1579874769541 + "created": 1579874769541, + "id": "32f5fda2-48e4-4fae-b89e-a18038294d23", + "kind": "metric", + "category": [ + "host" + ], + "type": [ + "info" + ], + "module": "endpoint", + "action": "endpoint_metadata", + "dataset": "endpoint.metadata" }, "host": { "hostname": "thurlow-9.example.com", + "name": "thurlow-9.example.com", "id": "2f735e3d-be14-483b-9822-bad06e9045ca", "ip": [ "10.46.229.234" @@ -417,6 +535,8 @@ "os": { "full": "Windows Server 2012R2", "name": "windows 6.3", + "platform": "Windows", + "family": "Windows", "version": "6.3", "Ext": { "variant" : "Windows Server 2012 R2" @@ -445,6 +565,7 @@ } }, "Endpoint": { + "status": "enrolled", "policy": { "applied": { "name": "With Eventing", @@ -454,11 +575,23 @@ } }, "event": { - "created": 1579874769541 + "created": 1579874769541, + "id": "32f5fda2-48e4-4fae-b89e-a18038294d35", + "kind": "metric", + "category": [ + "host" + ], + "type": [ + "info" + ], + "module": "endpoint", + "action": "endpoint_metadata", + "dataset": "endpoint.metadata" }, "host": { "architecture": "x86", "hostname": "rezzani-7.example.com", + "name": "rezzani-7.example.com", "id": "fc0ff548-feba-41b6-8367-65e8790d0eaf", "ip": [ "10.101.149.26", @@ -471,6 +604,8 @@ "full": "Windows Server 2012", "name": "windows 6.2", "version": "6.2", + "platform": "Windows", + "family": "Windows", "Ext": { "variant" : "Windows Server 2012" } diff --git a/x-pack/test/functional/es_archives/endpoint/policy/data.json.gz b/x-pack/test/functional/es_archives/endpoint/policy/data.json.gz index 1f492487d317b7522608650c2c2b8ca9e9278280..88c7995a2c26c7646ff35d8f9e8c23b5610719e0 100644 GIT binary patch literal 1545 zcmV+k2KMQOZ*BnXSxax@HW0q&R}en!0!u^hM|-K8I_POp zv|F@AQ5Y0S+1iyvL89Y0i~RSJdRfodPUT_`x&#q9GyG;aoFPAdy#rvuj|p9T2Ns0k z#Roa?h{C-b`6@odXF%la$A`~}s;__i>-g)ZN6!j>`t*61i%W@^&?hMrQJw}kgES@y z3u3-ZSQy|VB-BUykXP~<+o$-nf*ARM2NBIUiV`uZnvP=_x~*8cZhfdk(3Zx$p;IG6 zs)Dd(IuPj|hKBebx`u(E>&9Dh-( zz4X>wmmzO2Sj4}(B1)1lprkV_ComUYWxWY8g<0$e+kHwmrDqvMDc)_wcuaG(0WBa? zBFlu^(#4h{z$}U~P@5$n+hZ(*{e2i78~asT66wMB65FDbTy$h!x=IbuUuYWB@GQg! z6odi_XD48wRDBhZFZ6`$HHbpC-3IaYETrEWVQGe|(w{m~5eg3|rDt*d9@D2p?7VEgkkW*ue7Y@ef<2@Grx*N8$n{~B zQAMfD&D;u*R&I+nNtu*?V+IshOB=TW$_{ZJ&27j$6mG3AY$ZP zUA!J#Vf1oDvX>nJ0(q=h444o2j@HEp<%92v%JF_8uwqZdWu@HeO|jHB4mpqQg-Dx? z7$hjo1nd^LDKV9u;=+Aj=beVojHY=}m%XM=#qUAPfVjK{r0PG#xaa`_T_~bI5%84C zn$*1pT@u>Xv6p7Rx^pR-KFkH$t8%puv(UdCz9QSN!`FaUsNW7=dD*%YpC8bWfOnn)0_9XIygmFyGcL~EdAn#?C@<>%+_}YbooTP z*H*6L>T9^x92Xk*$_lPlh5?;>U0WHd@vyR2oyEnjWQllD4PB=fE%+l{_?7EKVF630 z$>#KO`+GpJ(*XJbYT-G(0S4s$8B%V(grr`8l_#a#zdE^bdoN-ZvqG>XspUr2}e)OeZ$u(c0lWzZ2)u><0Isy%L z$~BOxVCq6w#Tqm$*Mp9xQ>anJCcbVPo@IB{V-)4}_m6<@z*_En2lNSEpNGmbrAwo8!-kxCFWG*gENYY5slB8H*a#3WXe$omq+xt@$ii{ZoKd&|;e zCpi{7mhqDxjoqp^(3UjbA#u+imKPOc9~Pum#k)0TQN@SVvzvm5+tlr#Ax#iR8Rbp! zk~i(Min#{^9H`;zp6f^s)U^mSB!!4{&4QY)5ZktNVz;(Aw^MEN&*Fa=?Y!sb{n4@f zrq~4{^4c26aI1`mbd%j5s_mqbb9^NC5`2ZNIZe^qgI3ZC1q!h>0R~rHnGjp9<$p|IhUQfMBNq z3`k7M;!elv57dTBN!(jIIUPG+ABYjx27>Sg=dSwcVPYP>`FgwHCx_-!>f)cOMg-3V zcl5C_Y5rVQZTbes>3v0UJXa$Q^bjV{AXI~nX?f80d==@2W_ul?65O2L?=p8Mii_%4 vpmij(uR%jGO^C3kL8>bzMc8twug>)TOz+S1e!k*=ruY8^=_w@j-7f$D7A5}- literal 1323 zcmV+`1=RWQOZ*BnXm`iWtHW0_}`4ogtyFh5@LA}~b-PAem zq1~b_io&2s%Hc*91xb!KyU2Ghsh9PP?Lc1Spi2-_Gs7Q;!`J`&4FHSmIcAHWz=8_0 zxK#~LINYn2@A8NI$*KDK>HbTso9kcydH((LlW&K=eg5*0n=6ZmvaiZ$$FhWz6talM zf=Agh79l6kkg@>p!>qDT#Xcd&9%Rce8Q-xq!@F3v>Xz#wWICE{n)YpFg10Qn8a_1{ z)-_CQ%Z1qV2}JUL=plrm?NZODI+l9JEU5EmYK$m$AbPU6AG>^Hq8!T$K zmtKGCGUe?Ki~M(c#Bm&QMmx>w1Igs5(%uL8iHIoR+kL_|<;YTo6Y{W;?Gek}2E2eO zi8PhNRzqw#0wQ+16x3lUNcV(D<6s|#&yD%2Bgyi}_KIy$S}r;^FI}ew^dn0m7M?`B z$2^oMnK*)h()CrMK>A7BONc_T-STLA66uddTAJyq{HM;;4u=Pvu#;SWM(k@WXI`SP zBoRr?CZRwTWU{2G8asCqTN&dK&o~bGH_5q^KLQ?RESV^4lx2!b#$+22vuMMf{jc!eWtR(@k*`_K@(5olARM6biNA%mZ8(l<#E!a+zRyd0JIg zaiVH7j{;Pb>(a)FQ1)-EfP%G*aV=1Bi1KW1L*e6aBm3lm-0?3dfGP%9|Gfq~K~L?A zm%|l7&nHr6*)zZ(&y|RP2tfA0>dgq}jUS4gRyA+No{M{t1w{QxfD&G=7RR3TJ2*N{@3Fxs{Jy)243NQJ-qU=buM4PSxCW$(rZbX zzu!J_cK9eOGy{RyNAh5bO57F9pttU8^`SxIC0q8hqRFIxB`r|>J+i6R0~gQlC~ z%kAF*!_EQFPf&~J_y!oT`)|m(kzeT-sXI&0s#gd()c70dvf+*18ArxCaQ2K?j$WIu zot)PmVsA=Le+WNd$Rm%u;UTA)+=O=1R7Ag;PnP}YOTV&fzCtIR{)={_{$|!OXsA;j z!n#J72R)q_5ZRs&UBhJ1V46b%(?P!NbloGo%lF^Ec=iC+YUVr8N4&lcl~+nzgd%cM zvs`W(RQ8TSo^LoF!xqFk#Sj^m2{AD+bR0|(G8}4AyD8*7@+kK!M6OM})%KVkL^`M( zqQLY$S1E^{O(9ak8#WCa8m2}a$2O@WlT$^zss6s)yxiQVSm!FIVMjUrw*7QX<OvnA3K3-nbS>M5jvwgQM26$*+7DJnyUN{B;$kL|w5}3G h4Tv<$f|&RQWTs{@Ol*$@`a4lZ{|6b}@kdiC004dNmB0W1 From 8ef9833ac5cb66ca1a62a30a8e062677edbea649 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Thu, 9 Jul 2020 16:11:35 +0100 Subject: [PATCH 13/13] skip flaky suite (#69849) --- .../cypress/integration/alerts_detection_rules_export.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_export.spec.ts b/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_export.spec.ts index eb8448233c624..a7e6652613493 100644 --- a/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_export.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_export.spec.ts @@ -17,7 +17,8 @@ import { DETECTIONS_URL } from '../urls/navigation'; const EXPECTED_EXPORTED_RULE_FILE_PATH = 'cypress/test_files/expected_rules_export.ndjson'; -describe('Export rules', () => { +// Flakky: https://github.com/elastic/kibana/issues/69849 +describe.skip('Export rules', () => { before(() => { esArchiverLoad('export_rule'); cy.server();