From 09fd98a5d1a07fcc9d8128386acc5538fa645d72 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Tue, 17 Nov 2020 15:37:05 -0700 Subject: [PATCH 1/6] [Metrics UI] Add time range to inventory metadata request --- .../infra/common/http_api/inventory_meta_api.ts | 1 + .../metrics/inventory_view/components/layout.tsx | 2 +- .../components/toolbars/toolbar.tsx | 5 +++-- .../inventory_view/hooks/use_inventory_meta.ts | 7 ++++++- .../server/routes/inventory_metadata/index.ts | 5 +++-- .../inventory_metadata/lib/get_cloud_metadata.ts | 16 ++++++++++++++-- 6 files changed, 28 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/infra/common/http_api/inventory_meta_api.ts b/x-pack/plugins/infra/common/http_api/inventory_meta_api.ts index 77de515c9cc46..43f3b2037e381 100644 --- a/x-pack/plugins/infra/common/http_api/inventory_meta_api.ts +++ b/x-pack/plugins/infra/common/http_api/inventory_meta_api.ts @@ -21,6 +21,7 @@ export const InventoryMetaResponseRT = rt.type({ export const InventoryMetaRequestRT = rt.type({ sourceId: rt.string, nodeType: ItemTypeRT, + currentTime: rt.number, }); export type InventoryMetaRequest = rt.TypeOf; diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/layout.tsx b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/layout.tsx index 76512b8a366c5..dffaf64ba11b9 100644 --- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/layout.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/layout.tsx @@ -120,7 +120,7 @@ export const Layout = () => { <> - + diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/toolbars/toolbar.tsx b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/toolbars/toolbar.tsx index e9ffc56d8c47f..7bcb1270c30a5 100644 --- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/toolbars/toolbar.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/toolbars/toolbar.tsx @@ -54,11 +54,12 @@ const wrapToolbarItems = ( interface Props { nodeType: InventoryItemType; + currentTime: number; } -export const Toolbar = ({ nodeType }: Props) => { +export const Toolbar = ({ nodeType, currentTime }: Props) => { const { sourceId } = useSourceContext(); - const { accounts, regions } = useInventoryMeta(sourceId, nodeType); + const { accounts, regions } = useInventoryMeta(sourceId, nodeType, currentTime); const ToolbarItems = findToolbar(nodeType); return wrapToolbarItems(ToolbarItems, accounts, regions); }; diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_inventory_meta.ts b/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_inventory_meta.ts index b038491690a13..01811eb21a110 100644 --- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_inventory_meta.ts +++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_inventory_meta.ts @@ -15,7 +15,11 @@ import { } from '../../../../../common/http_api/inventory_meta_api'; import { InventoryItemType } from '../../../../../common/inventory_models/types'; -export function useInventoryMeta(sourceId: string, nodeType: InventoryItemType) { +export function useInventoryMeta( + sourceId: string, + nodeType: InventoryItemType, + currentTime: number +) { const decodeResponse = (response: any) => { return pipe( InventoryMetaResponseRT.decode(response), @@ -29,6 +33,7 @@ export function useInventoryMeta(sourceId: string, nodeType: InventoryItemType) JSON.stringify({ sourceId, nodeType, + currentTime, }), decodeResponse ); diff --git a/x-pack/plugins/infra/server/routes/inventory_metadata/index.ts b/x-pack/plugins/infra/server/routes/inventory_metadata/index.ts index 8b5271cb960c1..eff335fc2236f 100644 --- a/x-pack/plugins/infra/server/routes/inventory_metadata/index.ts +++ b/x-pack/plugins/infra/server/routes/inventory_metadata/index.ts @@ -33,7 +33,7 @@ export const initInventoryMetaRoute = (libs: InfraBackendLibs) => { }, async (requestContext, request, response) => { try { - const { sourceId, nodeType } = pipe( + const { sourceId, nodeType, currentTime } = pipe( InventoryMetaRequestRT.decode(request.body), fold(throwErrors(Boom.badRequest), identity) ); @@ -46,7 +46,8 @@ export const initInventoryMetaRoute = (libs: InfraBackendLibs) => { framework, requestContext, configuration, - nodeType + nodeType, + currentTime ); return response.ok({ diff --git a/x-pack/plugins/infra/server/routes/inventory_metadata/lib/get_cloud_metadata.ts b/x-pack/plugins/infra/server/routes/inventory_metadata/lib/get_cloud_metadata.ts index b4288dae0c221..7e8b26de75e8b 100644 --- a/x-pack/plugins/infra/server/routes/inventory_metadata/lib/get_cloud_metadata.ts +++ b/x-pack/plugins/infra/server/routes/inventory_metadata/lib/get_cloud_metadata.ts @@ -25,7 +25,8 @@ export const getCloudMetadata = async ( framework: KibanaFramework, req: RequestHandlerContext, sourceConfiguration: InfraSourceConfiguration, - nodeType: InventoryItemType + nodeType: InventoryItemType, + currentTime: number ): Promise => { const model = findInventoryModel(nodeType); @@ -36,7 +37,18 @@ export const getCloudMetadata = async ( body: { query: { bool: { - must: [{ match: { 'event.module': model.requiredModule } }], + must: [ + { + range: { + [sourceConfiguration.fields.timestamp]: { + gte: currentTime - 86400000, // 24 hours ago + lte: currentTime, + format: 'epoch_millis', + }, + }, + }, + { match: { 'event.module': model.requiredModule } }, + ], }, }, size: 0, From 06d1334299e8d34a68656d4644f41a241080747a Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Tue, 17 Nov 2020 17:04:24 -0700 Subject: [PATCH 2/6] Adding optimizations for snapshot request --- .../inventory_view/hooks/use_snaphot.ts | 2 +- .../lib/find_interval_for_metrics.ts | 3 ++- .../lib/get_dataset_for_field.ts | 20 +++++++++++++++++-- .../lib/create_timerange_with_interval.ts | 5 ++++- .../server/routes/snapshot/lib/get_nodes.ts | 13 ++++++------ 5 files changed, 31 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_snaphot.ts b/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_snaphot.ts index 702213516c123..eec46b0486287 100644 --- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_snaphot.ts +++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_snaphot.ts @@ -44,7 +44,7 @@ export function useSnapshot( interval: '1m', to: currentTime, from: currentTime - 1200 * 1000, - lookbackSize: 20, + lookbackSize: 5, }; const { error, loading, response, makeRequest } = useHTTPRequest( diff --git a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/find_interval_for_metrics.ts b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/find_interval_for_metrics.ts index 8ab0f4a44c85d..b3d960e30404f 100644 --- a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/find_interval_for_metrics.ts +++ b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/find_interval_for_metrics.ts @@ -34,7 +34,8 @@ export const findIntervalForMetrics = async ( const modules = await Promise.all( fields.map( - async (field) => await getDatasetForField(client, field as string, options.indexPattern) + async (field) => + await getDatasetForField(client, field as string, options.indexPattern, options.timerange) ) ); diff --git a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/get_dataset_for_field.ts b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/get_dataset_for_field.ts index 85bb5b106c87c..bd8c5f6beff3e 100644 --- a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/get_dataset_for_field.ts +++ b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/get_dataset_for_field.ts @@ -17,7 +17,8 @@ interface EventDatasetHit { export const getDatasetForField = async ( client: ESSearchClient, field: string, - indexPattern: string + indexPattern: string, + timerange: { field: string; to: number; from: number } ) => { const params = { allowNoIndices: true, @@ -25,7 +26,22 @@ export const getDatasetForField = async ( terminateAfter: 1, index: indexPattern, body: { - query: { exists: { field } }, + query: { + bool: { + filter: [ + { exists: { field } }, + { + range: { + [timerange.field]: { + gte: timerange.from, + lte: timerange.to, + format: 'epoch_millis', + }, + }, + }, + ], + }, + }, size: 1, _source: ['event.dataset'], }, diff --git a/x-pack/plugins/infra/server/routes/snapshot/lib/create_timerange_with_interval.ts b/x-pack/plugins/infra/server/routes/snapshot/lib/create_timerange_with_interval.ts index 827e0901c1c01..833b5349f4b56 100644 --- a/x-pack/plugins/infra/server/routes/snapshot/lib/create_timerange_with_interval.ts +++ b/x-pack/plugins/infra/server/routes/snapshot/lib/create_timerange_with_interval.ts @@ -75,7 +75,10 @@ const aggregationsToModules = async ( const fields = await Promise.all( uniqueFields.map( async (field) => - await getDatasetForField(client, field as string, options.sourceConfiguration.metricAlias) + await getDatasetForField(client, field as string, options.sourceConfiguration.metricAlias, { + ...options.timerange, + field: options.sourceConfiguration.fields.timestamp, + }) ) ); return fields.filter((f) => f) as string[]; diff --git a/x-pack/plugins/infra/server/routes/snapshot/lib/get_nodes.ts b/x-pack/plugins/infra/server/routes/snapshot/lib/get_nodes.ts index 9332d5aee1f52..7a2985188dccf 100644 --- a/x-pack/plugins/infra/server/routes/snapshot/lib/get_nodes.ts +++ b/x-pack/plugins/infra/server/routes/snapshot/lib/get_nodes.ts @@ -23,12 +23,11 @@ export const getNodes = async ( snapshotRequest ); const metricsApiResponse = await queryAllData(client, metricsApiRequest); - return copyMissingMetrics( - transformMetricsApiResponseToSnapshotResponse( - metricsApiRequest, - snapshotRequest, - source, - metricsApiResponse - ) + const snapshotResponse = transformMetricsApiResponseToSnapshotResponse( + metricsApiRequest, + snapshotRequest, + source, + metricsApiResponse ); + return copyMissingMetrics(snapshotResponse); }; From b2daf3d2fdb3e9154859b566dc140961ca467ebb Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Wed, 18 Nov 2020 11:03:46 -0700 Subject: [PATCH 3/6] Adding sorting to dataset request --- .../server/routes/metrics_explorer/lib/get_dataset_for_field.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/get_dataset_for_field.ts b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/get_dataset_for_field.ts index bd8c5f6beff3e..15e6f7ba86d01 100644 --- a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/get_dataset_for_field.ts +++ b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/get_dataset_for_field.ts @@ -44,6 +44,7 @@ export const getDatasetForField = async ( }, size: 1, _source: ['event.dataset'], + sort: [{ [timerange.field]: { order: 'desc' } }], }, }; From eafc72b8a758f83ef253c009b3e15137aaaa1870 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Wed, 18 Nov 2020 12:06:29 -0700 Subject: [PATCH 4/6] Only query inventory metadata for AWS --- .../server/routes/inventory_metadata/index.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/x-pack/plugins/infra/server/routes/inventory_metadata/index.ts b/x-pack/plugins/infra/server/routes/inventory_metadata/index.ts index eff335fc2236f..c2ca93baa2651 100644 --- a/x-pack/plugins/infra/server/routes/inventory_metadata/index.ts +++ b/x-pack/plugins/infra/server/routes/inventory_metadata/index.ts @@ -9,6 +9,7 @@ import Boom from '@hapi/boom'; import { pipe } from 'fp-ts/lib/pipeable'; import { fold } from 'fp-ts/lib/Either'; import { identity } from 'fp-ts/lib/function'; +import { findInventoryModel } from '../../../common/inventory_models'; import { InfraBackendLibs } from '../../lib/infra_types'; import { throwErrors } from '../../../common/runtime_types'; @@ -38,10 +39,24 @@ export const initInventoryMetaRoute = (libs: InfraBackendLibs) => { fold(throwErrors(Boom.badRequest), identity) ); + const model = findInventoryModel(nodeType); + + // Only run this for AWS modules, eventually we might have more. + if (model.requiredModule !== 'aws') { + return response.ok({ + body: { + accounts: [], + projects: [], + regions: [], + }, + }); + } + const { configuration } = await libs.sources.getSourceConfiguration( requestContext.core.savedObjects.client, sourceId ); + const awsMetadata = await getCloudMetadata( framework, requestContext, From 97a885fd98c66ad8233fa1e6d83a8ef88a62c494 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Wed, 18 Nov 2020 13:26:48 -0700 Subject: [PATCH 5/6] moving check inside getCloudMetadata --- .../infra/server/routes/inventory_metadata/index.ts | 13 ------------- .../inventory_metadata/lib/get_cloud_metadata.ts | 8 ++++++++ 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/x-pack/plugins/infra/server/routes/inventory_metadata/index.ts b/x-pack/plugins/infra/server/routes/inventory_metadata/index.ts index c2ca93baa2651..637cb8a864d0b 100644 --- a/x-pack/plugins/infra/server/routes/inventory_metadata/index.ts +++ b/x-pack/plugins/infra/server/routes/inventory_metadata/index.ts @@ -39,19 +39,6 @@ export const initInventoryMetaRoute = (libs: InfraBackendLibs) => { fold(throwErrors(Boom.badRequest), identity) ); - const model = findInventoryModel(nodeType); - - // Only run this for AWS modules, eventually we might have more. - if (model.requiredModule !== 'aws') { - return response.ok({ - body: { - accounts: [], - projects: [], - regions: [], - }, - }); - } - const { configuration } = await libs.sources.getSourceConfiguration( requestContext.core.savedObjects.client, sourceId diff --git a/x-pack/plugins/infra/server/routes/inventory_metadata/lib/get_cloud_metadata.ts b/x-pack/plugins/infra/server/routes/inventory_metadata/lib/get_cloud_metadata.ts index 7e8b26de75e8b..af9e9c5f57c5b 100644 --- a/x-pack/plugins/infra/server/routes/inventory_metadata/lib/get_cloud_metadata.ts +++ b/x-pack/plugins/infra/server/routes/inventory_metadata/lib/get_cloud_metadata.ts @@ -29,6 +29,14 @@ export const getCloudMetadata = async ( currentTime: number ): Promise => { const model = findInventoryModel(nodeType); + // Only run this for AWS modules, eventually we might have more. + if (model.requiredModule !== 'aws') { + return { + accounts: [], + projects: [], + regions: [], + }; + } const metricQuery = { allowNoIndices: true, From 5fbe6397975cdb2dfbbc7446bbc6fec9bbc941d6 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Wed, 18 Nov 2020 13:48:39 -0700 Subject: [PATCH 6/6] removing unused deps --- x-pack/plugins/infra/server/routes/inventory_metadata/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/infra/server/routes/inventory_metadata/index.ts b/x-pack/plugins/infra/server/routes/inventory_metadata/index.ts index 637cb8a864d0b..c784aa0f7d20b 100644 --- a/x-pack/plugins/infra/server/routes/inventory_metadata/index.ts +++ b/x-pack/plugins/infra/server/routes/inventory_metadata/index.ts @@ -9,7 +9,6 @@ import Boom from '@hapi/boom'; import { pipe } from 'fp-ts/lib/pipeable'; import { fold } from 'fp-ts/lib/Either'; import { identity } from 'fp-ts/lib/function'; -import { findInventoryModel } from '../../../common/inventory_models'; import { InfraBackendLibs } from '../../lib/infra_types'; import { throwErrors } from '../../../common/runtime_types';