From ecdfeb6ced953300170d699b669b019e7a6021b6 Mon Sep 17 00:00:00 2001 From: Norbert Nader Date: Mon, 20 Dec 2021 21:58:52 -0800 Subject: [PATCH] feat: customizable resolutions * added customizable resolutions for sitewise data source which means we can specify a resolution per asset property * refactored resolutionMapping to resolution and allowed to pass string if we want a specific resolution for all queries --- .../iot-bar-chart/iot-bar-chart.spec.ts | 2 +- .../iot-connector/iot-connector.spec.ts | 4 +- .../src/components/iot-kpi/iot-kpi.spec.ts | 2 +- .../iot-line-chart/iot-line-chart.spec.ts | 2 +- .../iot-scatter-chart.spec.ts | 2 +- .../iot-status-grid/iot-status-grid.spec.ts | 2 +- .../iot-status-timeline.spec.ts | 2 +- .../components/iot-table/iot-table.spec.ts | 2 +- .../src/testing/createMockSource.ts | 2 +- .../testing/testing-ground/siteWiseQueries.ts | 13 +- .../testing/testing-ground/testing-ground.tsx | 42 +- .../data-module/IotAppKitDataModule.spec.ts | 11 +- .../data-module/data-cache/requestTypes.ts | 6 +- .../subscriptionStore.spec.ts | 2 +- .../site-wise/client/client.spec.ts | 40 +- .../data-sources/site-wise/client/client.ts | 4 +- .../client/getAggregatedPropertyDataPoints.ts | 52 ++- .../client/getHistoricalPropertyDataPoints.ts | 4 +- .../client/getLatestPropertyDataPoint.ts | 4 +- .../site-wise/data-source.spec.ts | 360 +++++++++++++++++- .../src/data-sources/site-wise/data-source.ts | 153 ++++++-- .../src/data-sources/site-wise/types.d.ts | 7 +- 22 files changed, 619 insertions(+), 99 deletions(-) diff --git a/packages/components/src/components/iot-bar-chart/iot-bar-chart.spec.ts b/packages/components/src/components/iot-bar-chart/iot-bar-chart.spec.ts index 925177adb..e7e9f5caa 100644 --- a/packages/components/src/components/iot-bar-chart/iot-bar-chart.spec.ts +++ b/packages/components/src/components/iot-bar-chart/iot-bar-chart.spec.ts @@ -29,7 +29,7 @@ const barChartSpecPage = async (propOverrides: Partial = isEditing: false, query: { source: 'test-mock', - assets: [{ assetId: 'some-asset-id', propertyIds: ['some-property-id'] }], + assets: [{ assetId: 'some-asset-id', properties: [{ propertyId: 'some-property-id' }] }], } as SiteWiseDataStreamQuery, viewport, ...propOverrides, diff --git a/packages/components/src/components/iot-connector/iot-connector.spec.ts b/packages/components/src/components/iot-connector/iot-connector.spec.ts index 515519a6b..180eeab0e 100644 --- a/packages/components/src/components/iot-connector/iot-connector.spec.ts +++ b/packages/components/src/components/iot-connector/iot-connector.spec.ts @@ -58,7 +58,7 @@ it('provides data streams', async () => { renderFunc, query: { source: 'test-mock', - assets: [{ assetId, propertyIds: [propertyId] }], + assets: [{ assetId, properties: [{ propertyId }] }], } as SiteWiseDataStreamQuery, }); @@ -87,7 +87,7 @@ it('updates with new query', async () => { connector.query = { source: 'test-mock', - assets: [{ assetId, propertyIds: [propertyId] }], + assets: [{ assetId, properties: [{ propertyId }] }], } as SiteWiseDataStreamQuery; await page.waitForChanges(); diff --git a/packages/components/src/components/iot-kpi/iot-kpi.spec.ts b/packages/components/src/components/iot-kpi/iot-kpi.spec.ts index fa3e00b64..aee645e60 100644 --- a/packages/components/src/components/iot-kpi/iot-kpi.spec.ts +++ b/packages/components/src/components/iot-kpi/iot-kpi.spec.ts @@ -29,7 +29,7 @@ const kpiSpecPage = async (propOverrides: Partial = {}) => { isEditing: false, query: { source: 'test-mock', - assets: [{ assetId: 'some-asset-id', propertyIds: ['some-property-id'] }], + assets: [{ assetId: 'some-asset-id', properties: [{ propertyId: 'some-property-id' }] }], } as SiteWiseDataStreamQuery, // static casting because of legacy sw viewport, ...propOverrides, diff --git a/packages/components/src/components/iot-line-chart/iot-line-chart.spec.ts b/packages/components/src/components/iot-line-chart/iot-line-chart.spec.ts index e52a197b1..f20bff416 100644 --- a/packages/components/src/components/iot-line-chart/iot-line-chart.spec.ts +++ b/packages/components/src/components/iot-line-chart/iot-line-chart.spec.ts @@ -30,7 +30,7 @@ const lineChartSpecPage = async (propOverrides: Partial = {}) isEditing: false, query: { source: 'test-mock', - assets: [{ assetId: 'some-asset-id', propertyIds: ['some-property-id'] }], + assets: [{ assetId: 'some-asset-id', properties: [{ propertyId: 'some-property-id' }] }], } as SiteWiseDataStreamQuery, // static casting because of legacy sw viewport, ...propOverrides, diff --git a/packages/components/src/components/iot-scatter-chart/iot-scatter-chart.spec.ts b/packages/components/src/components/iot-scatter-chart/iot-scatter-chart.spec.ts index c5d6d1950..41f9e13d0 100644 --- a/packages/components/src/components/iot-scatter-chart/iot-scatter-chart.spec.ts +++ b/packages/components/src/components/iot-scatter-chart/iot-scatter-chart.spec.ts @@ -29,7 +29,7 @@ const scatterChartSpecPage = async (propOverrides: Partial = {} isEditing: false, query: { source: 'test-mock', - assets: [{ assetId: 'some-asset-id', propertyIds: ['some-property-id'] }], + assets: [{ assetId: 'some-asset-id', properties: [{ propertyId: 'some-property-id' }] }], } as SiteWiseDataStreamQuery, // static casting because of legacy sw viewport, ...propOverrides, diff --git a/packages/components/src/components/iot-status-timeline/iot-status-timeline.spec.ts b/packages/components/src/components/iot-status-timeline/iot-status-timeline.spec.ts index 62726e1e9..d814d84e7 100644 --- a/packages/components/src/components/iot-status-timeline/iot-status-timeline.spec.ts +++ b/packages/components/src/components/iot-status-timeline/iot-status-timeline.spec.ts @@ -31,7 +31,7 @@ const statusTimelineSpecPage = async (propOverrides: Partial = {}) => isEditing: false, query: { source: 'test-mock', - assets: [{ assetId: 'some-asset-id', propertyIds: ['some-property-id'] }], + assets: [{ assetId: 'some-asset-id', properties: [{ propertyId: 'some-property-id' }] }], } as SiteWiseDataStreamQuery, // static casting because of legacy sw viewport, ...propOverrides, diff --git a/packages/components/src/testing/createMockSource.ts b/packages/components/src/testing/createMockSource.ts index 9204dbad0..53a67c6bb 100644 --- a/packages/components/src/testing/createMockSource.ts +++ b/packages/components/src/testing/createMockSource.ts @@ -5,7 +5,7 @@ import { toDataStreamId } from './dataStreamId'; const dataStreamIds = (query: SiteWiseDataStreamQuery) => query.assets - .map(({ assetId, propertyIds }) => propertyIds.map((propertyId) => toDataStreamId({ assetId, propertyId }))) + .map(({ assetId, properties }) => properties.map(({ propertyId }) => toDataStreamId({ assetId, propertyId }))) .flat(); export const createMockSource = (dataStreams: DataStream[]): DataSource => ({ diff --git a/packages/components/src/testing/testing-ground/siteWiseQueries.ts b/packages/components/src/testing/testing-ground/siteWiseQueries.ts index 51baaaf36..b24732fb7 100644 --- a/packages/components/src/testing/testing-ground/siteWiseQueries.ts +++ b/packages/components/src/testing/testing-ground/siteWiseQueries.ts @@ -10,7 +10,7 @@ export const DEMO_TURBINE_ASSET_1_PROPERTY_4 = '8d9ed440-a8dd-48bd-a35f-70db6f2e export const STRING_QUERY = { source: 'site-wise', - assets: [{ assetId: STRING_ASSET_ID, propertyIds: [STRING_PROPERTY_ID] }], + assets: [{ assetId: STRING_ASSET_ID, properties: [{ propertyId: STRING_PROPERTY_ID }] }], }; export const ASSET_DETAILS_QUERY = { @@ -22,17 +22,24 @@ export const NUMBER_QUERY = { assets: [ { assetId: DEMO_TURBINE_ASSET_1, - propertyIds: [DEMO_TURBINE_ASSET_1_PROPERTY_1, DEMO_TURBINE_ASSET_1_PROPERTY_4], + properties: [{ propertyId: DEMO_TURBINE_ASSET_1_PROPERTY_1 }, { propertyId: DEMO_TURBINE_ASSET_1_PROPERTY_4 }], }, ], }; const AGGREGATED_DATA_ASSET = '099b1330-83ff-4fec-b165-c7186ec8eb23'; const AGGREGATED_DATA_PROPERTY = '05c5c47f-fd92-4823-828e-09ce63b90569'; +const AGGREGATED_DATA_PROPERTY_2 = '11d2599a-2547-451d-ab79-a47f878dbbe3'; export const AGGREGATED_DATA_QUERY = { source: 'site-wise', - assets: [{ assetId: AGGREGATED_DATA_ASSET, propertyIds: [AGGREGATED_DATA_PROPERTY] }], + assets: [{ + assetId: AGGREGATED_DATA_ASSET, + properties: [ + { propertyId: AGGREGATED_DATA_PROPERTY }, + { propertyId: AGGREGATED_DATA_PROPERTY_2, resolution: '1m' } + ] + }], }; // From demo turbine asset, found at https://p-rlvy2rj8.app.iotsitewise.aws/ diff --git a/packages/components/src/testing/testing-ground/testing-ground.tsx b/packages/components/src/testing/testing-ground/testing-ground.tsx index ac13a83ea..2bdf9c5a9 100755 --- a/packages/components/src/testing/testing-ground/testing-ground.tsx +++ b/packages/components/src/testing/testing-ground/testing-ground.tsx @@ -1,5 +1,5 @@ -import { Component, h } from '@stencil/core'; -import { initialize, DataModule } from '@iot-app-kit/core'; +import { Component, State, h } from '@stencil/core'; +import { initialize, DataModule, ResolutionConfig } from '@iot-app-kit/core'; import { ASSET_DETAILS_QUERY, DEMO_TURBINE_ASSET_1, @@ -15,7 +15,7 @@ const VIEWPORT = { duration: '5m' }; const THREE_MINUTES = 1000 * 60 * 3; -const resolutionMapping = { +const DEFAULT_RESOLUTION_MAPPING = { [THREE_MINUTES]: '1m', } @@ -24,10 +24,30 @@ const resolutionMapping = { styleUrl: 'testing-ground.css', }) export class TestingGround { + @State() resolution: ResolutionConfig = DEFAULT_RESOLUTION_MAPPING; + @State() viewport: { duration: string } = VIEWPORT; private dataModule: DataModule; componentWillLoad() { - this.dataModule = initialize({ awsCredentials: getEnvCredentials(), awsRegion: 'us-west-2' }); + this.dataModule = initialize({ awsCredentials: getEnvCredentials(), awsRegion: 'us-east-1' }); + } + + private changeResolution = (ev: Event) => { + const resolution = (ev.target as HTMLSelectElement)?.value; + + if (resolution === 'auto'){ + this.resolution = DEFAULT_RESOLUTION_MAPPING; + } else if (resolution === '0') { + this.resolution = {}; + } else { + this.resolution = resolution; + } + } + + private changeDuration = (ev: Event) => { + const duration = `${(ev.target as HTMLSelectElement)?.value}m`; + + this.viewport = { duration }; } render() { @@ -86,12 +106,22 @@ export class TestingGround { /> + resolution: + viewport:
diff --git a/packages/core/src/data-module/IotAppKitDataModule.spec.ts b/packages/core/src/data-module/IotAppKitDataModule.spec.ts index c5d2db101..a53aefa6b 100644 --- a/packages/core/src/data-module/IotAppKitDataModule.spec.ts +++ b/packages/core/src/data-module/IotAppKitDataModule.spec.ts @@ -25,7 +25,7 @@ const DATA_STREAM_QUERY: SiteWiseDataStreamQuery = { assets: [ { assetId: ASSET_ID, - propertyIds: [PROPERTY_ID], + properties: [{ propertyId: PROPERTY_ID }], }, ], }; @@ -39,8 +39,8 @@ const createMockSiteWiseDataSource = ( initiateRequest: jest.fn(({ onSuccess }: DataSourceRequest) => onSuccess(dataStreams)), getRequestsFromQuery: ({ query }) => query.assets - .map(({ assetId, propertyIds }) => - propertyIds.map((propertyId) => ({ + .map(({ assetId, properties }) => + properties.map(({ propertyId }) => ({ id: toDataStreamId({ assetId, propertyId }), resolution, })) @@ -183,7 +183,7 @@ it('subscribes to a single data stream', async () => { assets: [ { assetId, - propertyIds: [propertyId], + properties: [{ propertyId }], }, ], }, @@ -233,7 +233,7 @@ it('requests data from a custom data source', () => { dataModule.subscribeToDataStreams( { query: { - assets: [{ assetId, propertyIds: [propertyId] }], + assets: [{ assetId, properties: [{ propertyId }] }], source: customSource.name, }, requestInfo: { @@ -778,3 +778,4 @@ it('requests data range with buffer', () => { unsubscribe(); }); + diff --git a/packages/core/src/data-module/data-cache/requestTypes.ts b/packages/core/src/data-module/data-cache/requestTypes.ts index e25241d1c..8ce46ead2 100755 --- a/packages/core/src/data-module/data-cache/requestTypes.ts +++ b/packages/core/src/data-module/data-cache/requestTypes.ts @@ -26,13 +26,15 @@ export type OnRequestData = (opts: { dataStreamId: string; }) => void; -export type ResolutionMapping ={ +export type ResolutionMapping = { [viewportDuration: number]: number | string; }; +export type ResolutionConfig = ResolutionMapping | string; + export interface RequestConfig { fetchMostRecentBeforeStart?: boolean; requestBuffer?: number; fetchAggregatedData?: boolean; - resolutionMapping?: ResolutionMapping; + resolution?: ResolutionConfig; } diff --git a/packages/core/src/data-module/subscription-store/subscriptionStore.spec.ts b/packages/core/src/data-module/subscription-store/subscriptionStore.spec.ts index 461bb4d0a..14dee12f0 100644 --- a/packages/core/src/data-module/subscription-store/subscriptionStore.spec.ts +++ b/packages/core/src/data-module/subscription-store/subscriptionStore.spec.ts @@ -39,7 +39,7 @@ it('updates subscription', () => { const query = { source: SITEWISE_DATA_SOURCE, - assets: [{ assetId: '123', propertyIds: ['prop1', 'prop2'] }], + assets: [{ assetId: '123', properties: [{ propertyId: 'prop1' }, { propertyId: 'prop2' }] }], }; subscriptionStore.addSubscription(SUBSCRIPTION_ID, MOCK_SUBSCRIPTION); diff --git a/packages/core/src/data-sources/site-wise/client/client.spec.ts b/packages/core/src/data-sources/site-wise/client/client.spec.ts index 7872b1839..88378883a 100644 --- a/packages/core/src/data-sources/site-wise/client/client.spec.ts +++ b/packages/core/src/data-sources/site-wise/client/client.spec.ts @@ -25,7 +25,7 @@ describe('getHistoricalPropertyDataPoints', () => { const onError = jest.fn(); const query: SiteWiseDataStreamQuery = { source: SITEWISE_DATA_SOURCE, - assets: [{ assetId, propertyIds: [propertyId] }], + assets: [{ assetId, properties: [{ propertyId }] }], }; const client = new SiteWiseClient(createSiteWiseSDK({ getAssetPropertyValueHistory })); @@ -47,7 +47,7 @@ describe('getHistoricalPropertyDataPoints', () => { const onError = jest.fn(); const query: SiteWiseDataStreamQuery = { source: SITEWISE_DATA_SOURCE, - assets: [{ assetId, propertyIds: [propertyId] }], + assets: [{ assetId, properties: [{ propertyId }] }], }; const client = new SiteWiseClient(createSiteWiseSDK({ getAssetPropertyValueHistory })); @@ -91,7 +91,7 @@ describe('getLatestPropertyDataPoint', () => { const onError = jest.fn(); const query: SiteWiseDataStreamQuery = { source: SITEWISE_DATA_SOURCE, - assets: [{ assetId, propertyIds: [propertyId] }], + assets: [{ assetId, properties: [{ propertyId }] }], }; const client = new SiteWiseClient(createSiteWiseSDK({ getAssetPropertyValue })); @@ -126,7 +126,7 @@ describe('getLatestPropertyDataPoint', () => { const onError = jest.fn(); const query: SiteWiseDataStreamQuery = { source: SITEWISE_DATA_SOURCE, - assets: [{ assetId, propertyIds: [propertyId] }], + assets: [{ assetId, properties: [{ propertyId }] }], }; await client.getLatestPropertyDataPoint({ query, onSuccess, onError }); @@ -147,7 +147,7 @@ describe('getAggregatedPropertyDataPoints', () => { const onError = jest.fn(); const query: SiteWiseDataStreamQuery = { source: SITEWISE_DATA_SOURCE, - assets: [{ assetId, propertyIds: [propertyId] }], + assets: [{ assetId, properties: [{ propertyId }] }], }; const client = new SiteWiseClient(createSiteWiseSDK({ getAssetPropertyAggregates })); @@ -170,6 +170,34 @@ describe('getAggregatedPropertyDataPoints', () => { expect(onError).toBeCalled(); }); + it('throws error when no resolution specified', async () => { + const getAssetPropertyAggregates = jest.fn().mockResolvedValue(AGGREGATE_VALUES); + const assetId = 'some-asset-id'; + const propertyId = 'some-property-id'; + + const onSuccess = jest.fn(); + const onError = jest.fn(); + const query: SiteWiseDataStreamQuery = { + source: SITEWISE_DATA_SOURCE, + assets: [{ assetId, properties: [{ propertyId }] }], + }; + + const client = new SiteWiseClient(createSiteWiseSDK({ getAssetPropertyAggregates })); + + const startDate = new Date(2000, 0, 0); + const endDate = new Date(2001, 0, 0); + const aggregateTypes = [AggregateType.AVERAGE]; + + await expect(async () => { await client.getAggregatedPropertyDataPoints({ + query, + onSuccess, + onError, + start: startDate, + end: endDate, + aggregateTypes, + })}).rejects.toThrowError(); + }); + it('returns data point on success', async () => { const assetId = 'some-asset-id'; const propertyId = 'some-property-id'; @@ -178,7 +206,7 @@ describe('getAggregatedPropertyDataPoints', () => { const onError = jest.fn(); const query: SiteWiseDataStreamQuery = { source: SITEWISE_DATA_SOURCE, - assets: [{ assetId, propertyIds: [propertyId] }], + assets: [{ assetId, properties: [{ propertyId }] }], }; const getAssetPropertyAggregates = jest.fn().mockResolvedValue(AGGREGATE_VALUES); diff --git a/packages/core/src/data-sources/site-wise/client/client.ts b/packages/core/src/data-sources/site-wise/client/client.ts index 9c512bbfa..78bfb0275 100644 --- a/packages/core/src/data-sources/site-wise/client/client.ts +++ b/packages/core/src/data-sources/site-wise/client/client.ts @@ -35,10 +35,10 @@ export class SiteWiseClient { query: SiteWiseDataStreamQuery; start: Date; end: Date; - resolution: string; + resolution?: string; aggregateTypes: AggregateType[]; maxResults?: number; - onError: Function; + onError: ErrorCallback; onSuccess: DataStreamCallback; }): Promise { return getAggregatedPropertyDataPoints({ client: this.siteWiseSdk, ...options }); diff --git a/packages/core/src/data-sources/site-wise/client/getAggregatedPropertyDataPoints.ts b/packages/core/src/data-sources/site-wise/client/getAggregatedPropertyDataPoints.ts index e08a53a2d..056f4daa5 100644 --- a/packages/core/src/data-sources/site-wise/client/getAggregatedPropertyDataPoints.ts +++ b/packages/core/src/data-sources/site-wise/client/getAggregatedPropertyDataPoints.ts @@ -7,9 +7,10 @@ import { import { AssetId, AssetPropertyId, SiteWiseDataStreamQuery } from '../types'; import { aggregateToDataPoint } from '../util/toDataPoint'; import { dataStreamFromSiteWise } from '../dataStreamFromSiteWise'; -import { DataStreamCallback } from '../../../data-module/types'; +import { DataStreamCallback, ErrorCallback } from '../../../data-module/types'; import { isDefined } from '../../../common/predicates'; import { RESOLUTION_TO_MS_MAPPING } from '../util/resolution'; +import { toDataStreamId } from '../util/dataStreamId'; const getAggregatedPropertyDataPointsForProperty = ({ assetId, @@ -20,6 +21,7 @@ const getAggregatedPropertyDataPointsForProperty = ({ aggregateTypes, maxResults, onSuccess, + onError, nextToken: prevToken, client, }: { @@ -30,6 +32,7 @@ const getAggregatedPropertyDataPointsForProperty = ({ resolution: string; aggregateTypes: AggregateType[]; maxResults?: number; + onError: ErrorCallback; onSuccess: DataStreamCallback; client: IoTSiteWiseClient; nextToken?: string; @@ -72,11 +75,16 @@ const getAggregatedPropertyDataPointsForProperty = ({ resolution, aggregateTypes, maxResults, + onError, onSuccess, nextToken, client, }); } + }) + .catch((err) => { + const id = toDataStreamId({ assetId, propertyId }); + onError({ id, resolution, error: err.message }); }); }; @@ -94,32 +102,38 @@ export const getAggregatedPropertyDataPoints = async ({ query: SiteWiseDataStreamQuery; start: Date; end: Date; - resolution: string; + resolution?: string; aggregateTypes: AggregateType[]; maxResults?: number; - onError: Function; + onError: ErrorCallback; onSuccess: DataStreamCallback; client: IoTSiteWiseClient; }) => { const requests = query.assets - .map(({ assetId, propertyIds }) => - propertyIds.map((propertyId) => - getAggregatedPropertyDataPointsForProperty({ - client, - assetId, - propertyId, - start, - end, - resolution, - aggregateTypes, - maxResults, - onSuccess, - }) + .map(({ assetId, properties }) => + properties.map(({ propertyId, resolution: propertyResolution }) => { + const resolutionOverride = propertyResolution || resolution; + + if (resolutionOverride == null) { + throw new Error('Resolution must be either specified in requestConfig or query'); + } + + return getAggregatedPropertyDataPointsForProperty({ + client, + assetId, + propertyId, + start, + end, + resolution: resolutionOverride, + aggregateTypes, + maxResults, + onSuccess, + onError, + }) + } ) ) .flat(); - await Promise.all(requests).catch((err) => { - onError(err); - }); + await Promise.all(requests); }; diff --git a/packages/core/src/data-sources/site-wise/client/getHistoricalPropertyDataPoints.ts b/packages/core/src/data-sources/site-wise/client/getHistoricalPropertyDataPoints.ts index f341b2810..05b313205 100644 --- a/packages/core/src/data-sources/site-wise/client/getHistoricalPropertyDataPoints.ts +++ b/packages/core/src/data-sources/site-wise/client/getHistoricalPropertyDataPoints.ts @@ -87,8 +87,8 @@ export const getHistoricalPropertyDataPoints = async ({ client: IoTSiteWiseClient; }) => { const requests = query.assets - .map(({ assetId, propertyIds }) => - propertyIds.map((propertyId) => + .map(({ assetId, properties }) => + properties.map(({ propertyId }) => getHistoricalPropertyDataPointsForProperty({ client, assetId, diff --git a/packages/core/src/data-sources/site-wise/client/getLatestPropertyDataPoint.ts b/packages/core/src/data-sources/site-wise/client/getLatestPropertyDataPoint.ts index 5486e12a5..f0c691e15 100644 --- a/packages/core/src/data-sources/site-wise/client/getLatestPropertyDataPoint.ts +++ b/packages/core/src/data-sources/site-wise/client/getLatestPropertyDataPoint.ts @@ -18,8 +18,8 @@ export const getLatestPropertyDataPoint = async ({ client: IoTSiteWiseClient; }): Promise => { const requests = assets - .map(({ assetId, propertyIds }) => - propertyIds.map((propertyId) => { + .map(({ assetId,properties }) => + properties.map(({ propertyId }) => { return client .send(new GetAssetPropertyValueCommand({ assetId, propertyId })) .then((res) => ({ diff --git a/packages/core/src/data-sources/site-wise/data-source.spec.ts b/packages/core/src/data-sources/site-wise/data-source.spec.ts index 0b1125bfb..1afab77e6 100644 --- a/packages/core/src/data-sources/site-wise/data-source.spec.ts +++ b/packages/core/src/data-sources/site-wise/data-source.spec.ts @@ -3,7 +3,7 @@ import { IoTSiteWiseClient } from '@aws-sdk/client-iotsitewise'; import { createDataSource, SITEWISE_DATA_SOURCE } from './data-source'; import { MINUTE_IN_MS, HOUR_IN_MS } from '../../common/time'; import { SiteWiseDataStreamQuery } from './types.d'; -import { ASSET_PROPERTY_DOUBLE_VALUE, AGGREGATE_VALUES } from '../../common/tests/mocks/assetPropertyValue'; +import { ASSET_PROPERTY_DOUBLE_VALUE, AGGREGATE_VALUES, ASSET_PROPERTY_VALUE_HISTORY } from '../../common/tests/mocks/assetPropertyValue'; import { createSiteWiseSDK } from '../../common/tests/util'; import { toDataStreamId } from './util/dataStreamId'; import { IotAppKitDataModule } from '../../data-module/IotAppKitDataModule'; @@ -68,7 +68,7 @@ describe('initiateRequest', () => { const query: SiteWiseDataStreamQuery = { source: SITEWISE_DATA_SOURCE, - assets: [{ assetId: ASSET_1, propertyIds: [PROPERTY_1] }], + assets: [{ assetId: ASSET_1, properties: [{ propertyId: PROPERTY_1 }] }], }; const onError = jest.fn(); @@ -117,7 +117,7 @@ describe('initiateRequest', () => { const query: SiteWiseDataStreamQuery = { source: SITEWISE_DATA_SOURCE, - assets: [{ assetId: 'some-asset-id', propertyIds: ['some-property-id'] }], + assets: [{ assetId: 'some-asset-id', properties: [{ propertyId: 'some-property-id' }] }], }; const onError = jest.fn(); @@ -147,7 +147,7 @@ describe('initiateRequest', () => { expect(getAssetPropertyValue).toBeCalledTimes(1); expect(getAssetPropertyValue).toBeCalledWith({ assetId: query.assets[0].assetId, - propertyId: query.assets[0].propertyIds[0], + propertyId: query.assets[0].properties[0].propertyId, }); expect(onError).not.toBeCalled(); @@ -176,7 +176,7 @@ describe('initiateRequest', () => { const query: SiteWiseDataStreamQuery = { source: SITEWISE_DATA_SOURCE, - assets: [{ assetId: ASSET_ID, propertyIds: [PROPERTY_1, PROPERTY_2] }], + assets: [{ assetId: ASSET_ID, properties: [ { propertyId: PROPERTY_1 }, { propertyId: PROPERTY_2 }] }], }; dataSource.initiateRequest( @@ -222,8 +222,8 @@ describe('initiateRequest', () => { const query: SiteWiseDataStreamQuery = { source: SITEWISE_DATA_SOURCE, assets: [ - { assetId: ASSET_1, propertyIds: [PROPERTY_1] }, - { assetId: ASSET_2, propertyIds: [PROPERTY_2] }, + { assetId: ASSET_1, properties: [{ propertyId: PROPERTY_1 }] }, + { assetId: ASSET_2, properties: [{ propertyId: PROPERTY_2 }] }, ], }; @@ -277,7 +277,7 @@ describe('e2e through data-module', () => { dataModule.subscribeToDataStreams( { query: { - assets: [{ assetId, propertyIds: [propertyId] }], + assets: [{ assetId, properties: [{ propertyId }] }], source: dataSource.name, } as SiteWiseDataStreamQuery, requestInfo: { viewport: { start: new Date(2000, 0, 0), end: new Date() }, onlyFetchLatestValue: false }, @@ -318,7 +318,7 @@ describe('e2e through data-module', () => { dataModule.subscribeToDataStreams( { query: { - assets: [{ assetId, propertyIds: [propertyId] }], + assets: [{ assetId, properties: [{ propertyId }] }], source: dataSource.name, } as SiteWiseDataStreamQuery, requestInfo: { viewport: { start: new Date(2000, 0, 0), end: new Date() }, onlyFetchLatestValue: true }, @@ -342,7 +342,7 @@ describe('e2e through data-module', () => { }); describe('aggregated data', () => { - it('requests aggregated data with correct resolution based on mapping and uses default aggregate type', async () => { + it('determines resolution based on mapping', async () => { const getAssetPropertyValue = jest.fn(); const getAssetPropertyAggregates = jest.fn().mockResolvedValue(AGGREGATE_VALUES); const getAssetPropertyValueHistory = jest.fn(); @@ -359,7 +359,7 @@ describe('aggregated data', () => { const query: SiteWiseDataStreamQuery = { source: SITEWISE_DATA_SOURCE, - assets: [{ assetId: 'some-asset-id', propertyIds: ['some-property-id'] }], + assets: [{ assetId: 'some-asset-id', properties: [{ propertyId: 'some-property-id' }] }], }; const onError = jest.fn(); @@ -381,7 +381,7 @@ describe('aggregated data', () => { onlyFetchLatestValue: false, requestConfig: { fetchAggregatedData: true, - resolutionMapping: { + resolution: { [FIFTY_HOURS]: '1d', [FIFTY_MINUTES]: '1h', } @@ -401,7 +401,7 @@ describe('aggregated data', () => { expect(getAssetPropertyAggregates).toBeCalledWith( expect.objectContaining({ assetId: query.assets[0].assetId, - propertyId: query.assets[0].propertyIds[0], + propertyId: query.assets[0].properties[0].propertyId, aggregateTypes: ['AVERAGE'], resolution: '1h' }) @@ -435,6 +435,338 @@ describe('aggregated data', () => { ]); }); + it('requests specific resolution', async () => { + const getAssetPropertyValue = jest.fn(); + const getAssetPropertyAggregates = jest.fn().mockResolvedValue(AGGREGATE_VALUES); + const getAssetPropertyValueHistory = jest.fn(); + const getInterpolatedAssetPropertyValues = jest.fn(); + + const mockSDK = createSiteWiseSDK({ + getAssetPropertyValue, + getAssetPropertyValueHistory, + getAssetPropertyAggregates, + getInterpolatedAssetPropertyValues, + }); + + const dataSource = createDataSource(mockSDK); + + const query: SiteWiseDataStreamQuery = { + source: SITEWISE_DATA_SOURCE, + assets: [{ assetId: 'some-asset-id', properties: [{ propertyId: 'some-property-id' }] }], + }; + + const onError = jest.fn(); + const onSuccess = jest.fn(); + + const FIFTY_FIVE_MINUTES = MINUTE_IN_MS * 55; + + const resolution = '1m'; + + dataSource.initiateRequest( + { + onError, + onSuccess, + query, + requestInfo: { + viewport: { + duration: FIFTY_FIVE_MINUTES, + }, + onlyFetchLatestValue: false, + requestConfig: { + fetchAggregatedData: true, + resolution + } + }, + }, + [] + ); + + await flushPromises(); + + expect(getAssetPropertyValue).not.toBeCalled(); + expect(getAssetPropertyValueHistory).not.toBeCalled(); + expect(getInterpolatedAssetPropertyValues).not.toBeCalled(); + + expect(getAssetPropertyAggregates).toBeCalledTimes(1); + expect(getAssetPropertyAggregates).toBeCalledWith( + expect.objectContaining({ + assetId: query.assets[0].assetId, + propertyId: query.assets[0].properties[0].propertyId, + aggregateTypes: ['AVERAGE'], + resolution + }) + ); + + expect(onError).not.toBeCalled(); + + expect(onSuccess).toBeCalledTimes(1); + expect(onSuccess).toBeCalledWith([ + expect.objectContaining({ + id: toDataStreamId({ assetId: 'some-asset-id', propertyId: 'some-property-id' }), + aggregates: { + [MINUTE_IN_MS]: [ + { + x: 946602000000, + y: 5, + }, + { + x: 946605600000, + y: 7, + }, + { + x: 946609200000, + y: 10, + } + ] + }, + resolution: MINUTE_IN_MS, + dataType: 'NUMBER', + }), + ]); + }); + + it('requests specific resolution per asset property', async () => { + const getAssetPropertyValue = jest.fn(); + const getAssetPropertyAggregates = jest.fn().mockResolvedValue(AGGREGATE_VALUES); + const getAssetPropertyValueHistory = jest.fn().mockResolvedValue(ASSET_PROPERTY_VALUE_HISTORY); + const getInterpolatedAssetPropertyValues = jest.fn(); + + const mockSDK = createSiteWiseSDK({ + getAssetPropertyValue, + getAssetPropertyValueHistory, + getAssetPropertyAggregates, + getInterpolatedAssetPropertyValues, + }); + + const dataSource = createDataSource(mockSDK); + + const resolution = '1m'; + + const query: SiteWiseDataStreamQuery = { + source: SITEWISE_DATA_SOURCE, + assets: [ + { + assetId: 'some-asset-id', + properties: [ + { propertyId: 'some-property-id', resolution }, + { propertyId: 'some-property-id2' } + ] + }, + { + assetId: 'some-asset-id2', + properties: [ + { propertyId: 'some-property-id', resolution }, + { propertyId: 'some-property-id2' } + ] + }, + ], + }; + + const onError = jest.fn(); + const onSuccess = jest.fn(); + + const FIFTY_FIVE_MINUTES = MINUTE_IN_MS * 55; + + dataSource.initiateRequest( + { + onError, + onSuccess, + query, + requestInfo: { + viewport: { + duration: FIFTY_FIVE_MINUTES, + }, + onlyFetchLatestValue: false + }, + }, + [] + ); + + await flushPromises(); + + expect(getAssetPropertyValue).not.toBeCalled(); + expect(getInterpolatedAssetPropertyValues).not.toBeCalled(); + + expect(getAssetPropertyAggregates).toBeCalledTimes(2); + expect(getAssetPropertyAggregates).toBeCalledWith( + expect.objectContaining({ + assetId: query.assets[0].assetId, + propertyId: query.assets[0].properties[0].propertyId, + aggregateTypes: ['AVERAGE'], + resolution + }) + ); + expect(getAssetPropertyAggregates).toBeCalledWith( + expect.objectContaining({ + assetId: query.assets[1].assetId, + propertyId: query.assets[1].properties[0].propertyId, + aggregateTypes: ['AVERAGE'], + resolution + }) + ); + + expect(getAssetPropertyValueHistory).toBeCalledTimes(2); + expect(getAssetPropertyValueHistory).toBeCalledWith( + expect.objectContaining({ + assetId: query.assets[0].assetId, + propertyId: query.assets[0].properties[1].propertyId, + }) + ); + expect(getAssetPropertyValueHistory).toBeCalledWith( + expect.objectContaining({ + assetId: query.assets[1].assetId, + propertyId: query.assets[1].properties[1].propertyId, + }) + ); + + expect(onError).not.toBeCalled(); + + expect(onSuccess).toBeCalledTimes(4); + expect(onSuccess).toBeCalledWith([ + expect.objectContaining({ + id: toDataStreamId({ assetId: 'some-asset-id', propertyId: 'some-property-id' }), + aggregates: { + [MINUTE_IN_MS]: [ + { + x: 946602000000, + y: 5, + }, + { + x: 946605600000, + y: 7, + }, + { + x: 946609200000, + y: 10, + } + ] + }, + resolution: MINUTE_IN_MS, + dataType: 'NUMBER', + }), + ]); + expect(onSuccess).toBeCalledWith([ + expect.objectContaining({ + id: toDataStreamId({ assetId: 'some-asset-id2', propertyId: 'some-property-id' }), + aggregates: { + [MINUTE_IN_MS]: [ + { + x: 946602000000, + y: 5, + }, + { + x: 946605600000, + y: 7, + }, + { + x: 946609200000, + y: 10, + } + ] + }, + resolution: MINUTE_IN_MS, + dataType: 'NUMBER', + }), + ]); + + expect(onSuccess).toBeCalledWith([ + expect.objectContaining({ + id: toDataStreamId({ assetId: 'some-asset-id', propertyId: 'some-property-id2' }), + data: [{ x: 1000099, y: 10.123 }, { x: 2000000, y: 12.01 }], + resolution: 0, + dataType: 'NUMBER', + }), + ]); + expect(onSuccess).toBeCalledWith([ + expect.objectContaining({ + id: toDataStreamId({ assetId: 'some-asset-id2', propertyId: 'some-property-id2' }), + data: [{ x: 1000099, y: 10.123 }, { x: 2000000, y: 12.01 }], + resolution: 0, + dataType: 'NUMBER', + }), + ]); + }); + + it('requests raw data if specified per asset property', async () => { + const getAssetPropertyValue = jest.fn(); + const getAssetPropertyAggregates = jest.fn(); + const getAssetPropertyValueHistory = jest.fn().mockResolvedValue(ASSET_PROPERTY_VALUE_HISTORY); + const getInterpolatedAssetPropertyValues = jest.fn(); + + const mockSDK = createSiteWiseSDK({ + getAssetPropertyValue, + getAssetPropertyValueHistory, + getAssetPropertyAggregates, + getInterpolatedAssetPropertyValues, + }); + + const dataSource = createDataSource(mockSDK); + + const query: SiteWiseDataStreamQuery = { + source: SITEWISE_DATA_SOURCE, + assets: [ + { + assetId: 'some-asset-id', + properties: [ + { propertyId: 'some-property-id', resolution: '0' } + ] + } + ], + }; + + const onError = jest.fn(); + const onSuccess = jest.fn(); + + const FIFTY_FIVE_MINUTES = MINUTE_IN_MS * 55; + + dataSource.initiateRequest( + { + onError, + onSuccess, + query, + requestInfo: { + viewport: { + duration: FIFTY_FIVE_MINUTES, + }, + onlyFetchLatestValue: false, + requestConfig: { + fetchAggregatedData: true, + resolution: '1m' + } + }, + }, + [] + ); + + await flushPromises(); + + expect(getAssetPropertyValue).not.toBeCalled(); + expect(getInterpolatedAssetPropertyValues).not.toBeCalled(); + + expect(getAssetPropertyAggregates).not.toBeCalled(); + + expect(getAssetPropertyValueHistory).toBeCalledTimes(1); + expect(getAssetPropertyValueHistory).toBeCalledWith( + expect.objectContaining({ + assetId: query.assets[0].assetId, + propertyId: query.assets[0].properties[0].propertyId, + }) + ); + + expect(onError).not.toBeCalled(); + + expect(onSuccess).toBeCalledTimes(1); + + expect(onSuccess).toBeCalledWith([ + expect.objectContaining({ + id: toDataStreamId({ assetId: 'some-asset-id', propertyId: 'some-property-id' }), + data: [{ x: 1000099, y: 10.123 }, { x: 2000000, y: 12.01 }], + resolution: 0, + dataType: 'NUMBER', + }), + ]); + }); + it('throws error when invalid resolution used in mapping', () => { const mockSDK = createSiteWiseSDK({}); @@ -460,7 +792,7 @@ describe('aggregated data', () => { onlyFetchLatestValue: false, requestConfig: { fetchAggregatedData: true, - resolutionMapping: { + resolution: { [MINUTE_IN_MS]: 'not_a_valid_resolution' }, } diff --git a/packages/core/src/data-sources/site-wise/data-source.ts b/packages/core/src/data-sources/site-wise/data-source.ts index 94423edb4..462d0a197 100644 --- a/packages/core/src/data-sources/site-wise/data-source.ts +++ b/packages/core/src/data-sources/site-wise/data-source.ts @@ -1,10 +1,10 @@ import { IoTSiteWiseClient, AggregateType } from '@aws-sdk/client-iotsitewise'; import { DataSource } from '../../data-module/types.d'; -import { SiteWiseDataStreamQuery } from './types.d'; +import { PropertyQuery, SiteWiseAssetDataStreamQuery, SiteWiseDataStreamQuery } from './types.d'; import { SiteWiseClient } from './client/client'; import { toDataStreamId } from './util/dataStreamId'; import { viewportEndDate, viewportStartDate } from '../../common/viewport'; -import { ResolutionMapping } from '../../data-module/data-cache/requestTypes'; +import { ResolutionConfig } from '../../data-module/data-cache/requestTypes'; import { MINUTE_IN_MS, HOUR_IN_MS, DAY_IN_MS } from '../../common/time'; import { RESOLUTION_TO_MS_MAPPING, SupportedResolutions } from './util/resolution'; @@ -21,38 +21,106 @@ const isSiteWiseResolution = (resolution: string | SupportedResolutions): resolu } export const determineResolution = ({ - resolutionMapping, + resolution, fetchAggregatedData = false, start, end }: { - resolutionMapping?: ResolutionMapping; + resolution?: ResolutionConfig; fetchAggregatedData?: boolean, start: Date; end: Date; }): string => { - // by default request raw data - let resolution = '0'; - if (fetchAggregatedData) { const viewportTimeSpan = end.getTime() - start.getTime(); - const resolutionMappingOverride = resolutionMapping || DEFAULT_RESOLUTION_MAPPING; + const resolutionOverride = resolution || DEFAULT_RESOLUTION_MAPPING; + + if (typeof resolutionOverride === 'string') { + return resolutionOverride; + } - const matchedViewport = Object.keys(resolutionMappingOverride) + const matchedViewport = Object.keys(resolutionOverride) .sort((a, b) => parseInt(b) - parseInt(a)) .find(viewport => viewportTimeSpan >= parseInt(viewport)); if (matchedViewport) { - resolution = resolutionMappingOverride[parseInt(matchedViewport)] as string; + const matchedResolution = resolutionOverride[parseInt(matchedViewport)] as string; - if (!isSiteWiseResolution(resolution)) { - throw new Error(`${resolution} is not a valid SiteWise aggregation resolution, must match regex pattern '1m|1h|1d'`); + if (!isSiteWiseResolution(matchedResolution)) { + throw new Error(`${matchedResolution} is not a valid SiteWise aggregation resolution, must match regex pattern '1m|1h|1d'`); } + + return matchedResolution; } } - return resolution; + return '0'; +} + +const separateDataQueries = (query: SiteWiseDataStreamQuery): + { + aggregatedDataQueries?: SiteWiseAssetDataStreamQuery; + rawDataQueries?: SiteWiseAssetDataStreamQuery; + defaultResolutionDataQueries?: SiteWiseAssetDataStreamQuery; + } => { + let aggregatedDataQueries: SiteWiseAssetDataStreamQuery | undefined; + let rawDataQueries: SiteWiseAssetDataStreamQuery | undefined; + let defaultResolutionDataQueries: SiteWiseAssetDataStreamQuery | undefined; + + query.assets.forEach(({ assetId, properties }) => { + let aggregatedDataProperties: PropertyQuery[] | undefined; + let rawDataProperties: PropertyQuery[] | undefined; + let defaultResolutionDataProperties: PropertyQuery[] | undefined; + + properties.forEach(({ propertyId, resolution }) => { + if (resolution === '0') { + if (!rawDataProperties) { + rawDataProperties = [{ propertyId, resolution }] + } else { + rawDataProperties.push({ propertyId, resolution }); + } + } else if (typeof resolution === 'string' && isSiteWiseResolution(resolution)) { + if (!aggregatedDataProperties) { + aggregatedDataProperties = [{ propertyId, resolution }] + } else { + aggregatedDataProperties.push({ propertyId, resolution }); + } + } else { + if (!defaultResolutionDataProperties) { + defaultResolutionDataProperties = [{ propertyId, resolution }] + } else { + defaultResolutionDataProperties.push({ propertyId, resolution }); + } + } + }); + + if (aggregatedDataProperties) { + if (!aggregatedDataQueries) { + aggregatedDataQueries = { ...query, assets: [{ assetId, properties: aggregatedDataProperties }] } + } else { + aggregatedDataQueries.assets.push({ assetId, properties: aggregatedDataProperties }); + } + } + + if (rawDataProperties) { + if (!rawDataQueries) { + rawDataQueries = { ...query, assets: [{ assetId, properties: rawDataProperties }] } + } else { + rawDataQueries.assets.push({ assetId, properties: rawDataProperties }); + } + } + + if (defaultResolutionDataProperties) { + if (!defaultResolutionDataQueries) { + defaultResolutionDataQueries = { ...query, assets: [{ assetId, properties: defaultResolutionDataProperties }] } + } else { + defaultResolutionDataQueries.assets.push({ assetId, properties: defaultResolutionDataProperties }); + } + } + }); + + return { rawDataQueries, aggregatedDataQueries, defaultResolutionDataQueries }; } export const createDataSource = (siteWise: IoTSiteWiseClient): DataSource => { @@ -68,44 +136,77 @@ export const createDataSource = (siteWise: IoTSiteWiseClient): DataSource client.getAggregatedPropertyDataPoints({ + query: aggregatedDataQueries, onSuccess, onError, start, end, - resolution, aggregateTypes - }); + })); + } + + if (rawDataQueries) { + requests.push(() => client.getHistoricalPropertyDataPoints({ query: rawDataQueries, onSuccess, onError, start, end })); + } + + if (defaultResolutionDataQueries) { + if (resolution !== '0') { + requests.push(() => client.getAggregatedPropertyDataPoints({ + query: defaultResolutionDataQueries, + onSuccess, + onError, + start, + end, + resolution, + aggregateTypes + })); + } else { + requests.push(() => client.getHistoricalPropertyDataPoints({ + query: defaultResolutionDataQueries, + onSuccess, + onError, + start, + end + })); + } } - return client.getHistoricalPropertyDataPoints({ query, onSuccess, onError, start, end }); + return Promise.all(requests.map(async (request) => request())); }, getRequestsFromQuery: ({ query, requestInfo }) => { const start = viewportStartDate(requestInfo.viewport); const end = viewportEndDate(requestInfo.viewport); const resolution = determineResolution({ - resolutionMapping: requestInfo.requestConfig?.resolutionMapping, + resolution: requestInfo.requestConfig?.resolution, fetchAggregatedData: requestInfo.requestConfig?.fetchAggregatedData, start, end }); - return query.assets.flatMap(({assetId, propertyIds}) => - propertyIds.map((propertyId) => ({ + return query.assets.flatMap(({assetId, properties}) => + properties.map(({ propertyId, resolution: resolutionOverride }) => ({ id: toDataStreamId({assetId, propertyId}), - resolution: RESOLUTION_TO_MS_MAPPING[resolution], + resolution: RESOLUTION_TO_MS_MAPPING[resolutionOverride || resolution], })) ); }, diff --git a/packages/core/src/data-sources/site-wise/types.d.ts b/packages/core/src/data-sources/site-wise/types.d.ts index 289c23702..6045b5ea0 100644 --- a/packages/core/src/data-sources/site-wise/types.d.ts +++ b/packages/core/src/data-sources/site-wise/types.d.ts @@ -10,9 +10,14 @@ export type AssetId = string; export type PropertyAlias = string; +export type PropertyQuery = { + propertyId: string, + resolution?: string +} + export type AssetQuery = { assetId: AssetId; - propertyIds: AssetPropertyId[]; + properties: PropertyQuery[] }; type SiteWiseAssetDataStreamQuery = DataStreamQuery & {