From 0ba3f1b93d9260fbd3d787ad90feb1003375c257 Mon Sep 17 00:00:00 2001 From: Bowei Han Date: Tue, 25 Jan 2022 13:51:17 -0500 Subject: [PATCH] support cacheSettings at a query level --- .../data-module/IotAppKitDataModule.spec.ts | 48 +++++++++++++++++++ .../src/data-module/IotAppKitDataModule.ts | 4 +- .../data-cache/caching/caching.spec.ts | 23 +++++++++ .../data-module/data-cache/caching/caching.ts | 11 +++++ .../data-source-store/dataSourceStore.ts | 4 +- .../subscription-store/subscriptionStore.ts | 7 ++- packages/core/src/data-module/types.d.ts | 9 +++- .../src/data-sources/site-wise/types.d.ts | 2 + 8 files changed, 103 insertions(+), 5 deletions(-) diff --git a/packages/core/src/data-module/IotAppKitDataModule.spec.ts b/packages/core/src/data-module/IotAppKitDataModule.spec.ts index f5d26a614..29b8dcbe3 100644 --- a/packages/core/src/data-module/IotAppKitDataModule.spec.ts +++ b/packages/core/src/data-module/IotAppKitDataModule.spec.ts @@ -808,6 +808,54 @@ describe('caching', () => { }); }); +it('overrides module-level cache TTL if query-level cache TTL is provided', async () => { + const customCacheSettings = { + ttlDurationMapping: { + [MINUTE_IN_MS]: 0, + [5 * MINUTE_IN_MS]: 30 * SECOND_IN_MS, + }, + }; + + const dataModule = new IotAppKitDataModule({ cacheSettings: customCacheSettings }); + const dataSource = createMockSiteWiseDataSource([DATA_STREAM]); + dataModule.registerDataSource(dataSource); + + const END = new Date(); + const START = new Date(END.getTime() - HOUR_IN_MS); + + const dataStreamCallback = jest.fn(); + dataModule.subscribeToDataStreams( + { + queries: [ + { + ...DATA_STREAM_QUERY, + cacheSettings: { + ttlDurationMapping: { + [MINUTE_IN_MS]: 0, + [10 * MINUTE_IN_MS]: 30 * SECOND_IN_MS, + }, + }, + }, + ], + request: { viewport: { start: START, end: END }, settings: { refreshRate: MINUTE_IN_MS } }, + }, + dataStreamCallback + ); + + (dataSource.initiateRequest as Mock).mockClear(); + jest.advanceTimersByTime(MINUTE_IN_MS); + + expect(dataSource.initiateRequest).toBeCalledWith(expect.any(Object), [ + { + id: DATA_STREAM_INFO.id, + resolution: DATA_STREAM_INFO.resolution, + // 1 minute time advancement invalidates 10 minutes of cache with query-level mapping, which is 9 minutes from END_1 + start: new Date(END.getTime() - 9 * MINUTE_IN_MS), + end: END, + }, + ]); +}); + describe('request scheduler', () => { it('periodically requests duration based queries', async () => { const dataModule = new IotAppKitDataModule(); diff --git a/packages/core/src/data-module/IotAppKitDataModule.ts b/packages/core/src/data-module/IotAppKitDataModule.ts index 9ecd20a81..afaf011a3 100644 --- a/packages/core/src/data-module/IotAppKitDataModule.ts +++ b/packages/core/src/data-module/IotAppKitDataModule.ts @@ -100,14 +100,14 @@ export class IotAppKitDataModule implements DataModule { ); const requests = requiredStreams - .map(({ resolution, id }) => { + .map(({ resolution, id, cacheSettings }) => { const dateRanges = getDateRangesToRequest({ store: this.dataCache.getState(), start: adjustedStart, end: adjustedEnd, resolution, dataStreamId: id, - cacheSettings: this.cacheSettings, + cacheSettings: { ...this.cacheSettings, ...cacheSettings }, }); return { diff --git a/packages/core/src/data-module/data-cache/caching/caching.spec.ts b/packages/core/src/data-module/data-cache/caching/caching.spec.ts index d35482a04..3fbe4cd30 100755 --- a/packages/core/src/data-module/data-cache/caching/caching.spec.ts +++ b/packages/core/src/data-module/data-cache/caching/caching.spec.ts @@ -6,6 +6,7 @@ import { EMPTY_CACHE, getDateRangesToRequest, unexpiredCacheIntervals, + maxCacheDuration, } from './caching'; import { DEFAULT_CACHE_SETTINGS } from '../../IotAppKitDataModule'; import { HOUR_IN_MS, MINUTE_IN_MS, SECOND_IN_MS } from '../../../common/time'; @@ -1210,3 +1211,25 @@ describe('checkCacheForRecentPoint', () => { expect(presentInCache).toBeFalse(); }); }); + +describe('maxCacheDuration', () => { + it('returns the maximum cache TTL duration', () => { + expect( + maxCacheDuration({ + ttlDurationMapping: { + [1.2 * MINUTE_IN_MS]: 0, + [3 * MINUTE_IN_MS]: 30 * SECOND_IN_MS, + [20 * MINUTE_IN_MS]: 5 * MINUTE_IN_MS, + }, + }) + ).toBe(20 * MINUTE_IN_MS); + }); + + it('handles empty mappings', () => { + expect( + maxCacheDuration({ + ttlDurationMapping: {}, + }) + ).toBe(0); + }); +}); diff --git a/packages/core/src/data-module/data-cache/caching/caching.ts b/packages/core/src/data-module/data-cache/caching/caching.ts index 729798c19..852d71bcc 100755 --- a/packages/core/src/data-module/data-cache/caching/caching.ts +++ b/packages/core/src/data-module/data-cache/caching/caching.ts @@ -204,3 +204,14 @@ export const validateRequestConfig = (requestConfig: TimeSeriesDataRequestSettin return false; }; + +// Returns the maximum duration for possible uncached data for given CacheSettings +export const maxCacheDuration = (cacheSettings: CacheSettings) => { + const ttlDurations = Object.keys(cacheSettings.ttlDurationMapping).map((key) => Number(key)); + + if (ttlDurations.length === 0) { + return 0; + } + + return Math.max(...ttlDurations); +}; diff --git a/packages/core/src/data-module/data-source-store/dataSourceStore.ts b/packages/core/src/data-module/data-source-store/dataSourceStore.ts index ed3890a4f..226dfbc54 100644 --- a/packages/core/src/data-module/data-source-store/dataSourceStore.ts +++ b/packages/core/src/data-module/data-source-store/dataSourceStore.ts @@ -43,7 +43,9 @@ export default class DataSourceStore { request: TimeSeriesDataRequest; }): RequestInformation[] => { const dataSource = this.getDataSource(query.source); - return dataSource.getRequestsFromQuery({ query, request }); + return dataSource + .getRequestsFromQuery({ query, request }) + .map((request) => ({ ...request, cacheSettings: query.cacheSettings })); }; public initiateRequest = ( diff --git a/packages/core/src/data-module/subscription-store/subscriptionStore.ts b/packages/core/src/data-module/subscription-store/subscriptionStore.ts index 9c1c44b18..21ebe80ac 100644 --- a/packages/core/src/data-module/subscription-store/subscriptionStore.ts +++ b/packages/core/src/data-module/subscription-store/subscriptionStore.ts @@ -4,6 +4,7 @@ import { CacheSettings } from '../data-cache/types'; import DataSourceStore from '../data-source-store/dataSourceStore'; import RequestScheduler from '../request-scheduler/requestScheduler'; import { viewportEndDate } from '../../common/viewport'; +import { maxCacheDuration } from '../data-cache/caching/caching'; /** * Subscription store @@ -55,7 +56,11 @@ export default class SubscriptionStore { refreshRate: subscription.request.settings?.refreshRate, refreshExpiration: viewportEndDate(subscription.request.viewport).getTime() + - Math.max(...Object.keys(this.cacheSettings.ttlDurationMapping).map((key) => Number(key))), + Math.max( + ...subscription.queries.map((query) => + maxCacheDuration({ ...this.cacheSettings, ...query.cacheSettings }) + ) + ), }); } diff --git a/packages/core/src/data-module/types.d.ts b/packages/core/src/data-module/types.d.ts index 0f9529fa1..8a88ba273 100644 --- a/packages/core/src/data-module/types.d.ts +++ b/packages/core/src/data-module/types.d.ts @@ -13,8 +13,14 @@ import { ListAssociatedAssetsCommandOutput, } from '@aws-sdk/client-iotsitewise'; import { RefId } from '../data-sources/site-wise/types'; +import { CacheSettings } from './data-cache/types'; -export type RequestInformation = { id: DataStreamId; resolution: Resolution; refId?: RefId }; +export type RequestInformation = { + id: DataStreamId; + resolution: Resolution; + refId?: RefId; + cacheSettings?: CacheSettings; +}; export type RequestInformationAndRange = RequestInformation & { start: Date; end: Date }; export type DataSourceName = string; @@ -55,6 +61,7 @@ export type DataModuleSubscription = { export type DataStreamQuery = { source: DataSourceName; + cacheSettings?: CacheSettings; }; export type AnyDataStreamQuery = DataStreamQuery & any; diff --git a/packages/core/src/data-sources/site-wise/types.d.ts b/packages/core/src/data-sources/site-wise/types.d.ts index 4807dbc63..e129da31d 100644 --- a/packages/core/src/data-sources/site-wise/types.d.ts +++ b/packages/core/src/data-sources/site-wise/types.d.ts @@ -1,3 +1,4 @@ +import { CacheSettings } from '../../data-module/data-cache/types'; import { DataStreamQuery, SubscriptionUpdate } from '../../data-module/types.d'; /** @@ -17,6 +18,7 @@ export type PropertyQuery = { propertyId: string; refId?: RefId; resolution?: string; + cacheSettings?: CacheSettings; }; export type AssetQuery = {