PLAT-3758: Temporal AA support (#559)

* Temporal AA support To avoid unnecessary CPU and network overhead, the depth buffer is not sent for successive temporal AA frames. This adds support for this; the SDK will reuse the previous depth buffer when possible. [PLAT-3758](https://vertexvis.atlassian.net/browse/PLAT-3758) * lint * ooops, more lint * PR suggestion * oops
Vertexvis · Jan 10, 2024 · f0e4401 · f0e4401
1 parent 59be370
commit f0e4401
Show file tree

Hide file tree

Showing 10 changed files with 149 additions and 14 deletions.
diff --git a/packages/stream-api/package.json b/packages/stream-api/package.json
@@ -36,7 +36,7 @@
     "test:coverage": "yarn test --coverage"
   },
   "dependencies": {
-    "@vertexvis/frame-streaming-protos": "^0.12.0"
+    "@vertexvis/frame-streaming-protos": "^0.13.0"
   },
   "devDependencies": {
     "@types/jest": "^27.5.1",

diff --git a/packages/viewer/package.json b/packages/viewer/package.json
@@ -49,7 +49,7 @@
     "@improbable-eng/grpc-web": "^0.15.0",
     "@stencil/core": "^2.16.1",
     "@types/classnames": "^2.3.1",
-    "@vertexvis/frame-streaming-protos": "^0.12.0",
+    "@vertexvis/frame-streaming-protos": "^0.13.0",
     "@vertexvis/geometry": "0.19.2",
     "@vertexvis/html-templates": "0.19.2",
     "@vertexvis/scene-tree-protos": "^0.1.18",

diff --git a/packages/viewer/src/components/viewer-hit-result-indicator/lib/__tests__/indicator.spec.ts b/packages/viewer/src/components/viewer-hit-result-indicator/lib/__tests__/indicator.spec.ts
@@ -116,6 +116,7 @@ function updateFrameCameraPosition(
 ): Frame {
   return new Frame(
     baseFrame.correlationIds,
+    baseFrame.temporalRefinementCorrelationId,
     baseFrame.sequenceNumber,
     baseFrame.dimensions,
     baseFrame.image,

diff --git a/packages/viewer/src/components/viewer-transform-widget/__tests__/widget.spec.ts b/packages/viewer/src/components/viewer-transform-widget/__tests__/widget.spec.ts
@@ -121,6 +121,7 @@ function updateFrameCameraPosition(
 ): Frame {
   return new Frame(
     baseFrame.correlationIds,
+    baseFrame.temporalRefinementCorrelationId,
     baseFrame.sequenceNumber,
     baseFrame.dimensions,
     baseFrame.image,

diff --git a/packages/viewer/src/components/viewer/viewer.spec.tsx b/packages/viewer/src/components/viewer/viewer.spec.tsx
@@ -1,3 +1,5 @@
+import { vertexvis } from '@vertexvis/frame-streaming-protos';
+
 jest.mock('./utils');
 jest.mock('../../lib/rendering/imageLoaders');
 jest.mock('../../workers/png-decoder-pool');
@@ -7,14 +9,15 @@ import { h } from '@stencil/core';
 import { NewSpecPageOptions, SpecPage } from '@stencil/core/internal';
 import { newSpecPage } from '@stencil/core/testing';
 import { Dimensions } from '@vertexvis/geometry';
-import { Async } from '@vertexvis/utils';
+import { Async, UUID } from '@vertexvis/utils';
 
 import { MouseInteractionHandler } from '../../lib/interactions/mouseInteractionHandler';
 import { TapInteractionHandler } from '../../lib/interactions/tapInteractionHandler';
 import { TouchInteractionHandler } from '../../lib/interactions/touchInteractionHandler';
 import { loadImageBytes } from '../../lib/rendering/imageLoaders';
 import * as Storage from '../../lib/storage';
 import { random } from '../../testing';
+import * as Fixtures from '../../testing/fixtures';
 import { makeImagePng } from '../../testing/fixtures';
 import { triggerResizeObserver } from '../../testing/resizeObserver';
 import {
@@ -620,6 +623,106 @@ describe('vertex-viewer', () => {
     });
   });
 
+  describe('temporal AA', () => {
+    it('reuses previous depth buffer if temporal correlation id matches', async () => {
+      const { stream, ws } = makeViewerStream();
+      const viewer = await newViewerSpec({
+        template: () => <vertex-viewer clientId={clientId} stream={stream} />,
+      });
+
+      await loadViewerStreamKey(key1, { viewer, stream, ws }, { token });
+
+      const onFrameDrawn = jest.fn();
+
+      viewer.addEventListener('frameDrawn', onFrameDrawn);
+
+      const tcri = new vertexvis.protobuf.core.Uuid({ hex: UUID.create() });
+
+      receiveFrame(ws, (payload) => ({
+        ...payload,
+        sequenceNumber: 2,
+        temporalRefinementCorrelationId: tcri,
+      }));
+
+      receiveFrame(ws, (payload) => ({
+        ...payload,
+        sequenceNumber: 3,
+        temporalRefinementCorrelationId: tcri,
+        depthBuffer: null,
+      }));
+
+      await Async.delay(10);
+
+      expect(onFrameDrawn).toHaveBeenNthCalledWith(
+        2,
+        expect.objectContaining({
+          detail: expect.objectContaining({
+            depthBufferBytes:
+              Fixtures.drawFramePayloadPerspective.depthBuffer?.value,
+          }),
+        })
+      );
+      expect(onFrameDrawn).toHaveBeenNthCalledWith(
+        3,
+        expect.objectContaining({
+          detail: expect.objectContaining({
+            depthBufferBytes:
+              Fixtures.drawFramePayloadPerspective.depthBuffer?.value,
+          }),
+        })
+      );
+    });
+
+    it('does not reuse previous depth buffer if temporal correlation id does not match', async () => {
+      const { stream, ws } = makeViewerStream();
+      const viewer = await newViewerSpec({
+        template: () => <vertex-viewer clientId={clientId} stream={stream} />,
+      });
+
+      await loadViewerStreamKey(key1, { viewer, stream, ws }, { token });
+
+      const onFrameDrawn = jest.fn();
+
+      viewer.addEventListener('frameDrawn', onFrameDrawn);
+
+      const tcri = new vertexvis.protobuf.core.Uuid({ hex: UUID.create() });
+      const tcri2 = new vertexvis.protobuf.core.Uuid({ hex: UUID.create() });
+
+      receiveFrame(ws, (payload) => ({
+        ...payload,
+        sequenceNumber: 2,
+        temporalRefinementCorrelationId: tcri,
+      }));
+
+      receiveFrame(ws, (payload) => ({
+        ...payload,
+        sequenceNumber: 3,
+        temporalRefinementCorrelationId: tcri2,
+        depthBuffer: null,
+      }));
+
+      await Async.delay(10);
+
+      expect(onFrameDrawn).toHaveBeenNthCalledWith(
+        2,
+        expect.objectContaining({
+          detail: expect.objectContaining({
+            depthBufferBytes:
+              Fixtures.drawFramePayloadPerspective.depthBuffer?.value,
+          }),
+        })
+      );
+      expect(onFrameDrawn).toHaveBeenNthCalledWith(
+        3,
+        expect.objectContaining({
+          detail: expect.objectContaining({
+            depthBufferBytes: undefined,
+          }),
+        })
+      );
+    });
+  });
+
   describe('resizing', () => {
     it('handles resizes', async () => {
       const { stream, ws } = makeViewerStream();

diff --git a/packages/viewer/src/lib/mappers/frameStreaming.ts b/packages/viewer/src/lib/mappers/frameStreaming.ts
@@ -283,9 +283,13 @@ export function fromPbFrame(
       fromPbFrameScene(worldOrientation),
       fromPbFrameImage,
       M.mapProp('depthBuffer', fromPbBytesValue),
-      M.mapProp('featureMap', fromPbBytesValue)
+      M.mapProp('featureMap', fromPbBytesValue),
+      M.mapProp('temporalRefinementCorrelationId', (id) =>
+        id != null ? fromPbUuid(id) : null
+      )
     ),
-    ([cIds, seq, fd, s, i, db, fm]) => new Frame(cIds, seq, fd, i, s, db, fm)
+    ([cIds, seq, fd, s, i, db, fm, trci]) =>
+      new Frame(cIds, trci || '', seq, fd, i, s, db, fm)
   );
 }
 

diff --git a/packages/viewer/src/lib/stream/state.ts b/packages/viewer/src/lib/stream/state.ts
@@ -26,6 +26,7 @@ export interface Connected {
   readonly token: Token;
   readonly frame: Frame;
   readonly clock: SynchronizedClock;
+  readonly fallbackDepthBufferBytes: Uint8Array | undefined;
 }
 
 export interface ConnectionFailed {

diff --git a/packages/viewer/src/lib/stream/stream.ts b/packages/viewer/src/lib/stream/stream.ts
@@ -336,7 +336,24 @@ export class ViewerStream extends StreamApi {
         const frame = fromPbFrameOrThrow(stream.worldOrientation)(req);
 
         if (this.state.type === 'connected') {
-          this.updateState({ ...this.state, frame });
+          if (
+            frame.depthBufferBytes != null ||
+            this.state.frame.temporalRefinementCorrelationId !==
+              frame.temporalRefinementCorrelationId
+          ) {
+            this.updateState({
+              ...this.state,
+              frame: frame,
+              fallbackDepthBufferBytes: frame.depthBufferBytes,
+            });
+          } else {
+            this.updateState({
+              ...this.state,
+              frame: frame.copy({
+                depthBufferBytes: this.state.fallbackDepthBufferBytes,
+              }),
+            });
+          }
         }
       }
     });
@@ -376,6 +393,7 @@ export class ViewerStream extends StreamApi {
       token: stream.token,
       frame,
       clock,
+      fallbackDepthBufferBytes: undefined,
     });
   }
 
@@ -389,6 +407,7 @@ export class ViewerStream extends StreamApi {
         streamKey: { value: resource.resource.id },
         dimensions: this.getDimensions(),
         frameBackgroundColor: toPbColorOrThrow(this.frameBgColor),
+        clientSupportsTemporalRefinement: true,
         streamAttributes: toPbStreamAttributesOrThrow(this.streamAttributes),
         sceneViewStateId:
           resource.subResource?.type === 'scene-view-state' &&
@@ -412,6 +431,7 @@ export class ViewerStream extends StreamApi {
       token: res.token,
       worldOrientation: res.worldOrientation,
       frame: undefined,
+      fallbackDepthBufferBytes: undefined,
     };
   }
 
@@ -424,6 +444,7 @@ export class ViewerStream extends StreamApi {
         dimensions: this.getDimensions(),
         frameBackgroundColor: toPbColorOrThrow(this.frameBgColor),
         streamAttributes: toPbStreamAttributesOrThrow(this.streamAttributes),
+        clientSupportsTemporalRefinement: true,
       })
     );
     return { ...state, token: res.token };

diff --git a/packages/viewer/src/lib/types/frame.ts b/packages/viewer/src/lib/types/frame.ts
@@ -26,11 +26,12 @@ export class Frame {
 
   public constructor(
     public readonly correlationIds: string[],
+    public readonly temporalRefinementCorrelationId: string,
     public readonly sequenceNumber: number,
     public readonly dimensions: Dimensions.Dimensions,
     public readonly image: FrameImage,
     public readonly scene: FrameScene,
-    private readonly depthBufferBytes: Uint8Array | undefined,
+    public readonly depthBufferBytes: Uint8Array | undefined,
     private readonly featureMapBytes: Uint8Array | undefined,
     private readonly id = UUID.create()
   ) {}
@@ -71,6 +72,7 @@ export class Frame {
 
   public copy({
     correlationIds,
+    temporalRefinementCorrelationId,
     sequenceNumber,
     dimensions,
     image,
@@ -79,6 +81,7 @@ export class Frame {
     featureMapBytes,
   }: {
     correlationIds?: string[];
+    temporalRefinementCorrelationId?: string;
     sequenceNumber?: number;
     dimensions?: Dimensions.Dimensions;
     image?: FrameImage;
@@ -88,6 +91,7 @@ export class Frame {
   }): Frame {
     return new Frame(
       correlationIds ?? this.correlationIds,
+      temporalRefinementCorrelationId ?? this.temporalRefinementCorrelationId,
       sequenceNumber ?? this.sequenceNumber,
       dimensions ?? this.dimensions,
       image ?? this.image,

diff --git a/yarn.lock b/yarn.lock
@@ -2049,9 +2049,9 @@
     "@types/react" "*"
 
 "@types/react@*", "@types/react@^17", "@types/react@^17.0.22":
-  version "17.0.64"
-  resolved "https://registry.yarnpkg.com/@types/react/-/react-17.0.64.tgz#468162c66c33ddb4548eb1a0e36682028d9e9a62"
-  integrity sha512-IlgbX/vglDTwrCRgad6fTCzOT+D/5C0xwuvrzfuqfhg9gJrkFqAGADpUFlEtqbrP1IEo9QLSbo41MaFfoIu9Aw==
+  version "17.0.74"
+  resolved "https://registry.yarnpkg.com/@types/react/-/react-17.0.74.tgz#ea93059a55e5cfc7a76e7712fe8db5317dd29ee3"
+  integrity sha512-nBtFGaeTMzpiL/p73xbmCi00SiCQZDTJUk9ZuHOLtil3nI+y7l269LHkHIAYpav99ZwGnPJzuJsJpfLXjiQ52g==
   dependencies:
     "@types/prop-types" "*"
     "@types/scheduler" "*"
@@ -2216,10 +2216,10 @@
     eslint-plugin-simple-import-sort "^7.0.0"
     prettier "^2.5.1"
 
-"@vertexvis/frame-streaming-protos@^0.12.0":
-  version "0.12.0"
-  resolved "https://registry.yarnpkg.com/@vertexvis/frame-streaming-protos/-/frame-streaming-protos-0.12.0.tgz#fe7759e0b3a8e8a4f88eccd0a14414f9001c76d0"
-  integrity sha512-LcePeheuspMC11ylOXPyInyrBOBKYfKJQSpI4sqmkN3uo+ZTZxPh4TxKURoDfwsYIvtaPvE7CQgqJuaiccMa6w==
+"@vertexvis/frame-streaming-protos@^0.13.0":
+  version "0.13.0"
+  resolved "https://registry.yarnpkg.com/@vertexvis/frame-streaming-protos/-/frame-streaming-protos-0.13.0.tgz#b49c08855467edb429dcf7d9c6904b7c6215486d"
+  integrity sha512-8aIRjGNFZ/UHY1sS85MsISC0sIJgI6hQtgQKpKNQXHK2HIJ5Cwe3ZNT0Khk5pmcpLpGExczyvern5kTWaLMW7Q==
 
 "@vertexvis/jest-config-vertexvis@^0.5.4":
   version "0.5.4"