From feb4998dbc9567862e1a55fc20c9eb2d0411c062 Mon Sep 17 00:00:00 2001 From: Nate Bauernfeind Date: Fri, 26 May 2023 10:26:54 -0600 Subject: [PATCH] Do not break snapshots into multiple record batches (yet) (#3866) --- .../engine/table/impl/remote/ConstructSnapshot.java | 6 +++++- .../extensions/barrage/BarrageStreamGeneratorImpl.java | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java index 824f8055a14..f7796b62437 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java @@ -22,6 +22,7 @@ import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.updategraph.NotificationQueue; import io.deephaven.engine.updategraph.WaitNotification; +import io.deephaven.proto.backplane.grpc.Config; import io.deephaven.time.DateTime; import io.deephaven.util.datastructures.LongSizedDataStructure; import io.deephaven.engine.liveness.LivenessManager; @@ -79,7 +80,10 @@ public NoSnapshotAllowedException(String reason) { private static final int MAX_CONCURRENT_ATTEMPT_DURATION_MILLIS = Configuration.getInstance() .getIntegerWithDefault("ConstructSnapshot.maxConcurrentAttemptDurationMillis", 5000); - public static final int SNAPSHOT_CHUNK_SIZE = ChunkPoolConstants.LARGEST_POOLED_CHUNK_CAPACITY; + // TODO (deephaven-core#188): use ChunkPoolConstants.LARGEST_POOL_CHUNK_CAPACITY when JS API allows multiple batches + // default enables more than 100MB of 8-byte values in a single record batch + public static final int SNAPSHOT_CHUNK_SIZE = Configuration.getInstance() + .getIntegerWithDefault("ConstructSnapshot.snapshotChunkSize", 1 << 24); /** * Holder for thread-local state. diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java index 946257e1b9c..2edba986263 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java @@ -771,7 +771,7 @@ private void processBatches(Consumer visitor, final View view, final int maxMessageSize = view.clientMaxMessageSize() > 0 ? view.clientMaxMessageSize() : DEFAULT_MESSAGE_SIZE_LIMIT; - // TODO: remove this when JS API can accept multiple batches + // TODO (deephaven-core#188): remove this when JS API can accept multiple batches boolean sendAllowed = numRows <= batchSize; while (offset < numRows) {