Do not break snapshots into multiple record batches (yet) (#3866)

deephaven · May 26, 2023 · feb4998 · feb4998
1 parent b5990f1
commit feb4998
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 2 deletions.
diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java
@@ -22,6 +22,7 @@
 import io.deephaven.engine.table.TableDefinition;
 import io.deephaven.engine.updategraph.NotificationQueue;
 import io.deephaven.engine.updategraph.WaitNotification;
+import io.deephaven.proto.backplane.grpc.Config;
 import io.deephaven.time.DateTime;
 import io.deephaven.util.datastructures.LongSizedDataStructure;
 import io.deephaven.engine.liveness.LivenessManager;
@@ -79,7 +80,10 @@ public NoSnapshotAllowedException(String reason) {
     private static final int MAX_CONCURRENT_ATTEMPT_DURATION_MILLIS = Configuration.getInstance()
             .getIntegerWithDefault("ConstructSnapshot.maxConcurrentAttemptDurationMillis", 5000);
 
-    public static final int SNAPSHOT_CHUNK_SIZE = ChunkPoolConstants.LARGEST_POOLED_CHUNK_CAPACITY;
+    // TODO (deephaven-core#188): use ChunkPoolConstants.LARGEST_POOL_CHUNK_CAPACITY when JS API allows multiple batches
+    // default enables more than 100MB of 8-byte values in a single record batch
+    public static final int SNAPSHOT_CHUNK_SIZE = Configuration.getInstance()
+            .getIntegerWithDefault("ConstructSnapshot.snapshotChunkSize", 1 << 24);
 
     /**
      * Holder for thread-local state.

diff --git a/...ons/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java b/...ons/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java
@@ -771,7 +771,7 @@ private void processBatches(Consumer<InputStream> visitor, final View view,
         final int maxMessageSize =
                 view.clientMaxMessageSize() > 0 ? view.clientMaxMessageSize() : DEFAULT_MESSAGE_SIZE_LIMIT;
 
-        // TODO: remove this when JS API can accept multiple batches
+        // TODO (deephaven-core#188): remove this when JS API can accept multiple batches
         boolean sendAllowed = numRows <= batchSize;
 
         while (offset < numRows) {