Rust eager serialization + partial updates playground

crates/build/re_types_builder/src/lib.rs

@@ -194,15 +194,17 @@ pub const ATTR_RERUN_EXPERIMENTAL: &str = "attr.rerun.experimental";
 pub const ATTR_PYTHON_ALIASES: &str = "attr.python.aliases";
 pub const ATTR_PYTHON_ARRAY_ALIASES: &str = "attr.python.array_aliases";
 
+pub const ATTR_CPP_NO_FIELD_CTORS: &str = "attr.cpp.no_field_ctors";
+pub const ATTR_CPP_RENAME_FIELD: &str = "attr.cpp.rename_field";
+
+pub const ATTR_RUST_ARCHETYPE_EAGER: &str = "attr.rust.archetype_eager";
 pub const ATTR_RUST_CUSTOM_CLAUSE: &str = "attr.rust.custom_clause";
 pub const ATTR_RUST_DERIVE: &str = "attr.rust.derive";
 pub const ATTR_RUST_DERIVE_ONLY: &str = "attr.rust.derive_only";
 pub const ATTR_RUST_NEW_PUB_CRATE: &str = "attr.rust.new_pub_crate";
 pub const ATTR_RUST_OVERRIDE_CRATE: &str = "attr.rust.override_crate";
 pub const ATTR_RUST_REPR: &str = "attr.rust.repr";
 pub const ATTR_RUST_TUPLE_STRUCT: &str = "attr.rust.tuple_struct";
-pub const ATTR_CPP_NO_FIELD_CTORS: &str = "attr.cpp.no_field_ctors";
-pub const ATTR_CPP_RENAME_FIELD: &str = "attr.cpp.rename_field";
 
 pub const ATTR_DOCS_UNRELEASED: &str = "attr.docs.unreleased";
 pub const ATTR_DOCS_CATEGORY: &str = "attr.docs.category";

crates/build/re_types_builder/src/objects.rs

@@ -12,7 +12,7 @@ use itertools::Itertools;
 use crate::{
     root_as_schema, Docs, FbsBaseType, FbsEnum, FbsEnumVal, FbsField, FbsKeyValue, FbsObject,
     FbsSchema, FbsType, Reporter, ATTR_RERUN_COMPONENT_OPTIONAL, ATTR_RERUN_COMPONENT_RECOMMENDED,
-    ATTR_RERUN_COMPONENT_REQUIRED, ATTR_RERUN_OVERRIDE_TYPE,
+    ATTR_RERUN_COMPONENT_REQUIRED, ATTR_RERUN_OVERRIDE_TYPE, ATTR_RUST_ARCHETYPE_EAGER,
 };
 
 // ---
@@ -686,6 +686,14 @@ impl Object {
             self.kind.plural_snake_case().to_owned()
         }
     }
+
+    pub fn is_archetype(&self) -> bool {
+        self.kind == ObjectKind::Archetype
+    }
+
+    pub fn is_eager_rust_archetype(&self) -> bool {
+        self.is_archetype() && self.is_attr_set(ATTR_RUST_ARCHETYPE_EAGER)
+    }
 }
 
 pub fn is_testing_fqname(fqname: &str) -> bool {

crates/store/re_chunk/src/builder.rs

@@ -7,7 +7,7 @@ use itertools::Itertools;
 use nohash_hasher::IntMap;
 
 use re_log_types::{EntityPath, TimeInt, TimePoint, Timeline};
-use re_types_core::{AsComponents, ComponentBatch, ComponentDescriptor};
+use re_types_core::{AsComponents, ComponentBatch, ComponentDescriptor, SerializedComponentBatch};
 
 use crate::{arrow2_util, chunk::ChunkComponents, Chunk, ChunkId, ChunkResult, RowId, TimeColumn};
 
@@ -125,14 +125,8 @@ impl ChunkBuilder {
         timepoint: impl Into<TimePoint>,
         as_components: &dyn AsComponents,
     ) -> Self {
-        let batches = as_components.as_component_batches();
-        self.with_component_batches(
-            row_id,
-            timepoint,
-            batches
-                .iter()
-                .map(|batch| batch as &dyn re_types_core::ComponentBatch),
-        )
+        let batches = as_components.as_component_batches_v2();
+        self.with_component_batches_v2(row_id, timepoint, batches)
     }
 
     /// Add a row's worth of data by serializing a single [`ComponentBatch`].
@@ -197,6 +191,62 @@ impl ChunkBuilder {
         )
     }
 
+    /// Add a row's worth of data by serializing a single [`ComponentBatch`].
+    #[inline]
+    pub fn with_component_batch_v2(
+        self,
+        row_id: RowId,
+        timepoint: impl Into<TimePoint>,
+        component_batch: SerializedComponentBatch,
+    ) -> Self {
+        self.with_row(
+            row_id,
+            timepoint,
+            [(component_batch.descriptor, component_batch.array)],
+        )
+    }
+
+    /// Add a row's worth of data by serializing many [`ComponentBatch`]es.
+    #[inline]
+    pub fn with_component_batches_v2(
+        self,
+        row_id: RowId,
+        timepoint: impl Into<TimePoint>,
+        component_batches: impl IntoIterator<Item = SerializedComponentBatch>,
+    ) -> Self {
+        self.with_row(
+            row_id,
+            timepoint,
+            component_batches
+                .into_iter()
+                .map(|component_batch| (component_batch.descriptor, component_batch.array)),
+        )
+    }
+
+    /// Add a row's worth of data by serializing many sparse [`ComponentBatch`]es.
+    #[inline]
+    pub fn with_sparse_component_batches_v2(
+        self,
+        row_id: RowId,
+        timepoint: impl Into<TimePoint>,
+        component_batches: impl IntoIterator<
+            Item = (ComponentDescriptor, Option<SerializedComponentBatch>),
+        >,
+    ) -> Self {
+        self.with_sparse_row(
+            row_id,
+            timepoint,
+            component_batches
+                .into_iter()
+                .map(|(component_desc, component_batch)| {
+                    (
+                        component_desc,
+                        component_batch.map(|batch| batch.array.into()),
+                    )
+                }),
+        )
+    }
+
     /// Builds and returns the final [`Chunk`].
     ///
     /// The arrow datatype of each individual column will be guessed by inspecting the data.

crates/store/re_entity_db/tests/clear.rs

@@ -122,14 +122,7 @@ fn clears() -> anyhow::Result<()> {
         let timepoint = TimePoint::from_iter([(timeline_frame, 10)]);
         let clear = Clear::flat();
         let chunk = Chunk::builder(entity_path_parent.clone())
-            .with_component_batches(
-                row_id,
-                timepoint,
-                clear
-                    .as_component_batches()
-                    .iter()
-                    .map(|b| b as &dyn re_types_core::ComponentBatch),
-            )
+            .with_component_batches_v2(row_id, timepoint, clear.as_component_batches_v2())
             .build()?;
 
         db.add_chunk(&Arc::new(chunk))?;
@@ -163,14 +156,7 @@ fn clears() -> anyhow::Result<()> {
         let timepoint = TimePoint::from_iter([(timeline_frame, 10)]);
         let clear = Clear::recursive();
         let chunk = Chunk::builder(entity_path_parent.clone())
-            .with_component_batches(
-                row_id,
-                timepoint,
-                clear
-                    .as_component_batches()
-                    .iter()
-                    .map(|b| b as &dyn re_types_core::ComponentBatch),
-            )
+            .with_component_batches_v2(row_id, timepoint, clear.as_component_batches_v2())
             .build()?;
 
         db.add_chunk(&Arc::new(chunk))?;
@@ -351,13 +337,10 @@ fn clears_respect_index_order() -> anyhow::Result<()> {
 
     let clear = Clear::recursive();
     let chunk = Chunk::builder(entity_path.clone())
-        .with_component_batches(
+        .with_component_batches_v2(
             row_id1, // older row id!
             timepoint.clone(),
-            clear
-                .as_component_batches()
-                .iter()
-                .map(|b| b as &dyn re_types_core::ComponentBatch),
+            clear.as_component_batches_v2(),
         )
         .build()?;
 
@@ -378,13 +361,10 @@ fn clears_respect_index_order() -> anyhow::Result<()> {
 
     let clear = Clear::recursive();
     let chunk = Chunk::builder(entity_path.clone())
-        .with_component_batches(
+        .with_component_batches_v2(
             row_id3, // newer row id!
             timepoint.clone(),
-            clear
-                .as_component_batches()
-                .iter()
-                .map(|b| b as &dyn re_types_core::ComponentBatch),
+            clear.as_component_batches_v2(),
         )
         .build()?;
 

crates/store/re_types/definitions/attributes/rust.fbs

@@ -41,3 +41,8 @@ attribute "attr.rust.new_pub_crate";
 /// an object of kind `Blueprint` with `attr.rust.override_crate=re_viewport`, the final
 /// object will be generated in `re_viewport/src/blueprint`.
 attribute "attr.rust.override_crate";
+
+/// The generated Rust object should be eagerly serialized, i.e. only comprised of Arrow arrays.
+///
+/// Applies only to archetypes. No-op otherwise.
+attribute "attr.rust.archetype_eager";

crates/store/re_types/definitions/rerun/archetypes/points3d.fbs

@@ -9,6 +9,7 @@ namespace rerun.archetypes;
 /// \example archetypes/points3d_ui_radius title="Log points with radii given in UI points" image="https://static.rerun.io/point3d_ui_radius/e051a65b4317438bcaea8d0eee016ac9460b5336/1200w.png"
 /// \example archetypes/points3d_send_columns title="Send several point clouds with varying point count over time in a single call" image="https://static.rerun.io/points3d_send_columns/633b524a2ee439b0e3afc3f894f4927ce938a3ec/1200w.png" missing="rs"
 table Points3D (
+  "attr.rust.archetype_eager": "",
   "attr.rust.derive": "PartialEq",
   "attr.docs.category": "Spatial 3D",
   "attr.docs.view_types": "Spatial3DView, Spatial2DView: if logged above active projection"