From 9c2bab1aa92fc2b71ffec7fd3a0ac0d17c3dac0c Mon Sep 17 00:00:00 2001 From: Antoine Beyeler Date: Tue, 4 Jul 2023 18:48:23 +0200 Subject: [PATCH 01/17] Improved Python code generator Co-authored-by: Clement Rey --- .../definitions/rerun/components/color.fbs | 4 +- .../definitions/rerun/components/label.fbs | 1 + .../definitions/rerun/components/point2d.fbs | 23 +- .../re_types/definitions/rerun/datatypes.fbs | 2 + .../definitions/rerun/datatypes/point2d.fbs | 19 + .../definitions/rerun/datatypes/vec2d.fbs | 3 + .../rerun/testing/archetypes/fuzzy.fbs | 8 +- .../rerun/testing/components/fuzzy.fbs | 28 +- crates/re_types/tests/fuzzy.rs | 41 - crates/re_types_builder/src/codegen/python.rs | 791 +++++++++++------- crates/re_types_builder/src/lib.rs | 1 - crates/re_types_builder/src/objects.rs | 20 +- rerun_py/rerun_sdk/rerun/__init__.py | 5 + .../rerun_sdk/rerun/_rerun2/_baseclasses.py | 118 ++- .../rerun_sdk/rerun/_rerun2/_converters.py | 101 +++ .../_rerun2/archetypes/_overrides/__init__.py | 1 + .../_rerun2/components/_overrides/__init__.py | 9 + .../_rerun2/components/_overrides/class_id.py | 14 + .../_rerun2/components/_overrides/color.py | 75 ++ .../components/_overrides/draw_order.py | 14 + .../components/_overrides/instance_key.py | 14 + .../components/_overrides/keypoint_id.py | 14 + .../_rerun2/components/_overrides/label.py | 19 + .../_rerun2/components/_overrides/radius.py | 14 + .../rerun/_rerun2/components/class_id_ext.py | 21 - .../rerun/_rerun2/components/color_ext.py | 43 - .../_rerun2/components/draw_order_ext.py | 21 - .../rerun/_rerun2/components/fuzzy_ext.py | 71 -- .../_rerun2/components/instance_key_ext.py | 33 - .../_rerun2/components/keypoint_id_ext.py | 21 - .../rerun/_rerun2/components/label_ext.py | 22 - .../rerun/_rerun2/components/point2d_ext.py | 28 - .../rerun/_rerun2/components/radius_ext.py | 21 - .../_rerun2/datatypes/_overrides/__init__.py | 4 + .../_rerun2/datatypes/_overrides/point2d.py | 23 + .../_rerun2/datatypes/_overrides/vec2d.py | 14 + .../rerun/_rerun2/datatypes/fuzzy_ext.py | 26 - .../rerun/_rerun2/datatypes/vec2d_ext.py | 23 - rerun_py/rerun_sdk/rerun/_rerun2/log_any.py | 32 +- rerun_py/tests/unit/test_points2d.py | 69 +- 40 files changed, 1039 insertions(+), 772 deletions(-) create mode 100644 crates/re_types/definitions/rerun/datatypes/point2d.fbs create mode 100644 rerun_py/rerun_sdk/rerun/_rerun2/_converters.py create mode 100644 rerun_py/rerun_sdk/rerun/_rerun2/archetypes/_overrides/__init__.py create mode 100644 rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/__init__.py create mode 100644 rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/class_id.py create mode 100644 rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/color.py create mode 100644 rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/draw_order.py create mode 100644 rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/instance_key.py create mode 100644 rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/keypoint_id.py create mode 100644 rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/label.py create mode 100644 rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/radius.py delete mode 100644 rerun_py/rerun_sdk/rerun/_rerun2/components/class_id_ext.py delete mode 100644 rerun_py/rerun_sdk/rerun/_rerun2/components/color_ext.py delete mode 100644 rerun_py/rerun_sdk/rerun/_rerun2/components/draw_order_ext.py delete mode 100644 rerun_py/rerun_sdk/rerun/_rerun2/components/fuzzy_ext.py delete mode 100644 rerun_py/rerun_sdk/rerun/_rerun2/components/instance_key_ext.py delete mode 100644 rerun_py/rerun_sdk/rerun/_rerun2/components/keypoint_id_ext.py delete mode 100644 rerun_py/rerun_sdk/rerun/_rerun2/components/label_ext.py delete mode 100644 rerun_py/rerun_sdk/rerun/_rerun2/components/point2d_ext.py delete mode 100644 rerun_py/rerun_sdk/rerun/_rerun2/components/radius_ext.py create mode 100644 rerun_py/rerun_sdk/rerun/_rerun2/datatypes/_overrides/__init__.py create mode 100644 rerun_py/rerun_sdk/rerun/_rerun2/datatypes/_overrides/point2d.py create mode 100644 rerun_py/rerun_sdk/rerun/_rerun2/datatypes/_overrides/vec2d.py delete mode 100644 rerun_py/rerun_sdk/rerun/_rerun2/datatypes/fuzzy_ext.py delete mode 100644 rerun_py/rerun_sdk/rerun/_rerun2/datatypes/vec2d_ext.py diff --git a/crates/re_types/definitions/rerun/components/color.fbs b/crates/re_types/definitions/rerun/components/color.fbs index 3265ad53ee6d..a610bd8991fb 100644 --- a/crates/re_types/definitions/rerun/components/color.fbs +++ b/crates/re_types/definitions/rerun/components/color.fbs @@ -16,8 +16,8 @@ namespace rerun.components; /// \py If there is an alpha, we assume it is in linear space, and separate (NOT pre-multiplied). struct Color ( "attr.arrow.transparent", - "attr.python.aliases": "int, npt.NDArray[np.uint8], npt.NDArray[np.uint32], npt.NDArray[np.float32], npt.NDArray[np.float64]", - "attr.python.array_aliases": "Sequence[int], npt.NDArray[np.uint8], npt.NDArray[np.uint32], npt.NDArray[np.float32], npt.NDArray[np.float64]", + "attr.python.aliases": "int, Sequence[int], npt.NDArray[np.uint8 | np.float32 | np.float64]", + "attr.python.array_aliases": "Sequence[Sequence[int]], npt.NDArray[np.uint8 | np.uint32 | np.float32 | np.float64]", "attr.rerun.legacy_fqname": "rerun.colorrgba", "attr.rust.derive": "Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, bytemuck::Pod, bytemuck::Zeroable", "attr.rust.repr": "transparent", diff --git a/crates/re_types/definitions/rerun/components/label.fbs b/crates/re_types/definitions/rerun/components/label.fbs index 38c1c2a10952..af35f8b9402b 100644 --- a/crates/re_types/definitions/rerun/components/label.fbs +++ b/crates/re_types/definitions/rerun/components/label.fbs @@ -13,6 +13,7 @@ namespace rerun.components; table Label ( "attr.arrow.transparent", "attr.python.aliases": "str", + "attr.python.array_aliases": "Sequence[str]", "attr.rerun.legacy_fqname": "rerun.label", "attr.rust.derive": "Debug, Clone, PartialEq, Eq, PartialOrd, Ord", "attr.rust.repr": "transparent", diff --git a/crates/re_types/definitions/rerun/components/point2d.fbs b/crates/re_types/definitions/rerun/components/point2d.fbs index 62fd9406bf5a..09d00d3dd757 100644 --- a/crates/re_types/definitions/rerun/components/point2d.fbs +++ b/crates/re_types/definitions/rerun/components/point2d.fbs @@ -10,6 +10,7 @@ namespace rerun.components; // --- /// A point in 2D space. +// TODO(cmc): bring back attr.rust.tuple_struct struct Point2D ( "attr.python.aliases": "npt.NDArray[np.float32], Sequence[float], Tuple[float, float]", "attr.python.array_aliases": "npt.NDArray[np.float32], Sequence[float]", @@ -17,25 +18,5 @@ struct Point2D ( "attr.rust.derive": "Debug, Default, Clone, Copy, PartialEq, PartialOrd", order: 100 ) { - x: float (order: 100); - y: float (order: 200); + xy: rerun.datatypes.Point2D (order: 100); } - -// --- - -// TODO(cmc): use the following definition instead once we've finalized the switch to HOPE. - -// /// A point in 2D space. -// struct Point2D ( -// "attr.arrow.transparent", -// "attr.python.aliases": "npt.NDArray[np.float32], Sequence[float], Tuple[float, float]", -// "attr.python.array_aliases": "npt.NDArray[np.float32], Sequence[float]", -// "attr.rust.tuple_struct", -// "attr.rust.derive": "Debug, Default, Clone, Copy, PartialEq, PartialOrd", -// order: 100 -// ) { -// position: rerun.datatypes.Vec2D ( -// "attr.python.transparent", -// order: 100 -// ); -// } diff --git a/crates/re_types/definitions/rerun/datatypes.fbs b/crates/re_types/definitions/rerun/datatypes.fbs index 3048df042e85..4b9ef8379b01 100644 --- a/crates/re_types/definitions/rerun/datatypes.fbs +++ b/crates/re_types/definitions/rerun/datatypes.fbs @@ -1,3 +1,5 @@ +include "./datatypes/point2d.fbs"; include "./datatypes/vec2d.fbs"; + namespace rerun.datatypes; diff --git a/crates/re_types/definitions/rerun/datatypes/point2d.fbs b/crates/re_types/definitions/rerun/datatypes/point2d.fbs new file mode 100644 index 000000000000..a635023f4f9d --- /dev/null +++ b/crates/re_types/definitions/rerun/datatypes/point2d.fbs @@ -0,0 +1,19 @@ +include "arrow/attributes.fbs"; +include "python/attributes.fbs"; +include "fbs/attributes.fbs"; +include "rust/attributes.fbs"; + +namespace rerun.datatypes; + +// --- + +/// A point in 2D space. +struct Point2D ( + "attr.python.aliases": "Sequence[float]", + "attr.python.array_aliases": "npt.NDArray[np.float32], Sequence[npt.NDArray[np.float32]], Sequence[tuple[float, float]], Sequence[float]", + "attr.rust.derive": "Debug, Default, Clone, Copy, PartialEq, PartialOrd", + order: 100 +) { + x: float (order: 100); + y: float (order: 200); +} diff --git a/crates/re_types/definitions/rerun/datatypes/vec2d.fbs b/crates/re_types/definitions/rerun/datatypes/vec2d.fbs index 9fc31cbb3f18..c423871afa81 100644 --- a/crates/re_types/definitions/rerun/datatypes/vec2d.fbs +++ b/crates/re_types/definitions/rerun/datatypes/vec2d.fbs @@ -1,4 +1,5 @@ include "arrow/attributes.fbs"; +include "python/attributes.fbs"; include "fbs/attributes.fbs"; include "rust/attributes.fbs"; @@ -9,6 +10,8 @@ namespace rerun.datatypes; /// A vector in 2D space. struct Vec2D ( "attr.arrow.transparent", + "attr.python.aliases": "tuple[float, float]", + "attr.python.array_aliases": "npt.NDArray[np.float32], Sequence[tuple[float, float]], Sequence[float]", "attr.rust.derive": "Debug, Default, Clone, Copy, PartialEq, PartialOrd", "attr.rust.tuple_struct", order: 100 diff --git a/crates/re_types/definitions/rerun/testing/archetypes/fuzzy.fbs b/crates/re_types/definitions/rerun/testing/archetypes/fuzzy.fbs index 2484542f52c2..9e91be6e71ee 100644 --- a/crates/re_types/definitions/rerun/testing/archetypes/fuzzy.fbs +++ b/crates/re_types/definitions/rerun/testing/archetypes/fuzzy.fbs @@ -21,7 +21,7 @@ table AffixFuzzer1 ( fuzz1004: rerun.testing.components.AffixFuzzer4 ("attr.rerun.component_required", required, order: 1004); fuzz1005: rerun.testing.components.AffixFuzzer5 ("attr.rerun.component_required", required, order: 1005); fuzz1006: rerun.testing.components.AffixFuzzer6 ("attr.rerun.component_required", required, order: 1006); - fuzz1007: rerun.testing.components.AffixFuzzer7 ("attr.rerun.component_required", required, order: 1007); + // fuzz1007: rerun.testing.components.AffixFuzzer7 ("attr.rerun.component_required", required, order: 1007); fuzz1101: [rerun.testing.components.AffixFuzzer1] ("attr.rerun.component_required", required, order: 1101); fuzz1102: [rerun.testing.components.AffixFuzzer2] ("attr.rerun.component_required", required, order: 1102); @@ -29,7 +29,7 @@ table AffixFuzzer1 ( fuzz1104: [rerun.testing.components.AffixFuzzer4] ("attr.rerun.component_required", required, order: 1104); fuzz1105: [rerun.testing.components.AffixFuzzer5] ("attr.rerun.component_required", required, order: 1105); fuzz1106: [rerun.testing.components.AffixFuzzer6] ("attr.rerun.component_required", required, order: 1106); - fuzz1107: [rerun.testing.components.AffixFuzzer7] ("attr.rerun.component_required", required, order: 1107); + // fuzz1107: [rerun.testing.components.AffixFuzzer7] ("attr.rerun.component_required", required, order: 1107); fuzz2001: rerun.testing.components.AffixFuzzer1 ("attr.rerun.component_optional", order: 2001); fuzz2002: rerun.testing.components.AffixFuzzer2 ("attr.rerun.component_optional", order: 2002); @@ -37,7 +37,7 @@ table AffixFuzzer1 ( fuzz2004: rerun.testing.components.AffixFuzzer4 ("attr.rerun.component_optional", order: 2004); fuzz2005: rerun.testing.components.AffixFuzzer5 ("attr.rerun.component_optional", order: 2005); fuzz2006: rerun.testing.components.AffixFuzzer6 ("attr.rerun.component_optional", order: 2006); - fuzz2007: rerun.testing.components.AffixFuzzer7 ("attr.rerun.component_optional", order: 2007); + // fuzz2007: rerun.testing.components.AffixFuzzer7 ("attr.rerun.component_optional", order: 2007); fuzz2101: [rerun.testing.components.AffixFuzzer1] ("attr.rerun.component_optional", order: 2101); fuzz2102: [rerun.testing.components.AffixFuzzer2] ("attr.rerun.component_optional", order: 2102); @@ -45,6 +45,6 @@ table AffixFuzzer1 ( fuzz2104: [rerun.testing.components.AffixFuzzer4] ("attr.rerun.component_optional", order: 2104); fuzz2105: [rerun.testing.components.AffixFuzzer5] ("attr.rerun.component_optional", order: 2105); fuzz2106: [rerun.testing.components.AffixFuzzer6] ("attr.rerun.component_optional", order: 2106); - fuzz2107: [rerun.testing.components.AffixFuzzer7] ("attr.rerun.component_optional", order: 2107); + // fuzz2107: [rerun.testing.components.AffixFuzzer7] ("attr.rerun.component_optional", order: 2107); } diff --git a/crates/re_types/definitions/rerun/testing/components/fuzzy.fbs b/crates/re_types/definitions/rerun/testing/components/fuzzy.fbs index e8610f9c4a11..82c1796bf9df 100644 --- a/crates/re_types/definitions/rerun/testing/components/fuzzy.fbs +++ b/crates/re_types/definitions/rerun/testing/components/fuzzy.fbs @@ -56,17 +56,17 @@ table AffixFuzzer6 ( single_optional: rerun.testing.datatypes.AffixFuzzer1; } -table AffixFuzzer7 ( - "attr.rust.derive": "Debug, Clone, PartialEq", - order: 700 -) { - many_optional: [rerun.testing.datatypes.AffixFuzzer1] (order: 100); - single_float_optional: float (order: 101); - single_string_required: string (order: 102, required); - single_string_optional: string (order: 103); - many_floats_optional: [float] (order: 104); - many_strings_required: [string] (order: 105, required); - many_strings_optional: [string] (order: 106); - // TODO(cmc): the ugly bug we need to take care of at some point - // many_transparent_optionals: rerun.testing.datatypes.AffixFuzzer2 (order: 107); -} +// table AffixFuzzer7 ( +// "attr.rust.derive": "Debug, Clone, PartialEq", +// order: 700 +// ) { +// many_optional: [rerun.testing.datatypes.AffixFuzzer1] (order: 100); +// single_float_optional: float (order: 101); +// single_string_required: string (order: 102, required); +// single_string_optional: string (order: 103); +// many_floats_optional: [float] (order: 104); +// many_strings_required: [string] (order: 105, required); +// many_strings_optional: [string] (order: 106); +// // TODO(cmc): the ugly bug we need to take care of at some point +// // many_transparent_optionals: rerun.testing.datatypes.AffixFuzzer2 (order: 107); +// } diff --git a/crates/re_types/tests/fuzzy.rs b/crates/re_types/tests/fuzzy.rs index 419e6db1c7dc..3b6c764e84f1 100644 --- a/crates/re_types/tests/fuzzy.rs +++ b/crates/re_types/tests/fuzzy.rs @@ -66,40 +66,6 @@ fn roundtrip() { // // many_strings_optional: None, // }), // }; - let fuzzy7_1 = re_types::components::AffixFuzzer7 { - many_optional: None, - single_float_optional: None, - single_string_required: "b".into(), - single_string_optional: None, - many_floats_optional: Some(vec![1.0, 10.0]), - many_strings_required: vec!["1".into(), "10".into()], - many_strings_optional: None, - // TODO(cmc): this one is bugged. - // many_transparent_optionals: vec![], - }; - let fuzzy7_2 = re_types::components::AffixFuzzer7 { - many_optional: Some(vec![re_types::datatypes::AffixFuzzer1 { - single_float_optional: None, - single_string_required: "d".into(), - single_string_optional: None, - many_floats_optional: Some(vec![4.0, 40.0, 400.0]), - many_strings_required: vec!["7".into(), "8".into()], - many_strings_optional: None, - }]), - single_float_optional: Some(1.0), - single_string_required: "a".into(), - single_string_optional: Some("a".into()), - many_floats_optional: Some(vec![2.0, 20.0, 200.0]), - many_strings_required: vec!["20".into(), "200".into(), "2000".into()], - many_strings_optional: Some(vec![ - "30".into(), - "300".into(), - "3000".into(), - "30000".into(), - ]), - // TODO(cmc): this one is bugged. - // many_transparent_optionals: vec![], - }; let arch = AffixFuzzer1::new( fuzzy1.clone(), @@ -108,19 +74,16 @@ fn roundtrip() { fuzzy4.clone(), fuzzy5.clone(), fuzzy6.clone(), - fuzzy7_1.clone(), [fuzzy1.clone(), fuzzy1.clone(), fuzzy1.clone()], [fuzzy2.clone(), fuzzy2.clone(), fuzzy2.clone()], [fuzzy3.clone(), fuzzy3.clone(), fuzzy3.clone()], [fuzzy4.clone(), fuzzy4.clone(), fuzzy4.clone()], [fuzzy5.clone(), fuzzy5.clone(), fuzzy5.clone()], [fuzzy6.clone(), fuzzy6.clone(), fuzzy6.clone()], - [fuzzy7_1.clone(), fuzzy7_2.clone()], ) .with_fuzz2001(fuzzy1.clone()) .with_fuzz2003(fuzzy3.clone()) .with_fuzz2005(fuzzy5.clone()) - .with_fuzz2007(fuzzy7_2.clone()) .with_fuzz2102([fuzzy2.clone(), fuzzy2.clone(), fuzzy2.clone()]) .with_fuzz2104([fuzzy4.clone(), fuzzy4.clone(), fuzzy4.clone()]) .with_fuzz2106([fuzzy6.clone(), fuzzy6.clone(), fuzzy6.clone()]); @@ -133,7 +96,6 @@ fn roundtrip() { ("fuzz1004", vec!["rerun.testing.components.AffixFuzzer4", "rerun.testing.datatypes.AffixFuzzer1"]), ("fuzz1005", vec!["rerun.testing.components.AffixFuzzer5", "rerun.testing.datatypes.AffixFuzzer1"]), ("fuzz1006", vec!["rerun.testing.components.AffixFuzzer6", "rerun.testing.datatypes.AffixFuzzer1"]), - ("fuzz1007", vec!["rerun.testing.components.AffixFuzzer7", "rerun.testing.datatypes.AffixFuzzer1"]), ("fuzz1101", vec!["rerun.testing.components.AffixFuzzer1", "rerun.testing.datatypes.AffixFuzzer1"]), ("fuzz1102", vec!["rerun.testing.components.AffixFuzzer2", "rerun.testing.datatypes.AffixFuzzer1"]), @@ -141,7 +103,6 @@ fn roundtrip() { ("fuzz1104", vec!["rerun.testing.components.AffixFuzzer4", "rerun.testing.datatypes.AffixFuzzer1"]), ("fuzz1105", vec!["rerun.testing.components.AffixFuzzer5", "rerun.testing.datatypes.AffixFuzzer1"]), ("fuzz1106", vec!["rerun.testing.components.AffixFuzzer6", "rerun.testing.datatypes.AffixFuzzer1"]), - ("fuzz1107", vec!["rerun.testing.components.AffixFuzzer7", "rerun.testing.datatypes.AffixFuzzer1"]), ("fuzz2001", vec!["rerun.testing.components.AffixFuzzer1", "rerun.testing.datatypes.AffixFuzzer1"]), ("fuzz2002", vec!["rerun.testing.components.AffixFuzzer2", "rerun.testing.datatypes.AffixFuzzer1"]), @@ -149,7 +110,6 @@ fn roundtrip() { ("fuzz2004", vec!["rerun.testing.components.AffixFuzzer4", "rerun.testing.datatypes.AffixFuzzer1"]), ("fuzz2005", vec!["rerun.testing.components.AffixFuzzer5", "rerun.testing.datatypes.AffixFuzzer1"]), ("fuzz2006", vec!["rerun.testing.components.AffixFuzzer6", "rerun.testing.datatypes.AffixFuzzer1"]), - ("fuzz2007", vec!["rerun.testing.components.AffixFuzzer7", "rerun.testing.datatypes.AffixFuzzer1"]), ("fuzz2101", vec!["rerun.testing.components.AffixFuzzer1", "rerun.testing.datatypes.AffixFuzzer1"]), ("fuzz2102", vec!["rerun.testing.components.AffixFuzzer2", "rerun.testing.datatypes.AffixFuzzer1"]), @@ -157,7 +117,6 @@ fn roundtrip() { ("fuzz2104", vec!["rerun.testing.components.AffixFuzzer4", "rerun.testing.datatypes.AffixFuzzer1"]), ("fuzz2105", vec!["rerun.testing.components.AffixFuzzer5", "rerun.testing.datatypes.AffixFuzzer1"]), ("fuzz2106", vec!["rerun.testing.components.AffixFuzzer6", "rerun.testing.datatypes.AffixFuzzer1"]), - ("fuzz2107", vec!["rerun.testing.components.AffixFuzzer7", "rerun.testing.datatypes.AffixFuzzer1"]), ] .into(); diff --git a/crates/re_types_builder/src/codegen/python.rs b/crates/re_types_builder/src/codegen/python.rs index 4cca28137f1f..254cc4304e09 100644 --- a/crates/re_types_builder/src/codegen/python.rs +++ b/crates/re_types_builder/src/codegen/python.rs @@ -16,6 +16,52 @@ use crate::{ // --- +/// Python-specific helpers for [`Object`]. +trait PythonObjectExt { + /// Returns `true` if the object is a delegating component. + /// + /// Components can either use a native type, or a custom datatype. In the latter case, the + /// the component delegates its implementation to the datatype. + fn is_delegating_component(&self) -> bool; + + /// Returns `true` if the object is a non-delegating component. + fn is_non_delegating_component(&self) -> bool; + + /// If the object is a delegating component, returns the datatype it delegates to. + fn delegate_datatype<'a>(&self, objects: &'a Objects) -> Option<&'a Object>; +} + +impl PythonObjectExt for Object { + fn is_delegating_component(&self) -> bool { + if self.kind != ObjectKind::Component { + return false; + } + + if let Type::Object(_) = self.fields[0].typ { + return true; + } + + false + } + + fn is_non_delegating_component(&self) -> bool { + self.kind == ObjectKind::Component && !self.is_delegating_component() + } + + fn delegate_datatype<'a>(&self, objects: &'a Objects) -> Option<&'a Object> { + if !self.is_delegating_component() { + return None; + } + + let field = &self.fields[0]; + if let Type::Object(name) = &field.typ { + return Some(objects.get(name)); + } + + unreachable!() + } +} + pub struct PythonCodeGenerator { pkg_path: PathBuf, } @@ -28,11 +74,28 @@ impl PythonCodeGenerator { } } +/// Inspect `_overrides` sub-packages for manual override of the generated code. +/// +/// This is the hacky way. We extract all identifiers from `__init__.py` which contains, but don't +/// start with, a underscore (`_`). +fn load_overrides(path_buf: &Path) -> HashSet { + let contents = std::fs::read_to_string(path_buf.join("_overrides").join("__init__.py")) + .expect("_overrides/__init__.py must exist"); + + // extract words from contents + contents + .split_whitespace() + .filter(|word| !word.starts_with('_') && !word.starts_with('.') && word.contains('_')) + .map(|word| word.trim_end_matches(',').to_owned()) + .collect() +} + impl CodeGenerator for PythonCodeGenerator { fn generate(&mut self, objs: &Objects, arrow_registry: &ArrowRegistry) -> Vec { let mut filepaths = Vec::new(); let datatypes_path = self.pkg_path.join("datatypes"); + let datatype_overrides = load_overrides(&datatypes_path); std::fs::create_dir_all(&datatypes_path) .with_context(|| format!("{datatypes_path:?}")) .unwrap(); @@ -40,6 +103,7 @@ impl CodeGenerator for PythonCodeGenerator { quote_objects( datatypes_path, arrow_registry, + &datatype_overrides, objs, ObjectKind::Datatype, &objs.ordered_objects(ObjectKind::Datatype.into()), @@ -48,6 +112,7 @@ impl CodeGenerator for PythonCodeGenerator { ); let components_path = self.pkg_path.join("components"); + let component_overrides = load_overrides(&components_path); std::fs::create_dir_all(&components_path) .with_context(|| format!("{components_path:?}")) .unwrap(); @@ -55,6 +120,7 @@ impl CodeGenerator for PythonCodeGenerator { quote_objects( components_path, arrow_registry, + &component_overrides, objs, ObjectKind::Component, &objs.ordered_objects(ObjectKind::Component.into()), @@ -63,12 +129,14 @@ impl CodeGenerator for PythonCodeGenerator { ); let archetypes_path = self.pkg_path.join("archetypes"); + let archetype_overrides = load_overrides(&archetypes_path); std::fs::create_dir_all(&archetypes_path) .with_context(|| format!("{archetypes_path:?}")) .unwrap(); let (paths, archetype_names) = quote_objects( archetypes_path, arrow_registry, + &archetype_overrides, objs, ObjectKind::Archetype, &objs.ordered_objects(ObjectKind::Archetype.into()), @@ -119,8 +187,9 @@ fn quote_lib(out_path: impl AsRef, archetype_names: &[String]) -> PathBuf fn quote_objects( out_path: impl AsRef, arrow_registry: &ArrowRegistry, + overrides: &HashSet, all_objects: &Objects, - kind: ObjectKind, + _kind: ObjectKind, objs: &[&Object], ) -> (Vec, Vec) { let out_path = out_path.as_ref(); @@ -133,9 +202,9 @@ fn quote_objects( all_names.push(obj.name.clone()); let obj = if obj.is_struct() { - QuotedObject::from_struct(arrow_registry, all_objects, obj) + QuotedObject::from_struct(arrow_registry, overrides, all_objects, obj) } else { - QuotedObject::from_union(arrow_registry, all_objects, obj) + QuotedObject::from_union(arrow_registry, overrides, all_objects, obj) }; let filepath = out_path.join(obj.filepath.file_name().unwrap()); @@ -153,13 +222,17 @@ fn quote_objects( ObjectKind::Datatype | ObjectKind::Component => { let name = &obj.object.name; - vec![ - format!("{name}"), - format!("{name}Like"), - format!("{name}Array"), - format!("{name}ArrayLike"), - format!("{name}Type"), - ] + if obj.object.is_delegating_component() { + vec![format!("{name}Array"), format!("{name}Type")] + } else { + vec![ + format!("{name}"), + format!("{name}Like"), + format!("{name}Array"), + format!("{name}ArrayLike"), + format!("{name}Type"), + ] + } } ObjectKind::Archetype => vec![obj.object.name.clone()], }) @@ -180,32 +253,69 @@ fn quote_objects( code.push_text(&format!("# {AUTOGEN_WARNING}"), 2, 0); let manifest = quote_manifest(names); - let base_include = match kind { - ObjectKind::Archetype => "from .._baseclasses import Archetype", - ObjectKind::Component => "from .._baseclasses import Component", - ObjectKind::Datatype => "", - }; - code.push_unindented_text( - format!( - " - from __future__ import annotations - import numpy as np - import numpy.typing as npt - import pyarrow as pa - - from dataclasses import dataclass, field - from typing import Any, Dict, Iterable, Optional, Sequence, Set, Tuple, Union - - {base_include} - - __all__ = [{manifest}] - - ", - ), + code.push_unindented_text( + " + from __future__ import annotations + + import numpy as np + import numpy.typing as npt + import pyarrow as pa + + from attrs import define, field + from typing import Any, Dict, Iterable, Optional, Sequence, Set, Tuple, Union + + from .._baseclasses import ( + Archetype, + BaseExtensionType, + BaseExtensionArray, + BaseDelegatingExtensionType, + BaseDelegatingExtensionArray + ) + from .._converters import ( + to_np_uint8, + to_np_uint16, + to_np_uint32, + to_np_uint64, + to_np_int8, + to_np_int16, + to_np_int32, + to_np_int64, + to_np_bool, + to_np_float16, + to_np_float32, + to_np_float64 + ) + ", 0, ); + // import all overrides + let override_names: Vec<_> = objs + .iter() + .flat_map(|obj| { + let name = obj.object.name.as_str().to_lowercase(); + overrides + .iter() + .filter(|o| o.starts_with(name.as_str())) + .map(|o| o.as_str()) + .collect::>() + }) + .collect(); + + // TODO(ab): remove this noqa—useful for checking what overrides are extracted + if !override_names.is_empty() { + code.push_unindented_text( + format!( + " + from ._overrides import {} # noqa: F401 + ", + override_names.join(", ") + ), + 0, + ); + } + let import_clauses: HashSet<_> = objs .iter() .flat_map(|obj| obj.object.fields.iter()) @@ -215,6 +325,16 @@ fn quote_objects( code.push_text(&clause, 1, 0); } + code.push_unindented_text( + format!( + " + __all__ = [{manifest}] + + ", + ), + 0, + ); + for obj in objs { code.push_text(&obj.code, 1, 0); } @@ -231,23 +351,12 @@ fn quote_objects( let manifest = quote_manifest(mods.iter().flat_map(|(_, names)| names.iter())); - let (base_manifest, base_include) = match kind { - ObjectKind::Archetype => ("\"Archetype\", ", "from .._baseclasses import Archetype\n"), - ObjectKind::Component => ("\"Component\", ", "from .._baseclasses import Component\n"), - ObjectKind::Datatype => ("", ""), - }; - code.push_text(&format!("# {AUTOGEN_WARNING}"), 2, 0); code.push_unindented_text( - format!( - " - from __future__ import annotations - - __all__ = [{base_manifest}{manifest}] + " + from __future__ import annotations - {base_include} - ", - ), + ", 0, ); @@ -256,6 +365,8 @@ fn quote_objects( code.push_text(&format!("from .{module} import {names}"), 1, 0); } + code.push_unindented_text(format!("\n__all__ = [{manifest}]"), 0); + filepaths.push(path.clone()); std::fs::write(&path, code) .with_context(|| format!("{path:?}")) @@ -275,7 +386,12 @@ struct QuotedObject { } impl QuotedObject { - fn from_struct(arrow_registry: &ArrowRegistry, objects: &Objects, obj: &Object) -> Self { + fn from_struct( + arrow_registry: &ArrowRegistry, + overrides: &HashSet, + objects: &Objects, + obj: &Object, + ) -> Self { assert!(obj.is_struct()); let Object { @@ -293,82 +409,142 @@ impl QuotedObject { let mut code = String::new(); - let superclass = match *kind { - ObjectKind::Archetype => "(Archetype)", - ObjectKind::Component | ObjectKind::Datatype => "", - }; - code.push_unindented_text( - format!( - r#" + if *kind != ObjectKind::Component || obj.is_non_delegating_component() { + let superclass = match *kind { + ObjectKind::Archetype => "(Archetype)", + ObjectKind::Component | ObjectKind::Datatype => "", + }; - ## --- {name} --- ## + let define_args = if *kind == ObjectKind::Archetype { + "(str=False, repr=False)" + } else { + "" + }; + + code.push_unindented_text( + format!( + r#" - @dataclass + @define{define_args} class {name}{superclass}: "# - ), - 0, - ); + ), + 0, + ); - code.push_text(quote_doc_from_docs(docs), 0, 4); + code.push_text(quote_doc_from_docs(docs), 0, 4); - // NOTE: We need to add required fields first, and then optional ones, otherwise mypy - // complains. - let fields_in_order = fields - .iter() - .filter(|field| !field.is_nullable) - .chain(fields.iter().filter(|field| field.is_nullable)); - for field in fields_in_order { - let ObjectField { - filepath: _, - fqname: _, - pkg_name: _, - name, - docs, - typ: _, - attrs: _, - is_nullable, - is_deprecated: _, - datatype: _, - } = field; + // NOTE: We need to add required fields first, and then optional ones, otherwise mypy + // complains. + let fields_in_order = fields + .iter() + .filter(|field| !field.is_nullable) + .chain(fields.iter().filter(|field| field.is_nullable)); + for field in fields_in_order { + let ObjectField { + filepath: _, + fqname: _, + pkg_name: _, + name, + docs, + typ: _, + attrs: _, + is_nullable, + is_deprecated: _, + datatype: _, + } = field; + + let (typ, _, default_converter) = + quote_field_type_from_field(objects, field, false); + let (typ_unwrapped, _, _) = quote_field_type_from_field(objects, field, true); + let typ = if *kind == ObjectKind::Archetype { + format!("{typ_unwrapped}Array") + } else { + typ + }; - let (typ, _) = quote_field_type_from_field(objects, field, false); - let typ = if *kind == ObjectKind::Archetype { - let (typ_unwrapped, _) = quote_field_type_from_field(objects, field, true); - format!("{typ_unwrapped}Array") - } else { - typ - }; - let typ = if *kind == ObjectKind::Archetype { - if !*is_nullable { - format!("{typ} = field(metadata={{'component': 'primary'}})") + let converter = if *kind == ObjectKind::Archetype { + // archetype always delegate field init to the component array object + format!("converter={typ_unwrapped}Array.from_similar, # type: ignore[misc]\n") } else { + // components and datatypes have converters only if manually provided + let override_name = format!( + "{}_{}_converter", + obj.name.to_lowercase(), + name.to_lowercase() + ); + if overrides.contains(&override_name) { + format!("converter={override_name}") + } else if !default_converter.is_empty() { + format!("converter={default_converter}") + } else { + String::new() + } + }; + + let metadata = if *kind == ObjectKind::Archetype { format!( - "{typ} | None = field(default=None, metadata={{'component': 'secondary'}})" + "\nmetadata={{'component': '{}'}}, ", + if *is_nullable { "secondary" } else { "primary" } ) - } - } else if !*is_nullable { - typ - } else { - format!("{typ} | None = None") - }; + } else { + String::new() + }; - code.push_text(format!("{name}: {typ}"), 1, 4); + let typ = if !*is_nullable { + format!("{typ} = field({metadata}{converter})") + } else { + format!( + "{typ} | None = field({metadata}default=None{}{converter})", + if converter.is_empty() { "" } else { ", " }, + ) + }; - code.push_text(quote_doc_from_docs(docs), 0, 4); - } + code.push_text(format!("{name}: {typ}"), 1, 4); - code.push_text(quote_str_repr_from_obj(obj), 1, 4); - code.push_text(quote_array_method_from_obj(objects, obj), 1, 4); - code.push_text(quote_str_method_from_obj(objects, obj), 1, 4); + code.push_text(quote_doc_from_docs(docs), 0, 4); + } - if obj.kind == ObjectKind::Archetype { - code.push_text(quote_builder_from_obj(objects, obj), 1, 4); - } else { - code.push_text(quote_aliases_from_object(obj), 1, 0); + if *kind == ObjectKind::Archetype { + code.push_text("__str__ = Archetype.__str__", 1, 4); + code.push_text("__repr__ = Archetype.__repr__", 1, 4); + } + + code.push_text(quote_array_method_from_obj(overrides, objects, obj), 1, 4); + code.push_text(quote_native_types_method_from_obj(objects, obj), 1, 4); + + if *kind != ObjectKind::Archetype { + code.push_text(quote_aliases_from_object(obj), 1, 0); + } } - code.push_text(quote_arrow_support_from_obj(arrow_registry, obj), 1, 0); + match kind { + ObjectKind::Archetype => (), + ObjectKind::Component => { + // a component might be either delegating to a datatype or using a native type + if let Type::Object(ref dtype_fqname) = obj.fields[0].typ { + let dtype_obj = objects.get(dtype_fqname); + code.push_text( + quote_arrow_support_from_delegating_component(obj, dtype_obj), + 1, + 0, + ); + } else { + code.push_text( + quote_arrow_support_from_obj(arrow_registry, overrides, obj), + 1, + 0, + ); + } + } + ObjectKind::Datatype => { + code.push_text( + quote_arrow_support_from_obj(arrow_registry, overrides, obj), + 1, + 0, + ); + } + } let mut filepath = PathBuf::from(filepath); filepath.set_extension("py"); @@ -380,8 +556,15 @@ impl QuotedObject { } } - fn from_union(arrow_registry: &ArrowRegistry, objects: &Objects, obj: &Object) -> Self { + // TODO(ab): this function is likely broken, to be handled in next PR + fn from_union( + arrow_registry: &ArrowRegistry, + overrides: &HashSet, + objects: &Objects, + obj: &Object, + ) -> Self { assert!(!obj.is_struct()); + assert_eq!(obj.kind, ObjectKind::Datatype); let Object { filepath, @@ -389,7 +572,7 @@ impl QuotedObject { pkg_name: _, name, docs, - kind: _, + kind, attrs: _, fields, specifics: _, @@ -402,7 +585,7 @@ impl QuotedObject { format!( r#" - @dataclass + @define class {name}: "# ), @@ -425,7 +608,7 @@ impl QuotedObject { datatype: _, } = field; - let (typ, _) = quote_field_type_from_field(objects, field, false); + let (typ, _, _) = quote_field_type_from_field(objects, field, false); // NOTE: It's always optional since only one of the fields can be set at a time. code.push_text(format!("{name}: {typ} | None = None"), 1, 4); @@ -433,11 +616,23 @@ impl QuotedObject { } code.push_text(quote_str_repr_from_obj(obj), 1, 4); - code.push_text(quote_array_method_from_obj(objects, obj), 1, 4); - code.push_text(quote_str_method_from_obj(objects, obj), 1, 4); + code.push_text(quote_array_method_from_obj(overrides, objects, obj), 1, 4); + code.push_text(quote_native_types_method_from_obj(objects, obj), 1, 4); code.push_text(quote_aliases_from_object(obj), 1, 0); - code.push_text(quote_arrow_support_from_obj(arrow_registry, obj), 1, 0); + match kind { + ObjectKind::Archetype => (), + ObjectKind::Component => { + unreachable!("component may not be a union") + } + ObjectKind::Datatype => { + code.push_text( + quote_arrow_support_from_obj(arrow_registry, overrides, obj), + 1, + 0, + ); + } + } let mut filepath = PathBuf::from(filepath); filepath.set_extension("py"); @@ -510,16 +705,31 @@ fn quote_str_repr_from_obj(obj: &Object) -> String { /// `npt.ArrayLike`/integer/floating-point field. /// /// Only applies to datatypes and components. -fn quote_array_method_from_obj(objects: &Objects, obj: &Object) -> String { +fn quote_array_method_from_obj( + overrides: &HashSet, + objects: &Objects, + obj: &Object, +) -> String { // TODO(cmc): should be using native type, but need transparency let typ = quote_field_type_from_field(objects, &obj.fields[0], false).0; - if - // cannot be an archetype - obj.kind == ObjectKind::Archetype - // has to have a single field + + // allow overriding the __array__ function + let override_name = format!("{}_as_array", obj.name.to_lowercase()); + if overrides.contains(&override_name) { + return unindent::unindent(&format!( + " + def __array__(self, dtype: npt.DTypeLike=None) -> npt.ArrayLike: + return {override_name}(self, dtype=dtype) + " + )); + } + + // exclude archetypes, objects which dont have a single field, and anything that isn't an numpy + // array or scalar numbers + if obj.kind == ObjectKind::Archetype || obj.fields.len() != 1 - // that single field must be `npt.ArrayLike`/integer/floating-point - || !["npt.ArrayLike", "float", "int"].contains(&typ.as_str()) + || (!["npt.ArrayLike", "float", "int"].contains(&typ.as_str()) + && !typ.contains("npt.NDArray")) { return String::new(); } @@ -527,24 +737,29 @@ fn quote_array_method_from_obj(objects: &Objects, obj: &Object) -> String { let field_name = &obj.fields[0].name; unindent::unindent(&format!( " - def __array__(self) -> npt.ArrayLike: - return np.asarray(self.{field_name}) + def __array__(self, dtype: npt.DTypeLike=None) -> npt.ArrayLike: + return np.asarray(self.{field_name}, dtype=dtype) ", )) } -/// Automatically implement `__str__` if the object is a single `str` field. +/// Automatically implement `__str__`, `__int__`, or `__float__` method if the object has a single +/// field of the corresponding type that is not optional. /// /// Only applies to datatypes and components. -fn quote_str_method_from_obj(objects: &Objects, obj: &Object) -> String { +fn quote_native_types_method_from_obj(objects: &Objects, obj: &Object) -> String { + let typ = quote_field_type_from_field(objects, &obj.fields[0], false).0; + let typ = typ.as_str(); if // cannot be an archetype obj.kind == ObjectKind::Archetype // has to have a single field || obj.fields.len() != 1 - // that single field must be `str` + // that field cannot be optional + || obj.fields[0].is_nullable + // that single field must be of a supported native type // TODO(cmc): should be using native type, but need transparency - || quote_field_type_from_field(objects, &obj.fields[0], false).0 != "str" + || !["str", "int", "float"].contains(&typ) { return String::new(); } @@ -552,15 +767,15 @@ fn quote_str_method_from_obj(objects: &Objects, obj: &Object) -> String { let field_name = &obj.fields[0].name; unindent::unindent(&format!( " - def __str__(self) -> str: - return self.{field_name} + def __{typ}__(self) -> {typ}: + return {typ}(self.{field_name}) ", )) } /// Only applies to datatypes and components. fn quote_aliases_from_object(obj: &Object) -> String { - assert!(obj.kind != ObjectKind::Archetype); + assert_ne!(obj.kind, ObjectKind::Archetype); let aliases = obj.try_get_attr::(ATTR_PYTHON_ALIASES); let array_aliases = obj @@ -591,7 +806,7 @@ fn quote_aliases_from_object(obj: &Object) -> String { format!( r#" {name}ArrayLike = Union[ - {name}Like, + {name}, Sequence[{name}Like], {array_aliases} ] @@ -644,12 +859,14 @@ fn quote_import_clauses_from_field(field: &ObjectField) -> Option { /// Specifying `unwrap = true` will unwrap the final type before returning it, e.g. `Vec` /// becomes just `String`. /// The returned boolean indicates whether there was anything to unwrap at all. +/// The third returned value is a default converter function if any. fn quote_field_type_from_field( objects: &Objects, field: &ObjectField, unwrap: bool, -) -> (String, bool) { +) -> (String, bool, String) { let mut unwrapped = false; + let mut converter = String::new(); let typ = match &field.typ { Type::UInt8 | Type::UInt16 @@ -667,32 +884,44 @@ fn quote_field_type_from_field( length: _, } | Type::Vector { elem_type } => { - let array_like = matches!( - elem_type, - ElementType::UInt8 - | ElementType::UInt16 - | ElementType::UInt32 - | ElementType::UInt64 - | ElementType::Int8 - | ElementType::Int16 - | ElementType::Int32 - | ElementType::Int64 - | ElementType::Bool - | ElementType::Float16 - | ElementType::Float32 - | ElementType::Float64 - | ElementType::String - ); + match elem_type { + ElementType::UInt8 => converter = "to_np_uint8".to_owned(), + ElementType::UInt16 => converter = "to_np_uint16".to_owned(), + ElementType::UInt32 => converter = "to_np_uint32".to_owned(), + ElementType::UInt64 => converter = "to_np_uint64".to_owned(), + ElementType::Int8 => converter = "to_np_int8".to_owned(), + ElementType::Int16 => converter = "to_np_int16".to_owned(), + ElementType::Int32 => converter = "to_np_int32".to_owned(), + ElementType::Int64 => converter = "to_np_int64".to_owned(), + ElementType::Bool => converter = "to_np_bool".to_owned(), + ElementType::Float16 => converter = "to_np_float16".to_owned(), + ElementType::Float32 => converter = "to_np_float32".to_owned(), + ElementType::Float64 => converter = "to_np_float64".to_owned(), + _ => {} + }; - if array_like { - "npt.ArrayLike".to_owned() - } else { - let typ = quote_type_from_element_type(elem_type); - if unwrap { - unwrapped = true; - typ - } else { - format!("list[{typ}]") + match elem_type { + ElementType::UInt8 => "npt.NDArray[np.uint8]".to_owned(), + ElementType::UInt16 => "npt.NDArray[np.uint16]".to_owned(), + ElementType::UInt32 => "npt.NDArray[np.uint32]".to_owned(), + ElementType::UInt64 => "npt.NDArray[np.uint64]".to_owned(), + ElementType::Int8 => "npt.NDArray[np.int8]".to_owned(), + ElementType::Int16 => "npt.NDArray[np.int16]".to_owned(), + ElementType::Int32 => "npt.NDArray[np.int32]".to_owned(), + ElementType::Int64 => "npt.NDArray[np.int64]".to_owned(), + ElementType::Bool => "npt.NDArray[np.bool_]".to_owned(), + ElementType::Float16 => "npt.NDArray[np.float16]".to_owned(), + ElementType::Float32 => "npt.NDArray[np.float32]".to_owned(), + ElementType::Float64 => "npt.NDArray[np.float64]".to_owned(), + ElementType::String => "list[str]".to_owned(), + ElementType::Object(_) => { + let typ = quote_type_from_element_type(elem_type); + if unwrap { + unwrapped = true; + typ + } else { + format!("list[{typ}]") + } } } } @@ -705,11 +934,7 @@ fn quote_field_type_from_field( .is_some(); if is_transparent { let target = objects.get(fqname); - assert!( - target.fields.len() == 1, - "transparent field must point to an object with exactly 1 field, but {:?} has {}", - fqname, target.fields.len(), - ); + assert_eq!(target.fields.len(), 1, "transparent field must point to an object with exactly 1 field, but {:?} has {}", fqname, target.fields.len()); // NOTE: unwrap call is safe due to assertion just above return quote_field_type_from_field( objects, @@ -721,7 +946,7 @@ fn quote_field_type_from_field( } }; - (typ, unwrapped) + (typ, unwrapped, converter) } fn quote_type_from_element_type(typ: &ElementType) -> String { @@ -757,171 +982,125 @@ fn quote_type_from_element_type(typ: &ElementType) -> String { } } -fn quote_arrow_support_from_obj(arrow_registry: &ArrowRegistry, obj: &Object) -> String { +fn quote_arrow_support_from_delegating_component(obj: &Object, dtype_obj: &Object) -> String { let Object { - fqname, - name, - kind, - filepath, - .. + fqname, name, kind, .. } = obj; - match kind { - ObjectKind::Datatype | ObjectKind::Component => { - let datatype = quote_arrow_datatype(&arrow_registry.get(fqname)); + assert_eq!( + *kind, + ObjectKind::Component, + "this function only handles components" + ); - let mono = name.clone(); - let mono_aliases = format!("{name}Like"); - let many = format!("{name}Array"); - let many_aliases = format!("{name}ArrayLike"); - let arrow = format!("{name}Type"); + // extract datatype, must be *only* one + assert_eq!( + obj.fields.len(), + 1, + "component must have exactly one field, but {} has {}", + fqname, + obj.fields.len() + ); - let mut filepath = PathBuf::from(filepath); - filepath.set_extension("py"); - let filename = filepath.file_stem().unwrap().to_string_lossy(); + let extension_type = format!("{name}Type"); + let extension_array = format!("{name}Array"); - let legacy_fqname = obj - .try_get_attr::(ATTR_RERUN_LEGACY_FQNAME) - .unwrap_or_else(|| fqname.clone()); + let dtype_extension_type = format!("{}Type", dtype_obj.name); + let dtype_extension_array = format!("{}Array", dtype_obj.name); + let dtype_extension_array_like = format!("{}ArrayLike", dtype_obj.name); - let superclass = if kind == &ObjectKind::Component { - "Component, " - } else { - "" - }; + let legacy_fqname = obj + .try_get_attr::(ATTR_RERUN_LEGACY_FQNAME) + .unwrap_or_else(|| fqname.clone()); - unindent::unindent(&format!( - r#" + unindent::unindent(&format!( + r#" - # --- Arrow support --- - - from .{filename}_ext import {many}Ext # noqa: E402 - - class {arrow}(pa.ExtensionType): # type: ignore[misc] - def __init__(self: type[pa.ExtensionType]) -> None: - pa.ExtensionType.__init__( - self, {datatype}, "{legacy_fqname}" - ) - - def __arrow_ext_serialize__(self: type[pa.ExtensionType]) -> bytes: - # since we don't have a parameterized type, we don't need extra metadata to be deserialized - return b"" - - @classmethod - def __arrow_ext_deserialize__( - cls: type[pa.ExtensionType], storage_type: Any, serialized: Any - ) -> type[pa.ExtensionType]: - # return an instance of this subclass given the serialized metadata. - return {arrow}() - - def __arrow_ext_class__(self: type[pa.ExtensionType]) -> type[pa.ExtensionArray]: - return {many} - - # TODO(cmc): bring back registration to pyarrow once legacy types are gone - # pa.register_extension_type({arrow}()) - - class {many}({superclass}{many}Ext): # type: ignore[misc] - _extension_name = "{legacy_fqname}" - - @staticmethod - def from_similar(data: {many_aliases} | None) -> pa.Array: - if data is None: - return {arrow}().wrap_array(pa.array([], type={arrow}().storage_type)) - else: - return {many}Ext._from_similar( - data, - mono={mono}, - mono_aliases={mono_aliases}, - many={many}, - many_aliases={many_aliases}, - arrow={arrow}, - ) - "# - )) - } - ObjectKind::Archetype => String::new(), - } + class {extension_type}(BaseDelegatingExtensionType): + _TYPE_NAME = "{legacy_fqname}" + _DELEGATED_EXTENSION_TYPE = datatypes.{dtype_extension_type} + + class {extension_array}(BaseDelegatingExtensionArray[datatypes.{dtype_extension_array_like}]): + _EXTENSION_NAME = "{legacy_fqname}" + _EXTENSION_TYPE = {extension_type} + _DELEGATED_ARRAY_TYPE = datatypes.{dtype_extension_array} + + {extension_type}._ARRAY_TYPE = {extension_array} + + # TODO(cmc): bring back registration to pyarrow once legacy types are gone + # pa.register_extension_type({extension_type}()) + "# + )) } -/// Only makes sense for archetypes. -fn quote_builder_from_obj(objects: &Objects, obj: &Object) -> String { - assert_eq!(ObjectKind::Archetype, obj.kind); +/// Arrow support objects +/// +/// Generated for Components using native types and Datatypes. Components using a Datatype instead +/// delegate to the Datatype's arrow support. +fn quote_arrow_support_from_obj( + arrow_registry: &ArrowRegistry, + overrides: &HashSet, + obj: &Object, +) -> String { + let Object { fqname, name, .. } = obj; + + let (ext_type_base, ext_array_base) = + if obj.kind == ObjectKind::Datatype || obj.is_non_delegating_component() { + ("BaseExtensionType", "BaseExtensionArray") + } else if obj.is_delegating_component() { + ( + "BaseDelegatingExtensionType", + "BaseDelegatingExtensionArray", + ) + } else { + unreachable!("archetypes do not have arrow support") + }; - let required = obj - .fields - .iter() - .filter(|field| !field.is_nullable) - .collect::>(); - let optional = obj - .fields - .iter() - .filter(|field| field.is_nullable) - .collect::>(); + let datatype = quote_arrow_datatype(&arrow_registry.get(fqname)); + let extension_array = format!("{name}Array"); + let extension_type = format!("{name}Type"); + let many_aliases = format!("{name}ArrayLike"); + + let legacy_fqname = obj + .try_get_attr::(ATTR_RERUN_LEGACY_FQNAME) + .unwrap_or_else(|| fqname.clone()); + + let name_lower = name.to_lowercase(); + let override_name = format!("{name_lower}_native_to_pa_array"); + let override_ = if overrides.contains(&override_name) { + format!("return {name_lower}_native_to_pa_array(data, data_type)") + } else { + "raise NotImplementedError".to_owned() + }; - let mut code = String::new(); + unindent::unindent(&format!( + r#" - let required_args = required - .iter() - .map(|field| { - let (typ, unwrapped) = quote_field_type_from_field(objects, field, true); - if unwrapped { - // This was originally a vec/array! - format!("{}: {typ}ArrayLike", field.name) - } else { - format!("{}: {typ}Like", field.name) - } - }) - .collect::>() - .join(", "); - let optional_args = optional - .iter() - .map(|field| { - let (typ, unwrapped) = quote_field_type_from_field(objects, field, true); - if unwrapped { - // This was originally a vec/array! - format!("{}: {typ}ArrayLike | None = None", field.name) - } else { - format!("{}: {typ}Like | None = None", field.name) - } - }) - .collect::>() - .join(", "); + # --- Arrow support --- - code.push_text( - format!("def __init__(self, {required_args}, *, {optional_args}) -> None:"), - 1, - 0, - ); + class {extension_type}({ext_type_base}): + def __init__(self) -> None: + pa.ExtensionType.__init__( + self, {datatype}, "{legacy_fqname}" + ) - code.push_text("# Required components", 1, 4); - for field in required { - let name = &field.name; - let (typ, _) = quote_field_type_from_field(objects, field, true); - code.push_text( - format!("self.{name} = {typ}Array.from_similar({name})"), - 1, - 4, - ); - } + class {extension_array}({ext_array_base}[{many_aliases}]): + _EXTENSION_NAME = "{legacy_fqname}" + _EXTENSION_TYPE = {extension_type} - code.push('\n'); + @staticmethod + def _native_to_pa_array(data: {many_aliases}, data_type: pa.DataType) -> pa.Array: + {override_} - code.push_text("# Optional components\n", 1, 4); - for field in optional { - let name = &field.name; - let (typ, _) = quote_field_type_from_field(objects, field, true); - code.push_text( - format!("self.{name} = {typ}Array.from_similar({name})"), - 1, - 4, - ); - } + {extension_type}._ARRAY_TYPE = {extension_array} - code + # TODO(cmc): bring back registration to pyarrow once legacy types are gone + # pa.register_extension_type({extension_type}()) + "# + )) } // --- Arrow registry code generators --- - use arrow2::datatypes::{DataType, Field, UnionMode}; fn quote_arrow_datatype(datatype: &DataType) -> String { diff --git a/crates/re_types_builder/src/lib.rs b/crates/re_types_builder/src/lib.rs index 73bb5eaf039b..6487ace25092 100644 --- a/crates/re_types_builder/src/lib.rs +++ b/crates/re_types_builder/src/lib.rs @@ -258,7 +258,6 @@ pub fn generate_rust_code( // passes 1 through 3: bfbs, semantic, arrow registry let (objects, arrow_registry) = generate_lang_agnostic(include_dir_path, entrypoint_path); - // generate rust code let mut gen = RustCodeGenerator::new(output_crate_path.as_ref()); let _filepaths = gen.generate(&objects, &arrow_registry); } diff --git a/crates/re_types_builder/src/objects.rs b/crates/re_types_builder/src/objects.rs index 50559609100b..34262ae701ae 100644 --- a/crates/re_types_builder/src/objects.rs +++ b/crates/re_types_builder/src/objects.rs @@ -277,7 +277,7 @@ impl Object { let kind = ObjectKind::from_pkg_name(&pkg_name); let attrs = Attributes::from_raw_attrs(obj.attributes()); - let fields = { + let fields: Vec<_> = { let mut fields: Vec<_> = obj .fields() .iter() @@ -289,6 +289,14 @@ impl Object { fields }; + if kind == ObjectKind::Component { + assert!( + fields.len() == 1, + "components must have exactly 1 field, but {fqname} has {}", + fields.len() + ); + } + Self { filepath, fqname, @@ -337,7 +345,7 @@ impl Object { }; let attrs = Attributes::from_raw_attrs(enm.attributes()); - let fields = enm + let fields: Vec<_> = enm .values() .iter() // NOTE: `BaseType::None` is only used by internal flatbuffers fields, we don't care. @@ -349,6 +357,14 @@ impl Object { .map(|val| ObjectField::from_raw_enum_value(enums, objs, enm, &val)) .collect(); + if kind == ObjectKind::Component { + assert!( + fields.len() == 1, + "components must have exactly 1 field, but {fqname} has {}", + fields.len() + ); + } + Self { filepath, fqname, diff --git a/rerun_py/rerun_sdk/rerun/__init__.py b/rerun_py/rerun_sdk/rerun/__init__.py index 64184c10318c..609b2d2bf5b6 100644 --- a/rerun_py/rerun_sdk/rerun/__init__.py +++ b/rerun_py/rerun_sdk/rerun/__init__.py @@ -58,6 +58,11 @@ # Next-gen API imports ENABLE_NEXT_GEN_API = True if ENABLE_NEXT_GEN_API: + # As temporary next-gen API namespacing, we bring to the top level `rr` namespace the `log_any` function as well as + # all archetypes. As components and datatypes have many attributes, several of which collide with the legacy API, + # we bring them in the `rr.cmp` and `rr.dt` sub-namespaces, respectively. + from ._rerun2 import components as cmp + from ._rerun2 import datatypes as dt from ._rerun2.archetypes import * from ._rerun2.log_any import log_any diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/_baseclasses.py b/rerun_py/rerun_sdk/rerun/_rerun2/_baseclasses.py index a42f49a61ad3..03a4f88f3d75 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/_baseclasses.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/_baseclasses.py @@ -1,16 +1,124 @@ from __future__ import annotations -from dataclasses import dataclass +from typing import Any, Generic, Self, TypeVar, cast import pyarrow as pa +from attrs import define, fields +T = TypeVar("T") -@dataclass + +@define class Archetype: - pass + """Base class for all archetypes.""" + + def __str__(self) -> str: + cls = type(self) + + s = f"rr.{cls.__name__}(\n" + for fld in fields(cls): + if "component" in fld.metadata: + comp = getattr(self, fld.name) + datatype = getattr(comp, "type", None) + if datatype: + s += f" {datatype.extension_name}<{datatype.storage_type}>(\n {comp.to_pylist()}\n )\n" + s += ")" + + return s + + __repr__ = __str__ + + +class BaseExtensionType(pa.ExtensionType): # type: ignore[misc] + """Extension type for datatypes and non-delegating components.""" + + _ARRAY_TYPE: type[pa.ExtensionArray] = pa.ExtensionArray + """The extension array class associated with this class.""" + + # Note: (de)serialization is not used in the Python SDK + def __arrow_ext_serialize__(self) -> bytes: + return b"" + + # noinspection PyMethodOverriding + @classmethod + def __arrow_ext_deserialize__(cls, storage_type: Any, serialized: Any) -> pa.ExtensionType: + return cls() + + def __arrow_ext_class__(self) -> type[pa.ExtensionArray]: + return self._ARRAY_TYPE + + +class NamedExtensionArray(pa.ExtensionArray): # type: ignore[misc] + """Common base class for any extension array that has a name.""" + + _EXTENSION_NAME = "" + """The fully qualified name of this class.""" -class Component(pa.ExtensionArray): # type: ignore[misc] @property def extension_name(self) -> str: - return getattr(self, "_extension_name", "") + return self._EXTENSION_NAME + + +class BaseExtensionArray(NamedExtensionArray, Generic[T]): # type: ignore[misc] + """Extension array for datatypes and non-delegating components.""" + + _EXTENSION_TYPE = pa.ExtensionType + """The extension type class associated with this class.""" + + @classmethod + def from_similar(cls, data: T | None) -> Self: + data_type = cls._EXTENSION_TYPE() + + if data is None: + return cast(Self, data_type.wrap_array(pa.array([], type=data_type.storage_type))) + else: + return cast(Self, data_type.wrap_array(cls._native_to_pa_array(data, data_type.storage_type))) + + @staticmethod + def _native_to_pa_array(data: T, data_type: pa.DataType) -> pa.Array: + raise NotImplementedError + + +class BaseDelegatingExtensionType(pa.ExtensionType): # type: ignore[misc] + """Extension type for delegating components.""" + + _TYPE_NAME = "" + """The fully qualified name of the component.""" + + _ARRAY_TYPE = pa.ExtensionArray + """The extension array class associated with this component.""" + + _DELEGATED_EXTENSION_TYPE = BaseExtensionType + """The extension type class associated with this component's datatype.""" + + def __init__(self) -> None: + # TODO(ab, cmc): we unwrap the type here because we can't have two layers of extension types for now + pa.ExtensionType.__init__(self, self._DELEGATED_EXTENSION_TYPE().storage_type, self._TYPE_NAME) + + # Note: (de)serialization is not used in the Python SDK + + def __arrow_ext_serialize__(self) -> bytes: + return b"" + + # noinspection PyMethodOverriding + @classmethod + def __arrow_ext_deserialize__(cls, storage_type: Any, serialized: Any) -> pa.ExtensionType: + return cls() + + def __arrow_ext_class__(self) -> type[pa.ExtensionArray]: + return self._ARRAY_TYPE # type: ignore[no-any-return] + + +class BaseDelegatingExtensionArray(BaseExtensionArray[T]): # type: ignore[misc] + """Extension array for delegating components.""" + + _DELEGATED_ARRAY_TYPE = BaseExtensionArray[T] # type: ignore[valid-type] + """The extension array class associated with this component's datatype.""" + + @classmethod + def from_similar(cls, data: T | None) -> Self: + arr = cls._DELEGATED_ARRAY_TYPE.from_similar(data) + + # TODO(ab, cmc): we unwrap the type here because we can't have two layers of extension types for now + return cast(Self, cls._EXTENSION_TYPE().wrap_array(arr.storage)) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/_converters.py b/rerun_py/rerun_sdk/rerun/_rerun2/_converters.py new file mode 100644 index 000000000000..d70d3a45d4ea --- /dev/null +++ b/rerun_py/rerun_sdk/rerun/_rerun2/_converters.py @@ -0,0 +1,101 @@ +""" +Utility converter functions to make attrs and mypy happy. + +As of version 1.4.1, mypy (and possibly other tooling) doesn't properly recognize converters passed to attrs fields. For +example, consider this class: + +```python +from attrs import define, field +import numpy as np +import numpy.typing as npt + +@define +class ClassA: + xy: npt.NDArray[np.float32]= field(converter=CONVERTER) +``` + +mypy is only happy if `CONVERTER` is a regular, properly-typed function. In particular, it rejects both of these correct +and more compact alternatives: +- `lambda data: np.array(data, dtype=np.float32)` +- `functools.partial(np.array, dtype=np.float32)` +""" + +from __future__ import annotations + +import numpy as np +import numpy.typing as npt + +__all__ = [ + "to_np_uint8", + "to_np_uint16", + "to_np_uint32", + "to_np_uint64", + "to_np_int8", + "to_np_int16", + "to_np_int32", + "to_np_int64", + "to_np_bool", + "to_np_float16", + "to_np_float32", + "to_np_float64", +] + + +def to_np_uint8(data: npt.ArrayLike) -> npt.NDArray[np.uint8]: + """Convert some datat to a numpy uint8 array.""" + return np.asarray(data, dtype=np.uint8) + + +def to_np_uint16(data: npt.ArrayLike) -> npt.NDArray[np.uint16]: + """Convert some datat to a numpy uint16 array.""" + return np.asarray(data, dtype=np.uint16) + + +def to_np_uint32(data: npt.ArrayLike) -> npt.NDArray[np.uint32]: + """Convert some datat to a numpy uint32 array.""" + return np.asarray(data, dtype=np.uint32) + + +def to_np_uint64(data: npt.ArrayLike) -> npt.NDArray[np.uint64]: + """Convert some datat to a numpy uint64 array.""" + return np.asarray(data, dtype=np.uint64) + + +def to_np_int8(data: npt.ArrayLike) -> npt.NDArray[np.int8]: + """Convert some datat to a numpy int8 array.""" + return np.asarray(data, dtype=np.int8) + + +def to_np_int16(data: npt.ArrayLike) -> npt.NDArray[np.int16]: + """Convert some datat to a numpy int16 array.""" + return np.asarray(data, dtype=np.int16) + + +def to_np_int32(data: npt.ArrayLike) -> npt.NDArray[np.int32]: + """Convert some datat to a numpy int32 array.""" + return np.asarray(data, dtype=np.int32) + + +def to_np_int64(data: npt.ArrayLike) -> npt.NDArray[np.int64]: + """Convert some datat to a numpy int64 array.""" + return np.asarray(data, dtype=np.int64) + + +def to_np_bool(data: npt.ArrayLike) -> npt.NDArray[np.bool_]: + """Convert some datat to a numpy bool array.""" + return np.asarray(data, dtype=np.bool_) + + +def to_np_float16(data: npt.ArrayLike) -> npt.NDArray[np.float16]: + """Convert some datat to a numpy float16 array.""" + return np.asarray(data, dtype=np.float16) + + +def to_np_float32(data: npt.ArrayLike) -> npt.NDArray[np.float32]: + """Convert some datat to a numpy float32 array.""" + return np.asarray(data, dtype=np.float32) + + +def to_np_float64(data: npt.ArrayLike) -> npt.NDArray[np.float64]: + """Convert some datat to a numpy float64 array.""" + return np.asarray(data, dtype=np.float64) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/archetypes/_overrides/__init__.py b/rerun_py/rerun_sdk/rerun/_rerun2/archetypes/_overrides/__init__.py new file mode 100644 index 000000000000..9d48db4f9f85 --- /dev/null +++ b/rerun_py/rerun_sdk/rerun/_rerun2/archetypes/_overrides/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/__init__.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/__init__.py new file mode 100644 index 000000000000..4b0310028b3d --- /dev/null +++ b/rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/__init__.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +from .class_id import classid_native_to_pa_array +from .color import color_native_to_pa_array, color_rgba_converter +from .draw_order import draworder_native_to_pa_array +from .instance_key import instancekey_native_to_pa_array +from .keypoint_id import keypointid_native_to_pa_array +from .label import label_native_to_pa_array +from .radius import radius_native_to_pa_array diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/class_id.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/class_id.py new file mode 100644 index 000000000000..f259faf8f318 --- /dev/null +++ b/rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/class_id.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import numpy as np +import pyarrow as pa + +if TYPE_CHECKING: + from .. import ClassIdArrayLike + + +def classid_native_to_pa_array(data: ClassIdArrayLike, data_type: pa.DataType) -> pa.Array: + array = np.asarray(data, dtype=np.uint16).flatten() + return pa.array(array, type=data_type) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/color.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/color.py new file mode 100644 index 000000000000..6e4ceba62550 --- /dev/null +++ b/rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/color.py @@ -0,0 +1,75 @@ +""" +Overrides for `Color` component. + +Possible input for `Color`: +- Sequence[int]: interpreted as rgb or rgba values in 0-255 range +- numpy array: interpreted as rgb or rgba values, range depending on dtype +- anything else (int or convertible to int): interpreted as a 32-bit packed rgba value + +Possible inputs for `ColorArray.from_similar()`: +- a single `Color` instance +- a sequence of `Color` instances +- Nx3 or Nx4 numpy array, range depending on dtype +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Sequence, cast + +import numpy as np +import numpy.typing as npt +import pyarrow as pa + +from rerun.color_conversion import u8_array_to_rgba + +if TYPE_CHECKING: + from .. import ColorArrayLike, ColorLike + + +def _numpy_array_to_u32(data: npt.NDArray[np.uint8 | np.float32 | np.float64]) -> npt.NDArray[np.uint32]: + if data.size == 0: + return np.array([], dtype=np.uint32) + + if data.dtype.type in [np.float32, np.float64]: + array = u8_array_to_rgba(np.asarray(np.round(np.asarray(data) * 255.0), np.uint8)) + elif data.dtype.type == np.uint32: + array = np.asarray(data, dtype=np.uint32).flatten() + else: + array = u8_array_to_rgba(np.asarray(data, dtype=np.uint8)) + return array + + +def color_rgba_converter(data: ColorLike) -> int: + if isinstance(data, np.ndarray): + return int(_numpy_array_to_u32(data.reshape((1, -1)))[0]) + elif isinstance(data, Sequence): + return int(_numpy_array_to_u32(np.array(data, dtype=np.uint8).reshape((1, -1)))[0]) + else: + return int(data) + + +def color_native_to_pa_array(data: ColorArrayLike, data_type: pa.DataType) -> pa.Array: + from .. import Color + + if isinstance(data, Color): + data = [data] + + if isinstance(data, np.ndarray): + if data.dtype == np.uint32: + # these are already packed values + array = data.flatten() + else: + # these are component values + if len(data.shape) == 1: + if data.size > 4: + # multiple RGBA colors + data = data.reshape((-1, 4)) + else: + # a single color + data = data.reshape((1, -1)) + array = _numpy_array_to_u32(cast(npt.NDArray[np.uint8 | np.float32 | np.float64], data)) + else: + # Sequence of Color (which are int-like) or sequence of sequence + array = np.array([Color(datum) for datum in data], np.uint32) + + return pa.array(array, type=data_type) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/draw_order.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/draw_order.py new file mode 100644 index 000000000000..2e4fe7f71d10 --- /dev/null +++ b/rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/draw_order.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import numpy as np +import pyarrow as pa + +if TYPE_CHECKING: + from .. import DrawOrderArrayLike + + +def draworder_native_to_pa_array(data: DrawOrderArrayLike, data_type: pa.DataType) -> pa.Array: + array = np.asarray(data, dtype=np.float32).flatten() + return pa.array(array, type=data_type) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/instance_key.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/instance_key.py new file mode 100644 index 000000000000..29a428c6b75a --- /dev/null +++ b/rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/instance_key.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import numpy as np +import pyarrow as pa + +if TYPE_CHECKING: + from .. import InstanceKeyArrayLike + + +def instancekey_native_to_pa_array(data: InstanceKeyArrayLike, data_type: pa.DataType) -> pa.Array: + array = np.asarray(data, dtype=np.uint64).flatten() + return pa.array(array, type=data_type) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/keypoint_id.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/keypoint_id.py new file mode 100644 index 000000000000..20af6779fca2 --- /dev/null +++ b/rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/keypoint_id.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import numpy as np +import pyarrow as pa + +if TYPE_CHECKING: + from .. import KeypointIdArrayLike + + +def keypointid_native_to_pa_array(data: KeypointIdArrayLike, data_type: pa.DataType) -> pa.Array: + array = np.asarray(data, dtype=np.uint16).flatten() + return pa.array(array, type=data_type) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/label.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/label.py new file mode 100644 index 000000000000..6b7356928518 --- /dev/null +++ b/rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/label.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Sequence + +import pyarrow as pa + +if TYPE_CHECKING: + from .. import LabelArrayLike + + +def label_native_to_pa_array(data: LabelArrayLike, data_type: pa.DataType) -> pa.Array: + if isinstance(data, str): + array = [data] + elif isinstance(data, Sequence): + array = [str(datum) for datum in data] + else: + array = [str(data)] + + return pa.array(array, type=data_type) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/radius.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/radius.py new file mode 100644 index 000000000000..bb244ae53213 --- /dev/null +++ b/rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/radius.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import numpy as np +import pyarrow as pa + +if TYPE_CHECKING: + from .. import RadiusArrayLike + + +def radius_native_to_pa_array(data: RadiusArrayLike, data_type: pa.DataType) -> pa.Array: + array = np.asarray(data, dtype=np.float32).flatten() + return pa.array(array, type=data_type) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/class_id_ext.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/class_id_ext.py deleted file mode 100644 index 2d4e24227555..000000000000 --- a/rerun_py/rerun_sdk/rerun/_rerun2/components/class_id_ext.py +++ /dev/null @@ -1,21 +0,0 @@ -from __future__ import annotations - -__all__ = ["ClassIdArrayExt"] - -from typing import Any, Sequence - -import numpy as np -import pyarrow as pa - - -class ClassIdArrayExt: - @staticmethod - def _from_similar( - data: Any | None, *, mono: type, mono_aliases: Any, many: type, many_aliases: Any, arrow: type - ) -> pa.Array: - if isinstance(data, Sequence) and (len(data) > 0 and isinstance(data[0], mono)): - array = np.asarray([class_id.id for class_id in data], np.uint16) - else: - array = np.asarray(data, dtype=np.uint16).flatten() - - return arrow().wrap_array(pa.array(array, type=arrow().storage_type)) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/color_ext.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/color_ext.py deleted file mode 100644 index f868d353f7ed..000000000000 --- a/rerun_py/rerun_sdk/rerun/_rerun2/components/color_ext.py +++ /dev/null @@ -1,43 +0,0 @@ -from __future__ import annotations - -__all__ = ["ColorArrayExt"] - -from typing import Any, Sequence - -import numpy as np -import pyarrow as pa - -from rerun.color_conversion import u8_array_to_rgba - - -class ColorArrayExt: - @staticmethod - def _from_similar( - data: Any | None, *, mono: type, mono_aliases: Any, many: type, many_aliases: Any, arrow: type - ) -> pa.Array: - """ - Normalize flexible colors arrays. - - Float colors are assumed to be in 0-1 gamma sRGB space. - All other colors are assumed to be in 0-255 gamma sRGB space. - - If there is an alpha, we assume it is in linear space, and separate (NOT pre-multiplied). - """ - if isinstance(data, Sequence) and len(data) == 0: - array = np.array([], np.uint32) - elif isinstance(data, Sequence) and (len(data) > 0 and isinstance(data[0], mono)): - array = np.asarray([color.rgba for color in data], np.uint32) - elif isinstance(data, Sequence) and (len(data) > 0 and isinstance(data[0], int)): - array = np.asarray(data, np.uint32) - else: - array = np.asarray(data) - # Rust expects colors in 0-255 uint8 - if array.dtype.type in [np.float32, np.float64]: - # Assume gamma-space colors - array = u8_array_to_rgba(np.asarray(np.round(np.asarray(data).reshape((-1, 4)) * 255.0), np.uint8)) - elif array.dtype.type == np.uint32: - array = np.asarray(data).flatten() - else: - array = u8_array_to_rgba(np.asarray(data, dtype=np.uint8).reshape((-1, 4))) - - return arrow().wrap_array(pa.array(array, type=arrow().storage_type)) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/draw_order_ext.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/draw_order_ext.py deleted file mode 100644 index bf7f185c1d0f..000000000000 --- a/rerun_py/rerun_sdk/rerun/_rerun2/components/draw_order_ext.py +++ /dev/null @@ -1,21 +0,0 @@ -from __future__ import annotations - -__all__ = ["DrawOrderArrayExt"] - -from typing import Any, Sequence - -import numpy as np -import pyarrow as pa - - -class DrawOrderArrayExt: - @staticmethod - def _from_similar( - data: Any | None, *, mono: type, mono_aliases: Any, many: type, many_aliases: Any, arrow: type - ) -> pa.Array: - if isinstance(data, Sequence) and (len(data) > 0 and isinstance(data[0], mono)): - array = np.asarray([draw_order.value for draw_order in data], np.float32) - else: - array = np.require(np.asarray(data), np.float32).flatten() - - return arrow().wrap_array(pa.array(array, type=arrow().storage_type)) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/fuzzy_ext.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/fuzzy_ext.py deleted file mode 100644 index 73c1b3c61549..000000000000 --- a/rerun_py/rerun_sdk/rerun/_rerun2/components/fuzzy_ext.py +++ /dev/null @@ -1,71 +0,0 @@ -from __future__ import annotations - -__all__ = [ - "AffixFuzzer1ArrayExt", - "AffixFuzzer2ArrayExt", - "AffixFuzzer3ArrayExt", - "AffixFuzzer4ArrayExt", - "AffixFuzzer5ArrayExt", - "AffixFuzzer6ArrayExt", - "AffixFuzzer7ArrayExt", -] - -from typing import Any - -import pyarrow as pa - - -class AffixFuzzer1ArrayExt: - @staticmethod - def _from_similar( - data: Any | None, *, mono: type, mono_aliases: Any, many: type, many_aliases: Any, arrow: type - ) -> pa.Array: - raise NotImplementedError() - - -class AffixFuzzer2ArrayExt: - @staticmethod - def _from_similar( - data: Any | None, *, mono: type, mono_aliases: Any, many: type, many_aliases: Any, arrow: type - ) -> pa.Array: - raise NotImplementedError() - - -class AffixFuzzer3ArrayExt: - @staticmethod - def _from_similar( - data: Any | None, *, mono: type, mono_aliases: Any, many: type, many_aliases: Any, arrow: type - ) -> pa.Array: - raise NotImplementedError() - - -class AffixFuzzer4ArrayExt: - @staticmethod - def _from_similar( - data: Any | None, *, mono: type, mono_aliases: Any, many: type, many_aliases: Any, arrow: type - ) -> pa.Array: - raise NotImplementedError() - - -class AffixFuzzer5ArrayExt: - @staticmethod - def _from_similar( - data: Any | None, *, mono: type, mono_aliases: Any, many: type, many_aliases: Any, arrow: type - ) -> pa.Array: - raise NotImplementedError() - - -class AffixFuzzer6ArrayExt: - @staticmethod - def _from_similar( - data: Any | None, *, mono: type, mono_aliases: Any, many: type, many_aliases: Any, arrow: type - ) -> pa.Array: - raise NotImplementedError() - - -class AffixFuzzer7ArrayExt: - @staticmethod - def _from_similar( - data: Any | None, *, mono: type, mono_aliases: Any, many: type, many_aliases: Any, arrow: type - ) -> pa.Array: - raise NotImplementedError() diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/instance_key_ext.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/instance_key_ext.py deleted file mode 100644 index 55fdf14f9981..000000000000 --- a/rerun_py/rerun_sdk/rerun/_rerun2/components/instance_key_ext.py +++ /dev/null @@ -1,33 +0,0 @@ -from __future__ import annotations - -__all__ = ["InstanceKeyArrayExt"] - -from typing import TYPE_CHECKING, Any, Sequence - -import numpy as np -import pyarrow as pa - -if TYPE_CHECKING: - from .instance_key import InstanceKeyArray - -_MAX_U64 = 2**64 - 1 - - -class InstanceKeyArrayExt: - @staticmethod - def _from_similar( - data: Any | None, *, mono: type, mono_aliases: Any, many: type, many_aliases: Any, arrow: type - ) -> pa.Array: - if isinstance(data, Sequence) and (len(data) > 0 and isinstance(data[0], mono)): - array = np.asarray([datum.value for datum in data], np.uint64) - else: - array = np.asarray(data, dtype=np.uint64).flatten() - - return arrow().wrap_array(pa.array(array, type=arrow().storage_type)) - - @staticmethod - def splat() -> InstanceKeyArray: - from .instance_key import InstanceKeyType - - storage = pa.array([_MAX_U64], type=InstanceKeyType().storage_type) - return storage # type: ignore[no-any-return] diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/keypoint_id_ext.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/keypoint_id_ext.py deleted file mode 100644 index 96af2a059cdc..000000000000 --- a/rerun_py/rerun_sdk/rerun/_rerun2/components/keypoint_id_ext.py +++ /dev/null @@ -1,21 +0,0 @@ -from __future__ import annotations - -__all__ = ["KeypointIdArrayExt"] - -from typing import Any, Sequence - -import numpy as np -import pyarrow as pa - - -class KeypointIdArrayExt: - @staticmethod - def _from_similar( - data: Any | None, *, mono: type, mono_aliases: Any, many: type, many_aliases: Any, arrow: type - ) -> pa.Array: - if isinstance(data, Sequence) and (len(data) > 0 and isinstance(data[0], mono)): - array = np.asarray([class_id.id for class_id in data], np.uint16) - else: - array = np.asarray(data, dtype=np.uint16).flatten() - - return arrow().wrap_array(pa.array(array, type=arrow().storage_type)) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/label_ext.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/label_ext.py deleted file mode 100644 index 1a74aef9bb08..000000000000 --- a/rerun_py/rerun_sdk/rerun/_rerun2/components/label_ext.py +++ /dev/null @@ -1,22 +0,0 @@ -from __future__ import annotations - -__all__ = ["LabelArrayExt"] - -from typing import Any, Sequence - -import pyarrow as pa - - -class LabelArrayExt: - @staticmethod - def _from_similar( - data: Any | None, *, mono: type, mono_aliases: Any, many: type, many_aliases: Any, arrow: type - ) -> pa.Array: - if isinstance(data, str): - array = [data] - elif isinstance(data, Sequence): - array = [str(datum) for datum in data] - else: - array = [str(data)] - - return arrow().wrap_array(pa.array(array, type=arrow().storage_type)) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/point2d_ext.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/point2d_ext.py deleted file mode 100644 index 520ba8d78a97..000000000000 --- a/rerun_py/rerun_sdk/rerun/_rerun2/components/point2d_ext.py +++ /dev/null @@ -1,28 +0,0 @@ -from __future__ import annotations - -__all__ = ["Point2DArrayExt"] - -from typing import Any, Sequence - -import numpy as np -import pyarrow as pa - - -class Point2DArrayExt: - @staticmethod - def _from_similar( - data: Any | None, *, mono: type, mono_aliases: Any, many: type, many_aliases: Any, arrow: type - ) -> pa.Array: - if isinstance(data, Sequence) and (len(data) > 0 and isinstance(data[0], mono)): - points = np.concatenate([np.asarray([datum.x, datum.y], dtype=np.float32) for datum in data]) - else: - points = np.asarray(data, dtype=np.float32) - - points = points.reshape((-1, 2)) - - return arrow().wrap_array( - pa.StructArray.from_arrays( - arrays=[pa.array(c, type=pa.float32()) for c in points.T], - fields=list(arrow().storage_type), - ) - ) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/radius_ext.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/radius_ext.py deleted file mode 100644 index 9ec2174cd497..000000000000 --- a/rerun_py/rerun_sdk/rerun/_rerun2/components/radius_ext.py +++ /dev/null @@ -1,21 +0,0 @@ -from __future__ import annotations - -__all__ = ["RadiusArrayExt"] - -from typing import Any, Sequence - -import numpy as np -import pyarrow as pa - - -class RadiusArrayExt: - @staticmethod - def _from_similar( - data: Any | None, *, mono: type, mono_aliases: Any, many: type, many_aliases: Any, arrow: type - ) -> pa.Array: - if isinstance(data, Sequence) and (len(data) > 0 and isinstance(data[0], mono)): - array = np.asarray([radius.value for radius in data], np.float32) - else: - array = np.asarray(data, dtype=np.float32).flatten() - - return arrow().wrap_array(pa.array(array, type=arrow().storage_type)) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/_overrides/__init__.py b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/_overrides/__init__.py new file mode 100644 index 000000000000..fe6c3318a90c --- /dev/null +++ b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/_overrides/__init__.py @@ -0,0 +1,4 @@ +from __future__ import annotations + +from .point2d import point2d_as_array, point2d_native_to_pa_array +from .vec2d import vec2d_native_to_pa_array diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/_overrides/point2d.py b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/_overrides/point2d.py new file mode 100644 index 000000000000..6be1d24fbd97 --- /dev/null +++ b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/_overrides/point2d.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import numpy as np +import pyarrow as pa + +if TYPE_CHECKING: + import numpy.typing as npt + + from .. import Point2D, Point2DArrayLike + + +def point2d_as_array(data: Point2D, dtype: npt.DTypeLike = None) -> npt.ArrayLike: + return np.array([data.x, data.y], dtype=dtype) + + +def point2d_native_to_pa_array(data: Point2DArrayLike, data_type: pa.DataType) -> pa.Array: + points = np.asarray(data, dtype=np.float32).reshape((-1, 2)) + return pa.StructArray.from_arrays( + arrays=[pa.array(c, type=pa.float32()) for c in points.T], + fields=list(data_type), + ) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/_overrides/vec2d.py b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/_overrides/vec2d.py new file mode 100644 index 000000000000..aa4e65ec4e82 --- /dev/null +++ b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/_overrides/vec2d.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import numpy as np +import pyarrow as pa + +if TYPE_CHECKING: + from .. import Vec2DArrayLike + + +def vec2d_native_to_pa_array(data: Vec2DArrayLike, data_type: pa.DataType) -> pa.Array: + points = np.asarray(data, dtype=np.float32).reshape((-1,)) + return pa.FixedSizeListArray.from_arrays(points, type=data_type) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/fuzzy_ext.py b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/fuzzy_ext.py deleted file mode 100644 index bae6a5c15ca0..000000000000 --- a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/fuzzy_ext.py +++ /dev/null @@ -1,26 +0,0 @@ -from __future__ import annotations - -__all__ = [ - "AffixFuzzer1ArrayExt", - "AffixFuzzer2ArrayExt", -] - -from typing import Any - -import pyarrow as pa - - -class AffixFuzzer1ArrayExt: - @staticmethod - def _from_similar( - data: Any | None, *, mono: type, mono_aliases: Any, many: type, many_aliases: Any, arrow: type - ) -> pa.Array: - raise NotImplementedError() - - -class AffixFuzzer2ArrayExt: - @staticmethod - def _from_similar( - data: Any | None, *, mono: type, mono_aliases: Any, many: type, many_aliases: Any, arrow: type - ) -> pa.Array: - raise NotImplementedError() diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/vec2d_ext.py b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/vec2d_ext.py deleted file mode 100644 index 61eda31ffb6f..000000000000 --- a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/vec2d_ext.py +++ /dev/null @@ -1,23 +0,0 @@ -from __future__ import annotations - -__all__ = ["Vec2DArrayExt"] - -from typing import Any, Sequence - -import numpy as np -import pyarrow as pa - - -class Vec2DArrayExt: - @staticmethod - def _from_similar( - data: Any | None, *, mono: type, mono_aliases: Any, many: type, many_aliases: Any, arrow: type - ) -> pa.Array: - if isinstance(data, Sequence) and (len(data) > 0 and isinstance(data[0], mono)): - points = np.concatenate([np.asarray(datum.position, dtype=np.float32) for datum in data]) - else: - points = np.asarray(data, dtype=np.float32) - - points = points.reshape((-1,)) - - return arrow().wrap_array(pa.FixedSizeListArray.from_arrays(points, type=arrow().storage_type)) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/log_any.py b/rerun_py/rerun_sdk/rerun/_rerun2/log_any.py index 6d7c4bbd7bb0..998ccef9925b 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/log_any.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/log_any.py @@ -1,16 +1,16 @@ from __future__ import annotations -from dataclasses import fields from typing import Any, Iterable import numpy as np import numpy.typing as npt import pyarrow as pa +from attrs import fields from .. import RecordingStream, bindings from ..log import error_utils -from .archetypes import Archetype -from .components import Component, InstanceKeyArray +from ._baseclasses import Archetype, NamedExtensionArray +from .components import InstanceKeyArray, InstanceKeyType __all__ = ["log_any"] @@ -22,8 +22,8 @@ # adapted from rerun.log._add_extension_components def _add_extension_components( - instanced: dict[str, Component], - splats: dict[str, Component], + instanced: dict[str, pa.ExtensionArray], + splats: dict[str, pa.ExtensionArray], ext: dict[str, Any], identifiers: npt.NDArray[np.uint64] | None, ) -> None: @@ -65,11 +65,21 @@ def _add_extension_components( instanced[name] = pa_value # noqa -def _extract_components(entity: Archetype) -> Iterable[tuple[Component, bool]]: +def _extract_components(entity: Archetype) -> Iterable[tuple[NamedExtensionArray, bool]]: """Extract the components from an entity, yielding (component, is_primary) tuples.""" - for fld in fields(entity): + for fld in fields(type(entity)): if "component" in fld.metadata: - yield getattr(entity, fld.name), fld.metadata["component"] == "primary" + comp = getattr(entity, fld.name) + if comp is not None: + yield getattr(entity, fld.name), fld.metadata["component"] == "primary" + + +_MAX_U64 = 2**64 - 1 + + +def _splat() -> InstanceKeyArray: + """Helper to generate a splat InstanceKeyArray.""" + return pa.array([_MAX_U64], type=InstanceKeyType().storage_type) # type: ignore[no-any-return] def log_any( @@ -105,8 +115,8 @@ def log_any( if not isinstance(entity, Archetype): raise TypeError(f"Expected Archetype, got {type(entity)}") - instanced: dict[str, Component] = {} - splats: dict[str, Component] = {} + instanced: dict[str, NamedExtensionArray] = {} + splats: dict[str, NamedExtensionArray] = {} # find canonical length of this entity by based on the longest length of any primary component archetype_length = max(len(comp) for comp, primary in _extract_components(entity) if primary) @@ -123,7 +133,7 @@ def log_any( _add_extension_components(instanced, splats, ext, None) if splats: - splats["rerun.instance_key"] = InstanceKeyArray.splat() + splats["rerun.instance_key"] = _splat() bindings.log_arrow_msg(entity_path, components=splats, timeless=timeless, recording=recording) # Always the primary component last so range-based queries will include the other data. See(#1215) diff --git a/rerun_py/tests/unit/test_points2d.py b/rerun_py/tests/unit/test_points2d.py index 0e503df4f357..e7cb01543f1b 100644 --- a/rerun_py/tests/unit/test_points2d.py +++ b/rerun_py/tests/unit/test_points2d.py @@ -4,11 +4,10 @@ import numpy as np import rerun as rr -from rerun._rerun2 import components as rrc # NOTE: uncomment these to get a better auto-completion experience... # from rerun_sdk import rerun2 as rr -# from rerun_sdk.rerun2 import components as rrc +# from rerun_sdk.rerun2 import components as rr.ext # TODO(cmc): roundtrips (serialize in python, deserialize in rust) @@ -17,13 +16,13 @@ def test_points2d() -> None: - points_arrays: list[rrc.Point2DArrayLike] = [ + points_arrays: list[rr.dt.Point2DArrayLike] = [ [], np.array([]), # Point2DArrayLike: Sequence[Point2DLike]: Point2D [ - rrc.Point2D(1, 2), - rrc.Point2D(3, 4), + rr.dt.Point2D(1, 2), + rr.dt.Point2D(3, 4), ], # Point2DArrayLike: Sequence[Point2DLike]: npt.NDArray[np.float32] [ @@ -40,7 +39,7 @@ def test_points2d() -> None: np.array([1, 2, 3, 4], dtype=np.float32), ] - radii_arrays: list[rrc.RadiusArrayLike | None] = [ + radii_arrays: list[rr.cmp.RadiusArrayLike | None] = [ None, [], np.array([]), @@ -48,14 +47,14 @@ def test_points2d() -> None: [42, 43], # RadiusArrayLike: Sequence[RadiusLike]: Radius [ - rrc.Radius(42), - rrc.Radius(43), + rr.cmp.Radius(42), + rr.cmp.Radius(43), ], # RadiusArrayLike: npt.NDArray[np.float32] np.array([42, 43], dtype=np.float32), ] - colors_arrays: list[rrc.ColorArrayLike | None] = [ + colors_arrays: list[rr.cmp.ColorArrayLike | None] = [ None, [], np.array([]), @@ -66,8 +65,8 @@ def test_points2d() -> None: ], # ColorArrayLike: Sequence[ColorLike]: Color [ - rrc.Color(0xAA0000CC), - rrc.Color(0x00BB00DD), + rr.cmp.Color(0xAA0000CC), + rr.cmp.Color(0x00BB00DD), ], # ColorArrayLike: Sequence[ColorLike]: npt.NDArray[np.uint8] np.array( @@ -153,24 +152,24 @@ def test_points2d() -> None: ), ] - labels_arrays: list[rrc.LabelArrayLike | None] = [ + labels_arrays: list[rr.cmp.LabelArrayLike | None] = [ None, [], # LabelArrayLike: Sequence[LabelLike]: str ["hello", "friend"], # LabelArrayLike: Sequence[LabelLike]: Label [ - rrc.Label("hello"), - rrc.Label("friend"), + rr.cmp.Label("hello"), + rr.cmp.Label("friend"), ], ] - draw_orders: list[rrc.DrawOrderLike | None] = [ + draw_orders: list[rr.cmp.DrawOrderLike | None] = [ None, # DrawOrderLike: float 300, # DrawOrderLike: DrawOrder - rrc.DrawOrder(300), + rr.cmp.DrawOrder(300), ] class_id_arrays = [ @@ -179,7 +178,7 @@ def test_points2d() -> None: # ClassIdArrayLike: Sequence[ClassIdLike]: int [126, 127], # ClassIdArrayLike: Sequence[ClassIdLike]: ClassId - [rrc.ClassId(126), rrc.ClassId(127)], + [rr.cmp.ClassId(126), rr.cmp.ClassId(127)], # ClassIdArrayLike: np.NDArray[np.uint8] np.array([126, 127], dtype=np.uint8), # ClassIdArrayLike: np.NDArray[np.uint16] @@ -196,7 +195,7 @@ def test_points2d() -> None: # KeypointIdArrayLike: Sequence[KeypointIdLike]: int [2, 3], # KeypointIdArrayLike: Sequence[KeypointIdLike]: KeypointId - [rrc.KeypointId(2), rrc.KeypointId(3)], + [rr.cmp.KeypointId(2), rr.cmp.KeypointId(3)], # KeypointIdArrayLike: np.NDArray[np.uint8] np.array([2, 3], dtype=np.uint8), # KeypointIdArrayLike: np.NDArray[np.uint16] @@ -213,7 +212,7 @@ def test_points2d() -> None: # InstanceKeyArrayLike: Sequence[InstanceKeyLike]: int [U64_MAX_MINUS_1, U64_MAX], # InstanceKeyArrayLike: Sequence[InstanceKeyLike]: InstanceKey - [rrc.InstanceKey(U64_MAX_MINUS_1), rrc.InstanceKey(U64_MAX)], + [rr.cmp.InstanceKey(U64_MAX_MINUS_1), rr.cmp.InstanceKey(U64_MAX)], # InstanceKeyArrayLike: np.NDArray[np.uint64] np.array([U64_MAX_MINUS_1, U64_MAX], dtype=np.uint64), ] @@ -245,25 +244,25 @@ def test_points2d() -> None: f")" ) arch = rr.Points2D( - points, # type: ignore[arg-type] - radii=radii, # type: ignore[arg-type] - colors=colors, # type: ignore[arg-type] - labels=labels, # type: ignore[arg-type] - draw_order=draw_order, # type: ignore[arg-type] - class_ids=class_ids, # type: ignore[arg-type] - keypoint_ids=keypoint_ids, # type: ignore[arg-type] - instance_keys=instance_keys, # type: ignore[arg-type] + points, + radii=radii, + colors=colors, + labels=labels, + draw_order=draw_order, + class_ids=class_ids, + keypoint_ids=keypoint_ids, + instance_keys=instance_keys, ) print(f"{arch}\n") - assert arch.points == rrc.Point2DArray.from_similar([[1.0, 2.0], [3.0, 4.0]] if non_empty(points) else []) - assert arch.radii == rrc.RadiusArray.from_similar([42, 43] if non_empty(radii) else []) - assert arch.colors == rrc.ColorArray.from_similar([0xAA0000CC, 0x00BB00DD] if non_empty(colors) else []) - assert arch.labels == rrc.LabelArray.from_similar(["hello", "friend"] if non_empty(labels) else []) - assert arch.draw_order == rrc.DrawOrderArray.from_similar([300] if draw_order is not None else []) - assert arch.class_ids == rrc.ClassIdArray.from_similar([126, 127] if non_empty(class_ids) else []) - assert arch.keypoint_ids == rrc.KeypointIdArray.from_similar([2, 3] if non_empty(keypoint_ids) else []) - assert arch.instance_keys == rrc.InstanceKeyArray.from_similar( + assert arch.points == rr.cmp.Point2DArray.from_similar([[1.0, 2.0], [3.0, 4.0]] if non_empty(points) else []) + assert arch.radii == rr.cmp.RadiusArray.from_similar([42, 43] if non_empty(radii) else []) + assert arch.colors == rr.cmp.ColorArray.from_similar([0xAA0000CC, 0x00BB00DD] if non_empty(colors) else []) + assert arch.labels == rr.cmp.LabelArray.from_similar(["hello", "friend"] if non_empty(labels) else []) + assert arch.draw_order == rr.cmp.DrawOrderArray.from_similar([300] if draw_order is not None else []) + assert arch.class_ids == rr.cmp.ClassIdArray.from_similar([126, 127] if non_empty(class_ids) else []) + assert arch.keypoint_ids == rr.cmp.KeypointIdArray.from_similar([2, 3] if non_empty(keypoint_ids) else []) + assert arch.instance_keys == rr.cmp.InstanceKeyArray.from_similar( [U64_MAX_MINUS_1, U64_MAX] if non_empty(instance_keys) else [] ) From b992ee7133806f131b087e73f88fa4289b01fee1 Mon Sep 17 00:00:00 2001 From: Antoine Beyeler Date: Tue, 4 Jul 2023 19:19:17 +0200 Subject: [PATCH 02/17] Codegen --- crates/re_types/source_hash.txt | 2 +- crates/re_types/src/archetypes/fuzzy.rs | 172 +-- crates/re_types/src/components/fuzzy.rs | 1073 ----------------- crates/re_types/src/components/mod.rs | 1 - crates/re_types/src/components/point2d.rs | 135 +-- crates/re_types/src/components/point2d_ext.rs | 84 +- crates/re_types/src/datatypes/mod.rs | 3 + crates/re_types/src/datatypes/point2d.rs | 212 ++++ crates/re_types/src/datatypes/point2d_ext.rs | 51 + .../rerun/_rerun2/archetypes/__init__.py | 5 +- .../rerun/_rerun2/archetypes/fuzzy.py | 236 ++-- .../rerun/_rerun2/archetypes/points2d.py | 106 +- .../rerun/_rerun2/components/__init__.py | 97 +- .../rerun/_rerun2/components/class_id.py | 69 +- .../rerun/_rerun2/components/color.py | 79 +- .../rerun/_rerun2/components/draw_order.py | 69 +- .../rerun/_rerun2/components/fuzzy.py | 621 +--------- .../rerun/_rerun2/components/instance_key.py | 69 +- .../rerun/_rerun2/components/keypoint_id.py | 69 +- .../rerun/_rerun2/components/label.py | 67 +- .../rerun/_rerun2/components/point2d.py | 80 +- .../rerun/_rerun2/components/radius.py | 69 +- .../rerun/_rerun2/datatypes/__init__.py | 35 +- .../rerun/_rerun2/datatypes/fuzzy.py | 130 +- .../rerun/_rerun2/datatypes/point2d.py | 68 ++ .../rerun/_rerun2/datatypes/vec2d.py | 73 +- 26 files changed, 921 insertions(+), 2754 deletions(-) create mode 100644 crates/re_types/src/datatypes/point2d.rs create mode 100644 crates/re_types/src/datatypes/point2d_ext.rs create mode 100644 rerun_py/rerun_sdk/rerun/_rerun2/datatypes/point2d.py diff --git a/crates/re_types/source_hash.txt b/crates/re_types/source_hash.txt index 07a7ae9d92ab..f3d42b1d2026 100644 --- a/crates/re_types/source_hash.txt +++ b/crates/re_types/source_hash.txt @@ -1,4 +1,4 @@ # This is a sha256 hash for all direct and indirect dependencies of this crate's build script. # It can be safely removed at anytime to force the build script to run again. # Check out build.rs to see how it's computed. -5cac23400a6084eaaa53fba54885d12daefa92fdbdb057c8221ec343dcdd9586 \ No newline at end of file +cfb190847b7ffeee14ee498f1f513044c7bdf2ffe8459ac14b39d82fa5c503a0 \ No newline at end of file diff --git a/crates/re_types/src/archetypes/fuzzy.rs b/crates/re_types/src/archetypes/fuzzy.rs index 1f722a7667b6..c7ad3cd1d203 100644 --- a/crates/re_types/src/archetypes/fuzzy.rs +++ b/crates/re_types/src/archetypes/fuzzy.rs @@ -16,96 +16,84 @@ pub struct AffixFuzzer1 { pub fuzz1004: crate::components::AffixFuzzer4, pub fuzz1005: crate::components::AffixFuzzer5, pub fuzz1006: crate::components::AffixFuzzer6, - pub fuzz1007: crate::components::AffixFuzzer7, pub fuzz1101: Vec, pub fuzz1102: Vec, pub fuzz1103: Vec, pub fuzz1104: Vec, pub fuzz1105: Vec, pub fuzz1106: Vec, - pub fuzz1107: Vec, pub fuzz2001: Option, pub fuzz2002: Option, pub fuzz2003: Option, pub fuzz2004: Option, pub fuzz2005: Option, pub fuzz2006: Option, - pub fuzz2007: Option, pub fuzz2101: Option>, pub fuzz2102: Option>, pub fuzz2103: Option>, pub fuzz2104: Option>, pub fuzz2105: Option>, pub fuzz2106: Option>, - pub fuzz2107: Option>, } impl AffixFuzzer1 { - pub const REQUIRED_COMPONENTS: [crate::ComponentName; 14usize] = [ + pub const REQUIRED_COMPONENTS: [crate::ComponentName; 12usize] = [ crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer1"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer2"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer3"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer4"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer5"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer6"), - crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer7"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer1"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer2"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer3"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer4"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer5"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer6"), - crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer7"), ]; pub const RECOMMENDED_COMPONENTS: [crate::ComponentName; 0usize] = []; - pub const OPTIONAL_COMPONENTS: [crate::ComponentName; 14usize] = [ + pub const OPTIONAL_COMPONENTS: [crate::ComponentName; 12usize] = [ crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer1"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer2"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer3"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer4"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer5"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer6"), - crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer7"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer1"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer2"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer3"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer4"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer5"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer6"), - crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer7"), ]; - pub const ALL_COMPONENTS: [crate::ComponentName; 28usize] = [ + pub const ALL_COMPONENTS: [crate::ComponentName; 24usize] = [ crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer1"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer2"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer3"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer4"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer5"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer6"), - crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer7"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer1"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer2"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer3"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer4"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer5"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer6"), - crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer7"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer1"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer2"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer3"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer4"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer5"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer6"), - crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer7"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer1"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer2"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer3"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer4"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer5"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer6"), - crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer7"), ]; } @@ -246,24 +234,6 @@ impl crate::Archetype for AffixFuzzer1 { }) .transpose()? }, - { - Some({ - let array = - ::try_to_arrow([&self.fuzz1007], None); - array.map(|array| { - let datatype = ::arrow2::datatypes::DataType::Extension( - "rerun.testing.components.AffixFuzzer7".into(), - Box::new(array.data_type().clone()), - Some("rerun.testing.components.AffixFuzzer7".into()), - ); - ( - ::arrow2::datatypes::Field::new("fuzz1007", datatype, false), - array, - ) - }) - }) - .transpose()? - }, { Some({ let array = @@ -372,24 +342,6 @@ impl crate::Archetype for AffixFuzzer1 { }) .transpose()? }, - { - Some({ - let array = - ::try_to_arrow(self.fuzz1107.iter(), None); - array.map(|array| { - let datatype = ::arrow2::datatypes::DataType::Extension( - "rerun.testing.components.AffixFuzzer7".into(), - Box::new(array.data_type().clone()), - Some("rerun.testing.components.AffixFuzzer7".into()), - ); - ( - ::arrow2::datatypes::Field::new("fuzz1107", datatype, false), - array, - ) - }) - }) - .transpose()? - }, { self.fuzz2001 .as_ref() @@ -504,25 +456,6 @@ impl crate::Archetype for AffixFuzzer1 { }) .transpose()? }, - { - self.fuzz2007 - .as_ref() - .map(|single| { - let array = ::try_to_arrow([single], None); - array.map(|array| { - let datatype = ::arrow2::datatypes::DataType::Extension( - "rerun.testing.components.AffixFuzzer7".into(), - Box::new(array.data_type().clone()), - Some("rerun.testing.components.AffixFuzzer7".into()), - ); - ( - ::arrow2::datatypes::Field::new("fuzz2007", datatype, false), - array, - ) - }) - }) - .transpose()? - }, { self.fuzz2101 .as_ref() @@ -643,26 +576,6 @@ impl crate::Archetype for AffixFuzzer1 { }) .transpose()? }, - { - self.fuzz2107 - .as_ref() - .map(|many| { - let array = - ::try_to_arrow(many.iter(), None); - array.map(|array| { - let datatype = ::arrow2::datatypes::DataType::Extension( - "rerun.testing.components.AffixFuzzer7".into(), - Box::new(array.data_type().clone()), - Some("rerun.testing.components.AffixFuzzer7".into()), - ); - ( - ::arrow2::datatypes::Field::new("fuzz2107", datatype, false), - array, - ) - }) - }) - .transpose()? - }, ] .into_iter() .flatten() @@ -762,20 +675,6 @@ impl crate::Archetype for AffixFuzzer1 { datatype: ::arrow2::datatypes::DataType::Null, })? }; - let fuzz1007 = { - let array = arrays_by_name.get("fuzz1007").ok_or_else(|| { - crate::DeserializationError::MissingData { - datatype: ::arrow2::datatypes::DataType::Null, - } - })?; - ::try_from_arrow_opt(&**array)? - .into_iter() - .next() - .flatten() - .ok_or_else(|| crate::DeserializationError::MissingData { - datatype: ::arrow2::datatypes::DataType::Null, - })? - }; let fuzz1101 = { let array = arrays_by_name.get("fuzz1101").ok_or_else(|| { crate::DeserializationError::MissingData { @@ -866,21 +765,6 @@ impl crate::Archetype for AffixFuzzer1 { }) .collect::>>()? }; - let fuzz1107 = { - let array = arrays_by_name.get("fuzz1107").ok_or_else(|| { - crate::DeserializationError::MissingData { - datatype: ::arrow2::datatypes::DataType::Null, - } - })?; - ::try_from_arrow_opt(&**array)? - .into_iter() - .map(|v| { - v.ok_or_else(|| crate::DeserializationError::MissingData { - datatype: ::arrow2::datatypes::DataType::Null, - }) - }) - .collect::>>()? - }; let fuzz2001 = if let Some(array) = arrays_by_name.get("fuzz2001") { Some( ::try_from_arrow_opt(&**array)? @@ -959,19 +843,6 @@ impl crate::Archetype for AffixFuzzer1 { } else { None }; - let fuzz2007 = if let Some(array) = arrays_by_name.get("fuzz2007") { - Some( - ::try_from_arrow_opt(&**array)? - .into_iter() - .next() - .flatten() - .ok_or_else(|| crate::DeserializationError::MissingData { - datatype: ::arrow2::datatypes::DataType::Null, - })?, - ) - } else { - None - }; let fuzz2101 = if let Some(array) = arrays_by_name.get("fuzz2101") { Some( ::try_from_arrow_opt(&**array)? @@ -1056,20 +927,6 @@ impl crate::Archetype for AffixFuzzer1 { } else { None }; - let fuzz2107 = if let Some(array) = arrays_by_name.get("fuzz2107") { - Some( - ::try_from_arrow_opt(&**array)? - .into_iter() - .map(|v| { - v.ok_or_else(|| crate::DeserializationError::MissingData { - datatype: ::arrow2::datatypes::DataType::Null, - }) - }) - .collect::>>()?, - ) - } else { - None - }; Ok(Self { fuzz1001, fuzz1002, @@ -1077,28 +934,24 @@ impl crate::Archetype for AffixFuzzer1 { fuzz1004, fuzz1005, fuzz1006, - fuzz1007, fuzz1101, fuzz1102, fuzz1103, fuzz1104, fuzz1105, fuzz1106, - fuzz1107, fuzz2001, fuzz2002, fuzz2003, fuzz2004, fuzz2005, fuzz2006, - fuzz2007, fuzz2101, fuzz2102, fuzz2103, fuzz2104, fuzz2105, fuzz2106, - fuzz2107, }) } } @@ -1111,14 +964,12 @@ impl AffixFuzzer1 { fuzz1004: impl Into, fuzz1005: impl Into, fuzz1006: impl Into, - fuzz1007: impl Into, fuzz1101: impl IntoIterator>, fuzz1102: impl IntoIterator>, fuzz1103: impl IntoIterator>, fuzz1104: impl IntoIterator>, fuzz1105: impl IntoIterator>, fuzz1106: impl IntoIterator>, - fuzz1107: impl IntoIterator>, ) -> Self { Self { fuzz1001: fuzz1001.into(), @@ -1127,28 +978,24 @@ impl AffixFuzzer1 { fuzz1004: fuzz1004.into(), fuzz1005: fuzz1005.into(), fuzz1006: fuzz1006.into(), - fuzz1007: fuzz1007.into(), fuzz1101: fuzz1101.into_iter().map(Into::into).collect(), fuzz1102: fuzz1102.into_iter().map(Into::into).collect(), fuzz1103: fuzz1103.into_iter().map(Into::into).collect(), fuzz1104: fuzz1104.into_iter().map(Into::into).collect(), fuzz1105: fuzz1105.into_iter().map(Into::into).collect(), fuzz1106: fuzz1106.into_iter().map(Into::into).collect(), - fuzz1107: fuzz1107.into_iter().map(Into::into).collect(), fuzz2001: None, fuzz2002: None, fuzz2003: None, fuzz2004: None, fuzz2005: None, fuzz2006: None, - fuzz2007: None, fuzz2101: None, fuzz2102: None, fuzz2103: None, fuzz2104: None, fuzz2105: None, fuzz2106: None, - fuzz2107: None, } } @@ -1182,11 +1029,6 @@ impl AffixFuzzer1 { self } - pub fn with_fuzz2007(mut self, fuzz2007: impl Into) -> Self { - self.fuzz2007 = Some(fuzz2007.into()); - self - } - pub fn with_fuzz2101( mut self, fuzz2101: impl IntoIterator>, @@ -1234,12 +1076,4 @@ impl AffixFuzzer1 { self.fuzz2106 = Some(fuzz2106.into_iter().map(Into::into).collect()); self } - - pub fn with_fuzz2107( - mut self, - fuzz2107: impl IntoIterator>, - ) -> Self { - self.fuzz2107 = Some(fuzz2107.into_iter().map(Into::into).collect()); - self - } } diff --git a/crates/re_types/src/components/fuzzy.rs b/crates/re_types/src/components/fuzzy.rs index e9519edb171b..0086dc5691f3 100644 --- a/crates/re_types/src/components/fuzzy.rs +++ b/crates/re_types/src/components/fuzzy.rs @@ -972,1076 +972,3 @@ impl crate::Component for AffixFuzzer6 { }) } } - -#[derive(Debug, Clone, PartialEq)] -pub struct AffixFuzzer7 { - pub many_optional: Option>, - pub single_float_optional: Option, - pub single_string_required: String, - pub single_string_optional: Option, - pub many_floats_optional: Option>, - pub many_strings_required: Vec, - pub many_strings_optional: Option>, -} - -impl<'a> From for ::std::borrow::Cow<'a, AffixFuzzer7> { - #[inline] - fn from(value: AffixFuzzer7) -> Self { - std::borrow::Cow::Owned(value) - } -} - -impl<'a> From<&'a AffixFuzzer7> for ::std::borrow::Cow<'a, AffixFuzzer7> { - #[inline] - fn from(value: &'a AffixFuzzer7) -> Self { - std::borrow::Cow::Borrowed(value) - } -} - -impl crate::Component for AffixFuzzer7 { - #[inline] - fn name() -> crate::ComponentName { - crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer7") - } - - #[allow(unused_imports, clippy::wildcard_imports)] - #[inline] - fn to_arrow_datatype() -> arrow2::datatypes::DataType { - use ::arrow2::datatypes::*; - DataType::Struct(vec![ - Field { - name: "many_optional".to_owned(), - data_type: DataType::List(Box::new(Field { - name: "item".to_owned(), - data_type: DataType::Struct(vec![ - Field { - name: "single_float_optional".to_owned(), - data_type: DataType::Float32, - is_nullable: true, - metadata: [].into(), - }, - Field { - name: "single_string_required".to_owned(), - data_type: DataType::Utf8, - is_nullable: false, - metadata: [].into(), - }, - Field { - name: "single_string_optional".to_owned(), - data_type: DataType::Utf8, - is_nullable: true, - metadata: [].into(), - }, - Field { - name: "many_floats_optional".to_owned(), - data_type: DataType::List(Box::new(Field { - name: "item".to_owned(), - data_type: DataType::Float32, - is_nullable: true, - metadata: [].into(), - })), - is_nullable: true, - metadata: [].into(), - }, - Field { - name: "many_strings_required".to_owned(), - data_type: DataType::List(Box::new(Field { - name: "item".to_owned(), - data_type: DataType::Utf8, - is_nullable: false, - metadata: [].into(), - })), - is_nullable: false, - metadata: [].into(), - }, - Field { - name: "many_strings_optional".to_owned(), - data_type: DataType::List(Box::new(Field { - name: "item".to_owned(), - data_type: DataType::Utf8, - is_nullable: true, - metadata: [].into(), - })), - is_nullable: true, - metadata: [].into(), - }, - ]), - is_nullable: true, - metadata: [].into(), - })), - is_nullable: true, - metadata: [].into(), - }, - Field { - name: "single_float_optional".to_owned(), - data_type: DataType::Float32, - is_nullable: true, - metadata: [].into(), - }, - Field { - name: "single_string_required".to_owned(), - data_type: DataType::Utf8, - is_nullable: false, - metadata: [].into(), - }, - Field { - name: "single_string_optional".to_owned(), - data_type: DataType::Utf8, - is_nullable: true, - metadata: [].into(), - }, - Field { - name: "many_floats_optional".to_owned(), - data_type: DataType::List(Box::new(Field { - name: "item".to_owned(), - data_type: DataType::Float32, - is_nullable: true, - metadata: [].into(), - })), - is_nullable: true, - metadata: [].into(), - }, - Field { - name: "many_strings_required".to_owned(), - data_type: DataType::List(Box::new(Field { - name: "item".to_owned(), - data_type: DataType::Utf8, - is_nullable: false, - metadata: [].into(), - })), - is_nullable: false, - metadata: [].into(), - }, - Field { - name: "many_strings_optional".to_owned(), - data_type: DataType::List(Box::new(Field { - name: "item".to_owned(), - data_type: DataType::Utf8, - is_nullable: true, - metadata: [].into(), - })), - is_nullable: true, - metadata: [].into(), - }, - ]) - } - - #[allow(unused_imports, clippy::wildcard_imports)] - fn try_to_arrow_opt<'a>( - data: impl IntoIterator>>>, - extension_wrapper: Option<&str>, - ) -> crate::SerializationResult> - where - Self: Clone + 'a, - { - use crate::{Component as _, Datatype as _}; - use ::arrow2::{array::*, datatypes::*}; - Ok({ - let (somes, data): (Vec<_>, Vec<_>) = data - .into_iter() - .map(|datum| { - let datum: Option<::std::borrow::Cow<'a, Self>> = datum.map(Into::into); - (datum.is_some(), datum) - }) - .unzip(); - let bitmap: Option<::arrow2::bitmap::Bitmap> = { - let any_nones = somes.iter().any(|some| !*some); - any_nones.then(|| somes.into()) - }; - StructArray::new( - (if let Some(ext) = extension_wrapper { - DataType::Extension( - ext.to_owned(), - Box::new(::to_arrow_datatype()), - None, - ) - } else { - ::to_arrow_datatype() - }) - .to_logical_type() - .clone(), - vec![ - { - let (somes, many_optional): (Vec<_>, Vec<_>) = data - .iter() - .map(|datum| { - let datum = datum - .as_ref() - .map(|datum| { - let Self { many_optional, .. } = &**datum; - many_optional.clone() - }) - .flatten(); - (datum.is_some(), datum) - }) - .unzip(); - let many_optional_bitmap: Option<::arrow2::bitmap::Bitmap> = { - let any_nones = somes.iter().any(|some| !*some); - any_nones.then(|| somes.into()) - }; - { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let many_optional_inner_data: Vec<_> = many_optional - .iter() - .flatten() - .flatten() - .map(ToOwned::to_owned) - .map(Some) - .collect(); - let many_optional_inner_bitmap: Option<::arrow2::bitmap::Bitmap> = { - let any_nones = - many_optional_inner_data.iter().any(|v| v.is_none()); - any_nones.then(|| { - many_optional_inner_data - .iter() - .map(|v| v.is_some()) - .collect() - }) - }; - let offsets = ::arrow2::offset::Offsets::::try_from_lengths( - many_optional.iter().map(|opt| { - opt.as_ref().map(|datum| datum.len()).unwrap_or_default() - }), - ) - .unwrap() - .into(); - ListArray::new( - { - _ = extension_wrapper; - DataType::List(Box::new(Field { - name: "item".to_owned(), - data_type: DataType::Struct(vec![ - Field { - name: "single_float_optional".to_owned(), - data_type: DataType::Float32, - is_nullable: true, - metadata: [].into(), - }, - Field { - name: "single_string_required".to_owned(), - data_type: DataType::Utf8, - is_nullable: false, - metadata: [].into(), - }, - Field { - name: "single_string_optional".to_owned(), - data_type: DataType::Utf8, - is_nullable: true, - metadata: [].into(), - }, - Field { - name: "many_floats_optional".to_owned(), - data_type: DataType::List(Box::new(Field { - name: "item".to_owned(), - data_type: DataType::Float32, - is_nullable: true, - metadata: [].into(), - })), - is_nullable: true, - metadata: [].into(), - }, - Field { - name: "many_strings_required".to_owned(), - data_type: DataType::List(Box::new(Field { - name: "item".to_owned(), - data_type: DataType::Utf8, - is_nullable: false, - metadata: [].into(), - })), - is_nullable: false, - metadata: [].into(), - }, - Field { - name: "many_strings_optional".to_owned(), - data_type: DataType::List(Box::new(Field { - name: "item".to_owned(), - data_type: DataType::Utf8, - is_nullable: true, - metadata: [].into(), - })), - is_nullable: true, - metadata: [].into(), - }, - ]), - is_nullable: true, - metadata: [].into(), - })) - .to_logical_type() - .clone() - }, - offsets, - { - _ = many_optional_inner_bitmap; - _ = extension_wrapper; - crate::datatypes::AffixFuzzer1::try_to_arrow_opt( - many_optional_inner_data, - None::<&str>, - )? - }, - many_optional_bitmap, - ) - .boxed() - } - }, - { - let (somes, single_float_optional): (Vec<_>, Vec<_>) = data - .iter() - .map(|datum| { - let datum = datum - .as_ref() - .map(|datum| { - let Self { - single_float_optional, - .. - } = &**datum; - single_float_optional.clone() - }) - .flatten(); - (datum.is_some(), datum) - }) - .unzip(); - let single_float_optional_bitmap: Option<::arrow2::bitmap::Bitmap> = { - let any_nones = somes.iter().any(|some| !*some); - any_nones.then(|| somes.into()) - }; - PrimitiveArray::new( - { - _ = extension_wrapper; - DataType::Float32.to_logical_type().clone() - }, - single_float_optional - .into_iter() - .map(|v| v.unwrap_or_default()) - .collect(), - single_float_optional_bitmap, - ) - .boxed() - }, - { - let (somes, single_string_required): (Vec<_>, Vec<_>) = data - .iter() - .map(|datum| { - let datum = datum.as_ref().map(|datum| { - let Self { - single_string_required, - .. - } = &**datum; - single_string_required.clone() - }); - (datum.is_some(), datum) - }) - .unzip(); - let single_string_required_bitmap: Option<::arrow2::bitmap::Bitmap> = { - let any_nones = somes.iter().any(|some| !*some); - any_nones.then(|| somes.into()) - }; - { - let inner_data: ::arrow2::buffer::Buffer = single_string_required - .iter() - .flatten() - .flat_map(|s| s.bytes()) - .collect(); - let offsets = ::arrow2::offset::Offsets::::try_from_lengths( - single_string_required.iter().map(|opt| { - opt.as_ref().map(|datum| datum.len()).unwrap_or_default() - }), - ) - .unwrap() - .into(); - #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - { - _ = extension_wrapper; - DataType::Utf8.to_logical_type().clone() - }, - offsets, - inner_data, - single_string_required_bitmap, - ) - } - .boxed() - } - }, - { - let (somes, single_string_optional): (Vec<_>, Vec<_>) = data - .iter() - .map(|datum| { - let datum = datum - .as_ref() - .map(|datum| { - let Self { - single_string_optional, - .. - } = &**datum; - single_string_optional.clone() - }) - .flatten(); - (datum.is_some(), datum) - }) - .unzip(); - let single_string_optional_bitmap: Option<::arrow2::bitmap::Bitmap> = { - let any_nones = somes.iter().any(|some| !*some); - any_nones.then(|| somes.into()) - }; - { - let inner_data: ::arrow2::buffer::Buffer = single_string_optional - .iter() - .flatten() - .flat_map(|s| s.bytes()) - .collect(); - let offsets = ::arrow2::offset::Offsets::::try_from_lengths( - single_string_optional.iter().map(|opt| { - opt.as_ref().map(|datum| datum.len()).unwrap_or_default() - }), - ) - .unwrap() - .into(); - #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - { - _ = extension_wrapper; - DataType::Utf8.to_logical_type().clone() - }, - offsets, - inner_data, - single_string_optional_bitmap, - ) - } - .boxed() - } - }, - { - let (somes, many_floats_optional): (Vec<_>, Vec<_>) = data - .iter() - .map(|datum| { - let datum = datum - .as_ref() - .map(|datum| { - let Self { - many_floats_optional, - .. - } = &**datum; - many_floats_optional.clone() - }) - .flatten(); - (datum.is_some(), datum) - }) - .unzip(); - let many_floats_optional_bitmap: Option<::arrow2::bitmap::Bitmap> = { - let any_nones = somes.iter().any(|some| !*some); - any_nones.then(|| somes.into()) - }; - { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let many_floats_optional_inner_data: Vec<_> = many_floats_optional - .iter() - .flatten() - .flatten() - .map(ToOwned::to_owned) - .map(Some) - .collect(); - let many_floats_optional_inner_bitmap: Option< - ::arrow2::bitmap::Bitmap, - > = { - let any_nones = - many_floats_optional_inner_data.iter().any(|v| v.is_none()); - any_nones.then(|| { - many_floats_optional_inner_data - .iter() - .map(|v| v.is_some()) - .collect() - }) - }; - let offsets = ::arrow2::offset::Offsets::::try_from_lengths( - many_floats_optional.iter().map(|opt| { - opt.as_ref().map(|datum| datum.len()).unwrap_or_default() - }), - ) - .unwrap() - .into(); - ListArray::new( - { - _ = extension_wrapper; - DataType::List(Box::new(Field { - name: "item".to_owned(), - data_type: DataType::Float32, - is_nullable: true, - metadata: [].into(), - })) - .to_logical_type() - .clone() - }, - offsets, - PrimitiveArray::new( - { - _ = extension_wrapper; - DataType::Float32.to_logical_type().clone() - }, - many_floats_optional_inner_data - .into_iter() - .map(|v| v.unwrap_or_default()) - .collect(), - many_floats_optional_inner_bitmap, - ) - .boxed(), - many_floats_optional_bitmap, - ) - .boxed() - } - }, - { - let (somes, many_strings_required): (Vec<_>, Vec<_>) = data - .iter() - .map(|datum| { - let datum = datum.as_ref().map(|datum| { - let Self { - many_strings_required, - .. - } = &**datum; - many_strings_required.clone() - }); - (datum.is_some(), datum) - }) - .unzip(); - let many_strings_required_bitmap: Option<::arrow2::bitmap::Bitmap> = { - let any_nones = somes.iter().any(|some| !*some); - any_nones.then(|| somes.into()) - }; - { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let many_strings_required_inner_data: Vec<_> = many_strings_required - .iter() - .flatten() - .flatten() - .map(ToOwned::to_owned) - .map(Some) - .collect(); - let many_strings_required_inner_bitmap: Option< - ::arrow2::bitmap::Bitmap, - > = { - let any_nones = - many_strings_required_inner_data.iter().any(|v| v.is_none()); - any_nones.then(|| { - many_strings_required_inner_data - .iter() - .map(|v| v.is_some()) - .collect() - }) - }; - let offsets = ::arrow2::offset::Offsets::::try_from_lengths( - many_strings_required.iter().map(|opt| { - opt.as_ref().map(|datum| datum.len()).unwrap_or_default() - }), - ) - .unwrap() - .into(); - ListArray::new( - { - _ = extension_wrapper; - DataType::List(Box::new(Field { - name: "item".to_owned(), - data_type: DataType::Utf8, - is_nullable: false, - metadata: [].into(), - })) - .to_logical_type() - .clone() - }, - offsets, - { - let inner_data: ::arrow2::buffer::Buffer = - many_strings_required_inner_data - .iter() - .flatten() - .flat_map(|s| s.bytes()) - .collect(); - let offsets = - ::arrow2::offset::Offsets::::try_from_lengths( - many_strings_required_inner_data.iter().map(|opt| { - opt.as_ref() - .map(|datum| datum.len()) - .unwrap_or_default() - }), - ) - .unwrap() - .into(); - #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - { - _ = extension_wrapper; - DataType::Utf8.to_logical_type().clone() - }, - offsets, - inner_data, - many_strings_required_inner_bitmap, - ) - } - .boxed() - }, - many_strings_required_bitmap, - ) - .boxed() - } - }, - { - let (somes, many_strings_optional): (Vec<_>, Vec<_>) = data - .iter() - .map(|datum| { - let datum = datum - .as_ref() - .map(|datum| { - let Self { - many_strings_optional, - .. - } = &**datum; - many_strings_optional.clone() - }) - .flatten(); - (datum.is_some(), datum) - }) - .unzip(); - let many_strings_optional_bitmap: Option<::arrow2::bitmap::Bitmap> = { - let any_nones = somes.iter().any(|some| !*some); - any_nones.then(|| somes.into()) - }; - { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let many_strings_optional_inner_data: Vec<_> = many_strings_optional - .iter() - .flatten() - .flatten() - .map(ToOwned::to_owned) - .map(Some) - .collect(); - let many_strings_optional_inner_bitmap: Option< - ::arrow2::bitmap::Bitmap, - > = { - let any_nones = - many_strings_optional_inner_data.iter().any(|v| v.is_none()); - any_nones.then(|| { - many_strings_optional_inner_data - .iter() - .map(|v| v.is_some()) - .collect() - }) - }; - let offsets = ::arrow2::offset::Offsets::::try_from_lengths( - many_strings_optional.iter().map(|opt| { - opt.as_ref().map(|datum| datum.len()).unwrap_or_default() - }), - ) - .unwrap() - .into(); - ListArray::new( - { - _ = extension_wrapper; - DataType::List(Box::new(Field { - name: "item".to_owned(), - data_type: DataType::Utf8, - is_nullable: true, - metadata: [].into(), - })) - .to_logical_type() - .clone() - }, - offsets, - { - let inner_data: ::arrow2::buffer::Buffer = - many_strings_optional_inner_data - .iter() - .flatten() - .flat_map(|s| s.bytes()) - .collect(); - let offsets = - ::arrow2::offset::Offsets::::try_from_lengths( - many_strings_optional_inner_data.iter().map(|opt| { - opt.as_ref() - .map(|datum| datum.len()) - .unwrap_or_default() - }), - ) - .unwrap() - .into(); - #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - { - _ = extension_wrapper; - DataType::Utf8.to_logical_type().clone() - }, - offsets, - inner_data, - many_strings_optional_inner_bitmap, - ) - } - .boxed() - }, - many_strings_optional_bitmap, - ) - .boxed() - } - }, - ], - bitmap, - ) - .boxed() - }) - } - - #[allow(unused_imports, clippy::wildcard_imports)] - fn try_from_arrow_opt( - data: &dyn ::arrow2::array::Array, - ) -> crate::DeserializationResult>> - where - Self: Sized, - { - use crate::{Component as _, Datatype as _}; - use ::arrow2::{array::*, datatypes::*}; - Ok({ - let data = data - .as_any() - .downcast_ref::<::arrow2::array::StructArray>() - .ok_or_else(|| crate::DeserializationError::SchemaMismatch { - expected: data.data_type().clone(), - got: data.data_type().clone(), - })?; - let (data_fields, data_arrays, data_bitmap) = - (data.fields(), data.values(), data.validity()); - let is_valid = |i| data_bitmap.map_or(true, |bitmap| bitmap.get_bit(i)); - let arrays_by_name: ::std::collections::HashMap<_, _> = data_fields - .iter() - .map(|field| field.name.as_str()) - .zip(data_arrays) - .collect(); - let many_optional = { - let data = &**arrays_by_name["many_optional"]; - - { - let datatype = data.data_type(); - let data = data - .as_any() - .downcast_ref::<::arrow2::array::ListArray>() - .unwrap(); - let bitmap = data.validity().cloned(); - let offsets = { - let offsets = data.offsets(); - offsets.iter().copied().zip(offsets.iter().copied().skip(1)) - }; - let data = &**data.values(); - let data = crate::datatypes::AffixFuzzer1::try_from_arrow_opt(data)? - .into_iter() - .map(|v| { - v.ok_or_else(|| crate::DeserializationError::MissingData { - datatype: DataType::Struct(vec![ - Field { - name: "single_float_optional".to_owned(), - data_type: DataType::Float32, - is_nullable: true, - metadata: [].into(), - }, - Field { - name: "single_string_required".to_owned(), - data_type: DataType::Utf8, - is_nullable: false, - metadata: [].into(), - }, - Field { - name: "single_string_optional".to_owned(), - data_type: DataType::Utf8, - is_nullable: true, - metadata: [].into(), - }, - Field { - name: "many_floats_optional".to_owned(), - data_type: DataType::List(Box::new(Field { - name: "item".to_owned(), - data_type: DataType::Float32, - is_nullable: true, - metadata: [].into(), - })), - is_nullable: true, - metadata: [].into(), - }, - Field { - name: "many_strings_required".to_owned(), - data_type: DataType::List(Box::new(Field { - name: "item".to_owned(), - data_type: DataType::Utf8, - is_nullable: false, - metadata: [].into(), - })), - is_nullable: false, - metadata: [].into(), - }, - Field { - name: "many_strings_optional".to_owned(), - data_type: DataType::List(Box::new(Field { - name: "item".to_owned(), - data_type: DataType::Utf8, - is_nullable: true, - metadata: [].into(), - })), - is_nullable: true, - metadata: [].into(), - }, - ]), - }) - }) - .collect::>>()?; - offsets - .enumerate() - .map(move |(i, (start, end))| { - bitmap - .as_ref() - .map_or(true, |bitmap| bitmap.get_bit(i)) - .then(|| { - Ok(data - .get(start as usize..end as usize) - .ok_or_else(|| { - crate::DeserializationError::OffsetsMismatch { - bounds: (start as usize, end as usize), - len: data.len(), - datatype: datatype.clone(), - } - })? - .to_vec()) - }) - .transpose() - }) - .collect::>>>()? - .into_iter() - } - }; - let single_float_optional = { - let data = &**arrays_by_name["single_float_optional"]; - - data.as_any() - .downcast_ref::() - .unwrap() - .into_iter() - .map(|v| v.copied()) - }; - let single_string_required = { - let data = &**arrays_by_name["single_string_required"]; - - data.as_any() - .downcast_ref::>() - .unwrap() - .into_iter() - .map(|v| v.map(ToOwned::to_owned)) - }; - let single_string_optional = { - let data = &**arrays_by_name["single_string_optional"]; - - data.as_any() - .downcast_ref::>() - .unwrap() - .into_iter() - .map(|v| v.map(ToOwned::to_owned)) - }; - let many_floats_optional = { - let data = &**arrays_by_name["many_floats_optional"]; - - { - let datatype = data.data_type(); - let data = data - .as_any() - .downcast_ref::<::arrow2::array::ListArray>() - .unwrap(); - let bitmap = data.validity().cloned(); - let offsets = { - let offsets = data.offsets(); - offsets.iter().copied().zip(offsets.iter().copied().skip(1)) - }; - let data = &**data.values(); - let data = data - .as_any() - .downcast_ref::() - .unwrap() - .into_iter() - .map(|v| v.copied()) - .map(|v| { - v.ok_or_else(|| crate::DeserializationError::MissingData { - datatype: DataType::Float32, - }) - }) - .collect::>>()?; - offsets - .enumerate() - .map(move |(i, (start, end))| { - bitmap - .as_ref() - .map_or(true, |bitmap| bitmap.get_bit(i)) - .then(|| { - Ok(data - .get(start as usize..end as usize) - .ok_or_else(|| { - crate::DeserializationError::OffsetsMismatch { - bounds: (start as usize, end as usize), - len: data.len(), - datatype: datatype.clone(), - } - })? - .to_vec()) - }) - .transpose() - }) - .collect::>>>()? - .into_iter() - } - }; - let many_strings_required = { - let data = &**arrays_by_name["many_strings_required"]; - - { - let datatype = data.data_type(); - let data = data - .as_any() - .downcast_ref::<::arrow2::array::ListArray>() - .unwrap(); - let bitmap = data.validity().cloned(); - let offsets = { - let offsets = data.offsets(); - offsets.iter().copied().zip(offsets.iter().copied().skip(1)) - }; - let data = &**data.values(); - let data = data - .as_any() - .downcast_ref::>() - .unwrap() - .into_iter() - .map(|v| v.map(ToOwned::to_owned)) - .map(|v| { - v.ok_or_else(|| crate::DeserializationError::MissingData { - datatype: DataType::Utf8, - }) - }) - .collect::>>()?; - offsets - .enumerate() - .map(move |(i, (start, end))| { - bitmap - .as_ref() - .map_or(true, |bitmap| bitmap.get_bit(i)) - .then(|| { - Ok(data - .get(start as usize..end as usize) - .ok_or_else(|| { - crate::DeserializationError::OffsetsMismatch { - bounds: (start as usize, end as usize), - len: data.len(), - datatype: datatype.clone(), - } - })? - .to_vec()) - }) - .transpose() - }) - .collect::>>>()? - .into_iter() - } - }; - let many_strings_optional = { - let data = &**arrays_by_name["many_strings_optional"]; - - { - let datatype = data.data_type(); - let data = data - .as_any() - .downcast_ref::<::arrow2::array::ListArray>() - .unwrap(); - let bitmap = data.validity().cloned(); - let offsets = { - let offsets = data.offsets(); - offsets.iter().copied().zip(offsets.iter().copied().skip(1)) - }; - let data = &**data.values(); - let data = data - .as_any() - .downcast_ref::>() - .unwrap() - .into_iter() - .map(|v| v.map(ToOwned::to_owned)) - .map(|v| { - v.ok_or_else(|| crate::DeserializationError::MissingData { - datatype: DataType::Utf8, - }) - }) - .collect::>>()?; - offsets - .enumerate() - .map(move |(i, (start, end))| { - bitmap - .as_ref() - .map_or(true, |bitmap| bitmap.get_bit(i)) - .then(|| { - Ok(data - .get(start as usize..end as usize) - .ok_or_else(|| { - crate::DeserializationError::OffsetsMismatch { - bounds: (start as usize, end as usize), - len: data.len(), - datatype: datatype.clone(), - } - })? - .to_vec()) - }) - .transpose() - }) - .collect::>>>()? - .into_iter() - } - }; - ::itertools::izip!( - many_optional, - single_float_optional, - single_string_required, - single_string_optional, - many_floats_optional, - many_strings_required, - many_strings_optional - ) - .enumerate() - .map( - |( - i, - ( - many_optional, - single_float_optional, - single_string_required, - single_string_optional, - many_floats_optional, - many_strings_required, - many_strings_optional, - ), - )| { - is_valid(i) - .then(|| { - Ok(Self { - many_optional, - single_float_optional, - single_string_required: single_string_required.ok_or_else( - || crate::DeserializationError::MissingData { - datatype: data.data_type().clone(), - }, - )?, - single_string_optional, - many_floats_optional, - many_strings_required: many_strings_required.ok_or_else(|| { - crate::DeserializationError::MissingData { - datatype: data.data_type().clone(), - } - })?, - many_strings_optional, - }) - }) - .transpose() - }, - ) - .collect::>>()? - }) - } -} diff --git a/crates/re_types/src/components/mod.rs b/crates/re_types/src/components/mod.rs index 26a3b9cdeead..fc1d3e59e9e8 100644 --- a/crates/re_types/src/components/mod.rs +++ b/crates/re_types/src/components/mod.rs @@ -23,7 +23,6 @@ pub use self::color::Color; pub use self::draw_order::DrawOrder; pub use self::fuzzy::{ AffixFuzzer1, AffixFuzzer2, AffixFuzzer3, AffixFuzzer4, AffixFuzzer5, AffixFuzzer6, - AffixFuzzer7, }; pub use self::instance_key::InstanceKey; pub use self::keypoint_id::KeypointId; diff --git a/crates/re_types/src/components/point2d.rs b/crates/re_types/src/components/point2d.rs index 5064558b5f30..8990300c4f91 100644 --- a/crates/re_types/src/components/point2d.rs +++ b/crates/re_types/src/components/point2d.rs @@ -11,8 +11,7 @@ #[doc = "A point in 2D space."] #[derive(Debug, Default, Clone, Copy, PartialEq, PartialOrd)] pub struct Point2D { - pub x: f32, - pub y: f32, + pub xy: crate::datatypes::Point2D, } impl<'a> From for ::std::borrow::Cow<'a, Point2D> { @@ -39,20 +38,25 @@ impl crate::Component for Point2D { #[inline] fn to_arrow_datatype() -> arrow2::datatypes::DataType { use ::arrow2::datatypes::*; - DataType::Struct(vec![ - Field { - name: "x".to_owned(), - data_type: DataType::Float32, - is_nullable: false, - metadata: [].into(), - }, - Field { - name: "y".to_owned(), - data_type: DataType::Float32, - is_nullable: false, - metadata: [].into(), - }, - ]) + DataType::Struct(vec![Field { + name: "xy".to_owned(), + data_type: DataType::Struct(vec![ + Field { + name: "x".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "y".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }, + ]), + is_nullable: false, + metadata: [].into(), + }]) } #[allow(unused_imports, clippy::wildcard_imports)] @@ -89,58 +93,27 @@ impl crate::Component for Point2D { }) .to_logical_type() .clone(), - vec![ - { - let (somes, x): (Vec<_>, Vec<_>) = data - .iter() - .map(|datum| { - let datum = datum.as_ref().map(|datum| { - let Self { x, .. } = &**datum; - x.clone() - }); - (datum.is_some(), datum) - }) - .unzip(); - let x_bitmap: Option<::arrow2::bitmap::Bitmap> = { - let any_nones = somes.iter().any(|some| !*some); - any_nones.then(|| somes.into()) - }; - PrimitiveArray::new( - { - _ = extension_wrapper; - DataType::Float32.to_logical_type().clone() - }, - x.into_iter().map(|v| v.unwrap_or_default()).collect(), - x_bitmap, - ) - .boxed() - }, + vec![{ + let (somes, xy): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = datum.as_ref().map(|datum| { + let Self { xy, .. } = &**datum; + xy.clone() + }); + (datum.is_some(), datum) + }) + .unzip(); + let xy_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; { - let (somes, y): (Vec<_>, Vec<_>) = data - .iter() - .map(|datum| { - let datum = datum.as_ref().map(|datum| { - let Self { y, .. } = &**datum; - y.clone() - }); - (datum.is_some(), datum) - }) - .unzip(); - let y_bitmap: Option<::arrow2::bitmap::Bitmap> = { - let any_nones = somes.iter().any(|some| !*some); - any_nones.then(|| somes.into()) - }; - PrimitiveArray::new( - { - _ = extension_wrapper; - DataType::Float32.to_logical_type().clone() - }, - y.into_iter().map(|v| v.unwrap_or_default()).collect(), - y_bitmap, - ) - .boxed() - }, - ], + _ = xy_bitmap; + _ = extension_wrapper; + crate::datatypes::Point2D::try_to_arrow_opt(xy, None::<&str>)? + } + }], bitmap, ) .boxed() @@ -172,34 +145,18 @@ impl crate::Component for Point2D { .map(|field| field.name.as_str()) .zip(data_arrays) .collect(); - let x = { - let data = &**arrays_by_name["x"]; + let xy = { + let data = &**arrays_by_name["xy"]; - data.as_any() - .downcast_ref::() - .unwrap() - .into_iter() - .map(|v| v.copied()) + crate::datatypes::Point2D::try_from_arrow_opt(data)?.into_iter() }; - let y = { - let data = &**arrays_by_name["y"]; - - data.as_any() - .downcast_ref::() - .unwrap() - .into_iter() - .map(|v| v.copied()) - }; - ::itertools::izip!(x, y) + ::itertools::izip!(xy) .enumerate() - .map(|(i, (x, y))| { + .map(|(i, (xy))| { is_valid(i) .then(|| { Ok(Self { - x: x.ok_or_else(|| crate::DeserializationError::MissingData { - datatype: data.data_type().clone(), - })?, - y: y.ok_or_else(|| crate::DeserializationError::MissingData { + xy: xy.ok_or_else(|| crate::DeserializationError::MissingData { datatype: data.data_type().clone(), })?, }) diff --git a/crates/re_types/src/components/point2d_ext.rs b/crates/re_types/src/components/point2d_ext.rs index 663a80bf1267..ec8c4fde9d9f 100644 --- a/crates/re_types/src/components/point2d_ext.rs +++ b/crates/re_types/src/components/point2d_ext.rs @@ -1,26 +1,40 @@ use super::Point2D; +// --- + impl Point2D { pub const ZERO: Self = Self::new(0.0, 0.0); pub const ONE: Self = Self::new(1.0, 1.0); #[inline] pub const fn new(x: f32, y: f32) -> Self { - Self { x, y } + Self { + xy: crate::datatypes::Point2D::new(x, y), + } + } + + #[inline] + pub fn x(&self) -> f32 { + self.xy.x + } + + #[inline] + pub fn y(&self) -> f32 { + self.xy.y } } impl From<(f32, f32)> for Point2D { #[inline] fn from((x, y): (f32, f32)) -> Self { - Self { x, y } + Self::new(x, y) } } impl From<[f32; 2]> for Point2D { #[inline] fn from([x, y]: [f32; 2]) -> Self { - Self { x, y } + Self::new(x, y) } } @@ -36,7 +50,7 @@ impl From for Point2D { impl From for glam::Vec2 { #[inline] fn from(pt: Point2D) -> Self { - Self::new(pt.x, pt.y) + Self::new(pt.x(), pt.y()) } } @@ -44,66 +58,6 @@ impl From for glam::Vec2 { impl From for glam::Vec3 { #[inline] fn from(pt: Point2D) -> Self { - Self::new(pt.x, pt.y, 0.0) + Self::new(pt.x(), pt.y(), 0.0) } } - -// --- - -// TODO(cmc): use the following definition instead once we've finalized the switch to HOPE. - -// impl Point2D { -// pub const ZERO: Self = Self::new(0.0, 0.0); -// pub const ONE: Self = Self::new(1.0, 1.0); - -// #[inline] -// pub const fn new(x: f32, y: f32) -> Self { -// Self(crate::datatypes::Vec2D::new(x, y)) -// } -// } - -// impl std::ops::Deref for Point2D { -// type Target = crate::datatypes::Vec2D; - -// fn deref(&self) -> &Self::Target { -// &self.0 -// } -// } - -// impl From<(f32, f32)> for Point2D { -// #[inline] -// fn from(xy: (f32, f32)) -> Self { -// Self(xy.into()) -// } -// } - -// impl From<[f32; 2]> for Point2D { -// #[inline] -// fn from(p: [f32; 2]) -> Self { -// Self(p.into()) -// } -// } - -// #[cfg(feature = "glam")] -// impl From for Point2D { -// #[inline] -// fn from(pt: glam::Vec2) -> Self { -// Self::new(pt.x, pt.y) -// } -// } - -// #[cfg(feature = "glam")] -// impl From for glam::Vec2 { -// #[inline] -// fn from(pt: Point2D) -> Self { -// Self::new(pt.x(), pt.y()) -// } -// } - -// #[cfg(feature = "glam")] -// impl From for glam::Vec3 { -// #[inline] -// fn from(pt: Point2D) -> Self { -// Self::new(pt.x(), pt.y(), 0.0) -// } -// } diff --git a/crates/re_types/src/datatypes/mod.rs b/crates/re_types/src/datatypes/mod.rs index 5e5217d24e24..6567c6a2fc17 100644 --- a/crates/re_types/src/datatypes/mod.rs +++ b/crates/re_types/src/datatypes/mod.rs @@ -1,8 +1,11 @@ // NOTE: This file was autogenerated by re_types_builder; DO NOT EDIT. mod fuzzy; +mod point2d; +mod point2d_ext; mod vec2d; mod vec2d_ext; pub use self::fuzzy::{AffixFuzzer1, AffixFuzzer2}; +pub use self::point2d::Point2D; pub use self::vec2d::Vec2D; diff --git a/crates/re_types/src/datatypes/point2d.rs b/crates/re_types/src/datatypes/point2d.rs new file mode 100644 index 000000000000..98abf164c17b --- /dev/null +++ b/crates/re_types/src/datatypes/point2d.rs @@ -0,0 +1,212 @@ +// NOTE: This file was autogenerated by re_types_builder; DO NOT EDIT. + +#![allow(trivial_numeric_casts)] +#![allow(unused_parens)] +#![allow(clippy::clone_on_copy)] +#![allow(clippy::map_flatten)] +#![allow(clippy::needless_question_mark)] +#![allow(clippy::too_many_arguments)] +#![allow(clippy::unnecessary_cast)] + +#[doc = "A point in 2D space."] +#[derive(Debug, Default, Clone, Copy, PartialEq, PartialOrd)] +pub struct Point2D { + pub x: f32, + pub y: f32, +} + +impl<'a> From for ::std::borrow::Cow<'a, Point2D> { + #[inline] + fn from(value: Point2D) -> Self { + std::borrow::Cow::Owned(value) + } +} + +impl<'a> From<&'a Point2D> for ::std::borrow::Cow<'a, Point2D> { + #[inline] + fn from(value: &'a Point2D) -> Self { + std::borrow::Cow::Borrowed(value) + } +} + +impl crate::Datatype for Point2D { + #[inline] + fn name() -> crate::DatatypeName { + crate::DatatypeName::Borrowed("rerun.datatypes.Point2D") + } + + #[allow(unused_imports, clippy::wildcard_imports)] + #[inline] + fn to_arrow_datatype() -> arrow2::datatypes::DataType { + use ::arrow2::datatypes::*; + DataType::Struct(vec![ + Field { + name: "x".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "y".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }, + ]) + } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_to_arrow_opt<'a>( + data: impl IntoIterator>>>, + extension_wrapper: Option<&str>, + ) -> crate::SerializationResult> + where + Self: Clone + 'a, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let (somes, data): (Vec<_>, Vec<_>) = data + .into_iter() + .map(|datum| { + let datum: Option<::std::borrow::Cow<'a, Self>> = datum.map(Into::into); + (datum.is_some(), datum) + }) + .unzip(); + let bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + StructArray::new( + (if let Some(ext) = extension_wrapper { + DataType::Extension( + ext.to_owned(), + Box::new(::to_arrow_datatype()), + None, + ) + } else { + ::to_arrow_datatype() + }) + .to_logical_type() + .clone(), + vec![ + { + let (somes, x): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = datum.as_ref().map(|datum| { + let Self { x, .. } = &**datum; + x.clone() + }); + (datum.is_some(), datum) + }) + .unzip(); + let x_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + PrimitiveArray::new( + { + _ = extension_wrapper; + DataType::Float32.to_logical_type().clone() + }, + x.into_iter().map(|v| v.unwrap_or_default()).collect(), + x_bitmap, + ) + .boxed() + }, + { + let (somes, y): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = datum.as_ref().map(|datum| { + let Self { y, .. } = &**datum; + y.clone() + }); + (datum.is_some(), datum) + }) + .unzip(); + let y_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + PrimitiveArray::new( + { + _ = extension_wrapper; + DataType::Float32.to_logical_type().clone() + }, + y.into_iter().map(|v| v.unwrap_or_default()).collect(), + y_bitmap, + ) + .boxed() + }, + ], + bitmap, + ) + .boxed() + }) + } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_from_arrow_opt( + data: &dyn ::arrow2::array::Array, + ) -> crate::DeserializationResult>> + where + Self: Sized, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let data = data + .as_any() + .downcast_ref::<::arrow2::array::StructArray>() + .ok_or_else(|| crate::DeserializationError::SchemaMismatch { + expected: data.data_type().clone(), + got: data.data_type().clone(), + })?; + let (data_fields, data_arrays, data_bitmap) = + (data.fields(), data.values(), data.validity()); + let is_valid = |i| data_bitmap.map_or(true, |bitmap| bitmap.get_bit(i)); + let arrays_by_name: ::std::collections::HashMap<_, _> = data_fields + .iter() + .map(|field| field.name.as_str()) + .zip(data_arrays) + .collect(); + let x = { + let data = &**arrays_by_name["x"]; + + data.as_any() + .downcast_ref::() + .unwrap() + .into_iter() + .map(|v| v.copied()) + }; + let y = { + let data = &**arrays_by_name["y"]; + + data.as_any() + .downcast_ref::() + .unwrap() + .into_iter() + .map(|v| v.copied()) + }; + ::itertools::izip!(x, y) + .enumerate() + .map(|(i, (x, y))| { + is_valid(i) + .then(|| { + Ok(Self { + x: x.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: data.data_type().clone(), + })?, + y: y.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: data.data_type().clone(), + })?, + }) + }) + .transpose() + }) + .collect::>>()? + }) + } +} diff --git a/crates/re_types/src/datatypes/point2d_ext.rs b/crates/re_types/src/datatypes/point2d_ext.rs new file mode 100644 index 000000000000..e81c42ae0a0f --- /dev/null +++ b/crates/re_types/src/datatypes/point2d_ext.rs @@ -0,0 +1,51 @@ +use super::Point2D; + +// --- + +impl Point2D { + pub const ZERO: Self = Self::new(0.0, 0.0); + pub const ONE: Self = Self::new(1.0, 1.0); + + #[inline] + pub const fn new(x: f32, y: f32) -> Self { + Self { x, y } + } +} + +impl From<(f32, f32)> for Point2D { + #[inline] + fn from((x, y): (f32, f32)) -> Self { + Self { x, y } + } +} + +impl From<[f32; 2]> for Point2D { + #[inline] + fn from([x, y]: [f32; 2]) -> Self { + Self { x, y } + } +} + +#[cfg(feature = "glam")] +impl From for Point2D { + #[inline] + fn from(pt: glam::Vec2) -> Self { + Self::new(pt.x, pt.y) + } +} + +#[cfg(feature = "glam")] +impl From for glam::Vec2 { + #[inline] + fn from(pt: Point2D) -> Self { + Self::new(pt.x, pt.y) + } +} + +#[cfg(feature = "glam")] +impl From for glam::Vec3 { + #[inline] + fn from(pt: Point2D) -> Self { + Self::new(pt.x, pt.y, 0.0) + } +} diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/archetypes/__init__.py b/rerun_py/rerun_sdk/rerun/_rerun2/archetypes/__init__.py index b718bbc1bf28..204f3071334a 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/archetypes/__init__.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/archetypes/__init__.py @@ -2,8 +2,7 @@ from __future__ import annotations -__all__ = ["Archetype", "AffixFuzzer1", "Points2D"] - -from .._baseclasses import Archetype from .fuzzy import AffixFuzzer1 from .points2d import Points2D + +__all__ = ["AffixFuzzer1", "Points2D"] diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/archetypes/fuzzy.py b/rerun_py/rerun_sdk/rerun/_rerun2/archetypes/fuzzy.py index 131509f65096..efccca567ffa 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/archetypes/fuzzy.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/archetypes/fuzzy.py @@ -2,129 +2,125 @@ from __future__ import annotations -from dataclasses import dataclass, field - -from .._baseclasses import Archetype - -__all__ = ["AffixFuzzer1"] +from attrs import define, field from .. import components +from .._baseclasses import ( + Archetype, +) -## --- AffixFuzzer1 --- ## +__all__ = ["AffixFuzzer1"] -@dataclass +@define(str=False, repr=False) class AffixFuzzer1(Archetype): - fuzz1001: components.AffixFuzzer1Array = field(metadata={"component": "primary"}) - fuzz1002: components.AffixFuzzer2Array = field(metadata={"component": "primary"}) - fuzz1003: components.AffixFuzzer3Array = field(metadata={"component": "primary"}) - fuzz1004: components.AffixFuzzer4Array = field(metadata={"component": "primary"}) - fuzz1005: components.AffixFuzzer5Array = field(metadata={"component": "primary"}) - fuzz1006: components.AffixFuzzer6Array = field(metadata={"component": "primary"}) - fuzz1007: components.AffixFuzzer7Array = field(metadata={"component": "primary"}) - fuzz1101: components.AffixFuzzer1Array = field(metadata={"component": "primary"}) - fuzz1102: components.AffixFuzzer2Array = field(metadata={"component": "primary"}) - fuzz1103: components.AffixFuzzer3Array = field(metadata={"component": "primary"}) - fuzz1104: components.AffixFuzzer4Array = field(metadata={"component": "primary"}) - fuzz1105: components.AffixFuzzer5Array = field(metadata={"component": "primary"}) - fuzz1106: components.AffixFuzzer6Array = field(metadata={"component": "primary"}) - fuzz1107: components.AffixFuzzer7Array = field(metadata={"component": "primary"}) - fuzz2001: components.AffixFuzzer1Array | None = field(default=None, metadata={"component": "secondary"}) - fuzz2002: components.AffixFuzzer2Array | None = field(default=None, metadata={"component": "secondary"}) - fuzz2003: components.AffixFuzzer3Array | None = field(default=None, metadata={"component": "secondary"}) - fuzz2004: components.AffixFuzzer4Array | None = field(default=None, metadata={"component": "secondary"}) - fuzz2005: components.AffixFuzzer5Array | None = field(default=None, metadata={"component": "secondary"}) - fuzz2006: components.AffixFuzzer6Array | None = field(default=None, metadata={"component": "secondary"}) - fuzz2007: components.AffixFuzzer7Array | None = field(default=None, metadata={"component": "secondary"}) - fuzz2101: components.AffixFuzzer1Array | None = field(default=None, metadata={"component": "secondary"}) - fuzz2102: components.AffixFuzzer2Array | None = field(default=None, metadata={"component": "secondary"}) - fuzz2103: components.AffixFuzzer3Array | None = field(default=None, metadata={"component": "secondary"}) - fuzz2104: components.AffixFuzzer4Array | None = field(default=None, metadata={"component": "secondary"}) - fuzz2105: components.AffixFuzzer5Array | None = field(default=None, metadata={"component": "secondary"}) - fuzz2106: components.AffixFuzzer6Array | None = field(default=None, metadata={"component": "secondary"}) - fuzz2107: components.AffixFuzzer7Array | None = field(default=None, metadata={"component": "secondary"}) - - def __str__(self) -> str: - s = f"rr.{type(self).__name__}(\n" - - from dataclasses import fields - - for fld in fields(self): - if "component" in fld.metadata: - comp: components.Component = getattr(self, fld.name) - if datatype := getattr(comp, "type"): - name = comp.extension_name - typ = datatype.storage_type - s += f" {name}<{typ}>(\n {comp.to_pylist()}\n )\n" - - s += ")" - - return s - - def __repr__(self) -> str: - return str(self) - - def __init__( - self, - fuzz1001: components.AffixFuzzer1Like, - fuzz1002: components.AffixFuzzer2Like, - fuzz1003: components.AffixFuzzer3Like, - fuzz1004: components.AffixFuzzer4Like, - fuzz1005: components.AffixFuzzer5Like, - fuzz1006: components.AffixFuzzer6Like, - fuzz1007: components.AffixFuzzer7Like, - fuzz1101: components.AffixFuzzer1ArrayLike, - fuzz1102: components.AffixFuzzer2ArrayLike, - fuzz1103: components.AffixFuzzer3ArrayLike, - fuzz1104: components.AffixFuzzer4ArrayLike, - fuzz1105: components.AffixFuzzer5ArrayLike, - fuzz1106: components.AffixFuzzer6ArrayLike, - fuzz1107: components.AffixFuzzer7ArrayLike, - *, - fuzz2001: components.AffixFuzzer1Like | None = None, - fuzz2002: components.AffixFuzzer2Like | None = None, - fuzz2003: components.AffixFuzzer3Like | None = None, - fuzz2004: components.AffixFuzzer4Like | None = None, - fuzz2005: components.AffixFuzzer5Like | None = None, - fuzz2006: components.AffixFuzzer6Like | None = None, - fuzz2007: components.AffixFuzzer7Like | None = None, - fuzz2101: components.AffixFuzzer1ArrayLike | None = None, - fuzz2102: components.AffixFuzzer2ArrayLike | None = None, - fuzz2103: components.AffixFuzzer3ArrayLike | None = None, - fuzz2104: components.AffixFuzzer4ArrayLike | None = None, - fuzz2105: components.AffixFuzzer5ArrayLike | None = None, - fuzz2106: components.AffixFuzzer6ArrayLike | None = None, - fuzz2107: components.AffixFuzzer7ArrayLike | None = None, - ) -> None: - # Required components - self.fuzz1001 = components.AffixFuzzer1Array.from_similar(fuzz1001) - self.fuzz1002 = components.AffixFuzzer2Array.from_similar(fuzz1002) - self.fuzz1003 = components.AffixFuzzer3Array.from_similar(fuzz1003) - self.fuzz1004 = components.AffixFuzzer4Array.from_similar(fuzz1004) - self.fuzz1005 = components.AffixFuzzer5Array.from_similar(fuzz1005) - self.fuzz1006 = components.AffixFuzzer6Array.from_similar(fuzz1006) - self.fuzz1007 = components.AffixFuzzer7Array.from_similar(fuzz1007) - self.fuzz1101 = components.AffixFuzzer1Array.from_similar(fuzz1101) - self.fuzz1102 = components.AffixFuzzer2Array.from_similar(fuzz1102) - self.fuzz1103 = components.AffixFuzzer3Array.from_similar(fuzz1103) - self.fuzz1104 = components.AffixFuzzer4Array.from_similar(fuzz1104) - self.fuzz1105 = components.AffixFuzzer5Array.from_similar(fuzz1105) - self.fuzz1106 = components.AffixFuzzer6Array.from_similar(fuzz1106) - self.fuzz1107 = components.AffixFuzzer7Array.from_similar(fuzz1107) - - # Optional components - - self.fuzz2001 = components.AffixFuzzer1Array.from_similar(fuzz2001) - self.fuzz2002 = components.AffixFuzzer2Array.from_similar(fuzz2002) - self.fuzz2003 = components.AffixFuzzer3Array.from_similar(fuzz2003) - self.fuzz2004 = components.AffixFuzzer4Array.from_similar(fuzz2004) - self.fuzz2005 = components.AffixFuzzer5Array.from_similar(fuzz2005) - self.fuzz2006 = components.AffixFuzzer6Array.from_similar(fuzz2006) - self.fuzz2007 = components.AffixFuzzer7Array.from_similar(fuzz2007) - self.fuzz2101 = components.AffixFuzzer1Array.from_similar(fuzz2101) - self.fuzz2102 = components.AffixFuzzer2Array.from_similar(fuzz2102) - self.fuzz2103 = components.AffixFuzzer3Array.from_similar(fuzz2103) - self.fuzz2104 = components.AffixFuzzer4Array.from_similar(fuzz2104) - self.fuzz2105 = components.AffixFuzzer5Array.from_similar(fuzz2105) - self.fuzz2106 = components.AffixFuzzer6Array.from_similar(fuzz2106) - self.fuzz2107 = components.AffixFuzzer7Array.from_similar(fuzz2107) + fuzz1001: components.AffixFuzzer1Array = field( + metadata={"component": "primary"}, + converter=components.AffixFuzzer1Array.from_similar, # type: ignore[misc] + ) + fuzz1002: components.AffixFuzzer2Array = field( + metadata={"component": "primary"}, + converter=components.AffixFuzzer2Array.from_similar, # type: ignore[misc] + ) + fuzz1003: components.AffixFuzzer3Array = field( + metadata={"component": "primary"}, + converter=components.AffixFuzzer3Array.from_similar, # type: ignore[misc] + ) + fuzz1004: components.AffixFuzzer4Array = field( + metadata={"component": "primary"}, + converter=components.AffixFuzzer4Array.from_similar, # type: ignore[misc] + ) + fuzz1005: components.AffixFuzzer5Array = field( + metadata={"component": "primary"}, + converter=components.AffixFuzzer5Array.from_similar, # type: ignore[misc] + ) + fuzz1006: components.AffixFuzzer6Array = field( + metadata={"component": "primary"}, + converter=components.AffixFuzzer6Array.from_similar, # type: ignore[misc] + ) + fuzz1101: components.AffixFuzzer1Array = field( + metadata={"component": "primary"}, + converter=components.AffixFuzzer1Array.from_similar, # type: ignore[misc] + ) + fuzz1102: components.AffixFuzzer2Array = field( + metadata={"component": "primary"}, + converter=components.AffixFuzzer2Array.from_similar, # type: ignore[misc] + ) + fuzz1103: components.AffixFuzzer3Array = field( + metadata={"component": "primary"}, + converter=components.AffixFuzzer3Array.from_similar, # type: ignore[misc] + ) + fuzz1104: components.AffixFuzzer4Array = field( + metadata={"component": "primary"}, + converter=components.AffixFuzzer4Array.from_similar, # type: ignore[misc] + ) + fuzz1105: components.AffixFuzzer5Array = field( + metadata={"component": "primary"}, + converter=components.AffixFuzzer5Array.from_similar, # type: ignore[misc] + ) + fuzz1106: components.AffixFuzzer6Array = field( + metadata={"component": "primary"}, + converter=components.AffixFuzzer6Array.from_similar, # type: ignore[misc] + ) + fuzz2001: components.AffixFuzzer1Array | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.AffixFuzzer1Array.from_similar, # type: ignore[misc] + ) + fuzz2002: components.AffixFuzzer2Array | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.AffixFuzzer2Array.from_similar, # type: ignore[misc] + ) + fuzz2003: components.AffixFuzzer3Array | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.AffixFuzzer3Array.from_similar, # type: ignore[misc] + ) + fuzz2004: components.AffixFuzzer4Array | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.AffixFuzzer4Array.from_similar, # type: ignore[misc] + ) + fuzz2005: components.AffixFuzzer5Array | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.AffixFuzzer5Array.from_similar, # type: ignore[misc] + ) + fuzz2006: components.AffixFuzzer6Array | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.AffixFuzzer6Array.from_similar, # type: ignore[misc] + ) + fuzz2101: components.AffixFuzzer1Array | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.AffixFuzzer1Array.from_similar, # type: ignore[misc] + ) + fuzz2102: components.AffixFuzzer2Array | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.AffixFuzzer2Array.from_similar, # type: ignore[misc] + ) + fuzz2103: components.AffixFuzzer3Array | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.AffixFuzzer3Array.from_similar, # type: ignore[misc] + ) + fuzz2104: components.AffixFuzzer4Array | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.AffixFuzzer4Array.from_similar, # type: ignore[misc] + ) + fuzz2105: components.AffixFuzzer5Array | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.AffixFuzzer5Array.from_similar, # type: ignore[misc] + ) + fuzz2106: components.AffixFuzzer6Array | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.AffixFuzzer6Array.from_similar, # type: ignore[misc] + ) + __str__ = Archetype.__str__ + __repr__ = Archetype.__repr__ diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/archetypes/points2d.py b/rerun_py/rerun_sdk/rerun/_rerun2/archetypes/points2d.py index 137fefc78fd3..504663032786 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/archetypes/points2d.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/archetypes/points2d.py @@ -2,32 +2,42 @@ from __future__ import annotations -from dataclasses import dataclass, field - -from .._baseclasses import Archetype - -__all__ = ["Points2D"] +from attrs import define, field from .. import components +from .._baseclasses import ( + Archetype, +) -## --- Points2D --- ## +__all__ = ["Points2D"] -@dataclass +@define(str=False, repr=False) class Points2D(Archetype): """A 2D point cloud with positions and optional colors, radii, labels, etc.""" - points: components.Point2DArray = field(metadata={"component": "primary"}) + points: components.Point2DArray = field( + metadata={"component": "primary"}, + converter=components.Point2DArray.from_similar, # type: ignore[misc] + ) """ All the actual 2D points that make up the point cloud. """ - radii: components.RadiusArray | None = field(default=None, metadata={"component": "secondary"}) + radii: components.RadiusArray | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.RadiusArray.from_similar, # type: ignore[misc] + ) """ Optional radii for the points, effectively turning them into circles. """ - colors: components.ColorArray | None = field(default=None, metadata={"component": "secondary"}) + colors: components.ColorArray | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.ColorArray.from_similar, # type: ignore[misc] + ) """ Optional colors for the points. @@ -35,12 +45,20 @@ class Points2D(Archetype): As either 0-1 floats or 0-255 integers, with separate alpha. """ - labels: components.LabelArray | None = field(default=None, metadata={"component": "secondary"}) + labels: components.LabelArray | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.LabelArray.from_similar, # type: ignore[misc] + ) """ Optional text labels for the points. """ - draw_order: components.DrawOrderArray | None = field(default=None, metadata={"component": "secondary"}) + draw_order: components.DrawOrderArray | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.DrawOrderArray.from_similar, # type: ignore[misc] + ) """ An optional floating point value that specifies the 2D drawing order. Objects with higher values are drawn on top of those with lower values. @@ -48,14 +66,22 @@ class Points2D(Archetype): The default for 2D points is 30.0. """ - class_ids: components.ClassIdArray | None = field(default=None, metadata={"component": "secondary"}) + class_ids: components.ClassIdArray | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.ClassIdArray.from_similar, # type: ignore[misc] + ) """ Optional class Ids for the points. The class ID provides colors and labels if not specified explicitly. """ - keypoint_ids: components.KeypointIdArray | None = field(default=None, metadata={"component": "secondary"}) + keypoint_ids: components.KeypointIdArray | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.KeypointIdArray.from_similar, # type: ignore[misc] + ) """ Optional keypoint IDs for the points, identifying them within a class. @@ -67,52 +93,14 @@ class Points2D(Archetype): detected skeleton. """ - instance_keys: components.InstanceKeyArray | None = field(default=None, metadata={"component": "secondary"}) + instance_keys: components.InstanceKeyArray | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.InstanceKeyArray.from_similar, # type: ignore[misc] + ) """ Unique identifiers for each individual point in the batch. """ - def __str__(self) -> str: - s = f"rr.{type(self).__name__}(\n" - - from dataclasses import fields - - for fld in fields(self): - if "component" in fld.metadata: - comp: components.Component = getattr(self, fld.name) - if datatype := getattr(comp, "type"): - name = comp.extension_name - typ = datatype.storage_type - s += f" {name}<{typ}>(\n {comp.to_pylist()}\n )\n" - - s += ")" - - return s - - def __repr__(self) -> str: - return str(self) - - def __init__( - self, - points: components.Point2DArrayLike, - *, - radii: components.RadiusArrayLike | None = None, - colors: components.ColorArrayLike | None = None, - labels: components.LabelArrayLike | None = None, - draw_order: components.DrawOrderLike | None = None, - class_ids: components.ClassIdArrayLike | None = None, - keypoint_ids: components.KeypointIdArrayLike | None = None, - instance_keys: components.InstanceKeyArrayLike | None = None, - ) -> None: - # Required components - self.points = components.Point2DArray.from_similar(points) - - # Optional components - - self.radii = components.RadiusArray.from_similar(radii) - self.colors = components.ColorArray.from_similar(colors) - self.labels = components.LabelArray.from_similar(labels) - self.draw_order = components.DrawOrderArray.from_similar(draw_order) - self.class_ids = components.ClassIdArray.from_similar(class_ids) - self.keypoint_ids = components.KeypointIdArray.from_similar(keypoint_ids) - self.instance_keys = components.InstanceKeyArray.from_similar(instance_keys) + __str__ = Archetype.__str__ + __repr__ = Archetype.__repr__ diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/__init__.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/__init__.py index 544020ec3748..9c5bcfe20bf5 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/components/__init__.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/components/__init__.py @@ -2,43 +2,42 @@ from __future__ import annotations +from .class_id import ClassId, ClassIdArray, ClassIdArrayLike, ClassIdLike, ClassIdType +from .color import Color, ColorArray, ColorArrayLike, ColorLike, ColorType +from .draw_order import DrawOrder, DrawOrderArray, DrawOrderArrayLike, DrawOrderLike, DrawOrderType +from .fuzzy import ( + AffixFuzzer1Array, + AffixFuzzer1Type, + AffixFuzzer2Array, + AffixFuzzer2Type, + AffixFuzzer3Array, + AffixFuzzer3Type, + AffixFuzzer4Array, + AffixFuzzer4Type, + AffixFuzzer5Array, + AffixFuzzer5Type, + AffixFuzzer6Array, + AffixFuzzer6Type, +) +from .instance_key import InstanceKey, InstanceKeyArray, InstanceKeyArrayLike, InstanceKeyLike, InstanceKeyType +from .keypoint_id import KeypointId, KeypointIdArray, KeypointIdArrayLike, KeypointIdLike, KeypointIdType +from .label import Label, LabelArray, LabelArrayLike, LabelLike, LabelType +from .point2d import Point2DArray, Point2DType +from .radius import Radius, RadiusArray, RadiusArrayLike, RadiusLike, RadiusType + __all__ = [ - "Component", - "AffixFuzzer1", "AffixFuzzer1Array", - "AffixFuzzer1ArrayLike", - "AffixFuzzer1Like", "AffixFuzzer1Type", - "AffixFuzzer2", "AffixFuzzer2Array", - "AffixFuzzer2ArrayLike", - "AffixFuzzer2Like", "AffixFuzzer2Type", - "AffixFuzzer3", "AffixFuzzer3Array", - "AffixFuzzer3ArrayLike", - "AffixFuzzer3Like", "AffixFuzzer3Type", - "AffixFuzzer4", "AffixFuzzer4Array", - "AffixFuzzer4ArrayLike", - "AffixFuzzer4Like", "AffixFuzzer4Type", - "AffixFuzzer5", "AffixFuzzer5Array", - "AffixFuzzer5ArrayLike", - "AffixFuzzer5Like", "AffixFuzzer5Type", - "AffixFuzzer6", "AffixFuzzer6Array", - "AffixFuzzer6ArrayLike", - "AffixFuzzer6Like", "AffixFuzzer6Type", - "AffixFuzzer7", - "AffixFuzzer7Array", - "AffixFuzzer7ArrayLike", - "AffixFuzzer7Like", - "AffixFuzzer7Type", "ClassId", "ClassIdArray", "ClassIdArrayLike", @@ -69,10 +68,7 @@ "LabelArrayLike", "LabelLike", "LabelType", - "Point2D", "Point2DArray", - "Point2DArrayLike", - "Point2DLike", "Point2DType", "Radius", "RadiusArray", @@ -80,50 +76,3 @@ "RadiusLike", "RadiusType", ] - -from .._baseclasses import Component -from .class_id import ClassId, ClassIdArray, ClassIdArrayLike, ClassIdLike, ClassIdType -from .color import Color, ColorArray, ColorArrayLike, ColorLike, ColorType -from .draw_order import DrawOrder, DrawOrderArray, DrawOrderArrayLike, DrawOrderLike, DrawOrderType -from .fuzzy import ( - AffixFuzzer1, - AffixFuzzer1Array, - AffixFuzzer1ArrayLike, - AffixFuzzer1Like, - AffixFuzzer1Type, - AffixFuzzer2, - AffixFuzzer2Array, - AffixFuzzer2ArrayLike, - AffixFuzzer2Like, - AffixFuzzer2Type, - AffixFuzzer3, - AffixFuzzer3Array, - AffixFuzzer3ArrayLike, - AffixFuzzer3Like, - AffixFuzzer3Type, - AffixFuzzer4, - AffixFuzzer4Array, - AffixFuzzer4ArrayLike, - AffixFuzzer4Like, - AffixFuzzer4Type, - AffixFuzzer5, - AffixFuzzer5Array, - AffixFuzzer5ArrayLike, - AffixFuzzer5Like, - AffixFuzzer5Type, - AffixFuzzer6, - AffixFuzzer6Array, - AffixFuzzer6ArrayLike, - AffixFuzzer6Like, - AffixFuzzer6Type, - AffixFuzzer7, - AffixFuzzer7Array, - AffixFuzzer7ArrayLike, - AffixFuzzer7Like, - AffixFuzzer7Type, -) -from .instance_key import InstanceKey, InstanceKeyArray, InstanceKeyArrayLike, InstanceKeyLike, InstanceKeyType -from .keypoint_id import KeypointId, KeypointIdArray, KeypointIdArrayLike, KeypointIdLike, KeypointIdType -from .label import Label, LabelArray, LabelArrayLike, LabelLike, LabelType -from .point2d import Point2D, Point2DArray, Point2DArrayLike, Point2DLike, Point2DType -from .radius import Radius, RadiusArray, RadiusArrayLike, RadiusLike, RadiusType diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/class_id.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/class_id.py index 9b62102c302f..f711e63e9085 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/components/class_id.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/components/class_id.py @@ -2,35 +2,39 @@ from __future__ import annotations -from dataclasses import dataclass -from typing import Any, Sequence, Union +from typing import Sequence, Union import numpy as np import numpy.typing as npt import pyarrow as pa +from attrs import define, field -from .._baseclasses import Component +from .._baseclasses import ( + BaseExtensionArray, + BaseExtensionType, +) +from ._overrides import classid_native_to_pa_array # noqa: F401 __all__ = ["ClassId", "ClassIdArray", "ClassIdArrayLike", "ClassIdLike", "ClassIdType"] -## --- ClassId --- ## - - -@dataclass +@define class ClassId: """A 16-bit ID representing a type of semantic class.""" - id: int + id: int = field() + + def __array__(self, dtype: npt.DTypeLike = None) -> npt.ArrayLike: + return np.asarray(self.id, dtype=dtype) - def __array__(self) -> npt.ArrayLike: - return np.asarray(self.id) + def __int__(self) -> int: + return int(self.id) ClassIdLike = Union[ClassId, int] ClassIdArrayLike = Union[ - ClassIdLike, + ClassId, Sequence[ClassIdLike], npt.NDArray[np.uint8], npt.NDArray[np.uint16], @@ -41,45 +45,22 @@ def __array__(self) -> npt.ArrayLike: # --- Arrow support --- -from .class_id_ext import ClassIdArrayExt # noqa: E402 - -class ClassIdType(pa.ExtensionType): # type: ignore[misc] - def __init__(self: type[pa.ExtensionType]) -> None: +class ClassIdType(BaseExtensionType): + def __init__(self) -> None: pa.ExtensionType.__init__(self, pa.uint16(), "rerun.class_id") - def __arrow_ext_serialize__(self: type[pa.ExtensionType]) -> bytes: - # since we don't have a parameterized type, we don't need extra metadata to be deserialized - return b"" - @classmethod - def __arrow_ext_deserialize__( - cls: type[pa.ExtensionType], storage_type: Any, serialized: Any - ) -> type[pa.ExtensionType]: - # return an instance of this subclass given the serialized metadata. - return ClassIdType() +class ClassIdArray(BaseExtensionArray[ClassIdArrayLike]): + _EXTENSION_NAME = "rerun.class_id" + _EXTENSION_TYPE = ClassIdType + + @staticmethod + def _native_to_pa_array(data: ClassIdArrayLike, data_type: pa.DataType) -> pa.Array: + return classid_native_to_pa_array(data, data_type) - def __arrow_ext_class__(self: type[pa.ExtensionType]) -> type[pa.ExtensionArray]: - return ClassIdArray +ClassIdType._ARRAY_TYPE = ClassIdArray # TODO(cmc): bring back registration to pyarrow once legacy types are gone # pa.register_extension_type(ClassIdType()) - - -class ClassIdArray(Component, ClassIdArrayExt): # type: ignore[misc] - _extension_name = "rerun.class_id" - - @staticmethod - def from_similar(data: ClassIdArrayLike | None) -> pa.Array: - if data is None: - return ClassIdType().wrap_array(pa.array([], type=ClassIdType().storage_type)) - else: - return ClassIdArrayExt._from_similar( - data, - mono=ClassId, - mono_aliases=ClassIdLike, - many=ClassIdArray, - many_aliases=ClassIdArrayLike, - arrow=ClassIdType, - ) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/color.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/color.py index 76fc3e37c376..6b0371646fad 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/components/color.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/components/color.py @@ -2,22 +2,23 @@ from __future__ import annotations -from dataclasses import dataclass -from typing import Any, Sequence, Union +from typing import Sequence, Union import numpy as np import numpy.typing as npt import pyarrow as pa +from attrs import define, field -from .._baseclasses import Component +from .._baseclasses import ( + BaseExtensionArray, + BaseExtensionType, +) +from ._overrides import color_native_to_pa_array, color_rgba_converter # noqa: F401 __all__ = ["Color", "ColorArray", "ColorArrayLike", "ColorLike", "ColorType"] -## --- Color --- ## - - -@dataclass +@define class Color: """ An RGBA color tuple with unmultiplied/separate alpha, in sRGB gamma space with linear alpha. @@ -27,68 +28,40 @@ class Color: If there is an alpha, we assume it is in linear space, and separate (NOT pre-multiplied). """ - rgba: int + rgba: int = field(converter=color_rgba_converter) - def __array__(self) -> npt.ArrayLike: - return np.asarray(self.rgba) + def __array__(self, dtype: npt.DTypeLike = None) -> npt.ArrayLike: + return np.asarray(self.rgba, dtype=dtype) + def __int__(self) -> int: + return int(self.rgba) -ColorLike = Union[ - Color, int, npt.NDArray[np.uint8], npt.NDArray[np.uint32], npt.NDArray[np.float32], npt.NDArray[np.float64] -] + +ColorLike = Union[Color, int, Sequence[int], npt.NDArray[np.uint8 | np.float32 | np.float64]] ColorArrayLike = Union[ - ColorLike, - Sequence[ColorLike], - Sequence[int], - npt.NDArray[np.uint8], - npt.NDArray[np.uint32], - npt.NDArray[np.float32], - npt.NDArray[np.float64], + Color, Sequence[ColorLike], Sequence[Sequence[int]], npt.NDArray[np.uint8 | np.uint32 | np.float32 | np.float64] ] # --- Arrow support --- -from .color_ext import ColorArrayExt # noqa: E402 - -class ColorType(pa.ExtensionType): # type: ignore[misc] - def __init__(self: type[pa.ExtensionType]) -> None: +class ColorType(BaseExtensionType): + def __init__(self) -> None: pa.ExtensionType.__init__(self, pa.uint32(), "rerun.colorrgba") - def __arrow_ext_serialize__(self: type[pa.ExtensionType]) -> bytes: - # since we don't have a parameterized type, we don't need extra metadata to be deserialized - return b"" - @classmethod - def __arrow_ext_deserialize__( - cls: type[pa.ExtensionType], storage_type: Any, serialized: Any - ) -> type[pa.ExtensionType]: - # return an instance of this subclass given the serialized metadata. - return ColorType() +class ColorArray(BaseExtensionArray[ColorArrayLike]): + _EXTENSION_NAME = "rerun.colorrgba" + _EXTENSION_TYPE = ColorType - def __arrow_ext_class__(self: type[pa.ExtensionType]) -> type[pa.ExtensionArray]: - return ColorArray + @staticmethod + def _native_to_pa_array(data: ColorArrayLike, data_type: pa.DataType) -> pa.Array: + return color_native_to_pa_array(data, data_type) +ColorType._ARRAY_TYPE = ColorArray + # TODO(cmc): bring back registration to pyarrow once legacy types are gone # pa.register_extension_type(ColorType()) - - -class ColorArray(Component, ColorArrayExt): # type: ignore[misc] - _extension_name = "rerun.colorrgba" - - @staticmethod - def from_similar(data: ColorArrayLike | None) -> pa.Array: - if data is None: - return ColorType().wrap_array(pa.array([], type=ColorType().storage_type)) - else: - return ColorArrayExt._from_similar( - data, - mono=Color, - mono_aliases=ColorLike, - many=ColorArray, - many_aliases=ColorArrayLike, - arrow=ColorType, - ) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/draw_order.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/draw_order.py index 59335a0c4143..0e899fec5154 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/components/draw_order.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/components/draw_order.py @@ -2,22 +2,23 @@ from __future__ import annotations -from dataclasses import dataclass -from typing import Any, Sequence, Union +from typing import Sequence, Union import numpy as np import numpy.typing as npt import pyarrow as pa +from attrs import define, field -from .._baseclasses import Component +from .._baseclasses import ( + BaseExtensionArray, + BaseExtensionType, +) +from ._overrides import draworder_native_to_pa_array # noqa: F401 __all__ = ["DrawOrder", "DrawOrderArray", "DrawOrderArrayLike", "DrawOrderLike", "DrawOrderType"] -## --- DrawOrder --- ## - - -@dataclass +@define class DrawOrder: """ Draw order used for the display order of 2D elements. @@ -29,58 +30,38 @@ class DrawOrder: Draw order for entities with the same draw order is generally undefined. """ - value: float + value: float = field() + + def __array__(self, dtype: npt.DTypeLike = None) -> npt.ArrayLike: + return np.asarray(self.value, dtype=dtype) - def __array__(self) -> npt.ArrayLike: - return np.asarray(self.value) + def __float__(self) -> float: + return float(self.value) DrawOrderLike = Union[DrawOrder, float] -DrawOrderArrayLike = Union[DrawOrderLike, Sequence[DrawOrderLike], npt.NDArray[np.float32]] +DrawOrderArrayLike = Union[DrawOrder, Sequence[DrawOrderLike], npt.NDArray[np.float32]] # --- Arrow support --- -from .draw_order_ext import DrawOrderArrayExt # noqa: E402 - -class DrawOrderType(pa.ExtensionType): # type: ignore[misc] - def __init__(self: type[pa.ExtensionType]) -> None: +class DrawOrderType(BaseExtensionType): + def __init__(self) -> None: pa.ExtensionType.__init__(self, pa.float32(), "rerun.draw_order") - def __arrow_ext_serialize__(self: type[pa.ExtensionType]) -> bytes: - # since we don't have a parameterized type, we don't need extra metadata to be deserialized - return b"" - @classmethod - def __arrow_ext_deserialize__( - cls: type[pa.ExtensionType], storage_type: Any, serialized: Any - ) -> type[pa.ExtensionType]: - # return an instance of this subclass given the serialized metadata. - return DrawOrderType() +class DrawOrderArray(BaseExtensionArray[DrawOrderArrayLike]): + _EXTENSION_NAME = "rerun.draw_order" + _EXTENSION_TYPE = DrawOrderType + + @staticmethod + def _native_to_pa_array(data: DrawOrderArrayLike, data_type: pa.DataType) -> pa.Array: + return draworder_native_to_pa_array(data, data_type) - def __arrow_ext_class__(self: type[pa.ExtensionType]) -> type[pa.ExtensionArray]: - return DrawOrderArray +DrawOrderType._ARRAY_TYPE = DrawOrderArray # TODO(cmc): bring back registration to pyarrow once legacy types are gone # pa.register_extension_type(DrawOrderType()) - - -class DrawOrderArray(Component, DrawOrderArrayExt): # type: ignore[misc] - _extension_name = "rerun.draw_order" - - @staticmethod - def from_similar(data: DrawOrderArrayLike | None) -> pa.Array: - if data is None: - return DrawOrderType().wrap_array(pa.array([], type=DrawOrderType().storage_type)) - else: - return DrawOrderArrayExt._from_similar( - data, - mono=DrawOrder, - mono_aliases=DrawOrderLike, - many=DrawOrderArray, - many_aliases=DrawOrderArrayLike, - arrow=DrawOrderType, - ) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/fuzzy.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/fuzzy.py index 24e4de7d4737..d8392e7ebfff 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/components/fuzzy.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/components/fuzzy.py @@ -2,640 +2,125 @@ from __future__ import annotations -from dataclasses import dataclass -from typing import Any, Sequence, Union - -import numpy.typing as npt -import pyarrow as pa - -from .._baseclasses import Component +from .. import datatypes +from .._baseclasses import ( + BaseDelegatingExtensionArray, + BaseDelegatingExtensionType, +) __all__ = [ - "AffixFuzzer1", "AffixFuzzer1Array", - "AffixFuzzer1ArrayLike", - "AffixFuzzer1Like", "AffixFuzzer1Type", - "AffixFuzzer2", "AffixFuzzer2Array", - "AffixFuzzer2ArrayLike", - "AffixFuzzer2Like", "AffixFuzzer2Type", - "AffixFuzzer3", "AffixFuzzer3Array", - "AffixFuzzer3ArrayLike", - "AffixFuzzer3Like", "AffixFuzzer3Type", - "AffixFuzzer4", "AffixFuzzer4Array", - "AffixFuzzer4ArrayLike", - "AffixFuzzer4Like", "AffixFuzzer4Type", - "AffixFuzzer5", "AffixFuzzer5Array", - "AffixFuzzer5ArrayLike", - "AffixFuzzer5Like", "AffixFuzzer5Type", - "AffixFuzzer6", "AffixFuzzer6Array", - "AffixFuzzer6ArrayLike", - "AffixFuzzer6Like", "AffixFuzzer6Type", - "AffixFuzzer7", - "AffixFuzzer7Array", - "AffixFuzzer7ArrayLike", - "AffixFuzzer7Like", - "AffixFuzzer7Type", -] - -from .. import datatypes - -## --- AffixFuzzer1 --- ## - - -@dataclass -class AffixFuzzer1: - single_required: datatypes.AffixFuzzer1 - - -AffixFuzzer1Like = AffixFuzzer1 -AffixFuzzer1ArrayLike = Union[ - AffixFuzzer1Like, - Sequence[AffixFuzzer1Like], ] -# --- Arrow support --- +class AffixFuzzer1Type(BaseDelegatingExtensionType): + _TYPE_NAME = "rerun.testing.components.AffixFuzzer1" + _DELEGATED_EXTENSION_TYPE = datatypes.AffixFuzzer1Type -from .fuzzy_ext import AffixFuzzer1ArrayExt # noqa: E402 +class AffixFuzzer1Array(BaseDelegatingExtensionArray[datatypes.AffixFuzzer1ArrayLike]): + _EXTENSION_NAME = "rerun.testing.components.AffixFuzzer1" + _EXTENSION_TYPE = AffixFuzzer1Type + _DELEGATED_ARRAY_TYPE = datatypes.AffixFuzzer1Array -class AffixFuzzer1Type(pa.ExtensionType): # type: ignore[misc] - def __init__(self: type[pa.ExtensionType]) -> None: - pa.ExtensionType.__init__( - self, - pa.struct( - [ - pa.field("single_float_optional", pa.float32(), True, {}), - pa.field("single_string_required", pa.utf8(), False, {}), - pa.field("single_string_optional", pa.utf8(), True, {}), - pa.field("many_floats_optional", pa.list_(pa.field("item", pa.float32(), True, {})), True, {}), - pa.field("many_strings_required", pa.list_(pa.field("item", pa.utf8(), False, {})), False, {}), - pa.field("many_strings_optional", pa.list_(pa.field("item", pa.utf8(), True, {})), True, {}), - ] - ), - "rerun.testing.components.AffixFuzzer1", - ) - - def __arrow_ext_serialize__(self: type[pa.ExtensionType]) -> bytes: - # since we don't have a parameterized type, we don't need extra metadata to be deserialized - return b"" - - @classmethod - def __arrow_ext_deserialize__( - cls: type[pa.ExtensionType], storage_type: Any, serialized: Any - ) -> type[pa.ExtensionType]: - # return an instance of this subclass given the serialized metadata. - return AffixFuzzer1Type() - - def __arrow_ext_class__(self: type[pa.ExtensionType]) -> type[pa.ExtensionArray]: - return AffixFuzzer1Array +AffixFuzzer1Type._ARRAY_TYPE = AffixFuzzer1Array # TODO(cmc): bring back registration to pyarrow once legacy types are gone # pa.register_extension_type(AffixFuzzer1Type()) -class AffixFuzzer1Array(Component, AffixFuzzer1ArrayExt): # type: ignore[misc] - _extension_name = "rerun.testing.components.AffixFuzzer1" - - @staticmethod - def from_similar(data: AffixFuzzer1ArrayLike | None) -> pa.Array: - if data is None: - return AffixFuzzer1Type().wrap_array(pa.array([], type=AffixFuzzer1Type().storage_type)) - else: - return AffixFuzzer1ArrayExt._from_similar( - data, - mono=AffixFuzzer1, - mono_aliases=AffixFuzzer1Like, - many=AffixFuzzer1Array, - many_aliases=AffixFuzzer1ArrayLike, - arrow=AffixFuzzer1Type, - ) - +class AffixFuzzer2Type(BaseDelegatingExtensionType): + _TYPE_NAME = "rerun.testing.components.AffixFuzzer2" + _DELEGATED_EXTENSION_TYPE = datatypes.AffixFuzzer1Type -## --- AffixFuzzer2 --- ## - - -@dataclass -class AffixFuzzer2: - single_required: datatypes.AffixFuzzer1 - - -AffixFuzzer2Like = AffixFuzzer2 -AffixFuzzer2ArrayLike = Union[ - AffixFuzzer2Like, - Sequence[AffixFuzzer2Like], -] +class AffixFuzzer2Array(BaseDelegatingExtensionArray[datatypes.AffixFuzzer1ArrayLike]): + _EXTENSION_NAME = "rerun.testing.components.AffixFuzzer2" + _EXTENSION_TYPE = AffixFuzzer2Type + _DELEGATED_ARRAY_TYPE = datatypes.AffixFuzzer1Array -# --- Arrow support --- - -from .fuzzy_ext import AffixFuzzer2ArrayExt # noqa: E402 - - -class AffixFuzzer2Type(pa.ExtensionType): # type: ignore[misc] - def __init__(self: type[pa.ExtensionType]) -> None: - pa.ExtensionType.__init__( - self, - pa.struct( - [ - pa.field("single_float_optional", pa.float32(), True, {}), - pa.field("single_string_required", pa.utf8(), False, {}), - pa.field("single_string_optional", pa.utf8(), True, {}), - pa.field("many_floats_optional", pa.list_(pa.field("item", pa.float32(), True, {})), True, {}), - pa.field("many_strings_required", pa.list_(pa.field("item", pa.utf8(), False, {})), False, {}), - pa.field("many_strings_optional", pa.list_(pa.field("item", pa.utf8(), True, {})), True, {}), - ] - ), - "rerun.testing.components.AffixFuzzer2", - ) - - def __arrow_ext_serialize__(self: type[pa.ExtensionType]) -> bytes: - # since we don't have a parameterized type, we don't need extra metadata to be deserialized - return b"" - - @classmethod - def __arrow_ext_deserialize__( - cls: type[pa.ExtensionType], storage_type: Any, serialized: Any - ) -> type[pa.ExtensionType]: - # return an instance of this subclass given the serialized metadata. - return AffixFuzzer2Type() - - def __arrow_ext_class__(self: type[pa.ExtensionType]) -> type[pa.ExtensionArray]: - return AffixFuzzer2Array +AffixFuzzer2Type._ARRAY_TYPE = AffixFuzzer2Array # TODO(cmc): bring back registration to pyarrow once legacy types are gone # pa.register_extension_type(AffixFuzzer2Type()) -class AffixFuzzer2Array(Component, AffixFuzzer2ArrayExt): # type: ignore[misc] - _extension_name = "rerun.testing.components.AffixFuzzer2" - - @staticmethod - def from_similar(data: AffixFuzzer2ArrayLike | None) -> pa.Array: - if data is None: - return AffixFuzzer2Type().wrap_array(pa.array([], type=AffixFuzzer2Type().storage_type)) - else: - return AffixFuzzer2ArrayExt._from_similar( - data, - mono=AffixFuzzer2, - mono_aliases=AffixFuzzer2Like, - many=AffixFuzzer2Array, - many_aliases=AffixFuzzer2ArrayLike, - arrow=AffixFuzzer2Type, - ) - - -## --- AffixFuzzer3 --- ## - +class AffixFuzzer3Type(BaseDelegatingExtensionType): + _TYPE_NAME = "rerun.testing.components.AffixFuzzer3" + _DELEGATED_EXTENSION_TYPE = datatypes.AffixFuzzer1Type -@dataclass -class AffixFuzzer3: - single_required: datatypes.AffixFuzzer1 +class AffixFuzzer3Array(BaseDelegatingExtensionArray[datatypes.AffixFuzzer1ArrayLike]): + _EXTENSION_NAME = "rerun.testing.components.AffixFuzzer3" + _EXTENSION_TYPE = AffixFuzzer3Type + _DELEGATED_ARRAY_TYPE = datatypes.AffixFuzzer1Array -AffixFuzzer3Like = AffixFuzzer3 -AffixFuzzer3ArrayLike = Union[ - AffixFuzzer3Like, - Sequence[AffixFuzzer3Like], -] - - -# --- Arrow support --- - -from .fuzzy_ext import AffixFuzzer3ArrayExt # noqa: E402 - - -class AffixFuzzer3Type(pa.ExtensionType): # type: ignore[misc] - def __init__(self: type[pa.ExtensionType]) -> None: - pa.ExtensionType.__init__( - self, - pa.struct( - [ - pa.field( - "single_required", - pa.struct( - [ - pa.field("single_float_optional", pa.float32(), True, {}), - pa.field("single_string_required", pa.utf8(), False, {}), - pa.field("single_string_optional", pa.utf8(), True, {}), - pa.field( - "many_floats_optional", pa.list_(pa.field("item", pa.float32(), True, {})), True, {} - ), - pa.field( - "many_strings_required", pa.list_(pa.field("item", pa.utf8(), False, {})), False, {} - ), - pa.field( - "many_strings_optional", pa.list_(pa.field("item", pa.utf8(), True, {})), True, {} - ), - ] - ), - False, - {}, - ) - ] - ), - "rerun.testing.components.AffixFuzzer3", - ) - - def __arrow_ext_serialize__(self: type[pa.ExtensionType]) -> bytes: - # since we don't have a parameterized type, we don't need extra metadata to be deserialized - return b"" - - @classmethod - def __arrow_ext_deserialize__( - cls: type[pa.ExtensionType], storage_type: Any, serialized: Any - ) -> type[pa.ExtensionType]: - # return an instance of this subclass given the serialized metadata. - return AffixFuzzer3Type() - - def __arrow_ext_class__(self: type[pa.ExtensionType]) -> type[pa.ExtensionArray]: - return AffixFuzzer3Array +AffixFuzzer3Type._ARRAY_TYPE = AffixFuzzer3Array # TODO(cmc): bring back registration to pyarrow once legacy types are gone # pa.register_extension_type(AffixFuzzer3Type()) -class AffixFuzzer3Array(Component, AffixFuzzer3ArrayExt): # type: ignore[misc] - _extension_name = "rerun.testing.components.AffixFuzzer3" - - @staticmethod - def from_similar(data: AffixFuzzer3ArrayLike | None) -> pa.Array: - if data is None: - return AffixFuzzer3Type().wrap_array(pa.array([], type=AffixFuzzer3Type().storage_type)) - else: - return AffixFuzzer3ArrayExt._from_similar( - data, - mono=AffixFuzzer3, - mono_aliases=AffixFuzzer3Like, - many=AffixFuzzer3Array, - many_aliases=AffixFuzzer3ArrayLike, - arrow=AffixFuzzer3Type, - ) - - -## --- AffixFuzzer4 --- ## - - -@dataclass -class AffixFuzzer4: - single_optional: datatypes.AffixFuzzer1 | None = None - - -AffixFuzzer4Like = AffixFuzzer4 -AffixFuzzer4ArrayLike = Union[ - AffixFuzzer4Like, - Sequence[AffixFuzzer4Like], -] - - -# --- Arrow support --- - -from .fuzzy_ext import AffixFuzzer4ArrayExt # noqa: E402 - - -class AffixFuzzer4Type(pa.ExtensionType): # type: ignore[misc] - def __init__(self: type[pa.ExtensionType]) -> None: - pa.ExtensionType.__init__( - self, - pa.struct( - [ - pa.field("single_float_optional", pa.float32(), True, {}), - pa.field("single_string_required", pa.utf8(), False, {}), - pa.field("single_string_optional", pa.utf8(), True, {}), - pa.field("many_floats_optional", pa.list_(pa.field("item", pa.float32(), True, {})), True, {}), - pa.field("many_strings_required", pa.list_(pa.field("item", pa.utf8(), False, {})), False, {}), - pa.field("many_strings_optional", pa.list_(pa.field("item", pa.utf8(), True, {})), True, {}), - ] - ), - "rerun.testing.components.AffixFuzzer4", - ) +class AffixFuzzer4Type(BaseDelegatingExtensionType): + _TYPE_NAME = "rerun.testing.components.AffixFuzzer4" + _DELEGATED_EXTENSION_TYPE = datatypes.AffixFuzzer1Type - def __arrow_ext_serialize__(self: type[pa.ExtensionType]) -> bytes: - # since we don't have a parameterized type, we don't need extra metadata to be deserialized - return b"" - @classmethod - def __arrow_ext_deserialize__( - cls: type[pa.ExtensionType], storage_type: Any, serialized: Any - ) -> type[pa.ExtensionType]: - # return an instance of this subclass given the serialized metadata. - return AffixFuzzer4Type() +class AffixFuzzer4Array(BaseDelegatingExtensionArray[datatypes.AffixFuzzer1ArrayLike]): + _EXTENSION_NAME = "rerun.testing.components.AffixFuzzer4" + _EXTENSION_TYPE = AffixFuzzer4Type + _DELEGATED_ARRAY_TYPE = datatypes.AffixFuzzer1Array - def __arrow_ext_class__(self: type[pa.ExtensionType]) -> type[pa.ExtensionArray]: - return AffixFuzzer4Array +AffixFuzzer4Type._ARRAY_TYPE = AffixFuzzer4Array # TODO(cmc): bring back registration to pyarrow once legacy types are gone # pa.register_extension_type(AffixFuzzer4Type()) -class AffixFuzzer4Array(Component, AffixFuzzer4ArrayExt): # type: ignore[misc] - _extension_name = "rerun.testing.components.AffixFuzzer4" +class AffixFuzzer5Type(BaseDelegatingExtensionType): + _TYPE_NAME = "rerun.testing.components.AffixFuzzer5" + _DELEGATED_EXTENSION_TYPE = datatypes.AffixFuzzer1Type - @staticmethod - def from_similar(data: AffixFuzzer4ArrayLike | None) -> pa.Array: - if data is None: - return AffixFuzzer4Type().wrap_array(pa.array([], type=AffixFuzzer4Type().storage_type)) - else: - return AffixFuzzer4ArrayExt._from_similar( - data, - mono=AffixFuzzer4, - mono_aliases=AffixFuzzer4Like, - many=AffixFuzzer4Array, - many_aliases=AffixFuzzer4ArrayLike, - arrow=AffixFuzzer4Type, - ) +class AffixFuzzer5Array(BaseDelegatingExtensionArray[datatypes.AffixFuzzer1ArrayLike]): + _EXTENSION_NAME = "rerun.testing.components.AffixFuzzer5" + _EXTENSION_TYPE = AffixFuzzer5Type + _DELEGATED_ARRAY_TYPE = datatypes.AffixFuzzer1Array -## --- AffixFuzzer5 --- ## - - -@dataclass -class AffixFuzzer5: - single_optional: datatypes.AffixFuzzer1 | None = None - - -AffixFuzzer5Like = AffixFuzzer5 -AffixFuzzer5ArrayLike = Union[ - AffixFuzzer5Like, - Sequence[AffixFuzzer5Like], -] - - -# --- Arrow support --- - -from .fuzzy_ext import AffixFuzzer5ArrayExt # noqa: E402 - - -class AffixFuzzer5Type(pa.ExtensionType): # type: ignore[misc] - def __init__(self: type[pa.ExtensionType]) -> None: - pa.ExtensionType.__init__( - self, - pa.struct( - [ - pa.field("single_float_optional", pa.float32(), True, {}), - pa.field("single_string_required", pa.utf8(), False, {}), - pa.field("single_string_optional", pa.utf8(), True, {}), - pa.field("many_floats_optional", pa.list_(pa.field("item", pa.float32(), True, {})), True, {}), - pa.field("many_strings_required", pa.list_(pa.field("item", pa.utf8(), False, {})), False, {}), - pa.field("many_strings_optional", pa.list_(pa.field("item", pa.utf8(), True, {})), True, {}), - ] - ), - "rerun.testing.components.AffixFuzzer5", - ) - - def __arrow_ext_serialize__(self: type[pa.ExtensionType]) -> bytes: - # since we don't have a parameterized type, we don't need extra metadata to be deserialized - return b"" - - @classmethod - def __arrow_ext_deserialize__( - cls: type[pa.ExtensionType], storage_type: Any, serialized: Any - ) -> type[pa.ExtensionType]: - # return an instance of this subclass given the serialized metadata. - return AffixFuzzer5Type() - - def __arrow_ext_class__(self: type[pa.ExtensionType]) -> type[pa.ExtensionArray]: - return AffixFuzzer5Array +AffixFuzzer5Type._ARRAY_TYPE = AffixFuzzer5Array # TODO(cmc): bring back registration to pyarrow once legacy types are gone # pa.register_extension_type(AffixFuzzer5Type()) -class AffixFuzzer5Array(Component, AffixFuzzer5ArrayExt): # type: ignore[misc] - _extension_name = "rerun.testing.components.AffixFuzzer5" - - @staticmethod - def from_similar(data: AffixFuzzer5ArrayLike | None) -> pa.Array: - if data is None: - return AffixFuzzer5Type().wrap_array(pa.array([], type=AffixFuzzer5Type().storage_type)) - else: - return AffixFuzzer5ArrayExt._from_similar( - data, - mono=AffixFuzzer5, - mono_aliases=AffixFuzzer5Like, - many=AffixFuzzer5Array, - many_aliases=AffixFuzzer5ArrayLike, - arrow=AffixFuzzer5Type, - ) - +class AffixFuzzer6Type(BaseDelegatingExtensionType): + _TYPE_NAME = "rerun.testing.components.AffixFuzzer6" + _DELEGATED_EXTENSION_TYPE = datatypes.AffixFuzzer1Type -## --- AffixFuzzer6 --- ## +class AffixFuzzer6Array(BaseDelegatingExtensionArray[datatypes.AffixFuzzer1ArrayLike]): + _EXTENSION_NAME = "rerun.testing.components.AffixFuzzer6" + _EXTENSION_TYPE = AffixFuzzer6Type + _DELEGATED_ARRAY_TYPE = datatypes.AffixFuzzer1Array -@dataclass -class AffixFuzzer6: - single_optional: datatypes.AffixFuzzer1 | None = None - - -AffixFuzzer6Like = AffixFuzzer6 -AffixFuzzer6ArrayLike = Union[ - AffixFuzzer6Like, - Sequence[AffixFuzzer6Like], -] - - -# --- Arrow support --- - -from .fuzzy_ext import AffixFuzzer6ArrayExt # noqa: E402 - - -class AffixFuzzer6Type(pa.ExtensionType): # type: ignore[misc] - def __init__(self: type[pa.ExtensionType]) -> None: - pa.ExtensionType.__init__( - self, - pa.struct( - [ - pa.field( - "single_optional", - pa.struct( - [ - pa.field("single_float_optional", pa.float32(), True, {}), - pa.field("single_string_required", pa.utf8(), False, {}), - pa.field("single_string_optional", pa.utf8(), True, {}), - pa.field( - "many_floats_optional", pa.list_(pa.field("item", pa.float32(), True, {})), True, {} - ), - pa.field( - "many_strings_required", pa.list_(pa.field("item", pa.utf8(), False, {})), False, {} - ), - pa.field( - "many_strings_optional", pa.list_(pa.field("item", pa.utf8(), True, {})), True, {} - ), - ] - ), - True, - {}, - ) - ] - ), - "rerun.testing.components.AffixFuzzer6", - ) - - def __arrow_ext_serialize__(self: type[pa.ExtensionType]) -> bytes: - # since we don't have a parameterized type, we don't need extra metadata to be deserialized - return b"" - - @classmethod - def __arrow_ext_deserialize__( - cls: type[pa.ExtensionType], storage_type: Any, serialized: Any - ) -> type[pa.ExtensionType]: - # return an instance of this subclass given the serialized metadata. - return AffixFuzzer6Type() - - def __arrow_ext_class__(self: type[pa.ExtensionType]) -> type[pa.ExtensionArray]: - return AffixFuzzer6Array +AffixFuzzer6Type._ARRAY_TYPE = AffixFuzzer6Array # TODO(cmc): bring back registration to pyarrow once legacy types are gone # pa.register_extension_type(AffixFuzzer6Type()) - - -class AffixFuzzer6Array(Component, AffixFuzzer6ArrayExt): # type: ignore[misc] - _extension_name = "rerun.testing.components.AffixFuzzer6" - - @staticmethod - def from_similar(data: AffixFuzzer6ArrayLike | None) -> pa.Array: - if data is None: - return AffixFuzzer6Type().wrap_array(pa.array([], type=AffixFuzzer6Type().storage_type)) - else: - return AffixFuzzer6ArrayExt._from_similar( - data, - mono=AffixFuzzer6, - mono_aliases=AffixFuzzer6Like, - many=AffixFuzzer6Array, - many_aliases=AffixFuzzer6ArrayLike, - arrow=AffixFuzzer6Type, - ) - - -## --- AffixFuzzer7 --- ## - - -@dataclass -class AffixFuzzer7: - single_string_required: str - many_strings_required: npt.ArrayLike - many_optional: list[datatypes.AffixFuzzer1] | None = None - single_float_optional: float | None = None - single_string_optional: str | None = None - many_floats_optional: npt.ArrayLike | None = None - many_strings_optional: npt.ArrayLike | None = None - - -AffixFuzzer7Like = AffixFuzzer7 -AffixFuzzer7ArrayLike = Union[ - AffixFuzzer7Like, - Sequence[AffixFuzzer7Like], -] - - -# --- Arrow support --- - -from .fuzzy_ext import AffixFuzzer7ArrayExt # noqa: E402 - - -class AffixFuzzer7Type(pa.ExtensionType): # type: ignore[misc] - def __init__(self: type[pa.ExtensionType]) -> None: - pa.ExtensionType.__init__( - self, - pa.struct( - [ - pa.field( - "many_optional", - pa.list_( - pa.field( - "item", - pa.struct( - [ - pa.field("single_float_optional", pa.float32(), True, {}), - pa.field("single_string_required", pa.utf8(), False, {}), - pa.field("single_string_optional", pa.utf8(), True, {}), - pa.field( - "many_floats_optional", - pa.list_(pa.field("item", pa.float32(), True, {})), - True, - {}, - ), - pa.field( - "many_strings_required", - pa.list_(pa.field("item", pa.utf8(), False, {})), - False, - {}, - ), - pa.field( - "many_strings_optional", - pa.list_(pa.field("item", pa.utf8(), True, {})), - True, - {}, - ), - ] - ), - True, - {}, - ) - ), - True, - {}, - ), - pa.field("single_float_optional", pa.float32(), True, {}), - pa.field("single_string_required", pa.utf8(), False, {}), - pa.field("single_string_optional", pa.utf8(), True, {}), - pa.field("many_floats_optional", pa.list_(pa.field("item", pa.float32(), True, {})), True, {}), - pa.field("many_strings_required", pa.list_(pa.field("item", pa.utf8(), False, {})), False, {}), - pa.field("many_strings_optional", pa.list_(pa.field("item", pa.utf8(), True, {})), True, {}), - ] - ), - "rerun.testing.components.AffixFuzzer7", - ) - - def __arrow_ext_serialize__(self: type[pa.ExtensionType]) -> bytes: - # since we don't have a parameterized type, we don't need extra metadata to be deserialized - return b"" - - @classmethod - def __arrow_ext_deserialize__( - cls: type[pa.ExtensionType], storage_type: Any, serialized: Any - ) -> type[pa.ExtensionType]: - # return an instance of this subclass given the serialized metadata. - return AffixFuzzer7Type() - - def __arrow_ext_class__(self: type[pa.ExtensionType]) -> type[pa.ExtensionArray]: - return AffixFuzzer7Array - - -# TODO(cmc): bring back registration to pyarrow once legacy types are gone -# pa.register_extension_type(AffixFuzzer7Type()) - - -class AffixFuzzer7Array(Component, AffixFuzzer7ArrayExt): # type: ignore[misc] - _extension_name = "rerun.testing.components.AffixFuzzer7" - - @staticmethod - def from_similar(data: AffixFuzzer7ArrayLike | None) -> pa.Array: - if data is None: - return AffixFuzzer7Type().wrap_array(pa.array([], type=AffixFuzzer7Type().storage_type)) - else: - return AffixFuzzer7ArrayExt._from_similar( - data, - mono=AffixFuzzer7, - mono_aliases=AffixFuzzer7Like, - many=AffixFuzzer7Array, - many_aliases=AffixFuzzer7ArrayLike, - arrow=AffixFuzzer7Type, - ) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/instance_key.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/instance_key.py index 10a85e14daa1..e907d7ecf8f6 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/components/instance_key.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/components/instance_key.py @@ -2,77 +2,58 @@ from __future__ import annotations -from dataclasses import dataclass -from typing import Any, Sequence, Union +from typing import Sequence, Union import numpy as np import numpy.typing as npt import pyarrow as pa +from attrs import define, field -from .._baseclasses import Component +from .._baseclasses import ( + BaseExtensionArray, + BaseExtensionType, +) +from ._overrides import instancekey_native_to_pa_array # noqa: F401 __all__ = ["InstanceKey", "InstanceKeyArray", "InstanceKeyArrayLike", "InstanceKeyLike", "InstanceKeyType"] -## --- InstanceKey --- ## - - -@dataclass +@define class InstanceKey: """A unique numeric identifier for each individual instance within a batch.""" - value: int + value: int = field() + + def __array__(self, dtype: npt.DTypeLike = None) -> npt.ArrayLike: + return np.asarray(self.value, dtype=dtype) - def __array__(self) -> npt.ArrayLike: - return np.asarray(self.value) + def __int__(self) -> int: + return int(self.value) InstanceKeyLike = Union[InstanceKey, int] -InstanceKeyArrayLike = Union[InstanceKeyLike, Sequence[InstanceKeyLike], npt.NDArray[np.uint64]] +InstanceKeyArrayLike = Union[InstanceKey, Sequence[InstanceKeyLike], npt.NDArray[np.uint64]] # --- Arrow support --- -from .instance_key_ext import InstanceKeyArrayExt # noqa: E402 - -class InstanceKeyType(pa.ExtensionType): # type: ignore[misc] - def __init__(self: type[pa.ExtensionType]) -> None: +class InstanceKeyType(BaseExtensionType): + def __init__(self) -> None: pa.ExtensionType.__init__(self, pa.uint64(), "rerun.instance_key") - def __arrow_ext_serialize__(self: type[pa.ExtensionType]) -> bytes: - # since we don't have a parameterized type, we don't need extra metadata to be deserialized - return b"" - @classmethod - def __arrow_ext_deserialize__( - cls: type[pa.ExtensionType], storage_type: Any, serialized: Any - ) -> type[pa.ExtensionType]: - # return an instance of this subclass given the serialized metadata. - return InstanceKeyType() +class InstanceKeyArray(BaseExtensionArray[InstanceKeyArrayLike]): + _EXTENSION_NAME = "rerun.instance_key" + _EXTENSION_TYPE = InstanceKeyType + + @staticmethod + def _native_to_pa_array(data: InstanceKeyArrayLike, data_type: pa.DataType) -> pa.Array: + return instancekey_native_to_pa_array(data, data_type) - def __arrow_ext_class__(self: type[pa.ExtensionType]) -> type[pa.ExtensionArray]: - return InstanceKeyArray +InstanceKeyType._ARRAY_TYPE = InstanceKeyArray # TODO(cmc): bring back registration to pyarrow once legacy types are gone # pa.register_extension_type(InstanceKeyType()) - - -class InstanceKeyArray(Component, InstanceKeyArrayExt): # type: ignore[misc] - _extension_name = "rerun.instance_key" - - @staticmethod - def from_similar(data: InstanceKeyArrayLike | None) -> pa.Array: - if data is None: - return InstanceKeyType().wrap_array(pa.array([], type=InstanceKeyType().storage_type)) - else: - return InstanceKeyArrayExt._from_similar( - data, - mono=InstanceKey, - mono_aliases=InstanceKeyLike, - many=InstanceKeyArray, - many_aliases=InstanceKeyArrayLike, - arrow=InstanceKeyType, - ) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/keypoint_id.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/keypoint_id.py index e53b0bcb6764..9256169a9a87 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/components/keypoint_id.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/components/keypoint_id.py @@ -2,22 +2,23 @@ from __future__ import annotations -from dataclasses import dataclass -from typing import Any, Sequence, Union +from typing import Sequence, Union import numpy as np import numpy.typing as npt import pyarrow as pa +from attrs import define, field -from .._baseclasses import Component +from .._baseclasses import ( + BaseExtensionArray, + BaseExtensionType, +) +from ._overrides import keypointid_native_to_pa_array # noqa: F401 __all__ = ["KeypointId", "KeypointIdArray", "KeypointIdArrayLike", "KeypointIdLike", "KeypointIdType"] -## --- KeypointId --- ## - - -@dataclass +@define class KeypointId: """ A 16-bit ID representing a type of semantic keypoint within a class. @@ -28,60 +29,40 @@ class KeypointId: [`rerun.components.AnnotationContext`]. """ - id: int + id: int = field() + + def __array__(self, dtype: npt.DTypeLike = None) -> npt.ArrayLike: + return np.asarray(self.id, dtype=dtype) - def __array__(self) -> npt.ArrayLike: - return np.asarray(self.id) + def __int__(self) -> int: + return int(self.id) KeypointIdLike = Union[KeypointId, float] KeypointIdArrayLike = Union[ - KeypointIdLike, Sequence[KeypointIdLike], npt.NDArray[np.uint8], npt.NDArray[np.uint16], npt.NDArray[np.uint32] + KeypointId, Sequence[KeypointIdLike], npt.NDArray[np.uint8], npt.NDArray[np.uint16], npt.NDArray[np.uint32] ] # --- Arrow support --- -from .keypoint_id_ext import KeypointIdArrayExt # noqa: E402 - -class KeypointIdType(pa.ExtensionType): # type: ignore[misc] - def __init__(self: type[pa.ExtensionType]) -> None: +class KeypointIdType(BaseExtensionType): + def __init__(self) -> None: pa.ExtensionType.__init__(self, pa.uint16(), "rerun.keypoint_id") - def __arrow_ext_serialize__(self: type[pa.ExtensionType]) -> bytes: - # since we don't have a parameterized type, we don't need extra metadata to be deserialized - return b"" - @classmethod - def __arrow_ext_deserialize__( - cls: type[pa.ExtensionType], storage_type: Any, serialized: Any - ) -> type[pa.ExtensionType]: - # return an instance of this subclass given the serialized metadata. - return KeypointIdType() +class KeypointIdArray(BaseExtensionArray[KeypointIdArrayLike]): + _EXTENSION_NAME = "rerun.keypoint_id" + _EXTENSION_TYPE = KeypointIdType + + @staticmethod + def _native_to_pa_array(data: KeypointIdArrayLike, data_type: pa.DataType) -> pa.Array: + return keypointid_native_to_pa_array(data, data_type) - def __arrow_ext_class__(self: type[pa.ExtensionType]) -> type[pa.ExtensionArray]: - return KeypointIdArray +KeypointIdType._ARRAY_TYPE = KeypointIdArray # TODO(cmc): bring back registration to pyarrow once legacy types are gone # pa.register_extension_type(KeypointIdType()) - - -class KeypointIdArray(Component, KeypointIdArrayExt): # type: ignore[misc] - _extension_name = "rerun.keypoint_id" - - @staticmethod - def from_similar(data: KeypointIdArrayLike | None) -> pa.Array: - if data is None: - return KeypointIdType().wrap_array(pa.array([], type=KeypointIdType().storage_type)) - else: - return KeypointIdArrayExt._from_similar( - data, - mono=KeypointId, - mono_aliases=KeypointIdLike, - many=KeypointIdArray, - many_aliases=KeypointIdArrayLike, - arrow=KeypointIdType, - ) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/label.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/label.py index b88d61613761..b31cce7957e4 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/components/label.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/components/label.py @@ -2,78 +2,53 @@ from __future__ import annotations -from dataclasses import dataclass -from typing import Any, Sequence, Union +from typing import Sequence, Union import pyarrow as pa +from attrs import define, field -from .._baseclasses import Component +from .._baseclasses import ( + BaseExtensionArray, + BaseExtensionType, +) +from ._overrides import label_native_to_pa_array # noqa: F401 __all__ = ["Label", "LabelArray", "LabelArrayLike", "LabelLike", "LabelType"] -## --- Label --- ## - - -@dataclass +@define class Label: """A String label component.""" - value: str + value: str = field() def __str__(self) -> str: - return self.value + return str(self.value) LabelLike = Union[Label, str] -LabelArrayLike = Union[ - LabelLike, - Sequence[LabelLike], -] +LabelArrayLike = Union[Label, Sequence[LabelLike], Sequence[str]] # --- Arrow support --- -from .label_ext import LabelArrayExt # noqa: E402 - -class LabelType(pa.ExtensionType): # type: ignore[misc] - def __init__(self: type[pa.ExtensionType]) -> None: +class LabelType(BaseExtensionType): + def __init__(self) -> None: pa.ExtensionType.__init__(self, pa.utf8(), "rerun.label") - def __arrow_ext_serialize__(self: type[pa.ExtensionType]) -> bytes: - # since we don't have a parameterized type, we don't need extra metadata to be deserialized - return b"" - @classmethod - def __arrow_ext_deserialize__( - cls: type[pa.ExtensionType], storage_type: Any, serialized: Any - ) -> type[pa.ExtensionType]: - # return an instance of this subclass given the serialized metadata. - return LabelType() +class LabelArray(BaseExtensionArray[LabelArrayLike]): + _EXTENSION_NAME = "rerun.label" + _EXTENSION_TYPE = LabelType + + @staticmethod + def _native_to_pa_array(data: LabelArrayLike, data_type: pa.DataType) -> pa.Array: + return label_native_to_pa_array(data, data_type) - def __arrow_ext_class__(self: type[pa.ExtensionType]) -> type[pa.ExtensionArray]: - return LabelArray +LabelType._ARRAY_TYPE = LabelArray # TODO(cmc): bring back registration to pyarrow once legacy types are gone # pa.register_extension_type(LabelType()) - - -class LabelArray(Component, LabelArrayExt): # type: ignore[misc] - _extension_name = "rerun.label" - - @staticmethod - def from_similar(data: LabelArrayLike | None) -> pa.Array: - if data is None: - return LabelType().wrap_array(pa.array([], type=LabelType().storage_type)) - else: - return LabelArrayExt._from_similar( - data, - mono=Label, - mono_aliases=LabelLike, - many=LabelArray, - many_aliases=LabelArrayLike, - arrow=LabelType, - ) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/point2d.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/point2d.py index e456b9791440..1282311afaff 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/components/point2d.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/components/point2d.py @@ -2,79 +2,27 @@ from __future__ import annotations -from dataclasses import dataclass -from typing import Any, Sequence, Tuple, Union +from .. import datatypes +from .._baseclasses import ( + BaseDelegatingExtensionArray, + BaseDelegatingExtensionType, +) -import numpy as np -import numpy.typing as npt -import pyarrow as pa +__all__ = ["Point2DArray", "Point2DType"] -from .._baseclasses import Component -__all__ = ["Point2D", "Point2DArray", "Point2DArrayLike", "Point2DLike", "Point2DType"] +class Point2DType(BaseDelegatingExtensionType): + _TYPE_NAME = "rerun.point2d" + _DELEGATED_EXTENSION_TYPE = datatypes.Point2DType -## --- Point2D --- ## +class Point2DArray(BaseDelegatingExtensionArray[datatypes.Point2DArrayLike]): + _EXTENSION_NAME = "rerun.point2d" + _EXTENSION_TYPE = Point2DType + _DELEGATED_ARRAY_TYPE = datatypes.Point2DArray -@dataclass -class Point2D: - """A point in 2D space.""" - - x: float - y: float - - -Point2DLike = Union[Point2D, npt.NDArray[np.float32], Sequence[float], Tuple[float, float]] - -Point2DArrayLike = Union[Point2DLike, Sequence[Point2DLike], npt.NDArray[np.float32], Sequence[float]] - - -# --- Arrow support --- - -from .point2d_ext import Point2DArrayExt # noqa: E402 - - -class Point2DType(pa.ExtensionType): # type: ignore[misc] - def __init__(self: type[pa.ExtensionType]) -> None: - pa.ExtensionType.__init__( - self, - pa.struct([pa.field("x", pa.float32(), False, {}), pa.field("y", pa.float32(), False, {})]), - "rerun.point2d", - ) - - def __arrow_ext_serialize__(self: type[pa.ExtensionType]) -> bytes: - # since we don't have a parameterized type, we don't need extra metadata to be deserialized - return b"" - - @classmethod - def __arrow_ext_deserialize__( - cls: type[pa.ExtensionType], storage_type: Any, serialized: Any - ) -> type[pa.ExtensionType]: - # return an instance of this subclass given the serialized metadata. - return Point2DType() - - def __arrow_ext_class__(self: type[pa.ExtensionType]) -> type[pa.ExtensionArray]: - return Point2DArray - +Point2DType._ARRAY_TYPE = Point2DArray # TODO(cmc): bring back registration to pyarrow once legacy types are gone # pa.register_extension_type(Point2DType()) - - -class Point2DArray(Component, Point2DArrayExt): # type: ignore[misc] - _extension_name = "rerun.point2d" - - @staticmethod - def from_similar(data: Point2DArrayLike | None) -> pa.Array: - if data is None: - return Point2DType().wrap_array(pa.array([], type=Point2DType().storage_type)) - else: - return Point2DArrayExt._from_similar( - data, - mono=Point2D, - mono_aliases=Point2DLike, - many=Point2DArray, - many_aliases=Point2DArrayLike, - arrow=Point2DType, - ) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/radius.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/radius.py index 407755538e74..3b7e65fcf68d 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/components/radius.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/components/radius.py @@ -2,77 +2,58 @@ from __future__ import annotations -from dataclasses import dataclass -from typing import Any, Sequence, Union +from typing import Sequence, Union import numpy as np import numpy.typing as npt import pyarrow as pa +from attrs import define, field -from .._baseclasses import Component +from .._baseclasses import ( + BaseExtensionArray, + BaseExtensionType, +) +from ._overrides import radius_native_to_pa_array # noqa: F401 __all__ = ["Radius", "RadiusArray", "RadiusArrayLike", "RadiusLike", "RadiusType"] -## --- Radius --- ## - - -@dataclass +@define class Radius: """A Radius component.""" - value: float + value: float = field() + + def __array__(self, dtype: npt.DTypeLike = None) -> npt.ArrayLike: + return np.asarray(self.value, dtype=dtype) - def __array__(self) -> npt.ArrayLike: - return np.asarray(self.value) + def __float__(self) -> float: + return float(self.value) RadiusLike = Union[Radius, float] -RadiusArrayLike = Union[RadiusLike, Sequence[RadiusLike], npt.NDArray[np.float32]] +RadiusArrayLike = Union[Radius, Sequence[RadiusLike], npt.NDArray[np.float32]] # --- Arrow support --- -from .radius_ext import RadiusArrayExt # noqa: E402 - -class RadiusType(pa.ExtensionType): # type: ignore[misc] - def __init__(self: type[pa.ExtensionType]) -> None: +class RadiusType(BaseExtensionType): + def __init__(self) -> None: pa.ExtensionType.__init__(self, pa.float32(), "rerun.radius") - def __arrow_ext_serialize__(self: type[pa.ExtensionType]) -> bytes: - # since we don't have a parameterized type, we don't need extra metadata to be deserialized - return b"" - @classmethod - def __arrow_ext_deserialize__( - cls: type[pa.ExtensionType], storage_type: Any, serialized: Any - ) -> type[pa.ExtensionType]: - # return an instance of this subclass given the serialized metadata. - return RadiusType() +class RadiusArray(BaseExtensionArray[RadiusArrayLike]): + _EXTENSION_NAME = "rerun.radius" + _EXTENSION_TYPE = RadiusType + + @staticmethod + def _native_to_pa_array(data: RadiusArrayLike, data_type: pa.DataType) -> pa.Array: + return radius_native_to_pa_array(data, data_type) - def __arrow_ext_class__(self: type[pa.ExtensionType]) -> type[pa.ExtensionArray]: - return RadiusArray +RadiusType._ARRAY_TYPE = RadiusArray # TODO(cmc): bring back registration to pyarrow once legacy types are gone # pa.register_extension_type(RadiusType()) - - -class RadiusArray(Component, RadiusArrayExt): # type: ignore[misc] - _extension_name = "rerun.radius" - - @staticmethod - def from_similar(data: RadiusArrayLike | None) -> pa.Array: - if data is None: - return RadiusType().wrap_array(pa.array([], type=RadiusType().storage_type)) - else: - return RadiusArrayExt._from_similar( - data, - mono=Radius, - mono_aliases=RadiusLike, - many=RadiusArray, - many_aliases=RadiusArrayLike, - arrow=RadiusType, - ) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/__init__.py b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/__init__.py index cec81dfa19cf..ca164f0ad4db 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/__init__.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/__init__.py @@ -2,6 +2,21 @@ from __future__ import annotations +from .fuzzy import ( + AffixFuzzer1, + AffixFuzzer1Array, + AffixFuzzer1ArrayLike, + AffixFuzzer1Like, + AffixFuzzer1Type, + AffixFuzzer2, + AffixFuzzer2Array, + AffixFuzzer2ArrayLike, + AffixFuzzer2Like, + AffixFuzzer2Type, +) +from .point2d import Point2D, Point2DArray, Point2DArrayLike, Point2DLike, Point2DType +from .vec2d import Vec2D, Vec2DArray, Vec2DArrayLike, Vec2DLike, Vec2DType + __all__ = [ "AffixFuzzer1", "AffixFuzzer1Array", @@ -13,24 +28,14 @@ "AffixFuzzer2ArrayLike", "AffixFuzzer2Like", "AffixFuzzer2Type", + "Point2D", + "Point2DArray", + "Point2DArrayLike", + "Point2DLike", + "Point2DType", "Vec2D", "Vec2DArray", "Vec2DArrayLike", "Vec2DLike", "Vec2DType", ] - - -from .fuzzy import ( - AffixFuzzer1, - AffixFuzzer1Array, - AffixFuzzer1ArrayLike, - AffixFuzzer1Like, - AffixFuzzer1Type, - AffixFuzzer2, - AffixFuzzer2Array, - AffixFuzzer2ArrayLike, - AffixFuzzer2Like, - AffixFuzzer2Type, -) -from .vec2d import Vec2D, Vec2DArray, Vec2DArrayLike, Vec2DLike, Vec2DType diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/fuzzy.py b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/fuzzy.py index f095027ea27e..c73509a3dcdf 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/fuzzy.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/fuzzy.py @@ -2,12 +2,20 @@ from __future__ import annotations -from dataclasses import dataclass -from typing import Any, Sequence, Union +from typing import Sequence, Union import numpy as np import numpy.typing as npt import pyarrow as pa +from attrs import define, field + +from .._baseclasses import ( + BaseExtensionArray, + BaseExtensionType, +) +from .._converters import ( + to_np_float32, +) __all__ = [ "AffixFuzzer1", @@ -23,33 +31,28 @@ ] -## --- AffixFuzzer1 --- ## - - -@dataclass +@define class AffixFuzzer1: - single_string_required: str - many_strings_required: npt.ArrayLike - single_float_optional: float | None = None - single_string_optional: str | None = None - many_floats_optional: npt.ArrayLike | None = None - many_strings_optional: npt.ArrayLike | None = None + single_string_required: str = field() + many_strings_required: list[str] = field() + single_float_optional: float | None = field(default=None) + single_string_optional: str | None = field(default=None) + many_floats_optional: npt.NDArray[np.float32] | None = field(default=None, converter=to_np_float32) + many_strings_optional: list[str] | None = field(default=None) AffixFuzzer1Like = AffixFuzzer1 AffixFuzzer1ArrayLike = Union[ - AffixFuzzer1Like, + AffixFuzzer1, Sequence[AffixFuzzer1Like], ] # --- Arrow support --- -from .fuzzy_ext import AffixFuzzer1ArrayExt # noqa: E402 - -class AffixFuzzer1Type(pa.ExtensionType): # type: ignore[misc] - def __init__(self: type[pa.ExtensionType]) -> None: +class AffixFuzzer1Type(BaseExtensionType): + def __init__(self) -> None: pa.ExtensionType.__init__( self, pa.struct( @@ -65,102 +68,55 @@ def __init__(self: type[pa.ExtensionType]) -> None: "rerun.testing.datatypes.AffixFuzzer1", ) - def __arrow_ext_serialize__(self: type[pa.ExtensionType]) -> bytes: - # since we don't have a parameterized type, we don't need extra metadata to be deserialized - return b"" - @classmethod - def __arrow_ext_deserialize__( - cls: type[pa.ExtensionType], storage_type: Any, serialized: Any - ) -> type[pa.ExtensionType]: - # return an instance of this subclass given the serialized metadata. - return AffixFuzzer1Type() +class AffixFuzzer1Array(BaseExtensionArray[AffixFuzzer1ArrayLike]): + _EXTENSION_NAME = "rerun.testing.datatypes.AffixFuzzer1" + _EXTENSION_TYPE = AffixFuzzer1Type + + @staticmethod + def _native_to_pa_array(data: AffixFuzzer1ArrayLike, data_type: pa.DataType) -> pa.Array: + raise NotImplementedError - def __arrow_ext_class__(self: type[pa.ExtensionType]) -> type[pa.ExtensionArray]: - return AffixFuzzer1Array +AffixFuzzer1Type._ARRAY_TYPE = AffixFuzzer1Array # TODO(cmc): bring back registration to pyarrow once legacy types are gone # pa.register_extension_type(AffixFuzzer1Type()) -class AffixFuzzer1Array(AffixFuzzer1ArrayExt): # type: ignore[misc] - _extension_name = "rerun.testing.datatypes.AffixFuzzer1" - - @staticmethod - def from_similar(data: AffixFuzzer1ArrayLike | None) -> pa.Array: - if data is None: - return AffixFuzzer1Type().wrap_array(pa.array([], type=AffixFuzzer1Type().storage_type)) - else: - return AffixFuzzer1ArrayExt._from_similar( - data, - mono=AffixFuzzer1, - mono_aliases=AffixFuzzer1Like, - many=AffixFuzzer1Array, - many_aliases=AffixFuzzer1ArrayLike, - arrow=AffixFuzzer1Type, - ) - - -## --- AffixFuzzer2 --- ## - - -@dataclass +@define class AffixFuzzer2: - single_float_optional: float | None = None + single_float_optional: float | None = field(default=None) - def __array__(self) -> npt.ArrayLike: - return np.asarray(self.single_float_optional) + def __array__(self, dtype: npt.DTypeLike = None) -> npt.ArrayLike: + return np.asarray(self.single_float_optional, dtype=dtype) AffixFuzzer2Like = AffixFuzzer2 AffixFuzzer2ArrayLike = Union[ - AffixFuzzer2Like, + AffixFuzzer2, Sequence[AffixFuzzer2Like], ] # --- Arrow support --- -from .fuzzy_ext import AffixFuzzer2ArrayExt # noqa: E402 - -class AffixFuzzer2Type(pa.ExtensionType): # type: ignore[misc] - def __init__(self: type[pa.ExtensionType]) -> None: +class AffixFuzzer2Type(BaseExtensionType): + def __init__(self) -> None: pa.ExtensionType.__init__(self, pa.float32(), "rerun.testing.datatypes.AffixFuzzer2") - def __arrow_ext_serialize__(self: type[pa.ExtensionType]) -> bytes: - # since we don't have a parameterized type, we don't need extra metadata to be deserialized - return b"" - @classmethod - def __arrow_ext_deserialize__( - cls: type[pa.ExtensionType], storage_type: Any, serialized: Any - ) -> type[pa.ExtensionType]: - # return an instance of this subclass given the serialized metadata. - return AffixFuzzer2Type() +class AffixFuzzer2Array(BaseExtensionArray[AffixFuzzer2ArrayLike]): + _EXTENSION_NAME = "rerun.testing.datatypes.AffixFuzzer2" + _EXTENSION_TYPE = AffixFuzzer2Type + + @staticmethod + def _native_to_pa_array(data: AffixFuzzer2ArrayLike, data_type: pa.DataType) -> pa.Array: + raise NotImplementedError - def __arrow_ext_class__(self: type[pa.ExtensionType]) -> type[pa.ExtensionArray]: - return AffixFuzzer2Array +AffixFuzzer2Type._ARRAY_TYPE = AffixFuzzer2Array # TODO(cmc): bring back registration to pyarrow once legacy types are gone # pa.register_extension_type(AffixFuzzer2Type()) - - -class AffixFuzzer2Array(AffixFuzzer2ArrayExt): # type: ignore[misc] - _extension_name = "rerun.testing.datatypes.AffixFuzzer2" - - @staticmethod - def from_similar(data: AffixFuzzer2ArrayLike | None) -> pa.Array: - if data is None: - return AffixFuzzer2Type().wrap_array(pa.array([], type=AffixFuzzer2Type().storage_type)) - else: - return AffixFuzzer2ArrayExt._from_similar( - data, - mono=AffixFuzzer2, - mono_aliases=AffixFuzzer2Like, - many=AffixFuzzer2Array, - many_aliases=AffixFuzzer2ArrayLike, - arrow=AffixFuzzer2Type, - ) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/point2d.py b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/point2d.py new file mode 100644 index 000000000000..3c0de2f2b122 --- /dev/null +++ b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/point2d.py @@ -0,0 +1,68 @@ +# NOTE: This file was autogenerated by re_types_builder; DO NOT EDIT. + +from __future__ import annotations + +from typing import Sequence, Union + +import numpy as np +import numpy.typing as npt +import pyarrow as pa +from attrs import define, field + +from .._baseclasses import ( + BaseExtensionArray, + BaseExtensionType, +) +from ._overrides import point2d_as_array, point2d_native_to_pa_array # noqa: F401 + +__all__ = ["Point2D", "Point2DArray", "Point2DArrayLike", "Point2DLike", "Point2DType"] + + +@define +class Point2D: + """A point in 2D space.""" + + x: float = field() + y: float = field() + + def __array__(self, dtype: npt.DTypeLike = None) -> npt.ArrayLike: + return point2d_as_array(self, dtype=dtype) + + +Point2DLike = Union[Point2D, Sequence[float]] + +Point2DArrayLike = Union[ + Point2D, + Sequence[Point2DLike], + npt.NDArray[np.float32], + Sequence[npt.NDArray[np.float32]], + Sequence[tuple[float, float]], + Sequence[float], +] + + +# --- Arrow support --- + + +class Point2DType(BaseExtensionType): + def __init__(self) -> None: + pa.ExtensionType.__init__( + self, + pa.struct([pa.field("x", pa.float32(), False, {}), pa.field("y", pa.float32(), False, {})]), + "rerun.datatypes.Point2D", + ) + + +class Point2DArray(BaseExtensionArray[Point2DArrayLike]): + _EXTENSION_NAME = "rerun.datatypes.Point2D" + _EXTENSION_TYPE = Point2DType + + @staticmethod + def _native_to_pa_array(data: Point2DArrayLike, data_type: pa.DataType) -> pa.Array: + return point2d_native_to_pa_array(data, data_type) + + +Point2DType._ARRAY_TYPE = Point2DArray + +# TODO(cmc): bring back registration to pyarrow once legacy types are gone +# pa.register_extension_type(Point2DType()) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/vec2d.py b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/vec2d.py index 14aa1d0a2a1c..c0a553a9a0b9 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/vec2d.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/vec2d.py @@ -2,77 +2,60 @@ from __future__ import annotations -from dataclasses import dataclass -from typing import Any, Sequence, Union +from typing import Sequence, Union import numpy as np import numpy.typing as npt import pyarrow as pa +from attrs import define, field -__all__ = ["Vec2D", "Vec2DArray", "Vec2DArrayLike", "Vec2DLike", "Vec2DType"] - +from .._baseclasses import ( + BaseExtensionArray, + BaseExtensionType, +) +from .._converters import ( + to_np_float32, +) +from ._overrides import vec2d_native_to_pa_array # noqa: F401 -## --- Vec2D --- ## +__all__ = ["Vec2D", "Vec2DArray", "Vec2DArrayLike", "Vec2DLike", "Vec2DType"] -@dataclass +@define class Vec2D: """A vector in 2D space.""" - xy: npt.ArrayLike + xy: npt.NDArray[np.float32] = field(converter=to_np_float32) - def __array__(self) -> npt.ArrayLike: - return np.asarray(self.xy) + def __array__(self, dtype: npt.DTypeLike = None) -> npt.ArrayLike: + return np.asarray(self.xy, dtype=dtype) -Vec2DLike = Vec2D +Vec2DLike = Union[Vec2D, tuple[float, float]] + Vec2DArrayLike = Union[ - Vec2DLike, - Sequence[Vec2DLike], + Vec2D, Sequence[Vec2DLike], npt.NDArray[np.float32], Sequence[tuple[float, float]], Sequence[float] ] # --- Arrow support --- -from .vec2d_ext import Vec2DArrayExt # noqa: E402 - -class Vec2DType(pa.ExtensionType): # type: ignore[misc] - def __init__(self: type[pa.ExtensionType]) -> None: +class Vec2DType(BaseExtensionType): + def __init__(self) -> None: pa.ExtensionType.__init__(self, pa.list_(pa.field("item", pa.float32(), False, {}), 2), "rerun.datatypes.Vec2D") - def __arrow_ext_serialize__(self: type[pa.ExtensionType]) -> bytes: - # since we don't have a parameterized type, we don't need extra metadata to be deserialized - return b"" - @classmethod - def __arrow_ext_deserialize__( - cls: type[pa.ExtensionType], storage_type: Any, serialized: Any - ) -> type[pa.ExtensionType]: - # return an instance of this subclass given the serialized metadata. - return Vec2DType() +class Vec2DArray(BaseExtensionArray[Vec2DArrayLike]): + _EXTENSION_NAME = "rerun.datatypes.Vec2D" + _EXTENSION_TYPE = Vec2DType - def __arrow_ext_class__(self: type[pa.ExtensionType]) -> type[pa.ExtensionArray]: - return Vec2DArray + @staticmethod + def _native_to_pa_array(data: Vec2DArrayLike, data_type: pa.DataType) -> pa.Array: + return vec2d_native_to_pa_array(data, data_type) +Vec2DType._ARRAY_TYPE = Vec2DArray + # TODO(cmc): bring back registration to pyarrow once legacy types are gone # pa.register_extension_type(Vec2DType()) - - -class Vec2DArray(Vec2DArrayExt): # type: ignore[misc] - _extension_name = "rerun.datatypes.Vec2D" - - @staticmethod - def from_similar(data: Vec2DArrayLike | None) -> pa.Array: - if data is None: - return Vec2DType().wrap_array(pa.array([], type=Vec2DType().storage_type)) - else: - return Vec2DArrayExt._from_similar( - data, - mono=Vec2D, - mono_aliases=Vec2DLike, - many=Vec2DArray, - many_aliases=Vec2DArrayLike, - arrow=Vec2DType, - ) From 9d26e3124d1e22a0116d0f5376de9fe6a13baa01 Mon Sep 17 00:00:00 2001 From: Antoine Beyeler Date: Tue, 4 Jul 2023 21:33:26 +0200 Subject: [PATCH 03/17] Several project and typing fix - added missing attrs dependency - CI: use Python 3.11 for python linting - fixed a bunch of old-Python typing issue --- .../workflows/reusable_build_and_test_wheels.yml | 2 +- .github/workflows/reusable_checks.yml | 2 +- .../definitions/rerun/components/color.fbs | 4 ++-- .../definitions/rerun/datatypes/point2d.fbs | 2 +- .../re_types/definitions/rerun/datatypes/vec2d.fbs | 4 ++-- rerun_py/pyproject.toml | 1 + rerun_py/requirements-build.txt | 1 + rerun_py/rerun_sdk/rerun/_rerun2/_baseclasses.py | 14 ++++++++------ .../rerun/_rerun2/components/_overrides/color.py | 4 ++-- .../rerun_sdk/rerun/_rerun2/components/color.py | 4 ++-- .../rerun/_rerun2/datatypes/_overrides/point2d.py | 14 +++++++++++++- .../rerun_sdk/rerun/_rerun2/datatypes/point2d.py | 4 ++-- .../rerun_sdk/rerun/_rerun2/datatypes/vec2d.py | 6 +++--- 13 files changed, 39 insertions(+), 23 deletions(-) diff --git a/.github/workflows/reusable_build_and_test_wheels.yml b/.github/workflows/reusable_build_and_test_wheels.yml index 12fa4c009d13..196a9f03425c 100644 --- a/.github/workflows/reusable_build_and_test_wheels.yml +++ b/.github/workflows/reusable_build_and_test_wheels.yml @@ -243,7 +243,7 @@ jobs: # TODO(jleibs): understand why deps can't be installed in the same step as the wheel shell: bash run: | - pip install deprecated numpy>=1.23 pillow pyarrow==10.0.1 pytest==7.1.2 + pip install attrs>=23.1.0 deprecated numpy>=1.23 pillow pyarrow==10.0.1 pytest==7.1.2 - name: Install built wheel if: needs.set-config.outputs.RUN_TESTS == 'true' diff --git a/.github/workflows/reusable_checks.yml b/.github/workflows/reusable_checks.yml index 92932e7c3570..acf7afae1f12 100644 --- a/.github/workflows/reusable_checks.yml +++ b/.github/workflows/reusable_checks.yml @@ -62,7 +62,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: "3.11" cache: "pip" cache-dependency-path: "rerun_py/requirements-lint.txt" diff --git a/crates/re_types/definitions/rerun/components/color.fbs b/crates/re_types/definitions/rerun/components/color.fbs index a610bd8991fb..fd1fb2afcdbf 100644 --- a/crates/re_types/definitions/rerun/components/color.fbs +++ b/crates/re_types/definitions/rerun/components/color.fbs @@ -16,8 +16,8 @@ namespace rerun.components; /// \py If there is an alpha, we assume it is in linear space, and separate (NOT pre-multiplied). struct Color ( "attr.arrow.transparent", - "attr.python.aliases": "int, Sequence[int], npt.NDArray[np.uint8 | np.float32 | np.float64]", - "attr.python.array_aliases": "Sequence[Sequence[int]], npt.NDArray[np.uint8 | np.uint32 | np.float32 | np.float64]", + "attr.python.aliases": "int, Sequence[int], npt.NDArray[Union[np.uint8, np.float32, np.float64]]", + "attr.python.array_aliases": "Sequence[Sequence[int]], npt.NDArray[Union[np.uint8, np.uint32, np.float32, np.float64]]", "attr.rerun.legacy_fqname": "rerun.colorrgba", "attr.rust.derive": "Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, bytemuck::Pod, bytemuck::Zeroable", "attr.rust.repr": "transparent", diff --git a/crates/re_types/definitions/rerun/datatypes/point2d.fbs b/crates/re_types/definitions/rerun/datatypes/point2d.fbs index a635023f4f9d..6fd963408127 100644 --- a/crates/re_types/definitions/rerun/datatypes/point2d.fbs +++ b/crates/re_types/definitions/rerun/datatypes/point2d.fbs @@ -10,7 +10,7 @@ namespace rerun.datatypes; /// A point in 2D space. struct Point2D ( "attr.python.aliases": "Sequence[float]", - "attr.python.array_aliases": "npt.NDArray[np.float32], Sequence[npt.NDArray[np.float32]], Sequence[tuple[float, float]], Sequence[float]", + "attr.python.array_aliases": "npt.NDArray[np.float32], Sequence[npt.NDArray[np.float32]], Sequence[Tuple[float, float]], Sequence[float]", "attr.rust.derive": "Debug, Default, Clone, Copy, PartialEq, PartialOrd", order: 100 ) { diff --git a/crates/re_types/definitions/rerun/datatypes/vec2d.fbs b/crates/re_types/definitions/rerun/datatypes/vec2d.fbs index c423871afa81..e3cf2b5dc835 100644 --- a/crates/re_types/definitions/rerun/datatypes/vec2d.fbs +++ b/crates/re_types/definitions/rerun/datatypes/vec2d.fbs @@ -10,8 +10,8 @@ namespace rerun.datatypes; /// A vector in 2D space. struct Vec2D ( "attr.arrow.transparent", - "attr.python.aliases": "tuple[float, float]", - "attr.python.array_aliases": "npt.NDArray[np.float32], Sequence[tuple[float, float]], Sequence[float]", + "attr.python.aliases": "Tuple[float, float]", + "attr.python.array_aliases": "npt.NDArray[np.float32], Sequence[Tuple[float, float]], Sequence[float]", "attr.rust.derive": "Debug, Default, Clone, Copy, PartialEq, PartialOrd", "attr.rust.tuple_struct", order: 100 diff --git a/rerun_py/pyproject.toml b/rerun_py/pyproject.toml index e2779b57bfa3..be287415de01 100644 --- a/rerun_py/pyproject.toml +++ b/rerun_py/pyproject.toml @@ -13,6 +13,7 @@ classifiers = [ ] dependencies = [ # Must match list in `.github/workflows/reusable_build_and_test_wheels.yml` + "attrs>=23.1.0", "deprecated", "numpy>=1.23", "pillow", # Used for JPEG encoding diff --git a/rerun_py/requirements-build.txt b/rerun_py/requirements-build.txt index 4ab2b1e21499..8027d9bd118f 100644 --- a/rerun_py/requirements-build.txt +++ b/rerun_py/requirements-build.txt @@ -1,3 +1,4 @@ +attrs>=23.1.0 maturin>=0.14,<0.15 semver>=2.13,<2.14 wheel>=0.38,<0.39 diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/_baseclasses.py b/rerun_py/rerun_sdk/rerun/_rerun2/_baseclasses.py index 03a4f88f3d75..c17fb55c0915 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/_baseclasses.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/_baseclasses.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Generic, Self, TypeVar, cast +from typing import Any, Generic, TypeVar, cast import pyarrow as pa from attrs import define, fields @@ -67,13 +67,15 @@ class BaseExtensionArray(NamedExtensionArray, Generic[T]): # type: ignore[misc] """The extension type class associated with this class.""" @classmethod - def from_similar(cls, data: T | None) -> Self: + def from_similar(cls, data: T | None) -> BaseExtensionArray[T]: data_type = cls._EXTENSION_TYPE() if data is None: - return cast(Self, data_type.wrap_array(pa.array([], type=data_type.storage_type))) + return cast(BaseExtensionArray[T], data_type.wrap_array(pa.array([], type=data_type.storage_type))) else: - return cast(Self, data_type.wrap_array(cls._native_to_pa_array(data, data_type.storage_type))) + return cast( + BaseExtensionArray[T], data_type.wrap_array(cls._native_to_pa_array(data, data_type.storage_type)) + ) @staticmethod def _native_to_pa_array(data: T, data_type: pa.DataType) -> pa.Array: @@ -117,8 +119,8 @@ class BaseDelegatingExtensionArray(BaseExtensionArray[T]): # type: ignore[misc] """The extension array class associated with this component's datatype.""" @classmethod - def from_similar(cls, data: T | None) -> Self: + def from_similar(cls, data: T | None) -> BaseDelegatingExtensionArray[T]: arr = cls._DELEGATED_ARRAY_TYPE.from_similar(data) # TODO(ab, cmc): we unwrap the type here because we can't have two layers of extension types for now - return cast(Self, cls._EXTENSION_TYPE().wrap_array(arr.storage)) + return cast(BaseDelegatingExtensionArray[T], cls._EXTENSION_TYPE().wrap_array(arr.storage)) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/color.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/color.py index 6e4ceba62550..dc1a21d3bfe6 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/color.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/components/_overrides/color.py @@ -14,7 +14,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Sequence, cast +from typing import TYPE_CHECKING, Sequence, Union, cast import numpy as np import numpy.typing as npt @@ -67,7 +67,7 @@ def color_native_to_pa_array(data: ColorArrayLike, data_type: pa.DataType) -> pa else: # a single color data = data.reshape((1, -1)) - array = _numpy_array_to_u32(cast(npt.NDArray[np.uint8 | np.float32 | np.float64], data)) + array = _numpy_array_to_u32(cast(npt.NDArray[Union[np.uint8, np.float32, np.float64]], data)) else: # Sequence of Color (which are int-like) or sequence of sequence array = np.array([Color(datum) for datum in data], np.uint32) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/color.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/color.py index 6b0371646fad..4b6639dc78b0 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/components/color.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/components/color.py @@ -37,10 +37,10 @@ def __int__(self) -> int: return int(self.rgba) -ColorLike = Union[Color, int, Sequence[int], npt.NDArray[np.uint8 | np.float32 | np.float64]] +ColorLike = Union[Color, int, Sequence[int], npt.NDArray[Union[np.uint8, np.float32, np.float64]]] ColorArrayLike = Union[ - Color, Sequence[ColorLike], Sequence[Sequence[int]], npt.NDArray[np.uint8 | np.uint32 | np.float32 | np.float64] + Color, Sequence[ColorLike], Sequence[Sequence[int]], npt.NDArray[Union[np.uint8, np.uint32, np.float32, np.float64]] ] diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/_overrides/point2d.py b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/_overrides/point2d.py index 6be1d24fbd97..02fd917b9739 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/_overrides/point2d.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/_overrides/point2d.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Sequence import numpy as np import pyarrow as pa @@ -11,11 +11,23 @@ from .. import Point2D, Point2DArrayLike +NUMPY_VERSION = tuple(map(int, np.version.version.split(".")[:2])) + + def point2d_as_array(data: Point2D, dtype: npt.DTypeLike = None) -> npt.ArrayLike: return np.array([data.x, data.y], dtype=dtype) def point2d_native_to_pa_array(data: Point2DArrayLike, data_type: pa.DataType) -> pa.Array: + from .. import Point2D + + # TODO(ab): get rid of this once we drop support for Python 3.8. Make sure to pin numpy>=1.25. + if NUMPY_VERSION < (1, 25): + # Older numpy doesn't seem to support `data` in the form of [Point2D(1, 2), Point2D(3, 4)] + # this happens for python 3.8 (1.25 supports 3.9+) + if isinstance(data, Sequence): + data = [point2d_as_array(p) if isinstance(p, Point2D) else p for p in data] # type: ignore[assignment] + points = np.asarray(data, dtype=np.float32).reshape((-1, 2)) return pa.StructArray.from_arrays( arrays=[pa.array(c, type=pa.float32()) for c in points.T], diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/point2d.py b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/point2d.py index 3c0de2f2b122..23ff209c596d 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/point2d.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/point2d.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Sequence, Union +from typing import Sequence, Tuple, Union import numpy as np import numpy.typing as npt @@ -36,7 +36,7 @@ def __array__(self, dtype: npt.DTypeLike = None) -> npt.ArrayLike: Sequence[Point2DLike], npt.NDArray[np.float32], Sequence[npt.NDArray[np.float32]], - Sequence[tuple[float, float]], + Sequence[Tuple[float, float]], Sequence[float], ] diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/vec2d.py b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/vec2d.py index c0a553a9a0b9..8e4cb44e1e84 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/vec2d.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/vec2d.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Sequence, Union +from typing import Sequence, Tuple, Union import numpy as np import numpy.typing as npt @@ -31,10 +31,10 @@ def __array__(self, dtype: npt.DTypeLike = None) -> npt.ArrayLike: return np.asarray(self.xy, dtype=dtype) -Vec2DLike = Union[Vec2D, tuple[float, float]] +Vec2DLike = Union[Vec2D, Tuple[float, float]] Vec2DArrayLike = Union[ - Vec2D, Sequence[Vec2DLike], npt.NDArray[np.float32], Sequence[tuple[float, float]], Sequence[float] + Vec2D, Sequence[Vec2DLike], npt.NDArray[np.float32], Sequence[Tuple[float, float]], Sequence[float] ] From 87195213acf79af97e44e8a0bd0965f97b87a006 Mon Sep 17 00:00:00 2001 From: Antoine Beyeler Date: Tue, 4 Jul 2023 21:56:29 +0200 Subject: [PATCH 04/17] Added attrs to requirements-lint.txt --- rerun_py/requirements-lint.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/rerun_py/requirements-lint.txt b/rerun_py/requirements-lint.txt index 492df037a6eb..7ae26628df0b 100644 --- a/rerun_py/requirements-lint.txt +++ b/rerun_py/requirements-lint.txt @@ -1,3 +1,4 @@ +attrs>=23.1.0 # for mypy to work black==23.3.0 blackdoc==0.3.8 mypy==1.4.1 From 6f986d39f9e43239efb47b1cd3daa36b6ec105cf Mon Sep 17 00:00:00 2001 From: Antoine Beyeler Date: Wed, 5 Jul 2023 11:01:00 +0200 Subject: [PATCH 05/17] source_hash.txt --- crates/re_types/source_hash.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/re_types/source_hash.txt b/crates/re_types/source_hash.txt index a8ffe0e496e5..070d0bc1824d 100644 --- a/crates/re_types/source_hash.txt +++ b/crates/re_types/source_hash.txt @@ -1,4 +1,4 @@ # This is a sha256 hash for all direct and indirect dependencies of this crate's build script. # It can be safely removed at anytime to force the build script to run again. # Check out build.rs to see how it's computed. -d828fdb05c35a01b92dd30b7ce40987b87c5dfe8d1bb8f729004ae88b62c830c \ No newline at end of file +381e94250d2f9d08019c9c4991c27be59b2b9bc31a1c12ffc7e7b7577a0da4d5 \ No newline at end of file From 2f798581f5a8b4671c78717ab7a3029e258babd0 Mon Sep 17 00:00:00 2001 From: Clement Rey Date: Wed, 5 Jul 2023 13:13:48 +0200 Subject: [PATCH 06/17] remove old and awkward 'attr.python.transparent' --- crates/re_types/definitions/python/attributes.fbs | 5 ----- crates/re_types/source_hash.txt | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/crates/re_types/definitions/python/attributes.fbs b/crates/re_types/definitions/python/attributes.fbs index c24923751719..7e35944c6c8f 100644 --- a/crates/re_types/definitions/python/attributes.fbs +++ b/crates/re_types/definitions/python/attributes.fbs @@ -1,10 +1,5 @@ namespace python.attributes; -/// Marks a field as transparent, meaning its type will be replaced by the underlying type. -/// -/// Only applies to fields whose type is an object with a single-field. -attribute "attr.python.transparent"; - /// Defines the type aliases for a component, e.g. the types that make up `ComponentLike`. /// /// Only applies to structs/unions that are components. diff --git a/crates/re_types/source_hash.txt b/crates/re_types/source_hash.txt index 070d0bc1824d..fccf395423bf 100644 --- a/crates/re_types/source_hash.txt +++ b/crates/re_types/source_hash.txt @@ -1,4 +1,4 @@ # This is a sha256 hash for all direct and indirect dependencies of this crate's build script. # It can be safely removed at anytime to force the build script to run again. # Check out build.rs to see how it's computed. -381e94250d2f9d08019c9c4991c27be59b2b9bc31a1c12ffc7e7b7577a0da4d5 \ No newline at end of file +4c67b6346b4c454aa54d78c733cfeed7acdf8dbaeadd9ad26b49551a20ccd281 \ No newline at end of file From 7d25d36fd355052394f03afad9d7fb8757c2bab7 Mon Sep 17 00:00:00 2001 From: Clement Rey Date: Wed, 5 Jul 2023 13:16:51 +0200 Subject: [PATCH 07/17] ...and all the associated code --- crates/re_types/source_hash.txt | 2 +- crates/re_types_builder/src/codegen/python.rs | 29 ++++--------------- crates/re_types_builder/src/lib.rs | 1 - 3 files changed, 6 insertions(+), 26 deletions(-) diff --git a/crates/re_types/source_hash.txt b/crates/re_types/source_hash.txt index fccf395423bf..7fe6bbeed553 100644 --- a/crates/re_types/source_hash.txt +++ b/crates/re_types/source_hash.txt @@ -1,4 +1,4 @@ # This is a sha256 hash for all direct and indirect dependencies of this crate's build script. # It can be safely removed at anytime to force the build script to run again. # Check out build.rs to see how it's computed. -4c67b6346b4c454aa54d78c733cfeed7acdf8dbaeadd9ad26b49551a20ccd281 \ No newline at end of file +20c6f77ae15730361eec65f0007fb9f28496b1e737f9fce2ef0efa117ca157b9 \ No newline at end of file diff --git a/crates/re_types_builder/src/codegen/python.rs b/crates/re_types_builder/src/codegen/python.rs index 7d7dfadbd60f..7b85db6ac60f 100644 --- a/crates/re_types_builder/src/codegen/python.rs +++ b/crates/re_types_builder/src/codegen/python.rs @@ -11,8 +11,7 @@ use std::{ use crate::{ codegen::{StringExt as _, AUTOGEN_WARNING}, ArrowRegistry, CodeGenerator, Docs, ElementType, Object, ObjectField, ObjectKind, Objects, - Type, ATTR_PYTHON_ALIASES, ATTR_PYTHON_ARRAY_ALIASES, ATTR_PYTHON_TRANSPARENT, - ATTR_RERUN_LEGACY_FQNAME, + Type, ATTR_PYTHON_ALIASES, ATTR_PYTHON_ARRAY_ALIASES, ATTR_RERUN_LEGACY_FQNAME, }; // --- @@ -721,7 +720,7 @@ fn quote_array_method_from_obj( objects: &Objects, obj: &Object, ) -> String { - // TODO(cmc): should be using native type, but need transparency + // TODO(cmc): should be using the native type, but need to compare numpy types etc let typ = quote_field_type_from_field(objects, &obj.fields[0], false).0; // allow overriding the __array__ function @@ -769,7 +768,7 @@ fn quote_native_types_method_from_obj(objects: &Objects, obj: &Object) -> String // that field cannot be optional || obj.fields[0].is_nullable // that single field must be of a supported native type - // TODO(cmc): should be using native type, but need transparency + // TODO(cmc): should be using the native type, but need to compare numpy types etc || !["str", "int", "float"].contains(&typ) { return String::new(); @@ -872,7 +871,7 @@ fn quote_import_clauses_from_field(field: &ObjectField) -> Option { /// The returned boolean indicates whether there was anything to unwrap at all. /// The third returned value is a default converter function if any. fn quote_field_type_from_field( - objects: &Objects, + _objects: &Objects, field: &ObjectField, unwrap: bool, ) -> (String, bool, String) { @@ -936,25 +935,7 @@ fn quote_field_type_from_field( } } } - Type::Object(fqname) => { - // TODO(cmc): it is a bit weird to be doing the transparency logic (which is language - // agnostic) in a python specific quoting function... a static helper at the very least - // would be nice. - let is_transparent = field - .try_get_attr::(ATTR_PYTHON_TRANSPARENT) - .is_some(); - if is_transparent { - let target = objects.get(fqname); - assert_eq!(target.fields.len(), 1, "transparent field must point to an object with exactly 1 field, but {:?} has {}", fqname, target.fields.len()); - // NOTE: unwrap call is safe due to assertion just above - return quote_field_type_from_field( - objects, - target.fields.first().unwrap(), - unwrap, - ); - } - quote_type_from_element_type(&ElementType::Object(fqname.clone())) - } + Type::Object(fqname) => quote_type_from_element_type(&ElementType::Object(fqname.clone())), }; (typ, unwrapped, converter) diff --git a/crates/re_types_builder/src/lib.rs b/crates/re_types_builder/src/lib.rs index 1516a0468355..50d21171afae 100644 --- a/crates/re_types_builder/src/lib.rs +++ b/crates/re_types_builder/src/lib.rs @@ -146,7 +146,6 @@ pub const ATTR_RERUN_COMPONENT_RECOMMENDED: &str = "attr.rerun.component_recomme pub const ATTR_RERUN_COMPONENT_REQUIRED: &str = "attr.rerun.component_required"; pub const ATTR_RERUN_LEGACY_FQNAME: &str = "attr.rerun.legacy_fqname"; -pub const ATTR_PYTHON_TRANSPARENT: &str = "attr.python.transparent"; pub const ATTR_PYTHON_ALIASES: &str = "attr.python.aliases"; pub const ATTR_PYTHON_ARRAY_ALIASES: &str = "attr.python.array_aliases"; From f7f4e3b76c4c4bad7b10b0d817f4ef1fe1379987 Mon Sep 17 00:00:00 2001 From: Clement Rey Date: Wed, 5 Jul 2023 13:24:05 +0200 Subject: [PATCH 08/17] rerun.components.Point2D should def be arrow transparent --- .../definitions/rerun/components/point2d.fbs | 1 + crates/re_types/source_hash.txt | 2 +- crates/re_types/src/components/point2d.rs | 125 +++++------------- 3 files changed, 35 insertions(+), 93 deletions(-) diff --git a/crates/re_types/definitions/rerun/components/point2d.fbs b/crates/re_types/definitions/rerun/components/point2d.fbs index 09d00d3dd757..8a69d406e151 100644 --- a/crates/re_types/definitions/rerun/components/point2d.fbs +++ b/crates/re_types/definitions/rerun/components/point2d.fbs @@ -12,6 +12,7 @@ namespace rerun.components; /// A point in 2D space. // TODO(cmc): bring back attr.rust.tuple_struct struct Point2D ( + "attr.arrow.transparent", "attr.python.aliases": "npt.NDArray[np.float32], Sequence[float], Tuple[float, float]", "attr.python.array_aliases": "npt.NDArray[np.float32], Sequence[float]", "attr.rerun.legacy_fqname": "rerun.point2d", diff --git a/crates/re_types/source_hash.txt b/crates/re_types/source_hash.txt index 7fe6bbeed553..df20573e1880 100644 --- a/crates/re_types/source_hash.txt +++ b/crates/re_types/source_hash.txt @@ -1,4 +1,4 @@ # This is a sha256 hash for all direct and indirect dependencies of this crate's build script. # It can be safely removed at anytime to force the build script to run again. # Check out build.rs to see how it's computed. -20c6f77ae15730361eec65f0007fb9f28496b1e737f9fce2ef0efa117ca157b9 \ No newline at end of file +51003743447ae6a7b2985c1b6761319e42eb446fcb86248431871bddf5817985 \ No newline at end of file diff --git a/crates/re_types/src/components/point2d.rs b/crates/re_types/src/components/point2d.rs index 8990300c4f91..49087d07c6ae 100644 --- a/crates/re_types/src/components/point2d.rs +++ b/crates/re_types/src/components/point2d.rs @@ -38,25 +38,20 @@ impl crate::Component for Point2D { #[inline] fn to_arrow_datatype() -> arrow2::datatypes::DataType { use ::arrow2::datatypes::*; - DataType::Struct(vec![Field { - name: "xy".to_owned(), - data_type: DataType::Struct(vec![ - Field { - name: "x".to_owned(), - data_type: DataType::Float32, - is_nullable: false, - metadata: [].into(), - }, - Field { - name: "y".to_owned(), - data_type: DataType::Float32, - is_nullable: false, - metadata: [].into(), - }, - ]), - is_nullable: false, - metadata: [].into(), - }]) + DataType::Struct(vec![ + Field { + name: "x".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "y".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }, + ]) } #[allow(unused_imports, clippy::wildcard_imports)] @@ -70,53 +65,26 @@ impl crate::Component for Point2D { use crate::{Component as _, Datatype as _}; use ::arrow2::{array::*, datatypes::*}; Ok({ - let (somes, data): (Vec<_>, Vec<_>) = data + let (somes, xy): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { let datum: Option<::std::borrow::Cow<'a, Self>> = datum.map(Into::into); + let datum = datum.map(|datum| { + let Self { xy } = datum.into_owned(); + xy + }); (datum.is_some(), datum) }) .unzip(); - let bitmap: Option<::arrow2::bitmap::Bitmap> = { + let xy_bitmap: Option<::arrow2::bitmap::Bitmap> = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - (if let Some(ext) = extension_wrapper { - DataType::Extension( - ext.to_owned(), - Box::new(::to_arrow_datatype()), - None, - ) - } else { - ::to_arrow_datatype() - }) - .to_logical_type() - .clone(), - vec![{ - let (somes, xy): (Vec<_>, Vec<_>) = data - .iter() - .map(|datum| { - let datum = datum.as_ref().map(|datum| { - let Self { xy, .. } = &**datum; - xy.clone() - }); - (datum.is_some(), datum) - }) - .unzip(); - let xy_bitmap: Option<::arrow2::bitmap::Bitmap> = { - let any_nones = somes.iter().any(|some| !*some); - any_nones.then(|| somes.into()) - }; - { - _ = xy_bitmap; - _ = extension_wrapper; - crate::datatypes::Point2D::try_to_arrow_opt(xy, None::<&str>)? - } - }], - bitmap, - ) - .boxed() + { + _ = xy_bitmap; + _ = extension_wrapper; + crate::datatypes::Point2D::try_to_arrow_opt(xy, Some("rerun.components.Point2D"))? + } }) } @@ -129,41 +97,14 @@ impl crate::Component for Point2D { { use crate::{Component as _, Datatype as _}; use ::arrow2::{array::*, datatypes::*}; - Ok({ - let data = data - .as_any() - .downcast_ref::<::arrow2::array::StructArray>() - .ok_or_else(|| crate::DeserializationError::SchemaMismatch { - expected: data.data_type().clone(), - got: data.data_type().clone(), - })?; - let (data_fields, data_arrays, data_bitmap) = - (data.fields(), data.values(), data.validity()); - let is_valid = |i| data_bitmap.map_or(true, |bitmap| bitmap.get_bit(i)); - let arrays_by_name: ::std::collections::HashMap<_, _> = data_fields - .iter() - .map(|field| field.name.as_str()) - .zip(data_arrays) - .collect(); - let xy = { - let data = &**arrays_by_name["xy"]; - - crate::datatypes::Point2D::try_from_arrow_opt(data)?.into_iter() - }; - ::itertools::izip!(xy) - .enumerate() - .map(|(i, (xy))| { - is_valid(i) - .then(|| { - Ok(Self { - xy: xy.ok_or_else(|| crate::DeserializationError::MissingData { - datatype: data.data_type().clone(), - })?, - }) - }) - .transpose() + Ok(crate::datatypes::Point2D::try_from_arrow_opt(data)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: data.data_type().clone(), }) - .collect::>>()? - }) + }) + .map(|res| res.map(|xy| Some(Self { xy }))) + .collect::>>>()?) } } From aaa0a5e9cf0abd136287aa5a92bede688eaed4e1 Mon Sep 17 00:00:00 2001 From: Antoine Beyeler Date: Wed, 5 Jul 2023 15:31:01 +0200 Subject: [PATCH 09/17] Revived lots of fuzzer + added */_overrides to source_hash.txt --- crates/re_types/build.rs | 24 + .../rerun/testing/archetypes/fuzzy.fbs | 32 +- .../rerun/testing/components/fuzzy.fbs | 65 +- crates/re_types/source_hash.txt | 2 +- crates/re_types/src/archetypes/fuzzy.rs | 1313 ++++++++++++- crates/re_types/src/archetypes/points2d.rs | 1 + crates/re_types/src/components/class_id.rs | 1 + crates/re_types/src/components/color.rs | 1 + crates/re_types/src/components/draw_order.rs | 1 + crates/re_types/src/components/fuzzy.rs | 1690 +++++++++++++++++ .../re_types/src/components/instance_key.rs | 1 + crates/re_types/src/components/keypoint_id.rs | 1 + crates/re_types/src/components/label.rs | 1 + crates/re_types/src/components/mod.rs | 4 +- crates/re_types/src/components/point2d.rs | 1 + crates/re_types/src/components/radius.rs | 1 + crates/re_types/src/datatypes/fuzzy.rs | 1 + crates/re_types/src/datatypes/point2d.rs | 1 + crates/re_types/src/datatypes/vec2d.rs | 1 + crates/re_types_builder/src/codegen/rust.rs | 1 + .../rerun/_rerun2/archetypes/fuzzy.py | 126 ++ .../rerun/_rerun2/components/__init__.py | 70 + .../rerun/_rerun2/components/fuzzy.py | 369 ++++ 23 files changed, 3622 insertions(+), 86 deletions(-) diff --git a/crates/re_types/build.rs b/crates/re_types/build.rs index ec98cc46fbb9..ed86f2a61a2a 100644 --- a/crates/re_types/build.rs +++ b/crates/re_types/build.rs @@ -20,6 +20,11 @@ const DOC_EXAMPLES_DIR_PATH: &str = "../../docs/code-examples"; const RUST_OUTPUT_DIR_PATH: &str = "."; const PYTHON_OUTPUT_DIR_PATH: &str = "../../rerun_py/rerun_sdk/rerun/_rerun2"; +// located in PYTHON_OUTPUT_DIR_PATH +const ARCHETYPE_OVERRIDES_SUB_DIR_PATH: &str = "archetypes/_overrides"; +const COMPONENT_OVERRIDES_SUB_DIR_PATH: &str = "components/_overrides"; +const DATATYPE_OVERRIDES_SUB_DIR_PATH: &str = "datatypes/_overrides"; + fn main() { if cfg!(target_os = "windows") { // TODO(#2591): Codegen is temporarily disabled on Windows due to hashing issues. @@ -47,16 +52,35 @@ fn main() { let re_types_builder_hash = compute_crate_hash("re_types_builder"); let definitions_hash = compute_dir_hash(DEFINITIONS_DIR_PATH, Some(&["fbs"])); let doc_examples_hash = compute_dir_hash(DOC_EXAMPLES_DIR_PATH, Some(&["rs", "py"])); + let archetype_overrides_hash = compute_dir_hash( + PathBuf::from(PYTHON_OUTPUT_DIR_PATH).join(ARCHETYPE_OVERRIDES_SUB_DIR_PATH), + Some(&["py"]), + ); + let component_overrides_hash = compute_dir_hash( + PathBuf::from(PYTHON_OUTPUT_DIR_PATH).join(COMPONENT_OVERRIDES_SUB_DIR_PATH), + Some(&["py"]), + ); + let datatype_overrides_hash = compute_dir_hash( + PathBuf::from(PYTHON_OUTPUT_DIR_PATH).join(DATATYPE_OVERRIDES_SUB_DIR_PATH), + Some(&["py"]), + ); + let new_hash = compute_strings_hash(&[ &re_types_builder_hash, &definitions_hash, &doc_examples_hash, + &archetype_overrides_hash, + &component_overrides_hash, + &datatype_overrides_hash, ]); // Leave these be please, very useful when debugging. eprintln!("re_types_builder_hash: {re_types_builder_hash:?}"); eprintln!("definitions_hash: {definitions_hash:?}"); eprintln!("doc_examples_hash: {doc_examples_hash:?}"); + eprintln!("archetype_overrides_hash: {archetype_overrides_hash:?}"); + eprintln!("component_overrides_hash: {component_overrides_hash:?}"); + eprintln!("datatype_overrides_hash: {datatype_overrides_hash:?}"); eprintln!("new_hash: {new_hash:?}"); eprintln!("cur_hash: {cur_hash:?}"); diff --git a/crates/re_types/definitions/rerun/testing/archetypes/fuzzy.fbs b/crates/re_types/definitions/rerun/testing/archetypes/fuzzy.fbs index 9e91be6e71ee..96864a08551c 100644 --- a/crates/re_types/definitions/rerun/testing/archetypes/fuzzy.fbs +++ b/crates/re_types/definitions/rerun/testing/archetypes/fuzzy.fbs @@ -21,7 +21,13 @@ table AffixFuzzer1 ( fuzz1004: rerun.testing.components.AffixFuzzer4 ("attr.rerun.component_required", required, order: 1004); fuzz1005: rerun.testing.components.AffixFuzzer5 ("attr.rerun.component_required", required, order: 1005); fuzz1006: rerun.testing.components.AffixFuzzer6 ("attr.rerun.component_required", required, order: 1006); - // fuzz1007: rerun.testing.components.AffixFuzzer7 ("attr.rerun.component_required", required, order: 1007); + fuzz1007: rerun.testing.components.AffixFuzzer7 ("attr.rerun.component_required", required, order: 1007); + fuzz1008: rerun.testing.components.AffixFuzzer8 ("attr.rerun.component_required", required, order: 1008); + fuzz1009: rerun.testing.components.AffixFuzzer9 ("attr.rerun.component_required", required, order: 1009); + fuzz1010: rerun.testing.components.AffixFuzzer10 ("attr.rerun.component_required", required, order: 1010); + fuzz1011: rerun.testing.components.AffixFuzzer11 ("attr.rerun.component_required", required, order: 1011); + fuzz1012: rerun.testing.components.AffixFuzzer12 ("attr.rerun.component_required", required, order: 1012); + fuzz1013: rerun.testing.components.AffixFuzzer13 ("attr.rerun.component_required", required, order: 1013); fuzz1101: [rerun.testing.components.AffixFuzzer1] ("attr.rerun.component_required", required, order: 1101); fuzz1102: [rerun.testing.components.AffixFuzzer2] ("attr.rerun.component_required", required, order: 1102); @@ -29,7 +35,13 @@ table AffixFuzzer1 ( fuzz1104: [rerun.testing.components.AffixFuzzer4] ("attr.rerun.component_required", required, order: 1104); fuzz1105: [rerun.testing.components.AffixFuzzer5] ("attr.rerun.component_required", required, order: 1105); fuzz1106: [rerun.testing.components.AffixFuzzer6] ("attr.rerun.component_required", required, order: 1106); - // fuzz1107: [rerun.testing.components.AffixFuzzer7] ("attr.rerun.component_required", required, order: 1107); + fuzz1107: [rerun.testing.components.AffixFuzzer7] ("attr.rerun.component_required", required, order: 1107); + fuzz1108: [rerun.testing.components.AffixFuzzer8] ("attr.rerun.component_required", required, order: 1108); + fuzz1109: [rerun.testing.components.AffixFuzzer9] ("attr.rerun.component_required", required, order: 1109); + fuzz1110: [rerun.testing.components.AffixFuzzer10] ("attr.rerun.component_required", required, order: 1110); + fuzz1111: [rerun.testing.components.AffixFuzzer11] ("attr.rerun.component_required", required, order: 1111); + fuzz1112: [rerun.testing.components.AffixFuzzer12] ("attr.rerun.component_required", required, order: 1112); + fuzz1113: [rerun.testing.components.AffixFuzzer13] ("attr.rerun.component_required", required, order: 1113); fuzz2001: rerun.testing.components.AffixFuzzer1 ("attr.rerun.component_optional", order: 2001); fuzz2002: rerun.testing.components.AffixFuzzer2 ("attr.rerun.component_optional", order: 2002); @@ -37,7 +49,13 @@ table AffixFuzzer1 ( fuzz2004: rerun.testing.components.AffixFuzzer4 ("attr.rerun.component_optional", order: 2004); fuzz2005: rerun.testing.components.AffixFuzzer5 ("attr.rerun.component_optional", order: 2005); fuzz2006: rerun.testing.components.AffixFuzzer6 ("attr.rerun.component_optional", order: 2006); - // fuzz2007: rerun.testing.components.AffixFuzzer7 ("attr.rerun.component_optional", order: 2007); + fuzz2007: rerun.testing.components.AffixFuzzer7 ("attr.rerun.component_optional", order: 2007); + fuzz2008: rerun.testing.components.AffixFuzzer8 ("attr.rerun.component_optional", order: 2008); + fuzz2009: rerun.testing.components.AffixFuzzer9 ("attr.rerun.component_optional", order: 2009); + fuzz2010: rerun.testing.components.AffixFuzzer10 ("attr.rerun.component_optional", order: 2010); + fuzz2011: rerun.testing.components.AffixFuzzer11 ("attr.rerun.component_optional", order: 2011); + fuzz2012: rerun.testing.components.AffixFuzzer12 ("attr.rerun.component_optional", order: 2012); + fuzz2013: rerun.testing.components.AffixFuzzer13 ("attr.rerun.component_optional", order: 2013); fuzz2101: [rerun.testing.components.AffixFuzzer1] ("attr.rerun.component_optional", order: 2101); fuzz2102: [rerun.testing.components.AffixFuzzer2] ("attr.rerun.component_optional", order: 2102); @@ -45,6 +63,12 @@ table AffixFuzzer1 ( fuzz2104: [rerun.testing.components.AffixFuzzer4] ("attr.rerun.component_optional", order: 2104); fuzz2105: [rerun.testing.components.AffixFuzzer5] ("attr.rerun.component_optional", order: 2105); fuzz2106: [rerun.testing.components.AffixFuzzer6] ("attr.rerun.component_optional", order: 2106); - // fuzz2107: [rerun.testing.components.AffixFuzzer7] ("attr.rerun.component_optional", order: 2107); + fuzz2107: [rerun.testing.components.AffixFuzzer7] ("attr.rerun.component_optional", order: 2107); + fuzz2108: [rerun.testing.components.AffixFuzzer8] ("attr.rerun.component_optional", order: 2108); + fuzz2109: [rerun.testing.components.AffixFuzzer9] ("attr.rerun.component_optional", order: 2109); + fuzz2110: [rerun.testing.components.AffixFuzzer10] ("attr.rerun.component_optional", order: 2110); + fuzz2111: [rerun.testing.components.AffixFuzzer11] ("attr.rerun.component_optional", order: 2111); + fuzz2112: [rerun.testing.components.AffixFuzzer12] ("attr.rerun.component_optional", order: 2112); + fuzz2113: [rerun.testing.components.AffixFuzzer13] ("attr.rerun.component_optional", order: 2113); } diff --git a/crates/re_types/definitions/rerun/testing/components/fuzzy.fbs b/crates/re_types/definitions/rerun/testing/components/fuzzy.fbs index 82c1796bf9df..4bf9cd762d72 100644 --- a/crates/re_types/definitions/rerun/testing/components/fuzzy.fbs +++ b/crates/re_types/definitions/rerun/testing/components/fuzzy.fbs @@ -56,17 +56,60 @@ table AffixFuzzer6 ( single_optional: rerun.testing.datatypes.AffixFuzzer1; } -// table AffixFuzzer7 ( +table AffixFuzzer7 ( + "attr.rust.derive": "Debug, Clone, PartialEq", + order: 700 +) { + many_optional: [rerun.testing.datatypes.AffixFuzzer1] (order: 100); +} + +table AffixFuzzer8 ( + "attr.rust.derive": "Debug, Clone, PartialEq", + order: 800 +) { + single_float_optional: float (order: 101); +} + +table AffixFuzzer9 ( + "attr.rust.derive": "Debug, Clone, PartialEq, Eq", + order: 900 +) { + single_string_required: string (order: 102, required); +} + +table AffixFuzzer10 ( + "attr.rust.derive": "Debug, Clone, PartialEq, Eq", + order: 1000 +) { + single_string_optional: string (order: 103); +} + +table AffixFuzzer11 ( + "attr.rust.derive": "Debug, Clone, PartialEq", + order: 1100 +) { + many_floats_optional: [float] (order: 104); +} + +table AffixFuzzer12 ( + "attr.rust.derive": "Debug, Clone, PartialEq, Eq", + order: 1200 +) { + many_strings_required: [string] (order: 105, required); +} + +table AffixFuzzer13 ( + "attr.rust.derive": "Debug, Clone, PartialEq, Eq", + order: 1300 +) { + many_strings_optional: [string] (order: 106); +} + +// TODO(cmc): the ugly bug we need to take care of at some point +// table AffixFuzzer14 ( // "attr.rust.derive": "Debug, Clone, PartialEq", -// order: 700 +// order: 1400 // ) { -// many_optional: [rerun.testing.datatypes.AffixFuzzer1] (order: 100); -// single_float_optional: float (order: 101); -// single_string_required: string (order: 102, required); -// single_string_optional: string (order: 103); -// many_floats_optional: [float] (order: 104); -// many_strings_required: [string] (order: 105, required); -// many_strings_optional: [string] (order: 106); -// // TODO(cmc): the ugly bug we need to take care of at some point -// // many_transparent_optionals: rerun.testing.datatypes.AffixFuzzer2 (order: 107); +// +// many_transparent_optionals: rerun.testing.datatypes.AffixFuzzer2 (order: 107); // } diff --git a/crates/re_types/source_hash.txt b/crates/re_types/source_hash.txt index df20573e1880..e0a28faa88c3 100644 --- a/crates/re_types/source_hash.txt +++ b/crates/re_types/source_hash.txt @@ -1,4 +1,4 @@ # This is a sha256 hash for all direct and indirect dependencies of this crate's build script. # It can be safely removed at anytime to force the build script to run again. # Check out build.rs to see how it's computed. -51003743447ae6a7b2985c1b6761319e42eb446fcb86248431871bddf5817985 \ No newline at end of file +db9e1ce25f2c0b5e31b8fd5cb40adf0ff549811b7acdea660993f666731a648f \ No newline at end of file diff --git a/crates/re_types/src/archetypes/fuzzy.rs b/crates/re_types/src/archetypes/fuzzy.rs index c7ad3cd1d203..22609bed9f1b 100644 --- a/crates/re_types/src/archetypes/fuzzy.rs +++ b/crates/re_types/src/archetypes/fuzzy.rs @@ -6,6 +6,7 @@ #![allow(clippy::map_flatten)] #![allow(clippy::needless_question_mark)] #![allow(clippy::too_many_arguments)] +#![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] #[derive(Debug, Clone, PartialEq)] @@ -16,84 +17,168 @@ pub struct AffixFuzzer1 { pub fuzz1004: crate::components::AffixFuzzer4, pub fuzz1005: crate::components::AffixFuzzer5, pub fuzz1006: crate::components::AffixFuzzer6, + pub fuzz1007: crate::components::AffixFuzzer7, + pub fuzz1008: crate::components::AffixFuzzer8, + pub fuzz1009: crate::components::AffixFuzzer9, + pub fuzz1010: crate::components::AffixFuzzer10, + pub fuzz1011: crate::components::AffixFuzzer11, + pub fuzz1012: crate::components::AffixFuzzer12, + pub fuzz1013: crate::components::AffixFuzzer13, pub fuzz1101: Vec, pub fuzz1102: Vec, pub fuzz1103: Vec, pub fuzz1104: Vec, pub fuzz1105: Vec, pub fuzz1106: Vec, + pub fuzz1107: Vec, + pub fuzz1108: Vec, + pub fuzz1109: Vec, + pub fuzz1110: Vec, + pub fuzz1111: Vec, + pub fuzz1112: Vec, + pub fuzz1113: Vec, pub fuzz2001: Option, pub fuzz2002: Option, pub fuzz2003: Option, pub fuzz2004: Option, pub fuzz2005: Option, pub fuzz2006: Option, + pub fuzz2007: Option, + pub fuzz2008: Option, + pub fuzz2009: Option, + pub fuzz2010: Option, + pub fuzz2011: Option, + pub fuzz2012: Option, + pub fuzz2013: Option, pub fuzz2101: Option>, pub fuzz2102: Option>, pub fuzz2103: Option>, pub fuzz2104: Option>, pub fuzz2105: Option>, pub fuzz2106: Option>, + pub fuzz2107: Option>, + pub fuzz2108: Option>, + pub fuzz2109: Option>, + pub fuzz2110: Option>, + pub fuzz2111: Option>, + pub fuzz2112: Option>, + pub fuzz2113: Option>, } impl AffixFuzzer1 { - pub const REQUIRED_COMPONENTS: [crate::ComponentName; 12usize] = [ + pub const REQUIRED_COMPONENTS: [crate::ComponentName; 26usize] = [ crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer1"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer2"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer3"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer4"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer5"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer6"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer7"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer8"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer9"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer10"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer11"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer12"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer13"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer1"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer2"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer3"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer4"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer5"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer6"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer7"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer8"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer9"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer10"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer11"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer12"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer13"), ]; pub const RECOMMENDED_COMPONENTS: [crate::ComponentName; 0usize] = []; - pub const OPTIONAL_COMPONENTS: [crate::ComponentName; 12usize] = [ + pub const OPTIONAL_COMPONENTS: [crate::ComponentName; 26usize] = [ crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer1"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer2"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer3"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer4"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer5"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer6"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer7"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer8"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer9"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer10"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer11"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer12"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer13"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer1"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer2"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer3"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer4"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer5"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer6"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer7"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer8"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer9"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer10"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer11"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer12"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer13"), ]; - pub const ALL_COMPONENTS: [crate::ComponentName; 24usize] = [ + pub const ALL_COMPONENTS: [crate::ComponentName; 52usize] = [ crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer1"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer2"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer3"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer4"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer5"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer6"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer7"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer8"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer9"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer10"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer11"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer12"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer13"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer1"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer2"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer3"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer4"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer5"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer6"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer7"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer8"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer9"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer10"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer11"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer12"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer13"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer1"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer2"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer3"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer4"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer5"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer6"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer7"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer8"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer9"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer10"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer11"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer12"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer13"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer1"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer2"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer3"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer4"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer5"), crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer6"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer7"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer8"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer9"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer10"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer11"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer12"), + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer13"), ]; } @@ -234,6 +319,132 @@ impl crate::Archetype for AffixFuzzer1 { }) .transpose()? }, + { + Some({ + let array = + ::try_to_arrow([&self.fuzz1007], None); + array.map(|array| { + let datatype = ::arrow2::datatypes::DataType::Extension( + "rerun.testing.components.AffixFuzzer7".into(), + Box::new(array.data_type().clone()), + Some("rerun.testing.components.AffixFuzzer7".into()), + ); + ( + ::arrow2::datatypes::Field::new("fuzz1007", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + Some({ + let array = + ::try_to_arrow([&self.fuzz1008], None); + array.map(|array| { + let datatype = ::arrow2::datatypes::DataType::Extension( + "rerun.testing.components.AffixFuzzer8".into(), + Box::new(array.data_type().clone()), + Some("rerun.testing.components.AffixFuzzer8".into()), + ); + ( + ::arrow2::datatypes::Field::new("fuzz1008", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + Some({ + let array = + ::try_to_arrow([&self.fuzz1009], None); + array.map(|array| { + let datatype = ::arrow2::datatypes::DataType::Extension( + "rerun.testing.components.AffixFuzzer9".into(), + Box::new(array.data_type().clone()), + Some("rerun.testing.components.AffixFuzzer9".into()), + ); + ( + ::arrow2::datatypes::Field::new("fuzz1009", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + Some({ + let array = + ::try_to_arrow([&self.fuzz1010], None); + array.map(|array| { + let datatype = ::arrow2::datatypes::DataType::Extension( + "rerun.testing.components.AffixFuzzer10".into(), + Box::new(array.data_type().clone()), + Some("rerun.testing.components.AffixFuzzer10".into()), + ); + ( + ::arrow2::datatypes::Field::new("fuzz1010", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + Some({ + let array = + ::try_to_arrow([&self.fuzz1011], None); + array.map(|array| { + let datatype = ::arrow2::datatypes::DataType::Extension( + "rerun.testing.components.AffixFuzzer11".into(), + Box::new(array.data_type().clone()), + Some("rerun.testing.components.AffixFuzzer11".into()), + ); + ( + ::arrow2::datatypes::Field::new("fuzz1011", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + Some({ + let array = + ::try_to_arrow([&self.fuzz1012], None); + array.map(|array| { + let datatype = ::arrow2::datatypes::DataType::Extension( + "rerun.testing.components.AffixFuzzer12".into(), + Box::new(array.data_type().clone()), + Some("rerun.testing.components.AffixFuzzer12".into()), + ); + ( + ::arrow2::datatypes::Field::new("fuzz1012", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + Some({ + let array = + ::try_to_arrow([&self.fuzz1013], None); + array.map(|array| { + let datatype = ::arrow2::datatypes::DataType::Extension( + "rerun.testing.components.AffixFuzzer13".into(), + Box::new(array.data_type().clone()), + Some("rerun.testing.components.AffixFuzzer13".into()), + ); + ( + ::arrow2::datatypes::Field::new("fuzz1013", datatype, false), + array, + ) + }) + }) + .transpose()? + }, { Some({ let array = @@ -342,6 +553,140 @@ impl crate::Archetype for AffixFuzzer1 { }) .transpose()? }, + { + Some({ + let array = + ::try_to_arrow(self.fuzz1107.iter(), None); + array.map(|array| { + let datatype = ::arrow2::datatypes::DataType::Extension( + "rerun.testing.components.AffixFuzzer7".into(), + Box::new(array.data_type().clone()), + Some("rerun.testing.components.AffixFuzzer7".into()), + ); + ( + ::arrow2::datatypes::Field::new("fuzz1107", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + Some({ + let array = + ::try_to_arrow(self.fuzz1108.iter(), None); + array.map(|array| { + let datatype = ::arrow2::datatypes::DataType::Extension( + "rerun.testing.components.AffixFuzzer8".into(), + Box::new(array.data_type().clone()), + Some("rerun.testing.components.AffixFuzzer8".into()), + ); + ( + ::arrow2::datatypes::Field::new("fuzz1108", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + Some({ + let array = + ::try_to_arrow(self.fuzz1109.iter(), None); + array.map(|array| { + let datatype = ::arrow2::datatypes::DataType::Extension( + "rerun.testing.components.AffixFuzzer9".into(), + Box::new(array.data_type().clone()), + Some("rerun.testing.components.AffixFuzzer9".into()), + ); + ( + ::arrow2::datatypes::Field::new("fuzz1109", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + Some({ + let array = ::try_to_arrow( + self.fuzz1110.iter(), + None, + ); + array.map(|array| { + let datatype = ::arrow2::datatypes::DataType::Extension( + "rerun.testing.components.AffixFuzzer10".into(), + Box::new(array.data_type().clone()), + Some("rerun.testing.components.AffixFuzzer10".into()), + ); + ( + ::arrow2::datatypes::Field::new("fuzz1110", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + Some({ + let array = ::try_to_arrow( + self.fuzz1111.iter(), + None, + ); + array.map(|array| { + let datatype = ::arrow2::datatypes::DataType::Extension( + "rerun.testing.components.AffixFuzzer11".into(), + Box::new(array.data_type().clone()), + Some("rerun.testing.components.AffixFuzzer11".into()), + ); + ( + ::arrow2::datatypes::Field::new("fuzz1111", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + Some({ + let array = ::try_to_arrow( + self.fuzz1112.iter(), + None, + ); + array.map(|array| { + let datatype = ::arrow2::datatypes::DataType::Extension( + "rerun.testing.components.AffixFuzzer12".into(), + Box::new(array.data_type().clone()), + Some("rerun.testing.components.AffixFuzzer12".into()), + ); + ( + ::arrow2::datatypes::Field::new("fuzz1112", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + Some({ + let array = ::try_to_arrow( + self.fuzz1113.iter(), + None, + ); + array.map(|array| { + let datatype = ::arrow2::datatypes::DataType::Extension( + "rerun.testing.components.AffixFuzzer13".into(), + Box::new(array.data_type().clone()), + Some("rerun.testing.components.AffixFuzzer13".into()), + ); + ( + ::arrow2::datatypes::Field::new("fuzz1113", datatype, false), + array, + ) + }) + }) + .transpose()? + }, { self.fuzz2001 .as_ref() @@ -457,19 +802,18 @@ impl crate::Archetype for AffixFuzzer1 { .transpose()? }, { - self.fuzz2101 + self.fuzz2007 .as_ref() - .map(|many| { - let array = - ::try_to_arrow(many.iter(), None); + .map(|single| { + let array = ::try_to_arrow([single], None); array.map(|array| { let datatype = ::arrow2::datatypes::DataType::Extension( - "rerun.testing.components.AffixFuzzer1".into(), + "rerun.testing.components.AffixFuzzer7".into(), Box::new(array.data_type().clone()), - Some("rerun.testing.components.AffixFuzzer1".into()), + Some("rerun.testing.components.AffixFuzzer7".into()), ); ( - ::arrow2::datatypes::Field::new("fuzz2101", datatype, false), + ::arrow2::datatypes::Field::new("fuzz2007", datatype, false), array, ) }) @@ -477,19 +821,18 @@ impl crate::Archetype for AffixFuzzer1 { .transpose()? }, { - self.fuzz2102 + self.fuzz2008 .as_ref() - .map(|many| { - let array = - ::try_to_arrow(many.iter(), None); + .map(|single| { + let array = ::try_to_arrow([single], None); array.map(|array| { let datatype = ::arrow2::datatypes::DataType::Extension( - "rerun.testing.components.AffixFuzzer2".into(), + "rerun.testing.components.AffixFuzzer8".into(), Box::new(array.data_type().clone()), - Some("rerun.testing.components.AffixFuzzer2".into()), + Some("rerun.testing.components.AffixFuzzer8".into()), ); ( - ::arrow2::datatypes::Field::new("fuzz2102", datatype, false), + ::arrow2::datatypes::Field::new("fuzz2008", datatype, false), array, ) }) @@ -497,19 +840,38 @@ impl crate::Archetype for AffixFuzzer1 { .transpose()? }, { - self.fuzz2103 + self.fuzz2009 .as_ref() - .map(|many| { + .map(|single| { + let array = ::try_to_arrow([single], None); + array.map(|array| { + let datatype = ::arrow2::datatypes::DataType::Extension( + "rerun.testing.components.AffixFuzzer9".into(), + Box::new(array.data_type().clone()), + Some("rerun.testing.components.AffixFuzzer9".into()), + ); + ( + ::arrow2::datatypes::Field::new("fuzz2009", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.fuzz2010 + .as_ref() + .map(|single| { let array = - ::try_to_arrow(many.iter(), None); + ::try_to_arrow([single], None); array.map(|array| { let datatype = ::arrow2::datatypes::DataType::Extension( - "rerun.testing.components.AffixFuzzer3".into(), + "rerun.testing.components.AffixFuzzer10".into(), Box::new(array.data_type().clone()), - Some("rerun.testing.components.AffixFuzzer3".into()), + Some("rerun.testing.components.AffixFuzzer10".into()), ); ( - ::arrow2::datatypes::Field::new("fuzz2103", datatype, false), + ::arrow2::datatypes::Field::new("fuzz2010", datatype, false), array, ) }) @@ -517,19 +879,19 @@ impl crate::Archetype for AffixFuzzer1 { .transpose()? }, { - self.fuzz2104 + self.fuzz2011 .as_ref() - .map(|many| { + .map(|single| { let array = - ::try_to_arrow(many.iter(), None); + ::try_to_arrow([single], None); array.map(|array| { let datatype = ::arrow2::datatypes::DataType::Extension( - "rerun.testing.components.AffixFuzzer4".into(), + "rerun.testing.components.AffixFuzzer11".into(), Box::new(array.data_type().clone()), - Some("rerun.testing.components.AffixFuzzer4".into()), + Some("rerun.testing.components.AffixFuzzer11".into()), ); ( - ::arrow2::datatypes::Field::new("fuzz2104", datatype, false), + ::arrow2::datatypes::Field::new("fuzz2011", datatype, false), array, ) }) @@ -537,19 +899,19 @@ impl crate::Archetype for AffixFuzzer1 { .transpose()? }, { - self.fuzz2105 + self.fuzz2012 .as_ref() - .map(|many| { + .map(|single| { let array = - ::try_to_arrow(many.iter(), None); + ::try_to_arrow([single], None); array.map(|array| { let datatype = ::arrow2::datatypes::DataType::Extension( - "rerun.testing.components.AffixFuzzer5".into(), + "rerun.testing.components.AffixFuzzer12".into(), Box::new(array.data_type().clone()), - Some("rerun.testing.components.AffixFuzzer5".into()), + Some("rerun.testing.components.AffixFuzzer12".into()), ); ( - ::arrow2::datatypes::Field::new("fuzz2105", datatype, false), + ::arrow2::datatypes::Field::new("fuzz2012", datatype, false), array, ) }) @@ -557,31 +919,291 @@ impl crate::Archetype for AffixFuzzer1 { .transpose()? }, { - self.fuzz2106 + self.fuzz2013 .as_ref() - .map(|many| { + .map(|single| { let array = - ::try_to_arrow(many.iter(), None); + ::try_to_arrow([single], None); array.map(|array| { let datatype = ::arrow2::datatypes::DataType::Extension( - "rerun.testing.components.AffixFuzzer6".into(), + "rerun.testing.components.AffixFuzzer13".into(), Box::new(array.data_type().clone()), - Some("rerun.testing.components.AffixFuzzer6".into()), + Some("rerun.testing.components.AffixFuzzer13".into()), ); ( - ::arrow2::datatypes::Field::new("fuzz2106", datatype, false), + ::arrow2::datatypes::Field::new("fuzz2013", datatype, false), array, ) }) }) .transpose()? }, - ] - .into_iter() - .flatten() - .collect()) - } - + { + self.fuzz2101 + .as_ref() + .map(|many| { + let array = + ::try_to_arrow(many.iter(), None); + array.map(|array| { + let datatype = ::arrow2::datatypes::DataType::Extension( + "rerun.testing.components.AffixFuzzer1".into(), + Box::new(array.data_type().clone()), + Some("rerun.testing.components.AffixFuzzer1".into()), + ); + ( + ::arrow2::datatypes::Field::new("fuzz2101", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.fuzz2102 + .as_ref() + .map(|many| { + let array = + ::try_to_arrow(many.iter(), None); + array.map(|array| { + let datatype = ::arrow2::datatypes::DataType::Extension( + "rerun.testing.components.AffixFuzzer2".into(), + Box::new(array.data_type().clone()), + Some("rerun.testing.components.AffixFuzzer2".into()), + ); + ( + ::arrow2::datatypes::Field::new("fuzz2102", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.fuzz2103 + .as_ref() + .map(|many| { + let array = + ::try_to_arrow(many.iter(), None); + array.map(|array| { + let datatype = ::arrow2::datatypes::DataType::Extension( + "rerun.testing.components.AffixFuzzer3".into(), + Box::new(array.data_type().clone()), + Some("rerun.testing.components.AffixFuzzer3".into()), + ); + ( + ::arrow2::datatypes::Field::new("fuzz2103", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.fuzz2104 + .as_ref() + .map(|many| { + let array = + ::try_to_arrow(many.iter(), None); + array.map(|array| { + let datatype = ::arrow2::datatypes::DataType::Extension( + "rerun.testing.components.AffixFuzzer4".into(), + Box::new(array.data_type().clone()), + Some("rerun.testing.components.AffixFuzzer4".into()), + ); + ( + ::arrow2::datatypes::Field::new("fuzz2104", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.fuzz2105 + .as_ref() + .map(|many| { + let array = + ::try_to_arrow(many.iter(), None); + array.map(|array| { + let datatype = ::arrow2::datatypes::DataType::Extension( + "rerun.testing.components.AffixFuzzer5".into(), + Box::new(array.data_type().clone()), + Some("rerun.testing.components.AffixFuzzer5".into()), + ); + ( + ::arrow2::datatypes::Field::new("fuzz2105", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.fuzz2106 + .as_ref() + .map(|many| { + let array = + ::try_to_arrow(many.iter(), None); + array.map(|array| { + let datatype = ::arrow2::datatypes::DataType::Extension( + "rerun.testing.components.AffixFuzzer6".into(), + Box::new(array.data_type().clone()), + Some("rerun.testing.components.AffixFuzzer6".into()), + ); + ( + ::arrow2::datatypes::Field::new("fuzz2106", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.fuzz2107 + .as_ref() + .map(|many| { + let array = + ::try_to_arrow(many.iter(), None); + array.map(|array| { + let datatype = ::arrow2::datatypes::DataType::Extension( + "rerun.testing.components.AffixFuzzer7".into(), + Box::new(array.data_type().clone()), + Some("rerun.testing.components.AffixFuzzer7".into()), + ); + ( + ::arrow2::datatypes::Field::new("fuzz2107", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.fuzz2108 + .as_ref() + .map(|many| { + let array = + ::try_to_arrow(many.iter(), None); + array.map(|array| { + let datatype = ::arrow2::datatypes::DataType::Extension( + "rerun.testing.components.AffixFuzzer8".into(), + Box::new(array.data_type().clone()), + Some("rerun.testing.components.AffixFuzzer8".into()), + ); + ( + ::arrow2::datatypes::Field::new("fuzz2108", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.fuzz2109 + .as_ref() + .map(|many| { + let array = + ::try_to_arrow(many.iter(), None); + array.map(|array| { + let datatype = ::arrow2::datatypes::DataType::Extension( + "rerun.testing.components.AffixFuzzer9".into(), + Box::new(array.data_type().clone()), + Some("rerun.testing.components.AffixFuzzer9".into()), + ); + ( + ::arrow2::datatypes::Field::new("fuzz2109", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.fuzz2110 + .as_ref() + .map(|many| { + let array = + ::try_to_arrow(many.iter(), None); + array.map(|array| { + let datatype = ::arrow2::datatypes::DataType::Extension( + "rerun.testing.components.AffixFuzzer10".into(), + Box::new(array.data_type().clone()), + Some("rerun.testing.components.AffixFuzzer10".into()), + ); + ( + ::arrow2::datatypes::Field::new("fuzz2110", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.fuzz2111 + .as_ref() + .map(|many| { + let array = + ::try_to_arrow(many.iter(), None); + array.map(|array| { + let datatype = ::arrow2::datatypes::DataType::Extension( + "rerun.testing.components.AffixFuzzer11".into(), + Box::new(array.data_type().clone()), + Some("rerun.testing.components.AffixFuzzer11".into()), + ); + ( + ::arrow2::datatypes::Field::new("fuzz2111", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.fuzz2112 + .as_ref() + .map(|many| { + let array = + ::try_to_arrow(many.iter(), None); + array.map(|array| { + let datatype = ::arrow2::datatypes::DataType::Extension( + "rerun.testing.components.AffixFuzzer12".into(), + Box::new(array.data_type().clone()), + Some("rerun.testing.components.AffixFuzzer12".into()), + ); + ( + ::arrow2::datatypes::Field::new("fuzz2112", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + { + self.fuzz2113 + .as_ref() + .map(|many| { + let array = + ::try_to_arrow(many.iter(), None); + array.map(|array| { + let datatype = ::arrow2::datatypes::DataType::Extension( + "rerun.testing.components.AffixFuzzer13".into(), + Box::new(array.data_type().clone()), + Some("rerun.testing.components.AffixFuzzer13".into()), + ); + ( + ::arrow2::datatypes::Field::new("fuzz2113", datatype, false), + array, + ) + }) + }) + .transpose()? + }, + ] + .into_iter() + .flatten() + .collect()) + } + #[inline] fn try_from_arrow( data: impl IntoIterator)>, @@ -669,19 +1291,222 @@ impl crate::Archetype for AffixFuzzer1 { })?; ::try_from_arrow_opt(&**array)? .into_iter() - .next() - .flatten() - .ok_or_else(|| crate::DeserializationError::MissingData { + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + })? + }; + let fuzz1007 = { + let array = arrays_by_name.get("fuzz1007").ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + } + })?; + ::try_from_arrow_opt(&**array)? + .into_iter() + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + })? + }; + let fuzz1008 = { + let array = arrays_by_name.get("fuzz1008").ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + } + })?; + ::try_from_arrow_opt(&**array)? + .into_iter() + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + })? + }; + let fuzz1009 = { + let array = arrays_by_name.get("fuzz1009").ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + } + })?; + ::try_from_arrow_opt(&**array)? + .into_iter() + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + })? + }; + let fuzz1010 = { + let array = arrays_by_name.get("fuzz1010").ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + } + })?; + ::try_from_arrow_opt(&**array)? + .into_iter() + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + })? + }; + let fuzz1011 = { + let array = arrays_by_name.get("fuzz1011").ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + } + })?; + ::try_from_arrow_opt(&**array)? + .into_iter() + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + })? + }; + let fuzz1012 = { + let array = arrays_by_name.get("fuzz1012").ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + } + })?; + ::try_from_arrow_opt(&**array)? + .into_iter() + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + })? + }; + let fuzz1013 = { + let array = arrays_by_name.get("fuzz1013").ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + } + })?; + ::try_from_arrow_opt(&**array)? + .into_iter() + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + })? + }; + let fuzz1101 = { + let array = arrays_by_name.get("fuzz1101").ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + } + })?; + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()? + }; + let fuzz1102 = { + let array = arrays_by_name.get("fuzz1102").ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + } + })?; + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()? + }; + let fuzz1103 = { + let array = arrays_by_name.get("fuzz1103").ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + } + })?; + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()? + }; + let fuzz1104 = { + let array = arrays_by_name.get("fuzz1104").ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + } + })?; + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()? + }; + let fuzz1105 = { + let array = arrays_by_name.get("fuzz1105").ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + } + })?; + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()? + }; + let fuzz1106 = { + let array = arrays_by_name.get("fuzz1106").ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + } + })?; + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()? + }; + let fuzz1107 = { + let array = arrays_by_name.get("fuzz1107").ok_or_else(|| { + crate::DeserializationError::MissingData { datatype: ::arrow2::datatypes::DataType::Null, - })? + } + })?; + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()? }; - let fuzz1101 = { - let array = arrays_by_name.get("fuzz1101").ok_or_else(|| { + let fuzz1108 = { + let array = arrays_by_name.get("fuzz1108").ok_or_else(|| { crate::DeserializationError::MissingData { datatype: ::arrow2::datatypes::DataType::Null, } })?; - ::try_from_arrow_opt(&**array)? + ::try_from_arrow_opt(&**array)? .into_iter() .map(|v| { v.ok_or_else(|| crate::DeserializationError::MissingData { @@ -690,13 +1515,13 @@ impl crate::Archetype for AffixFuzzer1 { }) .collect::>>()? }; - let fuzz1102 = { - let array = arrays_by_name.get("fuzz1102").ok_or_else(|| { + let fuzz1109 = { + let array = arrays_by_name.get("fuzz1109").ok_or_else(|| { crate::DeserializationError::MissingData { datatype: ::arrow2::datatypes::DataType::Null, } })?; - ::try_from_arrow_opt(&**array)? + ::try_from_arrow_opt(&**array)? .into_iter() .map(|v| { v.ok_or_else(|| crate::DeserializationError::MissingData { @@ -705,13 +1530,13 @@ impl crate::Archetype for AffixFuzzer1 { }) .collect::>>()? }; - let fuzz1103 = { - let array = arrays_by_name.get("fuzz1103").ok_or_else(|| { + let fuzz1110 = { + let array = arrays_by_name.get("fuzz1110").ok_or_else(|| { crate::DeserializationError::MissingData { datatype: ::arrow2::datatypes::DataType::Null, } })?; - ::try_from_arrow_opt(&**array)? + ::try_from_arrow_opt(&**array)? .into_iter() .map(|v| { v.ok_or_else(|| crate::DeserializationError::MissingData { @@ -720,13 +1545,13 @@ impl crate::Archetype for AffixFuzzer1 { }) .collect::>>()? }; - let fuzz1104 = { - let array = arrays_by_name.get("fuzz1104").ok_or_else(|| { + let fuzz1111 = { + let array = arrays_by_name.get("fuzz1111").ok_or_else(|| { crate::DeserializationError::MissingData { datatype: ::arrow2::datatypes::DataType::Null, } })?; - ::try_from_arrow_opt(&**array)? + ::try_from_arrow_opt(&**array)? .into_iter() .map(|v| { v.ok_or_else(|| crate::DeserializationError::MissingData { @@ -735,13 +1560,13 @@ impl crate::Archetype for AffixFuzzer1 { }) .collect::>>()? }; - let fuzz1105 = { - let array = arrays_by_name.get("fuzz1105").ok_or_else(|| { + let fuzz1112 = { + let array = arrays_by_name.get("fuzz1112").ok_or_else(|| { crate::DeserializationError::MissingData { datatype: ::arrow2::datatypes::DataType::Null, } })?; - ::try_from_arrow_opt(&**array)? + ::try_from_arrow_opt(&**array)? .into_iter() .map(|v| { v.ok_or_else(|| crate::DeserializationError::MissingData { @@ -750,13 +1575,13 @@ impl crate::Archetype for AffixFuzzer1 { }) .collect::>>()? }; - let fuzz1106 = { - let array = arrays_by_name.get("fuzz1106").ok_or_else(|| { + let fuzz1113 = { + let array = arrays_by_name.get("fuzz1113").ok_or_else(|| { crate::DeserializationError::MissingData { datatype: ::arrow2::datatypes::DataType::Null, } })?; - ::try_from_arrow_opt(&**array)? + ::try_from_arrow_opt(&**array)? .into_iter() .map(|v| { v.ok_or_else(|| crate::DeserializationError::MissingData { @@ -843,6 +1668,97 @@ impl crate::Archetype for AffixFuzzer1 { } else { None }; + let fuzz2007 = if let Some(array) = arrays_by_name.get("fuzz2007") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + })?, + ) + } else { + None + }; + let fuzz2008 = if let Some(array) = arrays_by_name.get("fuzz2008") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + })?, + ) + } else { + None + }; + let fuzz2009 = if let Some(array) = arrays_by_name.get("fuzz2009") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + })?, + ) + } else { + None + }; + let fuzz2010 = if let Some(array) = arrays_by_name.get("fuzz2010") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + })?, + ) + } else { + None + }; + let fuzz2011 = if let Some(array) = arrays_by_name.get("fuzz2011") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + })?, + ) + } else { + None + }; + let fuzz2012 = if let Some(array) = arrays_by_name.get("fuzz2012") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + })?, + ) + } else { + None + }; + let fuzz2013 = if let Some(array) = arrays_by_name.get("fuzz2013") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .next() + .flatten() + .ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + })?, + ) + } else { + None + }; let fuzz2101 = if let Some(array) = arrays_by_name.get("fuzz2101") { Some( ::try_from_arrow_opt(&**array)? @@ -927,6 +1843,104 @@ impl crate::Archetype for AffixFuzzer1 { } else { None }; + let fuzz2107 = if let Some(array) = arrays_by_name.get("fuzz2107") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()?, + ) + } else { + None + }; + let fuzz2108 = if let Some(array) = arrays_by_name.get("fuzz2108") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()?, + ) + } else { + None + }; + let fuzz2109 = if let Some(array) = arrays_by_name.get("fuzz2109") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()?, + ) + } else { + None + }; + let fuzz2110 = if let Some(array) = arrays_by_name.get("fuzz2110") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()?, + ) + } else { + None + }; + let fuzz2111 = if let Some(array) = arrays_by_name.get("fuzz2111") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()?, + ) + } else { + None + }; + let fuzz2112 = if let Some(array) = arrays_by_name.get("fuzz2112") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()?, + ) + } else { + None + }; + let fuzz2113 = if let Some(array) = arrays_by_name.get("fuzz2113") { + Some( + ::try_from_arrow_opt(&**array)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: ::arrow2::datatypes::DataType::Null, + }) + }) + .collect::>>()?, + ) + } else { + None + }; Ok(Self { fuzz1001, fuzz1002, @@ -934,24 +1948,52 @@ impl crate::Archetype for AffixFuzzer1 { fuzz1004, fuzz1005, fuzz1006, + fuzz1007, + fuzz1008, + fuzz1009, + fuzz1010, + fuzz1011, + fuzz1012, + fuzz1013, fuzz1101, fuzz1102, fuzz1103, fuzz1104, fuzz1105, fuzz1106, + fuzz1107, + fuzz1108, + fuzz1109, + fuzz1110, + fuzz1111, + fuzz1112, + fuzz1113, fuzz2001, fuzz2002, fuzz2003, fuzz2004, fuzz2005, fuzz2006, + fuzz2007, + fuzz2008, + fuzz2009, + fuzz2010, + fuzz2011, + fuzz2012, + fuzz2013, fuzz2101, fuzz2102, fuzz2103, fuzz2104, fuzz2105, fuzz2106, + fuzz2107, + fuzz2108, + fuzz2109, + fuzz2110, + fuzz2111, + fuzz2112, + fuzz2113, }) } } @@ -964,12 +2006,26 @@ impl AffixFuzzer1 { fuzz1004: impl Into, fuzz1005: impl Into, fuzz1006: impl Into, + fuzz1007: impl Into, + fuzz1008: impl Into, + fuzz1009: impl Into, + fuzz1010: impl Into, + fuzz1011: impl Into, + fuzz1012: impl Into, + fuzz1013: impl Into, fuzz1101: impl IntoIterator>, fuzz1102: impl IntoIterator>, fuzz1103: impl IntoIterator>, fuzz1104: impl IntoIterator>, fuzz1105: impl IntoIterator>, fuzz1106: impl IntoIterator>, + fuzz1107: impl IntoIterator>, + fuzz1108: impl IntoIterator>, + fuzz1109: impl IntoIterator>, + fuzz1110: impl IntoIterator>, + fuzz1111: impl IntoIterator>, + fuzz1112: impl IntoIterator>, + fuzz1113: impl IntoIterator>, ) -> Self { Self { fuzz1001: fuzz1001.into(), @@ -978,24 +2034,52 @@ impl AffixFuzzer1 { fuzz1004: fuzz1004.into(), fuzz1005: fuzz1005.into(), fuzz1006: fuzz1006.into(), + fuzz1007: fuzz1007.into(), + fuzz1008: fuzz1008.into(), + fuzz1009: fuzz1009.into(), + fuzz1010: fuzz1010.into(), + fuzz1011: fuzz1011.into(), + fuzz1012: fuzz1012.into(), + fuzz1013: fuzz1013.into(), fuzz1101: fuzz1101.into_iter().map(Into::into).collect(), fuzz1102: fuzz1102.into_iter().map(Into::into).collect(), fuzz1103: fuzz1103.into_iter().map(Into::into).collect(), fuzz1104: fuzz1104.into_iter().map(Into::into).collect(), fuzz1105: fuzz1105.into_iter().map(Into::into).collect(), fuzz1106: fuzz1106.into_iter().map(Into::into).collect(), + fuzz1107: fuzz1107.into_iter().map(Into::into).collect(), + fuzz1108: fuzz1108.into_iter().map(Into::into).collect(), + fuzz1109: fuzz1109.into_iter().map(Into::into).collect(), + fuzz1110: fuzz1110.into_iter().map(Into::into).collect(), + fuzz1111: fuzz1111.into_iter().map(Into::into).collect(), + fuzz1112: fuzz1112.into_iter().map(Into::into).collect(), + fuzz1113: fuzz1113.into_iter().map(Into::into).collect(), fuzz2001: None, fuzz2002: None, fuzz2003: None, fuzz2004: None, fuzz2005: None, fuzz2006: None, + fuzz2007: None, + fuzz2008: None, + fuzz2009: None, + fuzz2010: None, + fuzz2011: None, + fuzz2012: None, + fuzz2013: None, fuzz2101: None, fuzz2102: None, fuzz2103: None, fuzz2104: None, fuzz2105: None, fuzz2106: None, + fuzz2107: None, + fuzz2108: None, + fuzz2109: None, + fuzz2110: None, + fuzz2111: None, + fuzz2112: None, + fuzz2113: None, } } @@ -1029,6 +2113,41 @@ impl AffixFuzzer1 { self } + pub fn with_fuzz2007(mut self, fuzz2007: impl Into) -> Self { + self.fuzz2007 = Some(fuzz2007.into()); + self + } + + pub fn with_fuzz2008(mut self, fuzz2008: impl Into) -> Self { + self.fuzz2008 = Some(fuzz2008.into()); + self + } + + pub fn with_fuzz2009(mut self, fuzz2009: impl Into) -> Self { + self.fuzz2009 = Some(fuzz2009.into()); + self + } + + pub fn with_fuzz2010(mut self, fuzz2010: impl Into) -> Self { + self.fuzz2010 = Some(fuzz2010.into()); + self + } + + pub fn with_fuzz2011(mut self, fuzz2011: impl Into) -> Self { + self.fuzz2011 = Some(fuzz2011.into()); + self + } + + pub fn with_fuzz2012(mut self, fuzz2012: impl Into) -> Self { + self.fuzz2012 = Some(fuzz2012.into()); + self + } + + pub fn with_fuzz2013(mut self, fuzz2013: impl Into) -> Self { + self.fuzz2013 = Some(fuzz2013.into()); + self + } + pub fn with_fuzz2101( mut self, fuzz2101: impl IntoIterator>, @@ -1076,4 +2195,60 @@ impl AffixFuzzer1 { self.fuzz2106 = Some(fuzz2106.into_iter().map(Into::into).collect()); self } + + pub fn with_fuzz2107( + mut self, + fuzz2107: impl IntoIterator>, + ) -> Self { + self.fuzz2107 = Some(fuzz2107.into_iter().map(Into::into).collect()); + self + } + + pub fn with_fuzz2108( + mut self, + fuzz2108: impl IntoIterator>, + ) -> Self { + self.fuzz2108 = Some(fuzz2108.into_iter().map(Into::into).collect()); + self + } + + pub fn with_fuzz2109( + mut self, + fuzz2109: impl IntoIterator>, + ) -> Self { + self.fuzz2109 = Some(fuzz2109.into_iter().map(Into::into).collect()); + self + } + + pub fn with_fuzz2110( + mut self, + fuzz2110: impl IntoIterator>, + ) -> Self { + self.fuzz2110 = Some(fuzz2110.into_iter().map(Into::into).collect()); + self + } + + pub fn with_fuzz2111( + mut self, + fuzz2111: impl IntoIterator>, + ) -> Self { + self.fuzz2111 = Some(fuzz2111.into_iter().map(Into::into).collect()); + self + } + + pub fn with_fuzz2112( + mut self, + fuzz2112: impl IntoIterator>, + ) -> Self { + self.fuzz2112 = Some(fuzz2112.into_iter().map(Into::into).collect()); + self + } + + pub fn with_fuzz2113( + mut self, + fuzz2113: impl IntoIterator>, + ) -> Self { + self.fuzz2113 = Some(fuzz2113.into_iter().map(Into::into).collect()); + self + } } diff --git a/crates/re_types/src/archetypes/points2d.rs b/crates/re_types/src/archetypes/points2d.rs index 934c65b7c2ae..57db44037fdb 100644 --- a/crates/re_types/src/archetypes/points2d.rs +++ b/crates/re_types/src/archetypes/points2d.rs @@ -6,6 +6,7 @@ #![allow(clippy::map_flatten)] #![allow(clippy::needless_question_mark)] #![allow(clippy::too_many_arguments)] +#![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] #[doc = "A 2D point cloud with positions and optional colors, radii, labels, etc."] diff --git a/crates/re_types/src/components/class_id.rs b/crates/re_types/src/components/class_id.rs index c920738d364e..22191277f35d 100644 --- a/crates/re_types/src/components/class_id.rs +++ b/crates/re_types/src/components/class_id.rs @@ -6,6 +6,7 @@ #![allow(clippy::map_flatten)] #![allow(clippy::needless_question_mark)] #![allow(clippy::too_many_arguments)] +#![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] #[doc = "A 16-bit ID representing a type of semantic class."] diff --git a/crates/re_types/src/components/color.rs b/crates/re_types/src/components/color.rs index ad4a881e3b3d..685af5c7fa85 100644 --- a/crates/re_types/src/components/color.rs +++ b/crates/re_types/src/components/color.rs @@ -6,6 +6,7 @@ #![allow(clippy::map_flatten)] #![allow(clippy::needless_question_mark)] #![allow(clippy::too_many_arguments)] +#![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] #[doc = "An RGBA color tuple with unmultiplied/separate alpha, in sRGB gamma space with linear alpha."] diff --git a/crates/re_types/src/components/draw_order.rs b/crates/re_types/src/components/draw_order.rs index 55357a24bebf..932cf424143d 100644 --- a/crates/re_types/src/components/draw_order.rs +++ b/crates/re_types/src/components/draw_order.rs @@ -6,6 +6,7 @@ #![allow(clippy::map_flatten)] #![allow(clippy::needless_question_mark)] #![allow(clippy::too_many_arguments)] +#![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] #[doc = "Draw order used for the display order of 2D elements."] diff --git a/crates/re_types/src/components/fuzzy.rs b/crates/re_types/src/components/fuzzy.rs index 0086dc5691f3..fc9c601a6af0 100644 --- a/crates/re_types/src/components/fuzzy.rs +++ b/crates/re_types/src/components/fuzzy.rs @@ -6,6 +6,7 @@ #![allow(clippy::map_flatten)] #![allow(clippy::needless_question_mark)] #![allow(clippy::too_many_arguments)] +#![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] #[derive(Debug, Clone, PartialEq)] @@ -972,3 +973,1692 @@ impl crate::Component for AffixFuzzer6 { }) } } + +#[derive(Debug, Clone, PartialEq)] +pub struct AffixFuzzer7 { + pub many_optional: Option>, +} + +impl<'a> From for ::std::borrow::Cow<'a, AffixFuzzer7> { + #[inline] + fn from(value: AffixFuzzer7) -> Self { + std::borrow::Cow::Owned(value) + } +} + +impl<'a> From<&'a AffixFuzzer7> for ::std::borrow::Cow<'a, AffixFuzzer7> { + #[inline] + fn from(value: &'a AffixFuzzer7) -> Self { + std::borrow::Cow::Borrowed(value) + } +} + +impl crate::Component for AffixFuzzer7 { + #[inline] + fn name() -> crate::ComponentName { + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer7") + } + + #[allow(unused_imports, clippy::wildcard_imports)] + #[inline] + fn to_arrow_datatype() -> arrow2::datatypes::DataType { + use ::arrow2::datatypes::*; + DataType::Struct(vec![Field { + name: "many_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Struct(vec![ + Field { + name: "single_float_optional".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "single_string_required".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "single_string_optional".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_floats_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_strings_required".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + })), + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "many_strings_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + ]), + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }]) + } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_to_arrow_opt<'a>( + data: impl IntoIterator>>>, + extension_wrapper: Option<&str>, + ) -> crate::SerializationResult> + where + Self: Clone + 'a, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let (somes, data): (Vec<_>, Vec<_>) = data + .into_iter() + .map(|datum| { + let datum: Option<::std::borrow::Cow<'a, Self>> = datum.map(Into::into); + (datum.is_some(), datum) + }) + .unzip(); + let bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + StructArray::new( + (if let Some(ext) = extension_wrapper { + DataType::Extension( + ext.to_owned(), + Box::new(::to_arrow_datatype()), + None, + ) + } else { + ::to_arrow_datatype() + }) + .to_logical_type() + .clone(), + vec![{ + let (somes, many_optional): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = datum + .as_ref() + .map(|datum| { + let Self { many_optional, .. } = &**datum; + many_optional.clone() + }) + .flatten(); + (datum.is_some(), datum) + }) + .unzip(); + let many_optional_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + { + use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; + let many_optional_inner_data: Vec<_> = many_optional + .iter() + .flatten() + .flatten() + .map(ToOwned::to_owned) + .map(Some) + .collect(); + let many_optional_inner_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = many_optional_inner_data.iter().any(|v| v.is_none()); + any_nones.then(|| { + many_optional_inner_data + .iter() + .map(|v| v.is_some()) + .collect() + }) + }; + let offsets = ::arrow2::offset::Offsets::::try_from_lengths( + many_optional.iter().map(|opt| { + opt.as_ref().map(|datum| datum.len()).unwrap_or_default() + }), + ) + .unwrap() + .into(); + ListArray::new( + { + _ = extension_wrapper; + DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Struct(vec![ + Field { + name: "single_float_optional".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "single_string_required".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "single_string_optional".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_floats_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_strings_required".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + })), + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "many_strings_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + ]), + is_nullable: true, + metadata: [].into(), + })) + .to_logical_type() + .clone() + }, + offsets, + { + _ = many_optional_inner_bitmap; + _ = extension_wrapper; + crate::datatypes::AffixFuzzer1::try_to_arrow_opt( + many_optional_inner_data, + None::<&str>, + )? + }, + many_optional_bitmap, + ) + .boxed() + } + }], + bitmap, + ) + .boxed() + }) + } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_from_arrow_opt( + data: &dyn ::arrow2::array::Array, + ) -> crate::DeserializationResult>> + where + Self: Sized, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let data = data + .as_any() + .downcast_ref::<::arrow2::array::StructArray>() + .ok_or_else(|| crate::DeserializationError::SchemaMismatch { + expected: data.data_type().clone(), + got: data.data_type().clone(), + })?; + let (data_fields, data_arrays, data_bitmap) = + (data.fields(), data.values(), data.validity()); + let is_valid = |i| data_bitmap.map_or(true, |bitmap| bitmap.get_bit(i)); + let arrays_by_name: ::std::collections::HashMap<_, _> = data_fields + .iter() + .map(|field| field.name.as_str()) + .zip(data_arrays) + .collect(); + let many_optional = { + let data = &**arrays_by_name["many_optional"]; + + { + let datatype = data.data_type(); + let data = data + .as_any() + .downcast_ref::<::arrow2::array::ListArray>() + .unwrap(); + let bitmap = data.validity().cloned(); + let offsets = { + let offsets = data.offsets(); + offsets.iter().copied().zip(offsets.iter().copied().skip(1)) + }; + let data = &**data.values(); + let data = crate::datatypes::AffixFuzzer1::try_from_arrow_opt(data)? + .into_iter() + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: DataType::Struct(vec![ + Field { + name: "single_float_optional".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "single_string_required".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "single_string_optional".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_floats_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + Field { + name: "many_strings_required".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + })), + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "many_strings_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }, + ]), + }) + }) + .collect::>>()?; + offsets + .enumerate() + .map(move |(i, (start, end))| { + bitmap + .as_ref() + .map_or(true, |bitmap| bitmap.get_bit(i)) + .then(|| { + Ok(data + .get(start as usize..end as usize) + .ok_or_else(|| { + crate::DeserializationError::OffsetsMismatch { + bounds: (start as usize, end as usize), + len: data.len(), + datatype: datatype.clone(), + } + })? + .to_vec()) + }) + .transpose() + }) + .collect::>>>()? + .into_iter() + } + }; + ::itertools::izip!(many_optional) + .enumerate() + .map(|(i, (many_optional))| { + is_valid(i).then(|| Ok(Self { many_optional })).transpose() + }) + .collect::>>()? + }) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct AffixFuzzer8 { + pub single_float_optional: Option, +} + +impl<'a> From for ::std::borrow::Cow<'a, AffixFuzzer8> { + #[inline] + fn from(value: AffixFuzzer8) -> Self { + std::borrow::Cow::Owned(value) + } +} + +impl<'a> From<&'a AffixFuzzer8> for ::std::borrow::Cow<'a, AffixFuzzer8> { + #[inline] + fn from(value: &'a AffixFuzzer8) -> Self { + std::borrow::Cow::Borrowed(value) + } +} + +impl crate::Component for AffixFuzzer8 { + #[inline] + fn name() -> crate::ComponentName { + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer8") + } + + #[allow(unused_imports, clippy::wildcard_imports)] + #[inline] + fn to_arrow_datatype() -> arrow2::datatypes::DataType { + use ::arrow2::datatypes::*; + DataType::Struct(vec![Field { + name: "single_float_optional".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }]) + } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_to_arrow_opt<'a>( + data: impl IntoIterator>>>, + extension_wrapper: Option<&str>, + ) -> crate::SerializationResult> + where + Self: Clone + 'a, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let (somes, data): (Vec<_>, Vec<_>) = data + .into_iter() + .map(|datum| { + let datum: Option<::std::borrow::Cow<'a, Self>> = datum.map(Into::into); + (datum.is_some(), datum) + }) + .unzip(); + let bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + StructArray::new( + (if let Some(ext) = extension_wrapper { + DataType::Extension( + ext.to_owned(), + Box::new(::to_arrow_datatype()), + None, + ) + } else { + ::to_arrow_datatype() + }) + .to_logical_type() + .clone(), + vec![{ + let (somes, single_float_optional): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = datum + .as_ref() + .map(|datum| { + let Self { + single_float_optional, + .. + } = &**datum; + single_float_optional.clone() + }) + .flatten(); + (datum.is_some(), datum) + }) + .unzip(); + let single_float_optional_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + PrimitiveArray::new( + { + _ = extension_wrapper; + DataType::Float32.to_logical_type().clone() + }, + single_float_optional + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect(), + single_float_optional_bitmap, + ) + .boxed() + }], + bitmap, + ) + .boxed() + }) + } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_from_arrow_opt( + data: &dyn ::arrow2::array::Array, + ) -> crate::DeserializationResult>> + where + Self: Sized, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let data = data + .as_any() + .downcast_ref::<::arrow2::array::StructArray>() + .ok_or_else(|| crate::DeserializationError::SchemaMismatch { + expected: data.data_type().clone(), + got: data.data_type().clone(), + })?; + let (data_fields, data_arrays, data_bitmap) = + (data.fields(), data.values(), data.validity()); + let is_valid = |i| data_bitmap.map_or(true, |bitmap| bitmap.get_bit(i)); + let arrays_by_name: ::std::collections::HashMap<_, _> = data_fields + .iter() + .map(|field| field.name.as_str()) + .zip(data_arrays) + .collect(); + let single_float_optional = { + let data = &**arrays_by_name["single_float_optional"]; + + data.as_any() + .downcast_ref::() + .unwrap() + .into_iter() + .map(|v| v.copied()) + }; + ::itertools::izip!(single_float_optional) + .enumerate() + .map(|(i, (single_float_optional))| { + is_valid(i) + .then(|| { + Ok(Self { + single_float_optional, + }) + }) + .transpose() + }) + .collect::>>()? + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AffixFuzzer9 { + pub single_string_required: String, +} + +impl<'a> From for ::std::borrow::Cow<'a, AffixFuzzer9> { + #[inline] + fn from(value: AffixFuzzer9) -> Self { + std::borrow::Cow::Owned(value) + } +} + +impl<'a> From<&'a AffixFuzzer9> for ::std::borrow::Cow<'a, AffixFuzzer9> { + #[inline] + fn from(value: &'a AffixFuzzer9) -> Self { + std::borrow::Cow::Borrowed(value) + } +} + +impl crate::Component for AffixFuzzer9 { + #[inline] + fn name() -> crate::ComponentName { + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer9") + } + + #[allow(unused_imports, clippy::wildcard_imports)] + #[inline] + fn to_arrow_datatype() -> arrow2::datatypes::DataType { + use ::arrow2::datatypes::*; + DataType::Struct(vec![Field { + name: "single_string_required".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + }]) + } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_to_arrow_opt<'a>( + data: impl IntoIterator>>>, + extension_wrapper: Option<&str>, + ) -> crate::SerializationResult> + where + Self: Clone + 'a, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let (somes, data): (Vec<_>, Vec<_>) = data + .into_iter() + .map(|datum| { + let datum: Option<::std::borrow::Cow<'a, Self>> = datum.map(Into::into); + (datum.is_some(), datum) + }) + .unzip(); + let bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + StructArray::new( + (if let Some(ext) = extension_wrapper { + DataType::Extension( + ext.to_owned(), + Box::new(::to_arrow_datatype()), + None, + ) + } else { + ::to_arrow_datatype() + }) + .to_logical_type() + .clone(), + vec![{ + let (somes, single_string_required): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = datum.as_ref().map(|datum| { + let Self { + single_string_required, + .. + } = &**datum; + single_string_required.clone() + }); + (datum.is_some(), datum) + }) + .unzip(); + let single_string_required_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + { + let inner_data: ::arrow2::buffer::Buffer = single_string_required + .iter() + .flatten() + .flat_map(|s| s.bytes()) + .collect(); + let offsets = ::arrow2::offset::Offsets::::try_from_lengths( + single_string_required.iter().map(|opt| { + opt.as_ref().map(|datum| datum.len()).unwrap_or_default() + }), + ) + .unwrap() + .into(); + #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] + unsafe { + Utf8Array::::new_unchecked( + { + _ = extension_wrapper; + DataType::Utf8.to_logical_type().clone() + }, + offsets, + inner_data, + single_string_required_bitmap, + ) + } + .boxed() + } + }], + bitmap, + ) + .boxed() + }) + } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_from_arrow_opt( + data: &dyn ::arrow2::array::Array, + ) -> crate::DeserializationResult>> + where + Self: Sized, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let data = data + .as_any() + .downcast_ref::<::arrow2::array::StructArray>() + .ok_or_else(|| crate::DeserializationError::SchemaMismatch { + expected: data.data_type().clone(), + got: data.data_type().clone(), + })?; + let (data_fields, data_arrays, data_bitmap) = + (data.fields(), data.values(), data.validity()); + let is_valid = |i| data_bitmap.map_or(true, |bitmap| bitmap.get_bit(i)); + let arrays_by_name: ::std::collections::HashMap<_, _> = data_fields + .iter() + .map(|field| field.name.as_str()) + .zip(data_arrays) + .collect(); + let single_string_required = { + let data = &**arrays_by_name["single_string_required"]; + + data.as_any() + .downcast_ref::>() + .unwrap() + .into_iter() + .map(|v| v.map(ToOwned::to_owned)) + }; + ::itertools::izip!(single_string_required) + .enumerate() + .map(|(i, (single_string_required))| { + is_valid(i) + .then(|| { + Ok(Self { + single_string_required: single_string_required.ok_or_else( + || crate::DeserializationError::MissingData { + datatype: data.data_type().clone(), + }, + )?, + }) + }) + .transpose() + }) + .collect::>>()? + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AffixFuzzer10 { + pub single_string_optional: Option, +} + +impl<'a> From for ::std::borrow::Cow<'a, AffixFuzzer10> { + #[inline] + fn from(value: AffixFuzzer10) -> Self { + std::borrow::Cow::Owned(value) + } +} + +impl<'a> From<&'a AffixFuzzer10> for ::std::borrow::Cow<'a, AffixFuzzer10> { + #[inline] + fn from(value: &'a AffixFuzzer10) -> Self { + std::borrow::Cow::Borrowed(value) + } +} + +impl crate::Component for AffixFuzzer10 { + #[inline] + fn name() -> crate::ComponentName { + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer10") + } + + #[allow(unused_imports, clippy::wildcard_imports)] + #[inline] + fn to_arrow_datatype() -> arrow2::datatypes::DataType { + use ::arrow2::datatypes::*; + DataType::Struct(vec![Field { + name: "single_string_optional".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + }]) + } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_to_arrow_opt<'a>( + data: impl IntoIterator>>>, + extension_wrapper: Option<&str>, + ) -> crate::SerializationResult> + where + Self: Clone + 'a, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let (somes, data): (Vec<_>, Vec<_>) = data + .into_iter() + .map(|datum| { + let datum: Option<::std::borrow::Cow<'a, Self>> = datum.map(Into::into); + (datum.is_some(), datum) + }) + .unzip(); + let bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + StructArray::new( + (if let Some(ext) = extension_wrapper { + DataType::Extension( + ext.to_owned(), + Box::new(::to_arrow_datatype()), + None, + ) + } else { + ::to_arrow_datatype() + }) + .to_logical_type() + .clone(), + vec![{ + let (somes, single_string_optional): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = datum + .as_ref() + .map(|datum| { + let Self { + single_string_optional, + .. + } = &**datum; + single_string_optional.clone() + }) + .flatten(); + (datum.is_some(), datum) + }) + .unzip(); + let single_string_optional_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + { + let inner_data: ::arrow2::buffer::Buffer = single_string_optional + .iter() + .flatten() + .flat_map(|s| s.bytes()) + .collect(); + let offsets = ::arrow2::offset::Offsets::::try_from_lengths( + single_string_optional.iter().map(|opt| { + opt.as_ref().map(|datum| datum.len()).unwrap_or_default() + }), + ) + .unwrap() + .into(); + #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] + unsafe { + Utf8Array::::new_unchecked( + { + _ = extension_wrapper; + DataType::Utf8.to_logical_type().clone() + }, + offsets, + inner_data, + single_string_optional_bitmap, + ) + } + .boxed() + } + }], + bitmap, + ) + .boxed() + }) + } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_from_arrow_opt( + data: &dyn ::arrow2::array::Array, + ) -> crate::DeserializationResult>> + where + Self: Sized, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let data = data + .as_any() + .downcast_ref::<::arrow2::array::StructArray>() + .ok_or_else(|| crate::DeserializationError::SchemaMismatch { + expected: data.data_type().clone(), + got: data.data_type().clone(), + })?; + let (data_fields, data_arrays, data_bitmap) = + (data.fields(), data.values(), data.validity()); + let is_valid = |i| data_bitmap.map_or(true, |bitmap| bitmap.get_bit(i)); + let arrays_by_name: ::std::collections::HashMap<_, _> = data_fields + .iter() + .map(|field| field.name.as_str()) + .zip(data_arrays) + .collect(); + let single_string_optional = { + let data = &**arrays_by_name["single_string_optional"]; + + data.as_any() + .downcast_ref::>() + .unwrap() + .into_iter() + .map(|v| v.map(ToOwned::to_owned)) + }; + ::itertools::izip!(single_string_optional) + .enumerate() + .map(|(i, (single_string_optional))| { + is_valid(i) + .then(|| { + Ok(Self { + single_string_optional, + }) + }) + .transpose() + }) + .collect::>>()? + }) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct AffixFuzzer11 { + pub many_floats_optional: Option>, +} + +impl<'a> From for ::std::borrow::Cow<'a, AffixFuzzer11> { + #[inline] + fn from(value: AffixFuzzer11) -> Self { + std::borrow::Cow::Owned(value) + } +} + +impl<'a> From<&'a AffixFuzzer11> for ::std::borrow::Cow<'a, AffixFuzzer11> { + #[inline] + fn from(value: &'a AffixFuzzer11) -> Self { + std::borrow::Cow::Borrowed(value) + } +} + +impl crate::Component for AffixFuzzer11 { + #[inline] + fn name() -> crate::ComponentName { + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer11") + } + + #[allow(unused_imports, clippy::wildcard_imports)] + #[inline] + fn to_arrow_datatype() -> arrow2::datatypes::DataType { + use ::arrow2::datatypes::*; + DataType::Struct(vec![Field { + name: "many_floats_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }]) + } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_to_arrow_opt<'a>( + data: impl IntoIterator>>>, + extension_wrapper: Option<&str>, + ) -> crate::SerializationResult> + where + Self: Clone + 'a, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let (somes, data): (Vec<_>, Vec<_>) = data + .into_iter() + .map(|datum| { + let datum: Option<::std::borrow::Cow<'a, Self>> = datum.map(Into::into); + (datum.is_some(), datum) + }) + .unzip(); + let bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + StructArray::new( + (if let Some(ext) = extension_wrapper { + DataType::Extension( + ext.to_owned(), + Box::new(::to_arrow_datatype()), + None, + ) + } else { + ::to_arrow_datatype() + }) + .to_logical_type() + .clone(), + vec![{ + let (somes, many_floats_optional): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = datum + .as_ref() + .map(|datum| { + let Self { + many_floats_optional, + .. + } = &**datum; + many_floats_optional.clone() + }) + .flatten(); + (datum.is_some(), datum) + }) + .unzip(); + let many_floats_optional_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + { + use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; + let many_floats_optional_inner_data: Vec<_> = many_floats_optional + .iter() + .flatten() + .flatten() + .map(ToOwned::to_owned) + .map(Some) + .collect(); + let many_floats_optional_inner_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = + many_floats_optional_inner_data.iter().any(|v| v.is_none()); + any_nones.then(|| { + many_floats_optional_inner_data + .iter() + .map(|v| v.is_some()) + .collect() + }) + }; + let offsets = ::arrow2::offset::Offsets::::try_from_lengths( + many_floats_optional.iter().map(|opt| { + opt.as_ref().map(|datum| datum.len()).unwrap_or_default() + }), + ) + .unwrap() + .into(); + ListArray::new( + { + _ = extension_wrapper; + DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + })) + .to_logical_type() + .clone() + }, + offsets, + PrimitiveArray::new( + { + _ = extension_wrapper; + DataType::Float32.to_logical_type().clone() + }, + many_floats_optional_inner_data + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect(), + many_floats_optional_inner_bitmap, + ) + .boxed(), + many_floats_optional_bitmap, + ) + .boxed() + } + }], + bitmap, + ) + .boxed() + }) + } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_from_arrow_opt( + data: &dyn ::arrow2::array::Array, + ) -> crate::DeserializationResult>> + where + Self: Sized, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let data = data + .as_any() + .downcast_ref::<::arrow2::array::StructArray>() + .ok_or_else(|| crate::DeserializationError::SchemaMismatch { + expected: data.data_type().clone(), + got: data.data_type().clone(), + })?; + let (data_fields, data_arrays, data_bitmap) = + (data.fields(), data.values(), data.validity()); + let is_valid = |i| data_bitmap.map_or(true, |bitmap| bitmap.get_bit(i)); + let arrays_by_name: ::std::collections::HashMap<_, _> = data_fields + .iter() + .map(|field| field.name.as_str()) + .zip(data_arrays) + .collect(); + let many_floats_optional = { + let data = &**arrays_by_name["many_floats_optional"]; + + { + let datatype = data.data_type(); + let data = data + .as_any() + .downcast_ref::<::arrow2::array::ListArray>() + .unwrap(); + let bitmap = data.validity().cloned(); + let offsets = { + let offsets = data.offsets(); + offsets.iter().copied().zip(offsets.iter().copied().skip(1)) + }; + let data = &**data.values(); + let data = data + .as_any() + .downcast_ref::() + .unwrap() + .into_iter() + .map(|v| v.copied()) + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: DataType::Float32, + }) + }) + .collect::>>()?; + offsets + .enumerate() + .map(move |(i, (start, end))| { + bitmap + .as_ref() + .map_or(true, |bitmap| bitmap.get_bit(i)) + .then(|| { + Ok(data + .get(start as usize..end as usize) + .ok_or_else(|| { + crate::DeserializationError::OffsetsMismatch { + bounds: (start as usize, end as usize), + len: data.len(), + datatype: datatype.clone(), + } + })? + .to_vec()) + }) + .transpose() + }) + .collect::>>>()? + .into_iter() + } + }; + ::itertools::izip!(many_floats_optional) + .enumerate() + .map(|(i, (many_floats_optional))| { + is_valid(i) + .then(|| { + Ok(Self { + many_floats_optional, + }) + }) + .transpose() + }) + .collect::>>()? + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AffixFuzzer12 { + pub many_strings_required: Vec, +} + +impl<'a> From for ::std::borrow::Cow<'a, AffixFuzzer12> { + #[inline] + fn from(value: AffixFuzzer12) -> Self { + std::borrow::Cow::Owned(value) + } +} + +impl<'a> From<&'a AffixFuzzer12> for ::std::borrow::Cow<'a, AffixFuzzer12> { + #[inline] + fn from(value: &'a AffixFuzzer12) -> Self { + std::borrow::Cow::Borrowed(value) + } +} + +impl crate::Component for AffixFuzzer12 { + #[inline] + fn name() -> crate::ComponentName { + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer12") + } + + #[allow(unused_imports, clippy::wildcard_imports)] + #[inline] + fn to_arrow_datatype() -> arrow2::datatypes::DataType { + use ::arrow2::datatypes::*; + DataType::Struct(vec![Field { + name: "many_strings_required".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + })), + is_nullable: false, + metadata: [].into(), + }]) + } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_to_arrow_opt<'a>( + data: impl IntoIterator>>>, + extension_wrapper: Option<&str>, + ) -> crate::SerializationResult> + where + Self: Clone + 'a, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let (somes, data): (Vec<_>, Vec<_>) = data + .into_iter() + .map(|datum| { + let datum: Option<::std::borrow::Cow<'a, Self>> = datum.map(Into::into); + (datum.is_some(), datum) + }) + .unzip(); + let bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + StructArray::new( + (if let Some(ext) = extension_wrapper { + DataType::Extension( + ext.to_owned(), + Box::new(::to_arrow_datatype()), + None, + ) + } else { + ::to_arrow_datatype() + }) + .to_logical_type() + .clone(), + vec![{ + let (somes, many_strings_required): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = datum.as_ref().map(|datum| { + let Self { + many_strings_required, + .. + } = &**datum; + many_strings_required.clone() + }); + (datum.is_some(), datum) + }) + .unzip(); + let many_strings_required_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + { + use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; + let many_strings_required_inner_data: Vec<_> = many_strings_required + .iter() + .flatten() + .flatten() + .map(ToOwned::to_owned) + .map(Some) + .collect(); + let many_strings_required_inner_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = + many_strings_required_inner_data.iter().any(|v| v.is_none()); + any_nones.then(|| { + many_strings_required_inner_data + .iter() + .map(|v| v.is_some()) + .collect() + }) + }; + let offsets = ::arrow2::offset::Offsets::::try_from_lengths( + many_strings_required.iter().map(|opt| { + opt.as_ref().map(|datum| datum.len()).unwrap_or_default() + }), + ) + .unwrap() + .into(); + ListArray::new( + { + _ = extension_wrapper; + DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: false, + metadata: [].into(), + })) + .to_logical_type() + .clone() + }, + offsets, + { + let inner_data: ::arrow2::buffer::Buffer = + many_strings_required_inner_data + .iter() + .flatten() + .flat_map(|s| s.bytes()) + .collect(); + let offsets = ::arrow2::offset::Offsets::::try_from_lengths( + many_strings_required_inner_data.iter().map(|opt| { + opt.as_ref().map(|datum| datum.len()).unwrap_or_default() + }), + ) + .unwrap() + .into(); + #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] + unsafe { + Utf8Array::::new_unchecked( + { + _ = extension_wrapper; + DataType::Utf8.to_logical_type().clone() + }, + offsets, + inner_data, + many_strings_required_inner_bitmap, + ) + } + .boxed() + }, + many_strings_required_bitmap, + ) + .boxed() + } + }], + bitmap, + ) + .boxed() + }) + } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_from_arrow_opt( + data: &dyn ::arrow2::array::Array, + ) -> crate::DeserializationResult>> + where + Self: Sized, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let data = data + .as_any() + .downcast_ref::<::arrow2::array::StructArray>() + .ok_or_else(|| crate::DeserializationError::SchemaMismatch { + expected: data.data_type().clone(), + got: data.data_type().clone(), + })?; + let (data_fields, data_arrays, data_bitmap) = + (data.fields(), data.values(), data.validity()); + let is_valid = |i| data_bitmap.map_or(true, |bitmap| bitmap.get_bit(i)); + let arrays_by_name: ::std::collections::HashMap<_, _> = data_fields + .iter() + .map(|field| field.name.as_str()) + .zip(data_arrays) + .collect(); + let many_strings_required = { + let data = &**arrays_by_name["many_strings_required"]; + + { + let datatype = data.data_type(); + let data = data + .as_any() + .downcast_ref::<::arrow2::array::ListArray>() + .unwrap(); + let bitmap = data.validity().cloned(); + let offsets = { + let offsets = data.offsets(); + offsets.iter().copied().zip(offsets.iter().copied().skip(1)) + }; + let data = &**data.values(); + let data = data + .as_any() + .downcast_ref::>() + .unwrap() + .into_iter() + .map(|v| v.map(ToOwned::to_owned)) + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: DataType::Utf8, + }) + }) + .collect::>>()?; + offsets + .enumerate() + .map(move |(i, (start, end))| { + bitmap + .as_ref() + .map_or(true, |bitmap| bitmap.get_bit(i)) + .then(|| { + Ok(data + .get(start as usize..end as usize) + .ok_or_else(|| { + crate::DeserializationError::OffsetsMismatch { + bounds: (start as usize, end as usize), + len: data.len(), + datatype: datatype.clone(), + } + })? + .to_vec()) + }) + .transpose() + }) + .collect::>>>()? + .into_iter() + } + }; + ::itertools::izip!(many_strings_required) + .enumerate() + .map(|(i, (many_strings_required))| { + is_valid(i) + .then(|| { + Ok(Self { + many_strings_required: many_strings_required.ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: data.data_type().clone(), + } + })?, + }) + }) + .transpose() + }) + .collect::>>()? + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AffixFuzzer13 { + pub many_strings_optional: Option>, +} + +impl<'a> From for ::std::borrow::Cow<'a, AffixFuzzer13> { + #[inline] + fn from(value: AffixFuzzer13) -> Self { + std::borrow::Cow::Owned(value) + } +} + +impl<'a> From<&'a AffixFuzzer13> for ::std::borrow::Cow<'a, AffixFuzzer13> { + #[inline] + fn from(value: &'a AffixFuzzer13) -> Self { + std::borrow::Cow::Borrowed(value) + } +} + +impl crate::Component for AffixFuzzer13 { + #[inline] + fn name() -> crate::ComponentName { + crate::ComponentName::Borrowed("rerun.testing.components.AffixFuzzer13") + } + + #[allow(unused_imports, clippy::wildcard_imports)] + #[inline] + fn to_arrow_datatype() -> arrow2::datatypes::DataType { + use ::arrow2::datatypes::*; + DataType::Struct(vec![Field { + name: "many_strings_optional".to_owned(), + data_type: DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + })), + is_nullable: true, + metadata: [].into(), + }]) + } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_to_arrow_opt<'a>( + data: impl IntoIterator>>>, + extension_wrapper: Option<&str>, + ) -> crate::SerializationResult> + where + Self: Clone + 'a, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let (somes, data): (Vec<_>, Vec<_>) = data + .into_iter() + .map(|datum| { + let datum: Option<::std::borrow::Cow<'a, Self>> = datum.map(Into::into); + (datum.is_some(), datum) + }) + .unzip(); + let bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + StructArray::new( + (if let Some(ext) = extension_wrapper { + DataType::Extension( + ext.to_owned(), + Box::new(::to_arrow_datatype()), + None, + ) + } else { + ::to_arrow_datatype() + }) + .to_logical_type() + .clone(), + vec![{ + let (somes, many_strings_optional): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = datum + .as_ref() + .map(|datum| { + let Self { + many_strings_optional, + .. + } = &**datum; + many_strings_optional.clone() + }) + .flatten(); + (datum.is_some(), datum) + }) + .unzip(); + let many_strings_optional_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + { + use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; + let many_strings_optional_inner_data: Vec<_> = many_strings_optional + .iter() + .flatten() + .flatten() + .map(ToOwned::to_owned) + .map(Some) + .collect(); + let many_strings_optional_inner_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = + many_strings_optional_inner_data.iter().any(|v| v.is_none()); + any_nones.then(|| { + many_strings_optional_inner_data + .iter() + .map(|v| v.is_some()) + .collect() + }) + }; + let offsets = ::arrow2::offset::Offsets::::try_from_lengths( + many_strings_optional.iter().map(|opt| { + opt.as_ref().map(|datum| datum.len()).unwrap_or_default() + }), + ) + .unwrap() + .into(); + ListArray::new( + { + _ = extension_wrapper; + DataType::List(Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Utf8, + is_nullable: true, + metadata: [].into(), + })) + .to_logical_type() + .clone() + }, + offsets, + { + let inner_data: ::arrow2::buffer::Buffer = + many_strings_optional_inner_data + .iter() + .flatten() + .flat_map(|s| s.bytes()) + .collect(); + let offsets = ::arrow2::offset::Offsets::::try_from_lengths( + many_strings_optional_inner_data.iter().map(|opt| { + opt.as_ref().map(|datum| datum.len()).unwrap_or_default() + }), + ) + .unwrap() + .into(); + #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] + unsafe { + Utf8Array::::new_unchecked( + { + _ = extension_wrapper; + DataType::Utf8.to_logical_type().clone() + }, + offsets, + inner_data, + many_strings_optional_inner_bitmap, + ) + } + .boxed() + }, + many_strings_optional_bitmap, + ) + .boxed() + } + }], + bitmap, + ) + .boxed() + }) + } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_from_arrow_opt( + data: &dyn ::arrow2::array::Array, + ) -> crate::DeserializationResult>> + where + Self: Sized, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let data = data + .as_any() + .downcast_ref::<::arrow2::array::StructArray>() + .ok_or_else(|| crate::DeserializationError::SchemaMismatch { + expected: data.data_type().clone(), + got: data.data_type().clone(), + })?; + let (data_fields, data_arrays, data_bitmap) = + (data.fields(), data.values(), data.validity()); + let is_valid = |i| data_bitmap.map_or(true, |bitmap| bitmap.get_bit(i)); + let arrays_by_name: ::std::collections::HashMap<_, _> = data_fields + .iter() + .map(|field| field.name.as_str()) + .zip(data_arrays) + .collect(); + let many_strings_optional = { + let data = &**arrays_by_name["many_strings_optional"]; + + { + let datatype = data.data_type(); + let data = data + .as_any() + .downcast_ref::<::arrow2::array::ListArray>() + .unwrap(); + let bitmap = data.validity().cloned(); + let offsets = { + let offsets = data.offsets(); + offsets.iter().copied().zip(offsets.iter().copied().skip(1)) + }; + let data = &**data.values(); + let data = data + .as_any() + .downcast_ref::>() + .unwrap() + .into_iter() + .map(|v| v.map(ToOwned::to_owned)) + .map(|v| { + v.ok_or_else(|| crate::DeserializationError::MissingData { + datatype: DataType::Utf8, + }) + }) + .collect::>>()?; + offsets + .enumerate() + .map(move |(i, (start, end))| { + bitmap + .as_ref() + .map_or(true, |bitmap| bitmap.get_bit(i)) + .then(|| { + Ok(data + .get(start as usize..end as usize) + .ok_or_else(|| { + crate::DeserializationError::OffsetsMismatch { + bounds: (start as usize, end as usize), + len: data.len(), + datatype: datatype.clone(), + } + })? + .to_vec()) + }) + .transpose() + }) + .collect::>>>()? + .into_iter() + } + }; + ::itertools::izip!(many_strings_optional) + .enumerate() + .map(|(i, (many_strings_optional))| { + is_valid(i) + .then(|| { + Ok(Self { + many_strings_optional, + }) + }) + .transpose() + }) + .collect::>>()? + }) + } +} diff --git a/crates/re_types/src/components/instance_key.rs b/crates/re_types/src/components/instance_key.rs index 452854675e91..1c8ea3978876 100644 --- a/crates/re_types/src/components/instance_key.rs +++ b/crates/re_types/src/components/instance_key.rs @@ -6,6 +6,7 @@ #![allow(clippy::map_flatten)] #![allow(clippy::needless_question_mark)] #![allow(clippy::too_many_arguments)] +#![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] #[doc = "A unique numeric identifier for each individual instance within a batch."] diff --git a/crates/re_types/src/components/keypoint_id.rs b/crates/re_types/src/components/keypoint_id.rs index 116b79addb2a..a9a84c28896a 100644 --- a/crates/re_types/src/components/keypoint_id.rs +++ b/crates/re_types/src/components/keypoint_id.rs @@ -6,6 +6,7 @@ #![allow(clippy::map_flatten)] #![allow(clippy::needless_question_mark)] #![allow(clippy::too_many_arguments)] +#![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] #[doc = "A 16-bit ID representing a type of semantic keypoint within a class."] diff --git a/crates/re_types/src/components/label.rs b/crates/re_types/src/components/label.rs index 64124c4a8b4e..45977308297e 100644 --- a/crates/re_types/src/components/label.rs +++ b/crates/re_types/src/components/label.rs @@ -6,6 +6,7 @@ #![allow(clippy::map_flatten)] #![allow(clippy::needless_question_mark)] #![allow(clippy::too_many_arguments)] +#![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] #[doc = "A String label component."] diff --git a/crates/re_types/src/components/mod.rs b/crates/re_types/src/components/mod.rs index fc1d3e59e9e8..c78f73fde1c8 100644 --- a/crates/re_types/src/components/mod.rs +++ b/crates/re_types/src/components/mod.rs @@ -22,7 +22,9 @@ pub use self::class_id::ClassId; pub use self::color::Color; pub use self::draw_order::DrawOrder; pub use self::fuzzy::{ - AffixFuzzer1, AffixFuzzer2, AffixFuzzer3, AffixFuzzer4, AffixFuzzer5, AffixFuzzer6, + AffixFuzzer1, AffixFuzzer10, AffixFuzzer11, AffixFuzzer12, AffixFuzzer13, AffixFuzzer2, + AffixFuzzer3, AffixFuzzer4, AffixFuzzer5, AffixFuzzer6, AffixFuzzer7, AffixFuzzer8, + AffixFuzzer9, }; pub use self::instance_key::InstanceKey; pub use self::keypoint_id::KeypointId; diff --git a/crates/re_types/src/components/point2d.rs b/crates/re_types/src/components/point2d.rs index 49087d07c6ae..472f08f7cdd6 100644 --- a/crates/re_types/src/components/point2d.rs +++ b/crates/re_types/src/components/point2d.rs @@ -6,6 +6,7 @@ #![allow(clippy::map_flatten)] #![allow(clippy::needless_question_mark)] #![allow(clippy::too_many_arguments)] +#![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] #[doc = "A point in 2D space."] diff --git a/crates/re_types/src/components/radius.rs b/crates/re_types/src/components/radius.rs index edd85f330480..9e0ba6a2babe 100644 --- a/crates/re_types/src/components/radius.rs +++ b/crates/re_types/src/components/radius.rs @@ -6,6 +6,7 @@ #![allow(clippy::map_flatten)] #![allow(clippy::needless_question_mark)] #![allow(clippy::too_many_arguments)] +#![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] #[doc = "A Radius component."] diff --git a/crates/re_types/src/datatypes/fuzzy.rs b/crates/re_types/src/datatypes/fuzzy.rs index 1a23d47aad3d..1bcced7de879 100644 --- a/crates/re_types/src/datatypes/fuzzy.rs +++ b/crates/re_types/src/datatypes/fuzzy.rs @@ -6,6 +6,7 @@ #![allow(clippy::map_flatten)] #![allow(clippy::needless_question_mark)] #![allow(clippy::too_many_arguments)] +#![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] #[derive(Debug, Clone, PartialEq)] diff --git a/crates/re_types/src/datatypes/point2d.rs b/crates/re_types/src/datatypes/point2d.rs index 98abf164c17b..d702f49f68e0 100644 --- a/crates/re_types/src/datatypes/point2d.rs +++ b/crates/re_types/src/datatypes/point2d.rs @@ -6,6 +6,7 @@ #![allow(clippy::map_flatten)] #![allow(clippy::needless_question_mark)] #![allow(clippy::too_many_arguments)] +#![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] #[doc = "A point in 2D space."] diff --git a/crates/re_types/src/datatypes/vec2d.rs b/crates/re_types/src/datatypes/vec2d.rs index 571e25a7d18c..89c42ec71741 100644 --- a/crates/re_types/src/datatypes/vec2d.rs +++ b/crates/re_types/src/datatypes/vec2d.rs @@ -6,6 +6,7 @@ #![allow(clippy::map_flatten)] #![allow(clippy::needless_question_mark)] #![allow(clippy::too_many_arguments)] +#![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] #[doc = "A vector in 2D space."] diff --git a/crates/re_types_builder/src/codegen/rust.rs b/crates/re_types_builder/src/codegen/rust.rs index 68d6e619308e..b3b61d781364 100644 --- a/crates/re_types_builder/src/codegen/rust.rs +++ b/crates/re_types_builder/src/codegen/rust.rs @@ -125,6 +125,7 @@ fn create_files( code.push_text("#![allow(clippy::map_flatten)]", 2, 0); code.push_text("#![allow(clippy::needless_question_mark)]", 2, 0); code.push_text("#![allow(clippy::too_many_arguments)]", 2, 0); + code.push_text("#![allow(clippy::too_many_lines)]", 2, 0); code.push_text("#![allow(clippy::unnecessary_cast)]", 2, 0); for obj in objs { diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/archetypes/fuzzy.py b/rerun_py/rerun_sdk/rerun/_rerun2/archetypes/fuzzy.py index efccca567ffa..ce6276757eaa 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/archetypes/fuzzy.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/archetypes/fuzzy.py @@ -38,6 +38,34 @@ class AffixFuzzer1(Archetype): metadata={"component": "primary"}, converter=components.AffixFuzzer6Array.from_similar, # type: ignore[misc] ) + fuzz1007: components.AffixFuzzer7Array = field( + metadata={"component": "primary"}, + converter=components.AffixFuzzer7Array.from_similar, # type: ignore[misc] + ) + fuzz1008: components.AffixFuzzer8Array = field( + metadata={"component": "primary"}, + converter=components.AffixFuzzer8Array.from_similar, # type: ignore[misc] + ) + fuzz1009: components.AffixFuzzer9Array = field( + metadata={"component": "primary"}, + converter=components.AffixFuzzer9Array.from_similar, # type: ignore[misc] + ) + fuzz1010: components.AffixFuzzer10Array = field( + metadata={"component": "primary"}, + converter=components.AffixFuzzer10Array.from_similar, # type: ignore[misc] + ) + fuzz1011: components.AffixFuzzer11Array = field( + metadata={"component": "primary"}, + converter=components.AffixFuzzer11Array.from_similar, # type: ignore[misc] + ) + fuzz1012: components.AffixFuzzer12Array = field( + metadata={"component": "primary"}, + converter=components.AffixFuzzer12Array.from_similar, # type: ignore[misc] + ) + fuzz1013: components.AffixFuzzer13Array = field( + metadata={"component": "primary"}, + converter=components.AffixFuzzer13Array.from_similar, # type: ignore[misc] + ) fuzz1101: components.AffixFuzzer1Array = field( metadata={"component": "primary"}, converter=components.AffixFuzzer1Array.from_similar, # type: ignore[misc] @@ -62,6 +90,34 @@ class AffixFuzzer1(Archetype): metadata={"component": "primary"}, converter=components.AffixFuzzer6Array.from_similar, # type: ignore[misc] ) + fuzz1107: components.AffixFuzzer7Array = field( + metadata={"component": "primary"}, + converter=components.AffixFuzzer7Array.from_similar, # type: ignore[misc] + ) + fuzz1108: components.AffixFuzzer8Array = field( + metadata={"component": "primary"}, + converter=components.AffixFuzzer8Array.from_similar, # type: ignore[misc] + ) + fuzz1109: components.AffixFuzzer9Array = field( + metadata={"component": "primary"}, + converter=components.AffixFuzzer9Array.from_similar, # type: ignore[misc] + ) + fuzz1110: components.AffixFuzzer10Array = field( + metadata={"component": "primary"}, + converter=components.AffixFuzzer10Array.from_similar, # type: ignore[misc] + ) + fuzz1111: components.AffixFuzzer11Array = field( + metadata={"component": "primary"}, + converter=components.AffixFuzzer11Array.from_similar, # type: ignore[misc] + ) + fuzz1112: components.AffixFuzzer12Array = field( + metadata={"component": "primary"}, + converter=components.AffixFuzzer12Array.from_similar, # type: ignore[misc] + ) + fuzz1113: components.AffixFuzzer13Array = field( + metadata={"component": "primary"}, + converter=components.AffixFuzzer13Array.from_similar, # type: ignore[misc] + ) fuzz2001: components.AffixFuzzer1Array | None = field( metadata={"component": "secondary"}, default=None, @@ -92,6 +148,41 @@ class AffixFuzzer1(Archetype): default=None, converter=components.AffixFuzzer6Array.from_similar, # type: ignore[misc] ) + fuzz2007: components.AffixFuzzer7Array | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.AffixFuzzer7Array.from_similar, # type: ignore[misc] + ) + fuzz2008: components.AffixFuzzer8Array | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.AffixFuzzer8Array.from_similar, # type: ignore[misc] + ) + fuzz2009: components.AffixFuzzer9Array | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.AffixFuzzer9Array.from_similar, # type: ignore[misc] + ) + fuzz2010: components.AffixFuzzer10Array | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.AffixFuzzer10Array.from_similar, # type: ignore[misc] + ) + fuzz2011: components.AffixFuzzer11Array | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.AffixFuzzer11Array.from_similar, # type: ignore[misc] + ) + fuzz2012: components.AffixFuzzer12Array | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.AffixFuzzer12Array.from_similar, # type: ignore[misc] + ) + fuzz2013: components.AffixFuzzer13Array | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.AffixFuzzer13Array.from_similar, # type: ignore[misc] + ) fuzz2101: components.AffixFuzzer1Array | None = field( metadata={"component": "secondary"}, default=None, @@ -122,5 +213,40 @@ class AffixFuzzer1(Archetype): default=None, converter=components.AffixFuzzer6Array.from_similar, # type: ignore[misc] ) + fuzz2107: components.AffixFuzzer7Array | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.AffixFuzzer7Array.from_similar, # type: ignore[misc] + ) + fuzz2108: components.AffixFuzzer8Array | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.AffixFuzzer8Array.from_similar, # type: ignore[misc] + ) + fuzz2109: components.AffixFuzzer9Array | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.AffixFuzzer9Array.from_similar, # type: ignore[misc] + ) + fuzz2110: components.AffixFuzzer10Array | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.AffixFuzzer10Array.from_similar, # type: ignore[misc] + ) + fuzz2111: components.AffixFuzzer11Array | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.AffixFuzzer11Array.from_similar, # type: ignore[misc] + ) + fuzz2112: components.AffixFuzzer12Array | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.AffixFuzzer12Array.from_similar, # type: ignore[misc] + ) + fuzz2113: components.AffixFuzzer13Array | None = field( + metadata={"component": "secondary"}, + default=None, + converter=components.AffixFuzzer13Array.from_similar, # type: ignore[misc] + ) __str__ = Archetype.__str__ __repr__ = Archetype.__repr__ diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/__init__.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/__init__.py index 9c5bcfe20bf5..2f3c5b4701d4 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/components/__init__.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/components/__init__.py @@ -18,6 +18,41 @@ AffixFuzzer5Type, AffixFuzzer6Array, AffixFuzzer6Type, + AffixFuzzer7, + AffixFuzzer7Array, + AffixFuzzer7ArrayLike, + AffixFuzzer7Like, + AffixFuzzer7Type, + AffixFuzzer8, + AffixFuzzer8Array, + AffixFuzzer8ArrayLike, + AffixFuzzer8Like, + AffixFuzzer8Type, + AffixFuzzer9, + AffixFuzzer9Array, + AffixFuzzer9ArrayLike, + AffixFuzzer9Like, + AffixFuzzer9Type, + AffixFuzzer10, + AffixFuzzer10Array, + AffixFuzzer10ArrayLike, + AffixFuzzer10Like, + AffixFuzzer10Type, + AffixFuzzer11, + AffixFuzzer11Array, + AffixFuzzer11ArrayLike, + AffixFuzzer11Like, + AffixFuzzer11Type, + AffixFuzzer12, + AffixFuzzer12Array, + AffixFuzzer12ArrayLike, + AffixFuzzer12Like, + AffixFuzzer12Type, + AffixFuzzer13, + AffixFuzzer13Array, + AffixFuzzer13ArrayLike, + AffixFuzzer13Like, + AffixFuzzer13Type, ) from .instance_key import InstanceKey, InstanceKeyArray, InstanceKeyArrayLike, InstanceKeyLike, InstanceKeyType from .keypoint_id import KeypointId, KeypointIdArray, KeypointIdArrayLike, KeypointIdLike, KeypointIdType @@ -26,6 +61,26 @@ from .radius import Radius, RadiusArray, RadiusArrayLike, RadiusLike, RadiusType __all__ = [ + "AffixFuzzer10", + "AffixFuzzer10Array", + "AffixFuzzer10ArrayLike", + "AffixFuzzer10Like", + "AffixFuzzer10Type", + "AffixFuzzer11", + "AffixFuzzer11Array", + "AffixFuzzer11ArrayLike", + "AffixFuzzer11Like", + "AffixFuzzer11Type", + "AffixFuzzer12", + "AffixFuzzer12Array", + "AffixFuzzer12ArrayLike", + "AffixFuzzer12Like", + "AffixFuzzer12Type", + "AffixFuzzer13", + "AffixFuzzer13Array", + "AffixFuzzer13ArrayLike", + "AffixFuzzer13Like", + "AffixFuzzer13Type", "AffixFuzzer1Array", "AffixFuzzer1Type", "AffixFuzzer2Array", @@ -38,6 +93,21 @@ "AffixFuzzer5Type", "AffixFuzzer6Array", "AffixFuzzer6Type", + "AffixFuzzer7", + "AffixFuzzer7Array", + "AffixFuzzer7ArrayLike", + "AffixFuzzer7Like", + "AffixFuzzer7Type", + "AffixFuzzer8", + "AffixFuzzer8Array", + "AffixFuzzer8ArrayLike", + "AffixFuzzer8Like", + "AffixFuzzer8Type", + "AffixFuzzer9", + "AffixFuzzer9Array", + "AffixFuzzer9ArrayLike", + "AffixFuzzer9Like", + "AffixFuzzer9Type", "ClassId", "ClassIdArray", "ClassIdArrayLike", diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/fuzzy.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/fuzzy.py index d8392e7ebfff..6a1f35eaf893 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/components/fuzzy.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/components/fuzzy.py @@ -2,13 +2,45 @@ from __future__ import annotations +from typing import Sequence, Union + +import numpy as np +import numpy.typing as npt +import pyarrow as pa +from attrs import define, field + from .. import datatypes from .._baseclasses import ( BaseDelegatingExtensionArray, BaseDelegatingExtensionType, + BaseExtensionArray, + BaseExtensionType, +) +from .._converters import ( + to_np_float32, ) __all__ = [ + "AffixFuzzer10", + "AffixFuzzer10Array", + "AffixFuzzer10ArrayLike", + "AffixFuzzer10Like", + "AffixFuzzer10Type", + "AffixFuzzer11", + "AffixFuzzer11Array", + "AffixFuzzer11ArrayLike", + "AffixFuzzer11Like", + "AffixFuzzer11Type", + "AffixFuzzer12", + "AffixFuzzer12Array", + "AffixFuzzer12ArrayLike", + "AffixFuzzer12Like", + "AffixFuzzer12Type", + "AffixFuzzer13", + "AffixFuzzer13Array", + "AffixFuzzer13ArrayLike", + "AffixFuzzer13Like", + "AffixFuzzer13Type", "AffixFuzzer1Array", "AffixFuzzer1Type", "AffixFuzzer2Array", @@ -21,6 +53,21 @@ "AffixFuzzer5Type", "AffixFuzzer6Array", "AffixFuzzer6Type", + "AffixFuzzer7", + "AffixFuzzer7Array", + "AffixFuzzer7ArrayLike", + "AffixFuzzer7Like", + "AffixFuzzer7Type", + "AffixFuzzer8", + "AffixFuzzer8Array", + "AffixFuzzer8ArrayLike", + "AffixFuzzer8Like", + "AffixFuzzer8Type", + "AffixFuzzer9", + "AffixFuzzer9Array", + "AffixFuzzer9ArrayLike", + "AffixFuzzer9Like", + "AffixFuzzer9Type", ] @@ -124,3 +171,325 @@ class AffixFuzzer6Array(BaseDelegatingExtensionArray[datatypes.AffixFuzzer1Array # TODO(cmc): bring back registration to pyarrow once legacy types are gone # pa.register_extension_type(AffixFuzzer6Type()) + + +@define +class AffixFuzzer7: + many_optional: list[datatypes.AffixFuzzer1] | None = field(default=None) + + +AffixFuzzer7Like = AffixFuzzer7 +AffixFuzzer7ArrayLike = Union[ + AffixFuzzer7, + Sequence[AffixFuzzer7Like], +] + + +# --- Arrow support --- + + +class AffixFuzzer7Type(BaseExtensionType): + def __init__(self) -> None: + pa.ExtensionType.__init__( + self, + pa.struct( + [ + pa.field( + "many_optional", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field("single_float_optional", pa.float32(), True, {}), + pa.field("single_string_required", pa.utf8(), False, {}), + pa.field("single_string_optional", pa.utf8(), True, {}), + pa.field( + "many_floats_optional", + pa.list_(pa.field("item", pa.float32(), True, {})), + True, + {}, + ), + pa.field( + "many_strings_required", + pa.list_(pa.field("item", pa.utf8(), False, {})), + False, + {}, + ), + pa.field( + "many_strings_optional", + pa.list_(pa.field("item", pa.utf8(), True, {})), + True, + {}, + ), + ] + ), + True, + {}, + ) + ), + True, + {}, + ) + ] + ), + "rerun.testing.components.AffixFuzzer7", + ) + + +class AffixFuzzer7Array(BaseExtensionArray[AffixFuzzer7ArrayLike]): + _EXTENSION_NAME = "rerun.testing.components.AffixFuzzer7" + _EXTENSION_TYPE = AffixFuzzer7Type + + @staticmethod + def _native_to_pa_array(data: AffixFuzzer7ArrayLike, data_type: pa.DataType) -> pa.Array: + raise NotImplementedError + + +AffixFuzzer7Type._ARRAY_TYPE = AffixFuzzer7Array + +# TODO(cmc): bring back registration to pyarrow once legacy types are gone +# pa.register_extension_type(AffixFuzzer7Type()) + + +@define +class AffixFuzzer8: + single_float_optional: float | None = field(default=None) + + def __array__(self, dtype: npt.DTypeLike = None) -> npt.ArrayLike: + return np.asarray(self.single_float_optional, dtype=dtype) + + +AffixFuzzer8Like = AffixFuzzer8 +AffixFuzzer8ArrayLike = Union[ + AffixFuzzer8, + Sequence[AffixFuzzer8Like], +] + + +# --- Arrow support --- + + +class AffixFuzzer8Type(BaseExtensionType): + def __init__(self) -> None: + pa.ExtensionType.__init__( + self, + pa.struct([pa.field("single_float_optional", pa.float32(), True, {})]), + "rerun.testing.components.AffixFuzzer8", + ) + + +class AffixFuzzer8Array(BaseExtensionArray[AffixFuzzer8ArrayLike]): + _EXTENSION_NAME = "rerun.testing.components.AffixFuzzer8" + _EXTENSION_TYPE = AffixFuzzer8Type + + @staticmethod + def _native_to_pa_array(data: AffixFuzzer8ArrayLike, data_type: pa.DataType) -> pa.Array: + raise NotImplementedError + + +AffixFuzzer8Type._ARRAY_TYPE = AffixFuzzer8Array + +# TODO(cmc): bring back registration to pyarrow once legacy types are gone +# pa.register_extension_type(AffixFuzzer8Type()) + + +@define +class AffixFuzzer9: + single_string_required: str = field() + + def __str__(self) -> str: + return str(self.single_string_required) + + +AffixFuzzer9Like = AffixFuzzer9 +AffixFuzzer9ArrayLike = Union[ + AffixFuzzer9, + Sequence[AffixFuzzer9Like], +] + + +# --- Arrow support --- + + +class AffixFuzzer9Type(BaseExtensionType): + def __init__(self) -> None: + pa.ExtensionType.__init__( + self, + pa.struct([pa.field("single_string_required", pa.utf8(), False, {})]), + "rerun.testing.components.AffixFuzzer9", + ) + + +class AffixFuzzer9Array(BaseExtensionArray[AffixFuzzer9ArrayLike]): + _EXTENSION_NAME = "rerun.testing.components.AffixFuzzer9" + _EXTENSION_TYPE = AffixFuzzer9Type + + @staticmethod + def _native_to_pa_array(data: AffixFuzzer9ArrayLike, data_type: pa.DataType) -> pa.Array: + raise NotImplementedError + + +AffixFuzzer9Type._ARRAY_TYPE = AffixFuzzer9Array + +# TODO(cmc): bring back registration to pyarrow once legacy types are gone +# pa.register_extension_type(AffixFuzzer9Type()) + + +@define +class AffixFuzzer10: + single_string_optional: str | None = field(default=None) + + +AffixFuzzer10Like = AffixFuzzer10 +AffixFuzzer10ArrayLike = Union[ + AffixFuzzer10, + Sequence[AffixFuzzer10Like], +] + + +# --- Arrow support --- + + +class AffixFuzzer10Type(BaseExtensionType): + def __init__(self) -> None: + pa.ExtensionType.__init__( + self, + pa.struct([pa.field("single_string_optional", pa.utf8(), True, {})]), + "rerun.testing.components.AffixFuzzer10", + ) + + +class AffixFuzzer10Array(BaseExtensionArray[AffixFuzzer10ArrayLike]): + _EXTENSION_NAME = "rerun.testing.components.AffixFuzzer10" + _EXTENSION_TYPE = AffixFuzzer10Type + + @staticmethod + def _native_to_pa_array(data: AffixFuzzer10ArrayLike, data_type: pa.DataType) -> pa.Array: + raise NotImplementedError + + +AffixFuzzer10Type._ARRAY_TYPE = AffixFuzzer10Array + +# TODO(cmc): bring back registration to pyarrow once legacy types are gone +# pa.register_extension_type(AffixFuzzer10Type()) + + +@define +class AffixFuzzer11: + many_floats_optional: npt.NDArray[np.float32] | None = field(default=None, converter=to_np_float32) + + def __array__(self, dtype: npt.DTypeLike = None) -> npt.ArrayLike: + return np.asarray(self.many_floats_optional, dtype=dtype) + + +AffixFuzzer11Like = AffixFuzzer11 +AffixFuzzer11ArrayLike = Union[ + AffixFuzzer11, + Sequence[AffixFuzzer11Like], +] + + +# --- Arrow support --- + + +class AffixFuzzer11Type(BaseExtensionType): + def __init__(self) -> None: + pa.ExtensionType.__init__( + self, + pa.struct([pa.field("many_floats_optional", pa.list_(pa.field("item", pa.float32(), True, {})), True, {})]), + "rerun.testing.components.AffixFuzzer11", + ) + + +class AffixFuzzer11Array(BaseExtensionArray[AffixFuzzer11ArrayLike]): + _EXTENSION_NAME = "rerun.testing.components.AffixFuzzer11" + _EXTENSION_TYPE = AffixFuzzer11Type + + @staticmethod + def _native_to_pa_array(data: AffixFuzzer11ArrayLike, data_type: pa.DataType) -> pa.Array: + raise NotImplementedError + + +AffixFuzzer11Type._ARRAY_TYPE = AffixFuzzer11Array + +# TODO(cmc): bring back registration to pyarrow once legacy types are gone +# pa.register_extension_type(AffixFuzzer11Type()) + + +@define +class AffixFuzzer12: + many_strings_required: list[str] = field() + + +AffixFuzzer12Like = AffixFuzzer12 +AffixFuzzer12ArrayLike = Union[ + AffixFuzzer12, + Sequence[AffixFuzzer12Like], +] + + +# --- Arrow support --- + + +class AffixFuzzer12Type(BaseExtensionType): + def __init__(self) -> None: + pa.ExtensionType.__init__( + self, + pa.struct([pa.field("many_strings_required", pa.list_(pa.field("item", pa.utf8(), False, {})), False, {})]), + "rerun.testing.components.AffixFuzzer12", + ) + + +class AffixFuzzer12Array(BaseExtensionArray[AffixFuzzer12ArrayLike]): + _EXTENSION_NAME = "rerun.testing.components.AffixFuzzer12" + _EXTENSION_TYPE = AffixFuzzer12Type + + @staticmethod + def _native_to_pa_array(data: AffixFuzzer12ArrayLike, data_type: pa.DataType) -> pa.Array: + raise NotImplementedError + + +AffixFuzzer12Type._ARRAY_TYPE = AffixFuzzer12Array + +# TODO(cmc): bring back registration to pyarrow once legacy types are gone +# pa.register_extension_type(AffixFuzzer12Type()) + + +@define +class AffixFuzzer13: + many_strings_optional: list[str] | None = field(default=None) + + +AffixFuzzer13Like = AffixFuzzer13 +AffixFuzzer13ArrayLike = Union[ + AffixFuzzer13, + Sequence[AffixFuzzer13Like], +] + + +# --- Arrow support --- + + +class AffixFuzzer13Type(BaseExtensionType): + def __init__(self) -> None: + pa.ExtensionType.__init__( + self, + pa.struct([pa.field("many_strings_optional", pa.list_(pa.field("item", pa.utf8(), True, {})), True, {})]), + "rerun.testing.components.AffixFuzzer13", + ) + + +class AffixFuzzer13Array(BaseExtensionArray[AffixFuzzer13ArrayLike]): + _EXTENSION_NAME = "rerun.testing.components.AffixFuzzer13" + _EXTENSION_TYPE = AffixFuzzer13Type + + @staticmethod + def _native_to_pa_array(data: AffixFuzzer13ArrayLike, data_type: pa.DataType) -> pa.Array: + raise NotImplementedError + + +AffixFuzzer13Type._ARRAY_TYPE = AffixFuzzer13Array + +# TODO(cmc): bring back registration to pyarrow once legacy types are gone +# pa.register_extension_type(AffixFuzzer13Type()) From e8c6caa3036e0bd20d1abb7ed61ab84cb4ad58f8 Mon Sep 17 00:00:00 2001 From: Clement Rey Date: Wed, 5 Jul 2023 15:46:26 +0200 Subject: [PATCH 10/17] bring fuzzy tests back to life --- crates/re_types/tests/fuzzy.rs | 85 +++++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/crates/re_types/tests/fuzzy.rs b/crates/re_types/tests/fuzzy.rs index 3b6c764e84f1..910a1100f696 100644 --- a/crates/re_types/tests/fuzzy.rs +++ b/crates/re_types/tests/fuzzy.rs @@ -16,6 +16,7 @@ fn roundtrip() { many_strings_optional: Some(vec!["10".into(), "20".into()]), }, }; + let fuzzy2 = re_types::components::AffixFuzzer2(re_types::datatypes::AffixFuzzer1 { single_float_optional: None, single_string_required: "b".into(), @@ -24,6 +25,7 @@ fn roundtrip() { many_strings_required: vec!["3".into(), "4".into()], many_strings_optional: None, }); + let fuzzy3 = re_types::components::AffixFuzzer3 { single_required: re_types::datatypes::AffixFuzzer1 { single_float_optional: Some(3.0), @@ -34,6 +36,7 @@ fn roundtrip() { many_strings_optional: Some(vec!["50".into(), "60".into()]), }, }; + let fuzzy4 = re_types::components::AffixFuzzer4 { single_optional: Some(re_types::datatypes::AffixFuzzer1 { single_float_optional: None, @@ -44,6 +47,7 @@ fn roundtrip() { many_strings_optional: None, }), }; + let fuzzy5 = re_types::components::AffixFuzzer5(None); // let fuzzy5 = re_types::components::AffixFuzzer5(Some(re_types::datatypes::AffixFuzzer1 { // single_float_optional: None, @@ -53,6 +57,7 @@ fn roundtrip() { // many_strings_required: vec!["7".into(), "8".into()], // many_strings_optional: None, // })); + let fuzzy6 = re_types::components::AffixFuzzer6 { single_optional: None, }; @@ -67,6 +72,67 @@ fn roundtrip() { // }), // }; + let fuzzy7_1 = re_types::components::AffixFuzzer7 { + many_optional: None, + }; + let fuzzy7_2 = re_types::components::AffixFuzzer7 { + many_optional: Some(vec![re_types::datatypes::AffixFuzzer1 { + single_float_optional: None, + single_string_required: "d".into(), + single_string_optional: None, + many_floats_optional: Some(vec![4.0, 40.0, 400.0]), + many_strings_required: vec!["7".into(), "8".into()], + many_strings_optional: None, + }]), + }; + + let fuzzy8_1 = re_types::components::AffixFuzzer8 { + single_float_optional: None, + }; + let fuzzy8_2 = re_types::components::AffixFuzzer8 { + single_float_optional: Some(1.0), + }; + + let fuzzy9_1 = re_types::components::AffixFuzzer9 { + single_string_required: "b".into(), + }; + let fuzzy9_2 = re_types::components::AffixFuzzer9 { + single_string_required: "a".into(), + }; + + let fuzzy10_1 = re_types::components::AffixFuzzer10 { + single_string_optional: None, + }; + let fuzzy10_2 = re_types::components::AffixFuzzer10 { + single_string_optional: Some("a".into()), + }; + + let fuzzy11_1 = re_types::components::AffixFuzzer11 { + many_floats_optional: Some(vec![1.0, 10.0]), + }; + let fuzzy11_2 = re_types::components::AffixFuzzer11 { + many_floats_optional: Some(vec![2.0, 20.0, 200.0]), + }; + + let fuzzy12_1 = re_types::components::AffixFuzzer12 { + many_strings_required: vec!["1".into(), "10".into()], + }; + let fuzzy12_2 = re_types::components::AffixFuzzer12 { + many_strings_required: vec!["20".into(), "200".into(), "2000".into()], + }; + + let fuzzy13_1 = re_types::components::AffixFuzzer13 { + many_strings_optional: None, + }; + let fuzzy13_2 = re_types::components::AffixFuzzer13 { + many_strings_optional: Some(vec![ + "30".into(), + "300".into(), + "3000".into(), + "30000".into(), + ]), + }; + let arch = AffixFuzzer1::new( fuzzy1.clone(), fuzzy2.clone(), @@ -74,19 +140,36 @@ fn roundtrip() { fuzzy4.clone(), fuzzy5.clone(), fuzzy6.clone(), + fuzzy7_1.clone(), + fuzzy8_1.clone(), + fuzzy9_1.clone(), + fuzzy10_1.clone(), + fuzzy11_1.clone(), + fuzzy12_1.clone(), + fuzzy13_1.clone(), [fuzzy1.clone(), fuzzy1.clone(), fuzzy1.clone()], [fuzzy2.clone(), fuzzy2.clone(), fuzzy2.clone()], [fuzzy3.clone(), fuzzy3.clone(), fuzzy3.clone()], [fuzzy4.clone(), fuzzy4.clone(), fuzzy4.clone()], [fuzzy5.clone(), fuzzy5.clone(), fuzzy5.clone()], [fuzzy6.clone(), fuzzy6.clone(), fuzzy6.clone()], + [fuzzy7_1.clone(), fuzzy7_2.clone(), fuzzy7_1.clone()], + [fuzzy8_1.clone(), fuzzy8_2.clone(), fuzzy8_1.clone()], + [fuzzy9_1.clone(), fuzzy9_2.clone(), fuzzy9_1.clone()], + [fuzzy10_1.clone(), fuzzy10_2.clone(), fuzzy10_1.clone()], + [fuzzy11_1.clone(), fuzzy11_2.clone(), fuzzy11_1.clone()], + [fuzzy12_1.clone(), fuzzy12_2.clone(), fuzzy12_1.clone()], + [fuzzy13_1.clone(), fuzzy13_2.clone(), fuzzy13_1.clone()], ) .with_fuzz2001(fuzzy1.clone()) .with_fuzz2003(fuzzy3.clone()) .with_fuzz2005(fuzzy5.clone()) .with_fuzz2102([fuzzy2.clone(), fuzzy2.clone(), fuzzy2.clone()]) .with_fuzz2104([fuzzy4.clone(), fuzzy4.clone(), fuzzy4.clone()]) - .with_fuzz2106([fuzzy6.clone(), fuzzy6.clone(), fuzzy6.clone()]); + .with_fuzz2106([fuzzy6.clone(), fuzzy6.clone(), fuzzy6.clone()]) + .with_fuzz2108([fuzzy8_1.clone(), fuzzy8_2.clone(), fuzzy8_1.clone()]) + .with_fuzz2110([fuzzy10_1.clone(), fuzzy10_2.clone(), fuzzy10_1.clone()]) + .with_fuzz2112([fuzzy12_1.clone(), fuzzy12_2.clone(), fuzzy12_1.clone()]); #[rustfmt::skip] let expected_extensions: HashMap<_, _> = [ From 63a3da66dd7ee0bf137108cee05a94d319cdc301 Mon Sep 17 00:00:00 2001 From: Antoine Beyeler Date: Wed, 5 Jul 2023 16:36:26 +0200 Subject: [PATCH 11/17] Minor changes + class_id type error fix --- crates/re_types/definitions/rerun/components/class_id.fbs | 2 +- crates/re_types/definitions/rerun/datatypes.fbs | 1 - crates/re_types/source_hash.txt | 2 +- rerun_py/rerun_sdk/rerun/_rerun2/components/class_id.py | 1 + rerun_py/rerun_sdk/rerun/_rerun2/log_any.py | 5 ++--- rerun_py/tests/unit/test_points2d.py | 3 --- 6 files changed, 5 insertions(+), 9 deletions(-) diff --git a/crates/re_types/definitions/rerun/components/class_id.fbs b/crates/re_types/definitions/rerun/components/class_id.fbs index d421a8e0d430..a864e27852cf 100644 --- a/crates/re_types/definitions/rerun/components/class_id.fbs +++ b/crates/re_types/definitions/rerun/components/class_id.fbs @@ -14,7 +14,7 @@ namespace rerun.components; struct ClassId ( "attr.arrow.transparent", "attr.python.aliases": "int", - "attr.python.array_aliases": "npt.NDArray[np.uint8], npt.NDArray[np.uint16], npt.NDArray[np.uint32], npt.NDArray[np.uint64]", + "attr.python.array_aliases": "int, npt.NDArray[np.uint8], npt.NDArray[np.uint16], npt.NDArray[np.uint32], npt.NDArray[np.uint64]", "attr.rerun.legacy_fqname": "rerun.class_id", "attr.rust.derive": "Copy, Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash", "attr.rust.tuple_struct", diff --git a/crates/re_types/definitions/rerun/datatypes.fbs b/crates/re_types/definitions/rerun/datatypes.fbs index 4b9ef8379b01..797d6f19f411 100644 --- a/crates/re_types/definitions/rerun/datatypes.fbs +++ b/crates/re_types/definitions/rerun/datatypes.fbs @@ -1,5 +1,4 @@ include "./datatypes/point2d.fbs"; include "./datatypes/vec2d.fbs"; - namespace rerun.datatypes; diff --git a/crates/re_types/source_hash.txt b/crates/re_types/source_hash.txt index e0a28faa88c3..1605847e6114 100644 --- a/crates/re_types/source_hash.txt +++ b/crates/re_types/source_hash.txt @@ -1,4 +1,4 @@ # This is a sha256 hash for all direct and indirect dependencies of this crate's build script. # It can be safely removed at anytime to force the build script to run again. # Check out build.rs to see how it's computed. -db9e1ce25f2c0b5e31b8fd5cb40adf0ff549811b7acdea660993f666731a648f \ No newline at end of file +463595fbba52247b68d3c25795ed90b9fdb9a92c0a0c665c55939a0236161506 \ No newline at end of file diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/class_id.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/class_id.py index f711e63e9085..7136f76dd49d 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/components/class_id.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/components/class_id.py @@ -36,6 +36,7 @@ def __int__(self) -> int: ClassIdArrayLike = Union[ ClassId, Sequence[ClassIdLike], + int, npt.NDArray[np.uint8], npt.NDArray[np.uint16], npt.NDArray[np.uint32], diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/log_any.py b/rerun_py/rerun_sdk/rerun/_rerun2/log_any.py index 998ccef9925b..48af40bb90cd 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/log_any.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/log_any.py @@ -74,11 +74,10 @@ def _extract_components(entity: Archetype) -> Iterable[tuple[NamedExtensionArray yield getattr(entity, fld.name), fld.metadata["component"] == "primary" -_MAX_U64 = 2**64 - 1 - - def _splat() -> InstanceKeyArray: """Helper to generate a splat InstanceKeyArray.""" + + _MAX_U64 = 2**64 - 1 return pa.array([_MAX_U64], type=InstanceKeyType().storage_type) # type: ignore[no-any-return] diff --git a/rerun_py/tests/unit/test_points2d.py b/rerun_py/tests/unit/test_points2d.py index e7cb01543f1b..f9b70348f566 100644 --- a/rerun_py/tests/unit/test_points2d.py +++ b/rerun_py/tests/unit/test_points2d.py @@ -5,9 +5,6 @@ import numpy as np import rerun as rr -# NOTE: uncomment these to get a better auto-completion experience... -# from rerun_sdk import rerun2 as rr -# from rerun_sdk.rerun2 import components as rr.ext # TODO(cmc): roundtrips (serialize in python, deserialize in rust) From 0d2a77c1d86b5436c8249b87475071fb149fcd0c Mon Sep 17 00:00:00 2001 From: Antoine Beyeler <49431240+abey79@users.noreply.github.com> Date: Wed, 5 Jul 2023 16:38:22 +0200 Subject: [PATCH 12/17] Update crates/re_types_builder/src/codegen/python.rs Co-authored-by: Clement Rey --- crates/re_types_builder/src/codegen/python.rs | 29 +++++++------------ 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/crates/re_types_builder/src/codegen/python.rs b/crates/re_types_builder/src/codegen/python.rs index 7b85db6ac60f..edebe9ceb189 100644 --- a/crates/re_types_builder/src/codegen/python.rs +++ b/crates/re_types_builder/src/codegen/python.rs @@ -33,15 +33,7 @@ trait PythonObjectExt { impl PythonObjectExt for Object { fn is_delegating_component(&self) -> bool { - if self.kind != ObjectKind::Component { - return false; - } - - if let Type::Object(_) = self.fields[0].typ { - return true; - } - - false + self.kind == ObjectKind::Component && matches!(self.fields[0].typ, Type::Object(_)) } fn is_non_delegating_component(&self) -> bool { @@ -49,16 +41,15 @@ impl PythonObjectExt for Object { } fn delegate_datatype<'a>(&self, objects: &'a Objects) -> Option<&'a Object> { - if !self.is_delegating_component() { - return None; - } - - let field = &self.fields[0]; - if let Type::Object(name) = &field.typ { - return Some(objects.get(name)); - } - - unreachable!() + self.is_delegating_component() + .then(|| { + if let Type::Object(name) = &self.fields[0].typ { + Some(objects.get(name)) + } else { + None + } + }) + .flatten() } } From 45033c7b0994595faf231da181812407d79863d9 Mon Sep 17 00:00:00 2001 From: Antoine Beyeler <49431240+abey79@users.noreply.github.com> Date: Wed, 5 Jul 2023 16:39:14 +0200 Subject: [PATCH 13/17] Update crates/re_types_builder/src/codegen/python.rs Co-authored-by: Clement Rey --- crates/re_types_builder/src/codegen/python.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/re_types_builder/src/codegen/python.rs b/crates/re_types_builder/src/codegen/python.rs index edebe9ceb189..d7299a54de29 100644 --- a/crates/re_types_builder/src/codegen/python.rs +++ b/crates/re_types_builder/src/codegen/python.rs @@ -69,9 +69,11 @@ impl PythonCodeGenerator { /// /// This is the hacky way. We extract all identifiers from `__init__.py` which contains, but don't /// start with, a underscore (`_`). -fn load_overrides(path_buf: &Path) -> HashSet { - let contents = std::fs::read_to_string(path_buf.join("_overrides").join("__init__.py")) - .expect("_overrides/__init__.py must exist"); +fn load_overrides(path: &Path) -> HashSet { + let path = path.join("_overrides").join("__init__.py"); + let contents = std::fs::read_to_string(&path) + .with_context(|| format!("couldn't load overrides module at {path:?}")) + .unwrap(); // extract words from contents contents From cb9507105b9a446561141dea3d863d6952284819 Mon Sep 17 00:00:00 2001 From: Antoine Beyeler Date: Wed, 5 Jul 2023 16:40:53 +0200 Subject: [PATCH 14/17] source_hash.txt --- crates/re_types/source_hash.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/re_types/source_hash.txt b/crates/re_types/source_hash.txt index 1605847e6114..d398b561733a 100644 --- a/crates/re_types/source_hash.txt +++ b/crates/re_types/source_hash.txt @@ -1,4 +1,4 @@ # This is a sha256 hash for all direct and indirect dependencies of this crate's build script. # It can be safely removed at anytime to force the build script to run again. # Check out build.rs to see how it's computed. -463595fbba52247b68d3c25795ed90b9fdb9a92c0a0c665c55939a0236161506 \ No newline at end of file +dec6b10b9e90c9c95b3e1436c1b53987573319efd121cc5e715c31466163b0bc \ No newline at end of file From bd32da12734ffca6185810f7bd533a4e14882d50 Mon Sep 17 00:00:00 2001 From: Antoine Beyeler Date: Wed, 5 Jul 2023 16:58:37 +0200 Subject: [PATCH 15/17] Added docs to BaseExtensionArray --- .../rerun_sdk/rerun/_rerun2/_baseclasses.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/_baseclasses.py b/rerun_py/rerun_sdk/rerun/_rerun2/_baseclasses.py index c17fb55c0915..1d73b3686684 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/_baseclasses.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/_baseclasses.py @@ -68,6 +68,24 @@ class BaseExtensionArray(NamedExtensionArray, Generic[T]): # type: ignore[misc] @classmethod def from_similar(cls, data: T | None) -> BaseExtensionArray[T]: + """ + Primary method for creating Arrow arrays for components. + + This method must flexibly accept native data (which comply with type `T`). Subclasses must provide a type + parameter specifying the type of the native data (this is automatically handled by the code generator). + + The actual creation of the Arrow array is delegated to the `_native_to_pa_array()` method, which is not + implemented by default. + + Parameters + ---------- + data : T | None + The data to convert into an Arrow array. + + Returns + ------- + The Arrow array encapsulating the data. + """ data_type = cls._EXTENSION_TYPE() if data is None: @@ -79,6 +97,34 @@ def from_similar(cls, data: T | None) -> BaseExtensionArray[T]: @staticmethod def _native_to_pa_array(data: T, data_type: pa.DataType) -> pa.Array: + """ + Converts native data into an Arrow array. + + Subclasses must provide an implementation of this method (via an override) if they are to be used as either + an archetype's field (which should be the case for all components), or a (delegating) component's field (for + datatypes). Datatypes which are used only within other datatypes may omit implementing this method, provided + that the top-level datatype implements it. + + A hand-coded override must be provided for the code generator to implement this method. The override must be + named `xxx_native_to_pa_array()`, where `xxx` is the lowercase name of the datatype. The override must be + located in the `_overrides` subpackage and *explicitly* imported by `_overrides/__init__.py` (to be noticed + by the code generator). + + `color_native_to_pa_array()` in `_overrides/color.py` is a good example of how to implement this method, in + conjunction with the native type's converter (see `color_converter()`, used to construct the native `Color` + object). + + Parameters + ---------- + data : T + The data to convert into an Arrow array. + data_type : pa.DataType + The Arrow data type of the data. + + Returns + ------- + The Arrow array encapsulating the data. + """ raise NotImplementedError From 6ca16645d68497bc117bc94208ab6ba24faa5ebd Mon Sep 17 00:00:00 2001 From: Antoine Beyeler Date: Wed, 5 Jul 2023 17:02:09 +0200 Subject: [PATCH 16/17] lint fix --- rerun_py/tests/unit/test_points2d.py | 1 - 1 file changed, 1 deletion(-) diff --git a/rerun_py/tests/unit/test_points2d.py b/rerun_py/tests/unit/test_points2d.py index f9b70348f566..86cfe99494d8 100644 --- a/rerun_py/tests/unit/test_points2d.py +++ b/rerun_py/tests/unit/test_points2d.py @@ -5,7 +5,6 @@ import numpy as np import rerun as rr - # TODO(cmc): roundtrips (serialize in python, deserialize in rust) U64_MAX_MINUS_1 = 2**64 - 2 From a14e37a0a6851498c5241c116da5923ba3439dcb Mon Sep 17 00:00:00 2001 From: Antoine Beyeler Date: Wed, 5 Jul 2023 17:18:51 +0200 Subject: [PATCH 17/17] Changed imports in test_points2d.py --- rerun_py/tests/unit/test_points2d.py | 52 +++++++++++++++------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/rerun_py/tests/unit/test_points2d.py b/rerun_py/tests/unit/test_points2d.py index 86cfe99494d8..0a1eebec403c 100644 --- a/rerun_py/tests/unit/test_points2d.py +++ b/rerun_py/tests/unit/test_points2d.py @@ -4,6 +4,8 @@ import numpy as np import rerun as rr +from rerun import cmp as rrc +from rerun import dt as rrd # TODO(cmc): roundtrips (serialize in python, deserialize in rust) @@ -12,13 +14,13 @@ def test_points2d() -> None: - points_arrays: list[rr.dt.Point2DArrayLike] = [ + points_arrays: list[rrd.Point2DArrayLike] = [ [], np.array([]), # Point2DArrayLike: Sequence[Point2DLike]: Point2D [ - rr.dt.Point2D(1, 2), - rr.dt.Point2D(3, 4), + rrd.Point2D(1, 2), + rrd.Point2D(3, 4), ], # Point2DArrayLike: Sequence[Point2DLike]: npt.NDArray[np.float32] [ @@ -35,7 +37,7 @@ def test_points2d() -> None: np.array([1, 2, 3, 4], dtype=np.float32), ] - radii_arrays: list[rr.cmp.RadiusArrayLike | None] = [ + radii_arrays: list[rrc.RadiusArrayLike | None] = [ None, [], np.array([]), @@ -43,14 +45,14 @@ def test_points2d() -> None: [42, 43], # RadiusArrayLike: Sequence[RadiusLike]: Radius [ - rr.cmp.Radius(42), - rr.cmp.Radius(43), + rrc.Radius(42), + rrc.Radius(43), ], # RadiusArrayLike: npt.NDArray[np.float32] np.array([42, 43], dtype=np.float32), ] - colors_arrays: list[rr.cmp.ColorArrayLike | None] = [ + colors_arrays: list[rrc.ColorArrayLike | None] = [ None, [], np.array([]), @@ -61,8 +63,8 @@ def test_points2d() -> None: ], # ColorArrayLike: Sequence[ColorLike]: Color [ - rr.cmp.Color(0xAA0000CC), - rr.cmp.Color(0x00BB00DD), + rrc.Color(0xAA0000CC), + rrc.Color(0x00BB00DD), ], # ColorArrayLike: Sequence[ColorLike]: npt.NDArray[np.uint8] np.array( @@ -148,24 +150,24 @@ def test_points2d() -> None: ), ] - labels_arrays: list[rr.cmp.LabelArrayLike | None] = [ + labels_arrays: list[rrc.LabelArrayLike | None] = [ None, [], # LabelArrayLike: Sequence[LabelLike]: str ["hello", "friend"], # LabelArrayLike: Sequence[LabelLike]: Label [ - rr.cmp.Label("hello"), - rr.cmp.Label("friend"), + rrc.Label("hello"), + rrc.Label("friend"), ], ] - draw_orders: list[rr.cmp.DrawOrderLike | None] = [ + draw_orders: list[rrc.DrawOrderLike | None] = [ None, # DrawOrderLike: float 300, # DrawOrderLike: DrawOrder - rr.cmp.DrawOrder(300), + rrc.DrawOrder(300), ] class_id_arrays = [ @@ -174,7 +176,7 @@ def test_points2d() -> None: # ClassIdArrayLike: Sequence[ClassIdLike]: int [126, 127], # ClassIdArrayLike: Sequence[ClassIdLike]: ClassId - [rr.cmp.ClassId(126), rr.cmp.ClassId(127)], + [rrc.ClassId(126), rrc.ClassId(127)], # ClassIdArrayLike: np.NDArray[np.uint8] np.array([126, 127], dtype=np.uint8), # ClassIdArrayLike: np.NDArray[np.uint16] @@ -191,7 +193,7 @@ def test_points2d() -> None: # KeypointIdArrayLike: Sequence[KeypointIdLike]: int [2, 3], # KeypointIdArrayLike: Sequence[KeypointIdLike]: KeypointId - [rr.cmp.KeypointId(2), rr.cmp.KeypointId(3)], + [rrc.KeypointId(2), rrc.KeypointId(3)], # KeypointIdArrayLike: np.NDArray[np.uint8] np.array([2, 3], dtype=np.uint8), # KeypointIdArrayLike: np.NDArray[np.uint16] @@ -208,7 +210,7 @@ def test_points2d() -> None: # InstanceKeyArrayLike: Sequence[InstanceKeyLike]: int [U64_MAX_MINUS_1, U64_MAX], # InstanceKeyArrayLike: Sequence[InstanceKeyLike]: InstanceKey - [rr.cmp.InstanceKey(U64_MAX_MINUS_1), rr.cmp.InstanceKey(U64_MAX)], + [rrc.InstanceKey(U64_MAX_MINUS_1), rrc.InstanceKey(U64_MAX)], # InstanceKeyArrayLike: np.NDArray[np.uint64] np.array([U64_MAX_MINUS_1, U64_MAX], dtype=np.uint64), ] @@ -251,14 +253,14 @@ def test_points2d() -> None: ) print(f"{arch}\n") - assert arch.points == rr.cmp.Point2DArray.from_similar([[1.0, 2.0], [3.0, 4.0]] if non_empty(points) else []) - assert arch.radii == rr.cmp.RadiusArray.from_similar([42, 43] if non_empty(radii) else []) - assert arch.colors == rr.cmp.ColorArray.from_similar([0xAA0000CC, 0x00BB00DD] if non_empty(colors) else []) - assert arch.labels == rr.cmp.LabelArray.from_similar(["hello", "friend"] if non_empty(labels) else []) - assert arch.draw_order == rr.cmp.DrawOrderArray.from_similar([300] if draw_order is not None else []) - assert arch.class_ids == rr.cmp.ClassIdArray.from_similar([126, 127] if non_empty(class_ids) else []) - assert arch.keypoint_ids == rr.cmp.KeypointIdArray.from_similar([2, 3] if non_empty(keypoint_ids) else []) - assert arch.instance_keys == rr.cmp.InstanceKeyArray.from_similar( + assert arch.points == rrc.Point2DArray.from_similar([[1.0, 2.0], [3.0, 4.0]] if non_empty(points) else []) + assert arch.radii == rrc.RadiusArray.from_similar([42, 43] if non_empty(radii) else []) + assert arch.colors == rrc.ColorArray.from_similar([0xAA0000CC, 0x00BB00DD] if non_empty(colors) else []) + assert arch.labels == rrc.LabelArray.from_similar(["hello", "friend"] if non_empty(labels) else []) + assert arch.draw_order == rrc.DrawOrderArray.from_similar([300] if draw_order is not None else []) + assert arch.class_ids == rrc.ClassIdArray.from_similar([126, 127] if non_empty(class_ids) else []) + assert arch.keypoint_ids == rrc.KeypointIdArray.from_similar([2, 3] if non_empty(keypoint_ids) else []) + assert arch.instance_keys == rrc.InstanceKeyArray.from_similar( [U64_MAX_MINUS_1, U64_MAX] if non_empty(instance_keys) else [] )