Feature/make properties typed (#1266)

* take property insertion apart and put it back together again

* fix tests that were testing broken behaviour

* remove `"_id"` from properties and change stray ints to floats in python tests

* fix warnings

* String deduplication (#1269)

* eliminate a lot of arc clones

* replace String by ArcStr (wrapped Arc<str>) for cheap clone and to make it possible to support string deduplication

* test string deduplication

* implement string deduplication for property values

* clean up warnings

* expose meta data in core ops and minor cleanup

* fix rebase issues and clean up warnings

* dubious warning fix

* attribute does not work, warning is still there

* simplify edge addition and deletion

* No more spin-locking for adding edges (instead get locks in consistent order)

examples/custom_python_extension/src/lib.rs

@@ -11,7 +11,7 @@ fn py_custom_algorithm(graph: DynamicGraph) -> usize {
 }
 
 #[pymodule]
-fn custom_python_extension(py: Python<'_>, m: &PyModule) -> PyResult<()> {
+fn custom_python_extension(_py: Python<'_>, m: &PyModule) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(py_custom_algorithm, m)?)?;
     Ok(())
 }

examples/rust/src/bin/lotr/main.rs

@@ -59,25 +59,22 @@ fn main() {
                 g.add_vertex(
                     lotr.time,
                     lotr.src_id.clone(),
-                    [("type".to_string(), Prop::Str("Character".to_string()))],
+                    [("type", Prop::str("Character"))],
                 )
                 .expect("Failed to add vertex");
 
                 g.add_vertex(
                     lotr.time,
                     lotr.dst_id.clone(),
-                    [("type".to_string(), Prop::Str("Character".to_string()))],
+                    [("type", Prop::str("Character"))],
                 )
                 .expect("Failed to add vertex");
 
                 g.add_edge(
                     lotr.time,
                     lotr.src_id.clone(),
                     lotr.dst_id.clone(),
-                    [(
-                        "type".to_string(),
-                        Prop::Str("Character Co-occurrence".to_string()),
-                    )],
+                    [("type", Prop::str("Character Co-occurrence"))],
                     None,
                 )
                 .expect("Failed to add edge");

js-raphtory/src/graph/edge.rs

@@ -4,7 +4,7 @@ use raphtory::db::{
     api::view::*,
     graph::{edge::EdgeView, graph::Graph as TGraph},
 };
-use std::{ops::Deref, sync::Arc};
+use std::sync::Arc;
 use wasm_bindgen::prelude::*;
 
 #[wasm_bindgen]
@@ -39,7 +39,7 @@ impl Edge {
         let t_props = self.0.properties();
         let obj = js_sys::Map::new();
         for (k, v) in t_props.iter() {
-            obj.set(&k.deref().into(), &JsProp(v).into());
+            obj.set(&k.to_string().into(), &JsProp(v).into());
         }
         obj
     }

js-raphtory/src/graph/misc.rs

@@ -21,7 +21,7 @@ impl From<JsProp> for JsValue {
         match value.0 {
             raphtory::core::Prop::U8(v) => v.into(),
             raphtory::core::Prop::U16(v) => v.into(),
-            raphtory::core::Prop::Str(v) => v.into(),
+            raphtory::core::Prop::Str(v) => v.to_string().into(),
             raphtory::core::Prop::I32(v) => v.into(),
             raphtory::core::Prop::I64(v) => v.into(),
             raphtory::core::Prop::U32(v) => v.into(),
@@ -68,6 +68,6 @@ impl From<JsObjectEntry> for Option<(String, Prop)> {
 
         let key = arr.at(0).as_string().unwrap();
         let value = arr.at(1).as_string().unwrap();
-        Some((key, Prop::Str(value)))
+        Some((key, Prop::str(value)))
     }
 }

js-raphtory/src/graph/mod.rs

@@ -99,7 +99,7 @@ impl Graph {
     #[wasm_bindgen(js_name = addVertex)]
     pub fn add_vertex_js(&self, t: i64, id: JsValue, js_props: Object) -> Result<Vertex, JSError> {
         let rust_props = if js_props.is_string() {
-            vec![("name".to_string(), Prop::Str(js_props.as_string().unwrap()))]
+            vec![("name".to_string(), Prop::str(js_props.as_string().unwrap()))]
         } else if js_props.is_object() {
             Object::entries(&js_props)
                 .iter()

js-raphtory/src/graph/vertex.rs

@@ -100,7 +100,7 @@ impl Vertex {
     pub fn properties(&self) -> js_sys::Map {
         let obj = js_sys::Map::new();
         for (k, v) in self.0.properties() {
-            obj.set(&k.into(), &JsProp(v).into());
+            obj.set(&k.to_string().into(), &JsProp(v).into());
         }
         obj
     }

python/src/graphql.rs

@@ -1,6 +1,6 @@
 use pyo3::{exceptions, prelude::*};
 use raphtory_core::{
-    db::api::view::internal::{DynamicGraph, MaterializedGraph},
+    db::api::view::internal::MaterializedGraph,
     prelude::Graph,
     python::{graph::graph::PyGraph, utils::errors::adapt_err_value},
 };

python/src/lib.rs

@@ -12,7 +12,7 @@ use raphtory_core::python::{
         properties::{PyConstProperties, PyProperties, PyTemporalProp, PyTemporalProperties},
         vertex::{PyVertex, PyVertices},
     },
-    packages::{algorithms::*, graph_gen::*, graph_loader::*, vectors::PyVectorizedGraph},
+    packages::{algorithms::*, graph_gen::*, graph_loader::*},
 };
 
 /// Raphtory graph analytics library

python/tests/expected/dataframe_output/vertex_df_all.json

@@ -1 +1 @@
-{"id":{"0":"ServerA","1":"ServerB","2":"ServerC","3":"ServerD","4":"ServerE"},"properties":{"0":{"server_name":"Alpha","hardware_type":"Blade Server","datasource":"data\/network_traffic_edges.csv","_id":"ServerA","primary_function":[[1693555200000,"Database"],[1693555260000,"Database"],[1693555320000,"Database"]],"uptime_days":[[1693555200000,120],[1693555260000,121],[1693555320000,122]],"OS_version":[[1693555200000,"Ubuntu 20.04"],[1693555260000,"Ubuntu 20.04"],[1693555320000,"Ubuntu 20.04"]]},"1":{"_id":"ServerB","datasource":"data\/network_traffic_edges.csv","hardware_type":"Rack Server","server_name":"Beta","uptime_days":[[1693555500000,45]],"OS_version":[[1693555500000,"Red Hat 8.1"]],"primary_function":[[1693555500000,"Web Server"]]},"2":{"_id":"ServerC","server_name":"Charlie","datasource":"data\/network_traffic_edges.csv","hardware_type":"Blade Server","OS_version":[[1693555800000,"Windows Server 2022"]],"primary_function":[[1693555800000,"File Storage"]],"uptime_days":[[1693555800000,90]]},"3":{"_id":"ServerD","datasource":"data\/network_traffic_edges.csv","server_name":"Delta","hardware_type":"Tower Server","uptime_days":[[1693556100000,60]],"OS_version":[[1693556100000,"Ubuntu 20.04"]],"primary_function":[[1693556100000,"Application Server"]]},"4":{"server_name":"Echo","hardware_type":"Rack Server","datasource":"data\/network_traffic_edges.csv","_id":"ServerE","primary_function":[[1693556400000,"Backup"]],"OS_version":[[1693556400000,"Red Hat 8.1"]],"uptime_days":[[1693556400000,30]]}},"update_history":{"0":[1693555200000,1693555260000,1693555320000,1693555500000,1693556400000],"1":[1693555200000,1693555500000,1693555800000,1693556700000],"2":[1693555500000,1693555800000,1693556400000,1693557000000,1693557060000,1693557120000],"3":[1693555800000,1693556100000,1693557000000,1693557060000,1693557120000],"4":[1693556100000,1693556400000,1693556700000]}}
+{"id":{"0":"ServerA","1":"ServerB","2":"ServerC","3":"ServerD","4":"ServerE"},"properties":{"0":{"server_name":"Alpha","hardware_type":"Blade Server","datasource":"data\/network_traffic_edges.csv","primary_function":[[1693555200000,"Database"],[1693555260000,"Database"],[1693555320000,"Database"]],"uptime_days":[[1693555200000,120],[1693555260000,121],[1693555320000,122]],"OS_version":[[1693555200000,"Ubuntu 20.04"],[1693555260000,"Ubuntu 20.04"],[1693555320000,"Ubuntu 20.04"]]},"1":{"datasource":"data\/network_traffic_edges.csv","hardware_type":"Rack Server","server_name":"Beta","uptime_days":[[1693555500000,45]],"OS_version":[[1693555500000,"Red Hat 8.1"]],"primary_function":[[1693555500000,"Web Server"]]},"2":{"server_name":"Charlie","datasource":"data\/network_traffic_edges.csv","hardware_type":"Blade Server","OS_version":[[1693555800000,"Windows Server 2022"]],"primary_function":[[1693555800000,"File Storage"]],"uptime_days":[[1693555800000,90]]},"3":{"datasource":"data\/network_traffic_edges.csv","server_name":"Delta","hardware_type":"Tower Server","uptime_days":[[1693556100000,60]],"OS_version":[[1693556100000,"Ubuntu 20.04"]],"primary_function":[[1693556100000,"Application Server"]]},"4":{"server_name":"Echo","hardware_type":"Rack Server","datasource":"data\/network_traffic_edges.csv","primary_function":[[1693556400000,"Backup"]],"OS_version":[[1693556400000,"Red Hat 8.1"]],"uptime_days":[[1693556400000,30]]}},"update_history":{"0":[1693555200000,1693555260000,1693555320000,1693555500000,1693556400000],"1":[1693555200000,1693555500000,1693555800000,1693556700000],"2":[1693555500000,1693555800000,1693556400000,1693557000000,1693557060000,1693557120000],"3":[1693555800000,1693556100000,1693557000000,1693557060000,1693557120000],"4":[1693556100000,1693556400000,1693556700000]}}

python/tests/expected/dataframe_output/vertex_df_no_hist.json

@@ -1 +1 @@
-{"id":{"0":"ServerA","1":"ServerB","2":"ServerC","3":"ServerD","4":"ServerE"},"properties":{"0":{"datasource":"data\/network_traffic_edges.csv","server_name":"Alpha","hardware_type":"Blade Server","_id":"ServerA","OS_version":[[1693555200000,"Ubuntu 20.04"],[1693555260000,"Ubuntu 20.04"],[1693555320000,"Ubuntu 20.04"]],"uptime_days":[[1693555200000,120],[1693555260000,121],[1693555320000,122]],"primary_function":[[1693555200000,"Database"],[1693555260000,"Database"],[1693555320000,"Database"]]},"1":{"datasource":"data\/network_traffic_edges.csv","_id":"ServerB","hardware_type":"Rack Server","server_name":"Beta","primary_function":[[1693555500000,"Web Server"]],"uptime_days":[[1693555500000,45]],"OS_version":[[1693555500000,"Red Hat 8.1"]]},"2":{"hardware_type":"Blade Server","_id":"ServerC","server_name":"Charlie","datasource":"data\/network_traffic_edges.csv","uptime_days":[[1693555800000,90]],"OS_version":[[1693555800000,"Windows Server 2022"]],"primary_function":[[1693555800000,"File Storage"]]},"3":{"server_name":"Delta","hardware_type":"Tower Server","datasource":"data\/network_traffic_edges.csv","_id":"ServerD","uptime_days":[[1693556100000,60]],"OS_version":[[1693556100000,"Ubuntu 20.04"]],"primary_function":[[1693556100000,"Application Server"]]},"4":{"datasource":"data\/network_traffic_edges.csv","hardware_type":"Rack Server","server_name":"Echo","_id":"ServerE","uptime_days":[[1693556400000,30]],"OS_version":[[1693556400000,"Red Hat 8.1"]],"primary_function":[[1693556400000,"Backup"]]}}}
+{"id":{"0":"ServerA","1":"ServerB","2":"ServerC","3":"ServerD","4":"ServerE"},"properties":{"0":{"datasource":"data\/network_traffic_edges.csv","server_name":"Alpha","hardware_type":"Blade Server","OS_version":[[1693555200000,"Ubuntu 20.04"],[1693555260000,"Ubuntu 20.04"],[1693555320000,"Ubuntu 20.04"]],"uptime_days":[[1693555200000,120],[1693555260000,121],[1693555320000,122]],"primary_function":[[1693555200000,"Database"],[1693555260000,"Database"],[1693555320000,"Database"]]},"1":{"datasource":"data\/network_traffic_edges.csv","hardware_type":"Rack Server","server_name":"Beta","primary_function":[[1693555500000,"Web Server"]],"uptime_days":[[1693555500000,45]],"OS_version":[[1693555500000,"Red Hat 8.1"]]},"2":{"hardware_type":"Blade Server","server_name":"Charlie","datasource":"data\/network_traffic_edges.csv","uptime_days":[[1693555800000,90]],"OS_version":[[1693555800000,"Windows Server 2022"]],"primary_function":[[1693555800000,"File Storage"]]},"3":{"server_name":"Delta","hardware_type":"Tower Server","datasource":"data\/network_traffic_edges.csv","uptime_days":[[1693556100000,60]],"OS_version":[[1693556100000,"Ubuntu 20.04"]],"primary_function":[[1693556100000,"Application Server"]]},"4":{"datasource":"data\/network_traffic_edges.csv","hardware_type":"Rack Server","server_name":"Echo","uptime_days":[[1693556400000,30]],"OS_version":[[1693556400000,"Red Hat 8.1"]],"primary_function":[[1693556400000,"Backup"]]}}}

python/tests/expected/dataframe_output/vertex_df_no_prop_hist.json

@@ -1 +1 @@
-{"id":{"0":"ServerA","1":"ServerB","2":"ServerC","3":"ServerD","4":"ServerE"},"properties":{"0":{"_id":"ServerA","server_name":"Alpha","datasource":"data\/network_traffic_edges.csv","hardware_type":"Blade Server","uptime_days":122,"OS_version":"Ubuntu 20.04","primary_function":"Database"},"1":{"primary_function":"Web Server","_id":"ServerB","server_name":"Beta","OS_version":"Red Hat 8.1","hardware_type":"Rack Server","datasource":"data\/network_traffic_edges.csv","uptime_days":45},"2":{"primary_function":"File Storage","server_name":"Charlie","datasource":"data\/network_traffic_edges.csv","hardware_type":"Blade Server","OS_version":"Windows Server 2022","_id":"ServerC","uptime_days":90},"3":{"primary_function":"Application Server","OS_version":"Ubuntu 20.04","_id":"ServerD","server_name":"Delta","datasource":"data\/network_traffic_edges.csv","uptime_days":60,"hardware_type":"Tower Server"},"4":{"primary_function":"Backup","hardware_type":"Rack Server","_id":"ServerE","OS_version":"Red Hat 8.1","datasource":"data\/network_traffic_edges.csv","uptime_days":30,"server_name":"Echo"}},"update_history":{"0":[1693555200000,1693555260000,1693555320000,1693555500000,1693556400000],"1":[1693555200000,1693555500000,1693555800000,1693556700000],"2":[1693555500000,1693555800000,1693556400000,1693557000000,1693557060000,1693557120000],"3":[1693555800000,1693556100000,1693557000000,1693557060000,1693557120000],"4":[1693556100000,1693556400000,1693556700000]}}
+{"id":{"0":"ServerA","1":"ServerB","2":"ServerC","3":"ServerD","4":"ServerE"},"properties":{"0":{"server_name":"Alpha","datasource":"data\/network_traffic_edges.csv","hardware_type":"Blade Server","uptime_days":122,"OS_version":"Ubuntu 20.04","primary_function":"Database"},"1":{"primary_function":"Web Server","server_name":"Beta","OS_version":"Red Hat 8.1","hardware_type":"Rack Server","datasource":"data\/network_traffic_edges.csv","uptime_days":45},"2":{"primary_function":"File Storage","server_name":"Charlie","datasource":"data\/network_traffic_edges.csv","hardware_type":"Blade Server","OS_version":"Windows Server 2022","uptime_days":90},"3":{"primary_function":"Application Server","OS_version":"Ubuntu 20.04","server_name":"Delta","datasource":"data\/network_traffic_edges.csv","uptime_days":60,"hardware_type":"Tower Server"},"4":{"primary_function":"Backup","hardware_type":"Rack Server","OS_version":"Red Hat 8.1","datasource":"data\/network_traffic_edges.csv","uptime_days":30,"server_name":"Echo"}},"update_history":{"0":[1693555200000,1693555260000,1693555320000,1693555500000,1693556400000],"1":[1693555200000,1693555500000,1693555800000,1693556700000],"2":[1693555500000,1693555800000,1693556400000,1693557000000,1693557060000,1693557120000],"3":[1693555800000,1693556100000,1693557000000,1693557060000,1693557120000],"4":[1693556100000,1693556400000,1693556700000]}}

python/tests/test_graph_conversions.py

@@ -161,7 +161,6 @@ def test_networkx_full_history():
                     (1693555260000, "Ubuntu 20.04"),
                     (1693555320000, "Ubuntu 20.04"),
                 ],
-                "_id": "ServerA",
                 "datasource": "data/network_traffic_edges.csv",
                 "hardware_type": "Blade Server",
                 "primary_function": [
@@ -188,7 +187,6 @@ def test_networkx_full_history():
             "ServerB",
             {
                 "OS_version": [(1693555500000, "Red Hat 8.1")],
-                "_id": "ServerB",
                 "datasource": "data/network_traffic_edges.csv",
                 "hardware_type": "Rack Server",
                 "primary_function": [(1693555500000, "Web Server")],
@@ -206,7 +204,6 @@ def test_networkx_full_history():
             "ServerC",
             {
                 "OS_version": [(1693555800000, "Windows Server 2022")],
-                "_id": "ServerC",
                 "datasource": "data/network_traffic_edges.csv",
                 "hardware_type": "Blade Server",
                 "primary_function": [(1693555800000, "File Storage")],
@@ -226,7 +223,6 @@ def test_networkx_full_history():
             "ServerD",
             {
                 "OS_version": [(1693556100000, "Ubuntu 20.04")],
-                "_id": "ServerD",
                 "datasource": "data/network_traffic_edges.csv",
                 "hardware_type": "Tower Server",
                 "primary_function": [(1693556100000, "Application Server")],
@@ -245,7 +241,6 @@ def test_networkx_full_history():
             "ServerE",
             {
                 "OS_version": [(1693556400000, "Red Hat 8.1")],
-                "_id": "ServerE",
                 "datasource": "data/network_traffic_edges.csv",
                 "hardware_type": "Rack Server",
                 "primary_function": [(1693556400000, "Backup")],
@@ -656,7 +651,6 @@ def test_networkx_no_history():
             "ServerA",
             {
                 "OS_version": "Ubuntu 20.04",
-                "_id": "ServerA",
                 "datasource": "data/network_traffic_edges.csv",
                 "hardware_type": "Blade Server",
                 "primary_function": "Database",
@@ -668,7 +662,6 @@ def test_networkx_no_history():
             "ServerB",
             {
                 "OS_version": "Red Hat 8.1",
-                "_id": "ServerB",
                 "datasource": "data/network_traffic_edges.csv",
                 "hardware_type": "Rack Server",
                 "primary_function": "Web Server",
@@ -680,7 +673,6 @@ def test_networkx_no_history():
             "ServerC",
             {
                 "OS_version": "Windows Server 2022",
-                "_id": "ServerC",
                 "datasource": "data/network_traffic_edges.csv",
                 "hardware_type": "Blade Server",
                 "primary_function": "File Storage",
@@ -692,7 +684,6 @@ def test_networkx_no_history():
             "ServerD",
             {
                 "OS_version": "Ubuntu 20.04",
-                "_id": "ServerD",
                 "datasource": "data/network_traffic_edges.csv",
                 "hardware_type": "Tower Server",
                 "primary_function": "Application Server",
@@ -704,7 +695,6 @@ def test_networkx_no_history():
             "ServerE",
             {
                 "OS_version": "Red Hat 8.1",
-                "_id": "ServerE",
                 "datasource": "data/network_traffic_edges.csv",
                 "hardware_type": "Rack Server",
                 "primary_function": "Backup",

python/tests/test_graphdb.py

@@ -19,7 +19,7 @@ def create_graph():
 
     g.add_vertex(0, 1, {"type": "wallet", "cost": 99.5})
     g.add_vertex(-1, 2, {"type": "wallet", "cost": 10.0})
-    g.add_vertex(6, 3, {"type": "wallet", "cost": 76})
+    g.add_vertex(6, 3, {"type": "wallet", "cost": 76.0})
 
     for e in edges:
         g.add_edge(e[0], e[1], e[2], {"prop1": 1, "prop2": 9.8, "prop3": "test"})
@@ -32,7 +32,7 @@ def create_graph_with_deletions():
 
     g.add_vertex(0, 1, {"type": "wallet", "cost": 99.5})
     g.add_vertex(-1, 2, {"type": "wallet", "cost": 10.0})
-    g.add_vertex(6, 3, {"type": "wallet", "cost": 76})
+    g.add_vertex(6, 3, {"type": "wallet", "cost": 76.0})
 
     for e in edges:
         g.add_edge(e[0], e[1], e[2], {"prop1": 1, "prop2": 9.8, "prop3": "test"})
@@ -832,7 +832,7 @@ def test_save_load_graph():
     g = create_graph()
     g.add_vertex(1, 11, {"type": "wallet", "balance": 99.5})
     g.add_vertex(2, 12, {"type": "wallet", "balance": 10.0})
-    g.add_vertex(3, 13, {"type": "wallet", "balance": 76})
+    g.add_vertex(3, 13, {"type": "wallet", "balance": 76.0})
     g.add_edge(4, 11, 12, {"prop1": 1, "prop2": 9.8, "prop3": "test"})
     g.add_edge(5, 12, 13, {"prop1": 1321, "prop2": 9.8, "prop3": "test"})
     g.add_edge(6, 13, 11, {"prop1": 645, "prop2": 9.8, "prop3": "test"})
@@ -1570,7 +1570,7 @@ def test_materialize_graph():
 
     g.add_vertex(0, 1, {"type": "wallet", "cost": 99.5})
     g.add_vertex(-1, 2, {"type": "wallet", "cost": 10.0})
-    g.add_vertex(6, 3, {"type": "wallet", "cost": 76})
+    g.add_vertex(6, 3, {"type": "wallet", "cost": 76.0})
     g.add_vertex(6, 4).add_constant_properties({"abc": "xyz"})
 
     for e in edges:

raphtory-graphql/src/data.rs

@@ -88,7 +88,7 @@ impl Data {
                 println!("loading graph from {path}");
                 let graph = Graph::load_from_file(&path).expect("Unable to load from graph");
                 graph
-                    .add_constant_properties([("path".to_string(), Prop::Str(path.clone()))])
+                    .add_constant_properties([("path".to_string(), Prop::str(path.clone()))])
                     .expect("Failed to add static property");
                 let maybe_graph_name = graph.properties().get("name");