From 13fcdf468eb3707e973f0fdc3f952d51bbcb0194 Mon Sep 17 00:00:00 2001 From: Fabian Murariu Date: Thu, 10 Oct 2024 10:35:10 +0000 Subject: [PATCH 1/2] add support to exclude edge temp properties on import --- pometry-storage-private | 2 +- raphtory-cypher/examples/raphtory_cypher.rs | 7 +++++++ raphtory-cypher/src/lib.rs | 2 ++ raphtory/src/disk_graph/graph_impl/mod.rs | 3 +++ raphtory/src/disk_graph/mod.rs | 5 +++-- raphtory/src/python/graph/disk_graph.rs | 4 ++++ 6 files changed, 20 insertions(+), 3 deletions(-) diff --git a/pometry-storage-private b/pometry-storage-private index 6b90b9179b..8a68cd7c44 160000 --- a/pometry-storage-private +++ b/pometry-storage-private @@ -1 +1 @@ -Subproject commit 6b90b9179b265a4df4db352b18f5a9105b71b4e6 +Subproject commit 8a68cd7c44b599b93446e194ca87c9d2d910a5b0 diff --git a/raphtory-cypher/examples/raphtory_cypher.rs b/raphtory-cypher/examples/raphtory_cypher.rs index 8359add122..f0c1084a28 100644 --- a/raphtory-cypher/examples/raphtory_cypher.rs +++ b/raphtory-cypher/examples/raphtory_cypher.rs @@ -116,6 +116,7 @@ mod cypher { src_col: String, dst_col: String, time_col: String, + exclude: Option>, } impl std::str::FromStr for ArgLayer { @@ -182,6 +183,7 @@ mod cypher { src_col, dst_col, time_col, + exclude: exclude_edge_props, }, ) = &layers[layer_id]; ParquetLayerCols { @@ -190,6 +192,11 @@ mod cypher { src_col: &src_col, dst_col: &dst_col, time_col: &time_col, + exclude_edge_props: exclude_edge_props + .into_iter() + .flatten() + .map(|s| s.as_str()) + .collect(), } }) .collect(); diff --git a/raphtory-cypher/src/lib.rs b/raphtory-cypher/src/lib.rs index 93dbaa36ac..e26831b1e0 100644 --- a/raphtory-cypher/src/lib.rs +++ b/raphtory-cypher/src/lib.rs @@ -337,6 +337,7 @@ mod cypher { src_col: "source", dst_col: "destination", time_col: "time", + exclude_edge_props: vec!["destination_port"], }, ParquetLayerCols { parquet_dir: v1_layer_path.to_str().unwrap(), @@ -344,6 +345,7 @@ mod cypher { src_col: "src", dst_col: "dst", time_col: "epoch_time", + exclude_edge_props: vec![] }, ]; diff --git a/raphtory/src/disk_graph/graph_impl/mod.rs b/raphtory/src/disk_graph/graph_impl/mod.rs index 2bcaef235d..2ce2320b73 100644 --- a/raphtory/src/disk_graph/graph_impl/mod.rs +++ b/raphtory/src/disk_graph/graph_impl/mod.rs @@ -15,6 +15,7 @@ pub struct ParquetLayerCols<'a> { pub src_col: &'a str, pub dst_col: &'a str, pub time_col: &'a str, + pub exclude_edge_props: Vec<&'a str>, } impl Graph { @@ -401,6 +402,7 @@ mod test { src_col: "source", dst_col: "destination", time_col: "time", + exclude_edge_props: vec![], }, ParquetLayerCols { parquet_dir: v1_layer_path.to_str().unwrap(), @@ -408,6 +410,7 @@ mod test { src_col: "src", dst_col: "dst", time_col: "Time", + exclude_edge_props: vec![], }, ]; diff --git a/raphtory/src/disk_graph/mod.rs b/raphtory/src/disk_graph/mod.rs index 65cecd6498..1d87a3c5fa 100644 --- a/raphtory/src/disk_graph/mod.rs +++ b/raphtory/src/disk_graph/mod.rs @@ -340,7 +340,7 @@ impl DiskGraphStorage { node_type_col: Option<&str>, ) -> Result { let layered_edge_list: Vec> = layer_parquet_cols - .iter() + .into_iter() .map( |ParquetLayerCols { parquet_dir, @@ -348,8 +348,9 @@ impl DiskGraphStorage { src_col, dst_col, time_col, + exclude_edge_props, }| { - ExternalEdgeList::new(layer, parquet_dir.as_ref(), src_col, dst_col, time_col) + ExternalEdgeList::new(layer, parquet_dir.as_ref(), src_col, dst_col, time_col, exclude_edge_props) .expect("Failed to load events") }, ) diff --git a/raphtory/src/python/graph/disk_graph.rs b/raphtory/src/python/graph/disk_graph.rs index 257fbad73d..69fcce81d5 100644 --- a/raphtory/src/python/graph/disk_graph.rs +++ b/raphtory/src/python/graph/disk_graph.rs @@ -90,6 +90,10 @@ impl<'a> FromPyObject<'a> for ParquetLayerCols<'a> { .get_item("time_col") .and_then(|item| item.expect("time_col is required").extract::<&PyString>()) .and_then(|s| s.to_str())?, + exclude_edge_props: dict.get_item("exclude_edge_props").and_then(|item| { + item.map(|item| item.extract::>()) + .unwrap_or_else(|| Ok(vec![])) + })?, }) } } From 4be0ef7d868ede7df600e0ea8fcda42faf267c4e Mon Sep 17 00:00:00 2001 From: Fabian Murariu Date: Thu, 10 Oct 2024 11:45:46 +0100 Subject: [PATCH 2/2] fmt --- pometry-storage-private | 2 +- raphtory-cypher/src/lib.rs | 2 +- raphtory/src/disk_graph/mod.rs | 11 +++++++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pometry-storage-private b/pometry-storage-private index 8a68cd7c44..93d65fd68d 160000 --- a/pometry-storage-private +++ b/pometry-storage-private @@ -1 +1 @@ -Subproject commit 8a68cd7c44b599b93446e194ca87c9d2d910a5b0 +Subproject commit 93d65fd68de26bc1f2702a932cfbc3b4998256ce diff --git a/raphtory-cypher/src/lib.rs b/raphtory-cypher/src/lib.rs index e26831b1e0..8cfac930fe 100644 --- a/raphtory-cypher/src/lib.rs +++ b/raphtory-cypher/src/lib.rs @@ -345,7 +345,7 @@ mod cypher { src_col: "src", dst_col: "dst", time_col: "epoch_time", - exclude_edge_props: vec![] + exclude_edge_props: vec![], }, ]; diff --git a/raphtory/src/disk_graph/mod.rs b/raphtory/src/disk_graph/mod.rs index 1d87a3c5fa..64587be6f3 100644 --- a/raphtory/src/disk_graph/mod.rs +++ b/raphtory/src/disk_graph/mod.rs @@ -350,8 +350,15 @@ impl DiskGraphStorage { time_col, exclude_edge_props, }| { - ExternalEdgeList::new(layer, parquet_dir.as_ref(), src_col, dst_col, time_col, exclude_edge_props) - .expect("Failed to load events") + ExternalEdgeList::new( + layer, + parquet_dir.as_ref(), + src_col, + dst_col, + time_col, + exclude_edge_props, + ) + .expect("Failed to load events") }, ) .collect::>();