Feature/materialise improvement

js-raphtory/src/graph/node.rs

@@ -81,7 +81,11 @@ impl Node {
 
     #[wasm_bindgen(js_name = edges)]
     pub fn edges(&self) -> js_sys::Array {
-        self.0.edges().iter().map(Edge).map(JsValue::from).collect()
+        self.0
+            .edges()
+            .iter()
+            .map(|e| JsValue::from(Edge(e.cloned())))
+            .collect()
     }
 
     // out_edges
@@ -90,8 +94,7 @@ impl Node {
         self.0
             .out_edges()
             .iter()
-            .map(Edge)
-            .map(JsValue::from)
+            .map(|e| JsValue::from(Edge(e.cloned())))
             .collect()
     }
 
@@ -101,8 +104,7 @@ impl Node {
         self.0
             .in_edges()
             .iter()
-            .map(Edge)
-            .map(JsValue::from)
+            .map(|e| JsValue::from(Edge(e.cloned())))
             .collect()
     }
 

raphtory-api/src/core/entities/edges/edge_ref.rs

@@ -76,8 +76,8 @@ impl EdgeRef {
     }
 
     #[inline(always)]
-    pub fn layer(&self) -> Option<&usize> {
-        self.layer_id.as_ref()
+    pub fn layer(&self) -> Option<usize> {
+        self.layer_id
     }
 
     #[inline(always)]

raphtory-api/src/core/entities/mod.rs

@@ -100,7 +100,7 @@ impl From<EdgeRef> for ELID {
     fn from(value: EdgeRef) -> Self {
         ELID {
             edge: value.pid(),
-            layer: value.layer().copied(),
+            layer: value.layer(),
         }
     }
 }
@@ -186,6 +186,13 @@ impl GID {
             GID::Str(v) => parse_u64_strict(v),
         }
     }
+
+    pub fn as_ref(&self) -> GidRef {
+        match self {
+            GID::U64(v) => GidRef::U64(*v),
+            GID::Str(v) => GidRef::Str(v),
+        }
+    }
 }
 
 impl From<u64> for GID {

raphtory-api/src/core/storage/dict_mapper.rs

@@ -87,6 +87,14 @@ impl<Index> BorrowMut<Index> for MaybeNew<Index> {
 }
 
 impl DictMapper {
+    pub fn deep_clone(&self) -> Self {
+        let reverse_map = self.reverse_map.read().clone();
+
+        Self {
+            map: self.map.clone(),
+            reverse_map: Arc::new(RwLock::new(reverse_map)),
+        }
+    }
     pub fn get_or_create_id<Q, T>(&self, name: &Q) -> MaybeNew<usize>
     where
         Q: Hash + Eq + ?Sized + ToOwned<Owned = T> + Borrow<str>,

raphtory-api/src/core/storage/timeindex.rs

@@ -3,7 +3,7 @@ use std::{fmt, ops::Range};
 use chrono::{DateTime, Utc};
 use serde::{Deserialize, Serialize};
 
-#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Ord, PartialOrd, Eq)]
+#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Ord, PartialOrd, Eq, Hash)]
 pub struct TimeIndexEntry(pub i64, pub usize);
 
 pub trait AsTime: fmt::Debug + Copy + Ord + Eq + Send + Sync + 'static {
@@ -52,7 +52,7 @@ impl TimeIndexEntry {
     }
 
     pub fn end(t: i64) -> Self {
-        Self(t.saturating_add(1), 0)
+        Self(t, usize::MAX)
     }
 }
 

raphtory-benchmark/Cargo.toml

@@ -7,7 +7,7 @@ edition = "2021"
 
 [dependencies]
 criterion = { workspace = true }
-raphtory = { path = "../raphtory", features = ["io"], version = "0.11.3" }
+raphtory = { path = "../raphtory", features = ["io", "proto"], version = "0.11.3" }
 raphtory-api = { path = "../raphtory-api", version = "0.11.3" }
 pometry-storage.workspace = true
 sorted_vector_map = { workspace = true }
@@ -42,6 +42,18 @@ harness = false
 name = "edge_add"
 harness = false
 
+[[bench]]
+name = "materialise"
+harness = false
+
+[[bench]]
+name = "proto_encode"
+harness = false
+
+[[bench]]
+name = "proto_decode"
+harness = false
+
 # [[bench]]
 # name = "arrow_algobench"
 # harness = false

raphtory-benchmark/benches/algobench.rs

@@ -1,4 +1,3 @@
-use crate::common::bench;
 use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, SamplingMode};
 use raphtory::{
     algorithms::{
@@ -16,9 +15,9 @@ use raphtory::{
     graphgen::random_attachment::random_attachment,
     prelude::*,
 };
+use raphtory_benchmark::common::bench;
 use rayon::prelude::*;
 
-mod common;
 pub fn local_triangle_count_analysis(c: &mut Criterion) {
     let mut group = c.benchmark_group("local_triangle_count");
     group.sample_size(10);

raphtory-benchmark/benches/arrow_algobench.rs

@@ -1,5 +1,3 @@
-mod common;
-
 #[cfg(feature = "storage")]
 pub mod arrow_bench {
 

raphtory-benchmark/benches/base.rs

@@ -1,9 +1,9 @@
-use crate::common::{bootstrap_graph, run_large_ingestion_benchmarks};
-use common::run_graph_ops_benches;
 use criterion::{criterion_group, criterion_main, Criterion, Throughput};
 use raphtory::{graph_loader::lotr_graph::lotr_graph, prelude::*};
-
-mod common;
+use raphtory_benchmark::common::{
+    bootstrap_graph, run_graph_ops_benches, run_large_ingestion_benchmarks,
+    run_proto_decode_benchmark, run_proto_encode_benchmark,
+};
 
 pub fn base(c: &mut Criterion) {
     // let mut ingestion_group = c.benchmark_group("ingestion");
@@ -48,7 +48,11 @@ pub fn base(c: &mut Criterion) {
         }
     }
 
-    run_graph_ops_benches(c, "lotr_graph", graph, layered_graph)
+    run_graph_ops_benches(c, "lotr_graph", graph.clone(), layered_graph);
+    let mut proto_group = c.benchmark_group("lotr_graph");
+    run_proto_decode_benchmark(&mut proto_group, graph.clone());
+    run_proto_encode_benchmark(&mut proto_group, graph.clone());
+    proto_group.finish();
 }
 
 criterion_group!(benches, base);

raphtory-benchmark/benches/edge_add.rs

@@ -1,11 +1,9 @@
 use criterion::{criterion_group, criterion_main, Criterion};
-use raphtory::prelude::*;
-
-mod common;
 use rand::{
     distributions::{Alphanumeric, DistString},
     thread_rng, Rng,
 };
+use raphtory::prelude::*;
 
 fn random_string(n: usize) -> String {
     Alphanumeric.sample_string(&mut thread_rng(), n)

raphtory-benchmark/benches/graph_ops.rs

@@ -1,13 +1,11 @@
-use common::run_graph_ops_benches;
 use criterion::{criterion_group, criterion_main, Criterion};
 use raphtory::{
     graph_loader::sx_superuser_graph::{sx_superuser_file, sx_superuser_graph, TEdge},
     io::csv_loader::CsvLoader,
     prelude::*,
 };
 use raphtory_api::core::utils::hashing::calculate_hash;
-
-mod common;
+use raphtory_benchmark::common::run_graph_ops_benches;
 
 pub fn graph(c: &mut Criterion) {
     let group_name = "analysis_graph";

raphtory-benchmark/benches/materialise.rs

@@ -0,0 +1,11 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use raphtory::{graph_loader::sx_superuser_graph::sx_superuser_graph, prelude::Graph};
+use raphtory_benchmark::common::bench_materialise;
+
+pub fn bench(c: &mut Criterion) {
+    let graph = sx_superuser_graph().unwrap();
+    bench_materialise("materialise", c, || graph.clone());
+}
+
+criterion_group!(benches, bench);
+criterion_main!(benches);

raphtory-benchmark/benches/parameterized.rs

@@ -1,10 +1,8 @@
-use crate::common::{bootstrap_graph, run_large_ingestion_benchmarks};
 use criterion::{
     criterion_group, criterion_main, AxisScale, Criterion, PlotConfiguration, Throughput,
 };
 use raphtory_api::core::entities::GID;
-
-mod common;
+use raphtory_benchmark::common::{bootstrap_graph, run_large_ingestion_benchmarks};
 
 pub fn parameterized(c: &mut Criterion) {
     let nodes_exponents = 1..6;

raphtory-benchmark/benches/proto_decode.rs

@@ -0,0 +1,20 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use raphtory::graph_loader::{lotr_graph::lotr_graph, sx_superuser_graph::sx_superuser_graph};
+use raphtory_benchmark::common::run_proto_decode_benchmark;
+
+fn bench(c: &mut Criterion) {
+    let graph = sx_superuser_graph().unwrap();
+    let mut group = c.benchmark_group("proto_sx_superuser");
+    group.sample_size(10);
+    run_proto_decode_benchmark(&mut group, graph);
+    group.finish();
+
+    let mut group = c.benchmark_group("proto_lotr");
+    let graph = lotr_graph();
+    group.sample_size(100);
+    run_proto_decode_benchmark(&mut group, graph);
+    group.finish();
+}
+
+criterion_group!(benches, bench);
+criterion_main!(benches);

raphtory-benchmark/benches/proto_encode.rs

@@ -0,0 +1,14 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use raphtory::graph_loader::sx_superuser_graph::sx_superuser_graph;
+use raphtory_benchmark::common::run_proto_encode_benchmark;
+
+fn bench(c: &mut Criterion) {
+    let graph = sx_superuser_graph().unwrap();
+    let mut group = c.benchmark_group("proto_sx_superuser");
+    group.sample_size(10);
+    run_proto_encode_benchmark(&mut group, graph);
+    group.finish();
+}
+
+criterion_group!(benches, bench);
+criterion_main!(benches);

raphtory-benchmark/benches/common/mod.rs → raphtory-benchmark/src/common/mod.rs

@@ -6,6 +6,7 @@ use criterion::{
 use rand::{distributions::Uniform, seq::*, Rng, SeedableRng};
 use raphtory::{db::api::view::StaticGraphViewOps, prelude::*};
 use std::collections::HashSet;
+use tempfile::NamedTempFile;
 
 fn make_index_gen() -> Box<dyn Iterator<Item = u64>> {
     let rng = rand::thread_rng();
@@ -570,7 +571,24 @@ pub fn run_graph_ops_benches(
     );
 }
 
-fn bench_materialise<F, G>(name: &str, c: &mut Criterion, make_graph: F)
+pub fn run_proto_encode_benchmark(group: &mut BenchmarkGroup<WallTime>, graph: Graph) {
+    println!("graph: {graph}");
+    let f = NamedTempFile::new().unwrap();
+    bench(group, "proto_encode", None, |b: &mut Bencher| {
+        b.iter(|| graph.encode(f.path()).unwrap())
+    });
+}
+
+pub fn run_proto_decode_benchmark(group: &mut BenchmarkGroup<WallTime>, graph: Graph) {
+    println!("graph: {graph}");
+    let f = NamedTempFile::new().unwrap();
+    graph.encode(f.path()).unwrap();
+    bench(group, "proto_decode", None, |b| {
+        b.iter(|| Graph::decode(f.path()).unwrap())
+    })
+}
+
+pub fn bench_materialise<F, G>(name: &str, c: &mut Criterion, make_graph: F)
 where
     F: Fn() -> G,
     G: StaticGraphViewOps,

raphtory-benchmark/src/lib.rs

@@ -0,0 +1 @@
+pub mod common;

raphtory-graphql/src/model/graph/edge.rs

@@ -29,6 +29,17 @@ impl<G: StaticGraphViewOps + IntoDynamic, GH: StaticGraphViewOps + IntoDynamic>
     }
 }
 
+impl Edge {
+    pub(crate) fn from_ref<
+        G: StaticGraphViewOps + IntoDynamic,
+        GH: StaticGraphViewOps + IntoDynamic,
+    >(
+        value: EdgeView<&G, &GH>,
+    ) -> Self {
+        value.cloned().into()
+    }
+}
+
 #[ResolvedObjectFields]
 impl Edge {
     ////////////////////////

raphtory-graphql/src/model/graph/edges.rs

@@ -22,7 +22,7 @@ impl GqlEdges {
     }
 
     fn iter(&self) -> Box<dyn Iterator<Item = Edge> + '_> {
-        let iter = self.ee.iter().map(Edge::from);
+        let iter = self.ee.iter().map(Edge::from_ref);
         Box::new(iter)
     }
 }

raphtory-graphql/src/model/schema/edge_schema.rs

@@ -5,7 +5,7 @@ use dynamic_graphql::{ResolvedObject, ResolvedObjectFields};
 use itertools::Itertools;
 use raphtory::{
     db::{api::view::StaticGraphViewOps, graph::edge::EdgeView},
-    prelude::EdgeViewOps,
+    prelude::{EdgeViewOps, GraphViewOps},
 };
 use std::collections::{HashMap, HashSet};
 
@@ -40,13 +40,13 @@ impl<G: StaticGraphViewOps> EdgeSchema<G> {
 
     /// Returns the list of property schemas for edges connecting these types of nodes
     async fn properties(&self) -> Vec<PropertySchema> {
-        let filter_types = |edge: &EdgeView<G>| {
+        let filter_types = |edge: &EdgeView<&G>| {
             let src_type = get_node_type(edge.src());
             let dst_type = get_node_type(edge.dst());
             src_type == self.src_type && dst_type == self.dst_type
         };
-
-        let filtered_edges = self.graph.edges().iter().filter(filter_types);
+        let edges = self.graph.edges();
+        let filtered_edges = edges.iter().filter(filter_types);
 
         let schema: SchemaAggregate = filtered_edges
             .map(collect_edge_schema)
@@ -57,7 +57,7 @@ impl<G: StaticGraphViewOps> EdgeSchema<G> {
     }
 }
 
-fn collect_edge_schema<G: StaticGraphViewOps>(edge: EdgeView<G>) -> SchemaAggregate {
+fn collect_edge_schema<'graph, G: GraphViewOps<'graph>>(edge: EdgeView<G>) -> SchemaAggregate {
     edge.properties()
         .iter()
         .map(|(key, value)| (key.to_string(), HashSet::from([value.to_string()])))

raphtory-graphql/src/model/schema/mod.rs

@@ -1,6 +1,6 @@
 use raphtory::{
-    db::{api::view::StaticGraphViewOps, graph::node::NodeView},
-    prelude::NodeViewOps,
+    db::graph::node::NodeView,
+    prelude::{GraphViewOps, NodeViewOps},
 };
 use std::collections::{HashMap, HashSet};
 
@@ -12,7 +12,7 @@ pub(crate) mod property_schema;
 
 const ENUM_BOUNDARY: usize = 20;
 
-fn get_node_type<G: StaticGraphViewOps>(node: NodeView<G>) -> String {
+fn get_node_type<'graph, G: GraphViewOps<'graph>>(node: NodeView<G>) -> String {
     let prop = node.properties().get("type");
     prop.map(|prop| prop.to_string())
         .unwrap_or_else(|| "NONE".to_string())

raphtory/src/algorithms/community_detection/modularity.rs

@@ -197,7 +197,7 @@ impl ModularityFunction for ModularityUnDir {
                         let w = weight_prop
                             .map(|w| e.properties().get(w).unwrap_f64())
                             .unwrap_or(1.0);
-                        let dst_id = local_id_map[&e.nbr()];
+                        let dst_id = local_id_map[&e.nbr().cloned()];
                         (dst_id, w)
                     })
                     .filter(|(_, w)| w >= &tol)