Allow props to be returned as JSON & Upgrade rust to 1.75 (#1482)

* expose props to json

* add multiple test to cover map+list+prop types

* UPGRADE RUST to 1.75

* this frankenstien almost works, its still a string

* fixed properties with new ScalarType

* commit cleaner names

* convert props to use actual gql types and vice versa

* delete crap off gh workflow runner to get more hdd space

.environment.yml

@@ -17,7 +17,7 @@ dependencies:
   - python_abi=3.10
   - readline>=8.1.2
   - libzlib>=1.2.13
-  - rust=1.73.0
+  - rust=1.75.0
   - setuptools=61.2.0
   - sqlite>=3.38.3
   - tk>=8.6.11

.github/workflows/_release_rust.yml

@@ -47,7 +47,7 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: 1.73.0
+          toolchain: 1.75.0
           override: true
           components: rustfmt, clippy
       - name: "Install cargo release"

.github/workflows/benchmark.yml

@@ -41,7 +41,7 @@ jobs:
         name: Setup Rust
         with:
           profile: minimal
-          toolchain: 1.73.0
+          toolchain: 1.75.0
           override: true
           components: rustfmt, clippy
       - name: Cargo update

.github/workflows/release_bump_versions.yml

@@ -54,7 +54,7 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: 1.73.0
+          toolchain: 1.75.0
           override: true
           components: rustfmt, clippy
       - name: "Install cargo release"

.github/workflows/test_python_workflow.yml

@@ -81,7 +81,7 @@ jobs:
         name: Setup Rust
         with:
           profile: minimal
-          toolchain: 1.73.0
+          toolchain: 1.75.0
           override: true
           components: rustfmt, clippy
       - name: Install sccache (macOS)

.github/workflows/test_rust_workflow.yml

@@ -50,9 +50,15 @@ jobs:
         name: Setup Rust
         with:
           profile: minimal
-          toolchain: 1.73.0
+          toolchain: 1.75.0
           override: true
           components: rustfmt, clippy
+      - name: Free up space (ubuntu)
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
       - name: Install sccache (macOS)
         if: matrix.os == 'macos-latest'
         run: brew install sccache
@@ -139,7 +145,7 @@ jobs:
         name: Setup Rust
         with:
           profile: minimal
-          toolchain: 1.73.0
+          toolchain: 1.75.0
           override: true
           components: rustfmt, clippy
       - name: Install sccache (Ubuntu)

raphtory-graphql/src/model/graph/property.rs

@@ -1,11 +1,78 @@
-use dynamic_graphql::{ResolvedObject, ResolvedObjectFields};
+use async_graphql::{Error, Name, Value as GqlValue};
+use dynamic_graphql::{ResolvedObject, ResolvedObjectFields, Scalar, ScalarValue};
 use raphtory::{
-    core::Prop,
+    core::{IntoPropMap, Prop},
     db::api::properties::{
         dyn_props::{DynConstProperties, DynProperties, DynProps, DynTemporalProperties},
         TemporalPropertyView,
     },
 };
+use serde_json::Number;
+use std::collections::HashMap;
+
+#[derive(Clone, Debug, Scalar)]
+pub struct GqlPropValue(pub Prop);
+
+impl ScalarValue for GqlPropValue {
+    fn from_value(value: GqlValue) -> Result<GqlPropValue, Error> {
+        Ok(GqlPropValue(gql_to_prop(value)?))
+    }
+
+    fn to_value(&self) -> GqlValue {
+        prop_to_gql(&self.0)
+    }
+}
+
+fn gql_to_prop(value: GqlValue) -> Result<Prop, Error> {
+    match value {
+        GqlValue::Number(n) => {
+            if let Some(n) = n.as_i64() {
+                Ok(Prop::I64(n))
+            } else if let Some(n) = n.as_f64() {
+                Ok(Prop::F64(n))
+            } else {
+                Err(async_graphql::Error::new("Unable to convert"))
+            }
+        }
+        GqlValue::Boolean(b) => Ok(Prop::Bool(b)),
+        GqlValue::Object(obj) => Ok(obj
+            .into_iter()
+            .map(|(k, v)| gql_to_prop(v).map(|vv| (k.to_string(), vv)))
+            .collect::<Result<HashMap<String, Prop>, Error>>()?
+            .into_prop_map()),
+        GqlValue::String(s) => Ok(Prop::Str(s.into())),
+        GqlValue::List(arr) => Ok(Prop::List(
+            arr.into_iter()
+                .map(gql_to_prop)
+                .collect::<Result<Vec<Prop>, Error>>()?
+                .into(),
+        )),
+        _ => Err(async_graphql::Error::new("Unable to convert")),
+    }
+}
+
+fn prop_to_gql(prop: &Prop) -> GqlValue {
+    match prop {
+        Prop::Str(s) => GqlValue::String(s.to_string()),
+        Prop::U8(u) => GqlValue::Number(Number::from(*u)),
+        Prop::U16(u) => GqlValue::Number(Number::from(*u)),
+        Prop::I32(u) => GqlValue::Number(Number::from(*u)),
+        Prop::I64(u) => GqlValue::Number(Number::from(*u)),
+        Prop::U32(u) => GqlValue::Number(Number::from(*u)),
+        Prop::U64(u) => GqlValue::Number(Number::from(*u)),
+        Prop::F32(u) => GqlValue::Number(Number::from_f64(*u as f64).unwrap()),
+        Prop::F64(u) => GqlValue::Number(Number::from_f64(*u).unwrap()),
+        Prop::Bool(b) => GqlValue::Boolean(*b),
+        Prop::List(l) => GqlValue::List(l.iter().map(|pp| prop_to_gql(pp)).collect()),
+        Prop::Map(m) => GqlValue::Object(
+            m.iter()
+                .map(|(k, v)| (Name::new(k.to_string()), prop_to_gql(v)))
+                .collect(),
+        ),
+        Prop::DTime(t) => GqlValue::Number(t.timestamp_millis().into()),
+        Prop::Graph(g) => GqlValue::String(g.to_string()),
+    }
+}
 
 #[derive(ResolvedObject)]
 pub(crate) struct GqlProp {
@@ -31,6 +98,10 @@ impl GqlProp {
     async fn as_string(&self) -> String {
         self.prop.to_string()
     }
+
+    async fn value(&self) -> GqlPropValue {
+        GqlPropValue(self.prop.clone())
+    }
 }
 
 #[derive(ResolvedObject)]

raphtory/Cargo.toml

@@ -37,14 +37,14 @@ enum_dispatch = "0.3"
 ordered-float = "4.1.1"
 glam = "0.25.0"
 quad-rand = "0.2.1"
+serde_json = "1"
 
 # io optional dependencies
 csv = {version="1.1.6", optional=true}
 zip = {version ="0.6.6", optional=true}
 neo4rs = {version="0.6.1", optional=true}
 bzip2 = {version="0.4", optional=true}
 flate2 = {version="1.0", optional=true}
-serde_json = {version="1", optional=true}
 reqwest = { version = "0.11.14", features = ["blocking"], optional=true}
 tokio = { version = "1.27.0", features = ["full"], optional=true}
 
@@ -80,7 +80,7 @@ proptest = "1.4.0"
 [features]
 default = []
 # Enables the graph loader io module
-io = ["dep:zip", "dep:neo4rs", "dep:bzip2", "dep:flate2", "dep:csv", "dep:serde_json", "dep:reqwest", "dep:tokio"]
+io = ["dep:zip", "dep:neo4rs", "dep:bzip2", "dep:flate2", "dep:csv", "dep:reqwest", "dep:tokio"]
 # Enables generating the pyo3 python bindings
 python = ["io", "dep:pyo3", "dep:pyo3-asyncio", "dep:num", "dep:display-error-chain", "dep:arrow2", "dep:kdam"]
 # search

raphtory/src/core/mod.rs

@@ -27,6 +27,7 @@
 use crate::{db::graph::graph::Graph, prelude::GraphViewOps};
 use chrono::NaiveDateTime;
 use serde::{Deserialize, Serialize};
+use serde_json::Value;
 use std::{
     borrow::Borrow,
     cmp::Ordering,
@@ -171,6 +172,34 @@ impl PartialOrd for Prop {
 }
 
 impl Prop {
+    pub fn to_json(&self) -> Value {
+        match self {
+            Prop::Str(value) => Value::String(value.to_string()),
+            Prop::U8(value) => Value::Number((*value).into()),
+            Prop::U16(value) => Value::Number((*value).into()),
+            Prop::I32(value) => Value::Number((*value).into()),
+            Prop::I64(value) => Value::Number((*value).into()),
+            Prop::U32(value) => Value::Number((*value).into()),
+            Prop::U64(value) => Value::Number((*value).into()),
+            Prop::F32(value) => Value::Number(serde_json::Number::from_f64(*value as f64).unwrap()),
+            Prop::F64(value) => Value::Number(serde_json::Number::from_f64(*value).unwrap()),
+            Prop::Bool(value) => Value::Bool(*value),
+            Prop::List(value) => {
+                let vec: Vec<Value> = value.iter().map(|v| v.to_json()).collect();
+                Value::Array(vec)
+            }
+            Prop::Map(value) => {
+                let map: serde_json::Map<String, Value> = value
+                    .iter()
+                    .map(|(k, v)| (k.to_string(), v.to_json()))
+                    .collect();
+                Value::Object(map)
+            }
+            Prop::DTime(value) => Value::String(value.to_string()),
+            Prop::Graph(_) => Value::String("Graph cannot be converted to JSON".to_string()),
+        }
+    }
+
     pub fn dtype(&self) -> PropType {
         match self {
             Prop::Str(_) => PropType::Str,

raphtory/src/db/graph/graph.rs

@@ -211,6 +211,7 @@ mod db_tests {
     use quickcheck::Arbitrary;
     use quickcheck_macros::quickcheck;
     use rayon::prelude::*;
+    use serde_json::Value;
     use std::collections::{HashMap, HashSet};
     use tempdir::TempDir;
 
@@ -303,6 +304,73 @@ mod db_tests {
             .all(|&(_, src, dst)| g.edge(src, dst).is_some())
     }
 
+    #[test]
+    fn prop_json_test() {
+        let g = Graph::new();
+        let _ = g.add_node(0, "A", NO_PROPS).unwrap();
+        let _ = g.add_node(0, "B", NO_PROPS).unwrap();
+        let e = g.add_edge(0, "A", "B", NO_PROPS, None).unwrap();
+        e.add_constant_properties(vec![("aprop".to_string(), Prop::Bool(true))], None)
+            .unwrap();
+        let ee = g.add_edge(0, "A", "B", NO_PROPS, Some(&"LAYERA")).unwrap();
+        ee.add_constant_properties(
+            vec![("aprop".to_string(), Prop::Bool(false))],
+            Some(&"LAYERA"),
+        )
+        .unwrap();
+        let json_res = g
+            .edge("A", "B")
+            .unwrap()
+            .properties()
+            .constant()
+            .get("aprop")
+            .unwrap()
+            .to_json();
+        let json_as_map = json_res.as_object().unwrap();
+        assert_eq!(json_as_map.len(), 2);
+        assert_eq!(json_as_map.get("LAYERA"), Some(&Value::Bool(false)));
+        assert_eq!(json_as_map.get("_default"), Some(&Value::Bool(true)));
+
+        let eee = g.add_edge(0, "A", "B", NO_PROPS, Some(&"LAYERB")).unwrap();
+        let v: Vec<Prop> = vec![Prop::Bool(true), Prop::Bool(false), Prop::U64(0)];
+        eee.add_constant_properties(
+            vec![("bprop".to_string(), Prop::List(Arc::new(v)))],
+            Some(&"LAYERB"),
+        )
+        .unwrap();
+        let json_res = g
+            .edge("A", "B")
+            .unwrap()
+            .properties()
+            .constant()
+            .get("bprop")
+            .unwrap()
+            .to_json();
+        let list_res = json_res.as_object().unwrap().get("LAYERB").unwrap();
+        assert_eq!(list_res.as_array().unwrap().len(), 3);
+
+        let eeee = g.add_edge(0, "A", "B", NO_PROPS, Some(&"LAYERC")).unwrap();
+        let v: HashMap<ArcStr, Prop> = HashMap::from([
+            (ArcStr::from("H".to_string()), Prop::Bool(false)),
+            (ArcStr::from("Y".to_string()), Prop::U64(0)),
+        ]);
+        eeee.add_constant_properties(
+            vec![("mymap".to_string(), Prop::Map(Arc::new(v)))],
+            Some(&"LAYERC"),
+        )
+        .unwrap();
+        let json_res = g
+            .edge("A", "B")
+            .unwrap()
+            .properties()
+            .constant()
+            .get("mymap")
+            .unwrap()
+            .to_json();
+        let map_res = json_res.as_object().unwrap().get("LAYERC").unwrap();
+        assert_eq!(map_res.as_object().unwrap().len(), 2);
+    }
+
     #[test]
     fn import_from_another_graph() {
         let g = Graph::new();
@@ -355,6 +423,20 @@ mod db_tests {
         assert_eq!(res.len(), 2);
     }
 
+    #[test]
+    fn props_with_layers() {
+        let g = Graph::new();
+        g.add_edge(0, "A", "B", NO_PROPS, None).unwrap();
+        let ed = g.edge("A", "B").unwrap();
+        ed.add_constant_properties(vec![("CCC", Prop::str("RED"))], None)
+            .unwrap();
+        println!("{:?}", ed.properties().constant().as_map());
+        g.add_edge(0, "A", "B", NO_PROPS, Some("LAYERONE")).unwrap();
+        ed.add_constant_properties(vec![("CCC", Prop::str("BLUE"))], Some("LAYERONE"))
+            .unwrap();
+        println!("{:?}", ed.properties().constant().as_map());
+    }
+
     #[test]
     fn graph_save_to_load_from_file() {
         let vs = vec![