Add serde to most streamable types

Cargo.lock

@@ -192,6 +192,15 @@ version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b"
 
+[[package]]
+name = "bincode"
+version = "1.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "bitflags"
 version = "2.6.0"
@@ -291,6 +300,7 @@ dependencies = [
  "chia-protocol",
  "chia-puzzle-types",
  "chia-secp",
+ "chia-serde",
  "chia-sha2 0.19.0",
  "chia-ssl",
  "chia-traits 0.19.0",
@@ -321,6 +331,7 @@ version = "0.19.0"
 dependencies = [
  "arbitrary",
  "blst",
+ "chia-serde",
  "chia-sha2 0.19.0",
  "chia-traits 0.19.0",
  "chia_py_streamable_macro",
@@ -331,6 +342,7 @@ dependencies = [
  "pyo3",
  "rand",
  "rstest",
+ "serde",
  "sha2",
  "thiserror",
 ]
@@ -404,8 +416,10 @@ dependencies = [
 name = "chia-protocol"
 version = "0.19.0"
 dependencies = [
+ "anyhow",
  "arbitrary",
  "chia-bls 0.19.0",
+ "chia-serde",
  "chia-sha2 0.19.0",
  "chia-traits 0.19.0",
  "chia_py_streamable_macro",
@@ -414,8 +428,11 @@ dependencies = [
  "clvm-utils",
  "clvmr",
  "hex",
+ "indoc",
  "pyo3",
  "rstest",
+ "serde",
+ "serde_json",
 ]
 
 [[package]]
@@ -484,6 +501,17 @@ dependencies = [
  "rand_chacha",
 ]
 
+[[package]]
+name = "chia-serde"
+version = "0.19.0"
+dependencies = [
+ "anyhow",
+ "bincode",
+ "hex",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "chia-sha2"
 version = "0.15.0"
@@ -2154,18 +2182,18 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b"
 
 [[package]]
 name = "serde"
-version = "1.0.204"
+version = "1.0.217"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12"
+checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.204"
+version = "1.0.217"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222"
+checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2174,9 +2202,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.121"
+version = "1.0.137"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ab380d7d9f22ef3f21ad3e6c1ebe8e4fc7a2000ccba2e4d71fc96f15b2cb609"
+checksum = "930cfb6e6abf99298aaad7d29abbef7a9999a9a8806a40088f55f0dcec03146b"
 dependencies = [
  "itoa",
  "memchr",

Cargo.toml

@@ -57,6 +57,7 @@ chia-client = { workspace = true, optional = true }
 chia-consensus = { workspace = true, optional = true }
 chia-protocol = { workspace = true, optional = true }
 chia-ssl = { workspace = true, optional = true }
+chia-serde = { workspace = true, optional = true }
 chia-traits = { workspace = true, optional = true }
 chia-puzzle-types = { workspace = true, optional = true }
 chia-sha2 = { workspace = true, optional = true }
@@ -77,6 +78,7 @@ default = [
     "consensus",
     "protocol",
     "ssl",
+    "serde",
     "traits",
     "puzzle-types",
     "sha2",
@@ -90,6 +92,7 @@ client = ["dep:chia-client"]
 consensus = ["dep:chia-consensus"]
 protocol = ["dep:chia-protocol"]
 ssl = ["dep:chia-ssl"]
+serde = ["dep:chia-serde", "chia-protocol/serde", "chia-bls/serde"]
 traits = ["dep:chia-traits"]
 puzzle-types = ["dep:chia-puzzle-types"]
 sha2 = ["dep:chia-sha2"]
@@ -113,6 +116,7 @@ chia-ssl = { path = "./crates/chia-ssl", version = "0.19.0" }
 chia-traits = { path = "./crates/chia-traits", version = "0.19.0" }
 chia-puzzle-types = { path = "./crates/chia-puzzle-types", version = "0.19.0" }
 chia-sha2 = { path = "./crates/chia-sha2", version = "0.19.0" }
+chia-serde = { path = "./crates/chia-serde", version = "0.19.0" }
 clvm-traits = { path = "./crates/clvm-traits", version = "0.19.0" }
 clvm-utils = { path = "./crates/clvm-utils", version = "0.19.0" }
 clvm-derive = { path = "./crates/clvm-derive", version = "0.19.0" }
@@ -161,3 +165,7 @@ k256 = "0.13.4"
 p256 = "0.13.2"
 rand_chacha = "0.3.1"
 chia-puzzles = "0.20.1"
+serde = "1.0.217"
+serde_json = "1.0.137"
+bincode = "1.3.3"
+indoc = "2.0.5"

crates/chia-bls/Cargo.toml

@@ -14,6 +14,7 @@ workspace = true
 [features]
 py-bindings = ["dep:pyo3", "chia_py_streamable_macro", "chia-traits/py-bindings"]
 arbitrary = ["dep:arbitrary"]
+serde = ["dep:serde", "dep:chia-serde"]
 
 [dependencies]
 chia-traits = { workspace = true }
@@ -27,6 +28,8 @@ thiserror = { workspace = true }
 pyo3 = { workspace = true, features = ["multiple-pymethods"], optional = true }
 arbitrary = { workspace = true, optional = true }
 linked-hash-map = "0.5.6"
+serde = { workspace = true, optional = true }
+chia-serde = { workspace = true, optional = true }
 
 [dev-dependencies]
 rand = { workspace = true }

crates/chia-bls/src/public_key.rs

@@ -161,6 +161,26 @@ impl PartialEq for PublicKey {
 }
 impl Eq for PublicKey {}
 
+#[cfg(feature = "serde")]
+impl serde::Serialize for PublicKey {
+    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        chia_serde::ser_bytes(&self.to_bytes(), serializer, true)
+    }
+}
+
+#[cfg(feature = "serde")]
+impl<'de> serde::Deserialize<'de> for PublicKey {
+    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        Self::from_bytes(&chia_serde::de_bytes(deserializer)?).map_err(serde::de::Error::custom)
+    }
+}
+
 impl Streamable for PublicKey {
     fn update_digest(&self, digest: &mut Sha256) {
         digest.update(self.to_bytes());

crates/chia-bls/src/signature.rs

@@ -228,6 +228,26 @@ impl Add<&Signature> for &Signature {
     }
 }
 
+#[cfg(feature = "serde")]
+impl serde::Serialize for Signature {
+    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        chia_serde::ser_bytes(&self.to_bytes(), serializer, true)
+    }
+}
+
+#[cfg(feature = "serde")]
+impl<'de> serde::Deserialize<'de> for Signature {
+    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        Self::from_bytes(&chia_serde::de_bytes(deserializer)?).map_err(serde::de::Error::custom)
+    }
+}
+
 // validate a series of public keys (G1 points) and G2 points. These points are
 // paired and the resulting GT points are multiplied. If the resulting GT point
 // is the identity, the function returns true, otherwise false. To validate an

crates/chia-protocol/Cargo.toml

@@ -14,6 +14,7 @@ workspace = true
 [features]
 py-bindings = ["dep:pyo3", "dep:chia_py_streamable_macro", "chia-traits/py-bindings", "chia-bls/py-bindings"]
 arbitrary = ["dep:arbitrary", "chia-bls/arbitrary"]
+serde = ["dep:serde", "dep:chia-serde", "chia-bls/serde"]
 
 [dependencies]
 pyo3 = { workspace = true, features = ["multiple-pymethods", "num-bigint"], optional = true }
@@ -27,9 +28,14 @@ clvm-traits = { workspace = true, features = ["derive"] }
 clvm-utils = { workspace = true }
 chia-bls = { workspace = true }
 arbitrary = { workspace = true, features = ["derive"], optional = true }
+serde = { workspace = true, optional = true, features = ["derive"] }
+chia-serde = { workspace = true, optional = true }
 
 [dev-dependencies]
 rstest = { workspace = true }
+serde_json = { workspace = true }
+anyhow = { workspace = true }
+indoc = { workspace = true }
 
 [lib]
 crate-type = ["rlib"]

crates/chia-protocol/src/bytes.rs

@@ -61,6 +61,26 @@ impl fmt::Display for Bytes {
     }
 }
 
+#[cfg(feature = "serde")]
+impl serde::Serialize for Bytes {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        chia_serde::ser_bytes(self, serializer, false)
+    }
+}
+
+#[cfg(feature = "serde")]
+impl<'de> serde::Deserialize<'de> for Bytes {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        chia_serde::de_bytes(deserializer)
+    }
+}
+
 impl Streamable for Bytes {
     fn update_digest(&self, digest: &mut Sha256) {
         (self.0.len() as u32).update_digest(digest);
@@ -209,6 +229,26 @@ impl<const N: usize> fmt::Display for BytesImpl<N> {
     }
 }
 
+#[cfg(feature = "serde")]
+impl<const N: usize> serde::Serialize for BytesImpl<N> {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        chia_serde::ser_bytes(self, serializer, true)
+    }
+}
+
+#[cfg(feature = "serde")]
+impl<'de, const N: usize> serde::Deserialize<'de> for BytesImpl<N> {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        chia_serde::de_bytes(deserializer)
+    }
+}
+
 impl<const N: usize> Streamable for BytesImpl<N> {
     fn update_digest(&self, digest: &mut Sha256) {
         digest.update(self.0);

crates/chia-protocol/src/chia_protocol.rs

@@ -179,7 +179,7 @@ impl chia_traits::ChiaToPython for NodeType {
     }
 }
 
-#[streamable]
+#[streamable(no_serde)]
 pub struct Message {
     msg_type: ProtocolMessageTypes,
     id: Option<u16>,

crates/chia-protocol/src/lib.rs

@@ -1,3 +1,9 @@
+// The Python bindings have unsafe methods so if you derive Deserialize,
+// Rust assumes that you may not be upholding invariants, and therefore
+// Deserialize (which is a safe trait) may not be safe to implement for the type.
+// We know that the Python bindings are safe with arbitrary values, so we can suppress this warning.
+#![allow(clippy::unsafe_derive_deserialize)]
+
 mod block_record;
 mod bytes;
 mod chia_protocol;

crates/chia-protocol/src/program.rs

@@ -21,6 +21,7 @@ use std::ops::Deref;
 
 #[cfg_attr(feature = "py-bindings", pyclass, derive(PyStreamable))]
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 pub struct Program(Bytes);
 
 impl Default for Program {
@@ -563,3 +564,21 @@ mod tests {
         assert_eq!(a.number(result), 1337.into());
     }
 }
+
+#[cfg(all(test, feature = "serde"))]
+mod serde_tests {
+    use super::*;
+
+    #[test]
+    fn test_program_is_bytes() -> anyhow::Result<()> {
+        let bytes = Bytes::new(vec![1, 2, 3]);
+        let program = Program::new(bytes.clone());
+
+        let bytes_json = serde_json::to_string(&bytes)?;
+        let program_json = serde_json::to_string(&program)?;
+
+        assert_eq!(program_json, bytes_json);
+
+        Ok(())
+    }
+}

crates/chia-protocol/src/spend_bundle.rs

@@ -272,3 +272,44 @@ ffff0101\
         });
     }
 }
+
+#[cfg(all(test, feature = "serde"))]
+mod serde_tests {
+    use chia_bls::Signature;
+    use indoc::indoc;
+
+    use crate::Program;
+
+    use super::*;
+
+    #[test]
+    fn test_json_spend_bundle() -> anyhow::Result<()> {
+        let json = serde_json::to_string_pretty(&SpendBundle::new(
+            vec![CoinSpend::new(
+                Coin::new([0; 32].into(), [1; 32].into(), 42),
+                Program::from(b"abc".to_vec()),
+                Program::from(b"xyz".to_vec()),
+            )],
+            Signature::default(),
+        ))?;
+
+        let output = indoc! {r#"{
+          "coin_spends": [
+            {
+              "coin": {
+                "parent_coin_info": "0x0000000000000000000000000000000000000000000000000000000000000000",
+                "puzzle_hash": "0x0101010101010101010101010101010101010101010101010101010101010101",
+                "amount": 42
+              },
+              "puzzle_reveal": "616263",
+              "solution": "78797a"
+            }
+          ],
+          "aggregated_signature": "0xc00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"
+        }"#};
+
+        assert_eq!(json, output);
+
+        Ok(())
+    }
+}