AbciInfo reimplemented as domain type with JSON serialization

proto-compiler/build.rs

@@ -4,8 +4,37 @@ use std::env::var;
 use std::path::{Path, PathBuf};
 use walkdir::WalkDir;
 
+/// Tendermint protobuf version
 const TENDERMINT_COMMITISH: &str = "tags/v0.34.0-rc5";
 
+/// Predefined custom attributes for annotations
+const FROM_STR: &str = r#"#[serde(with = "crate::serializers::from_str")]"#;
+const VEC_SKIP_IF_EMPTY: &str =
+    r#"#[serde(skip_serializing_if = "Vec::is_empty", with = "serde_bytes")]"#;
+
+// Custom type/field attributes:
+// The first item in the tuple defines the message or field where the annotation should apply and
+// the second item is the string that should be added on top of it.
+// The first item is a path as defined in the prost_build::Config::btree_map here:
+// https://docs.rs/prost-build/0.6.1/prost_build/struct.Config.html#method.btree_map
+
+/// Custom type attributes applied on top of protobuf structs
+static CUSTOM_TYPE_ATTRIBUTES: &[(&str, &str)] = &[
+    (".", "#[derive(::serde::Deserialize, ::serde::Serialize)]"), /* All types should be
+                                                                   * serializable */
+];
+
+/// Custom field attributes applied on top of protobuf fields in (a) struct(s)
+static CUSTOM_FIELD_ATTRIBUTES: &[(&str, &str)] = &[
+    (".tendermint.abci.ResponseInfo.app_version", FROM_STR),
+    (".tendermint.abci.ResponseInfo.last_block_height", FROM_STR),
+    (
+        ".tendermint.abci.ResponseInfo.last_block_app_hash",
+        VEC_SKIP_IF_EMPTY,
+    ),
+];
+
+/// Clone/open, fetch and check out a specific commitish
 fn git_commitish(dir: &Path, url: &str, commitish: &str) {
     // Open repo
     let repo = match dir.exists() {
@@ -84,9 +113,7 @@ fn main() {
         TENDERMINT_COMMITISH,
     );
 
-    let proto_paths = [
-        format!("{}/proto", tendermint_dir),
-    ];
+    let proto_paths = [format!("{}/proto", tendermint_dir)];
     let proto_includes_paths = [
         format!("{}/proto", tendermint_dir),
         format!("{}/third_party/proto", tendermint_dir),
@@ -112,9 +139,14 @@ fn main() {
     // List available paths for dependencies
     let includes: Vec<PathBuf> = proto_includes_paths.iter().map(PathBuf::from).collect();
 
-    // Compile all proto files
+    // Compile proto files, including well-known types (like Timestamp), with added annotations
     let mut pb = prost_build::Config::new();
     pb.compile_well_known_types();
-    pb.type_attribute(".", "#[derive(::serde::Deserialize, ::serde::Serialize)]");
+    for type_attribute in CUSTOM_TYPE_ATTRIBUTES {
+        pb.type_attribute(type_attribute.0, type_attribute.1);
+    }
+    for field_attribute in CUSTOM_FIELD_ATTRIBUTES {
+        pb.field_attribute(field_attribute.0, field_attribute.1);
+    }
     pb.compile_protos(&protos, &includes).unwrap();
 }

proto/Cargo.toml

@@ -23,3 +23,5 @@ bytes = "0.5"
 anomaly = "0.2"
 thiserror = "1.0"
 serde = { version = "1.0", features = ["derive"] }
+subtle-encoding = "0.5"
+serde_bytes = "0.11"

proto/src/lib.rs

@@ -70,11 +70,13 @@ mod tendermint {
 
 pub use tendermint::*;
 
-// Prost allows prost_types to be rebuilt, for so we can add JSON serialization annotations
+// Prost allows prost_types to be rebuilt, so we can add JSON serialization annotations
 pub use google::protobuf::Timestamp;
 
 mod domaintype;
 pub use domaintype::DomainType;
 
 mod error;
 pub use error::{Error, Kind};
+
+pub(crate) mod serializers;

proto/src/prost/tendermint.abci.rs

@@ -237,10 +237,13 @@ pub struct ResponseInfo {
     #[prost(string, tag="2")]
     pub version: std::string::String,
     #[prost(uint64, tag="3")]
+    #[serde(with = "crate::serializers::from_str")]
     pub app_version: u64,
     #[prost(int64, tag="4")]
+    #[serde(with = "crate::serializers::from_str")]
     pub last_block_height: i64,
     #[prost(bytes, tag="5")]
+    #[serde(skip_serializing_if = "Vec::is_empty", with = "serde_bytes")]
     pub last_block_app_hash: std::vec::Vec<u8>,
 }
 #[derive(Clone, PartialEq, ::prost::Message)]

proto/src/serializers.rs

@@ -0,0 +1,52 @@
+//! Serde JSON serializers
+//!
+//! Serializers and deserializers for a transparent developer experience.
+//!
+//! CAUTION: There are no guarantees for backwards compatibility, this module should be considered
+//! an internal implementation detail which can vanish without further warning. Use at your own
+//! risk.
+//!
+//! All serializers are presented in a serializers::<Rust_nickname>::<JSON_representation_name>
+//! format.
+//!
+//! This example shows how to serialize Vec<u8> into different types of strings:
+//! ```ignore
+//! use serde::{Serialize, Deserialize};
+//! use tendermint::serializers;
+//!
+//! #[derive(Serialize, Deserialize)]
+//! struct ByteTypes {
+//!
+//!     #[serde(with="serializers::bytes::hexstring")]
+//!     hexbytes: Vec<u8>,
+//!
+//!     #[serde(with="serializers::bytes::base64string")]
+//!     base64bytes: Vec<u8>,
+//!
+//!     #[serde(with="serializers::bytes::string")]
+//!     bytes: Vec<u8>,
+//!
+//! }
+//! ```
+//!
+//! Available serializers:
+//! i64                  <-> string:               #[serde(with="serializers::from_str")]
+//! u64                  <-> string:               #[serde(with="serializers::from_str")]
+//! std::time::Duration  <-> nanoseconds as string #[serde(with="serializers::time_duration")]
+//! Vec<u8>              <-> HexString:            #[serde(with="serializers::bytes::hexstring")]
+//! Vec<u8>              <-> Base64String:         #[serde(with="serializers::bytes::base64string")]
+//! Vec<u8>              <-> String:               #[serde(with="serializers::bytes::string")]
+//!
+//! Notes:
+//! * Any type that has the "FromStr" trait can be serialized into a string with
+//!   serializers::primitives::string.
+//! * serializers::bytes::* deserializes a null value into an empty vec![].
+
+// Todo: remove dead_code allowance as soon as more types are implemented
+#![allow(dead_code)]
+pub mod bytes;
+pub mod from_str;
+pub mod time_duration;
+
+mod custom;
+pub use custom::null_as_default;

proto/src/serializers/bytes.rs

@@ -0,0 +1,82 @@
+//! Serialize/deserialize bytes (Vec<u8>) type
+
+/// Serialize into hexstring, deserialize from hexstring
+pub mod hexstring {
+    use serde::{Deserialize, Deserializer, Serializer};
+    use subtle_encoding::hex;
+
+    /// Deserialize hexstring into Vec<u8>
+    pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let string = Option::<String>::deserialize(deserializer)?.unwrap_or_default();
+        hex::decode_upper(&string)
+            .or_else(|_| hex::decode(&string))
+            .map_err(serde::de::Error::custom)
+    }
+
+    /// Serialize from T into hexstring
+    pub fn serialize<S, T>(value: &T, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+        T: AsRef<[u8]>,
+    {
+        let hex_bytes = hex::encode(value.as_ref());
+        let hex_string = String::from_utf8(hex_bytes).map_err(serde::ser::Error::custom)?;
+        serializer.serialize_str(&hex_string)
+    }
+}
+
+/// Serialize into base64string, deserialize from base64string
+pub mod base64string {
+    use serde::{Deserialize, Deserializer, Serializer};
+    use subtle_encoding::base64;
+
+    /// Deserialize base64string into Vec<u8>
+    pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let string = Option::<String>::deserialize(deserializer)?.unwrap_or_default();
+        base64::decode(&string).map_err(serde::de::Error::custom)
+    }
+
+    /// Serialize from T into base64string
+    pub fn serialize<S, T>(value: &T, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+        T: AsRef<[u8]>,
+    {
+        let base64_bytes = base64::encode(value.as_ref());
+        let base64_string = String::from_utf8(base64_bytes).map_err(serde::ser::Error::custom)?;
+        serializer.serialize_str(&base64_string)
+    }
+}
+
+/// Serialize into string, deserialize from string
+pub(crate) mod string {
+    use serde::{Deserialize, Deserializer, Serializer};
+
+    /// Deserialize string into Vec<u8>
+    #[allow(dead_code)]
+    pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let string = Option::<String>::deserialize(deserializer)?.unwrap_or_default();
+        Ok(string.as_bytes().to_vec())
+    }
+
+    /// Serialize from T into string
+    #[allow(dead_code)]
+    pub(crate) fn serialize<S, T>(value: &T, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+        T: AsRef<[u8]>,
+    {
+        let string =
+            String::from_utf8(value.as_ref().to_vec()).map_err(serde::ser::Error::custom)?;
+        serializer.serialize_str(&string)
+    }
+}

proto/src/serializers/custom.rs

@@ -0,0 +1,13 @@
+//! Custom serializers
+
+use serde::{Deserialize, Deserializer};
+
+/// Parse null as default
+pub fn null_as_default<'de, D, T: Default + Deserialize<'de>>(
+    deserializer: D,
+) -> Result<T, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    Ok(<Option<T>>::deserialize(deserializer)?.unwrap_or_default())
+}

proto/src/serializers/from_str.rs

@@ -0,0 +1,25 @@
+//! Serialize and deserialize any `T` that implements [[std::str::FromStr]]
+//! and [[std::fmt::Display]] from or into string. Note this be used for
+//! all primitive data types (e.g. .
+use serde::{de::Error as _, Deserialize, Deserializer, Serialize, Serializer};
+
+/// Deserialize string into T
+pub fn deserialize<'de, D, T>(deserializer: D) -> Result<T, D::Error>
+where
+    D: Deserializer<'de>,
+    T: std::str::FromStr,
+    <T as std::str::FromStr>::Err: std::fmt::Display,
+{
+    String::deserialize(deserializer)?
+        .parse::<T>()
+        .map_err(|e| D::Error::custom(format!("{}", e)))
+}
+
+/// Serialize from T into string
+pub fn serialize<S, T>(value: &T, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+    T: std::fmt::Display,
+{
+    format!("{}", value).serialize(serializer)
+}

proto/src/serializers/time_duration.rs

@@ -0,0 +1,24 @@
+//! Serialize/deserialize std::time::Duration type from and into string:
+use serde::{de::Error as _, Deserialize, Deserializer, Serialize, Serializer};
+
+use std::time::Duration;
+
+/// Deserialize string into Duration
+pub fn deserialize<'de, D>(deserializer: D) -> Result<Duration, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let value = String::deserialize(deserializer)?
+        .parse::<u64>()
+        .map_err(|e| D::Error::custom(format!("{}", e)))?;
+
+    Ok(Duration::from_nanos(value))
+}
+
+/// Serialize from Duration into string
+pub fn serialize<S>(value: &Duration, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    format!("{}", value.as_nanos()).serialize(serializer)
+}