refactor: external memory, fallible AsMetadata properties

Cargo.toml

@@ -15,6 +15,7 @@ exclude = ["/for_tests", "/.github"]
 base58 = {version = "0.2.0", optional = true}
 bitvec = {version = "1.0.1", default-features = false, features = ["alloc"]}
 blake2 = {version = "0.10.6", default-features = false, optional = true}
+external-memory-tools = {git = "https://github.com/Alzymologist/external-memory-tools", default-features = false}
 frame-metadata = {version = "16.0.0", default-features = false, features = ["current", "decode"]}
 hex = {version = "0.4.3", default-features = false, features = ["alloc"]}
 num-bigint = {version = "0.4.3", default-features = false}
@@ -29,7 +30,7 @@ sp-runtime = {version = "27.0.0", optional = true}
 
 [features]
 default = ["std"]
-std = ["plot_icon", "sp-core/std", "sp-runtime/std"]
+std = ["external-memory-tools/std", "frame-metadata/std", "plot_icon", "sp-core/std", "sp-runtime/std"]
 embed-display = ["base58", "blake2"]
 
 [lib]

src/compacts.rs

@@ -1,8 +1,8 @@
 //! [`Compact`] search and processing.
+use external_memory_tools::{AddressableBuffer, BufferError, ExternalMemory};
 use parity_scale_codec::{Compact, Decode, HasCompact};
 
 use crate::error::ParserError;
-use crate::traits::{AddressableBuffer, ExternalMemory};
 
 /// Compact found in data.
 pub struct FoundCompact<T: HasCompact> {
@@ -34,10 +34,10 @@ where
     Compact<T>: Decode,
 {
     if data.total_len() < position {
-        return Err(ParserError::OutOfRange {
+        return Err(ParserError::Buffer(BufferError::OutOfRange {
             position,
             total_length: data.total_len(),
-        });
+        }));
     }
     let mut out = None;
     for i in 0..(data.total_len() - position) {

src/decoding_sci.rs

@@ -2,6 +2,9 @@
 #[cfg(any(target_pointer_width = "32", test))]
 use bitvec::prelude::BitOrder;
 use bitvec::prelude::{BitVec, Lsb0, Msb0};
+#[cfg(any(target_pointer_width = "32", test))]
+use external_memory_tools::BufferError;
+use external_memory_tools::{AddressableBuffer, ExternalMemory};
 use num_bigint::{BigInt, BigUint};
 use parity_scale_codec::DecodeAll;
 use primitive_types::{H160, H512};
@@ -44,7 +47,7 @@ use crate::special_indicators::{
 use crate::special_types::{
     special_case_era, special_case_h256, wrap_sequence, CheckCompact, UnsignedInteger,
 };
-use crate::traits::{AddressableBuffer, AsMetadata, ExternalMemory, ResolveType};
+use crate::traits::{AsMetadata, ResolveType};
 use crate::MarkedData;
 
 /// Finalize parsing of primitives (variants of [`TypeDefPrimitive`]).
@@ -208,8 +211,7 @@ where
     let extrinsic_ty = extrinsic.ty;
     let types = meta_v14.types();
 
-    let extrinsic_type_params = extrinsic_type_params::<E, M>(ext_memory, &types, &extrinsic_ty)
-        .map_err(SignableError::Parsing)?;
+    let extrinsic_type_params = extrinsic_type_params::<E, M>(ext_memory, &types, &extrinsic_ty)?;
 
     let mut found_all_calls_ty = None;
 
@@ -229,8 +231,7 @@ where
         position,
         &types,
         Propagated::new(),
-    )
-    .map_err(SignableError::Parsing)?;
+    )?;
     if let ParsedData::Call(call) = call_extended_data.data {
         Ok(call)
     } else {
@@ -1097,10 +1098,11 @@ macro_rules! impl_patched {
                                     .expect("constant size slice, always fits"),
                             ),
                             Err(_) => {
-                                return Err(ParserError::DataTooShort {
+                                return Err(ParserError::Buffer(
+                                BufferError::DataTooShort {
                                     position: bitvec_positions.bitvec_start,
                                     minimal_length: bitvec_positions.minimal_length,
-                                })
+                                }))
                             }
                         }
                     }

src/decoding_sci_ext.rs

@@ -1,4 +1,5 @@
 //! Decode signable transaction extensions using `RuntimeMetadataV14`.
+use external_memory_tools::{AddressableBuffer, ExternalMemory};
 use primitive_types::H256;
 
 #[cfg(not(feature = "std"))]
@@ -18,7 +19,7 @@ use crate::error::{ExtensionsError, SignableError};
 use crate::propagated::Propagated;
 use crate::special_indicators::SpecialtyUnsignedInteger;
 use crate::special_types::UnsignedInteger;
-use crate::traits::{AddressableBuffer, AsMetadata, ExternalMemory};
+use crate::traits::AsMetadata;
 use crate::MarkedData;
 
 /// Parse extensions part of the signable transaction [`MarkedData`] using
@@ -85,30 +86,24 @@ where
     let meta_v14_types = meta_v14.types();
     let extrinsic = meta_v14.extrinsic().map_err(SignableError::MetaStructure)?;
     for signed_extensions_metadata in extrinsic.signed_extensions.iter() {
-        extensions.push(
-            decode_with_type::<B, E, M>(
-                &Ty::Symbol(&signed_extensions_metadata.ty),
-                data,
-                ext_memory,
-                position,
-                &meta_v14_types,
-                Propagated::from_ext_meta(signed_extensions_metadata),
-            )
-            .map_err(SignableError::Parsing)?,
-        )
+        extensions.push(decode_with_type::<B, E, M>(
+            &Ty::Symbol(&signed_extensions_metadata.ty),
+            data,
+            ext_memory,
+            position,
+            &meta_v14_types,
+            Propagated::from_ext_meta(signed_extensions_metadata),
+        )?)
     }
     for signed_extensions_metadata in extrinsic.signed_extensions.iter() {
-        extensions.push(
-            decode_with_type::<B, E, M>(
-                &Ty::Symbol(&signed_extensions_metadata.additional_signed),
-                data,
-                ext_memory,
-                position,
-                &meta_v14_types,
-                Propagated::from_ext_meta(signed_extensions_metadata),
-            )
-            .map_err(SignableError::Parsing)?,
-        )
+        extensions.push(decode_with_type::<B, E, M>(
+            &Ty::Symbol(&signed_extensions_metadata.additional_signed),
+            data,
+            ext_memory,
+            position,
+            &meta_v14_types,
+            Propagated::from_ext_meta(signed_extensions_metadata),
+        )?)
     }
     // `position > data.total_len()` is ruled out elsewhere
     if *position != data.total_len() {

src/error.rs

@@ -1,4 +1,5 @@
 //! Errors.
+use external_memory_tools::BufferError;
 use primitive_types::H256;
 
 use crate::std::string::String;
@@ -12,7 +13,9 @@ use std::{
 #[cfg(not(feature = "std"))]
 use core::fmt::{Display, Formatter, Result as FmtResult};
 
-use crate::traits::{AsMetadata, ExternalMemory};
+use external_memory_tools::ExternalMemory;
+
+use crate::traits::AsMetadata;
 
 /// Errors in signable transactions parsing.
 #[derive(Debug, Eq, PartialEq)]
@@ -112,66 +115,31 @@ impl<E: ExternalMemory> StorageError<E> {
 /// Errors in data parsing.
 #[derive(Debug, Eq, PartialEq)]
 pub enum ParserError<E: ExternalMemory> {
-    DataTooShort {
-        position: usize,
-        minimal_length: usize,
-    },
-    CyclicMetadata {
-        id: u32,
-    },
-    External(E::ExternalMemoryError),
+    Buffer(BufferError<E>),
+    CyclicMetadata { id: u32 },
     ExtrinsicNoCallParam,
-    NoCompact {
-        position: usize,
-    },
-    NotBitOrderType {
-        id: u32,
-    },
-    NotBitStoreType {
-        id: u32,
-    },
-    OutOfRange {
-        position: usize,
-        total_length: usize,
-    },
-    SomeDataNotUsedBlob {
-        from: usize,
-    },
-    TypeFailure {
-        position: usize,
-        ty: &'static str,
-    },
-    UnexpectedCompactInsides {
-        id: u32,
-    },
-    UnexpectedEnumVariant {
-        position: usize,
-    },
-    UnexpectedExtrinsicType {
-        extrinsic_ty_id: u32,
-    },
-    V14ShortTypesIncomplete {
-        old_id: u32,
-    },
-    V14TypeNotResolved {
-        id: u32,
-    },
-    V14TypeNotResolvedShortened {
-        id: u32,
-    },
+    NoCompact { position: usize },
+    NotBitOrderType { id: u32 },
+    NotBitStoreType { id: u32 },
+    SomeDataNotUsedBlob { from: usize },
+    TypeFailure { position: usize, ty: &'static str },
+    UnexpectedCompactInsides { id: u32 },
+    UnexpectedEnumVariant { position: usize },
+    UnexpectedExtrinsicType { extrinsic_ty_id: u32 },
+    V14ShortTypesIncomplete { old_id: u32 },
+    V14TypeNotResolved { id: u32 },
+    V14TypeNotResolvedShortened { id: u32 },
 }
 
 impl<E: ExternalMemory> ParserError<E> {
     fn error_text(&self) -> String {
         match &self {
-            ParserError::DataTooShort { position, minimal_length } => format!("Data is too short for expected content. Expected at least {minimal_length} element(s) after position {position}."),
+            ParserError::Buffer(buffer_error) => format!("{buffer_error}"),
             ParserError::CyclicMetadata { id } => format!("Resolving type id {id} in metadata type registry results in cycling."),
-            ParserError::External(e) => format!("Error accessing external memory. {e}"),
             ParserError::ExtrinsicNoCallParam => String::from("Extrinsic type in provided metadata has no specified call parameter."),
             ParserError::NoCompact { position } => format!("Expected compact starting at position {position}, not found one."),
             ParserError::NotBitOrderType { id } => format!("BitVec type {id} in metadata type registry has unexpected BitOrder type."),
             ParserError::NotBitStoreType { id } => format!("BitVec type {id} in metadata type registry has unexpected BitStore type."),
-            ParserError::OutOfRange { position, total_length } => format!("Position {position} is out of range for data length {total_length}."),
             ParserError::SomeDataNotUsedBlob { from } => format!("Some data (input positions [{from}..]) remained unused after decoding."),
             ParserError::TypeFailure { position, ty } => format!("Unable to decode data starting at position {position} as {ty}."),
             ParserError::UnexpectedCompactInsides { id } => format!("Compact type {id} in metadata type registry has unexpected type inside compact."),
@@ -276,7 +244,7 @@ pub enum UncheckedExtrinsicError<E: ExternalMemory, M: AsMetadata<E>> {
     NoCallParam,
     NoExtraParam,
     NoSignatureParam,
-    Parser(ParserError<E>),
+    Parsing(ParserError<E>),
     VersionMismatch { version_byte: u8, version: u8 },
     UnexpectedCallTy { call_ty_id: u32 },
 }
@@ -294,7 +262,7 @@ where
             UncheckedExtrinsicError::NoCallParam => String::from("Unchecked extrinsic type in provided metadata has no specified call parameter."),
             UncheckedExtrinsicError::NoExtraParam => String::from("Unchecked extrinsic type in provided metadata has no specified extra parameter."),
             UncheckedExtrinsicError::NoSignatureParam => String::from("Unchecked extrinsic type in provided metadata has no specified signature parameter."),
-            UncheckedExtrinsicError::Parser(parser_error) => format!("Error parsing unchecked extrinsic data. {parser_error}"),
+            UncheckedExtrinsicError::Parsing(parser_error) => format!("Error parsing unchecked extrinsic data. {parser_error}"),
             UncheckedExtrinsicError::VersionMismatch { version_byte, version } => format!("Version byte in unchecked extrinsic {version_byte} does not match with version {version} from provided metadata. Last 7 bits were expected to be identical."),
             UncheckedExtrinsicError::UnexpectedCallTy { call_ty_id } => format!("Parameter type for call {call_ty_id} in metadata type registry is not a call type, and does not match known call type descriptors."),
         }
@@ -326,7 +294,7 @@ impl_display_and_error!(ExtensionsError, MetaVersionErrorPallets);
 
 /// Implement [`Display`] for errors in both `std` and `no_std` cases.
 /// Implement `Error` for `std` case.
-macro_rules! impl_display_and_error_traited {
+macro_rules! impl_display_and_error_gen {
     ($($ty: ty), *) => {
         $(
             impl <E: ExternalMemory> Display for $ty {
@@ -345,46 +313,52 @@ macro_rules! impl_display_and_error_traited {
     }
 }
 
-impl_display_and_error_traited!(ParserError<E>, StorageError<E>);
+impl_display_and_error_gen!(ParserError<E>, StorageError<E>);
 
-impl<E, M> Display for SignableError<E, M>
-where
-    E: ExternalMemory,
-    M: AsMetadata<E>,
-{
-    fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult {
-        write!(f, "{}", self.error_text())
+impl<E: ExternalMemory> From<BufferError<E>> for ParserError<E> {
+    fn from(buffer_error: BufferError<E>) -> Self {
+        ParserError::Buffer(buffer_error)
     }
 }
 
-#[cfg(feature = "std")]
-impl<E, M> Error for SignableError<E, M>
-where
-    E: ExternalMemory,
-    M: AsMetadata<E>,
-{
-    fn source(&self) -> Option<&(dyn Error + 'static)> {
-        None
-    }
-}
+/// Implement [`Display`] for errors in both `std` and `no_std` cases.
+/// Implement `Error` for `std` case.
+/// Implement `From<ParserError<E>>` for simplified error conversion.
+macro_rules! impl_display_error_from_2gen {
+    ($($ty: ty), *) => {
+        $(
+            impl <E, M> Display for $ty
+            where
+                E: ExternalMemory,
+                M: AsMetadata<E>,
+            {
+                fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult {
+                    write!(f, "{}", self.error_text())
+                }
+            }
 
-impl<E, M> Display for UncheckedExtrinsicError<E, M>
-where
-    E: ExternalMemory,
-    M: AsMetadata<E>,
-{
-    fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult {
-        write!(f, "{}", self.error_text())
-    }
-}
+            #[cfg(feature = "std")]
+            impl <E, M> Error for $ty
+            where
+                E: ExternalMemory,
+                M: AsMetadata<E>,
+            {
+                fn source(&self) -> Option<&(dyn Error + 'static)> {
+                    None
+                }
+            }
 
-#[cfg(feature = "std")]
-impl<E, M> Error for UncheckedExtrinsicError<E, M>
-where
-    E: ExternalMemory,
-    M: AsMetadata<E>,
-{
-    fn source(&self) -> Option<&(dyn Error + 'static)> {
-        None
+            impl <E, M> From<ParserError<E>> for $ty
+            where
+                E: ExternalMemory,
+                M: AsMetadata<E>,
+            {
+                fn from(parser_error: ParserError<E>) -> Self {
+                    <$ty>::Parsing(parser_error)
+                }
+            }
+        )*
     }
 }
+
+impl_display_error_from_2gen!(SignableError<E, M>, UncheckedExtrinsicError<E, M>);

src/lib.rs

@@ -455,6 +455,7 @@
 #![no_std]
 #![deny(unused_crate_dependencies)]
 
+pub use external_memory_tools::{AddressableBuffer, ExternalMemory};
 use parity_scale_codec::{Decode, Encode};
 use primitive_types::H256;
 use scale_info::interner::UntrackedSymbol;
@@ -496,7 +497,7 @@ use core::{any::TypeId, marker::PhantomData};
 
 pub use decoding_sci::{decode_as_call, decode_as_call_unmarked, ResolvedTy};
 pub use decoding_sci_ext::{decode_extensions, decode_extensions_unmarked};
-pub use traits::{AddressableBuffer, AsMetadata, ExternalMemory, ResolveType};
+pub use traits::{AsMetadata, ResolveType};
 
 use cards::{Call, ExtendedCard, ExtendedData};
 use compacts::get_compact;

src/propagated.rs

@@ -1,4 +1,5 @@
 //! Data that can propagate hierarchically during parsing.
+use external_memory_tools::ExternalMemory;
 use frame_metadata::v14::SignedExtensionMetadata;
 use scale_info::{form::PortableForm, Field, Path, Type};
 
@@ -7,7 +8,6 @@ use crate::std::vec::Vec;
 use crate::cards::Info;
 use crate::error::ParserError;
 use crate::special_indicators::{Hint, SpecialtyH256, SpecialtyUnsignedInteger};
-use crate::traits::ExternalMemory;
 
 /// Type specialty data (type specialty [`Hint`] and compact info) that
 /// hierarchically propagates during the decoding.