metadata: Use v15 internally (#912)

* Update frame-metadata to v15.1.0

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* Enable V15 unstable metadata in frame-metadata

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* metadata: Move validation hashing to dedicated file

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* Use sp-metadata-ir from substrate to work with metadata

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* Revert using sp-metadata-ir in favor of conversion to v15

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* metadata: Convert v14 to v15

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* metadata: Use v15 for validation

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* codegen: Use v15 for codegen

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* metadata/bench: Use v15

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* Adjust to v15 metadata

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* Adjust testing

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* Improve documentation

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* force CI

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* metadata: Address feedback

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* metadata: Use HASH_LEN

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* metadta: Remove `LatestRuntimeMetadata` type alias

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* metadata: Remove `metadata_to_latest` to avoid pancis

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

---------

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

cli/Cargo.toml

@@ -32,7 +32,7 @@ serde_json = "1.0.96"
 # hex encoded metadata to bytes
 hex = "0.4.3"
 # actual metadata types
-frame-metadata = { version = "15.0.0", features = ["v14", "std"] }
+frame-metadata = { version = "15.1.0", features = ["v14", "v15-unstable", "std"] }
 # decode bytes into the metadata types
 scale = { package = "parity-scale-codec", version = "3.0.0", default-features = false }
 # generate the item mod for codegen

cli/src/commands/compatibility.rs

@@ -4,12 +4,14 @@
 
 use clap::Parser as ClapParser;
 use color_eyre::eyre::{self, WrapErr};
-use frame_metadata::{RuntimeMetadata, RuntimeMetadataPrefixed, RuntimeMetadataV14, META_RESERVED};
+use frame_metadata::{
+    v15::RuntimeMetadataV15, RuntimeMetadata, RuntimeMetadataPrefixed, META_RESERVED,
+};
 use jsonrpsee::client_transport::ws::Uri;
 use scale::Decode;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
-use subxt_metadata::{get_metadata_hash, get_pallet_hash};
+use subxt_metadata::{get_metadata_hash, get_pallet_hash, metadata_v14_to_latest};
 
 /// Verify metadata compatibility between substrate nodes.
 #[derive(Debug, ClapParser)]
@@ -94,7 +96,7 @@ async fn handle_full_metadata(nodes: &[Uri]) -> color_eyre::Result<()> {
     Ok(())
 }
 
-async fn fetch_runtime_metadata(url: &Uri) -> color_eyre::Result<RuntimeMetadataV14> {
+async fn fetch_runtime_metadata(url: &Uri) -> color_eyre::Result<RuntimeMetadataV15> {
     let bytes = subxt_codegen::utils::fetch_metadata_bytes(url).await?;
 
     let metadata = <RuntimeMetadataPrefixed as Decode>::decode(&mut &bytes[..])?;
@@ -108,7 +110,8 @@ async fn fetch_runtime_metadata(url: &Uri) -> color_eyre::Result<RuntimeMetadata
     }
 
     match metadata.1 {
-        RuntimeMetadata::V14(v14) => Ok(v14),
+        RuntimeMetadata::V14(v14) => Ok(metadata_v14_to_latest(v14)),
+        RuntimeMetadata::V15(v15) => Ok(v15),
         _ => Err(eyre::eyre!(
             "Node {:?} with unsupported metadata version: {:?}",
             url,

cli/src/commands/metadata.rs

@@ -8,7 +8,7 @@ use color_eyre::eyre;
 use frame_metadata::{RuntimeMetadata, RuntimeMetadataPrefixed};
 use scale::{Decode, Encode};
 use std::io::{self, Write};
-use subxt_metadata::retain_metadata_pallets;
+use subxt_metadata::{metadata_v14_to_latest, retain_metadata_pallets};
 
 /// Download metadata from a substrate node, for use with `subxt` codegen.
 #[derive(Debug, ClapParser)]
@@ -20,6 +20,9 @@ pub struct Opts {
     format: String,
     /// Generate a subset of the metadata that contains only the
     /// types needed to represent the provided pallets.
+    ///
+    /// The returned metadata is updated to the latest available version
+    /// when using the option.
     #[clap(long, use_value_delimiter = true, value_parser)]
     pallets: Option<Vec<String>>,
 }
@@ -29,8 +32,9 @@ pub async fn run(opts: Opts) -> color_eyre::Result<()> {
     let mut metadata = <RuntimeMetadataPrefixed as Decode>::decode(&mut &bytes[..])?;
 
     if let Some(pallets) = opts.pallets {
-        let metadata_v14 = match &mut metadata.1 {
-            RuntimeMetadata::V14(metadata_v14) => metadata_v14,
+        let mut metadata_v15 = match metadata.1 {
+            RuntimeMetadata::V14(metadata_v14) => metadata_v14_to_latest(metadata_v14),
+            RuntimeMetadata::V15(metadata_v15) => metadata_v15,
             _ => {
                 return Err(eyre::eyre!(
                     "Unsupported metadata version {:?}, expected V14.",
@@ -39,9 +43,10 @@ pub async fn run(opts: Opts) -> color_eyre::Result<()> {
             }
         };
 
-        retain_metadata_pallets(metadata_v14, |pallet_name| {
+        retain_metadata_pallets(&mut metadata_v15, |pallet_name| {
             pallets.iter().any(|p| &**p == pallet_name)
         });
+        metadata = metadata_v15.into();
     }
 
     match opts.format.as_str() {

codegen/Cargo.toml

@@ -15,7 +15,7 @@ description = "Generate an API for interacting with a substrate node from FRAME
 [dependencies]
 codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "full"] }
 darling = "0.14.4"
-frame-metadata = "15.0.0"
+frame-metadata = { version = "15.1.0", features = ["v14", "v15-unstable", "std"] }
 heck = "0.4.1"
 proc-macro2 = "1.0.55"
 quote = "1.0.8"

codegen/src/api/calls.rs

@@ -7,7 +7,7 @@ use crate::{
     types::{CompositeDefFields, TypeGenerator},
     CratePath,
 };
-use frame_metadata::{v14::RuntimeMetadataV14, PalletMetadata};
+use frame_metadata::v15::{PalletMetadata, RuntimeMetadataV15};
 use heck::{ToSnakeCase as _, ToUpperCamelCase as _};
 use proc_macro2::TokenStream as TokenStream2;
 use quote::{format_ident, quote};
@@ -23,7 +23,7 @@ use scale_info::form::PortableForm;
 /// - `pallet` - Pallet metadata from which the calls are generated.
 /// - `types_mod_ident` - The ident of the base module that we can use to access the generated types from.
 pub fn generate_calls(
-    metadata: &RuntimeMetadataV14,
+    metadata: &RuntimeMetadataV15,
     type_gen: &TypeGenerator,
     pallet: &PalletMetadata<PortableForm>,
     types_mod_ident: &syn::Ident,

codegen/src/api/constants.rs

@@ -3,7 +3,7 @@
 // see LICENSE for license details.
 
 use crate::{types::TypeGenerator, CratePath};
-use frame_metadata::{v14::RuntimeMetadataV14, PalletMetadata};
+use frame_metadata::v15::{PalletMetadata, RuntimeMetadataV15};
 use heck::ToSnakeCase as _;
 use proc_macro2::TokenStream as TokenStream2;
 use quote::{format_ident, quote};
@@ -35,7 +35,7 @@ use super::CodegenError;
 /// - `pallet` - Pallet metadata from which the calls are generated.
 /// - `types_mod_ident` - The ident of the base module that we can use to access the generated types from.
 pub fn generate_constants(
-    metadata: &RuntimeMetadataV14,
+    metadata: &RuntimeMetadataV15,
     type_gen: &TypeGenerator,
     pallet: &PalletMetadata<PortableForm>,
     types_mod_ident: &syn::Ident,

codegen/src/api/events.rs

@@ -3,7 +3,7 @@
 // see LICENSE for license details.
 
 use crate::{types::TypeGenerator, CratePath};
-use frame_metadata::PalletMetadata;
+use frame_metadata::v15::PalletMetadata;
 use proc_macro2::TokenStream as TokenStream2;
 use quote::quote;
 use scale_info::form::PortableForm;

codegen/src/api/mod.rs

@@ -9,7 +9,8 @@ mod constants;
 mod events;
 mod storage;
 
-use subxt_metadata::get_metadata_per_pallet_hash;
+use frame_metadata::v15::RuntimeMetadataV15;
+use subxt_metadata::{get_metadata_per_pallet_hash, metadata_v14_to_latest};
 
 use super::DerivesRegistry;
 use crate::error::CodegenError;
@@ -20,7 +21,7 @@ use crate::{
     CratePath,
 };
 use codec::Decode;
-use frame_metadata::{v14::RuntimeMetadataV14, RuntimeMetadata, RuntimeMetadataPrefixed};
+use frame_metadata::{RuntimeMetadata, RuntimeMetadataPrefixed};
 use heck::ToSnakeCase as _;
 use proc_macro2::TokenStream as TokenStream2;
 use quote::{format_ident, quote};
@@ -152,7 +153,7 @@ pub fn generate_runtime_api_from_bytes(
 
 /// Create the API for interacting with a Substrate runtime.
 pub struct RuntimeGenerator {
-    metadata: RuntimeMetadataV14,
+    metadata: RuntimeMetadataV15,
 }
 
 impl RuntimeGenerator {
@@ -161,11 +162,20 @@ impl RuntimeGenerator {
     /// **Note:** If you have the metadata path, URL or bytes to hand, prefer to use
     /// one of the `generate_runtime_api_from_*` functions for generating the runtime API
     /// from that.
+    ///
+    /// # Panics
+    ///
+    /// Panics if the runtime metadata version is not supported.
+    ///
+    /// Supported versions: v14 and v15.
     pub fn new(metadata: RuntimeMetadataPrefixed) -> Self {
-        match metadata.1 {
-            RuntimeMetadata::V14(v14) => Self { metadata: v14 },
+        let metadata = match metadata.1 {
+            RuntimeMetadata::V14(v14) => metadata_v14_to_latest(v14),
+            RuntimeMetadata::V15(v15) => v15,
             _ => panic!("Unsupported metadata version {:?}", metadata.1),
-        }
+        };
+
+        RuntimeGenerator { metadata }
     }
 
     /// Generate the API for interacting with a Substrate runtime.

codegen/src/api/storage.rs

@@ -3,8 +3,8 @@
 // see LICENSE for license details.
 
 use crate::{types::TypeGenerator, CratePath};
-use frame_metadata::{
-    v14::RuntimeMetadataV14, PalletMetadata, StorageEntryMetadata, StorageEntryModifier,
+use frame_metadata::v15::{
+    PalletMetadata, RuntimeMetadataV15, StorageEntryMetadata, StorageEntryModifier,
     StorageEntryType,
 };
 use heck::ToSnakeCase as _;
@@ -24,7 +24,7 @@ use super::CodegenError;
 /// - `pallet` - Pallet metadata from which the storages are generated.
 /// - `types_mod_ident` - The ident of the base module that we can use to access the generated types from.
 pub fn generate_storage(
-    metadata: &RuntimeMetadataV14,
+    metadata: &RuntimeMetadataV15,
     type_gen: &TypeGenerator,
     pallet: &PalletMetadata<PortableForm>,
     types_mod_ident: &syn::Ident,
@@ -64,7 +64,7 @@ pub fn generate_storage(
 }
 
 fn generate_storage_entry_fns(
-    metadata: &RuntimeMetadataV14,
+    metadata: &RuntimeMetadataV15,
     type_gen: &TypeGenerator,
     pallet: &PalletMetadata<PortableForm>,
     storage_entry: &StorageEntryMetadata<PortableForm>,

codegen/src/error.rs

@@ -13,10 +13,10 @@ pub enum CodegenError {
     #[error("Could not find type with ID {0} in the type registry; please raise a support issue.")]
     TypeNotFound(u32),
     /// Cannot fetch the metadata bytes.
-    #[error("Failed to fetch metadata, make sure that you're pointing at a node which is providing V14 metadata: {0}")]
+    #[error("Failed to fetch metadata, make sure that you're pointing at a node which is providing substrate-based metadata: {0}")]
     Fetch(#[from] FetchMetadataError),
     /// Failed IO for the metadata file.
-    #[error("Failed IO for {0}, make sure that you are providing the correct file path for metadata V14: {1}")]
+    #[error("Failed IO for {0}, make sure that you are providing the correct file path for metadata: {1}")]
     Io(String, std::io::Error),
     /// Cannot decode the metadata bytes.
     #[error("Could not decode metadata, only V14 metadata is supported: {0}")]
@@ -25,7 +25,7 @@ pub enum CodegenError {
     #[error("Out-of-line subxt modules are not supported, make sure you are providing a body to your module: pub mod polkadot {{ ... }}")]
     InvalidModule(Span),
     /// Expected named or unnamed fields.
-    #[error("Fields should either be all named or all unnamed, make sure you are providing a valid metadata V14: {0}")]
+    #[error("Fields should either be all named or all unnamed, make sure you are providing a valid metadata: {0}")]
     InvalidFields(String),
     /// Substitute types must have a valid path.
     #[error("Type substitution error: {0}")]
@@ -34,20 +34,20 @@ pub enum CodegenError {
     #[error("Invalid type path {0}: {1}")]
     InvalidTypePath(String, syn::Error),
     /// Metadata for constant could not be found.
-    #[error("Metadata for constant entry {0}_{1} could not be found. Make sure you are providing a valid metadata V14")]
+    #[error("Metadata for constant entry {0}_{1} could not be found. Make sure you are providing a valid substrate-based metadata")]
     MissingConstantMetadata(String, String),
     /// Metadata for storage could not be found.
-    #[error("Metadata for storage entry {0}_{1} could not be found. Make sure you are providing a valid metadata V14")]
+    #[error("Metadata for storage entry {0}_{1} could not be found. Make sure you are providing a valid substrate-based metadata")]
     MissingStorageMetadata(String, String),
     /// Metadata for call could not be found.
-    #[error("Metadata for call entry {0}_{1} could not be found. Make sure you are providing a valid metadata V14")]
+    #[error("Metadata for call entry {0}_{1} could not be found. Make sure you are providing a valid substrate-based metadata")]
     MissingCallMetadata(String, String),
     /// Call variant must have all named fields.
-    #[error("Call variant for type {0} must have all named fields. Make sure you are providing a valid metadata V14")]
+    #[error("Call variant for type {0} must have all named fields. Make sure you are providing a valid substrate-based metadata")]
     InvalidCallVariant(u32),
     /// Type should be an variant/enum.
     #[error(
-        "{0} type should be an variant/enum type. Make sure you are providing a valid metadata V14"
+        "{0} type should be an variant/enum type. Make sure you are providing a valid substrate-based metadata"
     )]
     InvalidType(String),
 }

metadata/Cargo.toml

@@ -15,7 +15,7 @@ description = "Command line utilities for checking metadata compatibility betwee
 
 [dependencies]
 codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "full"] }
-frame-metadata = "15.0.0"
+frame-metadata = { version = "15.1.0", features = ["v14", "v15-unstable", "std"] }
 scale-info = "2.5.0"
 sp-core-hashing = "8.0.0"
 

metadata/benches/bench.rs

@@ -4,21 +4,23 @@
 
 use codec::Decode;
 use criterion::*;
-use frame_metadata::{RuntimeMetadata::V14, RuntimeMetadataPrefixed, RuntimeMetadataV14};
+use frame_metadata::{v15::RuntimeMetadataV15, RuntimeMetadata, RuntimeMetadataPrefixed};
 use scale_info::{form::PortableForm, TypeDef, TypeDefVariant};
 use std::{fs, path::Path};
 use subxt_metadata::{
     get_call_hash, get_constant_hash, get_metadata_hash, get_pallet_hash, get_storage_hash,
+    metadata_v14_to_latest,
 };
 
-fn load_metadata() -> RuntimeMetadataV14 {
+fn load_metadata() -> RuntimeMetadataV15 {
     let bytes = fs::read(Path::new("../artifacts/polkadot_metadata.scale"))
         .expect("Cannot read metadata blob");
     let meta: RuntimeMetadataPrefixed =
         Decode::decode(&mut &*bytes).expect("Cannot decode scale metadata");
 
     match meta.1 {
-        V14(v14) => v14,
+        RuntimeMetadata::V14(v14) => metadata_v14_to_latest(v14),
+        RuntimeMetadata::V15(v15) => v15,
         _ => panic!("Unsupported metadata version {:?}", meta.1),
     }
 }