State diff on root mismatch option

ipld/blockstore/Cargo.toml

@@ -8,10 +8,13 @@ edition = "2018"
 cid = { package = "forest_cid", path = "../cid" }
 db = { path = "../../node/db" }
 encoding = { package = "forest_encoding", path = "../../encoding" }
-forest_ipld = { path = "../" }
-commcid = { path = "../../utils/commcid", optional = true }
+forest_ipld = { path = "../", optional = true }
+
+[dev-dependencies]
+commcid = { path = "../../utils/commcid" }
 
 [features]
 rocksdb = ["db/rocksdb"]
-buffered = ["commcid"]
-tracking = []
+buffered = ["forest_ipld"]
+tracking = []
+resolve = ["forest_ipld"]

ipld/blockstore/src/buffered.rs

@@ -4,11 +4,7 @@
 #![cfg(feature = "buffered")]
 
 use super::BlockStore;
-use cid::{
-    multihash::{Code, MultihashDigest},
-    Cid,
-};
-use commcid::{POSEIDON_BLS12_381_A1_FC1, SHA2_256_TRUNC254_PADDED};
+use cid::{multihash::MultihashDigest, Cid, Codec};
 use db::{Error, Store};
 use encoding::{from_slice, ser::Serialize, to_vec};
 use forest_ipld::Ipld;
@@ -55,8 +51,7 @@ where
     BS: BlockStore,
 {
     // Skip identity and Filecoin commitment Cids
-    let ch = cid.hash.algorithm();
-    if ch == Code::Identity || ch == SHA2_256_TRUNC254_PADDED || ch == POSEIDON_BLS12_381_A1_FC1 {
+    if cid.codec != Codec::DagCBOR {
         return Ok(());
     }
 
@@ -186,6 +181,7 @@ mod tests {
     use cid::multihash::{Blake2b256, Identity};
     use cid::Codec;
     use commcid::commitment_to_cid;
+    use commcid::{POSEIDON_BLS12_381_A1_FC1, SHA2_256_TRUNC254_PADDED};
     use forest_ipld::{ipld, Ipld};
 
     #[test]
@@ -210,7 +206,7 @@ mod tests {
         let str_val = "value";
         let value = 8u8;
         let arr_cid = buf_store.put(&(str_val, value), Blake2b256).unwrap();
-        let identity_cid = buf_store.put(&0u8, Identity).unwrap();
+        let identity_cid = Cid::new_v1(Codec::Raw, Identity::digest(&[0u8]));
 
         // Create map to insert into store
         let sealed_comm_cid = commitment_to_cid(

ipld/blockstore/src/lib.rs

@@ -3,6 +3,8 @@
 
 #[cfg(feature = "buffered")]
 mod buffered;
+#[cfg(feature = "resolve")]
+pub mod resolve;
 #[cfg(feature = "tracking")]
 mod tracking;
 

ipld/blockstore/src/resolve.rs

@@ -0,0 +1,56 @@
+// Copyright 2020 ChainSafe Systems
+// SPDX-License-Identifier: Apache-2.0, MIT
+
+use super::BlockStore;
+use cid::{Cid, Codec};
+use forest_ipld::Ipld;
+use std::error::Error as StdError;
+
+/// Resolves link to recursively resolved Ipld with no hash links.
+pub fn resolve_cids_recursive<BS>(bs: &BS, cid: &Cid) -> Result<Ipld, Box<dyn StdError>>
+where
+    BS: BlockStore,
+{
+    let mut ipld = bs
+        .get(cid)?
+        .ok_or_else(|| "Cid does not exist in blockstore")?;
+
+    resolve_ipld(bs, &mut ipld)?;
+
+    Ok(ipld)
+}
+
+/// Resolves Ipld links recursively, building an Ipld structure with no hash links.
+pub fn resolve_ipld<BS>(bs: &BS, ipld: &mut Ipld) -> Result<(), Box<dyn StdError>>
+where
+    BS: BlockStore,
+{
+    match ipld {
+        Ipld::Map(m) => {
+            for (_, v) in m.iter_mut() {
+                resolve_ipld(bs, v)?;
+            }
+        }
+        Ipld::List(list) => {
+            for v in list.iter_mut() {
+                resolve_ipld(bs, v)?;
+            }
+        }
+        link @ Ipld::Link(_) => {
+            let resolved: Option<Ipld> = if let Ipld::Link(cid) = link {
+                if cid.codec != Codec::DagCBOR {
+                    return Ok(());
+                }
+                bs.get(cid)?
+            } else {
+                unreachable!()
+            };
+
+            if let Some(ipld) = resolved {
+                *link = ipld;
+            }
+        }
+        _ => (),
+    }
+    Ok(())
+}

ipld/hamt/Cargo.toml

@@ -20,6 +20,8 @@ lazycell = "1.2.1"
 [features]
 identity = []
 go-interop = []
+# This feature should just be used for testing (ignoring links that don't exist in store)
+ignore-dead-links = []
 
 [dev-dependencies]
 hex = "0.4.2"

ipld/hamt/src/node.rs

@@ -132,9 +132,15 @@ where
                     if let Some(cached_node) = cache.borrow() {
                         cached_node.for_each(store, f)?
                     } else {
-                        let node = store
-                            .get(cid)?
-                            .ok_or_else(|| format!("Node with cid {} not found", cid))?;
+                        let node = if let Some(node) = store.get(cid)? {
+                            node
+                        } else {
+                            #[cfg(not(feature = "ignore-dead-links"))]
+                            return Err(Error::CidNotFound(cid.to_string()).into());
+
+                            #[cfg(feature = "ignore-dead-links")]
+                            continue;
+                        };
 
                         // Ignore error intentionally, the cache value will always be the same
                         let _ = cache.fill(node);
@@ -194,10 +200,15 @@ where
                     // Link node is cached
                     cached_node.get_value(hashed_key, bit_width, depth + 1, key, store)
                 } else {
-                    // Link is not cached, need to load and fill cache, then traverse for value.
-                    let node = store
-                        .get::<Box<Node<K, V, H>>>(cid)?
-                        .ok_or_else(|| Error::CidNotFound(cid.to_string()))?;
+                    let node: Box<Node<K, V, H>> = if let Some(node) = store.get(cid)? {
+                        node
+                    } else {
+                        #[cfg(not(feature = "ignore-dead-links"))]
+                        return Err(Error::CidNotFound(cid.to_string()).into());
+
+                        #[cfg(feature = "ignore-dead-links")]
+                        return Ok(None);
+                    };
 
                     // Intentionally ignoring error, cache will always be the same.
                     let _ = cache.fill(node);

tests/conformance_tests/Cargo.toml

@@ -43,7 +43,7 @@ state_manager = { path = "../../blockchain/state_manager", optional = true }
 interpreter = { path = "../../vm/interpreter/", optional = true }
 num-bigint = { path = "../../utils/bigint", package = "forest_bigint", optional = true }
 vm = { package = "forest_vm", path = "../../vm", optional = true }
-blockstore = { package = "ipld_blockstore", path = "../../ipld/blockstore/", optional = true }
+blockstore = { package = "ipld_blockstore", path = "../../ipld/blockstore/", features = ["resolve"], optional = true }
 crypto = { package = "forest_crypto", path = "../../crypto", optional = true }
 encoding = { package = "forest_encoding", path = "../../encoding", optional = true }
 fil_types = { path = "../../types", optional = true }
@@ -57,4 +57,8 @@ forest_car = { path = "../../ipld/car" }
 flate2 = "1.0"
 lazy_static = "1.4"
 pretty_env_logger = "0.4.0"
+difference = "2.0"
+colored = "2.0"
+ipld = { package = "forest_ipld", path = "../../ipld", features = ["json"] }
+ipld_hamt = { path = "../../ipld/hamt", features = ["ignore-dead-links"] }
 log = "0.4"

tests/conformance_tests/src/lib.rs

@@ -20,7 +20,6 @@ use encoding::Cbor;
 use fil_types::{SealVerifyInfo, WindowPoStVerifyInfo};
 use forest_message::{ChainMessage, Message, MessageReceipt, SignedMessage, UnsignedMessage};
 use interpreter::{ApplyRet, BlockMessages, Rand, VM};
-use num_bigint::BigInt;
 use runtime::{ConsensusFault, Syscalls};
 use serde::{Deserialize, Deserializer};
 use std::error::Error as StdError;
@@ -112,7 +111,7 @@ pub struct PreConditions {
     pub epoch: ChainEpoch,
     pub state_tree: StateTreeVector,
     #[serde(default)]
-    pub basefee: Option<u64>,
+    pub basefee: Option<f64>,
 }
 
 #[derive(Debug, Deserialize)]

tests/conformance_tests/src/message.rs

@@ -17,17 +17,10 @@ pub fn execute_message(
     msg: &ChainMessage,
     pre_root: &Cid,
     epoch: ChainEpoch,
-    basefee: u64,
+    basefee: TokenAmount,
     selector: &Option<Selector>,
 ) -> Result<(ApplyRet, Cid), Box<dyn StdError>> {
-    let mut vm = VM::<_, _, _>::new(
-        pre_root,
-        bs,
-        epoch,
-        TestSyscalls,
-        &TestRand,
-        TokenAmount::from(basefee),
-    )?;
+    let mut vm = VM::<_, _, _>::new(pre_root, bs, epoch, TestSyscalls, &TestRand, basefee)?;
 
     if let Some(s) = &selector {
         if s.chaos_actor

tests/conformance_tests/src/tipset.rs

@@ -3,6 +3,7 @@
 
 use super::*;
 use fil_types::verifier::MockVerifier;
+use num_bigint::ToBigInt;
 use state_manager::StateManager;
 use std::sync::Arc;
 
@@ -56,7 +57,7 @@ mod block_messages_json {
 #[derive(Debug, Deserialize)]
 pub struct TipsetVector {
     pub epoch: ChainEpoch,
-    pub basefee: u64,
+    pub basefee: f64,
     #[serde(with = "block_messages_json")]
     pub blocks: Vec<BlockMessages>,
 }
@@ -83,7 +84,7 @@ pub fn execute_tipset(
         &tipset.blocks,
         tipset.epoch,
         &TestRand,
-        BigInt::from(tipset.basefee),
+        tipset.basefee.to_bigint().unwrap_or_default(),
         Some(|_, msg: &ChainMessage, ret| {
             _applied_messages.push(msg.clone());
             applied_results.push(ret);