diff --git a/.circleci/config.yml b/.circleci/config.yml index 010dfe4dad82..1db62477d18e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -126,12 +126,18 @@ jobs: steps: - build_setup - restore_cargo_package_cache - - run: - name: Build Unit Tests - command: make test-all-no-run - run: name: Run Unit Tests - command: make test-all + command: make test + test-vectors: + executor: test-executor + description: Run serialization and conformance tests + steps: + - build_setup + - restore_cargo_package_cache + - run: + name: Run test vectors + command: make run-vectors install: executor: test-executor description: Install forest binary @@ -152,3 +158,6 @@ workflows: - test: requires: - prefetch-crates + - test-vectors: + requires: + - prefetch-crates diff --git a/.github/workflows/ci-rust.yml b/.github/workflows/ci-rust.yml index 5bb1c4962368..1a218f3c40e9 100644 --- a/.github/workflows/ci-rust.yml +++ b/.github/workflows/ci-rust.yml @@ -32,12 +32,15 @@ jobs: profile: minimal toolchain: stable override: true + + - name: Pull submodules + run: git submodule update --init - - name: Cargo Build Tests - run: make test-all-no-run + - name: Run all unit tests + run: make test - - name: Cargo test all - run: make test-all + - name: Run test vectors + run: make run-vectors fmt: name: rustfmt diff --git a/.gitmodules b/.gitmodules index 3241b8730507..ec1dc40d31b8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -8,3 +8,6 @@ [submodule "ipld/tests/ipld-traversal-vectors"] path = ipld/tests/ipld-traversal-vectors url = git@github.com:ChainSafe/ipld-traversal-vectors.git +[submodule "tests/conformance_tests/test-vectors"] + path = tests/conformance_tests/test-vectors + url = https://github.com/filecoin-project/test-vectors.git diff --git a/Cargo.lock b/Cargo.lock index 9a28154d0661..c88580220261 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1388,6 +1388,32 @@ dependencies = [ "toml", ] +[[package]] +name = "conformance_tests" +version = "0.1.0" +dependencies = [ + "base64 0.12.3", + "clock", + "db", + "fil_types", + "flate2", + "forest_address", + "forest_car", + "forest_cid", + "forest_crypto", + "forest_encoding", + "forest_message", + "forest_vm", + "interpreter", + "ipld_blockstore", + "lazy_static", + "regex", + "runtime", + "serde", + "serde_json", + "walkdir", +] + [[package]] name = "console" version = "0.11.3" diff --git a/Cargo.toml b/Cargo.toml index db29f8b6a97c..ea040d8b000c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ members = [ "ipld/graphsync", "utils/bigint", "tests/serialization_tests", + "tests/conformance_tests", "utils/bitfield", "utils/test_utils", "utils/commcid", diff --git a/Makefile b/Makefile index 9154084871e4..6686d12b321d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ SER_TESTS = "tests/serialization_tests" +CONF_TESTS = "tests/conformance_tests" install: cargo install --path forest --force @@ -53,14 +54,19 @@ release: pull-serialization-tests: git submodule update --init -run-vectors: +run-serialization-vectors: cargo test --release --manifest-path=$(SER_TESTS)/Cargo.toml --features "submodule_tests" +run-conformance-vectors: + cargo test --release --manifest-path=$(CONF_TESTS)/Cargo.toml --features "submodule_tests" + +run-vectors: run-serialization-vectors run-conformance-vectors + test-vectors: pull-serialization-tests run-vectors # Test all without the submodule test vectors with release configuration test: - cargo test --all --exclude serialization_tests + cargo test --all --all-features --exclude serialization_tests --exclude conformance_tests # This will run all tests will all features enabled, which will exclude some tests with # specific features disabled diff --git a/README.md b/README.md index 9442590e0550..3a9379a23dfd 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ Will show all debug logs by default, but the `forest_libp2p::service` logs will # To run base tests cargo test # add --release flag for longer compilation but faster execution -# To pull serialization vectors submodule and run serialization tests +# To pull serialization vectors submodule and run serialization and conformance tests make test-vectors # To run all tests and all features enabled diff --git a/blockchain/state_manager/Cargo.toml b/blockchain/state_manager/Cargo.toml index d479aa256a41..f92438156d05 100644 --- a/blockchain/state_manager/Cargo.toml +++ b/blockchain/state_manager/Cargo.toml @@ -12,7 +12,7 @@ db = { path = "../../node/db/" } encoding = { package = "forest_encoding", path = "../../encoding/" } num-bigint = { path = "../../utils/bigint", package = "forest_bigint" } state_tree = { path = "../../vm/state_tree/" } -blockstore = { package = "ipld_blockstore", path = "../../ipld/blockstore/" } +blockstore = { package = "ipld_blockstore", path = "../../ipld/blockstore/", features = ["buffered"] } forest_blocks = { path = "../../blockchain/blocks" } thiserror = "1.0" interpreter = { path = "../../vm/interpreter/" } diff --git a/blockchain/state_manager/src/lib.rs b/blockchain/state_manager/src/lib.rs index 05ea7db4db16..694d7f9a1c92 100644 --- a/blockchain/state_manager/src/lib.rs +++ b/blockchain/state_manager/src/lib.rs @@ -18,7 +18,6 @@ use cid::Cid; use clock::ChainEpoch; use encoding::de::DeserializeOwned; use encoding::Cbor; -use fil_types::DevnetParams; use flo_stream::Subscriber; use forest_blocks::{Block, BlockHeader, FullTipset, Tipset, TipsetKeys}; use futures::channel::oneshot; @@ -180,7 +179,7 @@ where let mut buf_store = BufferedBlockStore::new(self.bs.as_ref()); // TODO possibly switch out syscalls to be saved at state manager level // TODO change from statically using devnet params when needed - let mut vm = VM::<_, _, DevnetParams>::new( + let mut vm = VM::<_, _, _>::new( ts.parent_state(), &buf_store, ts.epoch(), @@ -256,7 +255,7 @@ where span!("state_call_raw", { let block_store = self.get_block_store_ref(); let buf_store = BufferedBlockStore::new(block_store); - let mut vm = VM::<_, _, DevnetParams>::new( + let mut vm = VM::<_, _, _>::new( bstate, &buf_store, *bheight, @@ -331,7 +330,7 @@ where .map_err(|_| Error::Other("Could not load tipset state".to_string()))?; let chain_rand = ChainRand::new(ts.key().to_owned()); - let mut vm = VM::<_, _, DevnetParams>::new( + let mut vm = VM::<_, _, _>::new( &st, self.bs.as_ref(), ts.epoch() + 1, diff --git a/ipld/blockstore/Cargo.toml b/ipld/blockstore/Cargo.toml index b499d691b543..9e23844e3659 100644 --- a/ipld/blockstore/Cargo.toml +++ b/ipld/blockstore/Cargo.toml @@ -9,7 +9,8 @@ cid = { package = "forest_cid", path = "../cid" } db = { path = "../../node/db" } encoding = { package = "forest_encoding", path = "../../encoding" } forest_ipld = { path = "../" } -commcid = { path = "../../utils/commcid" } +commcid = { path = "../../utils/commcid", optional = true } [features] rocksdb = ["db/rocksdb"] +buffered = ["commcid"] \ No newline at end of file diff --git a/ipld/blockstore/src/buffered.rs b/ipld/blockstore/src/buffered.rs index b68dae2e0309..ba4deae4a431 100644 --- a/ipld/blockstore/src/buffered.rs +++ b/ipld/blockstore/src/buffered.rs @@ -1,6 +1,8 @@ // Copyright 2020 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT +#![cfg(feature = "buffered")] + use super::BlockStore; use cid::{ multihash::{Code, MultihashDigest}, @@ -163,12 +165,12 @@ where { self.base.bulk_read(keys) } - fn bulk_write(&self, keys: &[K], values: &[V]) -> Result<(), Error> + fn bulk_write(&self, values: &[(K, V)]) -> Result<(), Error> where K: AsRef<[u8]>, V: AsRef<[u8]>, { - self.base.bulk_write(keys, values) + self.base.bulk_write(values) } fn bulk_delete(&self, keys: &[K]) -> Result<(), Error> where diff --git a/ipld/blockstore/src/lib.rs b/ipld/blockstore/src/lib.rs index 73d1a52d5ef6..3a9f694af219 100644 --- a/ipld/blockstore/src/lib.rs +++ b/ipld/blockstore/src/lib.rs @@ -3,6 +3,7 @@ mod buffered; +#[cfg(feature = "buffered")] pub use self::buffered::BufferedBlockStore; use cid::{multihash::MultihashDigest, Cid}; diff --git a/ipld/car/Cargo.toml b/ipld/car/Cargo.toml index 2fb6d1adafa9..240ead2de88f 100644 --- a/ipld/car/Cargo.toml +++ b/ipld/car/Cargo.toml @@ -5,7 +5,7 @@ authors = ["ChainSafe Systems "] edition = "2018" [dependencies] -unsigned-varint = "0.5" +unsigned-varint = { version = "0.5", features = ["futures-codec"] } cid = { package = "forest_cid", path = "../cid", features = ["cbor"] } forest_encoding = { path = "../../encoding" } blockstore = { package = "ipld_blockstore", path = "../blockstore" } diff --git a/ipld/car/src/lib.rs b/ipld/car/src/lib.rs index cf8806ba46ec..aadf2d1ec633 100644 --- a/ipld/car/src/lib.rs +++ b/ipld/car/src/lib.rs @@ -71,10 +71,19 @@ pub fn load_car( ) -> Result, Error> { let mut car_reader = CarReader::new(buf_reader)?; + // Batch write key value pairs from car file + let mut buf: Vec<(Vec, Vec)> = Vec::with_capacity(100); + // TODO revisit, seems possible buffer could be empty when underlying reader isn't while !car_reader.buf_reader.buffer().is_empty() { let block = car_reader.next_block()?; - s.write(block.cid.to_bytes(), block.data) - .map_err(|e| Error::Other(e.to_string()))?; + buf.push((block.cid.to_bytes(), block.data)); + if buf.len() > 1000 { + s.bulk_write(&buf) + .map_err(|e| Error::Other(e.to_string()))?; + buf.clear(); + } } + s.bulk_write(&buf) + .map_err(|e| Error::Other(e.to_string()))?; Ok(car_reader.header.roots) } diff --git a/ipld/car/src/util.rs b/ipld/car/src/util.rs index d1a6f7944743..42261dc498dd 100644 --- a/ipld/car/src/util.rs +++ b/ipld/car/src/util.rs @@ -5,11 +5,10 @@ use super::error::Error; use cid::Cid; use std::io::Read; -pub(crate) fn ld_read(mut buf_reader: &mut R) -> Result, Error> { - let l = - unsigned_varint::io::read_u64(&mut buf_reader).map_err(|e| Error::Other(e.to_string()))?; +pub(crate) fn ld_read(mut reader: &mut R) -> Result, Error> { + let l = unsigned_varint::io::read_u64(&mut reader).map_err(|e| Error::Other(e.to_string()))?; let mut buf = Vec::with_capacity(l as usize); - buf_reader + reader .take(l) .read_to_end(&mut buf) .map_err(|e| Error::Other(e.to_string()))?; diff --git a/node/db/src/lib.rs b/node/db/src/lib.rs index d0ea6a67b376..68ea0e6fd3c9 100644 --- a/node/db/src/lib.rs +++ b/node/db/src/lib.rs @@ -49,16 +49,13 @@ pub trait Store { } /// Write slice of KV pairs. - fn bulk_write(&self, keys: &[K], values: &[V]) -> Result<(), Error> + fn bulk_write(&self, values: &[(K, V)]) -> Result<(), Error> where K: AsRef<[u8]>, V: AsRef<[u8]>, { - if keys.len() != values.len() { - return Err(Error::InvalidBulkLen); - } - keys.iter() - .zip(values) + values + .iter() .map(|(key, value)| self.write(key, value)) .collect() } diff --git a/node/db/src/rocks.rs b/node/db/src/rocks.rs index 0f08a21a4ff0..6817e72a838e 100644 --- a/node/db/src/rocks.rs +++ b/node/db/src/rocks.rs @@ -89,18 +89,13 @@ impl Store for RocksDb { Ok(self.db()?.delete(key)?) } - fn bulk_write(&self, keys: &[K], values: &[V]) -> Result<(), Error> + fn bulk_write(&self, values: &[(K, V)]) -> Result<(), Error> where K: AsRef<[u8]>, V: AsRef<[u8]>, { - // Safety check to make sure kv lengths are the same - if keys.len() != values.len() { - return Err(Error::InvalidBulkLen); - } - let mut batch = WriteBatch::default(); - for (k, v) in keys.iter().zip(values.iter()) { + for (k, v) in values { batch.put(k, v); } Ok(self.db()?.write(batch)?) diff --git a/node/db/tests/subtests/mod.rs b/node/db/tests/subtests/mod.rs index 94545e0a0513..9c939654263a 100644 --- a/node/db/tests/subtests/mod.rs +++ b/node/db/tests/subtests/mod.rs @@ -68,10 +68,9 @@ pub fn bulk_write(db: &DB) where DB: Store, { - let keys = [[0], [1], [2]]; - let values = [[0], [1], [2]]; - db.bulk_write(&keys, &values).unwrap(); - for k in keys.iter() { + let values = [([0], [0]), ([1], [1]), ([2], [2])]; + db.bulk_write(&values).unwrap(); + for (k, _) in values.iter() { let res = db.exists(k.clone()).unwrap(); assert_eq!(res, true); } @@ -83,7 +82,8 @@ where { let keys = [[0], [1], [2]]; let values = [[0], [1], [2]]; - db.bulk_write(&keys, &values).unwrap(); + let kvs: Vec<_> = keys.iter().zip(values.iter()).collect(); + db.bulk_write(&kvs).unwrap(); let results = db.bulk_read(&keys).unwrap(); for (result, value) in results.iter().zip(values.iter()) { match result { @@ -99,7 +99,8 @@ where { let keys = [[0], [1], [2]]; let values = [[0], [1], [2]]; - db.bulk_write(&keys, &values).unwrap(); + let kvs: Vec<_> = keys.iter().zip(values.iter()).collect(); + db.bulk_write(&kvs).unwrap(); db.bulk_delete(&keys).unwrap(); for k in keys.iter() { let res = db.exists(k.clone()).unwrap(); diff --git a/node/rpc/src/state_api.rs b/node/rpc/src/state_api.rs index 0f39dc93b818..891dda785e8e 100644 --- a/node/rpc/src/state_api.rs +++ b/node/rpc/src/state_api.rs @@ -16,7 +16,7 @@ use clock::ChainEpoch; use fil_types::SectorNumber; use jsonrpc_v2::{Data, Error as JsonRpcError, Params}; use message::{ - json::MessageReceiptJson, + message_receipt::json::MessageReceiptJson, unsigned_message::{json::UnsignedMessageJson, UnsignedMessage}, }; use serde::{Deserialize, Serialize}; diff --git a/tests/conformance_tests/Cargo.toml b/tests/conformance_tests/Cargo.toml new file mode 100644 index 000000000000..e55b076ea025 --- /dev/null +++ b/tests/conformance_tests/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "conformance_tests" +version = "0.1.0" +authors = ["ChainSafe Systems "] +edition = "2018" + +[features] +submodule_tests = [] + +[dependencies] + +[dev-dependencies] +base64 = { version = "0.12.1" } +walkdir = "2.3" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +clock = { path = "../../node/clock" } +cid = { package = "forest_cid", path = "../../ipld/cid", features = ["cbor", "json"] } +forest_message = { path = "../../vm/message", features = ["json"] } +vm = { package = "forest_vm", path = "../../vm" } +db = { path = "../../node/db/" } +blockstore = { package = "ipld_blockstore", path = "../../ipld/blockstore/" } +forest_car = { path = "../../ipld/car" } +flate2 = "1.0" +encoding = { package = "forest_encoding", path = "../../encoding" } +interpreter = { path = "../../vm/interpreter/" } +runtime = { path = "../../vm/runtime/" } +fil_types = { path = "../../types" } +crypto = { package = "forest_crypto", path = "../../crypto" } +address = { package = "forest_address", path = "../../vm/address" } +regex = "1.0" +lazy_static = "1.4" diff --git a/tests/conformance_tests/src/lib.rs b/tests/conformance_tests/src/lib.rs new file mode 100644 index 000000000000..ecc3668996b9 --- /dev/null +++ b/tests/conformance_tests/src/lib.rs @@ -0,0 +1,4 @@ +// Copyright 2020 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +#![cfg(feature = "submodule_tests")] diff --git a/tests/conformance_tests/test-vectors b/tests/conformance_tests/test-vectors new file mode 160000 index 000000000000..e3b6da3c2e3b --- /dev/null +++ b/tests/conformance_tests/test-vectors @@ -0,0 +1 @@ +Subproject commit e3b6da3c2e3b228515d69250999babbcd09198b4 diff --git a/tests/conformance_tests/tests/conformance_runner.rs b/tests/conformance_tests/tests/conformance_runner.rs new file mode 100644 index 000000000000..e620be83f428 --- /dev/null +++ b/tests/conformance_tests/tests/conformance_runner.rs @@ -0,0 +1,361 @@ +// Copyright 2020 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +#![cfg(feature = "submodule_tests")] + +#[macro_use] +extern crate lazy_static; + +use address::Address; +use blockstore::BlockStore; +use cid::Cid; +use clock::ChainEpoch; +use crypto::{DomainSeparationTag, Signature}; +use encoding::Cbor; +use fil_types::{SealVerifyInfo, WindowPoStVerifyInfo}; +use flate2::read::GzDecoder; +use forest_message::{MessageReceipt, UnsignedMessage}; +use interpreter::{ApplyRet, Rand, VM}; +use regex::Regex; +use runtime::{ConsensusFault, Syscalls}; +use serde::{Deserialize, Deserializer}; +use std::error::Error as StdError; +use std::fs::File; +use std::io::{BufReader, Read}; +use vm::{ExitCode, Serialized, TokenAmount}; +use walkdir::{DirEntry, WalkDir}; + +lazy_static! { + static ref SKIP_TESTS: [Regex; 7] = [ + Regex::new(r"actor_creation/.*").unwrap(), + Regex::new(r"msg_application/.*").unwrap(), + Regex::new(r"multisig/.*").unwrap(), + Regex::new(r"nested/.*").unwrap(), + Regex::new(r"paych/.*").unwrap(), + Regex::new(r"transfer/.*").unwrap(), + Regex::new(r"vm_violations/.*").unwrap(), + ]; + static ref BASE_FEE: TokenAmount = TokenAmount::from(100); +} + +mod base64_bytes { + use super::*; + use serde::de; + use std::borrow::Cow; + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let s: Cow<'de, str> = Deserialize::deserialize(deserializer)?; + Ok(base64::decode(s.as_ref()).map_err(de::Error::custom)?) + } +} + +mod message_receipt_vec { + use super::*; + + #[derive(Deserialize)] + struct MessageReceiptVector { + exit_code: ExitCode, + #[serde(rename = "return", with = "base64_bytes")] + return_value: Vec, + gas_used: i64, + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let s: Vec = Deserialize::deserialize(deserializer)?; + Ok(s.into_iter() + .map(|v| MessageReceipt { + exit_code: v.exit_code, + return_data: Serialized::new(v.return_value), + gas_used: v.gas_used, + }) + .collect()) + } +} + +#[derive(Debug, Deserialize)] +struct StateTreeVector { + #[serde(with = "cid::json")] + root_cid: Cid, +} + +#[derive(Debug, Deserialize)] +struct GenerationData { + #[serde(default)] + source: String, + #[serde(default)] + version: String, +} + +#[derive(Debug, Deserialize)] +struct MetaData { + id: String, + #[serde(default)] + version: String, + #[serde(default)] + description: String, + #[serde(default)] + comment: String, + gen: Vec, +} + +#[derive(Debug, Deserialize)] +struct PreConditions { + epoch: ChainEpoch, + state_tree: StateTreeVector, +} + +#[derive(Debug, Deserialize)] +struct PostConditions { + state_tree: StateTreeVector, + #[serde(with = "message_receipt_vec")] + receipts: Vec, +} + +#[derive(Debug, Deserialize)] +struct MessageVector { + #[serde(with = "base64_bytes")] + bytes: Vec, + #[serde(default)] + epoch: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(tag = "class")] +enum TestVector { + #[serde(rename = "message")] + Message { + selector: Option, + #[serde(rename = "_meta")] + meta: Option, + + #[serde(with = "base64_bytes")] + car: Vec, + preconditions: PreConditions, + apply_messages: Vec, + postconditions: PostConditions, + }, + #[serde(rename = "block")] + Block { + selector: Option, + #[serde(rename = "_meta")] + meta: Option, + }, + #[serde(rename = "tipset")] + Tipset { + selector: Option, + #[serde(rename = "_meta")] + meta: Option, + }, + #[serde(rename = "chain")] + Chain { + selector: Option, + #[serde(rename = "_meta")] + meta: Option, + }, +} + +fn is_valid_file(entry: &DirEntry) -> bool { + let file_name = match entry.path().to_str() { + Some(file) => file, + None => return false, + }; + for rx in SKIP_TESTS.iter() { + if rx.is_match(file_name) { + return false; + } + } + file_name.ends_with(".json") +} + +struct TestRand; +impl Rand for TestRand { + fn get_chain_randomness( + &self, + _: &DB, + _: DomainSeparationTag, + _: ChainEpoch, + _: &[u8], + ) -> Result<[u8; 32], Box> { + Ok(*b"i_am_random_____i_am_random_____") + } + fn get_beacon_randomness( + &self, + _: &DB, + _: DomainSeparationTag, + _: ChainEpoch, + _: &[u8], + ) -> Result<[u8; 32], Box> { + Ok(*b"i_am_random_____i_am_random_____") + } +} + +struct TestSyscalls; +impl Syscalls for TestSyscalls { + fn verify_signature( + &self, + _: &Signature, + _: &Address, + _: &[u8], + ) -> Result<(), Box> { + Ok(()) + } + fn verify_seal(&self, _: &SealVerifyInfo) -> Result<(), Box> { + Ok(()) + } + fn verify_post(&self, _: &WindowPoStVerifyInfo) -> Result<(), Box> { + Ok(()) + } + + // TODO check if this should be defaulted as well + fn verify_consensus_fault( + &self, + _: &[u8], + _: &[u8], + _: &[u8], + ) -> Result, Box> { + Ok(None) + } +} + +fn execute_message( + msg: &UnsignedMessage, + pre_root: &Cid, + bs: &db::MemoryDB, + epoch: ChainEpoch, +) -> Result<(ApplyRet, Cid), Box> { + let mut vm = VM::<_, _, _>::new( + pre_root, + bs, + epoch, + TestSyscalls, + &TestRand, + BASE_FEE.clone(), + )?; + + // TODO register puppet actor (and conditionally chaos actor) + + let ret = vm.apply_message(msg)?; + + let root = vm.flush()?; + Ok((ret, root)) +} + +fn execute_message_vector( + _selector: Option, + car: Vec, + preconditions: PreConditions, + apply_messages: Vec, + postconditions: PostConditions, +) -> Result<(), Box> { + let bs = db::MemoryDB::default(); + + let mut epoch = preconditions.epoch; + let mut root = preconditions.state_tree.root_cid; + + // Decode gzip bytes + let mut d = GzDecoder::new(car.as_slice()); + let mut decoded = Vec::new(); + d.read_to_end(&mut decoded)?; + + // Load car file with bytes + let reader = BufReader::new(decoded.as_slice()); + forest_car::load_car(&bs, reader)?; + + for (i, m) in apply_messages.iter().enumerate() { + let msg = UnsignedMessage::unmarshal_cbor(&m.bytes)?; + + if let Some(ep) = m.epoch { + epoch = ep; + } + + let (ret, post_root) = execute_message(&msg, &root, &bs, epoch)?; + root = post_root; + + let receipt = &postconditions.receipts[i]; + let (expected, actual) = (receipt.exit_code, ret.msg_receipt.exit_code); + if expected != actual { + return Err(format!( + "exit code of msg {} did not match; expected: {:?}, got {:?}", + i, expected, actual + ) + .into()); + } + + let (expected, actual) = (receipt.gas_used, ret.msg_receipt.gas_used); + if expected != actual { + return Err(format!( + "gas used of msg {} did not match; expected: {}, got {}", + i, expected, actual + ) + .into()); + } + } + + if root != postconditions.state_tree.root_cid { + return Err(format!( + "wrong post root cid; expected {}, but got {}", + postconditions.state_tree.root_cid, root + ) + .into()); + } + + Ok(()) +} + +#[test] +fn conformance_test_runner() { + let walker = WalkDir::new("test-vectors/corpus").into_iter(); + let mut failed = Vec::new(); + let mut succeeded = 0; + for entry in walker.filter_map(|e| e.ok()).filter(is_valid_file) { + let file = File::open(entry.path()).unwrap(); + let reader = BufReader::new(file); + let vector: TestVector = serde_json::from_reader(reader).unwrap(); + + match vector { + TestVector::Message { + selector, + meta, + car, + preconditions, + apply_messages, + postconditions, + } => { + if let Err(e) = execute_message_vector( + selector, + car, + preconditions, + apply_messages, + postconditions, + ) { + failed.push((entry.path().display().to_string(), meta, e)); + } else { + println!("{} succeeded", entry.path().display()); + succeeded += 1; + } + } + _ => panic!("Unsupported test vector class"), + } + } + + if !failed.is_empty() { + eprintln!( + "{}/{} tests failed:", + failed.len(), + failed.len() + succeeded + ); + for (path, meta, e) in failed { + eprintln!( + "file {} failed:\n\tMeta: {:?}\n\tError: {}\n", + path, meta, e + ); + } + panic!() + } +} diff --git a/vm/interpreter/src/default_runtime.rs b/vm/interpreter/src/default_runtime.rs index 24c47caf2e52..d8c428406ee3 100644 --- a/vm/interpreter/src/default_runtime.rs +++ b/vm/interpreter/src/default_runtime.rs @@ -4,14 +4,14 @@ use super::gas_block_store::GasBlockStore; use super::gas_syscalls::GasSyscalls; use super::gas_tracker::{price_list_by_epoch, GasTracker, PriceList}; -use super::ChainRand; +use super::Rand; use actor::*; use address::{Address, Protocol}; use byteorder::{BigEndian, WriteBytesExt}; use cid::{multihash::Blake2b256, Cid}; use clock::ChainEpoch; use crypto::DomainSeparationTag; -use fil_types::NetworkParams; +use fil_types::{DevnetParams, NetworkParams}; use forest_encoding::Cbor; use forest_encoding::{error::Error as EncodingError, to_vec}; use ipld_blockstore::BlockStore; @@ -50,7 +50,7 @@ impl MessageInfo for VMMsg { } /// Implementation of the Runtime trait. -pub struct DefaultRuntime<'db, 'msg, 'st, 'sys, 'r, BS, SYS, P> { +pub struct DefaultRuntime<'db, 'msg, 'st, 'sys, 'r, BS, SYS, R, P = DevnetParams> { state: &'st mut StateTree<'db, BS>, store: GasBlockStore<'db, BS>, syscalls: GasSyscalls<'sys, SYS>, @@ -62,17 +62,19 @@ pub struct DefaultRuntime<'db, 'msg, 'st, 'sys, 'r, BS, SYS, P> { origin_nonce: u64, num_actors_created: u64, price_list: PriceList, - rand: &'r ChainRand, + rand: &'r R, caller_validated: bool, allow_internal: bool, params: PhantomData

, } -impl<'db, 'msg, 'st, 'sys, 'r, BS, SYS, P> DefaultRuntime<'db, 'msg, 'st, 'sys, 'r, BS, SYS, P> +impl<'db, 'msg, 'st, 'sys, 'r, BS, SYS, R, P> + DefaultRuntime<'db, 'msg, 'st, 'sys, 'r, BS, SYS, R, P> where BS: BlockStore, SYS: Syscalls, P: NetworkParams, + R: Rand, { /// Constructs a new Runtime #[allow(clippy::too_many_arguments)] @@ -86,7 +88,7 @@ where origin: Address, origin_nonce: u64, num_actors_created: u64, - rand: &'r ChainRand, + rand: &'r R, ) -> Result { let price_list = price_list_by_epoch(epoch); let gas_tracker = Rc::new(RefCell::new(GasTracker::new(message.gas_limit(), gas_used))); @@ -248,7 +250,7 @@ where .snapshot() .map_err(|e| actor_error!(fatal("failed to create snapshot {}", e)))?; - let send_res = vm_send::(self, &msg, None); + let send_res = vm_send::(self, &msg, None); send_res.map_err(|e| { if let Err(e) = self.state.revert_to_snapshot(&snapshot) { actor_error!(fatal("failed to revert snapshot: {}", e)) @@ -298,11 +300,12 @@ where } } -impl Runtime for DefaultRuntime<'_, '_, '_, '_, '_, BS, SYS, P> +impl Runtime for DefaultRuntime<'_, '_, '_, '_, '_, BS, SYS, R, P> where BS: BlockStore, SYS: Syscalls, P: NetworkParams, + R: Rand, { fn message(&self) -> &dyn MessageInfo { &self.vm_msg @@ -419,10 +422,10 @@ where }) } - fn transaction(&mut self, f: F) -> Result + fn transaction(&mut self, f: F) -> Result where C: Cbor, - F: FnOnce(&mut C, &mut Self) -> R, + F: FnOnce(&mut C, &mut Self) -> RT, { // get actor let act = self.state.get_actor(self.message().receiver()) @@ -590,8 +593,8 @@ where /// Shared logic between the DefaultRuntime and the Interpreter. /// It invokes methods on different Actors based on the Message. -pub fn vm_send<'db, 'msg, 'st, 'sys, 'r, BS, SYS, P>( - rt: &mut DefaultRuntime<'db, 'msg, 'st, 'sys, 'r, BS, SYS, P>, +pub fn vm_send<'db, 'msg, 'st, 'sys, 'r, BS, SYS, R, P>( + rt: &mut DefaultRuntime<'db, 'msg, 'st, 'sys, 'r, BS, SYS, R, P>, msg: &UnsignedMessage, gas_cost: Option, ) -> Result @@ -599,6 +602,7 @@ where BS: BlockStore, SYS: Syscalls, P: NetworkParams, + R: Rand, { if let Some(cost) = gas_cost { rt.charge_gas(cost)?; @@ -707,8 +711,8 @@ fn transfer( } /// Calls actor code with method and parameters. -fn invoke<'db, 'msg, 'st, 'sys, 'r, BS, SYS, P>( - rt: &mut DefaultRuntime<'db, 'msg, 'st, 'sys, 'r, BS, SYS, P>, +fn invoke<'db, 'msg, 'st, 'sys, 'r, BS, SYS, R, P>( + rt: &mut DefaultRuntime<'db, 'msg, 'st, 'sys, 'r, BS, SYS, R, P>, code: Cid, method_num: MethodNum, params: &Serialized, @@ -718,6 +722,7 @@ where BS: BlockStore, SYS: Syscalls, P: NetworkParams, + R: Rand, { match code { x if x == *SYSTEM_ACTOR_CODE_ID => system::Actor.invoke_method(rt, method_num, params), diff --git a/vm/interpreter/src/gas_block_store.rs b/vm/interpreter/src/gas_block_store.rs index 797e275b466f..57be716109a8 100644 --- a/vm/interpreter/src/gas_block_store.rs +++ b/vm/interpreter/src/gas_block_store.rs @@ -99,12 +99,12 @@ where { self.store.bulk_read(keys) } - fn bulk_write(&self, keys: &[K], values: &[V]) -> Result<(), Error> + fn bulk_write(&self, values: &[(K, V)]) -> Result<(), Error> where K: AsRef<[u8]>, V: AsRef<[u8]>, { - self.store.bulk_write(keys, values) + self.store.bulk_write(values) } fn bulk_delete(&self, keys: &[K]) -> Result<(), Error> where diff --git a/vm/interpreter/src/rand.rs b/vm/interpreter/src/rand.rs index bcdd96c2a053..7414c5adf6b2 100644 --- a/vm/interpreter/src/rand.rs +++ b/vm/interpreter/src/rand.rs @@ -7,6 +7,28 @@ use crypto::DomainSeparationTag; use ipld_blockstore::BlockStore; use std::error::Error; +/// Randomness provider trait +pub trait Rand { + /// Gets 32 bytes of randomness for ChainRand paramaterized by the DomainSeparationTag, + /// ChainEpoch, Entropy from the ticket chain. + fn get_chain_randomness( + &self, + db: &DB, + pers: DomainSeparationTag, + round: ChainEpoch, + entropy: &[u8], + ) -> Result<[u8; 32], Box>; + /// Gets 32 bytes of randomness for ChainRand paramaterized by the DomainSeparationTag, + /// ChainEpoch, Entropy from the latest beacon entry. + fn get_beacon_randomness( + &self, + db: &DB, + pers: DomainSeparationTag, + round: ChainEpoch, + entropy: &[u8], + ) -> Result<[u8; 32], Box>; +} + /// Allows for deriving the randomness from a particular tipset #[derive(Debug, Clone)] pub struct ChainRand { @@ -18,10 +40,10 @@ impl ChainRand { pub fn new(blks: TipsetKeys) -> Self { Self { blks } } +} - /// Gets 32 bytes of randomness paramaterized by the DomainSeparationTag, ChainEpoch, - /// Entropy, and Tipset. The randomness is gathered from the ticket chain. - pub fn get_chain_randomness( +impl Rand for ChainRand { + fn get_chain_randomness( &self, db: &DB, pers: DomainSeparationTag, @@ -31,9 +53,7 @@ impl ChainRand { chain::get_chain_randomness(db, &self.blks, pers, round, entropy) } - /// Gets 32 bytes of randomness paramaterized by the DomainSeparationTag, ChainEpoch, - /// Entropy, and Tipset. This randomness is drawn from the latest beacon entry. - pub fn get_beacon_randomness( + fn get_beacon_randomness( &self, db: &DB, pers: DomainSeparationTag, diff --git a/vm/interpreter/src/vm.rs b/vm/interpreter/src/vm.rs index 952768ae60e3..4e96980e1f12 100644 --- a/vm/interpreter/src/vm.rs +++ b/vm/interpreter/src/vm.rs @@ -1,7 +1,7 @@ // Copyright 2020 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT -use super::{gas_tracker::price_list_by_epoch, vm_send, ChainRand, DefaultRuntime}; +use super::{gas_tracker::price_list_by_epoch, vm_send, DefaultRuntime, Rand}; use actor::{ cron, reward, ACCOUNT_ACTOR_CODE_ID, BURNT_FUNDS_ACTOR_ADDR, CRON_ACTOR_ADDR, REWARD_ACTOR_ADDR, SYSTEM_ACTOR_ADDR, @@ -9,7 +9,7 @@ use actor::{ use blocks::FullTipset; use cid::Cid; use clock::ChainEpoch; -use fil_types::NetworkParams; +use fil_types::{DevnetParams, NetworkParams}; use forest_encoding::Cbor; use ipld_blockstore::BlockStore; use log::warn; @@ -29,28 +29,29 @@ const GAS_OVERUSE_DENOM: i64 = 10; /// Interpreter which handles execution of state transitioning messages and returns receipts /// from the vm execution. -pub struct VM<'db, 'r, DB, SYS, P> { +pub struct VM<'db, 'r, DB, SYS, R, P = DevnetParams> { state: StateTree<'db, DB>, store: &'db DB, epoch: ChainEpoch, syscalls: SYS, - rand: &'r ChainRand, + rand: &'r R, base_fee: BigInt, params: PhantomData

, } -impl<'db, 'r, DB, SYS, P> VM<'db, 'r, DB, SYS, P> +impl<'db, 'r, DB, SYS, R, P> VM<'db, 'r, DB, SYS, R, P> where DB: BlockStore, SYS: Syscalls, P: NetworkParams, + R: Rand, { pub fn new( root: &Cid, store: &'db DB, epoch: ChainEpoch, syscalls: SYS, - rand: &'r ChainRand, + rand: &'r R, base_fee: BigInt, ) -> Result { let state = StateTree::new_from_root(store, root)?; @@ -405,13 +406,14 @@ where }) } /// Instantiates a new Runtime, and calls internal_send to do the execution. + #[allow(clippy::type_complexity)] fn send<'m>( &mut self, msg: &'m UnsignedMessage, gas_cost: Option, ) -> ( Serialized, - Option>, + Option>, Option, ) { let res = DefaultRuntime::new( diff --git a/vm/interpreter/tests/transfer_test.rs b/vm/interpreter/tests/transfer_test.rs index 388b4e1a41f3..b0f0120c507c 100644 --- a/vm/interpreter/tests/transfer_test.rs +++ b/vm/interpreter/tests/transfer_test.rs @@ -6,7 +6,6 @@ use address::Address; use blocks::TipsetKeys; use cid::multihash::{Blake2b256, Identity}; use db::MemoryDB; -use fil_types::DevnetParams; use interpreter::{vm_send, ChainRand, DefaultRuntime, DefaultSyscalls}; use ipld_blockstore::BlockStore; use ipld_hamt::Hamt; @@ -94,7 +93,7 @@ fn transfer_test() { let default_syscalls = DefaultSyscalls::new(&store); let dummy_rand = ChainRand::new(TipsetKeys::new(vec![])); - let mut runtime = DefaultRuntime::<_, _, DevnetParams>::new( + let mut runtime = DefaultRuntime::<_, _, _>::new( &mut state, &store, &default_syscalls, diff --git a/vm/message/src/lib.rs b/vm/message/src/lib.rs index cfadccb9c02a..5282a8f01ae8 100644 --- a/vm/message/src/lib.rs +++ b/vm/message/src/lib.rs @@ -5,12 +5,12 @@ extern crate serde; pub mod chain_message; -mod message_receipt; +pub mod message_receipt; pub mod signed_message; pub mod unsigned_message; pub use chain_message::ChainMessage; -pub use message_receipt::*; +pub use message_receipt::MessageReceipt; pub use signed_message::SignedMessage; pub use unsigned_message::UnsignedMessage; diff --git a/vm/message/src/message_receipt.rs b/vm/message/src/message_receipt.rs index 312db0d17802..9f7d9ecda4f8 100644 --- a/vm/message/src/message_receipt.rs +++ b/vm/message/src/message_receipt.rs @@ -5,7 +5,7 @@ use encoding::tuple::*; use vm::{ExitCode, Serialized}; /// Result of a state transition from a message -#[derive(PartialEq, Clone, Serialize_tuple, Deserialize_tuple)] +#[derive(Debug, PartialEq, Clone, Serialize_tuple, Deserialize_tuple)] pub struct MessageReceipt { pub exit_code: ExitCode, pub return_data: Serialized, @@ -85,4 +85,27 @@ pub mod json { gas_used, }) } + pub mod vec { + use super::*; + use forest_json_utils::GoVecVisitor; + use serde::ser::SerializeSeq; + + pub fn serialize(m: &[MessageReceipt], serializer: S) -> Result + where + S: Serializer, + { + let mut seq = serializer.serialize_seq(Some(m.len()))?; + for e in m { + seq.serialize_element(&MessageReceiptJsonRef(e))?; + } + seq.end() + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + deserializer.deserialize_any(GoVecVisitor::::new()) + } + } }