diff --git a/zebra-chain/src/sapling/tree.rs b/zebra-chain/src/sapling/tree.rs index f946617bc73..57e551a5115 100644 --- a/zebra-chain/src/sapling/tree.rs +++ b/zebra-chain/src/sapling/tree.rs @@ -169,9 +169,15 @@ impl ZcashDeserialize for Root { /// /// Note that it's handled as a byte buffer and not a point coordinate (jubjub::Fq) /// because that's how the spec handles the MerkleCRH^Sapling function inputs and outputs. -#[derive(Copy, Clone, Debug, Eq, PartialEq)] +#[derive(Copy, Clone, Eq, PartialEq)] struct Node([u8; 32]); +impl fmt::Debug for Node { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_tuple("Node").field(&hex::encode(self.0)).finish() + } +} + /// Required to convert [`NoteCommitmentTree`] into [`SerializedTree`]. /// /// Zebra stores Sapling note commitment trees as [`Frontier`][1]s while the diff --git a/zebra-chain/src/sprout/tree.rs b/zebra-chain/src/sprout/tree.rs index 5274790738f..099273ebf40 100644 --- a/zebra-chain/src/sprout/tree.rs +++ b/zebra-chain/src/sprout/tree.rs @@ -15,13 +15,13 @@ use std::{fmt, ops::Deref}; use byteorder::{BigEndian, ByteOrder}; use incrementalmerkletree::{bridgetree, Frontier}; use lazy_static::lazy_static; +use sha2::digest::generic_array::GenericArray; use thiserror::Error; use super::commitment::NoteCommitment; #[cfg(any(test, feature = "proptest-impl"))] use proptest_derive::Arbitrary; -use sha2::digest::generic_array::GenericArray; /// Sprout note commitment trees have a max depth of 29. /// @@ -127,9 +127,15 @@ impl From<&Root> for [u8; 32] { } /// A node of the Sprout note commitment tree. -#[derive(Clone, Debug)] +#[derive(Clone, Copy, Eq, PartialEq)] struct Node([u8; 32]); +impl fmt::Debug for Node { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_tuple("Node").field(&hex::encode(self.0)).finish() + } +} + impl incrementalmerkletree::Hashable for Node { /// Returns an empty leaf. fn empty_leaf() -> Self { diff --git a/zebra-state/src/service/non_finalized_state.rs b/zebra-state/src/service/non_finalized_state.rs index 3e343da7405..27bc7a15ef3 100644 --- a/zebra-state/src/service/non_finalized_state.rs +++ b/zebra-state/src/service/non_finalized_state.rs @@ -10,10 +10,8 @@ use std::{ use zebra_chain::{ block::{self, Block}, - history_tree::HistoryTree, - orchard, parameters::Network, - sapling, sprout, transparent, + sprout, transparent, }; use crate::{ @@ -157,13 +155,7 @@ impl NonFinalizedState { let parent_hash = prepared.block.header.previous_block_hash; let (height, hash) = (prepared.height, prepared.hash); - let parent_chain = self.parent_chain( - parent_hash, - finalized_state.sprout_note_commitment_tree(), - finalized_state.sapling_note_commitment_tree(), - finalized_state.orchard_note_commitment_tree(), - finalized_state.history_tree(), - )?; + let parent_chain = self.parent_chain(parent_hash)?; // If the block is invalid, return the error, // and drop the cloned parent Arc, or newly created chain fork. @@ -185,13 +177,23 @@ impl NonFinalizedState { /// Commit block to the non-finalized state as a new chain where its parent /// is the finalized tip. #[tracing::instrument(level = "debug", skip(self, finalized_state, prepared))] + #[allow(clippy::unwrap_in_result)] pub fn commit_new_chain( &mut self, prepared: PreparedBlock, finalized_state: &ZebraDb, ) -> Result<(), ValidateContextError> { + let finalized_tip_height = finalized_state.finalized_tip_height(); + + // TODO: fix tests that don't initialize the finalized state + #[cfg(not(test))] + let finalized_tip_height = finalized_tip_height.expect("finalized state contains blocks"); + #[cfg(test)] + let finalized_tip_height = finalized_tip_height.unwrap_or(zebra_chain::block::Height(0)); + let chain = Chain::new( self.network, + finalized_tip_height, finalized_state.sprout_note_commitment_tree(), finalized_state.sapling_note_commitment_tree(), finalized_state.orchard_note_commitment_tree(), @@ -279,7 +281,7 @@ impl NonFinalizedState { // Clone function arguments for different threads let block = contextual.block.clone(); let network = new_chain.network(); - let history_tree = new_chain.history_tree.clone(); + let history_tree = new_chain.history_block_commitment_tree(); let block2 = contextual.block.clone(); let height = contextual.height; @@ -435,7 +437,10 @@ impl NonFinalizedState { /// Returns `true` if the best chain contains `sapling_nullifier`. #[cfg(test)] - pub fn best_contains_sapling_nullifier(&self, sapling_nullifier: &sapling::Nullifier) -> bool { + pub fn best_contains_sapling_nullifier( + &self, + sapling_nullifier: &zebra_chain::sapling::Nullifier, + ) -> bool { self.best_chain() .map(|best_chain| best_chain.sapling_nullifiers.contains(sapling_nullifier)) .unwrap_or(false) @@ -443,7 +448,10 @@ impl NonFinalizedState { /// Returns `true` if the best chain contains `orchard_nullifier`. #[cfg(test)] - pub fn best_contains_orchard_nullifier(&self, orchard_nullifier: &orchard::Nullifier) -> bool { + pub fn best_contains_orchard_nullifier( + &self, + orchard_nullifier: &zebra_chain::orchard::Nullifier, + ) -> bool { self.best_chain() .map(|best_chain| best_chain.orchard_nullifiers.contains(orchard_nullifier)) .unwrap_or(false) @@ -456,19 +464,12 @@ impl NonFinalizedState { /// Return the chain whose tip block hash is `parent_hash`. /// - /// The chain can be an existing chain in the non-finalized state or a freshly - /// created fork, if needed. - /// - /// The trees must be the trees of the finalized tip. - /// They are used to recreate the trees if a fork is needed. + /// The chain can be an existing chain in the non-finalized state, or a freshly + /// created fork. #[allow(clippy::unwrap_in_result)] fn parent_chain( &mut self, parent_hash: block::Hash, - sprout_note_commitment_tree: Arc, - sapling_note_commitment_tree: Arc, - orchard_note_commitment_tree: Arc, - history_tree: Arc, ) -> Result, ValidateContextError> { match self.find_chain(|chain| chain.non_finalized_tip_hash() == parent_hash) { // Clone the existing Arc in the non-finalized state @@ -480,18 +481,7 @@ impl NonFinalizedState { let fork_chain = self .chain_set .iter() - .find_map(|chain| { - chain - .fork( - parent_hash, - sprout_note_commitment_tree.clone(), - sapling_note_commitment_tree.clone(), - orchard_note_commitment_tree.clone(), - history_tree.clone(), - ) - .transpose() - }) - .transpose()? + .find_map(|chain| chain.fork(parent_hash)) .ok_or(ValidateContextError::NotReadyToBeCommitted)?; Ok(Arc::new(fork_chain)) diff --git a/zebra-state/src/service/non_finalized_state/chain.rs b/zebra-state/src/service/non_finalized_state/chain.rs index c67a1d05ba3..bcef1bb9991 100644 --- a/zebra-state/src/service/non_finalized_state/chain.rs +++ b/zebra-state/src/service/non_finalized_state/chain.rs @@ -6,7 +6,6 @@ use std::{ collections::{BTreeMap, BTreeSet, HashMap, HashSet}, ops::{Deref, RangeInclusive}, sync::Arc, - time::Instant, }; use mset::MultiSet; @@ -15,7 +14,6 @@ use tracing::instrument; use zebra_chain::{ amount::{Amount, NegativeAllowed, NonNegative}, block::{self, Height}, - fmt::humantime_milliseconds, history_tree::HistoryTree, orchard, parallel::tree::NoteCommitmentTrees, @@ -40,10 +38,15 @@ pub mod index; #[derive(Debug, Clone)] pub struct Chain { - // The function `eq_internal_state` must be updated every time a field is added to [`Chain`]. + // Note: `eq_internal_state()` must be updated every time a field is added to [`Chain`]. + + // Config + // /// The configured network for this chain. network: Network, + // Blocks, heights, hashes, and transaction locations + // /// The contextually valid blocks which form this non-finalized partial chain, in height order. pub(crate) blocks: BTreeMap, @@ -53,6 +56,8 @@ pub struct Chain { /// An index of [`TransactionLocation`]s for each transaction hash in `blocks`. pub tx_loc_by_hash: HashMap, + // Transparent outputs and spends + // /// The [`transparent::Utxo`]s created by `blocks`. /// /// Note that these UTXOs may not be unspent. @@ -64,46 +69,82 @@ pub struct Chain { /// including those created by earlier transactions or blocks in the chain. pub(crate) spent_utxos: HashSet, - /// The Sprout note commitment tree of the tip of this [`Chain`], - /// including all finalized notes, and the non-finalized notes in this chain. - pub(super) sprout_note_commitment_tree: Arc, + // Note commitment trees + // /// The Sprout note commitment tree for each anchor. /// This is required for interstitial states. + /// + /// When a chain is forked from the finalized tip, also contains the finalized tip root. + /// This extra root is removed when the first non-finalized block is committed. pub(crate) sprout_trees_by_anchor: HashMap>, /// The Sprout note commitment tree for each height. + /// + /// When a chain is forked from the finalized tip, also contains the finalized tip tree. + /// This extra tree is removed when the first non-finalized block is committed. pub(crate) sprout_trees_by_height: BTreeMap>, - /// The Sapling note commitment tree of the tip of this [`Chain`], - /// including all finalized notes, and the non-finalized notes in this chain. - pub(super) sapling_note_commitment_tree: Arc, + /// The Sapling note commitment tree for each height. + /// + /// When a chain is forked from the finalized tip, also contains the finalized tip tree. + /// This extra tree is removed when the first non-finalized block is committed. pub(crate) sapling_trees_by_height: BTreeMap>, - /// The Orchard note commitment tree of the tip of this [`Chain`], - /// including all finalized notes, and the non-finalized notes in this chain. - pub(super) orchard_note_commitment_tree: Arc, + /// The Orchard note commitment tree for each height. + /// + /// When a chain is forked from the finalized tip, also contains the finalized tip tree. + /// This extra tree is removed when the first non-finalized block is committed. pub(crate) orchard_trees_by_height: BTreeMap>, - /// The ZIP-221 history tree of the tip of this [`Chain`], - /// including all finalized blocks, and the non-finalized `blocks` in this chain. - pub(crate) history_tree: Arc, + + // History trees + // + /// The ZIP-221 history tree for each height, including all finalized blocks, + /// and the non-finalized `blocks` below that height in this chain. + /// + /// When a chain is forked from the finalized tip, also contains the finalized tip tree. + /// This extra tree is removed when the first non-finalized block is committed. pub(crate) history_trees_by_height: BTreeMap>, + // Anchors + // /// The Sprout anchors created by `blocks`. + /// + /// When a chain is forked from the finalized tip, also contains the finalized tip root. + /// This extra root is removed when the first non-finalized block is committed. pub(crate) sprout_anchors: MultiSet, /// The Sprout anchors created by each block in `blocks`. + /// + /// When a chain is forked from the finalized tip, also contains the finalized tip root. + /// This extra root is removed when the first non-finalized block is committed. pub(crate) sprout_anchors_by_height: BTreeMap, + /// The Sapling anchors created by `blocks`. + /// + /// When a chain is forked from the finalized tip, also contains the finalized tip root. + /// This extra root is removed when the first non-finalized block is committed. pub(crate) sapling_anchors: MultiSet, /// The Sapling anchors created by each block in `blocks`. + /// + /// When a chain is forked from the finalized tip, also contains the finalized tip root. + /// This extra root is removed when the first non-finalized block is committed. pub(crate) sapling_anchors_by_height: BTreeMap, + /// The Orchard anchors created by `blocks`. + /// + /// When a chain is forked from the finalized tip, also contains the finalized tip root. + /// This extra root is removed when the first non-finalized block is committed. pub(crate) orchard_anchors: MultiSet, /// The Orchard anchors created by each block in `blocks`. + /// + /// When a chain is forked from the finalized tip, also contains the finalized tip root. + /// This extra root is removed when the first non-finalized block is committed. pub(crate) orchard_anchors_by_height: BTreeMap, + // Nullifiers + // /// The Sprout nullifiers revealed by `blocks`. pub(crate) sprout_nullifiers: HashSet, /// The Sapling nullifiers revealed by `blocks`. @@ -111,9 +152,14 @@ pub struct Chain { /// The Orchard nullifiers revealed by `blocks`. pub(crate) orchard_nullifiers: HashSet, + // Transparent Transfers + // TODO: move to the transparent section + // /// Partial transparent address index data from `blocks`. pub(super) partial_transparent_transfers: HashMap, + // Chain Work + // /// The cumulative work represented by `blocks`. /// /// Since the best chain is determined by the largest cumulative work, @@ -121,6 +167,8 @@ pub struct Chain { /// because they are common to all non-finalized chains. pub(super) partial_cumulative_work: PartialCumulativeWork, + // Chain Pools + // /// The chain value pool balances of the tip of this [`Chain`], /// including the block value pool changes from all finalized blocks, /// and the non-finalized blocks in this chain. @@ -131,24 +179,22 @@ pub struct Chain { } impl Chain { - /// Create a new Chain with the given trees and network. + /// Create a new Chain with the given finalized tip trees and network. pub(crate) fn new( network: Network, + finalized_tip_height: Height, sprout_note_commitment_tree: Arc, sapling_note_commitment_tree: Arc, orchard_note_commitment_tree: Arc, history_tree: Arc, finalized_tip_chain_value_pools: ValueBalance, ) -> Self { - Self { + let mut chain = Self { network, blocks: Default::default(), height_by_hash: Default::default(), tx_loc_by_hash: Default::default(), created_utxos: Default::default(), - sprout_note_commitment_tree, - sapling_note_commitment_tree, - orchard_note_commitment_tree, spent_utxos: Default::default(), sprout_anchors: MultiSet::new(), sprout_anchors_by_height: Default::default(), @@ -165,10 +211,16 @@ impl Chain { orchard_nullifiers: Default::default(), partial_transparent_transfers: Default::default(), partial_cumulative_work: Default::default(), - history_tree, history_trees_by_height: Default::default(), chain_value_pools: finalized_tip_chain_value_pools, - } + }; + + chain.add_sprout_tree_and_anchor(finalized_tip_height, sprout_note_commitment_tree); + chain.add_sapling_tree_and_anchor(finalized_tip_height, sapling_note_commitment_tree); + chain.add_orchard_tree_and_anchor(finalized_tip_height, orchard_note_commitment_tree); + chain.add_history_tree(finalized_tip_height, history_tree); + + chain } /// Is the internal state of `self` the same as `other`? @@ -193,16 +245,12 @@ impl Chain { self.spent_utxos == other.spent_utxos && // note commitment trees - self.sprout_note_commitment_tree.root() == other.sprout_note_commitment_tree.root() && self.sprout_trees_by_anchor == other.sprout_trees_by_anchor && self.sprout_trees_by_height == other.sprout_trees_by_height && - self.sapling_note_commitment_tree.root() == other.sapling_note_commitment_tree.root() && self.sapling_trees_by_height == other.sapling_trees_by_height && - self.orchard_note_commitment_tree.root() == other.orchard_note_commitment_tree.root() && self.orchard_trees_by_height == other.orchard_trees_by_height && // history trees - self.history_tree == other.history_tree && self.history_trees_by_height == other.history_trees_by_height && // anchors @@ -278,149 +326,21 @@ impl Chain { .expect("only called while blocks is populated") } - /// Fork a chain at the block with the given hash, if it is part of this - /// chain. - /// - /// The passed trees must match the trees of the finalized tip. They are - /// extended by the commitments from the newly forked chain up to the passed - /// `fork_tip`. - #[allow(clippy::unwrap_in_result)] - pub fn fork( - &self, - fork_tip: block::Hash, - sprout_note_commitment_tree: Arc, - sapling_note_commitment_tree: Arc, - orchard_note_commitment_tree: Arc, - history_tree: Arc, - ) -> Result, ValidateContextError> { + /// Fork and return a chain at the block with the given `fork_tip`, if it is part of this + /// chain. Otherwise, if this chain does not contain `fork_tip`, returns `None`. + pub fn fork(&self, fork_tip: block::Hash) -> Option { if !self.height_by_hash.contains_key(&fork_tip) { - return Ok(None); + return None; } - let mut forked = self.with_trees( - sprout_note_commitment_tree, - sapling_note_commitment_tree, - orchard_note_commitment_tree, - history_tree, - ); + let mut forked = self.clone(); // Revert blocks above the fork while forked.non_finalized_tip_hash() != fork_tip { forked.pop_tip(); } - // Rebuild trees from the finalized tip, because we haven't implemented reverts yet. - forked.rebuild_trees_parallel()?; - - Ok(Some(forked)) - } - - /// Rebuild the note commitment and history trees after a chain fork, - /// starting from the finalized tip trees, using parallel `rayon` threads. - /// - /// Note commitments and history trees are not removed from the Chain during a fork, - /// because we don't support that operation yet. Instead, we recreate the tree - /// from the finalized tip. - /// - /// TODO: remove trees and anchors above the fork, to save CPU time (#4794) - #[allow(clippy::unwrap_in_result)] - pub fn rebuild_trees_parallel(&mut self) -> Result<(), ValidateContextError> { - let start_time = Instant::now(); - let rebuilt_block_count = self.blocks.len(); - let fork_height = self.non_finalized_tip_height(); - let fork_tip = self.non_finalized_tip_hash(); - - info!( - ?rebuilt_block_count, - ?fork_height, - ?fork_tip, - "starting to rebuild note commitment trees after a non-finalized chain fork", - ); - - // Prepare data for parallel execution - let block_list = self - .blocks - .iter() - .map(|(height, block)| (*height, block.block.clone())) - .collect(); - - // TODO: use NoteCommitmentTrees to store the trees as well? - let mut nct = NoteCommitmentTrees { - sprout: self.sprout_note_commitment_tree.clone(), - sapling: self.sapling_note_commitment_tree.clone(), - orchard: self.orchard_note_commitment_tree.clone(), - }; - - let mut note_result = None; - let mut history_result = None; - - // Run 4 tasks in parallel: - // - sprout, sapling, and orchard tree updates and (redundant) root calculations - // - history tree updates - rayon::in_place_scope_fifo(|scope| { - // Spawns a separate rayon task for each note commitment tree. - // - // TODO: skip the unused root calculations? (redundant after #4794) - note_result = Some(nct.update_trees_parallel_list(block_list)); - - scope.spawn_fifo(|_scope| { - history_result = Some(self.rebuild_history_tree()); - }); - }); - - note_result.expect("scope has already finished")?; - history_result.expect("scope has already finished")?; - - // Update the note commitment trees in the chain. - self.sprout_note_commitment_tree = nct.sprout; - self.sapling_note_commitment_tree = nct.sapling; - self.orchard_note_commitment_tree = nct.orchard; - - let rebuild_time = start_time.elapsed(); - let rebuild_time_per_block = - rebuild_time / rebuilt_block_count.try_into().expect("fits in u32"); - info!( - rebuild_time = ?humantime_milliseconds(rebuild_time), - rebuild_time_per_block = ?humantime_milliseconds(rebuild_time_per_block), - ?rebuilt_block_count, - ?fork_height, - ?fork_tip, - "finished rebuilding note commitment trees after a non-finalized chain fork", - ); - - Ok(()) - } - - /// Rebuild the history tree after a chain fork. - /// - /// TODO: remove trees and anchors above the fork, to save CPU time (#4794) - #[allow(clippy::unwrap_in_result)] - pub fn rebuild_history_tree(&mut self) -> Result<(), ValidateContextError> { - for block in self.blocks.values() { - // Note that anchors don't need to be recreated since they are already - // handled in revert_chain_state_with. - let sapling_root = self - .sapling_anchors_by_height - .get(&block.height) - .expect("Sapling anchors must exist for pre-fork blocks"); - - let orchard_root = self - .orchard_anchors_by_height - .get(&block.height) - .expect("Orchard anchors must exist for pre-fork blocks"); - - let history_tree_mut = Arc::make_mut(&mut self.history_tree); - history_tree_mut - .push( - self.network, - block.block.clone(), - *sapling_root, - *orchard_root, - ) - .map_err(Arc::new)?; - } - - Ok(()) + Some(forked) } /// Returns the [`Network`] for this chain. @@ -508,9 +428,25 @@ impl Chain { ) } - /// Returns the Sprout - /// [`NoteCommitmentTree`](sprout::tree::NoteCommitmentTree) specified by a - /// [`HashOrHeight`], if it exists in the non-finalized [`Chain`]. + /// Returns the Sprout note commitment tree of the tip of this [`Chain`], + /// including all finalized notes, and the non-finalized notes in this chain. + /// + /// If the chain is empty, instead returns the tree of the finalized tip, + /// which was supplied in [`Chain::new()`] + /// + /// # Panics + /// + /// If this chain has no sprout trees. (This should be impossible.) + pub fn sprout_note_commitment_tree(&self) -> Arc { + self.sprout_trees_by_height + .last_key_value() + .expect("only called while sprout_trees_by_height is populated") + .1 + .clone() + } + + /// Returns the Sprout [`NoteCommitmentTree`](sprout::tree::NoteCommitmentTree) specified by + /// a [`HashOrHeight`], if it exists in the non-finalized [`Chain`]. pub fn sprout_tree( &self, hash_or_height: HashOrHeight, @@ -521,9 +457,113 @@ impl Chain { self.sprout_trees_by_height.get(&height).cloned() } - /// Returns the Sapling - /// [`NoteCommitmentTree`](sapling::tree::NoteCommitmentTree) specified by a - /// [`HashOrHeight`], if it exists in the non-finalized [`Chain`]. + /// Add the Sprout `tree` to the tree and anchor indexes at `height`. + /// + /// `height` can be either: + /// - the height of a new block that has just been added to the chain tip, or + /// - the finalized tip height: the height of the parent of the first block of a new chain. + fn add_sprout_tree_and_anchor( + &mut self, + height: Height, + tree: Arc, + ) { + // Having updated all the note commitment trees and nullifier sets in + // this block, the roots of the note commitment trees as of the last + // transaction are the anchor treestates of this block. + // + // Use the previously cached root which was calculated in parallel. + let anchor = tree.root(); + trace!(?height, ?anchor, "adding sprout tree"); + + // TODO: fix test code that incorrectly overwrites trees + #[cfg(not(test))] + { + assert_eq!( + self.sprout_trees_by_height.insert(height, tree.clone()), + None, + "incorrect overwrite of sprout tree: trees must be reverted then inserted", + ); + assert_eq!( + self.sprout_anchors_by_height.insert(height, anchor), + None, + "incorrect overwrite of sprout anchor: anchors must be reverted then inserted", + ); + } + + #[cfg(test)] + { + self.sprout_trees_by_height.insert(height, tree.clone()); + self.sprout_anchors_by_height.insert(height, anchor); + } + + // Multiple inserts are expected here, + // because the anchors only change if a block has shielded transactions. + self.sprout_anchors.insert(anchor); + self.sprout_trees_by_anchor.insert(anchor, tree); + } + + /// Remove the Sprout tree and anchor indexes at `height`. + /// + /// `height` can be at two different [`RevertPosition`]s in the chain: + /// - a tip block above a chain fork: only that height is removed, or + /// - a root block: all trees and anchors below that height are removed, + /// including temporary finalized tip trees. + fn remove_sprout_tree_and_anchor(&mut self, position: RevertPosition, height: Height) { + let removed_heights: Vec = if position == RevertPosition::Root { + // Remove all trees and anchors at or below the removed block. + // This makes sure the temporary trees from finalized tip forks are removed. + self.sprout_anchors_by_height + .keys() + .cloned() + .filter(|index_height| *index_height <= height) + .collect() + } else { + // Just remove the reverted tip trees and anchors. + vec![height] + }; + + for height in &removed_heights { + let anchor = self + .sprout_anchors_by_height + .remove(height) + .expect("Sprout anchor must be present if block was added to chain"); + self.sprout_trees_by_height + .remove(height) + .expect("Sprout note commitment tree must be present if block was added to chain"); + + trace!(?height, ?position, ?anchor, "removing sprout tree"); + + // Multiple removals are expected here, + // because the anchors only change if a block has shielded transactions. + assert!( + self.sprout_anchors.remove(&anchor), + "Sprout anchor must be present if block was added to chain" + ); + if !self.sprout_anchors.contains(&anchor) { + self.sprout_trees_by_anchor.remove(&anchor); + } + } + } + + /// Returns the Sapling note commitment tree of the tip of this [`Chain`], + /// including all finalized notes, and the non-finalized notes in this chain. + /// + /// If the chain is empty, instead returns the tree of the finalized tip, + /// which was supplied in [`Chain::new()`] + /// + /// # Panics + /// + /// If this chain has no sapling trees. (This should be impossible.) + pub fn sapling_note_commitment_tree(&self) -> Arc { + self.sapling_trees_by_height + .last_key_value() + .expect("only called while sapling_trees_by_height is populated") + .1 + .clone() + } + + /// Returns the Sapling [`NoteCommitmentTree`](sapling::tree::NoteCommitmentTree) specified + /// by a [`HashOrHeight`], if it exists in the non-finalized [`Chain`]. pub fn sapling_tree( &self, hash_or_height: HashOrHeight, @@ -534,6 +574,102 @@ impl Chain { self.sapling_trees_by_height.get(&height).cloned() } + /// Add the Sapling `tree` to the tree and anchor indexes at `height`. + /// + /// `height` can be either: + /// - the height of a new block that has just been added to the chain tip, or + /// - the finalized tip height: the height of the parent of the first block of a new chain. + fn add_sapling_tree_and_anchor( + &mut self, + height: Height, + tree: Arc, + ) { + let anchor = tree.root(); + trace!(?height, ?anchor, "adding sapling tree"); + + // TODO: fix test code that incorrectly overwrites trees + #[cfg(not(test))] + { + assert_eq!( + self.sapling_trees_by_height.insert(height, tree), + None, + "incorrect overwrite of sapling tree: trees must be reverted then inserted", + ); + assert_eq!( + self.sapling_anchors_by_height.insert(height, anchor), + None, + "incorrect overwrite of sapling anchor: anchors must be reverted then inserted", + ); + } + + #[cfg(test)] + { + self.sapling_trees_by_height.insert(height, tree); + self.sapling_anchors_by_height.insert(height, anchor); + } + + // Multiple inserts are expected here, + // because the anchors only change if a block has shielded transactions. + self.sapling_anchors.insert(anchor); + } + + /// Remove the Sapling tree and anchor indexes at `height`. + /// + /// `height` can be at two different [`RevertPosition`]s in the chain: + /// - a tip block above a chain fork: only that height is removed, or + /// - a root block: all trees and anchors below that height are removed, + /// including temporary finalized tip trees. + fn remove_sapling_tree_and_anchor(&mut self, position: RevertPosition, height: Height) { + let removed_heights: Vec = if position == RevertPosition::Root { + // Remove all trees and anchors at or below the removed block. + // This makes sure the temporary trees from finalized tip forks are removed. + self.sapling_anchors_by_height + .keys() + .cloned() + .filter(|index_height| *index_height <= height) + .collect() + } else { + // Just remove the reverted tip trees and anchors. + vec![height] + }; + + for height in &removed_heights { + let anchor = self + .sapling_anchors_by_height + .remove(height) + .expect("Sapling anchor must be present if block was added to chain"); + self.sapling_trees_by_height + .remove(height) + .expect("Sapling note commitment tree must be present if block was added to chain"); + + trace!(?height, ?position, ?anchor, "removing sapling tree"); + + // Multiple removals are expected here, + // because the anchors only change if a block has shielded transactions. + assert!( + self.sapling_anchors.remove(&anchor), + "Sapling anchor must be present if block was added to chain" + ); + } + } + + /// Returns the Orchard note commitment tree of the tip of this [`Chain`], + /// including all finalized notes, and the non-finalized notes in this chain. + /// + /// If the chain is empty, instead returns the tree of the finalized tip, + /// which was supplied in [`Chain::new()`] + /// + /// # Panics + /// + /// If this chain has no orchard trees. (This should be impossible.) + pub fn orchard_note_commitment_tree(&self) -> Arc { + self.orchard_trees_by_height + .last_key_value() + .expect("only called while orchard_trees_by_height is populated") + .1 + .clone() + } + /// Returns the Orchard /// [`NoteCommitmentTree`](orchard::tree::NoteCommitmentTree) specified by a /// [`HashOrHeight`], if it exists in the non-finalized [`Chain`]. @@ -547,6 +683,107 @@ impl Chain { self.orchard_trees_by_height.get(&height).cloned() } + /// Add the Orchard `tree` to the tree and anchor indexes at `height`. + /// + /// `height` can be either: + /// - the height of a new block that has just been added to the chain tip, or + /// - the finalized tip height: the height of the parent of the first block of a new chain. + fn add_orchard_tree_and_anchor( + &mut self, + height: Height, + tree: Arc, + ) { + // Having updated all the note commitment trees and nullifier sets in + // this block, the roots of the note commitment trees as of the last + // transaction are the anchor treestates of this block. + // + // Use the previously cached root which was calculated in parallel. + let anchor = tree.root(); + trace!(?height, ?anchor, "adding orchard tree"); + + // TODO: fix test code that incorrectly overwrites trees + #[cfg(not(test))] + { + assert_eq!( + self.orchard_trees_by_height.insert(height, tree), + None, + "incorrect overwrite of orchard tree: trees must be reverted then inserted", + ); + assert_eq!( + self.orchard_anchors_by_height.insert(height, anchor), + None, + "incorrect overwrite of orchard anchor: anchors must be reverted then inserted", + ); + } + + #[cfg(test)] + { + self.orchard_trees_by_height.insert(height, tree); + self.orchard_anchors_by_height.insert(height, anchor); + } + + // Multiple inserts are expected here, + // because the anchors only change if a block has shielded transactions. + self.orchard_anchors.insert(anchor); + } + + /// Remove the Orchard tree and anchor indexes at `height`. + /// + /// `height` can be at two different [`RevertPosition`]s in the chain: + /// - a tip block above a chain fork: only that height is removed, or + /// - a root block: all trees and anchors below that height are removed, + /// including temporary finalized tip trees. + fn remove_orchard_tree_and_anchor(&mut self, position: RevertPosition, height: Height) { + let removed_heights: Vec = if position == RevertPosition::Root { + // Remove all trees and anchors at or below the removed block. + // This makes sure the temporary trees from finalized tip forks are removed. + self.orchard_anchors_by_height + .keys() + .cloned() + .filter(|index_height| *index_height <= height) + .collect() + } else { + // Just remove the reverted tip trees and anchors. + vec![height] + }; + + for height in &removed_heights { + let anchor = self + .orchard_anchors_by_height + .remove(height) + .expect("Orchard anchor must be present if block was added to chain"); + self.orchard_trees_by_height + .remove(height) + .expect("Orchard note commitment tree must be present if block was added to chain"); + + trace!(?height, ?position, ?anchor, "removing orchard tree"); + + // Multiple removals are expected here, + // because the anchors only change if a block has shielded transactions. + assert!( + self.orchard_anchors.remove(&anchor), + "Orchard anchor must be present if block was added to chain" + ); + } + } + + /// Returns the History tree of the tip of this [`Chain`], + /// including all finalized blocks, and the non-finalized blocks below the chain tip. + /// + /// If the chain is empty, instead returns the tree of the finalized tip, + /// which was supplied in [`Chain::new()`] + /// + /// # Panics + /// + /// If this chain has no history trees. (This should be impossible.) + pub fn history_block_commitment_tree(&self) -> Arc { + self.history_trees_by_height + .last_key_value() + .expect("only called while history_trees_by_height is populated") + .1 + .clone() + } + /// Returns the [`HistoryTree`] specified by a [`HashOrHeight`], if it /// exists in the non-finalized [`Chain`]. pub fn history_tree(&self, hash_or_height: HashOrHeight) -> Option> { @@ -556,6 +793,51 @@ impl Chain { self.history_trees_by_height.get(&height).cloned() } + /// Add the History `tree` to the history tree index at `height`. + /// + /// `height` can be either: + /// - the height of a new block that has just been added to the chain tip, or + /// - the finalized tip height: the height of the parent of the first block of a new chain. + fn add_history_tree(&mut self, height: Height, tree: Arc) { + // The history tree commits to all the blocks before this block. + // + // Use the previously cached root which was calculated in parallel. + trace!(?height, "adding history tree"); + + // TODO: fix test code that incorrectly overwrites trees + #[cfg(not(test))] + assert_eq!( + self.history_trees_by_height.insert(height, tree), + None, + "incorrect overwrite of history tree: trees must be reverted then inserted", + ); + + #[cfg(test)] + self.history_trees_by_height.insert(height, tree); + } + + /// Remove the History tree index at `height`. + /// + /// `height` can be at two different [`RevertPosition`]s in the chain: + /// - a tip block above a chain fork: only that height is removed, or + /// - a root block: all trees below that height are removed, + /// including temporary finalized tip trees. + fn remove_history_tree(&mut self, position: RevertPosition, height: Height) { + trace!(?height, ?position, "removing history tree"); + + if position == RevertPosition::Root { + // Remove all trees at or below the reverted root block. + // This makes sure the temporary trees from finalized tip forks are removed. + self.history_trees_by_height + .retain(|index_height, _tree| *index_height > height); + } else { + // Just remove the reverted tip tree. + self.history_trees_by_height + .remove(&height) + .expect("History tree must be present if block was added to chain"); + } + } + fn treestate(&self, hash_or_height: HashOrHeight) -> Option { let sprout_tree = self.sprout_tree(hash_or_height)?; let sapling_tree = self.sapling_tree(hash_or_height)?; @@ -792,50 +1074,6 @@ impl Chain { .collect() } - // Cloning - - /// Clone the Chain but not the history and note commitment trees, using - /// the specified trees instead. - /// - /// Useful when forking, where the trees are rebuilt anyway. - fn with_trees( - &self, - sprout_note_commitment_tree: Arc, - sapling_note_commitment_tree: Arc, - orchard_note_commitment_tree: Arc, - history_tree: Arc, - ) -> Self { - Chain { - network: self.network, - blocks: self.blocks.clone(), - height_by_hash: self.height_by_hash.clone(), - tx_loc_by_hash: self.tx_loc_by_hash.clone(), - created_utxos: self.created_utxos.clone(), - spent_utxos: self.spent_utxos.clone(), - sprout_note_commitment_tree, - sprout_trees_by_anchor: self.sprout_trees_by_anchor.clone(), - sprout_trees_by_height: self.sprout_trees_by_height.clone(), - sapling_note_commitment_tree, - sapling_trees_by_height: self.sapling_trees_by_height.clone(), - orchard_note_commitment_tree, - orchard_trees_by_height: self.orchard_trees_by_height.clone(), - sprout_anchors: self.sprout_anchors.clone(), - sapling_anchors: self.sapling_anchors.clone(), - orchard_anchors: self.orchard_anchors.clone(), - sprout_anchors_by_height: self.sprout_anchors_by_height.clone(), - sapling_anchors_by_height: self.sapling_anchors_by_height.clone(), - orchard_anchors_by_height: self.orchard_anchors_by_height.clone(), - sprout_nullifiers: self.sprout_nullifiers.clone(), - sapling_nullifiers: self.sapling_nullifiers.clone(), - orchard_nullifiers: self.orchard_nullifiers.clone(), - partial_transparent_transfers: self.partial_transparent_transfers.clone(), - partial_cumulative_work: self.partial_cumulative_work, - history_tree, - history_trees_by_height: self.history_trees_by_height.clone(), - chain_value_pools: self.chain_value_pools, - } - } - /// Update the chain tip with the `contextually_valid` block, /// running note commitment tree updates in parallel with other updates. /// @@ -849,12 +1087,10 @@ impl Chain { let height = contextually_valid.height; // Prepare data for parallel execution - // - // TODO: use NoteCommitmentTrees to store the trees as well? let mut nct = NoteCommitmentTrees { - sprout: self.sprout_note_commitment_tree.clone(), - sapling: self.sapling_note_commitment_tree.clone(), - orchard: self.orchard_note_commitment_tree.clone(), + sprout: self.sprout_note_commitment_tree(), + sapling: self.sapling_note_commitment_tree(), + orchard: self.orchard_note_commitment_tree(), }; let mut tree_result = None; @@ -877,40 +1113,16 @@ impl Chain { partial_result.expect("scope has already finished")?; // Update the note commitment trees in the chain. - self.sprout_note_commitment_tree = nct.sprout; - self.sapling_note_commitment_tree = nct.sapling; - self.orchard_note_commitment_tree = nct.orchard; + self.add_sprout_tree_and_anchor(height, nct.sprout); + self.add_sapling_tree_and_anchor(height, nct.sapling); + self.add_orchard_tree_and_anchor(height, nct.orchard); - // Do the Chain updates with data dependencies on note commitment tree updates - - // Update the note commitment trees indexed by height. - self.sprout_trees_by_height - .insert(height, self.sprout_note_commitment_tree.clone()); - self.sapling_trees_by_height - .insert(height, self.sapling_note_commitment_tree.clone()); - self.orchard_trees_by_height - .insert(height, self.orchard_note_commitment_tree.clone()); - - // Having updated all the note commitment trees and nullifier sets in - // this block, the roots of the note commitment trees as of the last - // transaction are the treestates of this block. - // - // Use the previously cached roots, which were calculated in parallel. - let sprout_root = self.sprout_note_commitment_tree.root(); - self.sprout_anchors.insert(sprout_root); - self.sprout_anchors_by_height.insert(height, sprout_root); - self.sprout_trees_by_anchor - .insert(sprout_root, self.sprout_note_commitment_tree.clone()); - let sapling_root = self.sapling_note_commitment_tree.root(); - self.sapling_anchors.insert(sapling_root); - self.sapling_anchors_by_height.insert(height, sapling_root); - - let orchard_root = self.orchard_note_commitment_tree.root(); - self.orchard_anchors.insert(orchard_root); - self.orchard_anchors_by_height.insert(height, orchard_root); + let sapling_root = self.sapling_note_commitment_tree().root(); + let orchard_root = self.orchard_note_commitment_tree().root(); // TODO: update the history trees in a rayon thread, if they show up in CPU profiles - let history_tree_mut = Arc::make_mut(&mut self.history_tree); + let mut history_tree = self.history_block_commitment_tree(); + let history_tree_mut = Arc::make_mut(&mut history_tree); history_tree_mut .push( self.network, @@ -920,8 +1132,7 @@ impl Chain { ) .map_err(Arc::new)?; - self.history_trees_by_height - .insert(height, self.history_tree.clone()); + self.add_history_tree(height, history_tree); Ok(()) } @@ -1110,6 +1321,7 @@ impl UpdateWith for Chain { "hash must be present if block was added to chain" ); + // TODO: move this to a Work or block header UpdateWith.revert...()? // remove work from partial_cumulative_work let block_work = block .header @@ -1118,16 +1330,6 @@ impl UpdateWith for Chain { .expect("work has already been validated"); self.partial_cumulative_work -= block_work; - // Note: the history tree is not modified in this method. - // This method is called on two scenarios: - // - When popping the root: the history tree does not change. - // - When popping the tip: the history tree is rebuilt in fork(). - // - // However, `history_trees_by_height` is reverted. - self.history_trees_by_height - .remove(&height) - .expect("History tree must be present if block was added to chain"); - // for each transaction in block for (transaction, transaction_hash) in block.transactions.iter().zip(transaction_hashes.iter()) @@ -1171,6 +1373,7 @@ impl UpdateWith for Chain { // reset the utxos this consumed self.revert_chain_with(&(inputs, transaction_hash, spent_outputs), position); + // TODO: move this to the history tree UpdateWith.revert...()? // remove `transaction.hash` from `tx_loc_by_hash` assert!( self.tx_loc_by_hash.remove(transaction_hash).is_some(), @@ -1184,44 +1387,13 @@ impl UpdateWith for Chain { self.revert_chain_with(orchard_shielded_data, position); } - let anchor = self - .sprout_anchors_by_height - .remove(&height) - .expect("Sprout anchor must be present if block was added to chain"); - assert!( - self.sprout_anchors.remove(&anchor), - "Sprout anchor must be present if block was added to chain" - ); - if !self.sprout_anchors.contains(&anchor) { - self.sprout_trees_by_anchor.remove(&anchor); - } - self.sprout_trees_by_height - .remove(&height) - .expect("Sprout note commitment tree must be present if block was added to chain"); + // TODO: move these to the shielded UpdateWith.revert...()? + self.remove_sprout_tree_and_anchor(position, height); + self.remove_sapling_tree_and_anchor(position, height); + self.remove_orchard_tree_and_anchor(position, height); - let anchor = self - .sapling_anchors_by_height - .remove(&height) - .expect("Sapling anchor must be present if block was added to chain"); - assert!( - self.sapling_anchors.remove(&anchor), - "Sapling anchor must be present if block was added to chain" - ); - self.sapling_trees_by_height - .remove(&height) - .expect("Sapling note commitment tree must be present if block was added to chain"); - - let anchor = self - .orchard_anchors_by_height - .remove(&height) - .expect("Orchard anchor must be present if block was added to chain"); - assert!( - self.orchard_anchors.remove(&anchor), - "Orchard anchor must be present if block was added to chain" - ); - self.orchard_trees_by_height - .remove(&height) - .expect("Orchard note commitment tree must be present if block was added to chain"); + // TODO: move this to the history tree UpdateWith.revert...()? + self.remove_history_tree(position, height); // revert the chain value pool balances, if needed self.revert_chain_with(chain_value_pool_change, position); @@ -1470,11 +1642,9 @@ impl UpdateWith>> for Chain { _position: RevertPosition, ) { if let Some(joinsplit_data) = joinsplit_data { - // Note commitments are not removed from the Chain during a fork, - // because we don't support that operation yet. Instead, we - // recreate the tree from the finalized tip in Chain::fork(). - // - // TODO: remove trees and anchors above the fork, to save CPU time (#4794) + // Note commitments are removed from the Chain during a fork, + // by removing trees above the fork height from the note commitment index. + // This happens when reverting the block itself. check::nullifier::remove_from_non_finalized_chain( &mut self.sprout_nullifiers, @@ -1516,11 +1686,9 @@ where _position: RevertPosition, ) { if let Some(sapling_shielded_data) = sapling_shielded_data { - // Note commitments are not removed from the Chain during a fork, - // because we don't support that operation yet. Instead, we - // recreate the tree from the finalized tip in Chain::fork(). - // - // TODO: remove trees and anchors above the fork, to save CPU time (#4794) + // Note commitments are removed from the Chain during a fork, + // by removing trees above the fork height from the note commitment index. + // This happens when reverting the block itself. check::nullifier::remove_from_non_finalized_chain( &mut self.sapling_nullifiers, @@ -1559,11 +1727,9 @@ impl UpdateWith> for Chain { _position: RevertPosition, ) { if let Some(orchard_shielded_data) = orchard_shielded_data { - // Note commitments are not removed from the Chain during a fork, - // because we don't support that operation yet. Instead, we - // recreate the tree from the finalized tip in Chain::fork(). - // - // TODO: remove trees and anchors above the fork, to save CPU time (#4794) + // Note commitments are removed from the Chain during a fork, + // by removing trees above the fork height from the note commitment index. + // This happens when reverting the block itself. check::nullifier::remove_from_non_finalized_chain( &mut self.orchard_nullifiers, diff --git a/zebra-state/src/service/non_finalized_state/tests/prop.rs b/zebra-state/src/service/non_finalized_state/tests/prop.rs index a8aa175c8b0..c1034cbc252 100644 --- a/zebra-state/src/service/non_finalized_state/tests/prop.rs +++ b/zebra-state/src/service/non_finalized_state/tests/prop.rs @@ -6,7 +6,7 @@ use zebra_test::prelude::*; use zebra_chain::{ amount::NonNegative, - block::{self, arbitrary::allow_all_transparent_coinbase_spends, Block}, + block::{self, arbitrary::allow_all_transparent_coinbase_spends, Block, Height}, fmt::DisplayToDebug, history_tree::{HistoryTree, NonEmptyHistoryTree}, parameters::NetworkUpgrade::*, @@ -47,7 +47,7 @@ fn push_genesis_chain() -> Result<()> { |((chain, count, network, empty_tree) in PreparedChain::default())| { prop_assert!(empty_tree.is_none()); - let mut only_chain = Chain::new(network, Default::default(), Default::default(), Default::default(), empty_tree, ValueBalance::zero()); + let mut only_chain = Chain::new(network, Height(0), Default::default(), Default::default(), Default::default(), empty_tree, ValueBalance::zero()); // contains the block value pool changes and chain value pool balances for each height let mut chain_values = BTreeMap::new(); @@ -99,7 +99,7 @@ fn push_history_tree_chain() -> Result<()> { let count = std::cmp::min(count, chain.len() - 1); let chain = &chain[1..]; - let mut only_chain = Chain::new(network, Default::default(), Default::default(), Default::default(), finalized_tree, ValueBalance::zero()); + let mut only_chain = Chain::new(network, Height(0), Default::default(), Default::default(), Default::default(), finalized_tree, ValueBalance::zero()); for block in chain .iter() @@ -143,6 +143,7 @@ fn forked_equals_pushed_genesis() -> Result<()> { // correspond to the blocks in the original chain before the fork. let mut partial_chain = Chain::new( network, + Height(0), Default::default(), Default::default(), Default::default(), @@ -162,10 +163,11 @@ fn forked_equals_pushed_genesis() -> Result<()> { // This chain will be forked. let mut full_chain = Chain::new( network, + Height(0), Default::default(), Default::default(), Default::default(), - empty_tree.clone(), + empty_tree, ValueBalance::zero(), ); for block in chain.iter().cloned() { @@ -195,18 +197,12 @@ fn forked_equals_pushed_genesis() -> Result<()> { } // Use [`fork_at_count`] as the fork tip. - let fork_tip_hash = chain[fork_at_count - 1].hash; + let fork_tip_height = fork_at_count - 1; + let fork_tip_hash = chain[fork_tip_height].hash; // Fork the chain. let mut forked = full_chain - .fork( - fork_tip_hash, - Default::default(), - Default::default(), - Default::default(), - empty_tree, - ) - .expect("fork works") + .fork(fork_tip_hash) .expect("hash is present"); // This check is redundant, but it's useful for debugging. @@ -254,8 +250,8 @@ fn forked_equals_pushed_history_tree() -> Result<()> { // use `fork_at_count` as the fork tip let fork_tip_hash = chain[fork_at_count - 1].hash; - let mut full_chain = Chain::new(network, Default::default(), Default::default(), Default::default(), finalized_tree.clone(), ValueBalance::zero()); - let mut partial_chain = Chain::new(network, Default::default(), Default::default(), Default::default(), finalized_tree.clone(), ValueBalance::zero()); + let mut full_chain = Chain::new(network, Height(0), Default::default(), Default::default(), Default::default(), finalized_tree.clone(), ValueBalance::zero()); + let mut partial_chain = Chain::new(network, Height(0), Default::default(), Default::default(), Default::default(), finalized_tree, ValueBalance::zero()); for block in chain .iter() @@ -271,14 +267,7 @@ fn forked_equals_pushed_history_tree() -> Result<()> { } let mut forked = full_chain - .fork( - fork_tip_hash, - Default::default(), - Default::default(), - Default::default(), - finalized_tree, - ) - .expect("fork works") + .fork(fork_tip_hash) .expect("hash is present"); // the first check is redundant, but it's useful for debugging @@ -318,14 +307,17 @@ fn finalized_equals_pushed_genesis() -> Result<()> { prop_assert!(empty_tree.is_none()); + // TODO: fix this test or the code so the full_chain temporary trees aren't overwritten + let chain = chain.iter().filter(|block| block.height != Height(0)); + // use `end_count` as the number of non-finalized blocks at the end of the chain - let finalized_count = chain.len() - end_count; + let finalized_count = chain.clone().count() - end_count; let fake_value_pool = ValueBalance::::fake_populated_pool(); - let mut full_chain = Chain::new(network, Default::default(), Default::default(), Default::default(), empty_tree, fake_value_pool); + let mut full_chain = Chain::new(network, Height(0), Default::default(), Default::default(), Default::default(), empty_tree, fake_value_pool); for block in chain - .iter() + .clone() .take(finalized_count) .map(ContextuallyValidBlock::test_with_zero_spent_utxos) { full_chain = full_chain.push(block)?; @@ -333,21 +325,21 @@ fn finalized_equals_pushed_genesis() -> Result<()> { let mut partial_chain = Chain::new( network, - full_chain.sprout_note_commitment_tree.clone(), - full_chain.sapling_note_commitment_tree.clone(), - full_chain.orchard_note_commitment_tree.clone(), - full_chain.history_tree.clone(), + full_chain.non_finalized_tip_height(), + full_chain.sprout_note_commitment_tree(), + full_chain.sapling_note_commitment_tree(), + full_chain.orchard_note_commitment_tree(), + full_chain.history_block_commitment_tree(), full_chain.chain_value_pools, ); for block in chain - .iter() + .clone() .skip(finalized_count) .map(ContextuallyValidBlock::test_with_zero_spent_utxos) { partial_chain = partial_chain.push(block.clone())?; } for block in chain - .iter() .skip(finalized_count) .map(ContextuallyValidBlock::test_with_zero_spent_utxos) { full_chain = full_chain.push(block.clone())?; @@ -357,8 +349,18 @@ fn finalized_equals_pushed_genesis() -> Result<()> { full_chain.pop_root(); } + // Make sure the temporary trees from finalized tip forks are removed. + // TODO: update the test or the code so this extra step isn't needed? + full_chain.pop_root(); + partial_chain.pop_root(); + prop_assert_eq!(full_chain.blocks.len(), partial_chain.blocks.len()); - prop_assert!(full_chain.eq_internal_state(&partial_chain)); + prop_assert!( + full_chain.eq_internal_state(&partial_chain), + "\n\ + full chain:\n{full_chain:#?}\n\n\ + partial chain:\n{partial_chain:#?}\n", + ); }); Ok(()) @@ -393,7 +395,7 @@ fn finalized_equals_pushed_history_tree() -> Result<()> { let fake_value_pool = ValueBalance::::fake_populated_pool(); - let mut full_chain = Chain::new(network, Default::default(), Default::default(), Default::default(), finalized_tree, fake_value_pool); + let mut full_chain = Chain::new(network, Height(0), Default::default(), Default::default(), Default::default(), finalized_tree, fake_value_pool); for block in chain .iter() .take(finalized_count) @@ -403,10 +405,11 @@ fn finalized_equals_pushed_history_tree() -> Result<()> { let mut partial_chain = Chain::new( network, - full_chain.sprout_note_commitment_tree.clone(), - full_chain.sapling_note_commitment_tree.clone(), - full_chain.orchard_note_commitment_tree.clone(), - full_chain.history_tree.clone(), + Height(finalized_count.try_into().unwrap()), + full_chain.sprout_note_commitment_tree(), + full_chain.sapling_note_commitment_tree(), + full_chain.orchard_note_commitment_tree(), + full_chain.history_block_commitment_tree(), full_chain.chain_value_pools, ); @@ -428,8 +431,18 @@ fn finalized_equals_pushed_history_tree() -> Result<()> { full_chain.pop_root(); } + // Make sure the temporary trees from finalized tip forks are removed. + // TODO: update the test or the code so this extra step isn't needed? + full_chain.pop_root(); + partial_chain.pop_root(); + prop_assert_eq!(full_chain.blocks.len(), partial_chain.blocks.len()); - prop_assert!(full_chain.eq_internal_state(&partial_chain)); + prop_assert!( + full_chain.eq_internal_state(&partial_chain), + "\n\ + full chain:\n{full_chain:#?}\n\n\ + partial chain:\n{partial_chain:#?}\n", + ); }); Ok(()) @@ -570,8 +583,8 @@ fn different_blocks_different_chains() -> Result<()> { Default::default() }; - let chain1 = Chain::new(Network::Mainnet, Default::default(), Default::default(), Default::default(), finalized_tree1, ValueBalance::fake_populated_pool()); - let chain2 = Chain::new(Network::Mainnet, Default::default(), Default::default(), Default::default(), finalized_tree2, ValueBalance::fake_populated_pool()); + let chain1 = Chain::new(Network::Mainnet, Height(0), Default::default(), Default::default(), Default::default(), finalized_tree1, ValueBalance::fake_populated_pool()); + let chain2 = Chain::new(Network::Mainnet, Height(0), Default::default(), Default::default(), Default::default(), finalized_tree2, ValueBalance::fake_populated_pool()); let block1 = vec1[1].clone().prepare().test_with_zero_spent_utxos(); let block2 = vec2[1].clone().prepare().test_with_zero_spent_utxos(); @@ -606,16 +619,12 @@ fn different_blocks_different_chains() -> Result<()> { chain1.spent_utxos = chain2.spent_utxos.clone(); // note commitment trees - chain1.sprout_note_commitment_tree = chain2.sprout_note_commitment_tree.clone(); chain1.sprout_trees_by_anchor = chain2.sprout_trees_by_anchor.clone(); chain1.sprout_trees_by_height = chain2.sprout_trees_by_height.clone(); - chain1.sapling_note_commitment_tree = chain2.sapling_note_commitment_tree.clone(); chain1.sapling_trees_by_height = chain2.sapling_trees_by_height.clone(); - chain1.orchard_note_commitment_tree = chain2.orchard_note_commitment_tree.clone(); chain1.orchard_trees_by_height = chain2.orchard_trees_by_height.clone(); // history trees - chain1.history_tree = chain2.history_tree.clone(); chain1.history_trees_by_height = chain2.history_trees_by_height.clone(); // anchors diff --git a/zebra-state/src/service/non_finalized_state/tests/vectors.rs b/zebra-state/src/service/non_finalized_state/tests/vectors.rs index 62f01cad183..e241a401337 100644 --- a/zebra-state/src/service/non_finalized_state/tests/vectors.rs +++ b/zebra-state/src/service/non_finalized_state/tests/vectors.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use zebra_chain::{ amount::NonNegative, - block::Block, + block::{Block, Height}, history_tree::NonEmptyHistoryTree, parameters::{Network, NetworkUpgrade}, serialization::ZcashDeserializeInto, @@ -27,6 +27,7 @@ fn construct_empty() { let _init_guard = zebra_test::init(); let _chain = Chain::new( Network::Mainnet, + Height(0), Default::default(), Default::default(), Default::default(), @@ -43,6 +44,7 @@ fn construct_single() -> Result<()> { let mut chain = Chain::new( Network::Mainnet, + Height(0), Default::default(), Default::default(), Default::default(), @@ -73,6 +75,7 @@ fn construct_many() -> Result<()> { let mut chain = Chain::new( Network::Mainnet, + Height(block.coinbase_height().unwrap().0 - 1), Default::default(), Default::default(), Default::default(), @@ -99,6 +102,7 @@ fn ord_matches_work() -> Result<()> { let mut lesser_chain = Chain::new( Network::Mainnet, + Height(0), Default::default(), Default::default(), Default::default(), @@ -109,6 +113,7 @@ fn ord_matches_work() -> Result<()> { let mut bigger_chain = Chain::new( Network::Mainnet, + Height(0), Default::default(), Default::default(), Default::default(), @@ -434,12 +439,12 @@ fn history_tree_is_updated_for_network_upgrade( let chain = state.best_chain().unwrap(); if network_upgrade == NetworkUpgrade::Heartwood { assert!( - chain.history_tree.as_ref().is_none(), + chain.history_block_commitment_tree().as_ref().is_none(), "history tree must not exist yet" ); } else { assert!( - chain.history_tree.as_ref().is_some(), + chain.history_block_commitment_tree().as_ref().is_some(), "history tree must already exist" ); } @@ -453,11 +458,16 @@ fn history_tree_is_updated_for_network_upgrade( let chain = state.best_chain().unwrap(); assert!( - chain.history_tree.as_ref().is_some(), + chain.history_block_commitment_tree().as_ref().is_some(), "history tree must have been (re)created" ); assert_eq!( - chain.history_tree.as_ref().as_ref().unwrap().size(), + chain + .history_block_commitment_tree() + .as_ref() + .as_ref() + .unwrap() + .size(), 1, "history tree must have a single node" ); @@ -466,8 +476,8 @@ fn history_tree_is_updated_for_network_upgrade( let tree = NonEmptyHistoryTree::from_block( Network::Mainnet, activation_block.clone(), - &chain.sapling_note_commitment_tree.root(), - &chain.orchard_note_commitment_tree.root(), + &chain.sapling_note_commitment_tree().root(), + &chain.orchard_note_commitment_tree().root(), ) .unwrap(); @@ -480,7 +490,12 @@ fn history_tree_is_updated_for_network_upgrade( .unwrap(); assert!( - state.best_chain().unwrap().history_tree.as_ref().is_some(), + state + .best_chain() + .unwrap() + .history_block_commitment_tree() + .as_ref() + .is_some(), "history tree must still exist" ); @@ -541,8 +556,8 @@ fn commitment_is_validated_for_network_upgrade(network: Network, network_upgrade let tree = NonEmptyHistoryTree::from_block( Network::Mainnet, activation_block.clone(), - &chain.sapling_note_commitment_tree.root(), - &chain.orchard_note_commitment_tree.root(), + &chain.sapling_note_commitment_tree().root(), + &chain.orchard_note_commitment_tree().root(), ) .unwrap();