debug

chain/chain/src/chain.rs

@@ -1679,7 +1679,13 @@ impl Chain {
 
         tracing::debug!(target: "sync", ?min_height_included, ?new_tail, "adjusting tail for missing chunks");
         new_tail = std::cmp::min(new_tail, min_height_included.saturating_sub(1));
-        let new_chunk_tail = prev_block.chunks().iter().map(|x| x.height_created()).min().unwrap();
+
+        // In order to find the right new_chunk_tail we need to find the minimum
+        // of chunk height_created for chunks in the new tail block.
+        let new_tail_block = self.get_block_by_height(new_tail)?;
+        let new_chunk_tail =
+            new_tail_block.chunks().iter().map(|chunk| chunk.height_created()).min().unwrap();
+
         let tip = Tip::from_header(prev_block.header());
         let final_head = Tip::from_header(self.genesis.header());
         // Update related heads now.

chain/chain/src/garbage_collection.rs

@@ -997,6 +997,7 @@ impl<'a> ChainStoreUpdate<'a> {
             | DBCol::EpochSyncProof
             | DBCol::Misc
             | DBCol::_ReceiptIdToShardId
+            | DBCol::StateShardUIdMapping
             => unreachable!(),
         }
         self.merge(store_update);

chain/chain/src/stateless_validation/chunk_validation.rs

@@ -10,7 +10,7 @@ use crate::types::{
     RuntimeStorageConfig, StorageDataSource,
 };
 use crate::validate::validate_chunk_with_chunk_extra_and_receipts_root;
-use crate::{Chain, ChainStoreAccess};
+use crate::{Chain, ChainStore, ChainStoreAccess};
 use lru::LruCache;
 use near_async::futures::AsyncComputationSpawnerExt;
 use near_chain_primitives::Error;
@@ -28,7 +28,7 @@ use near_primitives::stateless_validation::state_witness::{
 };
 use near_primitives::transaction::SignedTransaction;
 use near_primitives::types::chunk_extra::ChunkExtra;
-use near_primitives::types::{ProtocolVersion, ShardId};
+use near_primitives::types::{ProtocolVersion, ShardId, ShardIndex};
 use near_primitives::utils::compression::CompressedData;
 use near_store::PartialStorage;
 use std::collections::HashMap;
@@ -42,8 +42,9 @@ pub enum MainTransition {
     NewChunk(NewChunkData),
     // TODO(#11881): this is temporary indicator that resharding happened in the
     // state transition covered by state witness. Won't happen in production
-    // until resharding release. Must be removed and replaced with proper
-    // resharding state transition validation.
+    // until resharding release.
+    // Instead, we can store a separate field `resharding_transition` in
+    // `ChunkStateWitness` and use it for proper validation of this case.
     ShardLayoutChange,
 }
 
@@ -111,6 +112,74 @@ pub fn validate_prepared_transactions(
     )
 }
 
+struct StateWitnessBlockRange {
+    /// Blocks from the last new chunk (exclusive) to the parent block
+    /// (inclusive).
+    blocks_after_last_chunk: Vec<Block>,
+    /// Blocks from the last last new chunk (exclusive) to the last new chunk
+    /// (inclusive).
+    blocks_after_last_last_chunk: Vec<Block>,
+    last_chunk_shard_id: ShardId,
+    last_chunk_shard_index: ShardIndex,
+}
+
+fn get_state_witness_block_range(
+    store: &ChainStore,
+    epoch_manager: &dyn EpochManagerAdapter,
+    state_witness: &ChunkStateWitness,
+) -> Result<StateWitnessBlockRange, Error> {
+    let mut blocks_after_last_chunk = Vec::new();
+    let mut blocks_after_last_last_chunk = Vec::new();
+
+    let mut block_hash = *state_witness.chunk_header.prev_block_hash();
+    let mut prev_chunks_seen = 0;
+
+    // It is ok to use the shard id from the header because it is a new
+    // chunk. An old chunk may have the shard id from the parent shard.
+    let (mut current_shard_id, mut current_shard_index) =
+        epoch_manager.get_prev_shard_id(&block_hash, state_witness.chunk_header.shard_id())?;
+
+    let mut last_chunk_shard_id = current_shard_id;
+    let mut last_chunk_shard_index = current_shard_index;
+    loop {
+        let block = store.get_block(&block_hash)?;
+        let prev_hash = *block.header().prev_hash();
+        let chunks = block.chunks();
+        let Some(chunk) = chunks.get(current_shard_index) else {
+            return Err(Error::InvalidChunkStateWitness(format!(
+                "Shard {} does not exist in block {:?}",
+                current_shard_id, block_hash
+            )));
+        };
+        let is_new_chunk = chunk.is_new_chunk(block.header().height());
+        let is_genesis = block.header().is_genesis();
+        if is_new_chunk {
+            prev_chunks_seen += 1;
+        }
+        if prev_chunks_seen == 0 {
+            last_chunk_shard_id = current_shard_id;
+            last_chunk_shard_index = current_shard_index;
+            blocks_after_last_chunk.push(block);
+        } else if prev_chunks_seen == 1 {
+            blocks_after_last_last_chunk.push(block);
+        }
+        if prev_chunks_seen == 2 || is_genesis {
+            break;
+        }
+
+        block_hash = prev_hash;
+        (current_shard_id, current_shard_index) =
+            epoch_manager.get_prev_shard_id(&prev_hash, current_shard_id)?;
+    }
+
+    Ok(StateWitnessBlockRange {
+        blocks_after_last_chunk,
+        blocks_after_last_last_chunk,
+        last_chunk_shard_id,
+        last_chunk_shard_index,
+    })
+}
+
 /// Pre-validates the chunk's receipts and transactions against the chain.
 /// We do this before handing off the computationally intensive part to a
 /// validation thread.
@@ -124,57 +193,12 @@ pub fn pre_validate_chunk_state_witness(
 
     // First, go back through the blockchain history to locate the last new chunk
     // and last last new chunk for the shard.
-
-    // Blocks from the last new chunk (exclusive) to the parent block (inclusive).
-    let mut blocks_after_last_chunk = Vec::new();
-    // Blocks from the last last new chunk (exclusive) to the last new chunk (inclusive).
-    let mut blocks_after_last_last_chunk = Vec::new();
-
-    let (last_chunk_shard_id, last_chunk_shard_index) = {
-        let mut block_hash = *state_witness.chunk_header.prev_block_hash();
-        let mut prev_chunks_seen = 0;
-
-        let epoch_id = state_witness.epoch_id;
-        let shard_layout = epoch_manager.get_shard_layout(&epoch_id)?;
-
-        // It is ok to use the shard id from the header because it is a new
-        // chunk. An old chunk may have the shard id from the parent shard.
-        let mut current_shard_id = state_witness.chunk_header.shard_id();
-        let mut current_shard_index = shard_layout.get_shard_index(current_shard_id);
-
-        let mut last_chunk_shard_id = current_shard_id;
-        let mut last_chunk_shard_index = current_shard_index;
-        loop {
-            let block = store.get_block(&block_hash)?;
-            (current_shard_id, current_shard_index) =
-                epoch_manager.get_prev_shard_id(&block_hash, current_shard_id)?;
-
-            let chunks = block.chunks();
-            let Some(chunk) = chunks.get(current_shard_index) else {
-                return Err(Error::InvalidChunkStateWitness(format!(
-                    "Shard {} does not exist in block {:?}",
-                    current_shard_id, block_hash
-                )));
-            };
-            let is_new_chunk = chunk.is_new_chunk(block.header().height());
-            let is_genesis = block.header().is_genesis();
-            block_hash = *block.header().prev_hash();
-            if is_new_chunk {
-                prev_chunks_seen += 1;
-            }
-            if prev_chunks_seen == 0 {
-                last_chunk_shard_id = current_shard_id;
-                last_chunk_shard_index = current_shard_index;
-                blocks_after_last_chunk.push(block);
-            } else if prev_chunks_seen == 1 {
-                blocks_after_last_last_chunk.push(block);
-            }
-            if prev_chunks_seen == 2 || is_genesis {
-                break;
-            }
-        }
-        (last_chunk_shard_id, last_chunk_shard_index)
-    };
+    let StateWitnessBlockRange {
+        blocks_after_last_chunk,
+        blocks_after_last_last_chunk,
+        last_chunk_shard_id,
+        last_chunk_shard_index,
+    } = get_state_witness_block_range(store, epoch_manager, state_witness)?;
 
     let last_chunk_block = blocks_after_last_last_chunk.first().ok_or_else(|| {
         Error::Other("blocks_after_last_last_chunk is empty, this should be impossible!".into())

chain/chain/src/store/mod.rs

@@ -214,6 +214,10 @@ pub trait ChainStoreAccess {
         mut block_hash: CryptoHash,
         last_chunk_height_included: BlockHeight,
     ) -> Result<Vec<ReceiptProofResponse>, Error> {
+        println!(
+            "get_incoming_receipts_for_shard {} {:?} {}",
+            shard_id, block_hash, last_chunk_height_included
+        );
         let _span =
             tracing::debug_span!(target: "chain", "get_incoming_receipts_for_shard", ?shard_id, ?block_hash, last_chunk_height_included).entered();
 

chain/client/src/client_actor.rs

@@ -620,6 +620,8 @@ impl Handler<StateResponse> for ClientActorInner {
                         shard_id,
                         state_response,
                         &mut self.client.chain,
+                        self.state_parts_future_spawner.as_ref(),
+                        self.client.runtime_adapter.clone(),
                     );
                     return;
                 }
@@ -637,6 +639,8 @@ impl Handler<StateResponse> for ClientActorInner {
                     shard_id,
                     state_response,
                     &mut self.client.chain,
+                    self.state_parts_future_spawner.as_ref(),
+                    self.client.runtime_adapter.clone(),
                 );
                 return;
             }

chain/client/src/stateless_validation/state_witness_producer.rs

@@ -57,6 +57,7 @@ impl Client {
         }
 
         let height = chunk_header.height_created();
+        println!("get_chunk_validator_assignments");
         if self
             .epoch_manager
             .get_chunk_validator_assignments(epoch_id, shard_id, height)?
@@ -88,6 +89,7 @@ impl Client {
         chunk: &ShardChunk,
         transactions_storage_proof: Option<PartialState>,
     ) -> Result<ChunkStateWitness, Error> {
+        println!("create_state_witness");
         let chunk_header = chunk.cloned_header();
         let epoch_id =
             self.epoch_manager.get_epoch_id_from_prev_block(chunk_header.prev_block_hash())?;
@@ -253,6 +255,7 @@ impl Client {
         )?;
 
         // Convert to the right format (from [block_hash -> Vec<ReceiptProof>] to [chunk_hash -> ReceiptProof])
+        println!("incoming_receipt_proofs");
         let mut source_receipt_proofs = HashMap::new();
         for receipt_proof_response in incoming_receipt_proofs {
             let from_block = self.chain.chain_store().get_block(&receipt_proof_response.0)?;
@@ -261,6 +264,7 @@ impl Client {
             for proof in receipt_proof_response.1.iter() {
                 let from_shard_id = proof.1.from_shard_id;
                 let from_shard_index = shard_layout.get_shard_index(from_shard_id);
+                println!("from: id: {}, index: {}", from_shard_id, from_shard_index);
                 let from_chunk_hash = from_block
                     .chunks()
                     .get(from_shard_index)