From 9e81c7b6257773ddca970982adb89a1e0d548e2b Mon Sep 17 00:00:00 2001 From: Stan Bondi Date: Tue, 11 Oct 2022 18:00:41 +0200 Subject: [PATCH] fix(core)!: adds utxo and block info to get_template_registrations request (#4789) Description --- - adds UTXO and block info to get_template_registrations request - adds end_height to get_template_registrations request - Changes validator node registration key to - uses CompositeKey for all composite keys - Removed some unused (previous contract code) response types Motivation and Context --- DAN template scanner requires block hash info to determine last scanned height. Indexing template validations by block height enable more efficient retrievals NOTE: blockchain db will need to be resynced How Has This Been Tested? --- Manually - DAN block scanner --- .../tari_app_grpc/proto/base_node.proto | 28 ++- .../tari_app_grpc/proto/validator_node.proto | 2 +- .../src/grpc/base_node_grpc_server.rs | 221 +++++++++++++----- .../comms_interface/comms_request.rs | 18 +- .../comms_interface/comms_response.rs | 31 +-- .../comms_interface/inbound_handlers.rs | 21 +- .../comms_interface/local_interface.rs | 29 ++- base_layer/core/src/chain_storage/async_db.rs | 6 +- .../src/chain_storage/blockchain_backend.rs | 8 +- .../src/chain_storage/blockchain_database.rs | 21 +- .../core/src/chain_storage/db_transaction.rs | 4 +- base_layer/core/src/chain_storage/error.rs | 2 + .../chain_storage/lmdb_db/composite_key.rs | 41 +++- .../core/src/chain_storage/lmdb_db/lmdb_db.rs | 136 ++++++----- .../core/src/chain_storage/lmdb_db/mod.rs | 1 + base_layer/core/src/chain_storage/mod.rs | 2 +- .../src/chain_storage/template_registation.rs | 7 +- .../core/src/test_helpers/blockchain.rs | 13 +- .../transaction_components/output_type.rs | 7 + .../side_chain/template_registration.rs | 38 +-- 20 files changed, 411 insertions(+), 225 deletions(-) diff --git a/applications/tari_app_grpc/proto/base_node.proto b/applications/tari_app_grpc/proto/base_node.proto index 7902ee8232..f18c39aefc 100644 --- a/applications/tari_app_grpc/proto/base_node.proto +++ b/applications/tari_app_grpc/proto/base_node.proto @@ -94,7 +94,8 @@ service BaseNode { rpc GetCommittee(GetCommitteeRequest) returns (GetCommitteeResponse); rpc GetShardKey(GetShardKeyRequest) returns (GetShardKeyResponse); // Get templates - rpc GetTemplateRegistrations(GetTemplateRegistrationsRequest) returns (stream TemplateRegistration); + rpc GetTemplateRegistrations(GetTemplateRegistrationsRequest) returns (stream GetTemplateRegistrationResponse); + rpc GetSideChainUtxos(GetSideChainUtxosRequest) returns (stream GetSideChainUtxosResponse); } message GetAssetMetadataRequest { @@ -471,5 +472,28 @@ message GetShardKeyResponse { } message GetTemplateRegistrationsRequest { - uint64 from_height = 1; + bytes start_hash = 1; + uint64 count = 2; +} + +message GetTemplateRegistrationResponse { + bytes utxo_hash = 1; + TemplateRegistration registration = 2; +} + +message BlockInfo { + uint64 height = 1; + bytes hash = 2; + bytes next_block_hash = 3; } + +message GetSideChainUtxosRequest { + bytes start_hash = 1; + uint64 count = 2; +} + +message GetSideChainUtxosResponse { + BlockInfo block_info = 1; + repeated TransactionOutput outputs = 2; +} + diff --git a/applications/tari_app_grpc/proto/validator_node.proto b/applications/tari_app_grpc/proto/validator_node.proto index a549134607..9b8d73ebb1 100644 --- a/applications/tari_app_grpc/proto/validator_node.proto +++ b/applications/tari_app_grpc/proto/validator_node.proto @@ -118,7 +118,7 @@ message Authority { bytes proxied_by = 3; } -message InvokeMethodRequest{ +message InvokeMethodRequest { bytes contract_id = 1; uint32 template_id = 2; string method = 3; diff --git a/applications/tari_base_node/src/grpc/base_node_grpc_server.rs b/applications/tari_base_node/src/grpc/base_node_grpc_server.rs index 1479520768..e82ca58d9c 100644 --- a/applications/tari_base_node/src/grpc/base_node_grpc_server.rs +++ b/applications/tari_base_node/src/grpc/base_node_grpc_server.rs @@ -33,7 +33,7 @@ use tari_app_grpc::{ tari_rpc::{CalcType, Sorting}, }; use tari_app_utilities::consts; -use tari_common_types::types::{Commitment, PublicKey, Signature}; +use tari_common_types::types::{Commitment, FixedHash, PublicKey, Signature}; use tari_comms::{Bytes, CommsNode}; use tari_core::{ base_node::{ @@ -140,7 +140,8 @@ impl tari_rpc::base_node_server::BaseNode for BaseNodeGrpcServer { type GetMempoolTransactionsStream = mpsc::Receiver>; type GetNetworkDifficultyStream = mpsc::Receiver>; type GetPeersStream = mpsc::Receiver>; - type GetTemplateRegistrationsStream = mpsc::Receiver>; + type GetSideChainUtxosStream = mpsc::Receiver>; + type GetTemplateRegistrationsStream = mpsc::Receiver>; type GetTokensInCirculationStream = mpsc::Receiver>; type ListHeadersStream = mpsc::Receiver>; type SearchKernelsStream = mpsc::Receiver>; @@ -1484,7 +1485,6 @@ impl tari_rpc::base_node_server::BaseNode for BaseNodeGrpcServer { request: Request, ) -> Result, Status> { let request = request.into_inner(); - let report_error_flag = self.report_error_flag(); debug!(target: LOG_TARGET, "Incoming GRPC request for GetActiveValidatorNodes"); let mut handler = self.node_service.clone(); @@ -1493,39 +1493,24 @@ impl tari_rpc::base_node_server::BaseNode for BaseNodeGrpcServer { task::spawn(async move { let active_validator_nodes = match handler.get_active_validator_nodes(request.height).await { Err(err) => { - warn!(target: LOG_TARGET, "Error communicating with base node: {}", err,); + warn!(target: LOG_TARGET, "Base node service error: {}", err,); return; }, Ok(data) => data, }; - // dbg!(&active_validator_nodes); + for (public_key, shard_key) in active_validator_nodes { let active_validator_node = tari_rpc::GetActiveValidatorNodesResponse { public_key: public_key.to_vec(), shard_key: shard_key.to_vec(), }; - match tx.send(Ok(active_validator_node)).await { - Ok(_) => (), - Err(err) => { - warn!( - target: LOG_TARGET, - "Error sending mempool transaction via GRPC: {}", err - ); - match tx - .send(Err(obscure_error_if_true( - report_error_flag, - Status::unknown("Error sending data"), - ))) - .await - { - Ok(_) => (), - Err(send_err) => { - warn!(target: LOG_TARGET, "Error sending error to GRPC client: {}", send_err) - }, - } - return; - }, + if tx.send(Ok(active_validator_node)).await.is_err() { + debug!( + target: LOG_TARGET, + "[get_active_validator_nodes] Client has disconnected before stream completed" + ); + return; } } }); @@ -1544,63 +1529,193 @@ impl tari_rpc::base_node_server::BaseNode for BaseNodeGrpcServer { let report_error_flag = self.report_error_flag(); debug!(target: LOG_TARGET, "Incoming GRPC request for GetTemplateRegistrations"); - let mut handler = self.node_service.clone(); - let (mut tx, rx) = mpsc::channel(1000); + let (mut tx, rx) = mpsc::channel(10); + + let start_hash = Some(request.start_hash) + .filter(|x| !x.is_empty()) + .map(FixedHash::try_from) + .transpose() + .map_err(|_| Status::invalid_argument("Invalid start_hash"))?; + + let mut node_service = self.node_service.clone(); + + let start_height = match start_hash { + Some(hash) => { + let header = node_service + .get_header_by_hash(hash) + .await + .map_err(|err| obscure_error_if_true(self.report_grpc_error, Status::internal(err.to_string())))?; + header + .map(|h| h.height()) + .ok_or_else(|| Status::not_found("Start hash not found"))? + }, + None => 0, + }; + + if request.count == 0 { + return Ok(Response::new(rx)); + } + + let end_height = start_height + .checked_add(request.count) + .ok_or_else(|| Status::invalid_argument("Request start height + count overflows u64"))?; task::spawn(async move { - let template_registrations = match handler.get_template_registrations(request.from_height).await { + let template_registrations = match node_service.get_template_registrations(start_height, end_height).await { Err(err) => { - warn!(target: LOG_TARGET, "Error communicating with base node: {}", err,); + warn!(target: LOG_TARGET, "Base node service error: {}", err); return; }, Ok(data) => data, }; for template_registration in template_registrations { - let template_registration = match tari_rpc::TemplateRegistration::try_from(template_registration) { + let registration = match template_registration.registration_data.try_into() { Ok(t) => t, Err(e) => { warn!( target: LOG_TARGET, "Error sending converting template registration for GRPC: {}", e ); - match tx + let _ignore = tx .send(Err(obscure_error_if_true( report_error_flag, - Status::internal("Error converting template_registration"), + Status::internal(format!("Error converting template_registration: {}", e)), ))) - .await - { - Ok(_) => (), - Err(send_err) => { - warn!(target: LOG_TARGET, "Error sending error to GRPC client: {}", send_err) - }, - } + .await; return; }, }; - match tx.send(Ok(template_registration)).await { - Ok(_) => (), - Err(err) => { + let resp = tari_rpc::GetTemplateRegistrationResponse { + utxo_hash: template_registration.output_hash.to_vec(), + registration: Some(registration), + }; + + if tx.send(Ok(resp)).await.is_err() { + debug!( + target: LOG_TARGET, + "[get_template_registrations] Client has disconnected before stream completed" + ); + return; + } + } + }); + debug!( + target: LOG_TARGET, + "Sending GetTemplateRegistrations response stream to client" + ); + Ok(Response::new(rx)) + } + + async fn get_side_chain_utxos( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + let report_error_flag = self.report_error_flag(); + debug!(target: LOG_TARGET, "Incoming GRPC request for GetTemplateRegistrations"); + + let (mut tx, rx) = mpsc::channel(10); + + let start_hash = Some(request.start_hash) + .filter(|x| !x.is_empty()) + .map(FixedHash::try_from) + .transpose() + .map_err(|_| Status::invalid_argument("Invalid start_hash"))?; + + let mut node_service = self.node_service.clone(); + + let start_header = match start_hash { + Some(hash) => node_service + .get_header_by_hash(hash) + .await + .map_err(|err| obscure_error_if_true(self.report_grpc_error, Status::internal(err.to_string())))? + .ok_or_else(|| Status::not_found("Start hash not found"))?, + None => node_service + .get_header(0) + .await + .map_err(|err| obscure_error_if_true(self.report_grpc_error, Status::internal(err.to_string())))? + .ok_or_else(|| Status::unavailable("Genesis block not available"))?, + }; + + if request.count == 0 { + return Ok(Response::new(rx)); + } + + let start_height = start_header.height(); + let end_height = start_height + .checked_add(request.count - 1) + .ok_or_else(|| Status::invalid_argument("Request start height + count overflows u64"))?; + + task::spawn(async move { + let mut current_header = start_header; + + for height in start_height..=end_height { + let header_hash = *current_header.hash(); + let utxos = match node_service.fetch_unspent_utxos_in_block(header_hash).await { + Ok(utxos) => utxos, + Err(e) => { + warn!(target: LOG_TARGET, "Base node service error: {}", e); + return; + }, + }; + + let next_header = match node_service.get_header(height + 1).await { + Ok(h) => h, + Err(e) => { + let _ignore = tx.send(Err(obscure_error_if_true( + report_error_flag, + Status::internal(e.to_string()), + ))); + return; + }, + }; + + let sidechain_outputs = utxos + .into_iter() + .filter(|u| u.features.output_type.is_sidechain_type()) + .collect::>(); + + match sidechain_outputs.into_iter().map(TryInto::try_into).collect() { + Ok(outputs) => { + let resp = tari_rpc::GetSideChainUtxosResponse { + block_info: Some(tari_rpc::BlockInfo { + height: current_header.height(), + hash: header_hash.to_vec(), + next_block_hash: next_header.as_ref().map(|h| h.hash().to_vec()).unwrap_or_default(), + }), + outputs, + }; + + if tx.send(Ok(resp)).await.is_err() { + debug!( + target: LOG_TARGET, + "[get_template_registrations] Client has disconnected before stream completed" + ); + return; + } + }, + Err(e) => { warn!( target: LOG_TARGET, - "Error sending template registration via GRPC: {}", err + "Error sending converting sidechain output for GRPC: {}", e ); - match tx + let _ignore = tx .send(Err(obscure_error_if_true( report_error_flag, - Status::unknown("Error sending data"), + Status::internal(format!("Error converting sidechain output: {}", e)), ))) - .await - { - Ok(_) => (), - Err(send_err) => { - warn!(target: LOG_TARGET, "Error sending error to GRPC client: {}", send_err) - }, - } + .await; return; }, + }; + + match next_header { + Some(header) => { + current_header = header; + }, + None => break, } } }); diff --git a/base_layer/core/src/base_node/comms_interface/comms_request.rs b/base_layer/core/src/base_node/comms_interface/comms_request.rs index 6e95c929bd..f47b4a0859 100644 --- a/base_layer/core/src/base_node/comms_interface/comms_request.rs +++ b/base_layer/core/src/base_node/comms_interface/comms_request.rs @@ -26,7 +26,7 @@ use std::{ }; use serde::{Deserialize, Serialize}; -use tari_common_types::types::{Commitment, HashOutput, PrivateKey, PublicKey, Signature}; +use tari_common_types::types::{BlockHash, Commitment, HashOutput, PrivateKey, PublicKey, Signature}; use tari_utilities::hex::Hex; use crate::{blocks::NewBlockTemplate, chain_storage::MmrTree, proof_of_work::PowAlgorithm}; @@ -76,7 +76,11 @@ pub enum NodeCommsRequest { public_key: PublicKey, }, FetchTemplateRegistrations { - from_height: u64, + start_height: u64, + end_height: u64, + }, + FetchUnspentUtxosInBlock { + block_hash: BlockHash, }, } @@ -127,8 +131,14 @@ impl Display for NodeCommsRequest { GetShardKey { height, public_key } => { write!(f, "GetShardKey height ({}), public key ({:?})", height, public_key) }, - FetchTemplateRegistrations { from_height } => { - write!(f, "FetchTemplateRegistrations ({})", from_height) + FetchTemplateRegistrations { + start_height: start, + end_height: end, + } => { + write!(f, "FetchTemplateRegistrations ({}..={})", start, end) + }, + FetchUnspentUtxosInBlock { block_hash } => { + write!(f, "FetchUnspentUtxosInBlock ({})", block_hash) }, } } diff --git a/base_layer/core/src/base_node/comms_interface/comms_response.rs b/base_layer/core/src/base_node/comms_interface/comms_response.rs index 30509fe6f3..5ad0bbe052 100644 --- a/base_layer/core/src/base_node/comms_interface/comms_response.rs +++ b/base_layer/core/src/base_node/comms_interface/comms_response.rs @@ -32,14 +32,9 @@ use tari_common_types::{ use crate::{ blocks::{Block, ChainHeader, HistoricalBlock, NewBlockTemplate}, - chain_storage::{ActiveValidatorNode, UtxoMinedInfo}, + chain_storage::{ActiveValidatorNode, TemplateRegistrationEntry}, proof_of_work::Difficulty, - transactions::transaction_components::{ - CodeTemplateRegistration, - Transaction, - TransactionKernel, - TransactionOutput, - }, + transactions::transaction_components::{Transaction, TransactionKernel, TransactionOutput}, }; /// API Response enum @@ -60,26 +55,11 @@ pub enum NodeCommsResponse { }, TargetDifficulty(Difficulty), MmrNodes(Vec, Vec), - FetchTokensResponse { - outputs: Vec<(TransactionOutput, u64)>, - }, - FetchAssetRegistrationsResponse { - outputs: Vec, - }, - FetchAssetMetadataResponse { - output: Box>, - }, FetchMempoolTransactionsByExcessSigsResponse(FetchMempoolTransactionsResponse), - FetchOutputsForBlockResponse { - outputs: Vec, - }, - FetchOutputsByContractIdResponse { - outputs: Vec, - }, FetchValidatorNodesKeysResponse(Vec<(PublicKey, [u8; 32])>), FetchCommitteeResponse(Vec), GetShardKeyResponse(Option<[u8; 32]>), - FetchTemplateRegistrationsResponse(Vec), + FetchTemplateRegistrationsResponse(Vec), } impl Display for NodeCommsResponse { @@ -107,17 +87,12 @@ impl Display for NodeCommsResponse { ), TargetDifficulty(_) => write!(f, "TargetDifficulty"), MmrNodes(_, _) => write!(f, "MmrNodes"), - FetchTokensResponse { .. } => write!(f, "FetchTokensResponse"), - FetchAssetRegistrationsResponse { .. } => write!(f, "FetchAssetRegistrationsResponse"), - FetchAssetMetadataResponse { .. } => write!(f, "FetchAssetMetadataResponse"), FetchMempoolTransactionsByExcessSigsResponse(resp) => write!( f, "FetchMempoolTransactionsByExcessSigsResponse({} transaction(s), {} not found)", resp.transactions.len(), resp.not_found.len() ), - FetchOutputsForBlockResponse { .. } => write!(f, "FetchConstitutionsResponse"), - FetchOutputsByContractIdResponse { .. } => write!(f, "FetchOutputsByContractIdResponse"), FetchValidatorNodesKeysResponse(_) => write!(f, "FetchValidatorNodesKeysResponse"), FetchCommitteeResponse(_) => write!(f, "FetchCommitteeResponse"), GetShardKeyResponse(_) => write!(f, "GetShardKeyResponse"), diff --git a/base_layer/core/src/base_node/comms_interface/inbound_handlers.rs b/base_layer/core/src/base_node/comms_interface/inbound_handlers.rs index 9916e00230..f1c1774015 100644 --- a/base_layer/core/src/base_node/comms_interface/inbound_handlers.rs +++ b/base_layer/core/src/base_node/comms_interface/inbound_handlers.rs @@ -377,18 +377,27 @@ where B: BlockchainBackend + 'static let shard_key = self.blockchain_db.get_shard_key(height, public_key).await?; Ok(NodeCommsResponse::GetShardKeyResponse(shard_key)) }, - NodeCommsRequest::FetchTemplateRegistrations { from_height } => { + NodeCommsRequest::FetchTemplateRegistrations { + start_height, + end_height, + } => { let template_registrations = self .blockchain_db - .fetch_template_registrations(from_height) - .await? - .into_iter() - .map(|tr| tr.registration_data) - .collect(); + .fetch_template_registrations(start_height..=end_height) + .await?; Ok(NodeCommsResponse::FetchTemplateRegistrationsResponse( template_registrations, )) }, + NodeCommsRequest::FetchUnspentUtxosInBlock { block_hash } => { + let utxos = self.blockchain_db.fetch_outputs_in_block(block_hash).await?; + Ok(NodeCommsResponse::TransactionOutputs( + utxos + .into_iter() + .filter_map(|utxo| utxo.into_unpruned_output()) + .collect(), + )) + }, } } diff --git a/base_layer/core/src/base_node/comms_interface/local_interface.rs b/base_layer/core/src/base_node/comms_interface/local_interface.rs index b2a7a115f9..48c093e757 100644 --- a/base_layer/core/src/base_node/comms_interface/local_interface.rs +++ b/base_layer/core/src/base_node/comms_interface/local_interface.rs @@ -38,9 +38,9 @@ use crate::{ NodeCommsResponse, }, blocks::{Block, ChainHeader, HistoricalBlock, NewBlockTemplate}, - chain_storage::ActiveValidatorNode, + chain_storage::{ActiveValidatorNode, TemplateRegistrationEntry}, proof_of_work::PowAlgorithm, - transactions::transaction_components::{CodeTemplateRegistration, TransactionKernel, TransactionOutput}, + transactions::transaction_components::{TransactionKernel, TransactionOutput}, }; pub type BlockEventSender = broadcast::Sender>; @@ -327,15 +327,34 @@ impl LocalNodeCommsInterface { pub async fn get_template_registrations( &mut self, - from_height: u64, - ) -> Result, CommsInterfaceError> { + start_height: u64, + end_height: u64, + ) -> Result, CommsInterfaceError> { match self .request_sender - .call(NodeCommsRequest::FetchTemplateRegistrations { from_height }) + .call(NodeCommsRequest::FetchTemplateRegistrations { + start_height, + end_height, + }) .await?? { NodeCommsResponse::FetchTemplateRegistrationsResponse(template_registrations) => Ok(template_registrations), _ => Err(CommsInterfaceError::UnexpectedApiResponse), } } + + /// Fetches UTXOs that are not spent for the given block hash up to the current chain tip. + pub async fn fetch_unspent_utxos_in_block( + &mut self, + block_hash: BlockHash, + ) -> Result, CommsInterfaceError> { + match self + .request_sender + .call(NodeCommsRequest::FetchUnspentUtxosInBlock { block_hash }) + .await?? + { + NodeCommsResponse::TransactionOutputs(outputs) => Ok(outputs), + _ => Err(CommsInterfaceError::UnexpectedApiResponse), + } + } } diff --git a/base_layer/core/src/chain_storage/async_db.rs b/base_layer/core/src/chain_storage/async_db.rs index 0540182a89..12ee7a45d2 100644 --- a/base_layer/core/src/chain_storage/async_db.rs +++ b/base_layer/core/src/chain_storage/async_db.rs @@ -30,7 +30,7 @@ use tari_common_types::{ }; use tari_utilities::epoch_time::EpochTime; -use super::{ActiveValidatorNode, TemplateRegistration}; +use super::{ActiveValidatorNode, TemplateRegistrationEntry}; use crate::{ blocks::{ Block, @@ -163,6 +163,8 @@ impl AsyncBlockchainDb { make_async_fn!(fetch_utxos_in_block(hash: HashOutput, deleted: Option>) -> (Vec, Bitmap), "fetch_utxos_in_block"); + make_async_fn!(fetch_outputs_in_block(hash: HashOutput) -> Vec, "fetch_outputs_in_block"); + make_async_fn!(utxo_count() -> usize, "utxo_count"); //---------------------------------- Kernel --------------------------------------------// @@ -271,7 +273,7 @@ impl AsyncBlockchainDb { make_async_fn!(get_shard_key(height:u64, public_key: PublicKey) -> Option<[u8;32]>, "get_shard_key"); - make_async_fn!(fetch_template_registrations(from_height: u64) -> Vec, "fetch_template_registrations"); + make_async_fn!(fetch_template_registrations>(range: T) -> Vec, "fetch_template_registrations"); } impl From> for AsyncBlockchainDb { diff --git a/base_layer/core/src/chain_storage/blockchain_backend.rs b/base_layer/core/src/chain_storage/blockchain_backend.rs index b653477bd8..05d8ca33d6 100644 --- a/base_layer/core/src/chain_storage/blockchain_backend.rs +++ b/base_layer/core/src/chain_storage/blockchain_backend.rs @@ -7,7 +7,7 @@ use tari_common_types::{ types::{Commitment, HashOutput, PublicKey, Signature}, }; -use super::{ActiveValidatorNode, TemplateRegistration}; +use super::{ActiveValidatorNode, TemplateRegistrationEntry}; use crate::{ blocks::{ Block, @@ -196,5 +196,9 @@ pub trait BlockchainBackend: Send + Sync { fn fetch_active_validator_nodes(&self, height: u64) -> Result, ChainStorageError>; fn fetch_committee(&self, height: u64, shard: [u8; 32]) -> Result, ChainStorageError>; fn get_shard_key(&self, height: u64, public_key: PublicKey) -> Result, ChainStorageError>; - fn fetch_template_registrations(&self, from_height: u64) -> Result, ChainStorageError>; + fn fetch_template_registrations( + &self, + start_height: u64, + end_height: u64, + ) -> Result, ChainStorageError>; } diff --git a/base_layer/core/src/chain_storage/blockchain_database.rs b/base_layer/core/src/chain_storage/blockchain_database.rs index 225fca8035..ca893c34ca 100644 --- a/base_layer/core/src/chain_storage/blockchain_database.rs +++ b/base_layer/core/src/chain_storage/blockchain_database.rs @@ -41,7 +41,7 @@ use tari_common_types::{ use tari_mmr::pruned_hashset::PrunedHashSet; use tari_utilities::{epoch_time::EpochTime, hex::Hex, ByteArray}; -use super::{ActiveValidatorNode, TemplateRegistration}; +use super::{ActiveValidatorNode, TemplateRegistrationEntry}; use crate::{ blocks::{ Block, @@ -439,6 +439,11 @@ where B: BlockchainBackend db.fetch_utxos_in_block(&hash, deleted.as_deref()) } + pub fn fetch_outputs_in_block(&self, hash: HashOutput) -> Result, ChainStorageError> { + let db = self.db_read_access()?; + db.fetch_outputs_in_block(&hash) + } + /// Returns the number of UTXOs in the current unspent set pub fn utxo_count(&self) -> Result { let db = self.db_read_access()?; @@ -1188,12 +1193,18 @@ where B: BlockchainBackend db.fetch_committee(height, shard) } - pub fn fetch_template_registrations( + pub fn fetch_template_registrations>( &self, - from_height: u64, - ) -> Result, ChainStorageError> { + range: T, + ) -> Result, ChainStorageError> { let db = self.db_read_access()?; - db.fetch_template_registrations(from_height) + let (start, mut end) = convert_to_option_bounds(range); + if end.is_none() { + // `(n..)` means fetch block headers until this node's tip + end = Some(db.fetch_last_header()?.height); + } + let (start, end) = (start.unwrap_or(0), end.unwrap()); + db.fetch_template_registrations(start, end) } } diff --git a/base_layer/core/src/chain_storage/db_transaction.rs b/base_layer/core/src/chain_storage/db_transaction.rs index e0cf01008e..d2c2f706db 100644 --- a/base_layer/core/src/chain_storage/db_transaction.rs +++ b/base_layer/core/src/chain_storage/db_transaction.rs @@ -30,7 +30,7 @@ use croaring::Bitmap; use tari_common_types::types::{BlockHash, Commitment, HashOutput}; use tari_utilities::hex::Hex; -use super::{ActiveValidatorNode, TemplateRegistration}; +use super::{ActiveValidatorNode, TemplateRegistrationEntry}; use crate::{ blocks::{Block, BlockHeader, BlockHeaderAccumulatedData, ChainBlock, ChainHeader, UpdateBlockAccumulatedData}, chain_storage::{error::ChainStorageError, HorizonData, Reorg}, @@ -363,7 +363,7 @@ pub enum WriteOperation { validator_node: ActiveValidatorNode, }, InsertTemplateRegistration { - template_registration: TemplateRegistration, + template_registration: TemplateRegistrationEntry, }, } diff --git a/base_layer/core/src/chain_storage/error.rs b/base_layer/core/src/chain_storage/error.rs index 93456647ce..62d33a6a91 100644 --- a/base_layer/core/src/chain_storage/error.rs +++ b/base_layer/core/src/chain_storage/error.rs @@ -134,6 +134,8 @@ pub enum ChainStorageError { UnspendableDueToDependentUtxos { details: String }, #[error("FixedHashSize Error: {0}")] FixedHashSizeError(#[from] FixedHashSizeError), + #[error("Composite key length was exceeded (THIS SHOULD NEVER HAPPEN)")] + CompositeKeyLengthExceeded, } impl ChainStorageError { diff --git a/base_layer/core/src/chain_storage/lmdb_db/composite_key.rs b/base_layer/core/src/chain_storage/lmdb_db/composite_key.rs index 565feb8104..0d6a7f5c09 100644 --- a/base_layer/core/src/chain_storage/lmdb_db/composite_key.rs +++ b/base_layer/core/src/chain_storage/lmdb_db/composite_key.rs @@ -25,15 +25,18 @@ use std::{ ops::{Deref, DerefMut}, }; +use lmdb_zero::traits::AsLmdbBytes; use tari_utilities::hex::to_hex; -#[derive(Debug, Clone, Copy)] -pub(super) struct CompositeKey { - bytes: [u8; KEY_LEN], +use crate::chain_storage::ChainStorageError; + +#[derive(Debug, Clone)] +pub(super) struct CompositeKey { + bytes: Box<[u8; L]>, len: usize, } -impl CompositeKey { +impl CompositeKey { pub fn new() -> Self { Self { bytes: Self::new_buf(), @@ -41,10 +44,20 @@ impl CompositeKey { } } + pub fn try_from_parts>(parts: &[T]) -> Result { + let mut key = Self::new(); + for part in parts { + if !key.push(part) { + return Err(ChainStorageError::CompositeKeyLengthExceeded); + } + } + Ok(key) + } + pub fn push>(&mut self, bytes: T) -> bool { let b = bytes.as_ref(); let new_len = self.len + b.len(); - if new_len > KEY_LEN { + if new_len > L { return false; } self.bytes[self.len..new_len].copy_from_slice(b); @@ -61,18 +74,18 @@ impl CompositeKey { } /// Returns a fixed 0-filled byte array. - const fn new_buf() -> [u8; KEY_LEN] { - [0x0u8; KEY_LEN] + fn new_buf() -> Box<[u8; L]> { + Box::new([0x0u8; L]) } } -impl Display for CompositeKey { +impl Display for CompositeKey { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "{}", to_hex(self.as_bytes())) } } -impl Deref for CompositeKey { +impl Deref for CompositeKey { type Target = [u8]; fn deref(&self) -> &Self::Target { @@ -80,14 +93,20 @@ impl Deref for CompositeKey { } } -impl DerefMut for CompositeKey { +impl DerefMut for CompositeKey { fn deref_mut(&mut self) -> &mut Self::Target { self.as_bytes_mut() } } -impl AsRef<[u8]> for CompositeKey { +impl AsRef<[u8]> for CompositeKey { fn as_ref(&self) -> &[u8] { self.as_bytes() } } + +impl AsLmdbBytes for CompositeKey { + fn as_lmdb_bytes(&self) -> &[u8] { + self.as_bytes() + } +} diff --git a/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs b/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs index 627125350f..7cbb3f1ac3 100644 --- a/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs +++ b/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs @@ -20,18 +20,12 @@ // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE // USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// Because we use dynamically sized u8 vectors for hash types through the type alias HashOutput, -// let's ignore this clippy error in this module - -#![allow(clippy::ptr_arg)] - use std::{ collections::HashMap, convert::TryFrom, fmt, fs, fs::File, - mem, ops::Deref, path::Path, sync::Arc, @@ -69,6 +63,7 @@ use crate::{ db_transaction::{DbKey, DbTransaction, DbValue, WriteOperation}, error::{ChainStorageError, OrNotFound}, lmdb_db::{ + composite_key::CompositeKey, lmdb::{ fetch_db_entry_sizes, lmdb_clear, @@ -103,7 +98,7 @@ use crate::{ MmrTree, PrunedOutput, Reorg, - TemplateRegistration, + TemplateRegistrationEntry, }, consensus::ConsensusManager, transactions::{ @@ -148,6 +143,15 @@ const LMDB_DB_VALIDATOR_NODES_MAPPING: &str = "validator_nodes_mapping"; const LMDB_DB_VALIDATOR_NODE_ENDING: &str = "validator_node_ending"; const LMDB_DB_TEMPLATE_REGISTRATIONS: &str = "template_registrations"; +/// HeaderHash(32), mmr_pos(4), hash(32) +type InputKey = CompositeKey<68>; +/// HeaderHash(32), mmr_pos(4), hash(32) +type KernelKey = CompositeKey<68>; +/// HeaderHash(32), mmr_pos(4), hash(32) +type OutputKey = CompositeKey<68>; +/// Height(8), Hash(32) +type ValidatorNodeRegistrationKey = CompositeKey<40>; + pub fn create_lmdb_database>( path: P, config: LMDBConfig, @@ -257,7 +261,7 @@ pub struct LMDBDatabase { validator_nodes_mapping: DatabaseRef, /// Maps the end block height of nodes validator_nodes_ending: DatabaseRef, - /// Maps CodeTemplateRegistration hash-> TemplateRegistration + /// Maps CodeTemplateRegistration -> TemplateRegistration template_registrations: DatabaseRef, _file_lock: Arc, consensus_manager: ConsensusManager, @@ -562,19 +566,16 @@ impl LMDBDatabase { key: &OutputKey, ) -> Result { let mut output: TransactionOutputRowData = - lmdb_get(txn, &self.utxos_db, key.as_bytes()).or_not_found("TransactionOutput", "key", key.to_hex())?; + lmdb_get(txn, &self.utxos_db, key).or_not_found("TransactionOutput", "key", key.to_string())?; let pruned_output = output .output .take() .ok_or_else(|| ChainStorageError::DataInconsistencyDetected { function: "prune_output", - details: format!( - "Attempt to prune output that has already been pruned for key {}", - key.to_hex() - ), + details: format!("Attempt to prune output that has already been pruned for key {}", key), })?; // output.output is None - lmdb_replace(txn, &self.utxos_db, key.as_bytes(), &output)?; + lmdb_replace(txn, &self.utxos_db, key, &output)?; Ok(pruned_output) } @@ -590,7 +591,7 @@ impl LMDBDatabase { let output_hash = output.hash(); let witness_hash = output.witness_hash(); - let output_key = OutputKey::new(header_hash.as_slice(), mmr_position, &[]); + let output_key = OutputKey::try_from_parts(&[header_hash.as_slice(), mmr_position.to_le_bytes().as_slice()])?; lmdb_insert( txn, @@ -604,13 +605,13 @@ impl LMDBDatabase { txn, &*self.txos_hash_to_index_db, output_hash.as_slice(), - &(mmr_position, output_key.as_bytes()), + &(mmr_position, output_key.to_vec()), "txos_hash_to_index_db", )?; lmdb_insert( txn, &*self.utxos_db, - output_key.as_bytes(), + &output_key, &TransactionOutputRowData { output: Some(output.clone()), header_hash: *header_hash, @@ -642,18 +643,18 @@ impl LMDBDatabase { header_hash.to_hex(), ))); } - let key = OutputKey::new(header_hash.as_slice(), mmr_position, &[]); + let key = OutputKey::try_from_parts(&[header_hash.as_slice(), mmr_position.to_le_bytes().as_slice()])?; lmdb_insert( txn, &*self.txos_hash_to_index_db, output_hash.as_slice(), - &(mmr_position, key.as_bytes()), + &(mmr_position, key.to_vec()), "txos_hash_to_index_db", )?; lmdb_insert( txn, &*self.utxos_db, - key.as_bytes(), + &key, &TransactionOutputRowData { output: None, header_hash: *header_hash, @@ -675,7 +676,11 @@ impl LMDBDatabase { mmr_position: u32, ) -> Result<(), ChainStorageError> { let hash = kernel.hash(); - let key = KernelKey::new(header_hash.as_slice(), mmr_position, hash.as_slice()); + let key = KernelKey::try_from_parts(&[ + header_hash.as_slice(), + mmr_position.to_le_bytes().as_slice(), + hash.as_slice(), + ])?; lmdb_insert( txn, @@ -699,7 +704,7 @@ impl LMDBDatabase { lmdb_insert( txn, &*self.kernels_db, - key.as_bytes(), + &key, &TransactionKernelRowData { kernel: kernel.clone(), header_hash: *header_hash, @@ -738,11 +743,15 @@ impl LMDBDatabase { )?; let hash = input.canonical_hash(); - let key = InputKey::new(header_hash.as_slice(), mmr_position, hash.as_slice()); + let key = InputKey::try_from_parts(&[ + header_hash.as_slice(), + mmr_position.to_le_bytes().as_slice(), + hash.as_slice(), + ])?; lmdb_insert( txn, &*self.inputs_db, - key.as_bytes(), + &key, &TransactionInputRowDataRef { input: &input.to_compact(), header_hash, @@ -1315,6 +1324,7 @@ impl LMDBDatabase { )) })?; + let output_hash = output.hash(); if let Some(vn_reg) = output .features .sidechain_feature @@ -1332,7 +1342,7 @@ impl LMDBDatabase { .consensus_constants(header.height) .validator_node_timeout(), public_key: vn_reg.public_key.clone(), - output_hash: output.hash(), + output_hash, }; self.insert_validator_node(txn, &validator_node)?; } @@ -1342,9 +1352,11 @@ impl LMDBDatabase { .as_ref() .and_then(|f| f.template_registration()) { - let record = TemplateRegistration { + let record = TemplateRegistrationEntry { registration_data: template_reg.clone(), - height: header.height, + output_hash, + block_height: header.height, + block_hash, }; self.insert_template_registration(txn, &record)?; @@ -1500,8 +1512,8 @@ impl LMDBDatabase { &u64::from(pos + 1).to_be_bytes(), ) .or_not_found("BlockHeader", "mmr_position", pos.to_string())?; - let key = OutputKey::new(&hash, *pos, &[]); - debug!(target: LOG_TARGET, "Pruning output: {}", key.to_hex()); + let key = OutputKey::try_from_parts(&[hash.as_slice(), pos.to_le_bytes().as_slice()])?; + debug!(target: LOG_TARGET, "Pruning output: {}", key); self.prune_output(write_txn, &key)?; } @@ -1642,13 +1654,16 @@ impl LMDBDatabase { fn insert_template_registration( &self, txn: &WriteTransaction<'_>, - template_registration: &TemplateRegistration, + template_registration: &TemplateRegistrationEntry, ) -> Result<(), ChainStorageError> { - let key = template_registration.registration_data.hash(); + let key = ValidatorNodeRegistrationKey::try_from_parts(&[ + template_registration.block_height.to_le_bytes().as_slice(), + template_registration.output_hash.as_slice(), + ])?; lmdb_insert( txn, &self.template_registrations, - key.as_bytes(), + &key, template_registration, "template_registrations", ) @@ -2050,8 +2065,12 @@ impl BlockchainBackend for LMDBDatabase { if let Some((header_hash, mmr_position, hash)) = lmdb_get::<_, (HashOutput, u32, HashOutput)>(&txn, &self.kernel_excess_sig_index, key.as_slice())? { - let key = KernelKey::new(header_hash.deref(), mmr_position, hash.deref()); - Ok(lmdb_get(&txn, &self.kernels_db, key.as_bytes())? + let key = KernelKey::try_from_parts(&[ + header_hash.as_slice(), + mmr_position.to_le_bytes().as_slice(), + hash.as_slice(), + ])?; + Ok(lmdb_get(&txn, &self.kernels_db, &key)? .map(|kernel: TransactionKernelRowData| (kernel.kernel, header_hash))) } else { Ok(None) @@ -2576,16 +2595,22 @@ impl BlockchainBackend for LMDBDatabase { Ok(validator_nodes.into_iter().map(|a| a.shard_key).last()) } - fn fetch_template_registrations(&self, from_height: u64) -> Result, ChainStorageError> { - // TODO: we can optimise this query by making using a compound key + fn fetch_template_registrations( + &self, + start_height: u64, + end_height: u64, + ) -> Result, ChainStorageError> { let txn = self.read_transaction()?; - lmdb_filter_map_values(&txn, &self.template_registrations, |tr: TemplateRegistration| { - if tr.height >= from_height { - Some(tr) - } else { - None + let mut result = vec![]; + for _ in start_height..=end_height { + let height = start_height.to_le_bytes(); + let mut cursor: KeyPrefixCursor = + lmdb_get_prefix_cursor(&txn, &self.template_registrations, &height)?; + while let Some((_, val)) = cursor.next()? { + result.push(val); } - }) + } + Ok(result) } } @@ -2842,30 +2867,3 @@ impl<'a, 'b> DeletedBitmapModel<'a, WriteTransaction<'b>> { Ok(()) } } - -struct CompositeKey { - key: Vec, -} - -impl CompositeKey { - pub fn new(header_hash: &[u8], mmr_position: u32, hash: &[u8]) -> CompositeKey { - let mut key = Vec::with_capacity(header_hash.len() + mem::size_of::() + hash.len()); - key.extend_from_slice(header_hash); - key.extend_from_slice(&mmr_position.to_be_bytes()); - key.extend_from_slice(hash); - - CompositeKey { key } - } - - pub fn as_bytes(&self) -> &[u8] { - &self.key - } - - pub fn to_hex(&self) -> String { - self.key.to_hex() - } -} - -type InputKey = CompositeKey; -type KernelKey = CompositeKey; -type OutputKey = CompositeKey; diff --git a/base_layer/core/src/chain_storage/lmdb_db/mod.rs b/base_layer/core/src/chain_storage/lmdb_db/mod.rs index e462684b17..dd25794e99 100644 --- a/base_layer/core/src/chain_storage/lmdb_db/mod.rs +++ b/base_layer/core/src/chain_storage/lmdb_db/mod.rs @@ -28,6 +28,7 @@ use tari_crypto::hash_domain; use crate::transactions::transaction_components::{TransactionInput, TransactionKernel, TransactionOutput}; // mod composite_key; +mod composite_key; pub(crate) mod helpers; pub(crate) mod key_prefix_cursor; mod lmdb; diff --git a/base_layer/core/src/chain_storage/mod.rs b/base_layer/core/src/chain_storage/mod.rs index aa65f98a95..6777a0fd05 100644 --- a/base_layer/core/src/chain_storage/mod.rs +++ b/base_layer/core/src/chain_storage/mod.rs @@ -84,4 +84,4 @@ mod active_validator_node; pub use active_validator_node::ActiveValidatorNode; mod template_registation; -pub use template_registation::TemplateRegistration; +pub use template_registation::TemplateRegistrationEntry; diff --git a/base_layer/core/src/chain_storage/template_registation.rs b/base_layer/core/src/chain_storage/template_registation.rs index 452fc02ef6..b13c8370b5 100644 --- a/base_layer/core/src/chain_storage/template_registation.rs +++ b/base_layer/core/src/chain_storage/template_registation.rs @@ -21,11 +21,14 @@ // USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. use serde::{Deserialize, Serialize}; +use tari_common_types::types::FixedHash; use crate::transactions::transaction_components::CodeTemplateRegistration; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct TemplateRegistration { +pub struct TemplateRegistrationEntry { pub registration_data: CodeTemplateRegistration, - pub height: u64, + pub output_hash: FixedHash, + pub block_height: u64, + pub block_hash: FixedHash, } diff --git a/base_layer/core/src/test_helpers/blockchain.rs b/base_layer/core/src/test_helpers/blockchain.rs index 917f1dd3c4..e12ee86d36 100644 --- a/base_layer/core/src/test_helpers/blockchain.rs +++ b/base_layer/core/src/test_helpers/blockchain.rs @@ -67,7 +67,7 @@ use crate::{ MmrTree, PrunedOutput, Reorg, - TemplateRegistration, + TemplateRegistrationEntry, UtxoMinedInfo, Validators, }, @@ -427,8 +427,15 @@ impl BlockchainBackend for TempDatabase { self.db.as_ref().unwrap().get_shard_key(height, public_key) } - fn fetch_template_registrations(&self, from_height: u64) -> Result, ChainStorageError> { - self.db.as_ref().unwrap().fetch_template_registrations(from_height) + fn fetch_template_registrations( + &self, + start_height: u64, + end_height: u64, + ) -> Result, ChainStorageError> { + self.db + .as_ref() + .unwrap() + .fetch_template_registrations(start_height, end_height) } } diff --git a/base_layer/core/src/transactions/transaction_components/output_type.rs b/base_layer/core/src/transactions/transaction_components/output_type.rs index 9db69b1ef2..1927d8c3e8 100644 --- a/base_layer/core/src/transactions/transaction_components/output_type.rs +++ b/base_layer/core/src/transactions/transaction_components/output_type.rs @@ -71,6 +71,13 @@ impl OutputType { OutputType::CodeTemplateRegistration, ] } + + pub fn is_sidechain_type(&self) -> bool { + matches!( + self, + OutputType::ValidatorNodeRegistration | OutputType::CodeTemplateRegistration + ) + } } impl Default for OutputType { diff --git a/base_layer/core/src/transactions/transaction_components/side_chain/template_registration.rs b/base_layer/core/src/transactions/transaction_components/side_chain/template_registration.rs index 8f3290cd1e..83f67156f6 100644 --- a/base_layer/core/src/transactions/transaction_components/side_chain/template_registration.rs +++ b/base_layer/core/src/transactions/transaction_components/side_chain/template_registration.rs @@ -23,19 +23,15 @@ use std::io::{Error, ErrorKind, Read, Write}; use serde::{Deserialize, Serialize}; -use tari_common_types::types::{FixedHash, PublicKey, Signature}; - -use crate::{ - consensus::{ - read_byte, - ConsensusDecoding, - ConsensusEncoding, - ConsensusEncodingSized, - DomainSeparatedConsensusHasher, - MaxSizeBytes, - MaxSizeString, - }, - transactions::TransactionHashDomain, +use tari_common_types::types::{PublicKey, Signature}; + +use crate::consensus::{ + read_byte, + ConsensusDecoding, + ConsensusEncoding, + ConsensusEncodingSized, + MaxSizeBytes, + MaxSizeString, }; #[derive(Debug, Clone, Hash, PartialEq, Eq, Deserialize, Serialize)] @@ -50,22 +46,6 @@ pub struct CodeTemplateRegistration { pub binary_url: MaxSizeString<255>, } -impl CodeTemplateRegistration { - pub fn hash(&self) -> FixedHash { - DomainSeparatedConsensusHasher::::new("template_registration") - .chain(&self.author_public_key) - .chain(&self.author_signature) - .chain(&self.template_name) - .chain(&self.template_version) - .chain(&self.template_type) - .chain(&self.build_info) - .chain(&self.binary_sha) - .chain(&self.binary_url) - .finalize() - .into() - } -} - impl ConsensusEncoding for CodeTemplateRegistration { fn consensus_encode(&self, writer: &mut W) -> Result<(), Error> { self.author_public_key.consensus_encode(writer)?;