From bd05f7c748360219fe2ab097047c2b3444c8796e Mon Sep 17 00:00:00 2001 From: Mirko von Leipzig Date: Wed, 31 Jul 2024 15:16:51 +0200 Subject: [PATCH 1/8] feat(rpc): impl SyncNotes endpoint --- CHANGELOG.md | 1 + crates/proto/src/generated/requests.rs | 21 +++++++ crates/proto/src/generated/responses.rs | 16 +++++ crates/proto/src/generated/rpc.rs | 78 +++++++++++++++++++++++++ crates/proto/src/generated/store.rs | 78 +++++++++++++++++++++++++ crates/rpc-proto/proto/requests.proto | 19 ++++++ crates/rpc-proto/proto/responses.proto | 14 +++++ crates/rpc-proto/proto/rpc.proto | 1 + crates/rpc-proto/proto/store.proto | 1 + crates/rpc/src/server/api.rs | 21 ++++++- crates/store/src/server/api.rs | 43 +++++++++++++- proto/requests.proto | 19 ++++++ proto/responses.proto | 14 +++++ proto/rpc.proto | 1 + proto/store.proto | 1 + 15 files changed, 324 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a9f57113..d17def80d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - Added `GetAccountStateDelta` endpoint (#418). - Added `CheckNullifiersByPrefix` endpoint (#419). - Support multiple inflight transactions on the same account (#407). +- Added `SyncNotes` endpoint (#424). ### Fixes diff --git a/crates/proto/src/generated/requests.rs b/crates/proto/src/generated/requests.rs index 918186430..c42dfb139 100644 --- a/crates/proto/src/generated/requests.rs +++ b/crates/proto/src/generated/requests.rs @@ -73,6 +73,27 @@ pub struct SyncStateRequest { #[prost(uint32, repeated, tag = "4")] pub nullifiers: ::prost::alloc::vec::Vec, } +/// State synchronization request. +/// +/// Specifies state updates the client is intersted in. The server will return the first block which +/// contains a note matching `note_tags` or the chain tip. And the corresponding updates to +/// `nullifiers` and `account_ids` for that block range. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SyncNoteRequest { + /// Last block known by the client. The response will contain the data starting from the next block, + /// until the + #[prost(fixed32, tag = "1")] + pub block_num: u32, + /// Determines the tags which the client is interested in. These are only the 16high bits of the + /// note's complete tag. + /// + /// The above means it is not possible to request an specific note, but only a "note family", + /// this is done to increase the privacy of the client, by hiding the note's the client is + /// intereted on. + #[prost(uint32, repeated, tag = "3")] + pub note_tags: ::prost::alloc::vec::Vec, +} #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct GetBlockInputsRequest { diff --git a/crates/proto/src/generated/responses.rs b/crates/proto/src/generated/responses.rs index 74f7fabb1..2dc92392b 100644 --- a/crates/proto/src/generated/responses.rs +++ b/crates/proto/src/generated/responses.rs @@ -63,6 +63,22 @@ pub struct SyncStateResponse { #[prost(message, repeated, tag = "8")] pub nullifiers: ::prost::alloc::vec::Vec, } +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SyncNoteResponse { + /// Number of the latest block in the chain + #[prost(fixed32, tag = "1")] + pub chain_tip: u32, + /// Block header of the block with the first note matching the specified criteria + #[prost(message, optional, tag = "2")] + pub block_header: ::core::option::Option, + /// Data needed to update the partial MMR from `request.block_num + 1` to `response.block_header.block_num` + #[prost(message, optional, tag = "3")] + pub mmr_delta: ::core::option::Option, + /// List of all notes together with the Merkle paths from `response.block_header.note_root` + #[prost(message, repeated, tag = "4")] + pub notes: ::prost::alloc::vec::Vec, +} /// An account returned as a response to the GetBlockInputs #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] diff --git a/crates/proto/src/generated/rpc.rs b/crates/proto/src/generated/rpc.rs index fe726b431..be76165b9 100644 --- a/crates/proto/src/generated/rpc.rs +++ b/crates/proto/src/generated/rpc.rs @@ -288,6 +288,28 @@ pub mod api_client { .insert(GrpcMethod::new("rpc.Api", "SubmitProvenTransaction")); self.inner.unary(req, path, codec).await } + pub async fn sync_notes( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/rpc.Api/SyncNotes"); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new("rpc.Api", "SyncNotes")); + self.inner.unary(req, path, codec).await + } pub async fn sync_state( &mut self, request: impl tonic::IntoRequest, @@ -381,6 +403,13 @@ pub mod api_server { tonic::Response, tonic::Status, >; + async fn sync_notes( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn sync_state( &self, request: tonic::Request, @@ -862,6 +891,55 @@ pub mod api_server { }; Box::pin(fut) } + "/rpc.Api/SyncNotes" => { + #[allow(non_camel_case_types)] + struct SyncNotesSvc(pub Arc); + impl< + T: Api, + > tonic::server::UnaryService< + super::super::requests::SyncNoteRequest, + > for SyncNotesSvc { + type Response = super::super::responses::SyncNoteResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request< + super::super::requests::SyncNoteRequest, + >, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::sync_notes(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = SyncNotesSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } "/rpc.Api/SyncState" => { #[allow(non_camel_case_types)] struct SyncStateSvc(pub Arc); diff --git a/crates/proto/src/generated/store.rs b/crates/proto/src/generated/store.rs index 5fab84afa..35b9eb598 100644 --- a/crates/proto/src/generated/store.rs +++ b/crates/proto/src/generated/store.rs @@ -408,6 +408,28 @@ pub mod api_client { req.extensions_mut().insert(GrpcMethod::new("store.Api", "ListNullifiers")); self.inner.unary(req, path, codec).await } + pub async fn sync_notes( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/store.Api/SyncNotes"); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new("store.Api", "SyncNotes")); + self.inner.unary(req, path, codec).await + } pub async fn sync_state( &mut self, request: impl tonic::IntoRequest, @@ -534,6 +556,13 @@ pub mod api_server { tonic::Response, tonic::Status, >; + async fn sync_notes( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn sync_state( &self, request: tonic::Request, @@ -1260,6 +1289,55 @@ pub mod api_server { }; Box::pin(fut) } + "/store.Api/SyncNotes" => { + #[allow(non_camel_case_types)] + struct SyncNotesSvc(pub Arc); + impl< + T: Api, + > tonic::server::UnaryService< + super::super::requests::SyncNoteRequest, + > for SyncNotesSvc { + type Response = super::super::responses::SyncNoteResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request< + super::super::requests::SyncNoteRequest, + >, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::sync_notes(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = SyncNotesSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } "/store.Api/SyncState" => { #[allow(non_camel_case_types)] struct SyncStateSvc(pub Arc); diff --git a/crates/rpc-proto/proto/requests.proto b/crates/rpc-proto/proto/requests.proto index 43a5e3be1..1f081e1a4 100644 --- a/crates/rpc-proto/proto/requests.proto +++ b/crates/rpc-proto/proto/requests.proto @@ -68,6 +68,25 @@ message SyncStateRequest { repeated uint32 nullifiers = 4; } +// State synchronization request. +// +// Specifies state updates the client is intersted in. The server will return the first block which +// contains a note matching `note_tags` or the chain tip. And the corresponding updates to +// `nullifiers` and `account_ids` for that block range. +message SyncNoteRequest { + // Last block known by the client. The response will contain the data starting from the next block, + // until the + fixed32 block_num = 1; + + // Determines the tags which the client is interested in. These are only the 16high bits of the + // note's complete tag. + // + // The above means it is not possible to request an specific note, but only a "note family", + // this is done to increase the privacy of the client, by hiding the note's the client is + // intereted on. + repeated uint32 note_tags = 3; +} + message GetBlockInputsRequest { // ID of the account against which a transaction is executed. repeated account.AccountId account_ids = 1; diff --git a/crates/rpc-proto/proto/responses.proto b/crates/rpc-proto/proto/responses.proto index 1bf7c8aa3..87abdf601 100644 --- a/crates/rpc-proto/proto/responses.proto +++ b/crates/rpc-proto/proto/responses.proto @@ -62,6 +62,20 @@ message SyncStateResponse { repeated NullifierUpdate nullifiers = 8; } +message SyncNoteResponse { + // Number of the latest block in the chain + fixed32 chain_tip = 1; + + // Block header of the block with the first note matching the specified criteria + block_header.BlockHeader block_header = 2; + + // Data needed to update the partial MMR from `request.block_num + 1` to `response.block_header.block_num` + mmr.MmrDelta mmr_delta = 3; + + // List of all notes together with the Merkle paths from `response.block_header.note_root` + repeated note.NoteSyncRecord notes = 4; +} + // An account returned as a response to the GetBlockInputs message AccountBlockInputRecord { account.AccountId account_id = 1; diff --git a/crates/rpc-proto/proto/rpc.proto b/crates/rpc-proto/proto/rpc.proto index 8e75bca77..da2c2bdd0 100644 --- a/crates/rpc-proto/proto/rpc.proto +++ b/crates/rpc-proto/proto/rpc.proto @@ -14,5 +14,6 @@ service Api { rpc GetBlockHeaderByNumber(requests.GetBlockHeaderByNumberRequest) returns (responses.GetBlockHeaderByNumberResponse) {} rpc GetNotesById(requests.GetNotesByIdRequest) returns (responses.GetNotesByIdResponse) {} rpc SubmitProvenTransaction(requests.SubmitProvenTransactionRequest) returns (responses.SubmitProvenTransactionResponse) {} + rpc SyncNotes(requests.SyncNoteRequest) returns (responses.SyncNoteResponse) {} rpc SyncState(requests.SyncStateRequest) returns (responses.SyncStateResponse) {} } diff --git a/crates/rpc-proto/proto/store.proto b/crates/rpc-proto/proto/store.proto index ced0a3f4d..1563b1261 100644 --- a/crates/rpc-proto/proto/store.proto +++ b/crates/rpc-proto/proto/store.proto @@ -21,5 +21,6 @@ service Api { rpc ListAccounts(requests.ListAccountsRequest) returns (responses.ListAccountsResponse) {} rpc ListNotes(requests.ListNotesRequest) returns (responses.ListNotesResponse) {} rpc ListNullifiers(requests.ListNullifiersRequest) returns (responses.ListNullifiersResponse) {} + rpc SyncNotes(requests.SyncNoteRequest) returns (responses.SyncNoteResponse) {} rpc SyncState(requests.SyncStateRequest) returns (responses.SyncStateResponse) {} } diff --git a/crates/rpc/src/server/api.rs b/crates/rpc/src/server/api.rs index 0ffd9c4a7..fc4836d08 100644 --- a/crates/rpc/src/server/api.rs +++ b/crates/rpc/src/server/api.rs @@ -4,12 +4,13 @@ use miden_node_proto::{ requests::{ CheckNullifiersByPrefixRequest, CheckNullifiersRequest, GetAccountDetailsRequest, GetAccountStateDeltaRequest, GetBlockByNumberRequest, GetBlockHeaderByNumberRequest, - GetNotesByIdRequest, SubmitProvenTransactionRequest, SyncStateRequest, + GetNotesByIdRequest, SubmitProvenTransactionRequest, SyncNoteRequest, SyncStateRequest, }, responses::{ CheckNullifiersByPrefixResponse, CheckNullifiersResponse, GetAccountDetailsResponse, GetAccountStateDeltaResponse, GetBlockByNumberResponse, GetBlockHeaderByNumberResponse, - GetNotesByIdResponse, SubmitProvenTransactionResponse, SyncStateResponse, + GetNotesByIdResponse, SubmitProvenTransactionResponse, SyncNoteResponse, + SyncStateResponse, }, rpc::api_server, store::api_client as store_client, @@ -130,6 +131,22 @@ impl api_server::Api for RpcApi { self.store.clone().sync_state(request).await } + #[instrument( + target = "miden-rpc", + name = "rpc:sync_notes", + skip_all, + ret(level = "debug"), + err + )] + async fn sync_notes( + &self, + request: Request, + ) -> Result, Status> { + debug!(target: COMPONENT, request = ?request.get_ref()); + + self.store.clone().sync_notes(request).await + } + #[instrument( target = "miden-rpc", name = "rpc:get_notes_by_id", diff --git a/crates/store/src/server/api.rs b/crates/store/src/server/api.rs index 47eb45ba2..99bfdb70c 100644 --- a/crates/store/src/server/api.rs +++ b/crates/store/src/server/api.rs @@ -12,7 +12,7 @@ use miden_node_proto::{ GetAccountDetailsRequest, GetAccountStateDeltaRequest, GetBlockByNumberRequest, GetBlockHeaderByNumberRequest, GetBlockInputsRequest, GetNotesByIdRequest, GetTransactionInputsRequest, ListAccountsRequest, ListNotesRequest, - ListNullifiersRequest, SyncStateRequest, + ListNullifiersRequest, SyncNoteRequest, SyncStateRequest, }, responses::{ AccountTransactionInputRecord, ApplyBlockResponse, CheckNullifiersByPrefixResponse, @@ -20,7 +20,7 @@ use miden_node_proto::{ GetBlockByNumberResponse, GetBlockHeaderByNumberResponse, GetBlockInputsResponse, GetNotesByIdResponse, GetTransactionInputsResponse, ListAccountsResponse, ListNotesResponse, ListNullifiersResponse, NullifierTransactionInputRecord, - NullifierUpdate, SyncStateResponse, + NullifierUpdate, SyncNoteResponse, SyncStateResponse, }, smt::SmtLeafEntry, store::api_server, @@ -220,6 +220,45 @@ impl api_server::Api for StoreApi { })) } + /// Returns info which can be used by the client to sync note state. + #[instrument( + target = "miden-store", + name = "store:sync_notes", + skip_all, + ret(level = "debug"), + err + )] + async fn sync_notes( + &self, + request: tonic::Request, + ) -> Result, Status> { + let request = request.into_inner(); + + let (state, delta) = self + .state + .sync_state(request.block_num, &[], &request.note_tags, &[]) + .await + .map_err(internal_error)?; + + let notes = state + .notes + .into_iter() + .map(|note| NoteSyncRecord { + note_index: note.note_index.to_absolute_index() as u32, + note_id: Some(note.note_id.into()), + metadata: Some(note.metadata.into()), + merkle_path: Some(note.merkle_path.into()), + }) + .collect(); + + Ok(Response::new(SyncNoteResponse { + chain_tip: state.chain_tip, + block_header: Some(state.block_header.into()), + mmr_delta: Some(delta.into()), + notes, + })) + } + /// Returns a list of Note's for the specified NoteId's. /// /// If the list is empty or no Note matched the requested NoteId and empty list is returned. diff --git a/proto/requests.proto b/proto/requests.proto index 43a5e3be1..1f081e1a4 100644 --- a/proto/requests.proto +++ b/proto/requests.proto @@ -68,6 +68,25 @@ message SyncStateRequest { repeated uint32 nullifiers = 4; } +// State synchronization request. +// +// Specifies state updates the client is intersted in. The server will return the first block which +// contains a note matching `note_tags` or the chain tip. And the corresponding updates to +// `nullifiers` and `account_ids` for that block range. +message SyncNoteRequest { + // Last block known by the client. The response will contain the data starting from the next block, + // until the + fixed32 block_num = 1; + + // Determines the tags which the client is interested in. These are only the 16high bits of the + // note's complete tag. + // + // The above means it is not possible to request an specific note, but only a "note family", + // this is done to increase the privacy of the client, by hiding the note's the client is + // intereted on. + repeated uint32 note_tags = 3; +} + message GetBlockInputsRequest { // ID of the account against which a transaction is executed. repeated account.AccountId account_ids = 1; diff --git a/proto/responses.proto b/proto/responses.proto index 1bf7c8aa3..87abdf601 100644 --- a/proto/responses.proto +++ b/proto/responses.proto @@ -62,6 +62,20 @@ message SyncStateResponse { repeated NullifierUpdate nullifiers = 8; } +message SyncNoteResponse { + // Number of the latest block in the chain + fixed32 chain_tip = 1; + + // Block header of the block with the first note matching the specified criteria + block_header.BlockHeader block_header = 2; + + // Data needed to update the partial MMR from `request.block_num + 1` to `response.block_header.block_num` + mmr.MmrDelta mmr_delta = 3; + + // List of all notes together with the Merkle paths from `response.block_header.note_root` + repeated note.NoteSyncRecord notes = 4; +} + // An account returned as a response to the GetBlockInputs message AccountBlockInputRecord { account.AccountId account_id = 1; diff --git a/proto/rpc.proto b/proto/rpc.proto index 8e75bca77..da2c2bdd0 100644 --- a/proto/rpc.proto +++ b/proto/rpc.proto @@ -14,5 +14,6 @@ service Api { rpc GetBlockHeaderByNumber(requests.GetBlockHeaderByNumberRequest) returns (responses.GetBlockHeaderByNumberResponse) {} rpc GetNotesById(requests.GetNotesByIdRequest) returns (responses.GetNotesByIdResponse) {} rpc SubmitProvenTransaction(requests.SubmitProvenTransactionRequest) returns (responses.SubmitProvenTransactionResponse) {} + rpc SyncNotes(requests.SyncNoteRequest) returns (responses.SyncNoteResponse) {} rpc SyncState(requests.SyncStateRequest) returns (responses.SyncStateResponse) {} } diff --git a/proto/store.proto b/proto/store.proto index ced0a3f4d..1563b1261 100644 --- a/proto/store.proto +++ b/proto/store.proto @@ -21,5 +21,6 @@ service Api { rpc ListAccounts(requests.ListAccountsRequest) returns (responses.ListAccountsResponse) {} rpc ListNotes(requests.ListNotesRequest) returns (responses.ListNotesResponse) {} rpc ListNullifiers(requests.ListNullifiersRequest) returns (responses.ListNullifiersResponse) {} + rpc SyncNotes(requests.SyncNoteRequest) returns (responses.SyncNoteResponse) {} rpc SyncState(requests.SyncStateRequest) returns (responses.SyncStateResponse) {} } From 0626e7622585bba92bb05c14ebe3102c695f0f46 Mon Sep 17 00:00:00 2001 From: Mirko von Leipzig Date: Thu, 1 Aug 2024 12:08:05 +0200 Subject: [PATCH 2/8] feat: review comments --- crates/proto/src/generated/requests.rs | 11 ++- crates/rpc-proto/proto/requests.proto | 11 ++- crates/store/src/db/mod.rs | 26 ++++++ crates/store/src/db/sql.rs | 111 ++++++++++++++++++++++++- crates/store/src/server/api.rs | 2 +- crates/store/src/state.rs | 47 ++++++++++- proto/requests.proto | 11 ++- 7 files changed, 198 insertions(+), 21 deletions(-) diff --git a/crates/proto/src/generated/requests.rs b/crates/proto/src/generated/requests.rs index c42dfb139..5171c1e27 100644 --- a/crates/proto/src/generated/requests.rs +++ b/crates/proto/src/generated/requests.rs @@ -73,16 +73,15 @@ pub struct SyncStateRequest { #[prost(uint32, repeated, tag = "4")] pub nullifiers: ::prost::alloc::vec::Vec, } -/// State synchronization request. +/// Note synchronization request. /// -/// Specifies state updates the client is intersted in. The server will return the first block which -/// contains a note matching `note_tags` or the chain tip. And the corresponding updates to -/// `nullifiers` and `account_ids` for that block range. +/// Specifies note tags that client is intersted in. The server will return the first block which +/// contains a note matching `note_tags` or the chain tip. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct SyncNoteRequest { - /// Last block known by the client. The response will contain the data starting from the next block, - /// until the + /// Last block known by the client. The response will contain data starting from the next block, + /// until the first block which contains a note of matching the requested tag. #[prost(fixed32, tag = "1")] pub block_num: u32, /// Determines the tags which the client is interested in. These are only the 16high bits of the diff --git a/crates/rpc-proto/proto/requests.proto b/crates/rpc-proto/proto/requests.proto index 1f081e1a4..82684bce2 100644 --- a/crates/rpc-proto/proto/requests.proto +++ b/crates/rpc-proto/proto/requests.proto @@ -68,14 +68,13 @@ message SyncStateRequest { repeated uint32 nullifiers = 4; } -// State synchronization request. +// Note synchronization request. // -// Specifies state updates the client is intersted in. The server will return the first block which -// contains a note matching `note_tags` or the chain tip. And the corresponding updates to -// `nullifiers` and `account_ids` for that block range. +// Specifies note tags that client is intersted in. The server will return the first block which +// contains a note matching `note_tags` or the chain tip. message SyncNoteRequest { - // Last block known by the client. The response will contain the data starting from the next block, - // until the + // Last block known by the client. The response will contain data starting from the next block, + // until the first block which contains a note of matching the requested tag. fixed32 block_num = 1; // Determines the tags which the client is interested in. These are only the 16high bits of the diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index ac9b5ef1e..a07891e25 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -91,6 +91,13 @@ pub struct StateSyncUpdate { pub nullifiers: Vec, } +#[derive(Debug, PartialEq)] +pub struct NoteSyncUpdate { + pub notes: Vec, + pub block_header: BlockHeader, + pub chain_tip: BlockNumber, +} + impl Db { /// Open a connection to the DB, apply any pending migrations, and ensure that the genesis block /// is as expected and present in the database. @@ -289,6 +296,25 @@ impl Db { })? } + #[instrument(target = "miden-store", skip_all, ret(level = "debug"), err)] + pub async fn get_note_sync( + &self, + block_num: BlockNumber, + note_tag_prefixes: &[u32], + ) -> Result { + let note_tag_prefixes = note_tag_prefixes.to_vec(); + + self.pool + .get() + .await + .map_err(DatabaseError::MissingDbConnection)? + .interact(move |conn| sql::get_note_sync(conn, block_num, ¬e_tag_prefixes)) + .await + .map_err(|err| { + DatabaseError::InteractError(format!("Get notes sync task failed: {err}")) + })? + } + /// Loads all the Note's matching a certain NoteId from the database. #[instrument(target = "miden-store", skip_all, ret(level = "debug"), err)] pub async fn select_notes_by_id(&self, note_ids: Vec) -> Result> { diff --git a/crates/store/src/db/sql.rs b/crates/store/src/db/sql.rs index 77afe7b87..54d4a2f27 100644 --- a/crates/store/src/db/sql.rs +++ b/crates/store/src/db/sql.rs @@ -18,7 +18,9 @@ use rusqlite::{ Connection, OptionalExtension, Transaction, }; -use super::{NoteRecord, NullifierInfo, Result, StateSyncUpdate, TransactionSummary}; +use super::{ + NoteRecord, NoteSyncUpdate, NullifierInfo, Result, StateSyncUpdate, TransactionSummary, +}; use crate::{ errors::{DatabaseError, StateSyncError}, types::{AccountId, BlockNumber}, @@ -604,6 +606,84 @@ pub fn select_notes_since_block_by_tag_and_sender( Ok(res) } +/// Select notes matching the tag search criteria using the given [Connection]. +/// +/// # Returns +/// +/// - Empty vector if no tag created after `block_num` match `tags`. +/// - Otherwise, notes which the 16 high bits match `tags`, or the `sender` is one of the +/// `account_ids`. +/// +/// # Note +/// +/// This method returns notes from a single block. To fetch all notes up to the chain tip, +/// multiple requests are necessary. +pub fn select_notes_since_block_by_tag( + conn: &mut Connection, + tags: &[u32], + block_num: BlockNumber, +) -> Result> { + let tags: Vec = tags.iter().copied().map(u32_to_value).collect(); + + let mut stmt = conn.prepare( + " + SELECT + block_num, + batch_index, + note_index, + note_id, + note_type, + sender, + tag, + aux, + merkle_path, + details + FROM + notes + -- find the next block which contains at least one note with a matching tag + WHERE + tag IN rarray(?1) AND + block_num > ?2 + ORDER BY + block_num ASC + LIMIT + 1; + ", + )?; + let mut rows = stmt.query(params![Rc::new(tags), block_num])?; + + let mut res = Vec::new(); + while let Some(row) = rows.next()? { + let block_num = row.get(0)?; + let note_index = BlockNoteIndex::new(row.get(1)?, row.get(2)?); + let note_id_data = row.get_ref(3)?.as_blob()?; + let note_id = RpoDigest::read_from_bytes(note_id_data)?; + let note_type = row.get::<_, u8>(4)?; + let sender = column_value_as_u64(row, 5)?; + let tag: u32 = row.get(6)?; + let aux: u64 = row.get(7)?; + let aux = aux.try_into().map_err(DatabaseError::InvalidFelt)?; + let merkle_path_data = row.get_ref(8)?.as_blob()?; + let merkle_path = MerklePath::read_from_bytes(merkle_path_data)?; + let details_data = row.get_ref(9)?.as_blob_or_null()?; + let details = details_data.map(>::read_from_bytes).transpose()?; + + let metadata = + NoteMetadata::new(sender.try_into()?, NoteType::try_from(note_type)?, tag.into(), aux)?; + + let note = NoteRecord { + block_num, + note_index, + note_id, + details, + metadata, + merkle_path, + }; + res.push(note); + } + Ok(res) +} + /// Select Note's matching the NoteId using the given [Connection]. /// /// # Returns @@ -877,6 +957,35 @@ pub fn get_state_sync( }) } +// NOTE SYNC +// ================================================================================================ + +/// Loads the data necessary for a note sync. +pub fn get_note_sync( + conn: &mut Connection, + block_num: BlockNumber, + note_tag_prefixes: &[u32], +) -> Result { + let notes = select_notes_since_block_by_tag(conn, note_tag_prefixes, block_num)?; + + let (block_header, chain_tip) = if !notes.is_empty() { + let block_header = select_block_header_by_block_num(conn, Some(notes[0].block_num))? + .ok_or(StateSyncError::EmptyBlockHeadersTable)?; + let tip = select_block_header_by_block_num(conn, None)? + .ok_or(StateSyncError::EmptyBlockHeadersTable)?; + + (block_header, tip.block_num()) + } else { + let block_header = select_block_header_by_block_num(conn, None)? + .ok_or(StateSyncError::EmptyBlockHeadersTable)?; + + let block_num = block_header.block_num(); + (block_header, block_num) + }; + + Ok(NoteSyncUpdate { notes, block_header, chain_tip }) +} + // APPLY BLOCK // ================================================================================================ diff --git a/crates/store/src/server/api.rs b/crates/store/src/server/api.rs index 99bfdb70c..da3f0e0cd 100644 --- a/crates/store/src/server/api.rs +++ b/crates/store/src/server/api.rs @@ -236,7 +236,7 @@ impl api_server::Api for StoreApi { let (state, delta) = self .state - .sync_state(request.block_num, &[], &request.note_tags, &[]) + .sync_notes(request.block_num, &request.note_tags) .await .map_err(internal_error)?; diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 795f335bb..b3ff9afff 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -29,7 +29,7 @@ use tracing::{info, info_span, instrument}; use crate::{ blocks::BlockStore, - db::{Db, NoteRecord, NullifierInfo, StateSyncUpdate}, + db::{Db, NoteRecord, NoteSyncUpdate, NullifierInfo, StateSyncUpdate}, errors::{ ApplyBlockError, DatabaseError, GetBlockHeaderError, GetBlockInputsError, StateInitializationError, StateSyncError, @@ -451,6 +451,51 @@ impl State { Ok((state_sync, delta)) } + /// Loads data to synchronize a client's notes. + /// + /// The client's request contains a list of tag prefixes, this method will return the first + /// block with a matching tag, or the chain tip. All the other values are filter based on this + /// block range. + /// + /// # Arguments + /// + /// - `block_num`: The last block *know* by the client, updates start from the next block. + /// - `note_tag_prefixes`: Only the 16 high bits of the tags the client is interested in, result + /// will include notes with matching prefixes, the first block with a matching note determines + /// the block range. + #[instrument(target = "miden-store", skip_all, ret(level = "debug"), err)] + pub async fn sync_notes( + &self, + block_num: BlockNumber, + note_tag_prefixes: &[u32], + ) -> Result<(NoteSyncUpdate, MmrDelta), StateSyncError> { + let inner = self.inner.read().await; + + let note_sync = self.db.get_note_sync(block_num, note_tag_prefixes).await?; + + let delta = if block_num == note_sync.block_header.block_num() { + // The client is in sync with the chain tip. + MmrDelta { forest: block_num as usize, data: vec![] } + } else { + // Important notes about the boundary conditions: + // + // - The Mmr forest is 1-indexed whereas the block number is 0-indexed. The Mmr root + // contained in the block header always lag behind by one block, this is because the Mmr + // leaves are hashes of block headers, and we can't have self-referential hashes. These two + // points cancel out and don't require adjusting. + // - Mmr::get_delta is inclusive, whereas the sync_state request block_num is defined to be + // exclusive, so the from_forest has to be adjusted with a +1 + let from_forest = (block_num + 1) as usize; + let to_forest = note_sync.block_header.block_num() as usize; + inner + .chain_mmr + .get_delta(from_forest, to_forest) + .map_err(StateSyncError::FailedToBuildMmrDelta)? + }; + + Ok((note_sync, delta)) + } + /// Returns data needed by the block producer to construct and prove the next block. pub async fn get_block_inputs( &self, diff --git a/proto/requests.proto b/proto/requests.proto index 1f081e1a4..82684bce2 100644 --- a/proto/requests.proto +++ b/proto/requests.proto @@ -68,14 +68,13 @@ message SyncStateRequest { repeated uint32 nullifiers = 4; } -// State synchronization request. +// Note synchronization request. // -// Specifies state updates the client is intersted in. The server will return the first block which -// contains a note matching `note_tags` or the chain tip. And the corresponding updates to -// `nullifiers` and `account_ids` for that block range. +// Specifies note tags that client is intersted in. The server will return the first block which +// contains a note matching `note_tags` or the chain tip. message SyncNoteRequest { - // Last block known by the client. The response will contain the data starting from the next block, - // until the + // Last block known by the client. The response will contain data starting from the next block, + // until the first block which contains a note of matching the requested tag. fixed32 block_num = 1; // Determines the tags which the client is interested in. These are only the 16high bits of the From 06678fdacb3133997dbd57fd56e761e31a9610d0 Mon Sep 17 00:00:00 2001 From: Mirko von Leipzig Date: Thu, 1 Aug 2024 17:24:58 +0200 Subject: [PATCH 3/8] fix(proto): note tag should be fixed32 --- crates/proto/src/generated/requests.rs | 18 ++++-------------- crates/rpc-proto/proto/requests.proto | 18 ++++-------------- proto/requests.proto | 18 ++++-------------- 3 files changed, 12 insertions(+), 42 deletions(-) diff --git a/crates/proto/src/generated/requests.rs b/crates/proto/src/generated/requests.rs index 5171c1e27..e385fdc3e 100644 --- a/crates/proto/src/generated/requests.rs +++ b/crates/proto/src/generated/requests.rs @@ -59,13 +59,8 @@ pub struct SyncStateRequest { /// it won't be included in the response. #[prost(message, repeated, tag = "2")] pub account_ids: ::prost::alloc::vec::Vec, - /// Determines the tags which the client is interested in. These are only the 16high bits of the - /// note's complete tag. - /// - /// The above means it is not possible to request an specific note, but only a "note family", - /// this is done to increase the privacy of the client, by hiding the note's the client is - /// intereted on. - #[prost(uint32, repeated, tag = "3")] + /// Determines the tags which the client is interested in. + #[prost(fixed32, repeated, tag = "3")] pub note_tags: ::prost::alloc::vec::Vec, /// Determines the nullifiers the client is interested in. /// @@ -84,13 +79,8 @@ pub struct SyncNoteRequest { /// until the first block which contains a note of matching the requested tag. #[prost(fixed32, tag = "1")] pub block_num: u32, - /// Determines the tags which the client is interested in. These are only the 16high bits of the - /// note's complete tag. - /// - /// The above means it is not possible to request an specific note, but only a "note family", - /// this is done to increase the privacy of the client, by hiding the note's the client is - /// intereted on. - #[prost(uint32, repeated, tag = "3")] + /// Determines the tags which the client is interested in. + #[prost(fixed32, repeated, tag = "2")] pub note_tags: ::prost::alloc::vec::Vec, } #[allow(clippy::derive_partial_eq_without_eq)] diff --git a/crates/rpc-proto/proto/requests.proto b/crates/rpc-proto/proto/requests.proto index 82684bce2..c10463753 100644 --- a/crates/rpc-proto/proto/requests.proto +++ b/crates/rpc-proto/proto/requests.proto @@ -54,13 +54,8 @@ message SyncStateRequest { // it won't be included in the response. repeated account.AccountId account_ids = 2; - // Determines the tags which the client is interested in. These are only the 16high bits of the - // note's complete tag. - // - // The above means it is not possible to request an specific note, but only a "note family", - // this is done to increase the privacy of the client, by hiding the note's the client is - // intereted on. - repeated uint32 note_tags = 3; + // Determines the tags which the client is interested in. + repeated fixed32 note_tags = 3; // Determines the nullifiers the client is interested in. // @@ -77,13 +72,8 @@ message SyncNoteRequest { // until the first block which contains a note of matching the requested tag. fixed32 block_num = 1; - // Determines the tags which the client is interested in. These are only the 16high bits of the - // note's complete tag. - // - // The above means it is not possible to request an specific note, but only a "note family", - // this is done to increase the privacy of the client, by hiding the note's the client is - // intereted on. - repeated uint32 note_tags = 3; + // Determines the tags which the client is interested in. + repeated fixed32 note_tags = 2; } message GetBlockInputsRequest { diff --git a/proto/requests.proto b/proto/requests.proto index 82684bce2..c10463753 100644 --- a/proto/requests.proto +++ b/proto/requests.proto @@ -54,13 +54,8 @@ message SyncStateRequest { // it won't be included in the response. repeated account.AccountId account_ids = 2; - // Determines the tags which the client is interested in. These are only the 16high bits of the - // note's complete tag. - // - // The above means it is not possible to request an specific note, but only a "note family", - // this is done to increase the privacy of the client, by hiding the note's the client is - // intereted on. - repeated uint32 note_tags = 3; + // Determines the tags which the client is interested in. + repeated fixed32 note_tags = 3; // Determines the nullifiers the client is interested in. // @@ -77,13 +72,8 @@ message SyncNoteRequest { // until the first block which contains a note of matching the requested tag. fixed32 block_num = 1; - // Determines the tags which the client is interested in. These are only the 16high bits of the - // note's complete tag. - // - // The above means it is not possible to request an specific note, but only a "note family", - // this is done to increase the privacy of the client, by hiding the note's the client is - // intereted on. - repeated uint32 note_tags = 3; + // Determines the tags which the client is interested in. + repeated fixed32 note_tags = 2; } message GetBlockInputsRequest { From 989293c7e4b29b5eaaeea83de596d24336b443e4 Mon Sep 17 00:00:00 2001 From: Mirko von Leipzig Date: Fri, 2 Aug 2024 14:10:09 +0200 Subject: [PATCH 4/8] chore: minor changes per review comments --- crates/proto/src/generated/requests.rs | 9 ++++----- crates/rpc-proto/proto/requests.proto | 9 ++++----- crates/store/src/db/mod.rs | 24 ++++++------------------ crates/store/src/db/sql.rs | 4 ++-- crates/store/src/server/api.rs | 4 ++-- crates/store/src/state.rs | 24 +++++++++++------------- proto/requests.proto | 9 ++++----- 7 files changed, 33 insertions(+), 50 deletions(-) diff --git a/crates/proto/src/generated/requests.rs b/crates/proto/src/generated/requests.rs index e385fdc3e..a75da6f50 100644 --- a/crates/proto/src/generated/requests.rs +++ b/crates/proto/src/generated/requests.rs @@ -59,12 +59,11 @@ pub struct SyncStateRequest { /// it won't be included in the response. #[prost(message, repeated, tag = "2")] pub account_ids: ::prost::alloc::vec::Vec, - /// Determines the tags which the client is interested in. + /// Specifies the tags which the client is interested in. #[prost(fixed32, repeated, tag = "3")] pub note_tags: ::prost::alloc::vec::Vec, - /// Determines the nullifiers the client is interested in. - /// - /// Similarly to the note_tags, this determins only the 16high bits of the target nullifier. + /// Determines the nullifiers the client is interested in by specifying the 16high bits of the + /// target nullifier. #[prost(uint32, repeated, tag = "4")] pub nullifiers: ::prost::alloc::vec::Vec, } @@ -79,7 +78,7 @@ pub struct SyncNoteRequest { /// until the first block which contains a note of matching the requested tag. #[prost(fixed32, tag = "1")] pub block_num: u32, - /// Determines the tags which the client is interested in. + /// Specifies the tags which the client is interested in. #[prost(fixed32, repeated, tag = "2")] pub note_tags: ::prost::alloc::vec::Vec, } diff --git a/crates/rpc-proto/proto/requests.proto b/crates/rpc-proto/proto/requests.proto index c10463753..02222f994 100644 --- a/crates/rpc-proto/proto/requests.proto +++ b/crates/rpc-proto/proto/requests.proto @@ -54,12 +54,11 @@ message SyncStateRequest { // it won't be included in the response. repeated account.AccountId account_ids = 2; - // Determines the tags which the client is interested in. + // Specifies the tags which the client is interested in. repeated fixed32 note_tags = 3; - // Determines the nullifiers the client is interested in. - // - // Similarly to the note_tags, this determins only the 16high bits of the target nullifier. + // Determines the nullifiers the client is interested in by specifying the 16high bits of the + // target nullifier. repeated uint32 nullifiers = 4; } @@ -72,7 +71,7 @@ message SyncNoteRequest { // until the first block which contains a note of matching the requested tag. fixed32 block_num = 1; - // Determines the tags which the client is interested in. + // Specifies the tags which the client is interested in. repeated fixed32 note_tags = 2; } diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index a07891e25..30aaad65d 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -269,26 +269,16 @@ impl Db { pub async fn get_state_sync( &self, block_num: BlockNumber, - account_ids: &[AccountId], - note_tag_prefixes: &[u32], - nullifier_prefixes: &[u32], + account_ids: Vec, + note_tags: Vec, + nullifier_prefixes: Vec, ) -> Result { - let account_ids = account_ids.to_vec(); - let note_tag_prefixes = note_tag_prefixes.to_vec(); - let nullifier_prefixes = nullifier_prefixes.to_vec(); - self.pool .get() .await .map_err(DatabaseError::MissingDbConnection)? .interact(move |conn| { - sql::get_state_sync( - conn, - block_num, - &account_ids, - ¬e_tag_prefixes, - &nullifier_prefixes, - ) + sql::get_state_sync(conn, block_num, &account_ids, ¬e_tags, &nullifier_prefixes) }) .await .map_err(|err| { @@ -300,15 +290,13 @@ impl Db { pub async fn get_note_sync( &self, block_num: BlockNumber, - note_tag_prefixes: &[u32], + note_tags: Vec, ) -> Result { - let note_tag_prefixes = note_tag_prefixes.to_vec(); - self.pool .get() .await .map_err(DatabaseError::MissingDbConnection)? - .interact(move |conn| sql::get_note_sync(conn, block_num, ¬e_tag_prefixes)) + .interact(move |conn| sql::get_note_sync(conn, block_num, ¬e_tags)) .await .map_err(|err| { DatabaseError::InteractError(format!("Get notes sync task failed: {err}")) diff --git a/crates/store/src/db/sql.rs b/crates/store/src/db/sql.rs index 54d4a2f27..c7ad18159 100644 --- a/crates/store/src/db/sql.rs +++ b/crates/store/src/db/sql.rs @@ -964,9 +964,9 @@ pub fn get_state_sync( pub fn get_note_sync( conn: &mut Connection, block_num: BlockNumber, - note_tag_prefixes: &[u32], + note_tags: &[u32], ) -> Result { - let notes = select_notes_since_block_by_tag(conn, note_tag_prefixes, block_num)?; + let notes = select_notes_since_block_by_tag(conn, note_tags, block_num)?; let (block_header, chain_tip) = if !notes.is_empty() { let block_header = select_block_header_by_block_num(conn, Some(notes[0].block_num))? diff --git a/crates/store/src/server/api.rs b/crates/store/src/server/api.rs index da3f0e0cd..888e3fefb 100644 --- a/crates/store/src/server/api.rs +++ b/crates/store/src/server/api.rs @@ -165,7 +165,7 @@ impl api_server::Api for StoreApi { let (state, delta) = self .state - .sync_state(request.block_num, &account_ids, &request.note_tags, &request.nullifiers) + .sync_state(request.block_num, account_ids, request.note_tags, request.nullifiers) .await .map_err(internal_error)?; @@ -236,7 +236,7 @@ impl api_server::Api for StoreApi { let (state, delta) = self .state - .sync_notes(request.block_num, &request.note_tags) + .sync_notes(request.block_num, request.note_tags) .await .map_err(internal_error)?; diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index b3ff9afff..2465cd199 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -405,27 +405,26 @@ impl State { /// /// # Arguments /// - /// - `block_num`: The last block *know* by the client, updates start from the next block. + /// - `block_num`: The last block *known* by the client, updates start from the next block. /// - `account_ids`: Include the account's hash if their _last change_ was in the result's block /// range. - /// - `note_tag_prefixes`: Only the 16 high bits of the tags the client is interested in, result - /// will include notes with matching prefixes, the first block with a matching note determines - /// the block range. + /// - `note_tags`: The tags the client is interested in, result is restricted to the first block + /// with any matches tags. /// - `nullifier_prefixes`: Only the 16 high bits of the nullifiers the client is interested in, /// results will include nullifiers matching prefixes produced in the given block range. #[instrument(target = "miden-store", skip_all, ret(level = "debug"), err)] pub async fn sync_state( &self, block_num: BlockNumber, - account_ids: &[AccountId], - note_tag_prefixes: &[u32], - nullifier_prefixes: &[u32], + account_ids: Vec, + note_tags: Vec, + nullifier_prefixes: Vec, ) -> Result<(StateSyncUpdate, MmrDelta), StateSyncError> { let inner = self.inner.read().await; let state_sync = self .db - .get_state_sync(block_num, account_ids, note_tag_prefixes, nullifier_prefixes) + .get_state_sync(block_num, account_ids, note_tags, nullifier_prefixes) .await?; let delta = if block_num == state_sync.block_header.block_num() { @@ -459,15 +458,14 @@ impl State { /// /// # Arguments /// - /// - `block_num`: The last block *know* by the client, updates start from the next block. - /// - `note_tag_prefixes`: Only the 16 high bits of the tags the client is interested in, result - /// will include notes with matching prefixes, the first block with a matching note determines - /// the block range. + /// - `block_num`: The last block *known* by the client, updates start from the next block. + /// - `note_tags`: The tags the client is interested in, resulting notes are restricted to + /// the first block containing a matching note. #[instrument(target = "miden-store", skip_all, ret(level = "debug"), err)] pub async fn sync_notes( &self, block_num: BlockNumber, - note_tag_prefixes: &[u32], + note_tag_prefixes: Vec, ) -> Result<(NoteSyncUpdate, MmrDelta), StateSyncError> { let inner = self.inner.read().await; diff --git a/proto/requests.proto b/proto/requests.proto index c10463753..02222f994 100644 --- a/proto/requests.proto +++ b/proto/requests.proto @@ -54,12 +54,11 @@ message SyncStateRequest { // it won't be included in the response. repeated account.AccountId account_ids = 2; - // Determines the tags which the client is interested in. + // Specifies the tags which the client is interested in. repeated fixed32 note_tags = 3; - // Determines the nullifiers the client is interested in. - // - // Similarly to the note_tags, this determins only the 16high bits of the target nullifier. + // Determines the nullifiers the client is interested in by specifying the 16high bits of the + // target nullifier. repeated uint32 nullifiers = 4; } @@ -72,7 +71,7 @@ message SyncNoteRequest { // until the first block which contains a note of matching the requested tag. fixed32 block_num = 1; - // Determines the tags which the client is interested in. + // Specifies the tags which the client is interested in. repeated fixed32 note_tags = 2; } From 1ef80746531fe631ab323a11b1104f39b624cf9f Mon Sep 17 00:00:00 2001 From: Mirko von Leipzig Date: Fri, 2 Aug 2024 14:15:07 +0200 Subject: [PATCH 5/8] fix: database query should be nested --- crates/store/src/db/sql.rs | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/crates/store/src/db/sql.rs b/crates/store/src/db/sql.rs index c7ad18159..f0a6cfa70 100644 --- a/crates/store/src/db/sql.rs +++ b/crates/store/src/db/sql.rs @@ -640,14 +640,23 @@ pub fn select_notes_since_block_by_tag( details FROM notes - -- find the next block which contains at least one note with a matching tag WHERE - tag IN rarray(?1) AND - block_num > ?2 - ORDER BY - block_num ASC - LIMIT - 1; + -- find the next block which contains at least one note with a matching tag + block_num = ( + SELECT + block_num + FROM + notes + WHERE + tag IN rarray(?1) AND + block_num > ?2 + ORDER BY + block_num ASC + LIMIT + 1 + ) AND + -- filter the block's notes and return only the ones matching the requested tags + tag IN rarray(?1); ", )?; let mut rows = stmt.query(params![Rc::new(tags), block_num])?; From 2eba60dc4af351dee05d5004534f74f99f958965 Mon Sep 17 00:00:00 2001 From: Mirko von Leipzig Date: Fri, 2 Aug 2024 14:32:36 +0200 Subject: [PATCH 6/8] feat: mmr proof instead of delta --- crates/proto/src/generated/responses.rs | 4 ++-- crates/rpc-proto/proto/responses.proto | 4 ++-- crates/store/src/db/mod.rs | 4 ++-- crates/store/src/db/sql.rs | 10 ++++----- crates/store/src/errors.rs | 10 +++++++++ crates/store/src/server/api.rs | 4 ++-- crates/store/src/state.rs | 28 ++++++------------------- proto/responses.proto | 4 ++-- 8 files changed, 31 insertions(+), 37 deletions(-) diff --git a/crates/proto/src/generated/responses.rs b/crates/proto/src/generated/responses.rs index 2dc92392b..a30c07bdb 100644 --- a/crates/proto/src/generated/responses.rs +++ b/crates/proto/src/generated/responses.rs @@ -72,9 +72,9 @@ pub struct SyncNoteResponse { /// Block header of the block with the first note matching the specified criteria #[prost(message, optional, tag = "2")] pub block_header: ::core::option::Option, - /// Data needed to update the partial MMR from `request.block_num + 1` to `response.block_header.block_num` + /// Proof for block header's MMR with respect to the chain tip. #[prost(message, optional, tag = "3")] - pub mmr_delta: ::core::option::Option, + pub mmr_path: ::core::option::Option, /// List of all notes together with the Merkle paths from `response.block_header.note_root` #[prost(message, repeated, tag = "4")] pub notes: ::prost::alloc::vec::Vec, diff --git a/crates/rpc-proto/proto/responses.proto b/crates/rpc-proto/proto/responses.proto index 87abdf601..99db3ab97 100644 --- a/crates/rpc-proto/proto/responses.proto +++ b/crates/rpc-proto/proto/responses.proto @@ -69,8 +69,8 @@ message SyncNoteResponse { // Block header of the block with the first note matching the specified criteria block_header.BlockHeader block_header = 2; - // Data needed to update the partial MMR from `request.block_num + 1` to `response.block_header.block_num` - mmr.MmrDelta mmr_delta = 3; + // Proof for block header's MMR with respect to the chain tip. + merkle.MerklePath mmr_path = 3; // List of all notes together with the Merkle paths from `response.block_header.note_root` repeated note.NoteSyncRecord notes = 4; diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 30aaad65d..d542a735e 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -26,7 +26,7 @@ use crate::{ blocks::BlockStore, config::StoreConfig, db::migrations::apply_migrations, - errors::{DatabaseError, DatabaseSetupError, GenesisError, StateSyncError}, + errors::{DatabaseError, DatabaseSetupError, GenesisError, NoteSyncError, StateSyncError}, genesis::GenesisState, types::{AccountId, BlockNumber}, COMPONENT, @@ -291,7 +291,7 @@ impl Db { &self, block_num: BlockNumber, note_tags: Vec, - ) -> Result { + ) -> Result { self.pool .get() .await diff --git a/crates/store/src/db/sql.rs b/crates/store/src/db/sql.rs index f0a6cfa70..dc58953e8 100644 --- a/crates/store/src/db/sql.rs +++ b/crates/store/src/db/sql.rs @@ -22,7 +22,7 @@ use super::{ NoteRecord, NoteSyncUpdate, NullifierInfo, Result, StateSyncUpdate, TransactionSummary, }; use crate::{ - errors::{DatabaseError, StateSyncError}, + errors::{DatabaseError, NoteSyncError, StateSyncError}, types::{AccountId, BlockNumber}, }; @@ -974,19 +974,19 @@ pub fn get_note_sync( conn: &mut Connection, block_num: BlockNumber, note_tags: &[u32], -) -> Result { +) -> Result { let notes = select_notes_since_block_by_tag(conn, note_tags, block_num)?; let (block_header, chain_tip) = if !notes.is_empty() { let block_header = select_block_header_by_block_num(conn, Some(notes[0].block_num))? - .ok_or(StateSyncError::EmptyBlockHeadersTable)?; + .ok_or(NoteSyncError::EmptyBlockHeadersTable)?; let tip = select_block_header_by_block_num(conn, None)? - .ok_or(StateSyncError::EmptyBlockHeadersTable)?; + .ok_or(NoteSyncError::EmptyBlockHeadersTable)?; (block_header, tip.block_num()) } else { let block_header = select_block_header_by_block_num(conn, None)? - .ok_or(StateSyncError::EmptyBlockHeadersTable)?; + .ok_or(NoteSyncError::EmptyBlockHeadersTable)?; let block_num = block_header.block_num(); (block_header, block_num) diff --git a/crates/store/src/errors.rs b/crates/store/src/errors.rs index c1bfd2f01..65461186d 100644 --- a/crates/store/src/errors.rs +++ b/crates/store/src/errors.rs @@ -214,3 +214,13 @@ pub enum StateSyncError { #[error("Failed to build MMR delta: {0}")] FailedToBuildMmrDelta(MmrError), } + +#[derive(Error, Debug)] +pub enum NoteSyncError { + #[error("Database error: {0}")] + DatabaseError(#[from] DatabaseError), + #[error("Block headers table is empty")] + EmptyBlockHeadersTable, + #[error("Error retrieving the merkle proof for the block: {0}")] + MmrError(#[from] MmrError), +} diff --git a/crates/store/src/server/api.rs b/crates/store/src/server/api.rs index 888e3fefb..3aad15d60 100644 --- a/crates/store/src/server/api.rs +++ b/crates/store/src/server/api.rs @@ -234,7 +234,7 @@ impl api_server::Api for StoreApi { ) -> Result, Status> { let request = request.into_inner(); - let (state, delta) = self + let (state, mmr_proof) = self .state .sync_notes(request.block_num, request.note_tags) .await @@ -254,7 +254,7 @@ impl api_server::Api for StoreApi { Ok(Response::new(SyncNoteResponse { chain_tip: state.chain_tip, block_header: Some(state.block_header.into()), - mmr_delta: Some(delta.into()), + mmr_path: Some(mmr_proof.merkle_path.into()), notes, })) } diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 2465cd199..a9abc3ead 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -31,7 +31,7 @@ use crate::{ blocks::BlockStore, db::{Db, NoteRecord, NoteSyncUpdate, NullifierInfo, StateSyncUpdate}, errors::{ - ApplyBlockError, DatabaseError, GetBlockHeaderError, GetBlockInputsError, + ApplyBlockError, DatabaseError, GetBlockHeaderError, GetBlockInputsError, NoteSyncError, StateInitializationError, StateSyncError, }, nullifier_tree::NullifierTree, @@ -466,32 +466,16 @@ impl State { &self, block_num: BlockNumber, note_tag_prefixes: Vec, - ) -> Result<(NoteSyncUpdate, MmrDelta), StateSyncError> { + ) -> Result<(NoteSyncUpdate, MmrProof), NoteSyncError> { let inner = self.inner.read().await; let note_sync = self.db.get_note_sync(block_num, note_tag_prefixes).await?; - let delta = if block_num == note_sync.block_header.block_num() { - // The client is in sync with the chain tip. - MmrDelta { forest: block_num as usize, data: vec![] } - } else { - // Important notes about the boundary conditions: - // - // - The Mmr forest is 1-indexed whereas the block number is 0-indexed. The Mmr root - // contained in the block header always lag behind by one block, this is because the Mmr - // leaves are hashes of block headers, and we can't have self-referential hashes. These two - // points cancel out and don't require adjusting. - // - Mmr::get_delta is inclusive, whereas the sync_state request block_num is defined to be - // exclusive, so the from_forest has to be adjusted with a +1 - let from_forest = (block_num + 1) as usize; - let to_forest = note_sync.block_header.block_num() as usize; - inner - .chain_mmr - .get_delta(from_forest, to_forest) - .map_err(StateSyncError::FailedToBuildMmrDelta)? - }; + let mmr_proof = inner + .chain_mmr + .open(note_sync.block_header.block_num() as usize, inner.chain_mmr.forest())?; - Ok((note_sync, delta)) + Ok((note_sync, mmr_proof)) } /// Returns data needed by the block producer to construct and prove the next block. diff --git a/proto/responses.proto b/proto/responses.proto index 87abdf601..99db3ab97 100644 --- a/proto/responses.proto +++ b/proto/responses.proto @@ -69,8 +69,8 @@ message SyncNoteResponse { // Block header of the block with the first note matching the specified criteria block_header.BlockHeader block_header = 2; - // Data needed to update the partial MMR from `request.block_num + 1` to `response.block_header.block_num` - mmr.MmrDelta mmr_delta = 3; + // Proof for block header's MMR with respect to the chain tip. + merkle.MerklePath mmr_path = 3; // List of all notes together with the Merkle paths from `response.block_header.note_root` repeated note.NoteSyncRecord notes = 4; From 4ec54b0828556747eba347373aa60d7099b9f46d Mon Sep 17 00:00:00 2001 From: Mirko von Leipzig Date: Mon, 5 Aug 2024 12:54:44 +0200 Subject: [PATCH 7/8] chore: review changes --- crates/proto/src/generated/responses.rs | 5 +++++ crates/rpc-proto/proto/responses.proto | 5 +++++ crates/store/src/db/sql.rs | 12 ++++++------ crates/store/src/state.rs | 6 +++--- proto/responses.proto | 5 +++++ 5 files changed, 24 insertions(+), 9 deletions(-) diff --git a/crates/proto/src/generated/responses.rs b/crates/proto/src/generated/responses.rs index a30c07bdb..e5cc8d2f9 100644 --- a/crates/proto/src/generated/responses.rs +++ b/crates/proto/src/generated/responses.rs @@ -73,6 +73,11 @@ pub struct SyncNoteResponse { #[prost(message, optional, tag = "2")] pub block_header: ::core::option::Option, /// Proof for block header's MMR with respect to the chain tip. + /// + /// More specifically, the full proof consists of `forest`, `position` and `path` components. This + /// value constitutes the `path`. The other two components can be obtained as follows: + /// - `position` is simply `resopnse.block_header.block_num` + /// - `forest` is the same as `response.chain_tip + 1` #[prost(message, optional, tag = "3")] pub mmr_path: ::core::option::Option, /// List of all notes together with the Merkle paths from `response.block_header.note_root` diff --git a/crates/rpc-proto/proto/responses.proto b/crates/rpc-proto/proto/responses.proto index 99db3ab97..4a0c9a044 100644 --- a/crates/rpc-proto/proto/responses.proto +++ b/crates/rpc-proto/proto/responses.proto @@ -70,6 +70,11 @@ message SyncNoteResponse { block_header.BlockHeader block_header = 2; // Proof for block header's MMR with respect to the chain tip. + // + // More specifically, the full proof consists of `forest`, `position` and `path` components. This + // value constitutes the `path`. The other two components can be obtained as follows: + // - `position` is simply `resopnse.block_header.block_num` + // - `forest` is the same as `response.chain_tip + 1` merkle.MerklePath mmr_path = 3; // List of all notes together with the Merkle paths from `response.block_header.note_root` diff --git a/crates/store/src/db/sql.rs b/crates/store/src/db/sql.rs index dc58953e8..49ffd5a5e 100644 --- a/crates/store/src/db/sql.rs +++ b/crates/store/src/db/sql.rs @@ -521,9 +521,9 @@ pub fn insert_notes(transaction: &Transaction, notes: &[NoteRecord]) -> Result, + note_tags: Vec, ) -> Result<(NoteSyncUpdate, MmrProof), NoteSyncError> { let inner = self.inner.read().await; - let note_sync = self.db.get_note_sync(block_num, note_tag_prefixes).await?; + let note_sync = self.db.get_note_sync(block_num, note_tags).await?; let mmr_proof = inner .chain_mmr diff --git a/proto/responses.proto b/proto/responses.proto index 99db3ab97..4a0c9a044 100644 --- a/proto/responses.proto +++ b/proto/responses.proto @@ -70,6 +70,11 @@ message SyncNoteResponse { block_header.BlockHeader block_header = 2; // Proof for block header's MMR with respect to the chain tip. + // + // More specifically, the full proof consists of `forest`, `position` and `path` components. This + // value constitutes the `path`. The other two components can be obtained as follows: + // - `position` is simply `resopnse.block_header.block_num` + // - `forest` is the same as `response.chain_tip + 1` merkle.MerklePath mmr_path = 3; // List of all notes together with the Merkle paths from `response.block_header.note_root` From c5bda0afef3b4d0032fb4b813a9b0166912315e8 Mon Sep 17 00:00:00 2001 From: Bobbin Threadbare Date: Mon, 5 Aug 2024 11:15:47 -0700 Subject: [PATCH 8/8] chore: fix lints --- crates/block-producer/src/batch_builder/batch.rs | 1 + crates/store/src/server/api.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/block-producer/src/batch_builder/batch.rs b/crates/block-producer/src/batch_builder/batch.rs index 916b63bd8..cda4992e0 100644 --- a/crates/block-producer/src/batch_builder/batch.rs +++ b/crates/block-producer/src/batch_builder/batch.rs @@ -180,6 +180,7 @@ impl TransactionBatch { /// Returns an iterator over (account_id, init_state_hash) tuples for accounts that were /// modified in this transaction batch. + #[cfg(test)] pub fn account_initial_states(&self) -> impl Iterator + '_ { self.updated_accounts .iter() diff --git a/crates/store/src/server/api.rs b/crates/store/src/server/api.rs index 8bd986b93..88fc63469 100644 --- a/crates/store/src/server/api.rs +++ b/crates/store/src/server/api.rs @@ -244,7 +244,7 @@ impl api_server::Api for StoreApi { .notes .into_iter() .map(|note| NoteSyncRecord { - note_index: note.note_index.to_absolute_index() as u32, + note_index: note.note_index.to_absolute_index(), note_id: Some(note.note_id.into()), metadata: Some(note.metadata.into()), merkle_path: Some(note.merkle_path.into()),