Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improvement(rpc-server): Add additional logs for shadow_data_consistency #93

Merged
merged 3 commits into from
Aug 3, 2023
Merged

Improvement(rpc-server): Add additional logs for shadow_data_consistency #93

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4161406
1. Add retry to get objects from s3 bucket
kobayurii Jul 20, 2023
3000155
rename variables to more clear and improve msg error for mismatch data
kobayurii Jul 31, 2023
e71da95
log inmprovement
kobayurii Aug 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 40 additions & 20 deletions rpc-server/src/modules/blocks/methods.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,12 +62,17 @@ pub async fn chunk(
{
let near_rpc_client = data.near_rpc_client.clone();
let error_meta = format!("CHUNK: {:?}", params);
let read_rpc_response_json = match &result {
Ok(res) => serde_json::to_value(&res.chunk_view),
Err(err) => serde_json::to_value(err),
let (read_rpc_response_json, is_response_ok) = match &result {
Ok(res) => (serde_json::to_value(&res.chunk_view), true),
Err(err) => (serde_json::to_value(err), false),
};
let comparison_result =
shadow_compare_results(read_rpc_response_json, near_rpc_client, params).await;
let comparison_result = shadow_compare_results(
read_rpc_response_json,
near_rpc_client,
params,
is_response_ok,
)
.await;

match comparison_result {
Ok(_) => {
Expand Down Expand Up @@ -172,20 +177,25 @@ async fn block_call(
{
let near_rpc_client = data.near_rpc_client.clone();
let error_meta = format!("BLOCK: {:?}", params);
let read_rpc_response_json = match &result {
let (read_rpc_response_json, is_response_ok) = match &result {
Ok(res) => {
if let near_primitives::types::BlockReference::Finality(_) = params.block_reference
{
params.block_reference = near_primitives::types::BlockReference::from(
near_primitives::types::BlockId::Height(res.block_view.header.height),
)
};
serde_json::to_value(&res.block_view)
(serde_json::to_value(&res.block_view), true)
}
Err(err) => serde_json::to_value(err),
Err(err) => (serde_json::to_value(err), false),
};
let comparison_result =
shadow_compare_results(read_rpc_response_json, near_rpc_client, params).await;
let comparison_result = shadow_compare_results(
read_rpc_response_json,
near_rpc_client,
params,
is_response_ok,
)
.await;

match comparison_result {
Ok(_) => {
Expand Down Expand Up @@ -225,12 +235,17 @@ async fn changes_in_block_call(
)
}
let error_meta = format!("CHANGES_IN_BLOCK: {:?}", params);
let read_rpc_response_json = match &result {
Ok(res) => serde_json::to_value(res),
Err(err) => serde_json::to_value(err),
let (read_rpc_response_json, is_response_ok) = match &result {
Ok(res) => (serde_json::to_value(res), true),
Err(err) => (serde_json::to_value(err), false),
};
let comparison_result =
shadow_compare_results(read_rpc_response_json, near_rpc_client, params).await;
let comparison_result = shadow_compare_results(
read_rpc_response_json,
near_rpc_client,
params,
is_response_ok,
)
.await;

match comparison_result {
Ok(_) => {
Expand Down Expand Up @@ -270,12 +285,17 @@ async fn changes_in_block_by_type_call(
)
}
let error_meta = format!("CHANGES_IN_BLOCK_BY_TYPE: {:?}", params);
let read_rpc_response_json = match &result {
Ok(res) => serde_json::to_value(res),
Err(err) => serde_json::to_value(err),
let (read_rpc_response_json, is_response_ok) = match &result {
Ok(res) => (serde_json::to_value(res), true),
Err(err) => (serde_json::to_value(err), false),
};
let comparison_result =
shadow_compare_results(read_rpc_response_json, near_rpc_client, params).await;
let comparison_result = shadow_compare_results(
read_rpc_response_json,
near_rpc_client,
params,
is_response_ok,
)
.await;

match comparison_result {
Ok(_) => {
Expand Down
15 changes: 10 additions & 5 deletions rpc-server/src/modules/queries/methods.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,13 +116,18 @@ async fn query_call(
)
}

let read_rpc_response_json = match &result {
Ok(res) => serde_json::to_value(res),
Err(err) => serde_json::to_value(err),
let (read_rpc_response_json, is_response_ok) = match &result {
Ok(res) => (serde_json::to_value(res), true),
Err(err) => (serde_json::to_value(err), false),
};

let comparison_result =
shadow_compare_results(read_rpc_response_json, near_rpc_client, params).await;
let comparison_result = shadow_compare_results(
read_rpc_response_json,
near_rpc_client,
params,
is_response_ok,
)
.await;

match comparison_result {
Ok(_) => {
Expand Down
16 changes: 10 additions & 6 deletions rpc-server/src/modules/receipts/methods.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,18 @@ pub async fn receipt(
{
let near_rpc_client = data.near_rpc_client.clone();
let error_meta = format!("TX: {:?}", params);

let read_rpc_response_json = match &result {
Ok(res) => serde_json::to_value(res),
Err(err) => serde_json::to_value(err),
let (read_rpc_response_json, is_response_ok) = match &result {
Ok(res) => (serde_json::to_value(res), true),
Err(err) => (serde_json::to_value(err), false),
};

let comparison_result =
shadow_compare_results(read_rpc_response_json, near_rpc_client, params).await;
let comparison_result = shadow_compare_results(
read_rpc_response_json,
near_rpc_client,
params,
is_response_ok,
)
.await;

match comparison_result {
Ok(_) => {
Expand Down
19 changes: 10 additions & 9 deletions rpc-server/src/modules/transactions/methods.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,9 @@ pub async fn tx(
{
let near_rpc_client = data.near_rpc_client.clone();
let error_meta = format!("TX: {:?}", params);

let read_rpc_response_json = match &result {
Ok(res) => serde_json::to_value(res),
Err(err) => serde_json::to_value(err),
let (read_rpc_response_json, is_response_ok) = match &result {
Ok(res) => (serde_json::to_value(res), true),
Err(err) => (serde_json::to_value(err), false),
};

let comparison_result = shadow_compare_results(
Expand All @@ -44,12 +43,13 @@ pub async fn tx(
near_jsonrpc_client::methods::tx::RpcTransactionStatusRequest {
transaction_info: tx_status_request.transaction_info,
},
is_response_ok,
)
.await;

match comparison_result {
Ok(_) => {
tracing::info!(target: "shadow-data-consistency", "Shadow data check: CORRECT\n{}", error_meta);
tracing::info!(target: "shadow_data_consistency", "Shadow data check: CORRECT\n{}", error_meta);
}
Err(err) => {
tracing::warn!(target: "shadow_data_consistency", "Shadow data check: ERROR\n{}\n{:?}", error_meta, err);
Expand Down Expand Up @@ -79,9 +79,9 @@ pub async fn tx_status(
let near_rpc_client = data.near_rpc_client.clone();
let error_meta = format!("EXPERIMENTAL_TX_STATUS: {:?}", params);

let read_rpc_response_json = match &result {
Ok(res) => serde_json::to_value(res),
Err(err) => serde_json::to_value(err),
let (read_rpc_response_json, is_response_ok) = match &result {
Ok(res) => (serde_json::to_value(res), true),
Err(err) => (serde_json::to_value(err), false),
};

let comparison_result = shadow_compare_results(
Expand All @@ -93,12 +93,13 @@ pub async fn tx_status(
near_jsonrpc_client::methods::EXPERIMENTAL_tx_status::RpcTransactionStatusRequest {
transaction_info: tx_status_request.transaction_info,
},
is_response_ok,
)
.await;

match comparison_result {
Ok(_) => {
tracing::info!(target: "shadow-data-consistency", "Shadow data check: CORRECT\n{}", error_meta);
tracing::info!(target: "shadow_data_consistency", "Shadow data check: CORRECT\n{}", error_meta);
}
Err(err) => {
tracing::warn!(target: "shadow_data_consistency", "Shadow data check: ERROR\n{}\n{:?}", error_meta, err);
Expand Down
75 changes: 65 additions & 10 deletions rpc-server/src/utils.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,27 +100,27 @@ pub async fn update_final_block_height_regularly(
/// Otherwise, it returns `Err(ShadowDataConsistencyError)`.
#[cfg(feature = "shadow_data_consistency")]
pub async fn shadow_compare_results<M>(
readrpc_response: Result<serde_json::Value, serde_json::Error>,
read_rpc_response: Result<serde_json::Value, serde_json::Error>,
client: near_jsonrpc_client::JsonRpcClient,
params: M,
read_rpc_response_is_ok: bool,
) -> Result<(), ShadowDataConsistencyError>
where
M: near_jsonrpc_client::methods::RpcMethod + std::fmt::Debug,
<M as near_jsonrpc_client::methods::RpcMethod>::Response: serde::ser::Serialize,
<M as near_jsonrpc_client::methods::RpcMethod>::Error: std::fmt::Debug + serde::ser::Serialize,
{
tracing::debug!(target: "shadow_data_consistency", "Compare results. {:?}", params);

let readrpc_response_json = match readrpc_response {
Ok(readrpc_response_json) => readrpc_response_json,
let read_rpc_response_json = match read_rpc_response {
Ok(read_rpc_response_json) => read_rpc_response_json,
Err(err) => {
return Err(ShadowDataConsistencyError::ReadRpcResponseParseError(err));
}
};

let mut near_rpc_response = client.call(&params).await;

for _ in 0..3 {
for _ in 0..crate::config::DEFAULT_RETRY_COUNT {
if let Err(json_rpc_err) = &near_rpc_response {
let retry = match json_rpc_err {
near_jsonrpc_client::errors::JsonRpcError::TransportError(_) => true,
Expand All @@ -144,17 +144,17 @@ where
}
}

let near_rpc_response_json = match near_rpc_response {
let (near_rpc_response_json, near_rpc_response_is_ok) = match near_rpc_response {
Ok(result) => match serde_json::to_value(&result) {
Ok(near_rpc_response_json) => near_rpc_response_json,
Ok(near_rpc_response_json) => (near_rpc_response_json, true),
Err(err) => {
return Err(ShadowDataConsistencyError::NearRpcResponseParseError(err));
}
},
Err(err) => {
if let Some(e) = err.handler_error() {
match serde_json::to_value(&e) {
Ok(near_rpc_response_json) => near_rpc_response_json,
Ok(near_rpc_response_json) => (near_rpc_response_json, false),
Err(err) => {
return Err(ShadowDataConsistencyError::NearRpcResponseParseError(err));
}
Expand All @@ -171,11 +171,66 @@ where
let config = Config::new(CompareMode::Strict).numeric_mode(NumericMode::AssumeFloat);

// Sorts the values of the JSON objects before comparing them.
let read_rpc_json = json_sort_value(readrpc_response_json);
let read_rpc_json = json_sort_value(read_rpc_response_json);
let near_rpc_json = json_sort_value(near_rpc_response_json);

if let Err(err) = assert_json_matches_no_panic(&read_rpc_json, &near_rpc_json, config) {
return Err(ShadowDataConsistencyError::ResultsDontMatch(err));
// separate mismatching successful and failure responses into different targets
// to make it easier to find reasons of the mismatching
// let reason = String::new();
// let description = String::new();
let (reason, description) = if read_rpc_response_is_ok && near_rpc_response_is_ok {
// Both services(read_rpc and near_rpc) have a successful response
// if data mismatch we are logging into `shadow_data_consistency_successful_responses`
// both response objects for future investigation.
(
"shadow_data_consistency_successful_responses",
format!(
"Shadow data check: DATA MISMATCH\n READ_RPC_DATA: {:?}\n NEAR_RPC_DATA: {:?}\n",
read_rpc_json,
near_rpc_json,
)
)
} else if !read_rpc_response_is_ok && near_rpc_response_is_ok {
// read_rpc service has failure response
// and near_rpc has successful response we are logging into
// `shadow_data_consistency_read_rpc_failure_response` just message about mismatch
// data without objects.
// In the future we should be investigate why we got error in the read_rpc.
(
"shadow_data_consistency_read_rpc_failure_response",
String::from(
"Shadow data check: read_rpc response failed, near_rpc response success",
),
)
} else if read_rpc_response_is_ok && !near_rpc_response_is_ok {
// read_rpc service has successful response
// and near_rpc has failure response we are logging into
// `shadow_data_consistency_near_failure_response` just message about mismatch
// data without objects. Expected that all error will be related with network issues.
(
"shadow_data_consistency_near_failure_response",
String::from(
"Shadow data check: read_rpc response success, near_rpc response failed",
),
)
} else {
// Both services(read_rpc and near_rpc) have a failure response
// we are logging into `shadow_data_consistency_failure_responses`
// both response objects for future investigation.
// expected we will only have a difference in the error text.
(
"shadow_data_consistency_failure_responses",
format!(
"Shadow data check: DATA MISMATCH\n READ_RPC_DATA: {:?}\n NEAR_RPC_DATA: {:?}\n",
read_rpc_json,
near_rpc_json,
),
)
};
return Err(ShadowDataConsistencyError::ResultsDontMatch(format!(
"{err}.\n Reason: {reason}.\n Description: {description}"
)));
};
Ok(())
}
Expand Down
Loading