Skip to content
This repository has been archived by the owner on Jan 13, 2025. It is now read-only.

Minor cleanup on bigtable_upload #29379

Merged
merged 1 commit into from
Dec 23, 2022
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 16 additions & 10 deletions ledger/src/bigtable_upload.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,22 +54,25 @@ pub async fn upload_confirmed_blocks(
) -> Result<Slot, Box<dyn std::error::Error>> {
let mut measure = Measure::start("entire upload");

info!("Loading ledger slots starting at {}...", starting_slot);
info!(
"Loading ledger slots from {} to {}",
starting_slot, ending_slot
);
CriesofCarrots marked this conversation as resolved.
Show resolved Hide resolved
let blockstore_slots: Vec<_> = blockstore
.rooted_slot_iterator(starting_slot)
.map_err(|err| {
format!("Failed to load entries starting from slot {starting_slot}: {err:?}")
})?
.map_while(|slot| (slot <= ending_slot).then_some(slot))
.take_while(|slot| *slot <= ending_slot)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

❤️

.collect();

if blockstore_slots.is_empty() {
warn!("Ledger has no slots from {starting_slot} to {ending_slot:?}");
return Ok(ending_slot);
}

let first_blockstore_slot = blockstore_slots.first().unwrap();
let last_blockstore_slot = blockstore_slots.last().unwrap();
let first_blockstore_slot = *blockstore_slots.first().unwrap();
let last_blockstore_slot = *blockstore_slots.last().unwrap();
CriesofCarrots marked this conversation as resolved.
Show resolved Hide resolved
info!(
"Found {} slots in the range ({}, {})",
blockstore_slots.len(),
Expand All @@ -85,8 +88,8 @@ pub async fn upload_confirmed_blocks(
first_blockstore_slot, last_blockstore_slot
);

let mut start_slot = *first_blockstore_slot;
while start_slot <= *last_blockstore_slot {
let mut start_slot = first_blockstore_slot;
while start_slot <= last_blockstore_slot {
let mut next_bigtable_slots = loop {
let num_bigtable_blocks = min(1000, config.max_num_slots_to_check * 2);
match bigtable
Expand All @@ -109,7 +112,7 @@ pub async fn upload_confirmed_blocks(
}
bigtable_slots
.into_iter()
.filter(|slot| slot <= last_blockstore_slot)
.filter(|slot| *slot <= last_blockstore_slot)
.collect::<Vec<_>>()
} else {
Vec::new()
Expand All @@ -118,7 +121,7 @@ pub async fn upload_confirmed_blocks(
// The blocks that still need to be uploaded is the difference between what's already in the
// bigtable and what's in blockstore...
let blocks_to_upload = {
let blockstore_slots = blockstore_slots.iter().cloned().collect::<HashSet<_>>();
let blockstore_slots = blockstore_slots.into_iter().collect::<HashSet<_>>();
let bigtable_slots = bigtable_slots.into_iter().collect::<HashSet<_>>();

let mut blocks_to_upload = blockstore_slots
Expand All @@ -131,8 +134,11 @@ pub async fn upload_confirmed_blocks(
};

if blocks_to_upload.is_empty() {
info!("No blocks need to be uploaded to bigtable");
return Ok(*last_blockstore_slot);
info!(
"No blocks between {} and {} need to be uploaded to bigtable",
CriesofCarrots marked this conversation as resolved.
Show resolved Hide resolved
starting_slot, ending_slot
);
return Ok(last_blockstore_slot);
}
let last_slot = *blocks_to_upload.last().unwrap();
info!(
Expand Down