From 7c5ebcebb47817f854ee9c8f20c629f8dc09b64d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Zemanovi=C4=8D?= Date: Tue, 2 May 2023 17:55:15 +0200 Subject: [PATCH 1/4] pos/epoched: add a test for epoched data trimming --- proof_of_stake/src/epoched.rs | 465 +++++++++++++++++++++++++++------- 1 file changed, 379 insertions(+), 86 deletions(-) diff --git a/proof_of_stake/src/epoched.rs b/proof_of_stake/src/epoched.rs index cc99555902..2d5e12c155 100644 --- a/proof_of_stake/src/epoched.rs +++ b/proof_of_stake/src/epoched.rs @@ -671,89 +671,382 @@ pub trait EpochOffset: fn dyn_offset() -> DynEpochOffset; } -// mod test { -// use namada_core::ledger::storage::testing::TestStorage; -// use namada_core::types::address::{self, Address}; -// use namada_core::types::storage::Key; -// -// use super::{ -// storage, storage_api, Epoch, LazyMap, NestedEpoched, NestedMap, -// OffsetPipelineLen, -// }; -// -// #[test] -// fn testing_epoched_new() -> storage_api::Result<()> { -// let mut storage = TestStorage::default(); -// -// let key1 = storage::Key::parse("test_nested1").unwrap(); -// let nested1 = -// NestedEpoched::, OffsetPipelineLen>::open( -// key1, -// ); -// nested1.init(&mut storage, Epoch(0))?; -// -// let key2 = storage::Key::parse("test_nested2").unwrap(); -// let nested2 = NestedEpoched::< -// NestedMap>, -// OffsetPipelineLen, -// >::open(key2); -// nested2.init(&mut storage, Epoch(0))?; -// -// dbg!(&nested1.get_last_update_storage_key()); -// dbg!(&nested1.get_last_update(&storage)); -// -// nested1.at(&Epoch(0)).insert( -// &mut storage, -// address::testing::established_address_1(), -// 1432, -// )?; -// dbg!(&nested1.at(&Epoch(0)).iter(&mut storage)?.next()); -// dbg!(&nested1.at(&Epoch(1)).iter(&mut storage)?.next()); -// -// nested2.at(&Epoch(0)).at(&100).insert( -// &mut storage, -// 1, -// address::testing::established_address_2(), -// )?; -// dbg!(&nested2.at(&Epoch(0)).iter(&mut storage)?.next()); -// dbg!(&nested2.at(&Epoch(1)).iter(&mut storage)?.next()); -// -// dbg!(&nested_epoched.get_epoch_key(&Epoch::from(0))); -// -// let epoch = Epoch::from(0); -// let addr = address::testing::established_address_1(); -// let amount: u64 = 234235; -// -// nested_epoched -// .at(&epoch) -// .insert(&mut storage, addr.clone(), amount)?; -// -// let epoch = epoch + 3_u64; -// nested_epoched.at(&epoch).insert( -// &mut storage, -// addr.clone(), -// 999_u64, -// )?; -// -// dbg!(nested_epoched.contains_epoch(&storage, &Epoch::from(0))?); -// dbg!( -// nested_epoched -// .get_data_handler() -// .get_data_key(&Epoch::from(3)) -// ); -// dbg!(nested_epoched.contains_epoch(&storage, &Epoch::from(3))?); -// dbg!( -// nested_epoched -// .at(&Epoch::from(0)) -// .get(&storage, &addr.clone())? -// ); -// dbg!( -// nested_epoched -// .at(&Epoch::from(3)) -// .get(&storage, &addr.clone())? -// ); -// dbg!(nested_epoched.at(&Epoch::from(3)).get_data_key(&addr)); -// -// Ok(()) -// } -// } +#[cfg(test)] +mod test { + use namada_core::ledger::storage::testing::TestWlStorage; + use test_log::test; + + use super::*; + + #[test] + fn test_epoched_data_trimming() -> storage_api::Result<()> { + let mut s = TestWlStorage::default(); + + const NUM_PAST_EPOCHS: u64 = 2; + let key_prefix = storage::Key::parse("test").unwrap(); + let epoched = Epoched::::open( + key_prefix, + ); + let data_handler = epoched.get_data_handler(); + assert!(epoched.get_last_update(&s)?.is_none()); + assert!(epoched.get_oldest_epoch(&s)?.is_none()); + + epoched.init_at_genesis(&mut s, 0, Epoch(0))?; + assert_eq!(epoched.get_last_update(&s)?, Some(Epoch(0))); + assert_eq!(epoched.get_oldest_epoch(&s)?, Some(Epoch(0))); + assert_eq!(data_handler.get(&s, &Epoch(0))?, Some(0)); + + epoched.set(&mut s, 1, Epoch(0), 0)?; + assert_eq!(epoched.get_last_update(&s)?, Some(Epoch(0))); + assert_eq!(epoched.get_oldest_epoch(&s)?, Some(Epoch(0))); + assert_eq!(data_handler.get(&s, &Epoch(0))?, Some(1)); + + epoched.set(&mut s, 2, Epoch(1), 0)?; + assert_eq!(epoched.get_last_update(&s)?, Some(Epoch(1))); + assert_eq!(epoched.get_oldest_epoch(&s)?, Some(Epoch(0))); + assert_eq!(data_handler.get(&s, &Epoch(0))?, Some(1)); + assert_eq!(data_handler.get(&s, &Epoch(1))?, Some(2)); + + // Nothing is trimmed yet, oldest kept epoch is 0 + epoched.set(&mut s, 3, Epoch(2), 0)?; + assert_eq!(epoched.get_last_update(&s)?, Some(Epoch(2))); + assert_eq!(epoched.get_oldest_epoch(&s)?, Some(Epoch(0))); + assert_eq!(data_handler.get(&s, &Epoch(0))?, Some(1)); + assert_eq!(data_handler.get(&s, &Epoch(1))?, Some(2)); + assert_eq!(data_handler.get(&s, &Epoch(2))?, Some(3)); + + // Epoch 0 should be trimmed now, oldest kept epoch is 1 + epoched.set(&mut s, 4, Epoch(3), 0)?; + assert_eq!(epoched.get_last_update(&s)?, Some(Epoch(3))); + assert_eq!(epoched.get_oldest_epoch(&s)?, Some(Epoch(1))); + assert_eq!(data_handler.get(&s, &Epoch(0))?, None); + assert_eq!(data_handler.get(&s, &Epoch(1))?, Some(2)); + assert_eq!(data_handler.get(&s, &Epoch(2))?, Some(3)); + assert_eq!(data_handler.get(&s, &Epoch(3))?, Some(4)); + + // Anything before epoch 3 should be trimmed + epoched.set(&mut s, 5, Epoch(5), 0)?; + assert_eq!(epoched.get_last_update(&s)?, Some(Epoch(5))); + assert_eq!(epoched.get_oldest_epoch(&s)?, Some(Epoch(3))); + assert_eq!(data_handler.get(&s, &Epoch(0))?, None); + assert_eq!(data_handler.get(&s, &Epoch(1))?, None); + assert_eq!(data_handler.get(&s, &Epoch(2))?, None); + assert_eq!(data_handler.get(&s, &Epoch(3))?, Some(4)); + assert_eq!(data_handler.get(&s, &Epoch(5))?, Some(5)); + + // Anything before epoch 8 should be trimmed + epoched.set(&mut s, 6, Epoch(10), 0)?; + assert_eq!(epoched.get_last_update(&s)?, Some(Epoch(10))); + assert_eq!(epoched.get_oldest_epoch(&s)?, Some(Epoch(8))); + for epoch in Epoch(0).iter_range(7) { + assert_eq!(data_handler.get(&s, &epoch)?, None); + } + // The value from the latest epoch 5 is assigned to epoch 8 + assert_eq!(data_handler.get(&s, &Epoch(8))?, Some(5)); + assert_eq!(data_handler.get(&s, &Epoch(9))?, None); + assert_eq!(data_handler.get(&s, &Epoch(10))?, Some(6)); + + Ok(()) + } + + #[test] + fn test_epoched_without_data_trimming() -> storage_api::Result<()> { + let mut s = TestWlStorage::default(); + + const NUM_PAST_EPOCHS: u64 = u64::MAX; + let key_prefix = storage::Key::parse("test").unwrap(); + let epoched = Epoched::::open( + key_prefix, + ); + let data_handler = epoched.get_data_handler(); + assert!(epoched.get_last_update(&s)?.is_none()); + assert!(epoched.get_oldest_epoch(&s)?.is_none()); + + epoched.init_at_genesis(&mut s, 0, Epoch(0))?; + assert_eq!(epoched.get_last_update(&s)?, Some(Epoch(0))); + assert_eq!(epoched.get_oldest_epoch(&s)?, Some(Epoch(0))); + assert_eq!(data_handler.get(&s, &Epoch(0))?, Some(0)); + + epoched.set(&mut s, 1, Epoch(0), 0)?; + assert_eq!(epoched.get_last_update(&s)?, Some(Epoch(0))); + assert_eq!(epoched.get_oldest_epoch(&s)?, Some(Epoch(0))); + assert_eq!(data_handler.get(&s, &Epoch(0))?, Some(1)); + + epoched.set(&mut s, 2, Epoch(1), 0)?; + assert_eq!(epoched.get_last_update(&s)?, Some(Epoch(1))); + assert_eq!(epoched.get_oldest_epoch(&s)?, Some(Epoch(0))); + assert_eq!(data_handler.get(&s, &Epoch(0))?, Some(1)); + assert_eq!(data_handler.get(&s, &Epoch(1))?, Some(2)); + + epoched.set(&mut s, 3, Epoch(2), 0)?; + assert_eq!(epoched.get_last_update(&s)?, Some(Epoch(2))); + assert_eq!(epoched.get_oldest_epoch(&s)?, Some(Epoch(0))); + assert_eq!(data_handler.get(&s, &Epoch(0))?, Some(1)); + assert_eq!(data_handler.get(&s, &Epoch(1))?, Some(2)); + assert_eq!(data_handler.get(&s, &Epoch(2))?, Some(3)); + + epoched.set(&mut s, 4, Epoch(3), 0)?; + assert_eq!(epoched.get_last_update(&s)?, Some(Epoch(3))); + assert_eq!(epoched.get_oldest_epoch(&s)?, Some(Epoch(0))); + assert_eq!(data_handler.get(&s, &Epoch(0))?, Some(1)); + assert_eq!(data_handler.get(&s, &Epoch(1))?, Some(2)); + assert_eq!(data_handler.get(&s, &Epoch(2))?, Some(3)); + assert_eq!(data_handler.get(&s, &Epoch(3))?, Some(4)); + + epoched.set(&mut s, 5, Epoch(5), 0)?; + assert_eq!(epoched.get_last_update(&s)?, Some(Epoch(5))); + assert_eq!(epoched.get_oldest_epoch(&s)?, Some(Epoch(0))); + assert_eq!(data_handler.get(&s, &Epoch(0))?, Some(1)); + assert_eq!(data_handler.get(&s, &Epoch(1))?, Some(2)); + assert_eq!(data_handler.get(&s, &Epoch(2))?, Some(3)); + assert_eq!(data_handler.get(&s, &Epoch(3))?, Some(4)); + assert_eq!(data_handler.get(&s, &Epoch(5))?, Some(5)); + + epoched.set(&mut s, 6, Epoch(10), 0)?; + assert_eq!(epoched.get_last_update(&s)?, Some(Epoch(10))); + assert_eq!(epoched.get_oldest_epoch(&s)?, Some(Epoch(0))); + assert_eq!(data_handler.get(&s, &Epoch(0))?, Some(1)); + assert_eq!(data_handler.get(&s, &Epoch(1))?, Some(2)); + assert_eq!(data_handler.get(&s, &Epoch(2))?, Some(3)); + assert_eq!(data_handler.get(&s, &Epoch(3))?, Some(4)); + assert_eq!(data_handler.get(&s, &Epoch(5))?, Some(5)); + assert_eq!(data_handler.get(&s, &Epoch(6))?, None); + assert_eq!(data_handler.get(&s, &Epoch(7))?, None); + assert_eq!(data_handler.get(&s, &Epoch(8))?, None); + assert_eq!(data_handler.get(&s, &Epoch(9))?, None); + assert_eq!(data_handler.get(&s, &Epoch(10))?, Some(6)); + + Ok(()) + } + + #[test] + fn test_epoched_delta_data_trimming() -> storage_api::Result<()> { + let mut s = TestWlStorage::default(); + + const NUM_PAST_EPOCHS: u64 = 2; + let key_prefix = storage::Key::parse("test").unwrap(); + let epoched = + EpochedDelta::::open( + key_prefix, + ); + let data_handler = epoched.get_data_handler(); + assert!(epoched.get_last_update(&s)?.is_none()); + assert!(epoched.get_oldest_epoch(&s)?.is_none()); + + epoched.init_at_genesis(&mut s, 0, Epoch(0))?; + assert_eq!(epoched.get_last_update(&s)?, Some(Epoch(0))); + assert_eq!(epoched.get_oldest_epoch(&s)?, Some(Epoch(0))); + assert_eq!(data_handler.get(&s, &Epoch(0))?, Some(0)); + + epoched.set(&mut s, 1, Epoch(0), 0)?; + assert_eq!(epoched.get_last_update(&s)?, Some(Epoch(0))); + assert_eq!(epoched.get_oldest_epoch(&s)?, Some(Epoch(0))); + assert_eq!(data_handler.get(&s, &Epoch(0))?, Some(1)); + + epoched.set(&mut s, 2, Epoch(1), 0)?; + assert_eq!(epoched.get_last_update(&s)?, Some(Epoch(1))); + assert_eq!(epoched.get_oldest_epoch(&s)?, Some(Epoch(0))); + assert_eq!(data_handler.get(&s, &Epoch(0))?, Some(1)); + assert_eq!(data_handler.get(&s, &Epoch(1))?, Some(2)); + + // Nothing is trimmed yet, oldest kept epoch is 0 + epoched.set(&mut s, 3, Epoch(2), 0)?; + assert_eq!(epoched.get_last_update(&s)?, Some(Epoch(2))); + assert_eq!(epoched.get_oldest_epoch(&s)?, Some(Epoch(0))); + assert_eq!(data_handler.get(&s, &Epoch(0))?, Some(1)); + assert_eq!(data_handler.get(&s, &Epoch(1))?, Some(2)); + assert_eq!(data_handler.get(&s, &Epoch(2))?, Some(3)); + + // Epoch 0 should be trimmed now, oldest kept epoch is 1 + epoched.set(&mut s, 4, Epoch(3), 0)?; + assert_eq!(epoched.get_last_update(&s)?, Some(Epoch(3))); + assert_eq!(epoched.get_oldest_epoch(&s)?, Some(Epoch(1))); + assert_eq!(data_handler.get(&s, &Epoch(0))?, None); + // The value from epoch 0 should be added to epoch 1 + assert_eq!(data_handler.get(&s, &Epoch(1))?, Some(3)); + assert_eq!(data_handler.get(&s, &Epoch(2))?, Some(3)); + assert_eq!(data_handler.get(&s, &Epoch(3))?, Some(4)); + + // Anything before epoch 3 should be trimmed + epoched.set(&mut s, 5, Epoch(5), 0)?; + assert_eq!(epoched.get_last_update(&s)?, Some(Epoch(5))); + assert_eq!(epoched.get_oldest_epoch(&s)?, Some(Epoch(3))); + assert_eq!(data_handler.get(&s, &Epoch(0))?, None); + assert_eq!(data_handler.get(&s, &Epoch(1))?, None); + assert_eq!(data_handler.get(&s, &Epoch(2))?, None); + // The values from epoch 1 and 2 should be added to epoch 3 + assert_eq!(data_handler.get(&s, &Epoch(3))?, Some(10)); + assert_eq!(data_handler.get(&s, &Epoch(5))?, Some(5)); + + // Anything before epoch 8 should be trimmed + epoched.set(&mut s, 6, Epoch(10), 0)?; + assert_eq!(epoched.get_last_update(&s)?, Some(Epoch(10))); + assert_eq!(epoched.get_oldest_epoch(&s)?, Some(Epoch(8))); + for epoch in Epoch(0).iter_range(7) { + assert_eq!(data_handler.get(&s, &epoch)?, None); + } + // The values from epoch 3 and 5 should be added to epoch 3 + assert_eq!(data_handler.get(&s, &Epoch(8))?, Some(15)); + assert_eq!(data_handler.get(&s, &Epoch(9))?, None); + assert_eq!(data_handler.get(&s, &Epoch(10))?, Some(6)); + + Ok(()) + } + + #[test] + fn test_epoched_delta_without_data_trimming() -> storage_api::Result<()> { + let mut s = TestWlStorage::default(); + + // Nothing should ever get trimmed + const NUM_PAST_EPOCHS: u64 = u64::MAX; + let key_prefix = storage::Key::parse("test").unwrap(); + let epoched = + EpochedDelta::::open( + key_prefix, + ); + let data_handler = epoched.get_data_handler(); + assert!(epoched.get_last_update(&s)?.is_none()); + assert!(epoched.get_oldest_epoch(&s)?.is_none()); + + epoched.init_at_genesis(&mut s, 0, Epoch(0))?; + assert_eq!(epoched.get_last_update(&s)?, Some(Epoch(0))); + assert_eq!(epoched.get_oldest_epoch(&s)?, Some(Epoch(0))); + assert_eq!(data_handler.get(&s, &Epoch(0))?, Some(0)); + + epoched.set(&mut s, 1, Epoch(0), 0)?; + assert_eq!(epoched.get_last_update(&s)?, Some(Epoch(0))); + assert_eq!(epoched.get_oldest_epoch(&s)?, Some(Epoch(0))); + assert_eq!(data_handler.get(&s, &Epoch(0))?, Some(1)); + + epoched.set(&mut s, 2, Epoch(1), 0)?; + assert_eq!(epoched.get_last_update(&s)?, Some(Epoch(1))); + assert_eq!(epoched.get_oldest_epoch(&s)?, Some(Epoch(0))); + assert_eq!(data_handler.get(&s, &Epoch(0))?, Some(1)); + assert_eq!(data_handler.get(&s, &Epoch(1))?, Some(2)); + + epoched.set(&mut s, 3, Epoch(2), 0)?; + assert_eq!(epoched.get_last_update(&s)?, Some(Epoch(2))); + assert_eq!(epoched.get_oldest_epoch(&s)?, Some(Epoch(0))); + assert_eq!(data_handler.get(&s, &Epoch(0))?, Some(1)); + assert_eq!(data_handler.get(&s, &Epoch(1))?, Some(2)); + assert_eq!(data_handler.get(&s, &Epoch(2))?, Some(3)); + + epoched.set(&mut s, 4, Epoch(3), 0)?; + assert_eq!(epoched.get_last_update(&s)?, Some(Epoch(3))); + assert_eq!(epoched.get_oldest_epoch(&s)?, Some(Epoch(0))); + assert_eq!(data_handler.get(&s, &Epoch(0))?, Some(1)); + assert_eq!(data_handler.get(&s, &Epoch(1))?, Some(2)); + assert_eq!(data_handler.get(&s, &Epoch(2))?, Some(3)); + assert_eq!(data_handler.get(&s, &Epoch(3))?, Some(4)); + + epoched.set(&mut s, 5, Epoch(5), 0)?; + assert_eq!(epoched.get_last_update(&s)?, Some(Epoch(5))); + assert_eq!(epoched.get_oldest_epoch(&s)?, Some(Epoch(0))); + assert_eq!(data_handler.get(&s, &Epoch(0))?, Some(1)); + assert_eq!(data_handler.get(&s, &Epoch(1))?, Some(2)); + assert_eq!(data_handler.get(&s, &Epoch(2))?, Some(3)); + assert_eq!(data_handler.get(&s, &Epoch(3))?, Some(4)); + assert_eq!(data_handler.get(&s, &Epoch(5))?, Some(5)); + + epoched.set(&mut s, 6, Epoch(10), 0)?; + assert_eq!(epoched.get_last_update(&s)?, Some(Epoch(10))); + assert_eq!(epoched.get_oldest_epoch(&s)?, Some(Epoch(0))); + assert_eq!(data_handler.get(&s, &Epoch(0))?, Some(1)); + assert_eq!(data_handler.get(&s, &Epoch(1))?, Some(2)); + assert_eq!(data_handler.get(&s, &Epoch(2))?, Some(3)); + assert_eq!(data_handler.get(&s, &Epoch(3))?, Some(4)); + assert_eq!(data_handler.get(&s, &Epoch(5))?, Some(5)); + assert_eq!(data_handler.get(&s, &Epoch(6))?, None); + assert_eq!(data_handler.get(&s, &Epoch(7))?, None); + assert_eq!(data_handler.get(&s, &Epoch(8))?, None); + assert_eq!(data_handler.get(&s, &Epoch(9))?, None); + assert_eq!(data_handler.get(&s, &Epoch(10))?, Some(6)); + + Ok(()) + } + + // use namada_core::ledger::storage::testing::TestStorage; + // use namada_core::types::address::{self, Address}; + // use namada_core::types::storage::Key; + // + // use super::{ + // storage, storage_api, Epoch, LazyMap, NestedEpoched, NestedMap, + // OffsetPipelineLen, + // }; + // + // #[test] + // fn testing_epoched_new() -> storage_api::Result<()> { + // let mut storage = TestStorage::default(); + // + // let key1 = storage::Key::parse("test_nested1").unwrap(); + // let nested1 = + // NestedEpoched::, OffsetPipelineLen>::open( + // key1, + // ); + // nested1.init(&mut storage, Epoch(0))?; + // + // let key2 = storage::Key::parse("test_nested2").unwrap(); + // let nested2 = NestedEpoched::< + // NestedMap>, + // OffsetPipelineLen, + // >::open(key2); + // nested2.init(&mut storage, Epoch(0))?; + // + // dbg!(&nested1.get_last_update_storage_key()); + // dbg!(&nested1.get_last_update(&storage)); + // + // nested1.at(&Epoch(0)).insert( + // &mut storage, + // address::testing::established_address_1(), + // 1432, + // )?; + // dbg!(&nested1.at(&Epoch(0)).iter(&mut storage)?.next()); + // dbg!(&nested1.at(&Epoch(1)).iter(&mut storage)?.next()); + // + // nested2.at(&Epoch(0)).at(&100).insert( + // &mut storage, + // 1, + // address::testing::established_address_2(), + // )?; + // dbg!(&nested2.at(&Epoch(0)).iter(&mut storage)?.next()); + // dbg!(&nested2.at(&Epoch(1)).iter(&mut storage)?.next()); + // + // dbg!(&nested_epoched.get_epoch_key(&Epoch::from(0))); + // + // let epoch = Epoch::from(0); + // let addr = address::testing::established_address_1(); + // let amount: u64 = 234235; + // + // nested_epoched + // .at(&epoch) + // .insert(&mut storage, addr.clone(), amount)?; + // + // let epoch = epoch + 3_u64; + // nested_epoched.at(&epoch).insert( + // &mut storage, + // addr.clone(), + // 999_u64, + // )?; + // + // dbg!(nested_epoched.contains_epoch(&storage, &Epoch::from(0))?); + // dbg!( + // nested_epoched + // .get_data_handler() + // .get_data_key(&Epoch::from(3)) + // ); + // dbg!(nested_epoched.contains_epoch(&storage, &Epoch::from(3))?); + // dbg!( + // nested_epoched + // .at(&Epoch::from(0)) + // .get(&storage, &addr.clone())? + // ); + // dbg!( + // nested_epoched + // .at(&Epoch::from(3)) + // .get(&storage, &addr.clone())? + // ); + // dbg!(nested_epoched.at(&Epoch::from(3)).get_data_key(&addr)); + // + // Ok(()) + // } +} From 9f996f14622d31ae3e076c192d2614aaccf858da Mon Sep 17 00:00:00 2001 From: brentstone Date: Mon, 24 Apr 2023 18:33:06 -0400 Subject: [PATCH 2/4] pos/epoched: fix update_data methods Avoid trimming data when there are no epochs to clear and write last update epoch even if no trimming occurs. --- proof_of_stake/src/epoched.rs | 127 ++++++++++++++++++++++++++++------ 1 file changed, 104 insertions(+), 23 deletions(-) diff --git a/proof_of_stake/src/epoched.rs b/proof_of_stake/src/epoched.rs index 2d5e12c155..4899ae1e1d 100644 --- a/proof_of_stake/src/epoched.rs +++ b/proof_of_stake/src/epoched.rs @@ -21,6 +21,8 @@ use crate::parameters::PosParams; pub const LAZY_MAP_SUB_KEY: &str = "lazy_map"; /// Sub-key for an epoched data structure's last (most recent) epoch of update pub const LAST_UPDATE_SUB_KEY: &str = "last_update"; +/// Sub-key for an epoched data structure's oldest epoch with some data +pub const OLDEST_EPOCH_SUB_KEY: &str = "oldest_epoch"; /// Discrete epoched data handle pub struct Epoched< @@ -86,7 +88,7 @@ where { let key = self.get_last_update_storage_key(); storage.write(&key, current_epoch)?; - + self.set_oldest_epoch(storage, current_epoch)?; self.set_at_epoch(storage, value, current_epoch, 0) } @@ -175,19 +177,22 @@ where S: StorageWrite + StorageRead, { let last_update = self.get_last_update(storage)?; - if let Some(last_update) = last_update { - if last_update < current_epoch { + let oldest_epoch = self.get_oldest_epoch(storage)?; + if let (Some(last_update), Some(oldest_epoch)) = + (last_update, oldest_epoch) + { + let oldest_to_keep = current_epoch + .0 + .checked_sub(NUM_PAST_EPOCHS) + .unwrap_or_default(); + if oldest_epoch.0 < oldest_to_keep { + let diff = oldest_to_keep - oldest_epoch.0; // Go through the epochs before the expected oldest epoch and // keep the latest one tracing::debug!( "Trimming data for epoched data in epoch {current_epoch}, \ last updated at {last_update}." ); - let diff = current_epoch - .checked_sub(last_update) - .unwrap_or_default() - .0; - let oldest_epoch = Self::sub_past_epochs(last_update); let data_handler = self.get_data_handler(); let mut latest_value: Option = None; // Remove data before the new oldest epoch, keep the latest @@ -213,15 +218,22 @@ where latest_value, )?; } + self.set_oldest_epoch(storage, new_oldest_epoch)?; } // Update the epoch of the last update to the current epoch let key = self.get_last_update_storage_key(); storage.write(&key, current_epoch)?; + return Ok(()); } - } else { - // Set the epoch of the last update to the current epoch - let key = self.get_last_update_storage_key(); - storage.write(&key, current_epoch)?; + } + + // Set the epoch of the last update to the current epoch + let key = self.get_last_update_storage_key(); + storage.write(&key, current_epoch)?; + + // If there's no oldest epoch written yet, set it to the current one + if oldest_epoch.is_none() { + self.set_oldest_epoch(storage, current_epoch)?; } Ok(()) } @@ -254,6 +266,35 @@ where fn sub_past_epochs(epoch: Epoch) -> Epoch { Epoch(epoch.0.checked_sub(NUM_PAST_EPOCHS).unwrap_or_default()) } + + fn get_oldest_epoch_storage_key(&self) -> storage::Key { + self.storage_prefix + .push(&OLDEST_EPOCH_SUB_KEY.to_owned()) + .unwrap() + } + + fn get_oldest_epoch( + &self, + storage: &S, + ) -> storage_api::Result> + where + S: StorageRead, + { + let key = self.get_oldest_epoch_storage_key(); + storage.read(&key) + } + + fn set_oldest_epoch( + &self, + storage: &mut S, + new_oldest_epoch: Epoch, + ) -> storage_api::Result<()> + where + S: StorageRead + StorageWrite, + { + let key = self.get_oldest_epoch_storage_key(); + storage.write(&key, new_oldest_epoch) + } } impl @@ -378,6 +419,7 @@ where { let key = self.get_last_update_storage_key(); storage.write(&key, current_epoch)?; + self.set_oldest_epoch(storage, current_epoch)?; self.set_at_epoch(storage, value, current_epoch, 0) } @@ -477,19 +519,22 @@ where S: StorageWrite + StorageRead, { let last_update = self.get_last_update(storage)?; - if let Some(last_update) = last_update { - if last_update < current_epoch { + let oldest_epoch = self.get_oldest_epoch(storage)?; + if let (Some(last_update), Some(oldest_epoch)) = + (last_update, oldest_epoch) + { + let oldest_to_keep = current_epoch + .0 + .checked_sub(NUM_PAST_EPOCHS) + .unwrap_or_default(); + if oldest_epoch.0 < oldest_to_keep { + let diff = oldest_to_keep - oldest_epoch.0; // Go through the epochs before the expected oldest epoch and // sum them into it tracing::debug!( "Trimming data for epoched delta data in epoch \ {current_epoch}, last updated at {last_update}." ); - let diff = current_epoch - .checked_sub(last_update) - .unwrap_or_default() - .0; - let oldest_epoch = Self::sub_past_epochs(last_update); let data_handler = self.get_data_handler(); // Find the sum of values before the new oldest epoch to be kept let mut sum: Option = None; @@ -521,15 +566,22 @@ where new_oldest_epoch, new_oldest_epoch_data, )?; + self.set_oldest_epoch(storage, new_oldest_epoch)?; } // Update the epoch of the last update to the current epoch let key = self.get_last_update_storage_key(); storage.write(&key, current_epoch)?; + return Ok(()); } - } else { - // Set the epoch of the last update to the current epoch - let key = self.get_last_update_storage_key(); - storage.write(&key, current_epoch)?; + } + + // Set the epoch of the last update to the current epoch + let key = self.get_last_update_storage_key(); + storage.write(&key, current_epoch)?; + + // If there's no oldest epoch written yet, set it to the current one + if oldest_epoch.is_none() { + self.set_oldest_epoch(storage, current_epoch)?; } Ok(()) } @@ -576,6 +628,35 @@ where fn sub_past_epochs(epoch: Epoch) -> Epoch { Epoch(epoch.0.checked_sub(NUM_PAST_EPOCHS).unwrap_or_default()) } + + fn get_oldest_epoch_storage_key(&self) -> storage::Key { + self.storage_prefix + .push(&OLDEST_EPOCH_SUB_KEY.to_owned()) + .unwrap() + } + + fn get_oldest_epoch( + &self, + storage: &S, + ) -> storage_api::Result> + where + S: StorageRead, + { + let key = self.get_oldest_epoch_storage_key(); + storage.read(&key) + } + + fn set_oldest_epoch( + &self, + storage: &mut S, + new_oldest_epoch: Epoch, + ) -> storage_api::Result<()> + where + S: StorageRead + StorageWrite, + { + let key = self.get_oldest_epoch_storage_key(); + storage.write(&key, new_oldest_epoch) + } } /// Offset at pipeline length. From 9de3b91be96fef02112de5383f59e3b0c2eaccd8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 4 May 2023 17:03:50 +0000 Subject: [PATCH 3/4] [ci] wasm checksums update --- wasm/checksums.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/wasm/checksums.json b/wasm/checksums.json index 8fcd7264e0..6bc139d8c3 100644 --- a/wasm/checksums.json +++ b/wasm/checksums.json @@ -1,13 +1,13 @@ { - "tx_bond.wasm": "tx_bond.897fbd5ca027d93fc03edac3310297f8fc5baa5241082f60484dc589e7372b47.wasm", - "tx_change_validator_commission.wasm": "tx_change_validator_commission.58c0eaf6593073d5d5a3e0f0ca287090b21c9b7b49d966006849e9805b05e5d7.wasm", + "tx_bond.wasm": "tx_bond.2c765016a1cc07a5577bd850c55f4d44015d0f2a1eddd3f7699f65915fc82025.wasm", + "tx_change_validator_commission.wasm": "tx_change_validator_commission.204fc1fee7223912e7cfbf077963d2ae967506a111c8e02f593bf5178c4c882c.wasm", "tx_ibc.wasm": "tx_ibc.4ac689d6bb4b8153e9c1e1aba1d6e985e0419ee2a62bbef2b24a34f3f0f90e8b.wasm", "tx_init_account.wasm": "tx_init_account.27a75bd5972baa54d69be89e546b60b98d6f36edff60abd647504b11a77d909f.wasm", "tx_init_proposal.wasm": "tx_init_proposal.d03ce22f9a6a8a9e9b6c531cedc05c4b649bcbe8d05e1da3a759e482f8abcf9d.wasm", - "tx_init_validator.wasm": "tx_init_validator.6f2c8aaf9462e6e3f8274631a702289ebff130f30040cb3dbdbf94aff424e76d.wasm", + "tx_init_validator.wasm": "tx_init_validator.298f37127a2d70e1b38ac11e9176f65291a6857c9ba4d81c164d563feb47e376.wasm", "tx_reveal_pk.wasm": "tx_reveal_pk.1c12867bbc2111395ca39a043ac7478cb59d09648d3c41c4d0489d5d6a8d9390.wasm", "tx_transfer.wasm": "tx_transfer.4c2168dc26839a770f238f562f249ae1f0a7c1bce1ec9c4293fee21068a825e8.wasm", - "tx_unbond.wasm": "tx_unbond.c546abbd03a7b8736b261f8b69fc094625df16f7443d23068f230d21376b848d.wasm", + "tx_unbond.wasm": "tx_unbond.b9e688588cf89a16959448f8cfe80d90d2570292bd23cdb0d53a72258ee742d3.wasm", "tx_update_vp.wasm": "tx_update_vp.cb5eb6fb079ceb44c39262476e0072b5952b71516e48911bc13d13b1c757d5c3.wasm", "tx_vote_proposal.wasm": "tx_vote_proposal.f186ae7f1b7ec145c96b12dfbe9f3918324abb0e2509729a99d2c9709f9eea44.wasm", "tx_withdraw.wasm": "tx_withdraw.c0ac8902f4d42f459b96caa5d0c9d0f62ac0caa1f19ec25c335ba45aa38a86b5.wasm", From b12eb34b1edbb97d0b166b46aafc28af54827bd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Zemanovi=C4=8D?= Date: Thu, 4 May 2023 19:19:25 +0200 Subject: [PATCH 4/4] changelog: #1325 --- .changelog/unreleased/bug-fixes/1325-fix-update-data-epoched.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .changelog/unreleased/bug-fixes/1325-fix-update-data-epoched.md diff --git a/.changelog/unreleased/bug-fixes/1325-fix-update-data-epoched.md b/.changelog/unreleased/bug-fixes/1325-fix-update-data-epoched.md new file mode 100644 index 0000000000..b36100c9eb --- /dev/null +++ b/.changelog/unreleased/bug-fixes/1325-fix-update-data-epoched.md @@ -0,0 +1,2 @@ +- PoS: fixed a function for clearing of historical epoched data + ([\#1325](https://github.com/anoma/namada/pull/1325)) \ No newline at end of file