From 7aec0b40fec7d9693021756bb58d2133ddb1dce4 Mon Sep 17 00:00:00 2001 From: Brooks Date: Fri, 27 Sep 2024 15:19:38 -0400 Subject: [PATCH 1/3] Supports deserializing accounts lt hash in snapshots (#2994) (cherry picked from commit 690fad08d47bb6aff66dce3ce1c5ec8d92950b63) # Conflicts: # accounts-db/src/accounts_hash.rs # runtime/src/serde_snapshot.rs --- Cargo.lock | 2 ++ accounts-db/src/accounts_hash.rs | 19 ++++++++++ programs/sbf/Cargo.lock | 2 ++ runtime/Cargo.toml | 2 ++ runtime/src/serde_snapshot.rs | 55 +++++++++++++++++++++++++++++ runtime/src/serde_snapshot/types.rs | 22 ++++++++++++ 6 files changed, 102 insertions(+) create mode 100644 runtime/src/serde_snapshot/types.rs diff --git a/Cargo.lock b/Cargo.lock index c513895564faf6..dbca3dabb4d71f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7187,6 +7187,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", + "serde_with", "solana-accounts-db", "solana-address-lookup-table-program", "solana-bpf-loader-program", @@ -7198,6 +7199,7 @@ dependencies = [ "solana-frozen-abi", "solana-frozen-abi-macro", "solana-inline-spl", + "solana-lattice-hash", "solana-loader-v4-program", "solana-logger", "solana-measure", diff --git a/accounts-db/src/accounts_hash.rs b/accounts-db/src/accounts_hash.rs index 7c7779f44581e5..4e79128dcf2df1 100644 --- a/accounts-db/src/accounts_hash.rs +++ b/accounts-db/src/accounts_hash.rs @@ -1236,6 +1236,25 @@ pub struct AccountHash(pub Hash); // This also ensures there are no padding bytes, which is required to safely implement Pod const _: () = assert!(std::mem::size_of::() == std::mem::size_of::()); +<<<<<<< HEAD +======= +/// The AccountHash for a zero-lamport account +pub const ZERO_LAMPORT_ACCOUNT_HASH: AccountHash = + AccountHash(Hash::new_from_array([0; HASH_BYTES])); + +/// Lattice hash of an account +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct AccountLtHash(pub LtHash); + +/// The AccountLtHash for a zero-lamport account +pub const ZERO_LAMPORT_ACCOUNT_LT_HASH: AccountLtHash = + AccountLtHash(LtHash([0; LtHash::NUM_ELEMENTS])); + +/// Lattice hash of all accounts +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct AccountsLtHash(pub LtHash); + +>>>>>>> 690fad08d4 (Supports deserializing accounts lt hash in snapshots (#2994)) /// Hash of accounts #[derive(Debug, Copy, Clone, Eq, PartialEq)] pub enum AccountsHashKind { diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index 5fbe614ecf6ec7..2741403e412a40 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -5604,6 +5604,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", + "serde_with", "solana-accounts-db", "solana-address-lookup-table-program", "solana-bpf-loader-program", @@ -5613,6 +5614,7 @@ dependencies = [ "solana-config-program", "solana-cost-model", "solana-inline-spl", + "solana-lattice-hash", "solana-loader-v4-program", "solana-measure", "solana-metrics", diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index da551f4ac4686d..6348bbec93838b 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -46,6 +46,7 @@ regex = { workspace = true } serde = { workspace = true, features = ["rc"] } serde_derive = { workspace = true } serde_json = { workspace = true } +serde_with = { workspace = true } solana-accounts-db = { workspace = true } solana-address-lookup-table-program = { workspace = true } solana-bpf-loader-program = { workspace = true } @@ -57,6 +58,7 @@ solana-cost-model = { workspace = true } solana-frozen-abi = { workspace = true, optional = true } solana-frozen-abi-macro = { workspace = true, optional = true } solana-inline-spl = { workspace = true } +solana-lattice-hash = { workspace = true } solana-loader-v4-program = { workspace = true } solana-measure = { workspace = true } solana-metrics = { workspace = true } diff --git a/runtime/src/serde_snapshot.rs b/runtime/src/serde_snapshot.rs index 28964394c21785..5d5df2a553dbe6 100644 --- a/runtime/src/serde_snapshot.rs +++ b/runtime/src/serde_snapshot.rs @@ -57,10 +57,12 @@ use { thread::Builder, }, storage::SerializableStorage, + types::SerdeAccountsLtHash, }; mod storage; mod tests; +mod types; mod utils; pub(crate) use { @@ -382,6 +384,48 @@ where deserialize_from::<_, _>(stream) } +<<<<<<< HEAD +======= +/// Extra fields that are deserialized from the end of snapshots. +/// +/// Note that this struct's fields should stay synced with the fields in +/// ExtraFieldsToSerialize with the exception that new "extra fields" should be +/// added to this struct a minor release before they are added to the serialize +/// struct. +#[cfg_attr(feature = "frozen-abi", derive(AbiExample))] +#[cfg_attr(feature = "dev-context-only-utils", derive(PartialEq))] +#[derive(Clone, Debug, Deserialize)] +struct ExtraFieldsToDeserialize { + #[serde(deserialize_with = "default_on_eof")] + lamports_per_signature: u64, + #[serde(deserialize_with = "default_on_eof")] + incremental_snapshot_persistence: Option, + #[serde(deserialize_with = "default_on_eof")] + epoch_accounts_hash: Option, + #[serde(deserialize_with = "default_on_eof")] + versioned_epoch_stakes: HashMap, + #[serde(deserialize_with = "default_on_eof")] + #[allow(dead_code)] + accounts_lt_hash: Option, +} + +/// Extra fields that are serialized at the end of snapshots. +/// +/// Note that this struct's fields should stay synced with the fields in +/// ExtraFieldsToDeserialize with the exception that new "extra fields" should +/// be added to the deserialize struct a minor release before they are added to +/// this one. +#[cfg_attr(feature = "frozen-abi", derive(AbiExample))] +#[cfg_attr(feature = "dev-context-only-utils", derive(Default, PartialEq))] +#[derive(Debug, Serialize)] +pub struct ExtraFieldsToSerialize<'a> { + pub lamports_per_signature: u64, + pub incremental_snapshot_persistence: Option<&'a BankIncrementalSnapshotPersistence>, + pub epoch_accounts_hash: Option, + pub versioned_epoch_stakes: HashMap, +} + +>>>>>>> 690fad08d4 (Supports deserializing accounts lt hash in snapshots (#2994)) fn deserialize_bank_fields( mut stream: &mut BufReader, ) -> Result< @@ -398,7 +442,18 @@ where deserialize_from::<_, DeserializableVersionedBank>(&mut stream)?.into(); let accounts_db_fields = deserialize_accounts_db_fields(stream)?; // Process extra fields +<<<<<<< HEAD let lamports_per_signature = ignore_eof_error(deserialize_from(&mut stream))?; +======= + let ExtraFieldsToDeserialize { + lamports_per_signature, + incremental_snapshot_persistence, + epoch_accounts_hash, + versioned_epoch_stakes, + accounts_lt_hash: _, + } = extra_fields; + +>>>>>>> 690fad08d4 (Supports deserializing accounts lt hash in snapshots (#2994)) bank_fields.fee_rate_governor = bank_fields .fee_rate_governor .clone_with_lamports_per_signature(lamports_per_signature); diff --git a/runtime/src/serde_snapshot/types.rs b/runtime/src/serde_snapshot/types.rs new file mode 100644 index 00000000000000..6dd9ef099cabaf --- /dev/null +++ b/runtime/src/serde_snapshot/types.rs @@ -0,0 +1,22 @@ +use {solana_accounts_db::accounts_hash::AccountsLtHash, solana_lattice_hash::lt_hash::LtHash}; + +/// Snapshot serde-safe AccountsLtHash +#[cfg_attr(feature = "frozen-abi", derive(AbiExample))] +#[serde_with::serde_as] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub struct SerdeAccountsLtHash( + // serde only has array support up to 32 elements; anything larger needs to be handled manually + // see https://github.com/serde-rs/serde/issues/1937 for more information + #[serde_as(as = "[_; LtHash::NUM_ELEMENTS]")] pub [u16; LtHash::NUM_ELEMENTS], +); + +impl From for AccountsLtHash { + fn from(accounts_lt_hash: SerdeAccountsLtHash) -> Self { + Self(LtHash(accounts_lt_hash.0)) + } +} +impl From for SerdeAccountsLtHash { + fn from(accounts_lt_hash: AccountsLtHash) -> Self { + Self(accounts_lt_hash.0 .0) + } +} From a1975d83aeaf0608599b2f5c4b05553ca995494f Mon Sep 17 00:00:00 2001 From: brooks Date: Fri, 27 Sep 2024 16:54:33 -0400 Subject: [PATCH 2/3] pr: fix merge conflicts --- Cargo.lock | 1 - accounts-db/src/accounts_hash.rs | 19 ---------- programs/sbf/Cargo.lock | 1 - runtime/Cargo.toml | 1 - runtime/src/serde_snapshot.rs | 57 ++--------------------------- runtime/src/serde_snapshot/types.rs | 15 +------- 6 files changed, 5 insertions(+), 89 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dbca3dabb4d71f..c975ffbbafa55e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7199,7 +7199,6 @@ dependencies = [ "solana-frozen-abi", "solana-frozen-abi-macro", "solana-inline-spl", - "solana-lattice-hash", "solana-loader-v4-program", "solana-logger", "solana-measure", diff --git a/accounts-db/src/accounts_hash.rs b/accounts-db/src/accounts_hash.rs index 4e79128dcf2df1..7c7779f44581e5 100644 --- a/accounts-db/src/accounts_hash.rs +++ b/accounts-db/src/accounts_hash.rs @@ -1236,25 +1236,6 @@ pub struct AccountHash(pub Hash); // This also ensures there are no padding bytes, which is required to safely implement Pod const _: () = assert!(std::mem::size_of::() == std::mem::size_of::()); -<<<<<<< HEAD -======= -/// The AccountHash for a zero-lamport account -pub const ZERO_LAMPORT_ACCOUNT_HASH: AccountHash = - AccountHash(Hash::new_from_array([0; HASH_BYTES])); - -/// Lattice hash of an account -#[derive(Debug, Clone, Eq, PartialEq)] -pub struct AccountLtHash(pub LtHash); - -/// The AccountLtHash for a zero-lamport account -pub const ZERO_LAMPORT_ACCOUNT_LT_HASH: AccountLtHash = - AccountLtHash(LtHash([0; LtHash::NUM_ELEMENTS])); - -/// Lattice hash of all accounts -#[derive(Debug, Clone, Eq, PartialEq)] -pub struct AccountsLtHash(pub LtHash); - ->>>>>>> 690fad08d4 (Supports deserializing accounts lt hash in snapshots (#2994)) /// Hash of accounts #[derive(Debug, Copy, Clone, Eq, PartialEq)] pub enum AccountsHashKind { diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index 2741403e412a40..5cb6231d88e960 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -5614,7 +5614,6 @@ dependencies = [ "solana-config-program", "solana-cost-model", "solana-inline-spl", - "solana-lattice-hash", "solana-loader-v4-program", "solana-measure", "solana-metrics", diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 6348bbec93838b..691ddedf625dbd 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -58,7 +58,6 @@ solana-cost-model = { workspace = true } solana-frozen-abi = { workspace = true, optional = true } solana-frozen-abi-macro = { workspace = true, optional = true } solana-inline-spl = { workspace = true } -solana-lattice-hash = { workspace = true } solana-loader-v4-program = { workspace = true } solana-measure = { workspace = true } solana-metrics = { workspace = true } diff --git a/runtime/src/serde_snapshot.rs b/runtime/src/serde_snapshot.rs index 5d5df2a553dbe6..ffc35fca4bac74 100644 --- a/runtime/src/serde_snapshot.rs +++ b/runtime/src/serde_snapshot.rs @@ -384,48 +384,6 @@ where deserialize_from::<_, _>(stream) } -<<<<<<< HEAD -======= -/// Extra fields that are deserialized from the end of snapshots. -/// -/// Note that this struct's fields should stay synced with the fields in -/// ExtraFieldsToSerialize with the exception that new "extra fields" should be -/// added to this struct a minor release before they are added to the serialize -/// struct. -#[cfg_attr(feature = "frozen-abi", derive(AbiExample))] -#[cfg_attr(feature = "dev-context-only-utils", derive(PartialEq))] -#[derive(Clone, Debug, Deserialize)] -struct ExtraFieldsToDeserialize { - #[serde(deserialize_with = "default_on_eof")] - lamports_per_signature: u64, - #[serde(deserialize_with = "default_on_eof")] - incremental_snapshot_persistence: Option, - #[serde(deserialize_with = "default_on_eof")] - epoch_accounts_hash: Option, - #[serde(deserialize_with = "default_on_eof")] - versioned_epoch_stakes: HashMap, - #[serde(deserialize_with = "default_on_eof")] - #[allow(dead_code)] - accounts_lt_hash: Option, -} - -/// Extra fields that are serialized at the end of snapshots. -/// -/// Note that this struct's fields should stay synced with the fields in -/// ExtraFieldsToDeserialize with the exception that new "extra fields" should -/// be added to the deserialize struct a minor release before they are added to -/// this one. -#[cfg_attr(feature = "frozen-abi", derive(AbiExample))] -#[cfg_attr(feature = "dev-context-only-utils", derive(Default, PartialEq))] -#[derive(Debug, Serialize)] -pub struct ExtraFieldsToSerialize<'a> { - pub lamports_per_signature: u64, - pub incremental_snapshot_persistence: Option<&'a BankIncrementalSnapshotPersistence>, - pub epoch_accounts_hash: Option, - pub versioned_epoch_stakes: HashMap, -} - ->>>>>>> 690fad08d4 (Supports deserializing accounts lt hash in snapshots (#2994)) fn deserialize_bank_fields( mut stream: &mut BufReader, ) -> Result< @@ -442,18 +400,7 @@ where deserialize_from::<_, DeserializableVersionedBank>(&mut stream)?.into(); let accounts_db_fields = deserialize_accounts_db_fields(stream)?; // Process extra fields -<<<<<<< HEAD let lamports_per_signature = ignore_eof_error(deserialize_from(&mut stream))?; -======= - let ExtraFieldsToDeserialize { - lamports_per_signature, - incremental_snapshot_persistence, - epoch_accounts_hash, - versioned_epoch_stakes, - accounts_lt_hash: _, - } = extra_fields; - ->>>>>>> 690fad08d4 (Supports deserializing accounts lt hash in snapshots (#2994)) bank_fields.fee_rate_governor = bank_fields .fee_rate_governor .clone_with_lamports_per_signature(lamports_per_signature); @@ -474,6 +421,10 @@ where .map(|(epoch, versioned_epoch_stakes)| (epoch, versioned_epoch_stakes.into())), ); + // This field is only deserialized (and ignored) in this version. + let _accounts_lt_hash: Option = + ignore_eof_error(deserialize_from(&mut stream))?; + Ok((bank_fields, accounts_db_fields)) } diff --git a/runtime/src/serde_snapshot/types.rs b/runtime/src/serde_snapshot/types.rs index 6dd9ef099cabaf..40e6f10e2a83d0 100644 --- a/runtime/src/serde_snapshot/types.rs +++ b/runtime/src/serde_snapshot/types.rs @@ -1,5 +1,3 @@ -use {solana_accounts_db::accounts_hash::AccountsLtHash, solana_lattice_hash::lt_hash::LtHash}; - /// Snapshot serde-safe AccountsLtHash #[cfg_attr(feature = "frozen-abi", derive(AbiExample))] #[serde_with::serde_as] @@ -7,16 +5,5 @@ use {solana_accounts_db::accounts_hash::AccountsLtHash, solana_lattice_hash::lt_ pub struct SerdeAccountsLtHash( // serde only has array support up to 32 elements; anything larger needs to be handled manually // see https://github.com/serde-rs/serde/issues/1937 for more information - #[serde_as(as = "[_; LtHash::NUM_ELEMENTS]")] pub [u16; LtHash::NUM_ELEMENTS], + #[serde_as(as = "[_; 1024]")] pub [u16; 1024], ); - -impl From for AccountsLtHash { - fn from(accounts_lt_hash: SerdeAccountsLtHash) -> Self { - Self(LtHash(accounts_lt_hash.0)) - } -} -impl From for SerdeAccountsLtHash { - fn from(accounts_lt_hash: AccountsLtHash) -> Self { - Self(accounts_lt_hash.0 .0) - } -} From 19106694ac19ac27399b0be6c24d5ee56cf8f98f Mon Sep 17 00:00:00 2001 From: brooks Date: Mon, 30 Sep 2024 19:26:36 -0400 Subject: [PATCH 3/3] pr: do not use serde_with --- Cargo.lock | 1 - programs/sbf/Cargo.lock | 1 - runtime/Cargo.toml | 1 - runtime/src/serde_snapshot/types.rs | 45 +++++++++++++++++++++++++++-- 4 files changed, 42 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c975ffbbafa55e..c513895564faf6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7187,7 +7187,6 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "serde_with", "solana-accounts-db", "solana-address-lookup-table-program", "solana-bpf-loader-program", diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index 5cb6231d88e960..5fbe614ecf6ec7 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -5604,7 +5604,6 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "serde_with", "solana-accounts-db", "solana-address-lookup-table-program", "solana-bpf-loader-program", diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 691ddedf625dbd..da551f4ac4686d 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -46,7 +46,6 @@ regex = { workspace = true } serde = { workspace = true, features = ["rc"] } serde_derive = { workspace = true } serde_json = { workspace = true } -serde_with = { workspace = true } solana-accounts-db = { workspace = true } solana-address-lookup-table-program = { workspace = true } solana-bpf-loader-program = { workspace = true } diff --git a/runtime/src/serde_snapshot/types.rs b/runtime/src/serde_snapshot/types.rs index 40e6f10e2a83d0..f80416eb4e72e2 100644 --- a/runtime/src/serde_snapshot/types.rs +++ b/runtime/src/serde_snapshot/types.rs @@ -1,9 +1,48 @@ +use { + serde::{de, Deserialize, Deserializer}, + std::{fmt, marker::PhantomData}, +}; + /// Snapshot serde-safe AccountsLtHash #[cfg_attr(feature = "frozen-abi", derive(AbiExample))] -#[serde_with::serde_as] -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[derive(Debug)] pub struct SerdeAccountsLtHash( // serde only has array support up to 32 elements; anything larger needs to be handled manually // see https://github.com/serde-rs/serde/issues/1937 for more information - #[serde_as(as = "[_; 1024]")] pub [u16; 1024], + #[allow(dead_code)] pub [u16; 1024], ); + +impl<'de> Deserialize<'de> for SerdeAccountsLtHash { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct ArrayVisitor { + element: PhantomData, + } + impl<'de> de::Visitor<'de> for ArrayVisitor { + type Value = [u16; 1024]; + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str(concat!("a u16 array of length 1024")) + } + fn visit_seq(self, mut seq: A) -> Result + where + A: de::SeqAccess<'de>, + { + let mut arr = [0u16; 1024]; + for (i, elem) in arr.iter_mut().enumerate() { + *elem = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(i, &self))?; + } + Ok(arr) + } + } + + let visitor = ArrayVisitor { + element: PhantomData, + }; + let array = deserializer.deserialize_tuple(1024, visitor)?; + Ok(Self(array)) + } +}