Introduce AccountsDataStorage

ledger-tool/src/main.rs

@@ -47,6 +47,7 @@ use {
         transaction_status_service::TransactionStatusService,
     },
     solana_runtime::{
+        account_storage::meta::StoredAccountMeta,
         accounts::Accounts,
         accounts_background_service::{
             AbsRequestHandlers, AbsRequestSender, AccountsBackgroundService,
@@ -57,6 +58,7 @@ use {
         },
         accounts_index::{AccountsIndexConfig, IndexLimitMb, ScanConfig},
         accounts_update_notifier_interface::AccountsUpdateNotifier,
+        append_vec::AppendVec,
         bank::{Bank, RewardCalculationEvent, TotalAccountsStats},
         bank_forks::BankForks,
         cost_model::CostModel,
@@ -71,6 +73,7 @@ use {
             self, create_accounts_run_and_snapshot_dirs, move_and_async_delete_path, ArchiveFormat,
             SnapshotVersion, DEFAULT_ARCHIVE_COMPRESSION, SUPPORTED_ARCHIVE_COMPRESSION,
         },
+        tiered_storage::TieredStorage,
     },
     solana_sdk::{
         account::{AccountSharedData, ReadableAccount, WritableAccount},
@@ -2372,6 +2375,24 @@ fn main() {
                            If no file name is specified, it will print the metadata of all ledger files.")
             )
         )
+        .subcommand(
+            SubCommand::with_name("new_ads_file")
+            .about("Create a new accounts-data-storage file from an existing append_vec file.")
+            .arg(
+                Arg::with_name("append_vec")
+                    .long("append-vec")
+                    .takes_value(true)
+                    .value_name("APPEND_VEC_FILE_NAME")
+                    .help("The name of the append vec file.")
+            )
+            .arg(
+                Arg::with_name("ads_file_name")
+                    .long("ads-file-name")
+                    .takes_value(true)
+                    .value_name("ADS_FILE_NAME")
+                    .help("The name of the output ads file.")
+            )
+        )
         .get_matches();
 
     info!("{} {}", crate_name!(), solana_version::version!());
@@ -4362,6 +4383,59 @@ fn main() {
                     eprintln!("{err}");
                 }
             }
+            ("new_ads_file", Some(arg_matches)) => {
+                let append_vec_path = value_t_or_exit!(arg_matches, "append_vec", String);
+                let ads_file_path =
+                    PathBuf::from(value_t_or_exit!(arg_matches, "ads_file_name", String));
+                let append_vec_len = std::fs::metadata(&append_vec_path).unwrap().len() as usize;
+                let mut append_vec =
+                    AppendVec::new_from_file_unchecked(append_vec_path, append_vec_len)
+                        .expect("should succeed");
+                append_vec.set_no_remove_on_drop();
+
+                let mut ads = TieredStorage::new(&ads_file_path, true /* create */);
+                ads.set_no_remove_on_drop();
+                ads.write_from_append_vec(&append_vec).unwrap();
+
+                // read append-vec
+                let mut num_accounts = 0;
+                let mut offset = 0;
+                let mut account_map: HashMap<Pubkey, StoredAccountMeta> = HashMap::new();
+                while let Some((account, next_offset)) = append_vec.get_account(offset) {
+                    offset = next_offset;
+                    num_accounts += 1;
+                    account_map.insert(*account.pubkey(), account);
+                }
+
+                // iterate through all accounts in the tiered storage and
+                // verify their correctness.
+                offset = 0;
+                let mut tiered_num_accounts = 0;
+                while let Some((account, next_offset)) = ads.get_account(offset) {
+                    tiered_num_accounts += 1;
+                    offset = next_offset;
+                    if *account.pubkey() == Pubkey::default() {
+                        continue;
+                    }
+                    if *account.owner() == Pubkey::default() {
+                        continue;
+                    }
+                    let av_account = &account_map[account.pubkey()];
+                    assert_eq!(*av_account.pubkey(), *account.pubkey());
+                    assert_eq!(*av_account.hash(), *account.hash());
+                    assert_eq!(av_account.data_len(), account.data_len());
+                    assert_eq!(av_account.write_version(), account.write_version());
+
+                    assert_eq!(av_account.lamports(), account.lamports());
+                    assert_eq!(av_account.data(), account.data());
+                    assert_eq!(*av_account.owner(), *account.owner());
+                    assert_eq!(av_account.rent_epoch(), account.rent_epoch());
+                    assert_eq!(av_account.executable(), account.executable());
+                }
+                assert_eq!(num_accounts, tiered_num_accounts);
+
+                info!("# accounts from append_vec = {:?}", num_accounts);
+            }
             ("", _) => {
                 eprintln!("{}", matches.usage());
                 exit(1);

runtime/Cargo.toml

@@ -31,6 +31,7 @@ lru = { workspace = true }
 lz4 = { workspace = true }
 memmap2 = { workspace = true }
 modular-bitfield = { workspace = true }
+num_enum = { workspace = true }
 num-derive = { workspace = true }
 num-traits = { workspace = true }
 num_cpus = { workspace = true }

runtime/src/account_storage/meta.rs

@@ -1,5 +1,8 @@
 use {
-    crate::{append_vec::AppendVecStoredAccountMeta, storable_accounts::StorableAccounts},
+    crate::{
+        append_vec::AppendVecStoredAccountMeta, storable_accounts::StorableAccounts,
+        tiered_storage::reader::TieredAccountMeta,
+    },
     solana_sdk::{
         account::{AccountSharedData, ReadableAccount},
         hash::Hash,
@@ -105,73 +108,85 @@ impl<'a: 'b, 'b, T: ReadableAccount + Sync + 'b, U: StorableAccounts<'a, T>, V:
 #[derive(PartialEq, Eq, Debug)]
 pub enum StoredAccountMeta<'a> {
     AppendVec(AppendVecStoredAccountMeta<'a>),
+    Tiered(TieredAccountMeta<'a>),
 }
 
 impl<'a> StoredAccountMeta<'a> {
     /// Return a new Account by copying all the data referenced by the `StoredAccountMeta`.
     pub fn clone_account(&self) -> AccountSharedData {
         match self {
             Self::AppendVec(av) => av.clone_account(),
+            Self::Tiered(ts) => ts.clone_account(),
         }
     }
 
     pub fn pubkey(&self) -> &'a Pubkey {
         match self {
             Self::AppendVec(av) => av.pubkey(),
+            Self::Tiered(ts) => ts.pubkey(),
         }
     }
 
     pub fn hash(&self) -> &'a Hash {
         match self {
             Self::AppendVec(av) => av.hash(),
+            Self::Tiered(ts) => ts.hash(),
         }
     }
 
     pub fn stored_size(&self) -> usize {
         match self {
             Self::AppendVec(av) => av.stored_size(),
+            Self::Tiered(ts) => ts.stored_size(),
         }
     }
 
     pub fn offset(&self) -> usize {
         match self {
             Self::AppendVec(av) => av.offset(),
+            Self::Tiered(ts) => ts.offset(),
         }
     }
 
     pub fn data(&self) -> &'a [u8] {
         match self {
             Self::AppendVec(av) => av.data(),
+            Self::Tiered(ts) => ts.data(),
         }
     }
 
     pub fn data_len(&self) -> u64 {
         match self {
             Self::AppendVec(av) => av.data_len(),
+            Self::Tiered(ts) => ts.data_len(),
         }
     }
 
     pub fn write_version(&self) -> StoredMetaWriteVersion {
         match self {
             Self::AppendVec(av) => av.write_version(),
+            Self::Tiered(ts) => ts.write_version(),
         }
     }
 
     pub fn meta(&self) -> &StoredMeta {
         match self {
             Self::AppendVec(av) => av.meta(),
+            Self::Tiered(_) => unreachable!(),
         }
     }
 
     pub fn set_meta(&mut self, meta: &'a StoredMeta) {
         match self {
             Self::AppendVec(av) => av.set_meta(meta),
+            Self::Tiered(_) => unreachable!(),
         }
     }
 
     pub(crate) fn sanitize(&self) -> bool {
         match self {
             Self::AppendVec(av) => av.sanitize(),
+            Self::Tiered(_) => unimplemented!(),
         }
     }
 }
@@ -180,26 +195,31 @@ impl<'a> ReadableAccount for StoredAccountMeta<'a> {
     fn lamports(&self) -> u64 {
         match self {
             Self::AppendVec(av) => av.lamports(),
+            Self::Tiered(ts) => ts.lamports(),
         }
     }
     fn data(&self) -> &[u8] {
         match self {
             Self::AppendVec(av) => av.data(),
+            Self::Tiered(ts) => ts.data(),
         }
     }
     fn owner(&self) -> &Pubkey {
         match self {
             Self::AppendVec(av) => av.owner(),
+            Self::Tiered(ts) => ts.owner(),
         }
     }
     fn executable(&self) -> bool {
         match self {
             Self::AppendVec(av) => av.executable(),
+            Self::Tiered(ts) => ts.executable(),
         }
     }
     fn rent_epoch(&self) -> Epoch {
         match self {
             Self::AppendVec(av) => av.rent_epoch(),
+            Self::Tiered(ts) => ts.rent_epoch(),
         }
     }
 }

runtime/src/accounts_file.rs

@@ -5,6 +5,7 @@ use {
         },
         append_vec::{AppendVec, MatchAccountOwnerError},
         storable_accounts::StorableAccounts,
+        tiered_storage::TieredStorage,
     },
     solana_sdk::{account::ReadableAccount, clock::Slot, hash::Hash, pubkey::Pubkey},
     std::{
@@ -19,6 +20,7 @@ use {
 /// under different formats.
 pub enum AccountsFile {
     AppendVec(AppendVec),
+    Cold(TieredStorage),
 }
 
 impl AccountsFile {
@@ -31,48 +33,59 @@ impl AccountsFile {
         Ok((Self::AppendVec(av), num_accounts))
     }
 
+    pub fn new_cold_entry(file_path: &Path, create: bool) -> Self {
+        Self::Cold(TieredStorage::new(file_path, create))
+    }
+
     /// By default, all AccountsFile will remove its underlying file on
     /// drop.  Calling this function to disable such behavior for this
     /// instance.
     pub fn set_no_remove_on_drop(&mut self) {
         match self {
             Self::AppendVec(av) => av.set_no_remove_on_drop(),
+            Self::Cold(ts) => ts.set_no_remove_on_drop(),
         }
     }
 
     pub fn flush(&self) -> io::Result<()> {
         match self {
             Self::AppendVec(av) => av.flush(),
+            Self::Cold(..) => Ok(()),
         }
     }
 
     pub fn reset(&self) {
         match self {
             Self::AppendVec(av) => av.reset(),
+            Self::Cold(..) => {}
         }
     }
 
     pub fn remaining_bytes(&self) -> u64 {
         match self {
             Self::AppendVec(av) => av.remaining_bytes(),
+            Self::Cold(ts) => ts.remaining_bytes(),
         }
     }
 
     pub fn len(&self) -> usize {
         match self {
             Self::AppendVec(av) => av.len(),
+            Self::Cold(ts) => ts.len(),
         }
     }
 
     pub fn is_empty(&self) -> bool {
         match self {
             Self::AppendVec(av) => av.is_empty(),
+            Self::Cold(ts) => ts.is_empty(),
         }
     }
 
     pub fn capacity(&self) -> u64 {
         match self {
             Self::AppendVec(av) => av.capacity(),
+            Self::Cold(ts) => ts.capacity(),
         }
     }
 
@@ -86,6 +99,7 @@ impl AccountsFile {
     pub fn get_account(&self, index: usize) -> Option<(StoredAccountMeta<'_>, usize)> {
         match self {
             Self::AppendVec(av) => av.get_account(index),
+            Self::Cold(ts) => ts.get_account(index),
         }
     }
 
@@ -96,13 +110,15 @@ impl AccountsFile {
     ) -> Result<usize, MatchAccountOwnerError> {
         match self {
             Self::AppendVec(av) => av.account_matches_owners(offset, owners),
+            Self::Cold(..) => todo!(),
         }
     }
 
     /// Return the path of the underlying account file.
     pub fn get_path(&self) -> PathBuf {
         match self {
             Self::AppendVec(av) => av.get_path(),
+            Self::Cold(ts) => ts.get_path(),
         }
     }
 
@@ -115,6 +131,7 @@ impl AccountsFile {
     pub fn accounts(&self, offset: usize) -> Vec<StoredAccountMeta> {
         match self {
             Self::AppendVec(av) => av.accounts(offset),
+            Self::Cold(ts) => ts.accounts(offset),
         }
     }
 
@@ -138,6 +155,7 @@ impl AccountsFile {
     ) -> Option<Vec<StoredAccountInfo>> {
         match self {
             Self::AppendVec(av) => av.append_accounts(accounts, skip),
+            Self::Cold(ts) => ts.append_accounts(accounts, skip),
         }
     }
 }
@@ -176,6 +194,7 @@ pub mod tests {
         pub(crate) fn set_current_len_for_tests(&self, len: usize) {
             match self {
                 Self::AppendVec(av) => av.set_current_len_for_tests(len),
+                Self::Cold(..) => todo!(),
             }
         }
     }

runtime/src/ancient_append_vecs.rs

@@ -752,6 +752,9 @@ pub fn get_ancient_append_vec_capacity() -> u64 {
 pub fn is_ancient(storage: &AccountsFile) -> bool {
     match storage {
         AccountsFile::AppendVec(storage) => storage.capacity() >= get_ancient_append_vec_capacity(),
+        AccountsFile::Cold(..) => {
+            return false;
+        }
     }
 }