Introduce AccountsDataStorage

Making new account file created in cold mode

Introduce AccountsDataStorage

Making new account file created in cold mode

Introduce AccountsDataStorage

Making new account file created in cold mode

Test

Test 2

ledger-tool/src/main.rs

@@ -53,6 +53,7 @@ use {
         transaction_status_service::TransactionStatusService,
     },
     solana_runtime::{
+        account_storage::meta::StoredAccountMeta,
         accounts::Accounts,
         accounts_background_service::{
             AbsRequestHandlers, AbsRequestSender, AccountsBackgroundService,
@@ -63,6 +64,7 @@ use {
         },
         accounts_index::{AccountsIndexConfig, IndexLimitMb, ScanConfig},
         accounts_update_notifier_interface::AccountsUpdateNotifier,
+        append_vec::AppendVec,
         bank::{Bank, RewardCalculationEvent, TotalAccountsStats},
         bank_forks::BankForks,
         cost_model::CostModel,
@@ -78,6 +80,7 @@ use {
             move_and_async_delete_path, ArchiveFormat, SnapshotVersion,
             DEFAULT_ARCHIVE_COMPRESSION, SUPPORTED_ARCHIVE_COMPRESSION,
         },
+        tiered_storage::TieredStorage,
     },
     solana_sdk::{
         account::{AccountSharedData, ReadableAccount, WritableAccount},
@@ -2468,6 +2471,24 @@ fn main() {
                            If no file name is specified, it will print the metadata of all ledger files.")
             )
         )
+        .subcommand(
+            SubCommand::with_name("new_ads_file")
+            .about("Create a new accounts-data-storage file from an existing append_vec file.")
+            .arg(
+                Arg::with_name("append_vec")
+                    .long("append-vec")
+                    .takes_value(true)
+                    .value_name("APPEND_VEC_FILE_NAME")
+                    .help("The name of the append vec file.")
+            )
+            .arg(
+                Arg::with_name("ads_file_name")
+                    .long("ads-file-name")
+                    .takes_value(true)
+                    .value_name("ADS_FILE_NAME")
+                    .help("The name of the output ads file.")
+            )
+        )
         .get_matches();
 
     info!("{} {}", crate_name!(), solana_version::version!());
@@ -4498,6 +4519,59 @@ fn main() {
                     eprintln!("{err}");
                 }
             }
+            ("new_ads_file", Some(arg_matches)) => {
+                let append_vec_path = value_t_or_exit!(arg_matches, "append_vec", String);
+                let ads_file_path =
+                    PathBuf::from(value_t_or_exit!(arg_matches, "ads_file_name", String));
+                let append_vec_len = std::fs::metadata(&append_vec_path).unwrap().len() as usize;
+                let mut append_vec =
+                    AppendVec::new_from_file_unchecked(append_vec_path, append_vec_len)
+                        .expect("should succeed");
+                append_vec.set_no_remove_on_drop();
+
+                let mut ads = TieredStorage::new(&ads_file_path, true /* create */);
+                ads.set_no_remove_on_drop();
+                ads.write_from_append_vec(&append_vec).unwrap();
+
+                // read append-vec
+                let mut num_accounts = 0;
+                let mut offset = 0;
+                let mut account_map: HashMap<Pubkey, StoredAccountMeta> = HashMap::new();
+                while let Some((account, next_offset)) = append_vec.get_account(offset) {
+                    offset = next_offset;
+                    num_accounts += 1;
+                    account_map.insert(*account.pubkey(), account);
+                }
+
+                // iterate through all accounts in the tiered storage and
+                // verify their correctness.
+                offset = 0;
+                let mut tiered_num_accounts = 0;
+                while let Some((account, next_offset)) = ads.get_account(offset) {
+                    tiered_num_accounts += 1;
+                    offset = next_offset;
+                    if *account.pubkey() == Pubkey::default() {
+                        continue;
+                    }
+                    if *account.owner() == Pubkey::default() {
+                        continue;
+                    }
+                    let av_account = &account_map[account.pubkey()];
+                    assert_eq!(*av_account.pubkey(), *account.pubkey());
+                    assert_eq!(*av_account.hash(), *account.hash());
+                    assert_eq!(av_account.data_len(), account.data_len());
+                    assert_eq!(av_account.write_version(), account.write_version());
+
+                    assert_eq!(av_account.lamports(), account.lamports());
+                    assert_eq!(av_account.data(), account.data());
+                    assert_eq!(*av_account.owner(), *account.owner());
+                    assert_eq!(av_account.rent_epoch(), account.rent_epoch());
+                    assert_eq!(av_account.executable(), account.executable());
+                }
+                assert_eq!(num_accounts, tiered_num_accounts);
+
+                info!("# accounts from append_vec = {:?}", num_accounts);
+            }
             ("", _) => {
                 eprintln!("{}", matches.usage());
                 exit(1);

runtime/Cargo.toml

@@ -31,6 +31,7 @@ lru = { workspace = true }
 lz4 = { workspace = true }
 memmap2 = { workspace = true }
 modular-bitfield = { workspace = true }
+num_enum = { workspace = true }
 num-derive = { workspace = true }
 num-traits = { workspace = true }
 num_cpus = { workspace = true }

runtime/src/account_storage.rs

@@ -270,6 +270,8 @@ pub(crate) mod tests {
     use {super::*, std::path::Path};
 
     #[test]
+    /// The new cold storage will fail this test as it does not have
+    /// the concept of capacity yet.
     fn test_shrink_in_progress() {
         // test that we check in order map then shrink_in_progress_map
         let storage = AccountStorage::default();
@@ -283,13 +285,13 @@ pub(crate) mod tests {
         let store_file_size = 4000;
         let store_file_size2 = store_file_size * 2;
         // 2 append vecs with same id, but different sizes
-        let entry = Arc::new(AccountStorageEntry::new(
+        let entry = Arc::new(AccountStorageEntry::new_av(
             common_store_path,
             slot,
             id,
             store_file_size,
         ));
-        let entry2 = Arc::new(AccountStorageEntry::new(
+        let entry2 = Arc::new(AccountStorageEntry::new_av(
             common_store_path,
             slot,
             id,

runtime/src/account_storage/meta.rs

@@ -1,5 +1,11 @@
 use {
-    crate::{append_vec::AppendVecStoredAccountMeta, storable_accounts::StorableAccounts},
+    crate::{
+        append_vec::AppendVecStoredAccountMeta,
+        storable_accounts::StorableAccounts,
+        tiered_storage::{
+            cold::ColdAccountMeta, hot::HotAccountMeta, reader::TieredStoredAccountMeta,
+        },
+    },
     solana_sdk::{
         account::{AccountSharedData, ReadableAccount},
         hash::Hash,
@@ -105,73 +111,97 @@ impl<'a: 'b, 'b, T: ReadableAccount + Sync + 'b, U: StorableAccounts<'a, T>, V:
 #[derive(PartialEq, Eq, Debug)]
 pub enum StoredAccountMeta<'a> {
     AppendVec(AppendVecStoredAccountMeta<'a>),
+    Cold(TieredStoredAccountMeta<'a, ColdAccountMeta>),
+    Hot(TieredStoredAccountMeta<'a, HotAccountMeta>),
 }
 
 impl<'a> StoredAccountMeta<'a> {
     /// Return a new Account by copying all the data referenced by the `StoredAccountMeta`.
     pub fn clone_account(&self) -> AccountSharedData {
         match self {
             Self::AppendVec(av) => av.clone_account(),
+            Self::Cold(cs) => cs.clone_account(),
+            Self::Hot(hs) => hs.clone_account(),
         }
     }
 
     pub fn pubkey(&self) -> &'a Pubkey {
         match self {
             Self::AppendVec(av) => av.pubkey(),
+            Self::Cold(cs) => cs.pubkey(),
+            Self::Hot(hs) => hs.pubkey(),
         }
     }
 
     pub fn hash(&self) -> &'a Hash {
         match self {
             Self::AppendVec(av) => av.hash(),
+            Self::Cold(cs) => cs.hash(),
+            Self::Hot(hs) => hs.hash(),
         }
     }
 
     pub fn stored_size(&self) -> usize {
         match self {
             Self::AppendVec(av) => av.stored_size(),
+            Self::Cold(cs) => cs.stored_size(),
+            Self::Hot(hs) => hs.stored_size(),
         }
     }
 
     pub fn offset(&self) -> usize {
         match self {
             Self::AppendVec(av) => av.offset(),
+            Self::Cold(cs) => cs.offset(),
+            Self::Hot(hs) => hs.offset(),
         }
     }
 
     pub fn data(&self) -> &'a [u8] {
         match self {
             Self::AppendVec(av) => av.data(),
+            Self::Cold(cs) => cs.data(),
+            Self::Hot(hs) => hs.data(),
         }
     }
 
     pub fn data_len(&self) -> u64 {
         match self {
             Self::AppendVec(av) => av.data_len(),
+            Self::Cold(cs) => cs.data_len(),
+            Self::Hot(hs) => hs.data_len(),
         }
     }
 
     pub fn write_version(&self) -> StoredMetaWriteVersion {
         match self {
             Self::AppendVec(av) => av.write_version(),
+            Self::Cold(cs) => cs.write_version(),
+            Self::Hot(hs) => hs.write_version(),
         }
     }
 
     pub fn meta(&self) -> &StoredMeta {
         match self {
             Self::AppendVec(av) => av.meta(),
+            Self::Cold(_) => unreachable!(),
+            Self::Hot(_) => unreachable!(),
         }
     }
 
     pub fn set_meta(&mut self, meta: &'a StoredMeta) {
         match self {
             Self::AppendVec(av) => av.set_meta(meta),
+            Self::Cold(_) => unreachable!(),
+            Self::Hot(_) => unreachable!(),
         }
     }
 
     pub(crate) fn sanitize(&self) -> bool {
         match self {
             Self::AppendVec(av) => av.sanitize(),
+            Self::Cold(_) => unimplemented!(),
+            Self::Hot(_) => unreachable!(),
         }
     }
 }
@@ -180,26 +210,36 @@ impl<'a> ReadableAccount for StoredAccountMeta<'a> {
     fn lamports(&self) -> u64 {
         match self {
             Self::AppendVec(av) => av.lamports(),
+            Self::Cold(cs) => cs.lamports(),
+            Self::Hot(hs) => hs.lamports(),
         }
     }
     fn data(&self) -> &[u8] {
         match self {
             Self::AppendVec(av) => av.data(),
+            Self::Cold(cs) => cs.data(),
+            Self::Hot(hs) => hs.data(),
         }
     }
     fn owner(&self) -> &Pubkey {
         match self {
             Self::AppendVec(av) => av.owner(),
+            Self::Cold(cs) => cs.owner(),
+            Self::Hot(hs) => hs.owner(),
         }
     }
     fn executable(&self) -> bool {
         match self {
             Self::AppendVec(av) => av.executable(),
+            Self::Cold(cs) => cs.executable(),
+            Self::Hot(hs) => hs.executable(),
         }
     }
     fn rent_epoch(&self) -> Epoch {
         match self {
             Self::AppendVec(av) => av.rent_epoch(),
+            Self::Cold(cs) => cs.rent_epoch(),
+            Self::Hot(hs) => hs.rent_epoch(),
         }
     }
 }

runtime/src/accounts_db.rs

@@ -66,6 +66,7 @@ use {
         snapshot_utils::create_accounts_run_and_snapshot_dirs,
         sorted_storages::SortedStorages,
         storable_accounts::StorableAccounts,
+        tiered_storage::{cold::COLD_FORMAT, TieredStorage},
         verify_accounts_hash_in_background::VerifyAccountsHashInBackground,
     },
     blake3::traits::digest::Digest,
@@ -1046,6 +1047,10 @@ pub struct AccountStorageEntry {
 
 impl AccountStorageEntry {
     pub fn new(path: &Path, slot: Slot, id: AppendVecId, file_size: u64) -> Self {
+        Self::new_cold(path, slot, id, file_size)
+    }
+
+    pub fn new_av(path: &Path, slot: Slot, id: AppendVecId, file_size: u64) -> Self {
         let tail = AccountsFile::file_name(slot, id);
         let path = Path::new(path).join(tail);
         let accounts = AccountsFile::AppendVec(AppendVec::new(&path, true, file_size as usize));
@@ -1060,6 +1065,22 @@ impl AccountStorageEntry {
         }
     }
 
+    pub fn new_cold(path: &Path, slot: Slot, id: AppendVecId, _file_size: u64) -> Self {
+        let tail = AccountsFile::file_name(slot, id);
+        let path = Path::new(path).join(tail);
+        let accounts = AccountsFile::Cold(TieredStorage::new(&path, Some(&COLD_FORMAT)));
+        info!("YH: Create new cold storage at path {:?}", path);
+
+        Self {
+            id: AtomicAppendVecId::new(id),
+            slot: AtomicU64::new(slot),
+            accounts,
+            count_and_status: RwLock::new((0, AccountStorageStatus::Available)),
+            approx_store_count: AtomicUsize::new(0),
+            alive_bytes: AtomicUsize::new(0),
+        }
+    }
+
     pub(crate) fn new_existing(
         slot: Slot,
         id: AppendVecId,