Implement serialize/deserialize for obsolete accounts

accounts-db/src/accounts_db.rs

@@ -913,6 +913,13 @@ impl AccountStorageEntry {
         self.alive_bytes.load(Ordering::Acquire)
     }
 
+    /// Returns the accounts that were marked obsolete as of the passed in slot
+    /// or earlier. Returned data includes the slots that the accounts were marked
+    /// obsolete at
+    pub fn obsolete_accounts_for_snapshots(&self, slot: Slot) -> ObsoleteAccounts {
+        self.obsolete_accounts_read_lock()
+            .obsolete_accounts_for_snapshots(slot)
+    }
     /// Locks obsolete accounts with a read lock and returns the the accounts with the guard
     pub(crate) fn obsolete_accounts_read_lock(&self) -> RwLockReadGuard<ObsoleteAccounts> {
         self.obsolete_accounts.read().unwrap()

accounts-db/src/lib.rs

@@ -47,7 +47,7 @@ pub mod waitable_condvar;
 
 pub use {
     buffered_reader::large_file_buf_reader, file_io::validate_memlock_limit_for_disk_io,
-    obsolete_accounts::ObsoleteAccounts,
+    obsolete_accounts::{ObsoleteAccounts, ObsoleteAccountItem},
 };
 
 #[macro_use]

accounts-db/src/obsolete_accounts.rs

@@ -1,18 +1,18 @@
 use {crate::account_info::Offset, solana_clock::Slot};
 
 #[derive(Debug, Clone, PartialEq)]
-struct ObsoleteAccountItem {
+pub struct ObsoleteAccountItem {
     /// Offset of the account in the account storage entry
-    offset: Offset,
+    pub offset: Offset,
     /// Length of the account data
-    data_len: usize,
+    pub data_len: usize,
     /// Slot when the account was marked obsolete
-    slot: Slot,
+    pub slot: Slot,
 }
 
 #[derive(Debug, Clone, PartialEq, Default)]
 pub struct ObsoleteAccounts {
-    accounts: Vec<ObsoleteAccountItem>,
+   pub accounts: Vec<ObsoleteAccountItem>,
 }
 
 impl ObsoleteAccounts {
@@ -45,6 +45,22 @@ impl ObsoleteAccounts {
             .filter(move |obsolete_account| slot.is_none_or(|s| obsolete_account.slot <= s))
             .map(|obsolete_account| (obsolete_account.offset, obsolete_account.data_len))
     }
+
+    /// Returns the accounts that were marked obsolete as of the passed in slot
+    /// or earlier. Returned data includes the slots that the accounts were marked
+    /// obsolete at
+    pub fn obsolete_accounts_for_snapshots(&self, slot: Slot) -> ObsoleteAccounts {
+        let filtered_accounts = self
+            .accounts
+            .iter()
+            .filter(|account| account.slot <= slot)
+            .cloned()
+            .collect();
+
+        ObsoleteAccounts {
+            accounts: filtered_accounts,
+        }
+    }
 }
 #[cfg(test)]
 mod tests {
@@ -99,4 +115,39 @@ mod tests {
 
         assert_eq!(filtered_accounts, vec![(10, 100), (20, 200), (30, 300)]);
     }
+
+    #[test]
+    fn test_obsolete_accounts_for_snapshots() {
+        let mut obsolete_accounts = ObsoleteAccounts::default();
+        let new_accounts = vec![(10, 100, 40), (20, 200, 42), (30, 300, 44)]
+            .into_iter()
+            .map(|(offset, data_len, slot)| ObsoleteAccountItem {
+                offset,
+                data_len,
+                slot,
+            })
+            .collect::<Vec<_>>();
+
+        // Mark accounts obsolete with different slots
+        new_accounts.iter().for_each(|item| {
+            obsolete_accounts
+                .mark_accounts_obsolete([(item.offset, item.data_len)].into_iter(), item.slot)
+        });
+
+        // Filter accounts obsolete as of slot 42
+        let obsolete_accounts_for_snapshots = obsolete_accounts.obsolete_accounts_for_snapshots(42);
+
+        let expected_accounts: Vec<_> = new_accounts
+            .iter()
+            .filter(|account| account.slot <= 42)
+            .cloned()
+            .collect();
+
+        assert_eq!(obsolete_accounts_for_snapshots.accounts, expected_accounts);
+
+        // Filter accounts obsolete passing in no slot (i.e., all obsolete accounts)
+        let obsolete_accounts_for_snapshots = obsolete_accounts.obsolete_accounts_for_snapshots(100);
+
+        assert_eq!(obsolete_accounts_for_snapshots.accounts, new_accounts);
+    }
 }

runtime/src/serde_snapshot.rs

@@ -11,6 +11,7 @@ use {
         stakes::{serialize_stake_accounts_to_delegation_format, Stakes},
     },
     bincode::{self, config::Options, Error},
+    dashmap::DashMap,
     log::*,
     serde::{de::DeserializeOwned, Deserialize, Serialize},
     solana_accounts_db::{
@@ -24,7 +25,7 @@ use {
         accounts_update_notifier_interface::AccountsUpdateNotifier,
         ancestors::AncestorsForSerialization,
         blockhash_queue::BlockhashQueue,
-        ObsoleteAccounts,
+        ObsoleteAccountItem, ObsoleteAccounts,
     },
     solana_clock::{Epoch, Slot, UnixTimestamp},
     solana_epoch_schedule::EpochSchedule,
@@ -58,7 +59,7 @@ mod status_cache;
 mod storage;
 mod tests;
 mod types;
-mod utils;
+pub mod utils;
 
 pub(crate) use {
     status_cache::{deserialize_status_cache, serialize_status_cache},
@@ -649,6 +650,34 @@ where
     (serializable_bank, serializable_accounts_db, extra_fields).serialize(serializer)
 }
 
+pub(crate) fn serialize_obsolete_accounts<W>(
+    stream: &mut BufWriter<W>,
+    obsolete_accounts_map: &HashMap<Slot, SerdeObsoleteAccounts>,
+) -> Result<(), Error>
+where
+    W: Write,
+{
+    let bincode = bincode::DefaultOptions::new().with_fixint_encoding();
+    bincode.serialize_into(
+        stream,
+        &utils::serialize_iter_as_tuple(obsolete_accounts_map.iter()),
+    )
+}
+
+pub(crate) fn deserialize_obsolete_accounts<R>(
+    stream: &mut BufReader<R>,
+) -> Result<DashMap<Slot, SerdeObsoleteAccounts>, Error>
+where
+    R: Read,
+{
+    let obsolete_accounts = DashMap::default();
+    while let Ok((slot, accounts)) = { deserialize_from(&mut *stream) } {
+        obsolete_accounts.insert(slot, accounts);
+    }
+
+    Ok(obsolete_accounts)
+}
+
 #[cfg(test)]
 struct SerializableBankAndStorage<'a> {
     bank: &'a Bank,
@@ -869,8 +898,19 @@ pub(crate) fn reconstruct_single_storage(
                 obsolete_accounts.id,
             ));
         }
+        let accounts = ObsoleteAccounts {
+            accounts: obsolete_accounts
+                .accounts
+                .into_iter()
+                .map(|(offset, data_len, slot)| ObsoleteAccountItem {
+                    offset,
+                    data_len,
+                    slot,
+                })
+                .collect(),
+        };
 
-        (updated_len, obsolete_accounts.accounts)
+        (updated_len, accounts)
     } else {
         (current_len, ObsoleteAccounts::default())
     };

runtime/src/serde_snapshot/storage.rs

@@ -1,6 +1,6 @@
 use {
     serde::{Deserialize, Serialize},
-    solana_accounts_db::{accounts_db::AccountStorageEntry, ObsoleteAccounts},
+    solana_accounts_db::{account_info::Offset, accounts_db::AccountStorageEntry},
     solana_clock::Slot,
 };
 
@@ -51,7 +51,7 @@ impl SerializableStorage for SerializableAccountStorageEntry {
 impl solana_frozen_abi::abi_example::TransparentAsHelper for SerializableAccountStorageEntry {}
 
 /// This structure handles the load/store of obsolete accounts during snapshot restoration.
-#[derive(Debug, Default)]
+#[derive(Debug, Default, Serialize, Deserialize)]
 pub(crate) struct SerdeObsoleteAccounts {
     /// The ID of the associated account file. Used for verification to ensure the restored
     /// obsolete accounts correspond to the correct account file
@@ -61,5 +61,25 @@ pub(crate) struct SerdeObsoleteAccounts {
     /// is used to validate the size when creating the accounts file.
     pub bytes: u64,
     /// A list of accounts that are obsolete in the storage being restored.
-    pub accounts: ObsoleteAccounts,
+    pub accounts: Vec<(Offset, usize, Slot)>,
+}
+
+impl SerdeObsoleteAccounts {
+    pub fn new_from_storage_entry_at_slot(
+        storage: &AccountStorageEntry,
+        snapshot_slot: Slot,
+    ) -> Self {
+        let accounts: Vec<(usize, usize, u64)> = storage
+            .obsolete_accounts_for_snapshots(snapshot_slot)
+            .accounts
+            .into_iter()
+            .map(|item| (item.offset, item.data_len, item.slot))
+            .collect();
+
+        SerdeObsoleteAccounts {
+            id: storage.id() as SerializedAccountsFileId,
+            bytes: storage.get_obsolete_bytes(Some(snapshot_slot)) as u64,
+            accounts,
+        }
+    }
 }

runtime/src/serde_snapshot/tests.rs

@@ -4,8 +4,10 @@ mod serde_snapshot_tests {
         crate::{
             bank::BankHashStats,
             serde_snapshot::{
-                deserialize_accounts_db_fields, reconstruct_accountsdb_from_fields,
-                remap_append_vec_file, SerializableAccountsDb, SnapshotAccountsDbFields,
+                deserialize_accounts_db_fields, deserialize_obsolete_accounts,
+                reconstruct_accountsdb_from_fields, remap_append_vec_file,
+                serialize_obsolete_accounts, SerdeObsoleteAccounts, SerializableAccountsDb,
+                SnapshotAccountsDbFields,
             },
             snapshot_utils::{get_storages_to_serialize, StorageAndNextAccountsFileId},
         },
@@ -30,8 +32,9 @@ mod serde_snapshot_tests {
         solana_nohash_hasher::BuildNoHashHasher,
         solana_pubkey::Pubkey,
         std::{
+            collections::HashMap,
             fs::File,
-            io::{self, BufReader, Cursor, Read, Write},
+            io::{self, BufReader, BufWriter, Cursor, Read, Write},
             ops::RangeFull,
             path::{Path, PathBuf},
             sync::{
@@ -937,4 +940,48 @@ mod serde_snapshot_tests {
         )
         .unwrap();
     }
+
+    #[test_case(0, 0)]
+    #[test_case(1, 0)]
+    #[test_case(10, 15)]
+    fn test_serialize_and_deserialize_obsolete_accounts(
+        num_storages: u64,
+        num_accounts_per_storage: usize,
+    ) {
+        // Create a sample obsolete accounts map
+        let mut obsolete_accounts_map = HashMap::new();
+        for slot in 1..=num_storages {
+            let obsolete_accounts = (0..num_accounts_per_storage)
+                .map(|j| (j, j * 10, slot + 1))
+                .collect();
+
+            obsolete_accounts_map.insert(
+                slot,
+                SerdeObsoleteAccounts {
+                    bytes: num_accounts_per_storage as u64 * 1000,
+                    id: slot as usize,
+                    accounts: obsolete_accounts,
+                },
+            );
+        }
+
+        // Serialize the obsolete accounts
+        let mut buf = Vec::new();
+        let cursor = Cursor::new(&mut buf);
+        let mut writer = BufWriter::new(cursor);
+        serialize_obsolete_accounts(&mut writer, &obsolete_accounts_map).unwrap();
+        drop(writer);
+
+        // Deserialize the obsolete accounts
+        let cursor = Cursor::new(buf.as_slice());
+        let mut reader = BufReader::new(cursor);
+        let deserialized_accounts = deserialize_obsolete_accounts(&mut reader).unwrap();
+
+        // Verify the deserialized data matches the original
+        assert_eq!(deserialized_accounts.len(), obsolete_accounts_map.len());
+        for (slot, accounts) in obsolete_accounts_map {
+            let deserialized_accounts = deserialized_accounts.get(&slot).unwrap();
+            assert_eq!(accounts.accounts, deserialized_accounts.accounts);
+        }
+    }
 }