[tracing] Add tracing for Garbage Collection (#10177)

Adding logs for generic GC.
Tracing for GC during the last block of the epoch during which
resharding happened.

chain/chain/src/chain.rs

@@ -1006,7 +1006,7 @@ impl Chain {
         tries: ShardTries,
         gc_config: &near_chain_configs::GCConfig,
     ) -> Result<(), Error> {
-        let _span = tracing::debug_span!(target: "chain", "clear_data").entered();
+        let _span = tracing::debug_span!(target: "garbage_collection", "clear_data").entered();
 
         let head = self.store.head()?;
         let tail = self.store.tail()?;

chain/chain/src/store.rs

@@ -1,6 +1,6 @@
 use std::collections::hash_map::Entry;
 use std::collections::{HashMap, HashSet};
-use std::io;
+use std::{fmt, io};
 
 use borsh::{BorshDeserialize, BorshSerialize};
 use chrono::Utc;
@@ -74,6 +74,16 @@ pub enum GCMode {
     StateSync { clear_block_info: bool },
 }
 
+impl fmt::Debug for GCMode {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            GCMode::Fork(_) => write!(f, "GCMode::Fork"),
+            GCMode::Canonical(_) => write!(f, "GCMode::Canonical"),
+            GCMode::StateSync { .. } => write!(f, "GCMode::StateSync"),
+        }
+    }
+}
+
 /// Accesses the chain store. Used to create atomic editable views that can be reverted.
 pub trait ChainStoreAccess {
     /// Returns underlaying store.
@@ -2344,6 +2354,7 @@ impl<'a> ChainStoreUpdate<'a> {
         // Now we can proceed to removing the trie state and flat state
         let mut store_update = self.store().store_update();
         for shard_uid in prev_shard_layout.get_shard_uids() {
+            tracing::info!(target: "garbage_collection", ?block_hash, ?shard_uid, "GC resharding");
             runtime.get_tries().delete_trie_for_shard(shard_uid, &mut store_update);
             runtime
                 .get_flat_storage_manager()
@@ -2364,6 +2375,8 @@ impl<'a> ChainStoreUpdate<'a> {
     ) -> Result<(), Error> {
         let mut store_update = self.store().store_update();
 
+        tracing::info!(target: "garbage_collection", ?gc_mode, ?block_hash, "GC block_hash");
+
         // 1. Apply revert insertions or deletions from DBCol::TrieChanges for Trie
         {
             let shard_uids_to_gc: Vec<_> = self.get_shard_uids_to_gc(epoch_manager, &block_hash);

core/store/src/flat/manager.rs

@@ -226,6 +226,7 @@ impl FlatStorageManager {
         let mut flat_storages = self.0.flat_storages.lock().expect(POISONED_LOCK_ERR);
         if let Some(flat_store) = flat_storages.remove(&shard_uid) {
             flat_store.clear_state(store_update)?;
+            tracing::info!(target: "store", ?shard_uid, "remove_flat_storage_for_shard successful");
             Ok(true)
         } else {
             Ok(false)