feat: cache new block template (#6222)

Description
---
In the merge mining proxy for the block template protocol:
- Cached a new block template and used it in case it is asked
repetitively for the same best block.
- Added an exit clause to the potential endless loop.

Motivation and Context
---
Under certain conditions, a merge mining proxy request for a new block
template from the base node would be repeated many times for the same
best block height, without the final request for a block template that
includes the coinbase transaction. This wastes many resources,
especially within the mempool where the template must be constructed.
```
2024-03-15 16:49:43.873749100 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1675
2024-03-15 16:56:35.702267000 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1676
2024-03-15 16:56:57.709323100 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1677
2024-03-15 16:59:04.747989700 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:04.840957100 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:04.849282000 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:04.852855000 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:04.953873500 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:04.954461600 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:04.958619200 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:05.054517300 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:05.056071600 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:05.057842800 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:05.058706200 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:05.184297100 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:05.184568300 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:05.186171600 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:05.308025900 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:05.308188600 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:05.411993800 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:05.413148500 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:05.436028800 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:05.457553200 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:05.569510200 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:05.573500600 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:05.658578700 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:05.659258900 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:05.693116700 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:05.695350400 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:05.713246600 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:05.724627700 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:05.734541000 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:05.904546600 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:05.909569300 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:05.913773800 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:05.914761900 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:05.918061100 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:05.928847000 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:06.125242100 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:06.125427400 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:06.130678700 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:06.367898700 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:06.385011900 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:06.386722300 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:06.388034000 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:06.406540100 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:06.645604600 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:06.654252400 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:06.659640400 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:06.661591500 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:06.829952300 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:06.838337400 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:06.843845500 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:06.847976400 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:07.036353600 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:07.038593100 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:07.042252500 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:07.070333200 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:07.204174900 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:07.207776400 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:07.208302900 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:07.211634900 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:07.218055300 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:07.367296100 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:07.367378600 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:07.369022900 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:07.372211400 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:07.504687000 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:07.505760400 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:07.505823500 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:09.578821600 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 16:59:09.829455600 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1678
2024-03-15 17:00:11.282291600 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1679
2024-03-15 17:01:13.476999400 [minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new block template at height: #1680
```

How Has This Been Tested?
---
System-level testing

What process can a PR reviewer use to test or verify this change?
---
- Review code changes
- System-level testing:
- These log entries should not be repeated for the same best block
height:
  ```
2024-03-18 15:12:50.362269600
[minotari_mm_proxy::proxy::block_template_protocol] DEBUG Requested new
block template at height: #1832 (try 4)
  ```
  - Look for these log entries instead:
  ```
2024-03-18 15:13:46.016447400
[minotari_mm_proxy::proxy::block_template_protocol] DEBUG Used existing
new block template at height: #1836 (try 1)
  ```

<!-- Checklist -->
<!-- 1. Is the title of your PR in the form that would make nice release
notes? The title, excluding the conventional commit
tag, will be included exactly as is in the CHANGELOG, so please think
about it carefully. -->


Breaking Changes
---

- [x] None
- [ ] Requires data directory on base node to be deleted
- [ ] Requires hard fork
- [ ] Other - Please specify

<!-- Does this include a breaking change? If so, include this line as a
footer -->
<!-- BREAKING CHANGE: Description what the user should do, e.g. delete a
database, resync the chain -->

---------

Co-authored-by: SW van Heerden <swvheerden@gmail.com>

applications/minotari_merge_mining_proxy/src/block_template_data.rs

@@ -33,28 +33,56 @@ use tari_core::proof_of_work::monero_rx::FixedByteArray;
 use tokio::sync::RwLock;
 use tracing::trace;
 
-use crate::{block_template_protocol::FinalBlockTemplateData, error::MmProxyError};
+use crate::{
+    block_template_protocol::{FinalBlockTemplateData, NewBlockTemplateData},
+    error::MmProxyError,
+};
 
 const LOG_TARGET: &str = "minotari_mm_proxy::xmrig";
 
-/// Structure for holding hashmap of hashes -> [BlockTemplateRepositoryItem]
+/// Structure for holding hashmap of hashes -> [BlockRepositoryItem] and [TemplateRepositoryItem].
 #[derive(Debug, Clone)]
 pub struct BlockTemplateRepository {
-    blocks: Arc<RwLock<HashMap<Vec<u8>, BlockTemplateRepositoryItem>>>,
+    blocks: Arc<RwLock<HashMap<Vec<u8>, BlockRepositoryItem>>>,
+    templates: Arc<RwLock<HashMap<Vec<u8>, TemplateRepositoryItem>>>,
 }
 
-/// Structure holding [BlockTemplateData] along with a timestamp.
+/// Structure holding [NewBlockTemplate] along with a timestamp.
 #[derive(Debug, Clone)]
-pub struct BlockTemplateRepositoryItem {
+pub struct TemplateRepositoryItem {
+    pub new_block_template: NewBlockTemplateData,
+    pub template_with_coinbase: grpc::NewBlockTemplate,
+    datetime: DateTime<Utc>,
+}
+
+impl TemplateRepositoryItem {
+    /// Create new [Self] with current time in UTC.
+    pub fn new(new_block_template: NewBlockTemplateData, template_with_coinbase: grpc::NewBlockTemplate) -> Self {
+        Self {
+            new_block_template,
+            template_with_coinbase,
+            datetime: Utc::now(),
+        }
+    }
+
+    /// Get the timestamp of creation.
+    pub fn datetime(&self) -> DateTime<Utc> {
+        self.datetime
+    }
+}
+
+/// Structure holding [FinalBlockTemplateData] along with a timestamp.
+#[derive(Debug, Clone)]
+pub struct BlockRepositoryItem {
     pub data: FinalBlockTemplateData,
     datetime: DateTime<Utc>,
 }
 
-impl BlockTemplateRepositoryItem {
+impl BlockRepositoryItem {
     /// Create new [Self] with current time in UTC.
-    pub fn new(block_template: FinalBlockTemplateData) -> Self {
+    pub fn new(final_block: FinalBlockTemplateData) -> Self {
         Self {
-            data: block_template,
+            data: final_block,
             datetime: Utc::now(),
         }
     }
@@ -69,35 +97,71 @@ impl BlockTemplateRepository {
     pub fn new() -> Self {
         Self {
             blocks: Arc::new(RwLock::new(HashMap::new())),
+            templates: Arc::new(RwLock::new(HashMap::new())),
         }
     }
 
     /// Return [BlockTemplateData] with the associated hash. None if the hash is not stored.
-    pub async fn get<T: AsRef<[u8]>>(&self, hash: T) -> Option<FinalBlockTemplateData> {
-        trace!(
-            target: LOG_TARGET,
-            "Retrieving blocktemplate with merge mining hash: {:?}",
-            hex::encode(hash.as_ref())
-        );
+    pub async fn get_final_template<T: AsRef<[u8]>>(&self, merge_mining_hash: T) -> Option<FinalBlockTemplateData> {
         let b = self.blocks.read().await;
-        b.get(hash.as_ref()).map(|item| item.data.clone())
+        b.get(merge_mining_hash.as_ref()).map(|item| {
+            trace!(
+                target: LOG_TARGET,
+                "Retrieving block template at height #{} with merge mining hash: {:?}",
+                item.data.clone().template.new_block_template.header.unwrap_or_default().height,
+                hex::encode(merge_mining_hash.as_ref())
+            );
+            item.data.clone()
+        })
+    }
+
+    /// Return [BlockTemplateData] with the associated hash. None if the hash is not stored.
+    pub async fn get_new_template<T: AsRef<[u8]>>(
+        &self,
+        best_block_hash: T,
+    ) -> Option<(NewBlockTemplateData, grpc::NewBlockTemplate)> {
+        let b = self.templates.read().await;
+        b.get(best_block_hash.as_ref())
+            .map(|item| (item.new_block_template.clone(), item.template_with_coinbase.clone()))
     }
 
-    /// Store [BlockTemplateData] at the hash value if the key does not exist.
-    pub async fn save_if_key_unique(&self, hash: Vec<u8>, block_template: FinalBlockTemplateData) {
+    /// Store [FinalBlockTemplateData] at the hash value if the key does not exist.
+    pub async fn save_final_block_template_if_key_unique(
+        &self,
+        merge_mining_hash: Vec<u8>,
+        block_template: FinalBlockTemplateData,
+    ) {
         let mut b = self.blocks.write().await;
-        b.entry(hash.clone()).or_insert_with(|| {
+        b.entry(merge_mining_hash.clone()).or_insert_with(|| {
+            trace!(
+                target: LOG_TARGET,
+                "Saving final block template with merge mining hash: {:?}",
+                hex::encode(&merge_mining_hash)
+            );
+            BlockRepositoryItem::new(block_template)
+        });
+    }
+
+    /// Store [NewBlockTemplate] at the hash value if the key does not exist.
+    pub async fn save_new_block_template_if_key_unique(
+        &self,
+        best_block_hash: Vec<u8>,
+        new_block_template: NewBlockTemplateData,
+        template_with_coinbase: grpc::NewBlockTemplate,
+    ) {
+        let mut b = self.templates.write().await;
+        b.entry(best_block_hash.clone()).or_insert_with(|| {
             trace!(
                 target: LOG_TARGET,
-                "Saving blocktemplate with merge mining hash: {:?}",
-                hex::encode(&hash)
+                "Saving new block template for best block hash: {:?}",
+                hex::encode(&best_block_hash)
             );
-            BlockTemplateRepositoryItem::new(block_template)
+            TemplateRepositoryItem::new(new_block_template, template_with_coinbase)
         });
     }
 
     /// Check if the repository contains a block template with best_previous_block_hash
-    pub async fn contains(&self, current_best_block_hash: FixedHash) -> Option<FinalBlockTemplateData> {
+    pub async fn blocks_contains(&self, current_best_block_hash: FixedHash) -> Option<FinalBlockTemplateData> {
         let b = self.blocks.read().await;
         b.values()
             .find(|item| {
@@ -109,25 +173,43 @@ impl BlockTemplateRepository {
 
     /// Remove any data that is older than 20 minutes.
     pub async fn remove_outdated(&self) {
-        trace!(target: LOG_TARGET, "Removing outdated blocktemplates");
+        trace!(target: LOG_TARGET, "Removing outdated final block templates");
         let mut b = self.blocks.write().await;
         #[cfg(test)]
         let threshold = Utc::now();
         #[cfg(not(test))]
         let threshold = Utc::now() - Duration::minutes(20);
         *b = b.drain().filter(|(_, i)| i.datetime() >= threshold).collect();
+        trace!(target: LOG_TARGET, "Removing outdated new block templates");
+        let mut b = self.templates.write().await;
+        #[cfg(test)]
+        let threshold = Utc::now();
+        #[cfg(not(test))]
+        let threshold = Utc::now() - Duration::minutes(20);
+        *b = b.drain().filter(|(_, i)| i.datetime() >= threshold).collect();
     }
 
-    /// Remove a particular hash and return the associated [BlockTemplateRepositoryItem] if any.
-    pub async fn remove<T: AsRef<[u8]>>(&self, hash: T) -> Option<BlockTemplateRepositoryItem> {
+    /// Remove a particularfinla block template for hash and return the associated [BlockRepositoryItem] if any.
+    pub async fn remove_final_block_template<T: AsRef<[u8]>>(&self, hash: T) -> Option<BlockRepositoryItem> {
         trace!(
             target: LOG_TARGET,
-            "Blocktemplate removed with merge mining hash {:?}",
+            "Final block template removed with merge mining hash {:?}",
             hex::encode(hash.as_ref())
         );
         let mut b = self.blocks.write().await;
         b.remove(hash.as_ref())
     }
+
+    /// Remove a particular new block template for hash and return the associated [BlockRepositoryItem] if any.
+    pub async fn remove_new_block_template<T: AsRef<[u8]>>(&self, hash: T) -> Option<TemplateRepositoryItem> {
+        trace!(
+            target: LOG_TARGET,
+            "New block template removed with best block hash {:?}",
+            hex::encode(hash.as_ref())
+        );
+        let mut b = self.templates.write().await;
+        b.remove(hash.as_ref())
+    }
 }
 
 /// Setup values for the new block.
@@ -299,19 +381,21 @@ pub mod test {
         let hash2 = vec![2; 32];
         let hash3 = vec![3; 32];
         let block_template = create_block_template_data();
-        btr.save_if_key_unique(hash1.clone(), block_template.clone()).await;
-        btr.save_if_key_unique(hash2.clone(), block_template).await;
-        assert!(btr.get(hash1.clone()).await.is_some());
-        assert!(btr.get(hash2.clone()).await.is_some());
-        assert!(btr.get(hash3.clone()).await.is_none());
-        assert!(btr.remove(hash1.clone()).await.is_some());
-        assert!(btr.get(hash1.clone()).await.is_none());
-        assert!(btr.get(hash2.clone()).await.is_some());
-        assert!(btr.get(hash3.clone()).await.is_none());
+        btr.save_final_block_template_if_key_unique(hash1.clone(), block_template.clone())
+            .await;
+        btr.save_final_block_template_if_key_unique(hash2.clone(), block_template)
+            .await;
+        assert!(btr.get_final_template(hash1.clone()).await.is_some());
+        assert!(btr.get_final_template(hash2.clone()).await.is_some());
+        assert!(btr.get_final_template(hash3.clone()).await.is_none());
+        assert!(btr.remove_final_block_template(hash1.clone()).await.is_some());
+        assert!(btr.get_final_template(hash1.clone()).await.is_none());
+        assert!(btr.get_final_template(hash2.clone()).await.is_some());
+        assert!(btr.get_final_template(hash3.clone()).await.is_none());
         btr.remove_outdated().await;
-        assert!(btr.get(hash1).await.is_none());
-        assert!(btr.get(hash2).await.is_none());
-        assert!(btr.get(hash3).await.is_none());
+        assert!(btr.get_final_template(hash1).await.is_none());
+        assert!(btr.get_final_template(hash2).await.is_none());
+        assert!(btr.get_final_template(hash3).await.is_none());
     }
 
     #[test]