wip try to implement scanblocks rpc to speed up full wallet scans

Cargo.toml

@@ -14,6 +14,7 @@ members = [
     "example-crates/wallet_electrum",
     "example-crates/wallet_esplora_blocking",
     "example-crates/wallet_esplora_async",
+    "example-crates/wallet_rpc",
     "nursery/tmp_plan",
     "nursery/coin_select"
 ]

crates/bdk/src/wallet/mod.rs

@@ -31,8 +31,8 @@ use bdk_chain::{
 use bitcoin::secp256k1::{All, Secp256k1};
 use bitcoin::sighash::{EcdsaSighashType, TapSighashType};
 use bitcoin::{
-    absolute, Address, Network, OutPoint, Script, ScriptBuf, Sequence, Transaction, TxOut, Txid,
-    Weight, Witness,
+    absolute, Address, Block, Network, OutPoint, Script, ScriptBuf, Sequence, Transaction, TxOut,
+    Txid, Weight, Witness,
 };
 use bitcoin::{consensus::encode::serialize, BlockHash};
 use bitcoin::{constants::genesis_block, psbt};
@@ -2288,7 +2288,7 @@ impl<D> Wallet<D> {
         self.persist.commit().map(|c| c.is_some())
     }
 
-    /// Returns the changes that will be staged with the next call to [`commit`].
+    /// Returns the changes that will be committed with the next call to [`commit`].
     ///
     /// [`commit`]: Self::commit
     pub fn staged(&self) -> &ChangeSet
@@ -2312,6 +2312,55 @@ impl<D> Wallet<D> {
     pub fn local_chain(&self) -> &LocalChain {
         &self.chain
     }
+
+    /// Set lookahead for all keychains.
+    ///
+    /// Passing a lookahead of 0 will result in an error. This is because if we sync
+    /// the chain with a lookahead of 0, we would not find any update since we don't
+    /// have any scripts stored.
+    pub fn set_lookahead_for_all(&mut self, lookahead: u32) -> Result<(), Error> {
+        if lookahead == 0 {
+            return Err(Error::Generic(
+                "lookahead must be greater than 0".to_string(),
+            ));
+        }
+        self.indexed_graph.index.set_lookahead_for_all(lookahead);
+        Ok(())
+    }
+
+    /// Insert all the block's relevant transactions into the IndexedTxGraph.
+    pub fn apply_block_relevant(
+        &mut self,
+        block: Block,
+        height: u32,
+    ) -> Result<(), CannotConnectError>
+    where
+        D: PersistBackend<ChangeSet>,
+    {
+        let chain_update = CheckPoint::from_header(&block.header, height).into_update(false);
+        let mut changeset = ChangeSet::from(self.chain.apply_update(chain_update)?);
+        changeset.append(ChangeSet::from(
+            self.indexed_graph.apply_block_relevant(block, height),
+        ));
+        self.persist.stage(changeset);
+        Ok(())
+    }
+
+    /// Batch insert unconfirmed transactions into the IndexedTxGraph,
+    /// filtering out those that are not relevant.
+    ///
+    /// Read more here: [`self.indexed_graph.batch_insert_relevant_unconfirmed()`]
+    pub fn batch_insert_relevant_unconfirmed<'t>(
+        &mut self,
+        unconfirmed_txs: impl IntoIterator<Item = (&'t Transaction, u64)>,
+    ) where
+        D: PersistBackend<ChangeSet>,
+    {
+        let indexed_graph_changeset = self
+            .indexed_graph
+            .batch_insert_relevant_unconfirmed(unconfirmed_txs);
+        self.persist.stage(ChangeSet::from(indexed_graph_changeset));
+    }
 }
 
 impl<D> AsRef<bdk_chain::tx_graph::TxGraph<ConfirmationTimeHeightAnchor>> for Wallet<D> {

crates/bitcoind_rpc/Cargo.toml

@@ -15,7 +15,8 @@ readme = "README.md"
 [dependencies]
 # For no-std, remember to enable the bitcoin/no-std feature
 bitcoin = { version = "0.30", default-features = false }
-bitcoincore-rpc = { version = "0.17" }
+# bitcoincore-rpc = { version = "0.17" }
+bitcoincore-rpc = { git = "https://github.com/chrisguida/rust-bitcoincore-rpc", branch = "feat/scanblocks" }
 bdk_chain = { path = "../chain", version = "0.6", default-features = false }
 
 [dev-dependencies]

example-crates/example_bitcoind_rpc_polling/README.md

@@ -0,0 +1,68 @@
+# Example RPC CLI
+
+### Simple Regtest Test
+
+1. Start local regtest bitcoind.
+   ```
+    mkdir -p /tmp/regtest/bitcoind
+    bitcoind -regtest -server -fallbackfee=0.0002 -rpcuser=<your-rpc-username> -rpcpassword=<your-rpc-password> -datadir=/tmp/regtest/bitcoind -daemon
+   ```
+2. Create a test bitcoind wallet and set bitcoind env.
+   ```
+   bitcoin-cli -datadir=/tmp/regtest/bitcoind -regtest -rpcuser=<your-rpc-username> -rpcpassword=<your-rpc-password> -named createwallet wallet_name="test"
+   export RPC_URL=127.0.0.1:18443
+   export RPC_USER=<your-rpc-username>
+   export RPC_PASS=<your-rpc-password>
+   ```
+3. Get test bitcoind wallet info.
+   ```
+   bitcoin-cli -rpcwallet="test" -rpcuser=<your-rpc-username> -rpcpassword=<your-rpc-password> -datadir=/tmp/regtest/bitcoind -regtest getwalletinfo
+   ```
+4. Get new test bitcoind wallet address.
+   ```
+   BITCOIND_ADDRESS=$(bitcoin-cli -rpcwallet="test" -datadir=/tmp/regtest/bitcoind -regtest -rpcuser=<your-rpc-username> -rpcpassword=<your-rpc-password> getnewaddress)
+   echo $BITCOIND_ADDRESS
+   ```
+5. Generate 101 blocks with reward to test bitcoind wallet address.
+   ```
+   bitcoin-cli -datadir=/tmp/regtest/bitcoind -regtest -rpcuser=<your-rpc-username> -rpcpassword=<your-rpc-password> generatetoaddress 101 $BITCOIND_ADDRESS
+   ```
+6. Verify test bitcoind wallet balance.
+   ```
+   bitcoin-cli -rpcwallet="test" -datadir=/tmp/regtest/bitcoind -regtest -rpcuser=<your-rpc-username> -rpcpassword=<your-rpc-password> getbalances
+   ```
+7. Set descriptor env and get address from RPC CLI wallet.
+   ```
+   export DESCRIPTOR="wpkh(tprv8ZgxMBicQKsPfK9BTf82oQkHhawtZv19CorqQKPFeaHDMA4dXYX6eWsJGNJ7VTQXWmoHdrfjCYuDijcRmNFwSKcVhswzqs4fugE8turndGc/1/*)"
+   cargo run -- --network regtest address next
+   ```
+8. Send 5 test bitcoin to RPC CLI wallet.
+   ```
+   bitcoin-cli -rpcwallet="test" -datadir=/tmp/regtest/bitcoind -regtest -rpcuser=<your-rpc-username> -rpcpassword=<your-rpc-password> sendtoaddress <address> 5
+   ```
+9. Sync blockchain with RPC CLI wallet.
+   ```
+   cargo run -- --network regtest sync
+   <CNTRL-C to stop syncing>
+   ```
+10. Get RPC CLI wallet unconfirmed balances.
+   ```
+   cargo run -- --network regtest balance
+   ```
+11. Generate 1 block with reward to test bitcoind wallet address.
+   ```
+   bitcoin-cli -datadir=/tmp/regtest/bitcoind -rpcuser=<your-rpc-username> -rpcpassword=<your-rpc-password> -regtest generatetoaddress 10 $BITCOIND_ADDRESS
+   ```
+12. Sync the blockchain with RPC CLI wallet.
+   ```
+   cargo run -- --network regtest sync
+   <CNTRL-C to stop syncing>
+   ```
+13. Get RPC CLI wallet confirmed balances.
+   ```
+   cargo run -- --network regtest balance
+   ```
+14. Get RPC CLI wallet transactions.
+   ```
+   cargo run -- --network regtest txout list
+   ```

example-crates/wallet_rpc/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "wallet_rpc"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+bdk = { path = "../../crates/bdk" }
+bdk_file_store = { path = "../../crates/file_store" }
+bdk_bitcoind_rpc = { path = "../../crates/bitcoind_rpc" }

example-crates/wallet_rpc/README.md

@@ -0,0 +1,13 @@
+# Wallet RPC Example 
+
+# To run the wallet example, execute the following code (replace arguments with values that match your setup)
+
+```
+cargo run -- <RPC_URL> <RPC_USER> <RPC_PASS> <LOOKAHEAD> <FALLBACK_HEIGHT>
+```
+
+Here is the command we used during testing
+
+```
+cargo run -- 127.0.0.1:18332 bitcoin password 20 2532323
+```

example-crates/wallet_rpc/src/main.rs

@@ -0,0 +1,135 @@
+use bdk::{
+    bitcoin::{Address, Network},
+    wallet::{AddressIndex, Wallet},
+    SignOptions,
+};
+use bdk_bitcoind_rpc::{
+    bitcoincore_rpc::{
+        bitcoincore_rpc_json::{
+            ScanBlocksOptions, ScanBlocksRequest, ScanBlocksRequestDescriptor, ScanBlocksResult,
+        },
+        Auth, Client, RpcApi,
+    },
+    Emitter,
+};
+use bdk_file_store::Store;
+use std::{str::FromStr, time::Duration};
+
+const DB_MAGIC: &str = "bdk-rpc-wallet-example";
+const SEND_AMOUNT: u64 = 5000;
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let args = std::env::args().collect::<Vec<_>>();
+    let db_path = std::env::temp_dir().join("bdk-rpc-example");
+    let db = Store::<bdk::wallet::ChangeSet>::new_from_path(DB_MAGIC.as_bytes(), db_path)?;
+
+    let external_descriptor = "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/1'/0'/0/*)";
+    let internal_descriptor = "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/1'/0'/1/*)";
+
+    if args.len() < 6 {
+        println!("Usage: wallet_rpc <RPC_URL> <RPC_USER> <RPC_PASS> <LOOKAHEAD> <FALLBACK_HEIGHT>");
+        std::process::exit(1);
+    }
+
+    let mut wallet = Wallet::new(
+        external_descriptor,
+        Some(internal_descriptor),
+        db,
+        Network::Testnet,
+    )?;
+
+    let address = wallet.get_address(AddressIndex::New);
+    println!("Generated Address: {}", address);
+
+    let balance = wallet.get_balance();
+    println!("Wallet balance before syncing: {} sats", balance.total());
+
+    let rpc_client = Client::new_with_timeout(
+        &args[1],
+        Auth::UserPass(args[2].clone(), args[3].clone()),
+        Duration::from_secs(3600),
+    )?;
+
+    println!(
+        "Connected to Bitcoin Core RPC at {:?}",
+        rpc_client.get_blockchain_info().unwrap()
+    );
+
+    let external_descriptor = ScanBlocksRequestDescriptor::Extended {
+        desc: external_descriptor.to_string(),
+        range: None,
+    };
+    let internal_descriptor = ScanBlocksRequestDescriptor::Extended {
+        desc: internal_descriptor.to_string(),
+        range: None,
+    };
+    let descriptors = &[external_descriptor, internal_descriptor];
+    let request = ScanBlocksRequest {
+        scanobjects: descriptors,
+        start_height: None,
+        stop_height: None,
+        filtertype: None,
+        options: Some(ScanBlocksOptions {
+            filter_false_positives: Some(true),
+        }),
+    };
+    let res: ScanBlocksResult = rpc_client.scan_blocks_blocking(request)?;
+    println!("scanblocks result: {:?}", res);
+
+    wallet.set_lookahead_for_all(args[4].parse::<u32>()?)?;
+
+    let chain_tip = wallet.latest_checkpoint();
+    let mut emitter = match chain_tip {
+        Some(cp) => Emitter::from_checkpoint(&rpc_client, cp),
+        None => Emitter::from_height(&rpc_client, args[5].parse::<u32>()?),
+    };
+
+    for bh in res.relevant_blocks {
+        // self.get_relevant_txs(bh, &conn);
+        let block = rpc_client.get_block(&bh)?;
+        let height: u32 = block.bip34_block_height()?.try_into().unwrap();
+        println!("adding block height {} to wallet", height);
+        wallet.apply_block_relevant(block.clone(), height)?;
+        wallet.commit()?;
+    }
+
+    // while let Some((height, block)) = emitter.next_block()? {
+    //     println!("Applying block {} at height {}", block.block_hash(), height);
+    //     wallet.apply_block_relevant(block, height)?;
+    //     wallet.commit()?;
+    // }
+
+    let unconfirmed_txs = emitter.mempool()?;
+    println!("Applying unconfirmed transactions: ...");
+    wallet.batch_insert_relevant_unconfirmed(unconfirmed_txs.iter().map(|(tx, time)| (tx, *time)));
+    wallet.commit()?;
+
+    let balance = wallet.get_balance();
+    println!("Wallet balance after syncing: {} sats", balance.total());
+
+    if balance.total() < SEND_AMOUNT {
+        println!(
+            "Please send at least {} sats to the receiving address",
+            SEND_AMOUNT
+        );
+        std::process::exit(1);
+    }
+
+    let faucet_address = Address::from_str("tb1qw2c3lxufxqe2x9s4rdzh65tpf4d7fssjgh8nv6")?
+        .require_network(Network::Testnet)?;
+
+    let mut tx_builder = wallet.build_tx();
+    tx_builder
+        .add_recipient(faucet_address.script_pubkey(), SEND_AMOUNT)
+        .enable_rbf();
+
+    let mut psbt = tx_builder.finish()?;
+    let finalized = wallet.sign(&mut psbt, SignOptions::default())?;
+    assert!(finalized);
+
+    let tx = psbt.extract_tx();
+    rpc_client.send_raw_transaction(&tx)?;
+    println!("Tx broadcasted! Txid: {}", tx.txid());
+
+    Ok(())
+}