perf(wallet api): improve wallet transaction list performance

BREAKING CHANGE: no longer page based on _id
nitsujlangston · Nov 13, 2018 · 7491e6f · 7491e6f
1 parent b2917e6
commit 7491e6f
Show file tree

Hide file tree

Showing 5 changed files with 56 additions and 92 deletions.
diff --git a/packages/bitcore-client/bin/wallet-transaction-list b/packages/bitcore-client/bin/wallet-transaction-list
@@ -15,10 +15,7 @@ const main = async () => {
   const { name, path, startDate, endDate } = program;
   try {
     const wallet = await Wallet.loadWallet({ name, path });
-    const txStream = await wallet.listTransactions({ startDate, endDate });
-    for(let tx of txStream) {
-      console.log(tx);
-    }
+    wallet.listTransactions({ startDate, endDate }).pipe(process.stdout);
   } catch (e) {
     console.error(e);
   }

diff --git a/packages/bitcore-client/lib/client.js b/packages/bitcore-client/lib/client.js
@@ -1,4 +1,5 @@
 const request = require('request-promise-native');
+const requestStream = require('request');
 const bitcoreLib = require('bitcore-lib');
 const secp256k1 = require('secp256k1');
 const stream = require('stream');
@@ -62,37 +63,13 @@ Client.prototype.getCoins = async function (params) {
 };
 
 Client.prototype.listTransactions = async function(params) {
-  const getTransactions = ({pubKey, startDate, endDate, since}) => {
-    let url = `${this.baseUrl}/wallet/${pubKey}/transactions?startDate=${startDate}&endDate=${endDate}&paging=_id&limit=1000`;
-    if(since) {
-      url += `&since=${since}`;
-    }
-    const signature = this.sign({ method: 'GET', url});
-    return request.get(url, {
-      headers: { 'x-signature': signature },
-      json: true
-    });
-  };
-
-  let totalResults = [];
-  let since = '';
-  let splitResults = null;
-  do {
-    try {
-      let results = await getTransactions({...params, since });
-      if(!results) {
-        throw new Error('No more results');
-      }
-      splitResults = results.split('\n').filter(r => r!= '');
-      totalResults = totalResults.concat(splitResults);
-      const last = JSON.parse(splitResults[splitResults.length - 1]);
-      since = last.id;
-    } catch (e) {
-      splitResults = null;
-    }
-  }
-  while(splitResults && splitResults != []);
-  return totalResults;
+  const { pubKey, startDate, endDate } = params;
+  const url = `${this.baseUrl}/wallet/${pubKey}/transactions?startDate=${startDate}&endDate=${endDate}`;
+  const signature = this.sign({ method: 'GET', url });
+  return requestStream.get(url, {
+    headers: { 'x-signature': signature },
+    json: true
+  });
 };
 
 Client.prototype.getFee = async function (params) {

diff --git a/packages/bitcore-node/src/models/transaction.ts b/packages/bitcore-node/src/models/transaction.ts
@@ -42,7 +42,8 @@ export class Transaction extends BaseModel<ITransaction> {
     this.collection.createIndex({ blockHeight: 1, chain: 1, network: 1 }, { background: true });
     this.collection.createIndex({ blockHash: 1 }, { background: true });
     this.collection.createIndex({ blockTimeNormalized: 1, chain: 1, network: 1 }, { background: true });
-    this.collection.createIndex({ wallets: 1, blockTimeNormalized: 1, _id: -1 }, { background: true, sparse: true });
+    this.collection.createIndex({ wallets: 1, blockTimeNormalized: 1 }, { background: true, sparse: true });
+    this.collection.createIndex({ wallets: 1, blockHeight: 1 }, { background: true, sparse: true });
   }
 
   async batchImport(params: {

diff --git a/packages/bitcore-node/src/providers/chain-state/internal/internal.ts b/packages/bitcore-node/src/providers/chain-state/internal/internal.ts
@@ -282,36 +282,43 @@ export class InternalStateProvider implements CSP.IChainStateService {
   }
 
   async streamWalletTransactions(params: CSP.StreamWalletTransactionsParams) {
-    let { chain, network, wallet, stream, args } = params;
-    let finalQuery: any = {
-      chain: chain,
+    const { chain, network, wallet, stream, args } = params;
+    const query: any = {
+      chain,
       network,
       wallets: wallet._id
     };
+    const options: any = {};
     if (args) {
-      if (args.startBlock) {
-        finalQuery.blockHeight = { $gte: Number(args.startBlock) };
-      }
-      if (args.endBlock) {
-        finalQuery.blockHeight = finalQuery.blockHeight || {};
-        finalQuery.blockHeight.$lte = Number(args.endBlock);
-      }
-      if (args.startDate) {
-        const startDate = new Date(args.startDate);
-        if (startDate.getTime()) {
-          finalQuery.blockTimeNormalized = { $gte: new Date(args.startDate) };
+      if (args.startBlock || args.endBlock) {
+        options.sort = {blockHeight: args.direction || -1};
+        if (args.startBlock) {
+          query.blockHeight = { $gte: Number(args.startBlock) };
         }
-      }
-      if (args.endDate) {
-        const endDate = new Date(args.endDate);
-        if (endDate.getTime()) {
-          finalQuery.blockTimeNormalized = finalQuery.blockTimeNormalized || {};
-          finalQuery.blockTimeNormalized.$lt = new Date(args.endDate);
+        if (args.endBlock) {
+          query.blockHeight = query.blockHeight || {};
+          query.blockHeight.$lte = Number(args.endBlock);
+        }
+      } else {
+        options.sort = { blockTimeNormalized: args.direction || -1 };
+        if (args.startDate) {
+          const startDate = new Date(args.startDate);
+          if (startDate.getTime()) {
+            query.blockTimeNormalized = { $gte: new Date(args.startDate) };
+          }
+        }
+        if (args.endDate) {
+          const endDate = new Date(args.endDate);
+          if (endDate.getTime()) {
+            query.blockTimeNormalized = query.blockTimeNormalized || {};
+            query.blockTimeNormalized.$lt = new Date(args.endDate);
+          }
         }
       }
     }
-    let transactionStream = TransactionModel.getTransactions({ query: finalQuery, options: args });
-    let listTransactionsStream = new ListTransactionsStream(wallet);
+
+    const transactionStream = TransactionModel.collection.find(query, options).addCursorFlag('noCursorTimeout', true);
+    const listTransactionsStream = new ListTransactionsStream(wallet);
     transactionStream.pipe(listTransactionsStream).pipe(stream);
   }
 

diff --git a/packages/bitcore-node/src/providers/chain-state/internal/transforms.ts b/packages/bitcore-node/src/providers/chain-state/internal/transforms.ts
@@ -8,7 +8,6 @@ export class ListTransactionsStream extends Transform {
   }
 
   async _transform(transaction, _, done) {
-    var self = this;
     transaction.inputs = await CoinModel.collection
       .find(
         {
@@ -32,34 +31,20 @@ export class ListTransactionsStream extends Transform {
       .addCursorFlag('noCursorTimeout', true)
       .toArray();
 
-    var wallet = this.wallet._id!.toString();
-    var totalInputs = transaction.inputs.reduce((total, input) => {
-      return total + input.value;
-    }, 0);
-    var totalOutputs = transaction.outputs.reduce((total, output) => {
-      return total + output.value;
-    }, 0);
-    var fee = totalInputs - totalOutputs;
-    var sending = transaction.inputs.some(function(input) {
-      var contains = false;
-      input.wallets.forEach(function(inputWallet) {
-        if (inputWallet.equals(wallet)) {
-          contains = true;
-        }
+    const wallet = this.wallet._id!.toString();
+    const sending = transaction.inputs.some((input) => {
+      return input.wallets.some((inputWallet) => {
+        return inputWallet.equals(wallet);
       });
-      return contains;
     });
 
     if (sending) {
-      transaction.outputs.forEach(function(output) {
-        var sendingToOurself = false;
-        output.wallets.forEach(function(outputWallet) {
-          if (outputWallet.equals(wallet)) {
-            sendingToOurself = true;
-          }
+      transaction.outputs.forEach((output) => {
+        const sendingToOurself = output.wallets.some((outputWallet) => {
+          return outputWallet.equals(wallet);
         });
         if (!sendingToOurself) {
-          self.push(
+          this.push(
             JSON.stringify({
               id: transaction._id,
               txid: transaction.txid,
@@ -74,7 +59,7 @@ export class ListTransactionsStream extends Transform {
             }) + '\n'
           );
         } else {
-          self.push(
+          this.push(
             JSON.stringify({
               id: transaction._id,
               txid: transaction.txid,
@@ -90,29 +75,26 @@ export class ListTransactionsStream extends Transform {
           );
         }
       });
-      if (fee > 0) {
-        self.push(
+      if (transaction.fee > 0) {
+        this.push(
           JSON.stringify({
             id: transaction._id,
             txid: transaction.txid,
             category: 'fee',
-            satoshis: -fee,
+            satoshis: -transaction.fee,
             height: transaction.blockHeight,
             blockTime: transaction.blockTimeNormalized
           }) + '\n'
         );
       }
       return done();
     } else {
-      transaction.outputs.forEach(function(output) {
-        var weReceived = false;
-        output.wallets.forEach(function(outputWallet) {
-          if (outputWallet.equals(wallet)) {
-            weReceived = true;
-          }
+      transaction.outputs.forEach((output) => {
+        const weReceived = output.wallets.some((outputWallet) => {
+          return outputWallet.equals(wallet);
         });
         if (weReceived) {
-          self.push(
+          this.push(
             JSON.stringify({
               id: transaction._id,
               txid: transaction.txid,