Update to use address history rather than bulk unspent

action_xpub.go

@@ -82,59 +82,56 @@ func (c *Client) GetXpubByID(ctx context.Context, xPubID string) (*Xpub, error)
 }
 
 // ImportXpub will import a given xPub and all related destinations and transactions
-func (c *Client) ImportXpub(ctx context.Context, xPubKey string, depth uint32, opts ...ModelOps) (*ImportResults, error) {
+func (c *Client) ImportXpub(ctx context.Context, xPubKey string, opts ...ModelOps) (*ImportResults, error) {
 
 	// Validate the xPub
 	xPub, err := utils.ValidateXPub(xPubKey)
 	if err != nil {
 		return nil, err
 	}
 
-	// todo: add opts to each model for storing metadata
-	// opts
-
 	// Start an accumulator
 	results := &ImportResults{Key: xPub.String()}
 
-	// Derive internal addresses until depth
-	c.Logger().Info(ctx, "Deriving internal addresses...")
-	addressList := whatsonchain.AddressList{}
-	var destination *Destination
-	for i := uint32(0); i < depth; i++ {
-		// log.Printf("path m/1/%v", i)
-		if destination, err = c.NewDestination(
-			ctx, xPub.String(), utils.ChainInternal, utils.ScriptTypePubKeyHash, nil,
-		); err != nil {
+	// Set the WOC client
+	woc := c.Chainstate().WhatsOnChain()
+	var allTransactions []*whatsonchain.HistoryRecord
+
+	// Assume gap of 20 addresses, if no txs are found for 20
+	// internal/external derivations, assume we've found everything
+	gapHit := false
+	for !gapHit {
+		// Derive internal addresses until depth
+		c.Logger().Info(ctx, "Deriving internal addresses...")
+		addressList := whatsonchain.AddressList{}
+		addresses, err := c.deriveAddresses(ctx, xPub.String(), utils.ChainInternal, 20)
+		if err != nil {
 			return nil, err
 		}
-		addressList.Addresses = append(addressList.Addresses, destination.Address)
-		results.InternalAddresses++
-	}
+		results.InternalAddresses += 20
+		addressList.Addresses = append(addressList.Addresses, addresses...)
 
-	// Derive external addresses until gap limit
-	c.Logger().Info(ctx, "Deriving external addresses...")
-	for i := uint32(0); i < depth; i++ {
-		// log.Printf("path m/0/%v", i)
-		if destination, err = c.NewDestination(
-			ctx, xPub.String(), utils.ChainExternal, utils.ScriptTypePubKeyHash, nil,
-		); err != nil {
+		// Derive external addresses until gap limit
+		c.Logger().Info(ctx, "Deriving external addresses...")
+		addresses, err = c.deriveAddresses(ctx, xPub.String(), utils.ChainExternal, 20)
+		if err != nil {
 			return nil, err
 		}
-		addressList.Addresses = append(addressList.Addresses, destination.Address)
-		results.ExternalAddresses++
-	}
-
-	// Set the WOC client
-	woc := c.Chainstate().WhatsOnChain()
-
-	// Get all transactions for those addresses
-	var allTransactions []*whatsonchain.HistoryRecord
-	if allTransactions, err = getTransactionsFromAddresses(
-		ctx, woc, addressList,
-	); err != nil {
-		return nil, err
+		results.ExternalAddresses += 20
+		addressList.Addresses = append(addressList.Addresses, addresses...)
+
+		// Get all transactions for those addresses
+		transactions, err := getAllTransactionsFromAddresses(
+			ctx, woc, addressList,
+		)
+		if err != nil {
+			return nil, err
+		}
+		if len(transactions) == 0 {
+			gapHit = true
+		}
+		allTransactions = append(allTransactions, transactions...)
 	}
-
 	// Remove any duplicate transactions from all historical txs
 	allTransactions = removeDuplicates(allTransactions)
 

import.go

@@ -3,6 +3,7 @@ package bux
 import (
 	"context"
 
+	"github.com/BuxOrg/bux/utils"
 	"github.com/mrz1836/go-whatsonchain"
 )
 
@@ -15,8 +16,8 @@ type ImportResults struct {
 	TransactionsImported int    `json:"transactions_imported"`
 }
 
-// getTransactionsFromAddresses will get all transactions related to addresses
-func getTransactionsFromAddresses(ctx context.Context, client whatsonchain.ClientInterface, addressList whatsonchain.AddressList) ([]*whatsonchain.HistoryRecord, error) {
+// getUnspentTransactionsFromAddresses will get all unspent transactions related to addresses
+func getUnspentTransactionsFromAddresses(ctx context.Context, client whatsonchain.ClientInterface, addressList whatsonchain.AddressList) ([]*whatsonchain.HistoryRecord, error) {
 	histories, err := client.BulkUnspentTransactionsProcessor(
 		ctx, &addressList,
 	)
@@ -30,6 +31,32 @@ func getTransactionsFromAddresses(ctx context.Context, client whatsonchain.Clien
 	return txs, nil
 }
 
+// getAllTransactionsFromAddresses will get all transactions related to addresses
+func getAllTransactionsFromAddresses(ctx context.Context, client whatsonchain.ClientInterface, addressList whatsonchain.AddressList) ([]*whatsonchain.HistoryRecord, error) {
+	var txs []*whatsonchain.HistoryRecord
+	for _, address := range addressList.Addresses {
+		history, err := client.AddressHistory(ctx, address)
+		if err != nil {
+			return nil, err
+		}
+		txs = append(txs, history...)
+	}
+	return txs, nil
+}
+
+// deriveAddresses will derive a new set of addresses for an xpub
+func (c *Client) deriveAddresses(ctx context.Context, xpub string, chain uint32, amount int) ([]string, error) {
+	addressList := []string{}
+	for i := 0; i < amount; i++ {
+		destination, err := c.NewDestination(ctx, xpub, chain, utils.ScriptTypePubKeyHash, nil)
+		if err != nil {
+			return []string{}, err
+		}
+		addressList = append(addressList, destination.Address)
+	}
+	return addressList, nil
+}
+
 // removeDuplicates will remove duplicate transactions
 func removeDuplicates(transactions []*whatsonchain.HistoryRecord) []*whatsonchain.HistoryRecord {
 	keys := make(map[string]bool)

interface.go

@@ -46,7 +46,7 @@ type UTXOService interface {
 type XPubService interface {
 	GetXpub(ctx context.Context, xPubKey string) (*Xpub, error)
 	GetXpubByID(ctx context.Context, xPubID string) (*Xpub, error)
-	ImportXpub(ctx context.Context, xPubKey string, depth uint32, opts ...ModelOps) (*ImportResults, error)
+	ImportXpub(ctx context.Context, xPubKey string, opts ...ModelOps) (*ImportResults, error)
 	NewXpub(ctx context.Context, xPubKey string, opts ...ModelOps) (*Xpub, error)
 }