fix syncing and gap address handling

examples/create_testnet_wallet.py

@@ -19,4 +19,4 @@
 # Print the seed words and an address.
 print("Mnemonic seed\n-------------")
 print(" ".join(mnemonicSeed))
-print("Receive DCR at %s" % wallet.paymentAddress())
+print("Receive DCR at %s" % wallet.currentAddress())

pydecred/README.md

@@ -30,8 +30,7 @@ print(json.dumps(bestBlock, indent=4, sort_keys=True))
 Because the available dcrdata endpoints can change with new versions, the 
 interface is generated by pulling a list of endpoints from the API itself.
 The acquired list does not include some endpoints, particularly the Insight API
-endpoints. Additional paths can be provided to the constructor in a list of 
-paths as the `customPaths` parameter.
+endpoints.
 
 You can print an endpoint guide to the console with  `client.endpointGuide()`,
 

pydecred/account.py

@@ -6,9 +6,17 @@
 support.
 """
 
-from tinydecred.accounts import Account, EXTERNAL_BRANCH
+from tinydecred.accounts import Account
 from tinydecred.util import tinyjson, helpers
 from tinydecred.crypto.crypto import AddressSecpPubKey
+from tinydecred.pydecred import txscript
+
+log = helpers.getLogger("DCRACCT")
+
+# In addition to the standard internal and external branches, we'll have a third
+# branch. This should help accomodate upcoming changes to dcrstakepool. See also
+# https://github.com/decred/dcrstakepool/pull/514
+STAKE_BRANCH = 2
 
 class KeySource(object):
     """
@@ -91,6 +99,8 @@ def __init__(self, *a, **k):
         self.tickets = []
         self.stakeStats = TicketStats()
         self.stakePools = []
+        self.blockchain = None
+        self.signals = None
     def __tojson__(self):
         obj = super().__tojson__()
         return helpers.recursiveUpdate(obj, {
@@ -155,17 +165,16 @@ def allAddresses(self):
         Overload the base class to add the voting address.
         """
         return self.addTicketAddresses(super().allAddresses())
-    def addressesOfInterest(self):
+    def watchAddrs(self):
         """
         Overload the base class to add the voting address.
         """
-        return self.addTicketAddresses(super().addressesOfInterest())
+        return self.addTicketAddresses(super().watchAddrs())
     def votingKey(self):
         """
-        This may change, but for now, the voting key is from the zeroth 
-        child of the zeroth child of the external branch.
+        For now, the voting key is the zeroth child 
         """
-        return self.privKey.child(EXTERNAL_BRANCH).child(0).child(0).privateKey()
+        return self.privKey.child(STAKE_BRANCH).child(0).privateKey()
     def votingAddress(self):
         """
         The voting address is the pubkey address (not pubkey-hash) for the
@@ -204,7 +213,60 @@ def ticketStats(self):
         A getter for the stakeStats.
         """
         return self.stakeStats
-    def sendToAddress(self, value, address, feeRate, blockchain):
+    def blockSignal(self, sig):
+        """
+        Process a new block from the explorer. 
+
+        Arg:
+            sig (obj or string): The block explorer's json-decoded block
+            notification.
+        """
+        block = sig["message"]["block"]
+        for newTx in block["Tx"]:
+            txid = newTx["TxID"]
+            # only grab the tx if its a transaction we care about.
+            if self.caresAboutTxid(txid):
+                tx = self.blockchain.tx(txid)
+                self.confirmTx(tx, self.blockchain.tipHeight)
+        # "Spendable" balance can change as utxo's mature, so update the 
+        # balance at every block.
+        self.signals.balance(self.calcBalance(self.blockchain.tipHeight))
+    def addressSignal(self, addr, txid):
+        """
+        Process an address notification from the block explorer.
+
+        Arg:
+            sig (obj or string): The block explorer's json-decoded address
+            notification.
+        """
+        tx = self.blockchain.tx(txid)
+        self.addTxid(addr, tx.txid())
+
+        matches = False
+        # scan the inputs for any spends.
+        for txin in tx.txIn:
+            op = txin.previousOutPoint
+            # spendTxidVout is a no-op if output is unknown
+            match = self.spendTxidVout(op.txid(), op.index)
+            if match:
+                matches += 1
+        # scan the outputs for any new UTXOs
+        for vout, txout in enumerate(tx.txOut):
+            try:
+                _, addresses, _ = txscript.extractPkScriptAddrs(0, txout.pkScript, self.net)
+            except Exception:
+                # log.debug("unsupported script %s" % txout.pkScript.hex())
+                continue
+            # convert the Address objects to strings.
+            if addr in (a.string() for a in addresses):
+                utxo = self.blockchain.txVout(txid, vout)
+                utxo.address = addr
+                self.addUTXO(utxo)
+                matches += 1
+        if matches:
+            # signal the balance update
+            self.signals.balance(self.calcBalance(self.blockchain.tip["height"]))
+    def sendToAddress(self, value, address, feeRate):
         """
         Send the value to the address.
 
@@ -217,15 +279,15 @@ def sendToAddress(self, value, address, feeRate, blockchain):
         """
         keysource = KeySource(
             priv = self.getPrivKeyForAddress,
-            internal = self.getChangeAddress,
+            internal = self.nextInternalAddress,
         )
-        tx, spentUTXOs, newUTXOs = blockchain.sendToAddress(value, address, keysource, self.getUTXOs, feeRate)
+        tx, spentUTXOs, newUTXOs = self.blockchain.sendToAddress(value, address, keysource, self.getUTXOs, feeRate)
         self.addMempoolTx(tx)
         self.spendUTXOs(spentUTXOs)
         for utxo in newUTXOs:
             self.addUTXO(utxo)
         return tx
-    def purchaseTickets(self, qty, price, blockchain):
+    def purchaseTickets(self, qty, price):
         """
         purchaseTickets completes the purchase of the specified tickets. The 
         DecredAccount uses the blockchain to do the heavy lifting, but must 
@@ -234,7 +296,7 @@ def purchaseTickets(self, qty, price, blockchain):
         """
         keysource = KeySource(
             priv = self.getPrivKeyForAddress,
-            internal = self.getChangeAddress,
+            internal = self.nextInternalAddress,
         )
         pool = self.stakePool()
         pi = pool.purchaseInfo
@@ -249,7 +311,7 @@ def purchaseTickets(self, qty, price, blockchain):
             count = qty, 
             txFee = 0, # use network default
         )
-        txs, spentUTXOs, newUTXOs = blockchain.purchaseTickets(keysource, self.getUTXOs, req)
+        txs, spentUTXOs, newUTXOs = self.blockchain.purchaseTickets(keysource, self.getUTXOs, req)
         if txs:
             # Add the split transactions
             self.addMempoolTx(txs[0])
@@ -265,5 +327,51 @@ def purchaseTickets(self, qty, price, blockchain):
         for utxo in newUTXOs:
             self.addUTXO(utxo)
         return txs[1]
+    def sync(self, blockchain, signals):
+        """
+        Synchronize the UTXO set with the server. This should be the first
+        action after the account is opened or changed.
+        """
+        self.blockchain = blockchain
+        self.signals = signals
+        signals.balance(self.balance)
+        self.generateGapAddresses()
+
+        # First, look at addresses that have been generated but not seen. Run in 
+        # loop until the gap limit is reached.
+        requestedTxs = 0
+        addrs = self.unseenAddrs()
+        while addrs:
+            for addr in addrs:
+                for txid in blockchain.txsForAddr(addr):
+                    requestedTxs += 1
+                    self.addTxid(addr, txid)
+            addrs = self.generateGapAddresses()
+        log.info("%d address transactions sets fetched" % requestedTxs)
+
+        # start with a search for all known addresses
+        addresses = self.allAddresses()
+
+        # Until the server stops returning UTXOs, keep requesting more addresses
+        # to check.
+        while True:
+            # Update the account with known UTXOs.
+            blockchainUTXOs = blockchain.UTXOs(addresses)
+            if not blockchainUTXOs:
+                break
+            self.resolveUTXOs(blockchainUTXOs)
+            addresses = self.generateGapAddresses()
+            if not addresses:
+                break
+
+        # Subscribe to block and address updates.
+        blockchain.addressReceiver = self.addressSignal
+        blockchain.subscribeBlocks(self.blockSignal)
+        watchAddresses = self.watchAddrs()
+        if watchAddresses:
+            blockchain.subscribeAddresses(watchAddresses)
+        # Signal the new balance.
+        signals.balance(self.calcBalance(self.blockchain.tip["height"]))
+        return True
 
 tinyjson.register(DecredAccount)

pydecred/dcrdata.py

@@ -91,6 +91,14 @@ def getSocketURIs(uri):
     ps = fmt.format(prot, uri.netloc, "ps")
     return ws, ps
 
+# To Do: Get the full list here. 
+InsightPaths = [
+    "/tx/send",
+    "/insight/api/addr/{address}/utxo",
+    "/insight/api/addr/{address}/txs",
+    "insight/api/tx/send"
+]
+
 class DcrdataClient(object):
     """
     DcrdataClient represents the base node. The only argument to the
@@ -99,7 +107,7 @@ class DcrdataClient(object):
     timeFmt = "%Y-%m-%d %H:%M:%S"
     rfc3339Z = "%Y-%m-%dT%H:%M:%SZ"
 
-    def __init__(self, baseURI, customPaths=None, emitter=None):
+    def __init__(self, baseURI, emitter=None):
         """
         Build the DcrdataPath tree. 
         """
@@ -113,10 +121,9 @@ def __init__(self, baseURI, customPaths=None, emitter=None):
         atexit.register(self.close)
         root = self.root = DcrdataPath()
         self.listEntries = []
-        customPaths = customPaths if customPaths else []
         # /list returns a json list of enpoints with parameters in template format, base/A/{param}/B
         endpoints = http.get(self.baseApi + "/list", headers=GET_HEADERS)
-        endpoints += customPaths
+        endpoints += InsightPaths
 
         def getParam(part):
             if part.startswith('{') and part.endswith('}'):
@@ -581,11 +588,6 @@ def connect(self):
         """
         self.dcrdata = DcrdataClient(
             self.datapath, 
-            customPaths=(
-                "/tx/send",
-                "/insight/api/addr/{address}/utxo",
-                "insight/api/tx/send"
-            ),
             emitter=self.pubsubSignal,
         )
         self.updateTip()
@@ -663,6 +665,17 @@ def UTXOs(self, addrs):
                 ads = addrs[start:end]
                 utxos += [self.processNewUTXO(u) for u in get(ads)]
         return utxos
+    def txsForAddr(self, addr):
+        """
+        Get the transaction IDs for the provided address.
+
+        Args:
+            addrs (string): Base-58 encoded address
+        """
+        addrInfo = self.dcrdata.insight.api.addr.txs(addr)
+        if "transactions" not in addrInfo:
+            return  []
+        return addrInfo["transactions"]
     def txVout(self, txid, vout):
         """
         Get a UTXO from the outpoint. The UTXO will not have the address set.
@@ -889,8 +902,12 @@ def pubsubSignal(self, sig):
             elif sigType == "subscribeResp":
                 # should check for error.
                 pass
+            elif sigType == "ping":
+                # nothing to do here right now. May want to implement a
+                # auto-reconnect using this signal.
+                pass
             else:
-                raise Exception("unknown signal")
+                raise Exception("unknown signal %s" % repr(sigType))
         except Exception as e:
             log.error("failed to process pubsub message: %s" % formatTraceback(e))
     def changeScript(self, changeAddress):

pydecred/tests.py

@@ -1879,11 +1879,7 @@ def test_calc_signature_hash_reference(self):
 
 class TestDcrdata(unittest.TestCase):
     def client(self, **k):
-        return dcrdata.DcrdataClient("https://alpha.dcrdata.org", customPaths={
-            "/tx/send",
-            "/insight/api/addr/{address}/utxo",
-            "insight/api/tx/send"
-        }, **k)
+        return dcrdata.DcrdataClient("https://alpha.dcrdata.org", **k)
     def test_websocket(self):
         """
         "newblock":       SigNewBlock,
@@ -1904,6 +1900,7 @@ def test_get_block_header(self):
             blockchain = dcrdata.DcrdataBlockchain(os.path.join(tempDir, "db.db"), mainnet, "https://alpha.dcrdata.org")
             blockchain.connect()
             blockchain.blockHeader("298e5cc3d985bfe7f81dc135f360abe089edd4396b86d2de66b0cef42b21d980")
+            blockchain.close()
     def test_purchase_ticket(self):
         from tinydecred.crypto.secp256k1 import curve as Curve
         from tinydecred.crypto import rando

ui/screens.py

@@ -490,7 +490,7 @@ def showEvent(self, e):
         """
         app = self.app
         if app.wallet:
-            address = app.wallet.paymentAddress()
+            address = app.wallet.currentAddress()
             self.address.setText(address)
     def balanceClicked(self):
         """
@@ -1156,7 +1156,7 @@ def buyTickets(self, wallet, qty):
         """
         tip = self.blockchain.tip["height"]
         acct = wallet.openAccount
-        txs = acct.purchaseTickets(qty, self.lastPrice, self.blockchain)
+        txs = acct.purchaseTickets(qty, self.lastPrice)
         if txs:
             wallet.signals.balance(acct.calcBalance(tip))
             self.app.home()

util/database.py

@@ -52,18 +52,20 @@ def __init__(self, database, name, datatypes, unique):
 		self.existsQuery = KVExists.format(tablename=name) # = "SELECT EXISTS(SELECT * FROM kvtable WHERE k = ?);"
 		self.deleteQuery = KVDelete.format(tablename=name) # = "DELETE FROM kvtable WHERE k = ?;"
 		self.countQuery = KVCount.format(tablename=name) # = "SELECT COUNT(*) FROM kvtable;"
-		self.tid = None
+		self.conn = None
 	def __enter__(self):
 		"""
 		Create a new connection for a every requesting thread.
 		"""
-		tid = threadID()
-		if self.tid != tid:
-			self.conn = self.database.openDB()
-			self.tid = tid
+		if self.conn:
+			self.conn.close()
+		self.conn = self.database.openDB()
 		self.open()
 		return self
 	def __exit__(self, xType, xVal, xTB):
+		if self.conn:
+			self.conn.close()
+		self.conn = None
 		pass
 	def __setitem__(self, k, v):
 		cursor = self.conn.cursor()