[feature] [BIT 578] speed up metagraph storage query

bittensor/_cli/cli_impl.py

@@ -487,7 +487,7 @@ def metagraph(self):
         subtensor = bittensor.subtensor( config = self.config )
         metagraph = bittensor.metagraph( subtensor = subtensor )
         console.print(":satellite: Syncing with chain: [white]{}[/white] ...".format(self.config.subtensor.network))
-        metagraph.sync()
+        metagraph.sync( cached=not self.config.subtensor.get('use_fast_sync', bittensor.defaults.subtensor.use_fast_sync) )
         metagraph.save()
         issuance = subtensor.total_issuance
         difficulty = subtensor.difficulty
@@ -779,8 +779,8 @@ def full(self):
         console = bittensor.__console__
         subtensor = bittensor.subtensor( config = self.config )
         meta: bittensor.Metagraph = bittensor.metagraph( subtensor = subtensor )
-        # Get metagraph, use no_cache if flagged
-        meta.sync(cached = not self.config.get('no_cache', False))
+        # Get metagraph, use no_cache if set or if use_fast_sync is set
+        meta.sync(cached = not (self.config.get('no_cache', False) or self.config.get('subtensor.use_fast_sync', bittensor.defaults.subtensor.use_fast_sync)))
         neurons = []
         block = subtensor.block
 

bittensor/_subtensor/__init__.py

@@ -17,6 +17,7 @@
 import argparse
 import copy
 import os
+from typing import Optional
 
 import bittensor
 from loguru import logger
@@ -75,6 +76,7 @@ def __new__(
             network: str = None,
             chain_endpoint: str = None,
             _mock: bool = None,
+            use_fast_sync: Optional[bool] = None,
         ) -> 'bittensor.Subtensor':
         r""" Initializes a subtensor chain interface.
             Args:
@@ -92,6 +94,8 @@ def __new__(
                     The subtensor endpoint flag. If set, overrides the network argument.
                 _mock (bool, `optional`):
                     Returned object is mocks the underlying chain connection.
+                use_fast_sync (bool, `optional`):
+                    Returned object uses the fast neuron implementation.
         """
         if config == None: config = subtensor.config()
         config = copy.deepcopy( config )
@@ -153,6 +157,7 @@ def __new__(
             substrate = substrate,
             network = config.subtensor.get('network', bittensor.defaults.subtensor.network),
             chain_endpoint = config.subtensor.chain_endpoint,
+            use_fast_sync=use_fast_sync if use_fast_sync is not None else config.subtensor.get('use_fast_sync', bittensor.defaults.subtensor.use_fast_sync),
         )
 
     @staticmethod   
@@ -196,6 +201,8 @@ def add_args(cls, parser: argparse.ArgumentParser, prefix: str = None ):
 
             parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.TPB', '--' + prefix_str + 'cuda.TPB', type=int, default=bittensor.defaults.subtensor.register.cuda.TPB, help='''Set the number of Threads Per Block for CUDA.''', required=False )
 
+            parser.add_argument('--' + prefix_str + 'subtensor.no_fast_sync', '--' + prefix_str + 'no_fast_sync', action='store_false', dest='subtensor.use_fast_sync', help='''Set flag to disable fast sync feature.''', default=True)
+
         except argparse.ArgumentError:
             # re-parsing arguments.
             pass
@@ -218,6 +225,8 @@ def add_defaults(cls, defaults ):
         defaults.subtensor.register.cuda.use_cuda = False
         defaults.subtensor.register.cuda.TPB = 256
 
+        defaults.subtensor.use_fast_sync = os.getenv('BT_SUBTENSOR_NEURONS_FAST') if os.getenv('BT_SUBTENSOR_NEURONS_FAST') != None else False
+
     @staticmethod   
     def check_config( config: 'bittensor.Config' ):
         assert config.subtensor

bittensor/_subtensor/subtensor_impl.py

@@ -32,6 +32,7 @@
 # Mocking imports
 import os
 import random
+import json
 import scalecodec
 import time
 import subprocess
@@ -49,6 +50,7 @@ def __init__(
         substrate: 'SubstrateInterface',
         network: str,
         chain_endpoint: str,
+        use_fast_sync: bool = False,
     ):
         r""" Initializes a subtensor chain interface.
             Args:
@@ -63,10 +65,13 @@ def __init__(
                     an entry point node from that network.
                 chain_endpoint (default=None, type=str)
                     The subtensor endpoint flag. If set, overrides the network argument.
+                use_fast_sync (default=False, type=bool)
+                    If true, uses the fast neuron implementation.
         """
         self.network = network
         self.chain_endpoint = chain_endpoint
         self.substrate = substrate
+        self.use_fast_sync = use_fast_sync
 
     def __str__(self) -> str:
         if self.network == self.chain_endpoint:
@@ -1490,6 +1495,14 @@ def neurons(self, block: int = None ) -> List[SimpleNamespace]:
             neuron (List[SimpleNamespace]):
                 List of neuron objects.
         """
+        if self.use_fast_sync:
+            try:
+                return self.neurons_fast(block)
+            except Exception as e:
+                logger.warning("Failed to get neurons fast, falling back to manual sync.")
+                self.use_fast_sync = False
+                return self.neurons(block)
+
         neurons = []
         for id in tqdm(range(self.get_n( block ))): 
             try:
@@ -1500,6 +1513,91 @@ def neurons(self, block: int = None ) -> List[SimpleNamespace]:
                 break
         return neurons
 
+    def neurons_fast(self, block: int = None ) -> List[SimpleNamespace]: 
+        r""" Returns a list of neuron from the chain, using the bundled subtensor-node-api
+        Args:
+            block (int):
+                block to sync from.
+        Returns:
+            neuron (List[SimpleNamespace]):
+                List of neuron objects.
+        """
+        neurons = []
+        if block is None:
+            block: int = self.get_current_block()
+
+        block_hash: str = self.substrate.get_block_hash( block )
+        endpoint_url: str = self.chain_endpoint
+        # format endpoint with ws://
+        if "ws://" not in endpoint_url and "wss://" not in endpoint_url:
+            endpoint_url = "ws://" + endpoint_url
+
+        try:
+            if platform == "linux" or platform == "linux2":
+                # linux
+                os_name = "linux"
+            elif platform == "darwin":
+                # OS X
+                os_name = 'macos'
+            else:
+                raise Exception('Unsupported platform for subtensor-node-api')
+            bittensor.__console__.print("Using subtensor-node-api for neuron retrieval...")
+            path_to_bin = os.path.join(os.path.dirname(__file__), f"../../bin/subtensor-node-api-{os_name}")
+            # will write to ~/.bittensor/metagraph.json by default
+            subprocess.run([path_to_bin, "sync_and_save", "-u", endpoint_url, '-b', block_hash], check=True, stdout=subprocess.PIPE)
+            with open(os.path.join(os.path.expanduser('~'), '.bittensor/metagraph.json')) as f:
+                data = json.load(f)
+
+            # all the large ints are strings
+            RAOPERTAO = 1000000000
+            U64MAX = 18446744073709551615
+            """
+            We expect a JSON array of:
+            {
+                "uid": int,
+                "ip": str,
+                "ip_type": int,
+                "port": int,
+                "stake": str(int),
+                "rank": str(int),
+                "emission": str(int),
+                "incentive": str(int),
+                "consensus": str(int),
+                "trust": str(int),
+                "dividends": str(int),
+                "modality": int,
+                "last_update": str(int),
+                "version": int,
+                "priority": str(int),
+                "last_update": int,
+                "weights": [
+                    [int, int],
+                ],
+                "bonds": [
+                    [int, str(int)],
+                ],
+            }
+            """
+            for neuron_data in data:
+                neuron = SimpleNamespace( **neuron_data )
+                neuron.stake = int(neuron.stake) / RAOPERTAO
+                neuron.rank = int(neuron.rank) / U64MAX
+                neuron.trust = int(neuron.trust) / U64MAX
+                neuron.consensus = int(neuron.consensus) / U64MAX
+                neuron.incentive = int(neuron.incentive) / U64MAX
+                neuron.dividends = int(neuron.dividends) / U64MAX
+                neuron.emission = int(neuron.emission) / RAOPERTAO
+                neuron.last_update = int(neuron.last_update)
+                neuron.priority = int(neuron.priority)
+                neuron.bonds = [ [bond[0], int(bond[1])] for bond in neuron.bonds ]
+                neuron.is_null = False
+                neurons.append( neuron )
+        except Exception as e:
+            logger.error('Exception encountered when pulling neurons: {}'.format(e))
+
+        return neurons
+
+
     @staticmethod
     def _null_neuron() -> SimpleNamespace:
         neuron = SimpleNamespace()

setup.py

@@ -15,13 +15,15 @@
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
 # DEALINGS IN THE SOFTWARE.
 
+from tkinter import E
 from setuptools import setup, find_packages
 from pkg_resources import parse_requirements
 from os import path
 from io import open
 import codecs
 import re
 import os
+from sys import platform
 
 here = path.abspath(path.dirname(__file__))
 
@@ -36,6 +38,25 @@
     version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", init_file.read(), re.M)
     version_string = version_match.group(1)
 
+try:
+    # Check platform and remove unsupported subtensor node api binaries.
+    if platform == "linux" or platform == "linux2":
+        # linux
+        # remove macos binaries
+        os.remove('bin/subtensor-node-api-macos')
+    elif platform == "darwin":
+        # OS X
+        # remove linux binaries
+        os.remove('bin/subtensor-node-api-linux')
+    else:
+        # neither linux or macos
+        # remove both binaries
+        os.remove('bin/subtensor-node-api-linux')
+        os.remove('bin/subtensor-node-api-macos')
+except Exception as e:
+    print('Failed to remove unsupported subtensor node api binaries. Error: {}'.format(e))
+    pass
+
 setup(
     name='bittensor',
     version=version_string,